From f51d753f81c0862e18af48d701a57128572883ed Mon Sep 17 00:00:00 2001 From: Christian Gmeiner Date: Fri, 3 Jun 2022 14:37:05 +0200 Subject: [PATCH 0001/1927] drm/etnaviv: print offender task information on hangcheck recovery Track the pid per submit, so we can print the name and cmdline of the task which submitted the batch that caused the gpu to hang. Signed-off-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gem.h | 1 + drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c | 6 ++++++ drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 18 +++++++++++++++++- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 2 +- drivers/gpu/drm/etnaviv/etnaviv_sched.c | 2 +- 5 files changed, 26 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h index 63688e6e45804..baa81cbf701a6 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h @@ -96,6 +96,7 @@ struct etnaviv_gem_submit { int out_fence_id; struct list_head node; /* GPU active submit list */ struct etnaviv_cmdbuf cmdbuf; + struct pid *pid; /* submitting process */ bool runtime_resumed; u32 exec_state; u32 flags; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c index 1ac916b248917..1491159d0d209 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c @@ -399,6 +399,9 @@ static void submit_cleanup(struct kref *kref) mutex_unlock(&submit->gpu->fence_lock); dma_fence_put(submit->out_fence); } + + put_pid(submit->pid); + kfree(submit->pmrs); kfree(submit); } @@ -422,6 +425,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, struct sync_file *sync_file = NULL; struct ww_acquire_ctx ticket; int out_fence_fd = -1; + struct pid *pid = get_pid(task_pid(current)); void *stream; int ret; @@ -519,6 +523,8 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, goto err_submit_ww_acquire; } + submit->pid = pid; + ret = etnaviv_cmdbuf_init(priv->cmdbuf_suballoc, &submit->cmdbuf, ALIGN(args->stream_size, 8) + 8); if (ret) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 37018bc55810d..7d9bf4673e2d7 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -1045,12 +1045,28 @@ int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) } #endif -void etnaviv_gpu_recover_hang(struct etnaviv_gpu *gpu) +void etnaviv_gpu_recover_hang(struct etnaviv_gem_submit *submit) { + struct etnaviv_gpu *gpu = submit->gpu; + char *comm = NULL, *cmd = NULL; + struct task_struct *task; unsigned int i; dev_err(gpu->dev, "recover hung GPU!\n"); + task = get_pid_task(submit->pid, PIDTYPE_PID); + if (task) { + comm = kstrdup(task->comm, GFP_KERNEL); + cmd = kstrdup_quotable_cmdline(task, GFP_KERNEL); + put_task_struct(task); + } + + if (comm && cmd) + dev_err(gpu->dev, "offending task: %s (%s)\n", comm, cmd); + + kfree(cmd); + kfree(comm); + if (pm_runtime_get_sync(gpu->dev) < 0) goto pm_put; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h index 85eddd492774d..b3a0941d56fd3 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h @@ -168,7 +168,7 @@ bool etnaviv_fill_identity_from_hwdb(struct etnaviv_gpu *gpu); int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m); #endif -void etnaviv_gpu_recover_hang(struct etnaviv_gpu *gpu); +void etnaviv_gpu_recover_hang(struct etnaviv_gem_submit *submit); void etnaviv_gpu_retire(struct etnaviv_gpu *gpu); int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu, u32 fence, struct drm_etnaviv_timespec *timeout); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c index 72e2553fbc984..d29f467eee138 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -67,7 +67,7 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job /* get the GPU back into the init state */ etnaviv_core_dump(submit); - etnaviv_gpu_recover_hang(gpu); + etnaviv_gpu_recover_hang(submit); drm_sched_resubmit_jobs(&gpu->sched); -- GitLab From 6cffb1c27f2d05047c7f569c425fd56d11ae89b2 Mon Sep 17 00:00:00 2001 From: "T.J. Mercier" Date: Wed, 6 Jul 2022 18:29:24 +0000 Subject: [PATCH 0002/1927] drm/etnaviv: Remove duplicate call to drm_gem_free_mmap_offset The docs explicitly say the drm_gem_object_release function already calls this, and this does not appear to be a prerequisite for the call to etnaviv_gem_ops.release. Signed-off-by: T.J. Mercier Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index cc386f8a7116e..ecb828e4e1565 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -504,7 +504,6 @@ void etnaviv_gem_free_object(struct drm_gem_object *obj) kfree(mapping); } - drm_gem_free_mmap_offset(obj); etnaviv_obj->ops->release(etnaviv_obj); drm_gem_object_release(obj); -- GitLab From 5a40837debaa9dcc71765d32ce1a15be068b6cc2 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Thu, 14 Jul 2022 12:31:42 +0200 Subject: [PATCH 0003/1927] drm/etnaviv: move idle mapping reaping into separate function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same logic is already used in two different places and now it will also be needed outside of the compilation unit, so split it into a separate function. Cc: stable@vger.kernel.org # 5.19 Signed-off-by: Lucas Stach Reviewed-by: Guido Günther --- drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 23 +++++++++++++++-------- drivers/gpu/drm/etnaviv/etnaviv_mmu.h | 1 + 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index dc1aa738c4f18..55479cb8b1ac3 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -135,6 +135,19 @@ static void etnaviv_iommu_remove_mapping(struct etnaviv_iommu_context *context, drm_mm_remove_node(&mapping->vram_node); } +void etnaviv_iommu_reap_mapping(struct etnaviv_vram_mapping *mapping) +{ + struct etnaviv_iommu_context *context = mapping->context; + + lockdep_assert_held(&context->lock); + WARN_ON(mapping->use); + + etnaviv_iommu_remove_mapping(context, mapping); + etnaviv_iommu_context_put(mapping->context); + mapping->context = NULL; + list_del_init(&mapping->mmu_node); +} + static int etnaviv_iommu_find_iova(struct etnaviv_iommu_context *context, struct drm_mm_node *node, size_t size) { @@ -202,10 +215,7 @@ static int etnaviv_iommu_find_iova(struct etnaviv_iommu_context *context, * this mapping. */ list_for_each_entry_safe(m, n, &list, scan_node) { - etnaviv_iommu_remove_mapping(context, m); - etnaviv_iommu_context_put(m->context); - m->context = NULL; - list_del_init(&m->mmu_node); + etnaviv_iommu_reap_mapping(m); list_del_init(&m->scan_node); } @@ -257,10 +267,7 @@ static int etnaviv_iommu_insert_exact(struct etnaviv_iommu_context *context, } list_for_each_entry_safe(m, n, &scan_list, scan_node) { - etnaviv_iommu_remove_mapping(context, m); - etnaviv_iommu_context_put(m->context); - m->context = NULL; - list_del_init(&m->mmu_node); + etnaviv_iommu_reap_mapping(m); list_del_init(&m->scan_node); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h index e4a0b7d09c2ea..c01a147f0dfdd 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.h @@ -91,6 +91,7 @@ int etnaviv_iommu_map_gem(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping, u64 va); void etnaviv_iommu_unmap_gem(struct etnaviv_iommu_context *context, struct etnaviv_vram_mapping *mapping); +void etnaviv_iommu_reap_mapping(struct etnaviv_vram_mapping *mapping); int etnaviv_iommu_get_suballoc_va(struct etnaviv_iommu_context *ctx, struct etnaviv_vram_mapping *mapping, -- GitLab From 332f847212e43d584019a8264895f25cf92aa647 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Thu, 14 Jul 2022 12:31:43 +0200 Subject: [PATCH 0004/1927] drm/etnaviv: reap idle mapping if it doesn't match the softpin address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a idle BO, which is held open by another process, gets freed by userspace and subsequently referenced again by e.g. importing it again, userspace may assign a different softpin VA than the last time around. As the kernel GEM object still exists, we likely have a idle mapping with the old VA still cached, if it hasn't been reaped in the meantime. As the context matches, we then simply try to resurrect this mapping by increasing the refcount. As the VA in this mapping does not match the new softpin address, we consequently fail the otherwise valid submit. Instead of failing, reap the idle mapping. Cc: stable@vger.kernel.org # 5.19 Signed-off-by: Lucas Stach Reviewed-by: Guido Günther --- drivers/gpu/drm/etnaviv/etnaviv_gem.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index ecb828e4e1565..d45bf03683394 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -258,7 +258,12 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( if (mapping->use == 0) { mutex_lock(&mmu_context->lock); if (mapping->context == mmu_context) - mapping->use += 1; + if (va && mapping->iova != va) { + etnaviv_iommu_reap_mapping(mapping); + mapping = NULL; + } else { + mapping->use += 1; + } else mapping = NULL; mutex_unlock(&mmu_context->lock); -- GitLab From 6ca53a75c425cb1cb0929ce07f58d9e41094500c Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 9 Sep 2022 11:20:21 +0200 Subject: [PATCH 0005/1927] dt-bindings: hwlock: qcom-hwspinlock: add support for MMIO on older SoCs Older Qualcomm SoCs have TCSR mutex registers with 0x80 stride, instead of 0x1000. Add dedicated compatibles for such case. Unfortunately the binding started using a generic "qcom,tcsr-mutex" compatible without specifying the SoC part, thus it looks now quite inconsistent. Signed-off-by: Krzysztof Kozlowski Acked-by: Rob Herring Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220909092035.223915-2-krzysztof.kozlowski@linaro.org --- .../bindings/hwlock/qcom-hwspinlock.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml b/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml index 1c7149f7d171a..de98b961fb381 100644 --- a/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml +++ b/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml @@ -15,9 +15,18 @@ description: properties: compatible: - enum: - - qcom,sfpb-mutex - - qcom,tcsr-mutex + oneOf: + - enum: + - qcom,sfpb-mutex + - qcom,tcsr-mutex + - items: + - enum: + - qcom,apq8084-tcsr-mutex + - qcom,ipq6018-tcsr-mutex + - qcom,msm8226-tcsr-mutex + - qcom,msm8974-tcsr-mutex + - qcom,msm8994-tcsr-mutex + - const: qcom,tcsr-mutex reg: maxItems: 1 -- GitLab From 276a4f1a5fb118bfd6980d42732d530e43f2916a Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 9 Sep 2022 11:20:22 +0200 Subject: [PATCH 0006/1927] dt-bindings: hwlock: qcom-hwspinlock: correct example indentation Use some consistent indentation (4-space) for DTS example. Signed-off-by: Krzysztof Kozlowski Acked-by: Rob Herring Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220909092035.223915-3-krzysztof.kozlowski@linaro.org --- .../devicetree/bindings/hwlock/qcom-hwspinlock.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml b/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml index de98b961fb381..1a3adf75934bc 100644 --- a/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml +++ b/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml @@ -43,9 +43,9 @@ additionalProperties: false examples: - | - tcsr_mutex: hwlock@1f40000 { - compatible = "qcom,tcsr-mutex"; - reg = <0x01f40000 0x40000>; - #hwlock-cells = <1>; - }; + hwlock@1f40000 { + compatible = "qcom,tcsr-mutex"; + reg = <0x01f40000 0x40000>; + #hwlock-cells = <1>; + }; ... -- GitLab From 90cb380f9ceb811059340d06ff5fd0c0e93ecbe1 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 9 Sep 2022 11:20:23 +0200 Subject: [PATCH 0007/1927] hwspinlock: qcom: correct MMIO max register for newer SoCs Newer ARMv8 Qualcomm SoCs using 0x1000 register stride have maximum register 0x20000 (32 mutexes * 0x1000). Fixes: 7a1e6fb1c606 ("hwspinlock: qcom: Allow mmio usage in addition to syscon") Signed-off-by: Krzysztof Kozlowski Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220909092035.223915-4-krzysztof.kozlowski@linaro.org --- drivers/hwspinlock/qcom_hwspinlock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hwspinlock/qcom_hwspinlock.c b/drivers/hwspinlock/qcom_hwspinlock.c index 80ea45b3a815f..9734e149d981b 100644 --- a/drivers/hwspinlock/qcom_hwspinlock.c +++ b/drivers/hwspinlock/qcom_hwspinlock.c @@ -121,7 +121,7 @@ static const struct regmap_config tcsr_mutex_config = { .reg_bits = 32, .reg_stride = 4, .val_bits = 32, - .max_register = 0x40000, + .max_register = 0x20000, .fast_io = true, }; -- GitLab From 5d4753f741d824e04e7ba46f46ec016be120f383 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 9 Sep 2022 11:20:24 +0200 Subject: [PATCH 0008/1927] hwspinlock: qcom: add support for MMIO on older SoCs Older Qualcomm SoCs have TCSR mutex registers with 0x80 stride, instead of 0x1000. Add dedicated compatibles and regmap for such case. Signed-off-by: Krzysztof Kozlowski Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220909092035.223915-5-krzysztof.kozlowski@linaro.org --- drivers/hwspinlock/qcom_hwspinlock.c | 42 +++++++++++++++++++++------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/drivers/hwspinlock/qcom_hwspinlock.c b/drivers/hwspinlock/qcom_hwspinlock.c index 9734e149d981b..9cf186362ae2f 100644 --- a/drivers/hwspinlock/qcom_hwspinlock.c +++ b/drivers/hwspinlock/qcom_hwspinlock.c @@ -22,6 +22,7 @@ struct qcom_hwspinlock_of_data { u32 offset; u32 stride; + const struct regmap_config *regmap_config; }; static int qcom_hwspinlock_trylock(struct hwspinlock *lock) @@ -73,15 +74,42 @@ static const struct qcom_hwspinlock_of_data of_sfpb_mutex = { .stride = 0x4, }; -/* All modern platform has offset 0 and stride of 4k */ +static const struct regmap_config tcsr_msm8226_mutex_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x1000, + .fast_io = true, +}; + +static const struct qcom_hwspinlock_of_data of_msm8226_tcsr_mutex = { + .offset = 0, + .stride = 0x80, + .regmap_config = &tcsr_msm8226_mutex_config, +}; + +static const struct regmap_config tcsr_mutex_config = { + .reg_bits = 32, + .reg_stride = 4, + .val_bits = 32, + .max_register = 0x20000, + .fast_io = true, +}; + static const struct qcom_hwspinlock_of_data of_tcsr_mutex = { .offset = 0, .stride = 0x1000, + .regmap_config = &tcsr_mutex_config, }; static const struct of_device_id qcom_hwspinlock_of_match[] = { { .compatible = "qcom,sfpb-mutex", .data = &of_sfpb_mutex }, { .compatible = "qcom,tcsr-mutex", .data = &of_tcsr_mutex }, + { .compatible = "qcom,apq8084-tcsr-mutex", .data = &of_msm8226_tcsr_mutex }, + { .compatible = "qcom,ipq6018-tcsr-mutex", .data = &of_msm8226_tcsr_mutex }, + { .compatible = "qcom,msm8226-tcsr-mutex", .data = &of_msm8226_tcsr_mutex }, + { .compatible = "qcom,msm8974-tcsr-mutex", .data = &of_msm8226_tcsr_mutex }, + { .compatible = "qcom,msm8994-tcsr-mutex", .data = &of_msm8226_tcsr_mutex }, { } }; MODULE_DEVICE_TABLE(of, qcom_hwspinlock_of_match); @@ -117,14 +145,6 @@ static struct regmap *qcom_hwspinlock_probe_syscon(struct platform_device *pdev, return regmap; } -static const struct regmap_config tcsr_mutex_config = { - .reg_bits = 32, - .reg_stride = 4, - .val_bits = 32, - .max_register = 0x20000, - .fast_io = true, -}; - static struct regmap *qcom_hwspinlock_probe_mmio(struct platform_device *pdev, u32 *offset, u32 *stride) { @@ -133,6 +153,8 @@ static struct regmap *qcom_hwspinlock_probe_mmio(struct platform_device *pdev, void __iomem *base; data = of_device_get_match_data(dev); + if (!data->regmap_config) + return ERR_PTR(-EINVAL); *offset = data->offset; *stride = data->stride; @@ -141,7 +163,7 @@ static struct regmap *qcom_hwspinlock_probe_mmio(struct platform_device *pdev, if (IS_ERR(base)) return ERR_CAST(base); - return devm_regmap_init_mmio(dev, base, &tcsr_mutex_config); + return devm_regmap_init_mmio(dev, base, data->regmap_config); } static int qcom_hwspinlock_probe(struct platform_device *pdev) -- GitLab From cc7d3fb446a91f24978a6aa59cbb578f92e22242 Mon Sep 17 00:00:00 2001 From: Doug Brown Date: Sat, 10 Sep 2022 13:29:38 -0700 Subject: [PATCH 0009/1927] drm/etnaviv: add missing quirks for GC300 The GC300's features register doesn't specify that a 2D pipe is available, and like the GC600, its idle register reports zero bits where modules aren't present. Signed-off-by: Doug Brown Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 7d9bf4673e2d7..a96392c2dd02c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -416,6 +416,12 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) if (gpu->identity.model == chipModel_GC700) gpu->identity.features &= ~chipFeatures_FAST_CLEAR; + /* These models/revisions don't have the 2D pipe bit */ + if ((gpu->identity.model == chipModel_GC500 && + gpu->identity.revision <= 2) || + gpu->identity.model == chipModel_GC300) + gpu->identity.features |= chipFeatures_PIPE_2D; + if ((gpu->identity.model == chipModel_GC500 && gpu->identity.revision < 2) || (gpu->identity.model == chipModel_GC300 && @@ -449,8 +455,9 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_5); } - /* GC600 idle register reports zero bits where modules aren't present */ - if (gpu->identity.model == chipModel_GC600) + /* GC600/300 idle register reports zero bits where modules aren't present */ + if (gpu->identity.model == chipModel_GC600 || + gpu->identity.model == chipModel_GC300) gpu->idle_mask = VIVS_HI_IDLE_STATE_TX | VIVS_HI_IDLE_STATE_RA | VIVS_HI_IDLE_STATE_SE | -- GitLab From 61a6920bb604df3a0e389a2a9479e1e233e4461d Mon Sep 17 00:00:00 2001 From: Doug Brown Date: Sat, 10 Sep 2022 13:29:39 -0700 Subject: [PATCH 0010/1927] drm/etnaviv: fix power register offset on GC300 Older GC300 revisions have their power registers at an offset of 0x200 rather than 0x100. Add new gpu_read_power and gpu_write_power functions to encapsulate accesses to the power addresses and fix the addresses. Signed-off-by: Doug Brown Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_dump.c | 7 ++++++- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 20 ++++++++++---------- drivers/gpu/drm/etnaviv/etnaviv_gpu.h | 21 +++++++++++++++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index f418e0b75772e..44b5f3c35aabe 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -83,10 +83,15 @@ static void etnaviv_core_dump_registers(struct core_dump_iterator *iter, { struct etnaviv_dump_registers *reg = iter->data; unsigned int i; + u32 read_addr; for (i = 0; i < ARRAY_SIZE(etnaviv_dump_registers); i++, reg++) { + read_addr = etnaviv_dump_registers[i]; + if (read_addr >= VIVS_PM_POWER_CONTROLS && + read_addr <= VIVS_PM_PULSE_EATER) + read_addr = gpu_fix_power_address(gpu, read_addr); reg->reg = cpu_to_le32(etnaviv_dump_registers[i]); - reg->value = cpu_to_le32(gpu_read(gpu, etnaviv_dump_registers[i])); + reg->value = cpu_to_le32(gpu_read(gpu, read_addr)); } etnaviv_core_dump_header(iter, ETDUMP_BUF_REG, reg); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index a96392c2dd02c..ed2f9c8f288c5 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -590,7 +590,7 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) u32 pmc, ppc; /* enable clock gating */ - ppc = gpu_read(gpu, VIVS_PM_POWER_CONTROLS); + ppc = gpu_read_power(gpu, VIVS_PM_POWER_CONTROLS); ppc |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; /* Disable stall module clock gating for 4.3.0.1 and 4.3.0.2 revs */ @@ -598,9 +598,9 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) gpu->identity.revision == 0x4302) ppc |= VIVS_PM_POWER_CONTROLS_DISABLE_STALL_MODULE_CLOCK_GATING; - gpu_write(gpu, VIVS_PM_POWER_CONTROLS, ppc); + gpu_write_power(gpu, VIVS_PM_POWER_CONTROLS, ppc); - pmc = gpu_read(gpu, VIVS_PM_MODULE_CONTROLS); + pmc = gpu_read_power(gpu, VIVS_PM_MODULE_CONTROLS); /* Disable PA clock gating for GC400+ without bugfix except for GC420 */ if (gpu->identity.model >= chipModel_GC400 && @@ -635,7 +635,7 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ; pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ; - gpu_write(gpu, VIVS_PM_MODULE_CONTROLS, pmc); + gpu_write_power(gpu, VIVS_PM_MODULE_CONTROLS, pmc); } void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch) @@ -695,11 +695,11 @@ static void etnaviv_gpu_setup_pulse_eater(struct etnaviv_gpu *gpu) (gpu->identity.features & chipFeatures_PIPE_3D)) { /* Performance fix: disable internal DFS */ - pulse_eater = gpu_read(gpu, VIVS_PM_PULSE_EATER); + pulse_eater = gpu_read_power(gpu, VIVS_PM_PULSE_EATER); pulse_eater |= BIT(18); } - gpu_write(gpu, VIVS_PM_PULSE_EATER, pulse_eater); + gpu_write_power(gpu, VIVS_PM_PULSE_EATER, pulse_eater); } static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) @@ -1317,9 +1317,9 @@ static void sync_point_perfmon_sample_pre(struct etnaviv_gpu *gpu, u32 val; /* disable clock gating */ - val = gpu_read(gpu, VIVS_PM_POWER_CONTROLS); + val = gpu_read_power(gpu, VIVS_PM_POWER_CONTROLS); val &= ~VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; - gpu_write(gpu, VIVS_PM_POWER_CONTROLS, val); + gpu_write_power(gpu, VIVS_PM_POWER_CONTROLS, val); /* enable debug register */ val = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); @@ -1350,9 +1350,9 @@ static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu, gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, val); /* enable clock gating */ - val = gpu_read(gpu, VIVS_PM_POWER_CONTROLS); + val = gpu_read_power(gpu, VIVS_PM_POWER_CONTROLS); val |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; - gpu_write(gpu, VIVS_PM_POWER_CONTROLS, val); + gpu_write_power(gpu, VIVS_PM_POWER_CONTROLS, val); } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h index b3a0941d56fd3..f1204b070fb8d 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h @@ -10,6 +10,7 @@ #include "etnaviv_gem.h" #include "etnaviv_mmu.h" #include "etnaviv_drv.h" +#include "common.xml.h" struct etnaviv_gem_submit; struct etnaviv_vram_mapping; @@ -159,6 +160,26 @@ static inline u32 gpu_read(struct etnaviv_gpu *gpu, u32 reg) return readl(gpu->mmio + reg); } +static inline u32 gpu_fix_power_address(struct etnaviv_gpu *gpu, u32 reg) +{ + /* Power registers in GC300 < 2.0 are offset by 0x100 */ + if (gpu->identity.model == chipModel_GC300 && + gpu->identity.revision < 0x2000) + reg += 0x100; + + return reg; +} + +static inline void gpu_write_power(struct etnaviv_gpu *gpu, u32 reg, u32 data) +{ + writel(data, gpu->mmio + gpu_fix_power_address(gpu, reg)); +} + +static inline u32 gpu_read_power(struct etnaviv_gpu *gpu, u32 reg) +{ + return readl(gpu->mmio + gpu_fix_power_address(gpu, reg)); +} + int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value); int etnaviv_gpu_init(struct etnaviv_gpu *gpu); -- GitLab From d37c120b73128690434cc093952439eef9d56af1 Mon Sep 17 00:00:00 2001 From: Lucas Stach Date: Fri, 16 Sep 2022 12:40:31 +0200 Subject: [PATCH 0011/1927] drm/etnaviv: don't truncate physical page address While the interface for the MMU mapping takes phys_addr_t to hold a full 64bit address when necessary and MMUv2 is able to map physical addresses with up to 40bit, etnaviv_iommu_map() truncates the address to 32bits. Fix this by using the correct type. Fixes: 931e97f3afd8 ("drm/etnaviv: mmuv2: support 40 bit phys address") Signed-off-by: Lucas Stach Reviewed-by: Philipp Zabel --- drivers/gpu/drm/etnaviv/etnaviv_mmu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index 55479cb8b1ac3..67bdce5326c6e 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -80,10 +80,10 @@ static int etnaviv_iommu_map(struct etnaviv_iommu_context *context, u32 iova, return -EINVAL; for_each_sgtable_dma_sg(sgt, sg, i) { - u32 pa = sg_dma_address(sg) - sg->offset; + phys_addr_t pa = sg_dma_address(sg) - sg->offset; size_t bytes = sg_dma_len(sg) + sg->offset; - VERB("map[%d]: %08x %08x(%zx)", i, iova, pa, bytes); + VERB("map[%d]: %08x %pap(%zx)", i, iova, &pa, bytes); ret = etnaviv_context_map(context, da, pa, bytes, prot); if (ret) -- GitLab From 4bce244272513ebb4d13c570e9fbca28497015b2 Mon Sep 17 00:00:00 2001 From: Marco Felsch Date: Fri, 26 Aug 2022 21:07:28 +0200 Subject: [PATCH 0012/1927] drm/etnaviv: disable tx clock gating for GC7000 rev6203 The i.MX8MN SoC errata sheet mentions ERR050226: "GPU: Texture L2 Cache idle signal may incorrectly clock gate the texture engine in GPU". The workaround is to disable the corresponding clock gatings. While on it move the clock gating check for rev6202 into the same check to bundle them. Signed-off-by: Marco Felsch Reviewed-by: Christian Gmeiner Signed-off-by: Lucas Stach --- drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index ed2f9c8f288c5..51320eeebfcff 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -623,14 +623,15 @@ static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) /* Disable TX clock gating on affected core revisions. */ if (etnaviv_is_model_rev(gpu, GC4000, 0x5222) || - etnaviv_is_model_rev(gpu, GC2000, 0x5108)) + etnaviv_is_model_rev(gpu, GC2000, 0x5108) || + etnaviv_is_model_rev(gpu, GC2000, 0x6202) || + etnaviv_is_model_rev(gpu, GC2000, 0x6203)) pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX; - /* Disable SE, RA and TX clock gating on affected core revisions. */ + /* Disable SE and RA clock gating on affected core revisions. */ if (etnaviv_is_model_rev(gpu, GC7000, 0x6202)) pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_SE | - VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA | - VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX; + VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA; pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ; pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ; -- GitLab From 084b9e1732f71e36c21a820162c9f601577932c6 Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Fri, 23 Sep 2022 15:40:44 -0700 Subject: [PATCH 0013/1927] drm/msm/gem: Unpin objects slightly later The introduction of "drm/msm/gem: Evict active GEM objects when necessary" exposes a problem with "drm/msm/gem: Unpin buffers earlier", in that we need to keep the object pinned in the time the submit is queued up in the gpu scheduler. Otherwise the shrinker will see it as a thing that can be evicted if we wait for it to be signaled. But if the shrinker path is waiting on it with the obj lock held, the job cannot be scheduled, as that also requires briefly grabbing the obj lock, leading to deadlock. (Not to mention, we don't want the shrinker to evict an obj queued up in gpu scheduler.) Fixes: f371bcc0c2ac ("drm/msm/gem: Unpin buffers earlier") Fixes: 025d27239a2f ("drm/msm/gem: Evict active GEM objects when necessary") Closes: https://gitlab.freedesktop.org/drm/msm/-/issues/19 Signed-off-by: Rob Clark Tested-by: Chia-I Wu Patchwork: https://patchwork.freedesktop.org/patch/504528/ Link: https://lore.kernel.org/r/20220923224043.2449152-1-robdclark@gmail.com --- drivers/gpu/drm/msm/msm_gem_submit.c | 4 ++-- drivers/gpu/drm/msm/msm_ringbuffer.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index 5599d93ec0d21..c670591995e6f 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -501,11 +501,11 @@ static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *ob */ static void submit_cleanup(struct msm_gem_submit *submit, bool error) { - unsigned cleanup_flags = BO_LOCKED | BO_OBJ_PINNED; + unsigned cleanup_flags = BO_LOCKED; unsigned i; if (error) - cleanup_flags |= BO_VMA_PINNED; + cleanup_flags |= BO_VMA_PINNED | BO_OBJ_PINNED; for (i = 0; i < submit->nr_bos; i++) { struct msm_gem_object *msm_obj = submit->bos[i].obj; diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index cad4c3525f0be..57a8e9564540e 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -25,7 +25,8 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job) msm_gem_lock(obj); msm_gem_unpin_vma_fenced(submit->bos[i].vma, fctx); - submit->bos[i].flags &= ~BO_VMA_PINNED; + msm_gem_unpin_locked(obj); + submit->bos[i].flags &= ~(BO_VMA_PINNED | BO_OBJ_PINNED); msm_gem_unlock(obj); } -- GitLab From ec8f1813bf8d0737898f99a8c1c69df0cde0d7dd Mon Sep 17 00:00:00 2001 From: Akhil P Oommen Date: Wed, 28 Sep 2022 12:48:59 +0530 Subject: [PATCH 0014/1927] drm/msm/a6xx: Replace kcalloc() with kvzalloc() In order to reduce chance of allocation failure while capturing a6xx gpu state, use kvzalloc() instead of kcalloc() in state_kcalloc(). Indirectly, this patch helps to fix leaking memory allocated for gmu_debug object. Fixes: b859f9b009b (drm/msm/gpu: Snapshot GMU debug buffer) Signed-off-by: Akhil P Oommen Patchwork: https://patchwork.freedesktop.org/patch/505074/ Link: https://lore.kernel.org/r/20220928124830.1.I8ea24a8d586b4978823b848adde000f92f74d5c2@changeid Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 55f443328d8e7..3c112a6cc8a2f 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -91,7 +91,7 @@ struct a6xx_state_memobj { static void *state_kcalloc(struct a6xx_gpu_state *a6xx_state, int nr, size_t objsize) { struct a6xx_state_memobj *obj = - kzalloc((nr * objsize) + sizeof(*obj), GFP_KERNEL); + kvzalloc((nr * objsize) + sizeof(*obj), GFP_KERNEL); if (!obj) return NULL; @@ -819,7 +819,7 @@ static struct msm_gpu_state_bo *a6xx_snapshot_gmu_bo( snapshot->iova = bo->iova; snapshot->size = bo->size; - snapshot->data = kvzalloc(snapshot->size, GFP_KERNEL); + snapshot->data = state_kcalloc(a6xx_state, 1, snapshot->size); if (!snapshot->data) return NULL; @@ -1034,14 +1034,8 @@ static void a6xx_gpu_state_destroy(struct kref *kref) struct a6xx_gpu_state *a6xx_state = container_of(state, struct a6xx_gpu_state, base); - if (a6xx_state->gmu_log) - kvfree(a6xx_state->gmu_log->data); - - if (a6xx_state->gmu_hfi) - kvfree(a6xx_state->gmu_hfi->data); - list_for_each_entry_safe(obj, tmp, &a6xx_state->objs, node) - kfree(obj); + kvfree(obj); adreno_gpu_state_destroy(state); kfree(a6xx_state); -- GitLab From 76efc2453d0e8e5d6692ef69981b183ad674edea Mon Sep 17 00:00:00 2001 From: Akhil P Oommen Date: Wed, 28 Sep 2022 12:49:00 +0530 Subject: [PATCH 0015/1927] drm/msm/gpu: Fix crash during system suspend after unbind In adreno_unbind, we should clean up gpu device's drvdata to avoid accessing a stale pointer during system suspend. Also, check for NULL ptr in both system suspend/resume callbacks. Signed-off-by: Akhil P Oommen Patchwork: https://patchwork.freedesktop.org/patch/505075/ Link: https://lore.kernel.org/r/20220928124830.2.I5ee0ac073ccdeb81961e5ec0cce5f741a7207a71@changeid Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/adreno/adreno_device.c | 10 +++++++++- drivers/gpu/drm/msm/msm_gpu.c | 2 ++ drivers/gpu/drm/msm/msm_gpu.h | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 24b489b6129a0..628806423f7d2 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -679,6 +679,9 @@ static int adreno_system_suspend(struct device *dev) struct msm_gpu *gpu = dev_to_gpu(dev); int remaining, ret; + if (!gpu) + return 0; + suspend_scheduler(gpu); remaining = wait_event_timeout(gpu->retire_event, @@ -700,7 +703,12 @@ static int adreno_system_suspend(struct device *dev) static int adreno_system_resume(struct device *dev) { - resume_scheduler(dev_to_gpu(dev)); + struct msm_gpu *gpu = dev_to_gpu(dev); + + if (!gpu) + return 0; + + resume_scheduler(gpu); return pm_runtime_force_resume(dev); } diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 0098ee8438aae..021f4e29b613b 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -997,4 +997,6 @@ void msm_gpu_cleanup(struct msm_gpu *gpu) } msm_devfreq_cleanup(gpu); + + platform_set_drvdata(gpu->pdev, NULL); } diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h index ff911e7305ce9..58a72e6b14008 100644 --- a/drivers/gpu/drm/msm/msm_gpu.h +++ b/drivers/gpu/drm/msm/msm_gpu.h @@ -280,6 +280,10 @@ struct msm_gpu { static inline struct msm_gpu *dev_to_gpu(struct device *dev) { struct adreno_smmu_priv *adreno_smmu = dev_get_drvdata(dev); + + if (!adreno_smmu) + return NULL; + return container_of(adreno_smmu, struct msm_gpu, adreno_smmu); } -- GitLab From 3a661247967a6f3c99a95a8ba4c8073c5846ea4b Mon Sep 17 00:00:00 2001 From: Kuogee Hsieh Date: Wed, 28 Sep 2022 16:36:51 -0700 Subject: [PATCH 0016/1927] drm/msm/dp: add atomic_check to bridge ops DRM commit_tails() will disable downstream crtc/encoder/bridge if both disable crtc is required and crtc->active is set before pushing a new frame downstream. There is a rare case that user space display manager issue an extra screen update immediately followed by close DRM device while down stream display interface is disabled. This extra screen update will timeout due to the downstream interface is disabled but will cause crtc->active be set. Hence the followed commit_tails() called by drm_release() will pass the disable downstream crtc/encoder/bridge conditions checking even downstream interface is disabled. This cause the crash to happen at dp_bridge_disable() due to it trying to access the main link register to push the idle pattern out while main link clocks is disabled. This patch adds atomic_check to prevent the extra frame will not be pushed down if display interface is down so that crtc->active will not be set neither. This will fail the conditions checking of disabling down stream crtc/encoder/bridge which prevent drm_release() from calling dp_bridge_disable() so that crash at dp_bridge_disable() prevented. There is no protection in the DRM framework to check if the display pipeline has been already disabled before trying again. The only check is the crtc_state->active but this is controlled by usermode using UAPI. Hence if the usermode sets this and then crashes, the driver needs to protect against double disable. SError Interrupt on CPU7, code 0x00000000be000411 -- SError CPU: 7 PID: 3878 Comm: Xorg Not tainted 5.19.0-stb-cbq #19 Hardware name: Google Lazor (rev3 - 8) (DT) pstate: a04000c9 (NzCv daIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : __cmpxchg_case_acq_32+0x14/0x2c lr : do_raw_spin_lock+0xa4/0xdc sp : ffffffc01092b6a0 x29: ffffffc01092b6a0 x28: 0000000000000028 x27: 0000000000000038 x26: 0000000000000004 x25: ffffffd2973dce48 x24: 0000000000000000 x23: 00000000ffffffff x22: 00000000ffffffff x21: ffffffd2978d0008 x20: ffffffd2978d0008 x19: ffffff80ff759fc0 x18: 0000000000000000 x17: 004800a501260460 x16: 0441043b04600438 x15: 04380000089807d0 x14: 07b0089807800780 x13: 0000000000000000 x12: 0000000000000000 x11: 0000000000000438 x10: 00000000000007d0 x9 : ffffffd2973e09e4 x8 : ffffff8092d53300 x7 : ffffff808902e8b8 x6 : 0000000000000001 x5 : ffffff808902e880 x4 : 0000000000000000 x3 : ffffff80ff759fc0 x2 : 0000000000000001 x1 : 0000000000000000 x0 : ffffff80ff759fc0 Kernel panic - not syncing: Asynchronous SError Interrupt CPU: 7 PID: 3878 Comm: Xorg Not tainted 5.19.0-stb-cbq #19 Hardware name: Google Lazor (rev3 - 8) (DT) Call trace: dump_backtrace.part.0+0xbc/0xe4 show_stack+0x24/0x70 dump_stack_lvl+0x68/0x84 dump_stack+0x18/0x34 panic+0x14c/0x32c nmi_panic+0x58/0x7c arm64_serror_panic+0x78/0x84 do_serror+0x40/0x64 el1h_64_error_handler+0x30/0x48 el1h_64_error+0x68/0x6c __cmpxchg_case_acq_32+0x14/0x2c _raw_spin_lock_irqsave+0x38/0x4c lock_timer_base+0x40/0x78 __mod_timer+0xf4/0x25c schedule_timeout+0xd4/0xfc __wait_for_common+0xac/0x140 wait_for_completion_timeout+0x2c/0x54 dp_ctrl_push_idle+0x40/0x88 dp_bridge_disable+0x24/0x30 drm_atomic_bridge_chain_disable+0x90/0xbc drm_atomic_helper_commit_modeset_disables+0x198/0x444 msm_atomic_commit_tail+0x1d0/0x374 commit_tail+0x80/0x108 drm_atomic_helper_commit+0x118/0x11c drm_atomic_commit+0xb4/0xe0 drm_client_modeset_commit_atomic+0x184/0x224 drm_client_modeset_commit_locked+0x58/0x160 drm_client_modeset_commit+0x3c/0x64 __drm_fb_helper_restore_fbdev_mode_unlocked+0x98/0xac drm_fb_helper_set_par+0x74/0x80 drm_fb_helper_hotplug_event+0xdc/0xe0 __drm_fb_helper_restore_fbdev_mode_unlocked+0x7c/0xac drm_fb_helper_restore_fbdev_mode_unlocked+0x20/0x2c drm_fb_helper_lastclose+0x20/0x2c drm_lastclose+0x44/0x6c drm_release+0x88/0xd4 __fput+0x104/0x220 ____fput+0x1c/0x28 task_work_run+0x8c/0x100 do_exit+0x450/0x8d0 do_group_exit+0x40/0xac __wake_up_parent+0x0/0x38 invoke_syscall+0x84/0x11c el0_svc_common.constprop.0+0xb8/0xe4 do_el0_svc+0x8c/0xb8 el0_svc+0x2c/0x54 el0t_64_sync_handler+0x120/0x1c0 el0t_64_sync+0x190/0x194 SMP: stopping secondary CPUs Kernel Offset: 0x128e800000 from 0xffffffc008000000 PHYS_OFFSET: 0x80000000 CPU features: 0x800,00c2a015,19801c82 Memory Limit: none Changes in v2: -- add more commit text Changes in v3: -- add comments into dp_bridge_atomic_check() Changes in v4: -- rewording the comment into dp_bridge_atomic_check() Changes in v5: -- removed quote x at end of commit text Changes in v6: -- removed quote x at end of comment in dp_bridge_atomic_check() Fixes: 8a3b4c17f863 ("drm/msm/dp: employ bridge mechanism for display enable and disable") Reported-by: Leonard Lausen Suggested-by: Rob Clark Closes: https://gitlab.freedesktop.org/drm/msm/-/issues/17 Signed-off-by: Kuogee Hsieh Reviewed-by: Abhinav Kumar Patchwork: https://patchwork.freedesktop.org/patch/505331/ Link: https://lore.kernel.org/r/1664408211-25314-1-git-send-email-quic_khsieh@quicinc.com Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dp/dp_drm.c | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/drivers/gpu/drm/msm/dp/dp_drm.c b/drivers/gpu/drm/msm/dp/dp_drm.c index 6df25f7662e79..6db82f9b03afb 100644 --- a/drivers/gpu/drm/msm/dp/dp_drm.c +++ b/drivers/gpu/drm/msm/dp/dp_drm.c @@ -31,6 +31,36 @@ static enum drm_connector_status dp_bridge_detect(struct drm_bridge *bridge) connector_status_disconnected; } +static int dp_bridge_atomic_check(struct drm_bridge *bridge, + struct drm_bridge_state *bridge_state, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) +{ + struct msm_dp *dp; + + dp = to_dp_bridge(bridge)->dp_display; + + drm_dbg_dp(dp->drm_dev, "is_connected = %s\n", + (dp->is_connected) ? "true" : "false"); + + /* + * There is no protection in the DRM framework to check if the display + * pipeline has been already disabled before trying to disable it again. + * Hence if the sink is unplugged, the pipeline gets disabled, but the + * crtc->active is still true. Any attempt to set the mode or manually + * disable this encoder will result in the crash. + * + * TODO: add support for telling the DRM subsystem that the pipeline is + * disabled by the hardware and thus all access to it should be forbidden. + * After that this piece of code can be removed. + */ + if (bridge->ops & DRM_BRIDGE_OP_HPD) + return (dp->is_connected) ? 0 : -ENOTCONN; + + return 0; +} + + /** * dp_bridge_get_modes - callback to add drm modes via drm_mode_probed_add() * @bridge: Poiner to drm bridge @@ -61,6 +91,9 @@ static int dp_bridge_get_modes(struct drm_bridge *bridge, struct drm_connector * } static const struct drm_bridge_funcs dp_bridge_ops = { + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .atomic_reset = drm_atomic_helper_bridge_reset, .enable = dp_bridge_enable, .disable = dp_bridge_disable, .post_disable = dp_bridge_post_disable, @@ -68,6 +101,7 @@ static const struct drm_bridge_funcs dp_bridge_ops = { .mode_valid = dp_bridge_mode_valid, .get_modes = dp_bridge_get_modes, .detect = dp_bridge_detect, + .atomic_check = dp_bridge_atomic_check, }; struct drm_bridge *dp_bridge_init(struct msm_dp *dp_display, struct drm_device *dev, -- GitLab From 0b33a33bd15d5bab73b87152b220a8d0153a4587 Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Tue, 13 Sep 2022 13:55:48 -0700 Subject: [PATCH 0017/1927] drm/msm: Fix return type of mdp4_lvds_connector_mode_valid The mode_valid field in drm_connector_helper_funcs is expected to be of type: enum drm_mode_status (* mode_valid) (struct drm_connector *connector, struct drm_display_mode *mode); The mismatched return type breaks forward edge kCFI since the underlying function definition does not match the function hook definition. The return type of mdp4_lvds_connector_mode_valid should be changed from int to enum drm_mode_status. Reported-by: Dan Carpenter Link: https://github.com/ClangBuiltLinux/linux/issues/1703 Cc: llvm@lists.linux.dev Signed-off-by: Nathan Huckleberry Fixes: 3e87599b68e7 ("drm/msm/mdp4: add LVDS panel support") Reviewed-by: Abhinav Kumar Reviewed-by: Nathan Chancellor Patchwork: https://patchwork.freedesktop.org/patch/502878/ Link: https://lore.kernel.org/r/20220913205551.155128-1-nhuck@google.com Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/disp/mdp4/mdp4_lvds_connector.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_lvds_connector.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_lvds_connector.c index 7288041dd86ad..7444b75c42157 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_lvds_connector.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_lvds_connector.c @@ -56,8 +56,9 @@ static int mdp4_lvds_connector_get_modes(struct drm_connector *connector) return ret; } -static int mdp4_lvds_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) +static enum drm_mode_status +mdp4_lvds_connector_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) { struct mdp4_lvds_connector *mdp4_lvds_connector = to_mdp4_lvds_connector(connector); -- GitLab From 6808abdb33bf90330e70a687d29f038507e06ebb Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:11 +0200 Subject: [PATCH 0018/1927] drm/msm: fix use-after-free on probe deferral The bridge counter was never reset when tearing down the DRM device so that stale pointers to deallocated structures would be accessed on the next tear down (e.g. after a second late bind deferral). Given enough bridges and a few probe deferrals this could currently also lead to data beyond the bridge array being corrupted. Fixes: d28ea556267c ("drm/msm: properly add and remove internal bridges") Fixes: a3376e3ec81c ("drm/msm: convert to drm_bridge") Cc: stable@vger.kernel.org # 3.12 Reviewed-by: Dmitry Baryshkov Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Patchwork: https://patchwork.freedesktop.org/patch/502665/ Link: https://lore.kernel.org/r/20220913085320.8577-2-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/msm_drv.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 07f66412533b9..fec6d449eded5 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -247,6 +247,7 @@ static int msm_drm_uninit(struct device *dev) for (i = 0; i < priv->num_bridges; i++) drm_bridge_remove(priv->bridges[i]); + priv->num_bridges = 0; pm_runtime_get_sync(dev); msm_irq_uninstall(ddev); -- GitLab From 74466e46e7543c7f74f1502181e9ba93f7521374 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:12 +0200 Subject: [PATCH 0019/1927] drm/msm/dp: fix memory corruption with too many bridges Add the missing sanity check on the bridge counter to avoid corrupting data beyond the fixed-sized bridge array in case there are ever more than eight bridges. Fixes: 8a3b4c17f863 ("drm/msm/dp: employ bridge mechanism for display enable and disable") Cc: stable@vger.kernel.org # 5.17 Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Reviewed-by: Dmitry Baryshkov Patchwork: https://patchwork.freedesktop.org/patch/502664/ Link: https://lore.kernel.org/r/20220913085320.8577-3-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dp/dp_display.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index bfd0aeff3f0d0..be9ed891dc3f8 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -1597,6 +1597,12 @@ int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev, return -EINVAL; priv = dev->dev_private; + + if (priv->num_bridges == ARRAY_SIZE(priv->bridges)) { + DRM_DEV_ERROR(dev->dev, "too many bridges\n"); + return -ENOSPC; + } + dp_display->drm_dev = dev; dp_priv = container_of(dp_display, struct dp_display_private, dp_display); -- GitLab From 2e786eb2f9cebb07e317226b60054df510b60c65 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:13 +0200 Subject: [PATCH 0020/1927] drm/msm/dsi: fix memory corruption with too many bridges Add the missing sanity check on the bridge counter to avoid corrupting data beyond the fixed-sized bridge array in case there are ever more than eight bridges. Fixes: a689554ba6ed ("drm/msm: Initial add DSI connector support") Cc: stable@vger.kernel.org # 4.1 Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Reviewed-by: Dmitry Baryshkov Patchwork: https://patchwork.freedesktop.org/patch/502668/ Link: https://lore.kernel.org/r/20220913085320.8577-4-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dsi/dsi.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c index 39bbabb5daf6f..8a95c744972a1 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.c +++ b/drivers/gpu/drm/msm/dsi/dsi.c @@ -218,6 +218,12 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev, return -EINVAL; priv = dev->dev_private; + + if (priv->num_bridges == ARRAY_SIZE(priv->bridges)) { + DRM_DEV_ERROR(dev->dev, "too many bridges\n"); + return -ENOSPC; + } + msm_dsi->dev = dev; ret = msm_dsi_host_modeset_init(msm_dsi->host, dev); -- GitLab From 4c1294da6aed1f16d47a417dcfe6602833c3c95c Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:14 +0200 Subject: [PATCH 0021/1927] drm/msm/hdmi: fix memory corruption with too many bridges Add the missing sanity check on the bridge counter to avoid corrupting data beyond the fixed-sized bridge array in case there are ever more than eight bridges. Fixes: a3376e3ec81c ("drm/msm: convert to drm_bridge") Cc: stable@vger.kernel.org # 3.12 Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Reviewed-by: Dmitry Baryshkov Patchwork: https://patchwork.freedesktop.org/patch/502670/ Link: https://lore.kernel.org/r/20220913085320.8577-5-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/hdmi/hdmi.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c index 93fe61b869670..a0ed6aa8e4e1a 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi.c @@ -300,6 +300,11 @@ int msm_hdmi_modeset_init(struct hdmi *hdmi, struct platform_device *pdev = hdmi->pdev; int ret; + if (priv->num_bridges == ARRAY_SIZE(priv->bridges)) { + DRM_DEV_ERROR(dev->dev, "too many bridges\n"); + return -ENOSPC; + } + hdmi->dev = dev; hdmi->encoder = encoder; -- GitLab From a79343dcaba4b11adb57350e0b6426906a9b658e Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:15 +0200 Subject: [PATCH 0022/1927] drm/msm/dp: fix IRQ lifetime Device-managed resources allocated post component bind must be tied to the lifetime of the aggregate DRM device or they will not necessarily be released when binding of the aggregate device is deferred. This is specifically true for the DP IRQ, which will otherwise remain requested so that the next bind attempt fails when requesting the IRQ a second time. Since commit c3bf8e21b38a ("drm/msm/dp: Add eDP support via aux_bus") this can happen when the aux-bus panel driver has not yet been loaded so that probe is deferred. Fix this by tying the device-managed lifetime of the DP IRQ to the DRM device so that it is released when bind fails. Fixes: c943b4948b58 ("drm/msm/dp: add displayPort driver support") Cc: stable@vger.kernel.org # 5.10 Reviewed-by: Dmitry Baryshkov Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Patchwork: https://patchwork.freedesktop.org/patch/502679/ Link: https://lore.kernel.org/r/20220913085320.8577-6-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dp/dp_display.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index be9ed891dc3f8..352cc09f20698 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -1249,7 +1249,7 @@ int dp_display_request_irq(struct msm_dp *dp_display) return -EINVAL; } - rc = devm_request_irq(&dp->pdev->dev, dp->irq, + rc = devm_request_irq(dp_display->drm_dev->dev, dp->irq, dp_display_irq_handler, IRQF_TRIGGER_HIGH, "dp_display_isr", dp); if (rc < 0) { -- GitLab From 2b57f726611e294dc4297dd48eb8c98ef1938e82 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:16 +0200 Subject: [PATCH 0023/1927] drm/msm/dp: fix aux-bus EP lifetime Device-managed resources allocated post component bind must be tied to the lifetime of the aggregate DRM device or they will not necessarily be released when binding of the aggregate device is deferred. This can lead resource leaks or failure to bind the aggregate device when binding is later retried and a second attempt to allocate the resources is made. For the DP aux-bus, an attempt to populate the bus a second time will simply fail ("DP AUX EP device already populated"). Fix this by tying the lifetime of the EP device to the DRM device rather than DP controller platform device. Fixes: c3bf8e21b38a ("drm/msm/dp: Add eDP support via aux_bus") Cc: stable@vger.kernel.org # 5.19 Signed-off-by: Johan Hovold Reviewed-by: Douglas Anderson Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Patchwork: https://patchwork.freedesktop.org/patch/502672/ Link: https://lore.kernel.org/r/20220913085320.8577-7-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dp/dp_display.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 352cc09f20698..42de690132cf5 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -1528,6 +1528,11 @@ void msm_dp_debugfs_init(struct msm_dp *dp_display, struct drm_minor *minor) } } +static void of_dp_aux_depopulate_bus_void(void *data) +{ + of_dp_aux_depopulate_bus(data); +} + static int dp_display_get_next_bridge(struct msm_dp *dp) { int rc; @@ -1552,10 +1557,16 @@ static int dp_display_get_next_bridge(struct msm_dp *dp) * panel driver is probed asynchronously but is the best we * can do without a bigger driver reorganization. */ - rc = devm_of_dp_aux_populate_ep_devices(dp_priv->aux); + rc = of_dp_aux_populate_bus(dp_priv->aux, NULL); of_node_put(aux_bus); if (rc) goto error; + + rc = devm_add_action_or_reset(dp->drm_dev->dev, + of_dp_aux_depopulate_bus_void, + dp_priv->aux); + if (rc) + goto error; } else if (dp->is_edp) { DRM_ERROR("eDP aux_bus not found\n"); return -ENODEV; -- GitLab From 16194958f888d63839042d1190f7001e5ddec47b Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:17 +0200 Subject: [PATCH 0024/1927] drm/msm/dp: fix bridge lifetime Device-managed resources allocated post component bind must be tied to the lifetime of the aggregate DRM device or they will not necessarily be released when binding of the aggregate device is deferred. This can lead resource leaks or failure to bind the aggregate device when binding is later retried and a second attempt to allocate the resources is made. For the DP bridges, previously allocated bridges will leak on probe deferral. Fix this by amending the DP parser interface and tying the lifetime of the bridge device to the DRM device rather than DP platform device. Fixes: c3bf8e21b38a ("drm/msm/dp: Add eDP support via aux_bus") Cc: stable@vger.kernel.org # 5.19 Reviewed-by: Dmitry Baryshkov Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Patchwork: https://patchwork.freedesktop.org/patch/502667/ Link: https://lore.kernel.org/r/20220913085320.8577-8-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dp/dp_display.c | 2 +- drivers/gpu/drm/msm/dp/dp_parser.c | 6 +++--- drivers/gpu/drm/msm/dp/dp_parser.h | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index 42de690132cf5..a49f6dbbe8883 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -1579,7 +1579,7 @@ static int dp_display_get_next_bridge(struct msm_dp *dp) * For DisplayPort interfaces external bridges are optional, so * silently ignore an error if one is not present (-ENODEV). */ - rc = dp_parser_find_next_bridge(dp_priv->parser); + rc = devm_dp_parser_find_next_bridge(dp->drm_dev->dev, dp_priv->parser); if (!dp->is_edp && rc == -ENODEV) return 0; diff --git a/drivers/gpu/drm/msm/dp/dp_parser.c b/drivers/gpu/drm/msm/dp/dp_parser.c index dd732215d55b4..dcbe893d66d7b 100644 --- a/drivers/gpu/drm/msm/dp/dp_parser.c +++ b/drivers/gpu/drm/msm/dp/dp_parser.c @@ -240,12 +240,12 @@ static int dp_parser_clock(struct dp_parser *parser) return 0; } -int dp_parser_find_next_bridge(struct dp_parser *parser) +int devm_dp_parser_find_next_bridge(struct device *dev, struct dp_parser *parser) { - struct device *dev = &parser->pdev->dev; + struct platform_device *pdev = parser->pdev; struct drm_bridge *bridge; - bridge = devm_drm_of_get_bridge(dev, dev->of_node, 1, 0); + bridge = devm_drm_of_get_bridge(dev, pdev->dev.of_node, 1, 0); if (IS_ERR(bridge)) return PTR_ERR(bridge); diff --git a/drivers/gpu/drm/msm/dp/dp_parser.h b/drivers/gpu/drm/msm/dp/dp_parser.h index 866c1a82bf1af..d30ab773db46d 100644 --- a/drivers/gpu/drm/msm/dp/dp_parser.h +++ b/drivers/gpu/drm/msm/dp/dp_parser.h @@ -138,8 +138,9 @@ struct dp_parser { struct dp_parser *dp_parser_get(struct platform_device *pdev); /** - * dp_parser_find_next_bridge() - find an additional bridge to DP + * devm_dp_parser_find_next_bridge() - find an additional bridge to DP * + * @dev: device to tie bridge lifetime to * @parser: dp_parser data from client * * This function is used to find any additional bridge attached to @@ -147,6 +148,6 @@ struct dp_parser *dp_parser_get(struct platform_device *pdev); * * Return: 0 if able to get the bridge, otherwise negative errno for failure. */ -int dp_parser_find_next_bridge(struct dp_parser *parser); +int devm_dp_parser_find_next_bridge(struct device *dev, struct dp_parser *parser); #endif -- GitLab From 152d394842bb564148e68b92486a87db0bf54859 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Tue, 13 Sep 2022 10:53:18 +0200 Subject: [PATCH 0025/1927] drm/msm/hdmi: fix IRQ lifetime Device-managed resources allocated post component bind must be tied to the lifetime of the aggregate DRM device or they will not necessarily be released when binding of the aggregate device is deferred. This is specifically true for the HDMI IRQ, which will otherwise remain requested so that the next bind attempt fails when requesting the IRQ a second time. Fix this by tying the device-managed lifetime of the HDMI IRQ to the DRM device so that it is released when bind fails. Fixes: 067fef372c73 ("drm/msm/hdmi: refactor bind/init") Cc: stable@vger.kernel.org # 3.19 Reviewed-by: Dmitry Baryshkov Signed-off-by: Johan Hovold Tested-by: Kuogee Hsieh Reviewed-by: Kuogee Hsieh Patchwork: https://patchwork.freedesktop.org/patch/502666/ Link: https://lore.kernel.org/r/20220913085320.8577-9-johan+linaro@kernel.org Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/hdmi/hdmi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c index a0ed6aa8e4e1a..f28fb21e38911 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi.c +++ b/drivers/gpu/drm/msm/hdmi/hdmi.c @@ -344,7 +344,7 @@ int msm_hdmi_modeset_init(struct hdmi *hdmi, goto fail; } - ret = devm_request_irq(&pdev->dev, hdmi->irq, + ret = devm_request_irq(dev->dev, hdmi->irq, msm_hdmi_irq, IRQF_TRIGGER_HIGH, "hdmi_isr", hdmi); if (ret < 0) { -- GitLab From 70445dee1b4cf44c9fecc580dfa08079011866f1 Mon Sep 17 00:00:00 2001 From: Kuogee Hsieh Date: Mon, 12 Sep 2022 09:23:48 -0700 Subject: [PATCH 0026/1927] drm/msm/dp: cleared DP_DOWNSPREAD_CTRL register before start link training DOWNSPREAD_CTRL (0x107) shall be cleared to 0 upon power-on reset or an upstream device disconnect. This patch will enforce this rule by always cleared DOWNSPREAD_CTRL register to 0 before start link training. At rare case that DP MSA timing parameters may be mis-interpreted by the sink which causes audio sampling rate be calculated wrongly and cause audio did not work at sink if DOWNSPREAD_CTRL register is not cleared to 0. Changes in v2: 1) fix spelling at commit text 2) merge ssc variable into encoding[0] Changes in v3: -- correct spelling of DOWNSPREAD_CTRL -- replace err with len of ssize_t Changes in v4: -- split into 2 patches Signed-off-by: Kuogee Hsieh Reviewed-by: Dmitry Baryshkov Fixes: c943b4948b58 ("drm/msm/dp: add displayPort driver support") Patchwork: https://patchwork.freedesktop.org/patch/502532/ Link: https://lore.kernel.org/r/1662999830-13916-2-git-send-email-quic_khsieh@quicinc.com Signed-off-by: Abhinav Kumar --- drivers/gpu/drm/msm/dp/dp_ctrl.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index 3854c9f1f7e90..dd26ca651a054 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -1243,8 +1243,7 @@ static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl, { int ret = 0; const u8 *dpcd = ctrl->panel->dpcd; - u8 encoding = DP_SET_ANSI_8B10B; - u8 ssc; + u8 encoding[] = { 0, DP_SET_ANSI_8B10B }; u8 assr; struct dp_link_info link_info = {0}; @@ -1256,13 +1255,11 @@ static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl, dp_aux_link_configure(ctrl->aux, &link_info); - if (drm_dp_max_downspread(dpcd)) { - ssc = DP_SPREAD_AMP_0_5; - drm_dp_dpcd_write(ctrl->aux, DP_DOWNSPREAD_CTRL, &ssc, 1); - } + if (drm_dp_max_downspread(dpcd)) + encoding[0] |= DP_SPREAD_AMP_0_5; - drm_dp_dpcd_write(ctrl->aux, DP_MAIN_LINK_CHANNEL_CODING_SET, - &encoding, 1); + /* config DOWNSPREAD_CTRL and MAIN_LINK_CHANNEL_CODING_SET */ + drm_dp_dpcd_write(ctrl->aux, DP_DOWNSPREAD_CTRL, encoding, 2); if (drm_dp_alternate_scrambler_reset_cap(dpcd)) { assr = DP_ALTERNATE_SCRAMBLER_RESET_ENABLE; -- GitLab From 1f1009ea8ca5a0271ad69afe8a86c887d530b5c8 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Fri, 9 Sep 2022 12:04:22 +0300 Subject: [PATCH 0027/1927] scsi: target: core: Fix preempt and abort for allreg res Match a key only if SARK is not zero according to SPC-4 and the comment above the code: If an all registrants persistent reservation is present and the SERVICE ACTION RESERVATION KEY field is set to zero, then all registrations shall be removed except for that of the I_T nexus that is being used for the PERSISTENT RESERVE OUT command; Without this patch in case of SARK==0 no registrants will be removed. Link: https://lore.kernel.org/r/20220909090425.14479-2-d.bogdanov@yadro.com Reviewed-by: Mike Christie Signed-off-by: Dmitry Bogdanov Signed-off-by: Martin K. Petersen --- drivers/target/target_core_pr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index a1d67554709f3..1521a97ddac25 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -3022,7 +3022,7 @@ core_scsi3_pro_preempt(struct se_cmd *cmd, int type, int scope, u64 res_key, if (calling_it_nexus) continue; - if (pr_reg->pr_res_key != sa_res_key) + if (sa_res_key && pr_reg->pr_res_key != sa_res_key) continue; pr_reg_nacl = pr_reg->pr_reg_nacl; -- GitLab From f050a7c66ca56aa2f49ab9b53e01d04b3e7e94c5 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Fri, 9 Sep 2022 12:04:23 +0300 Subject: [PATCH 0028/1927] scsi: target: core: Fix memory leak in preempt_and_abort Always release preempt_and_abort_list to avoid memory leak of t10_pr_registration objects in it. Link: https://lore.kernel.org/r/20220909090425.14479-3-d.bogdanov@yadro.com Reviewed-by: Mike Christie Signed-off-by: Dmitry Bogdanov Signed-off-by: Martin K. Petersen --- drivers/target/target_core_pr.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index 1521a97ddac25..e3869576f254f 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -2956,13 +2956,14 @@ core_scsi3_pro_preempt(struct se_cmd *cmd, int type, int scope, u64 res_key, __core_scsi3_complete_pro_preempt(dev, pr_reg_n, (preempt_type == PREEMPT_AND_ABORT) ? &preempt_and_abort_list : NULL, type, scope, preempt_type); - - if (preempt_type == PREEMPT_AND_ABORT) - core_scsi3_release_preempt_and_abort( - &preempt_and_abort_list, pr_reg_n); } + spin_unlock(&dev->dev_reservation_lock); + if (preempt_type == PREEMPT_AND_ABORT) + core_scsi3_release_preempt_and_abort( + &preempt_and_abort_list, pr_reg_n); + if (pr_tmpl->pr_aptpl_active) core_scsi3_update_and_write_aptpl(cmd->se_dev, true); -- GitLab From 49790e6a582012c36ca17174cda228444f9a2414 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Fri, 9 Sep 2022 12:04:24 +0300 Subject: [PATCH 0029/1927] scsi: target: core: Abort all preempted regs if requested According to SPC the preempted commands shall be always aborted. SPC-4: 5.12.11.2.6 Preempting and aborting c) all commands from the I_T nexus(es) associated with the persistent reservations or registrations being preempted (i.e., preempted commands) except the PERSISTENT RESERVE OUT command itself shall be aborted as defined in SAM-5; Link: https://lore.kernel.org/r/20220909090425.14479-4-d.bogdanov@yadro.com Reviewed-by: Mike Christie Signed-off-by: Dmitry Bogdanov Signed-off-by: Martin K. Petersen --- drivers/target/target_core_pr.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index e3869576f254f..6a5f9504a4811 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -2960,9 +2960,23 @@ core_scsi3_pro_preempt(struct se_cmd *cmd, int type, int scope, u64 res_key, spin_unlock(&dev->dev_reservation_lock); - if (preempt_type == PREEMPT_AND_ABORT) + /* + * SPC-4 5.12.11.2.6 Preempting and aborting + * The actions described in this subclause shall be performed + * for all I_T nexuses that are registered with the non-zero + * SERVICE ACTION RESERVATION KEY value, without regard for + * whether the preempted I_T nexuses hold the persistent + * reservation. If the SERVICE ACTION RESERVATION KEY field is + * set to zero and an all registrants persistent reservation is + * present, the device server shall abort all commands for all + * registered I_T nexuses. + */ + if (preempt_type == PREEMPT_AND_ABORT) { + core_tmr_lun_reset(dev, NULL, &preempt_and_abort_list, + cmd); core_scsi3_release_preempt_and_abort( &preempt_and_abort_list, pr_reg_n); + } if (pr_tmpl->pr_aptpl_active) core_scsi3_update_and_write_aptpl(cmd->se_dev, true); -- GitLab From 3e2deba7aa662862c8046aa24148b83b49298a9b Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Fri, 9 Sep 2022 12:04:25 +0300 Subject: [PATCH 0030/1927] scsi: target: core: New key must be used for moved PR According to SPC4 5.12.8: e) Retain the reservation key specified in the SERVICE ACTION RESERVATION KEY field and associated information; But currently sa_res_key is only used for the not existing I_T nexus. Add a changing of the key for the existing I_T nexus the PR moved to. Link: https://lore.kernel.org/r/20220909090425.14479-5-d.bogdanov@yadro.com Reviewed-by: Mike Christie Signed-off-by: Dmitry Bogdanov Signed-off-by: Martin K. Petersen --- drivers/target/target_core_pr.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c index 6a5f9504a4811..1493b1d01194f 100644 --- a/drivers/target/target_core_pr.c +++ b/drivers/target/target_core_pr.c @@ -3440,8 +3440,6 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, * transport protocols where port names are not required; * d) Register the reservation key specified in the SERVICE ACTION * RESERVATION KEY field; - * e) Retain the reservation key specified in the SERVICE ACTION - * RESERVATION KEY field and associated information; * * Also, It is not an error for a REGISTER AND MOVE service action to * register an I_T nexus that is already registered with the same @@ -3463,6 +3461,12 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key, dest_pr_reg = __core_scsi3_locate_pr_reg(dev, dest_node_acl, iport_ptr); new_reg = 1; + } else { + /* + * e) Retain the reservation key specified in the SERVICE ACTION + * RESERVATION KEY field and associated information; + */ + dest_pr_reg->pr_res_key = sa_res_key; } /* * f) Release the persistent reservation for the persistent reservation -- GitLab From 6290e23f3bd8cee52fb8fd98980bb1eb31c8284d Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Tue, 13 Sep 2022 19:36:02 +0300 Subject: [PATCH 0031/1927] scsi: target: core: UA on all LUNs after reset Allocate UNIT ATTENTION "BUS DEVICE RESET OCCURRED" on all LUNs on all target ports of the device upon reception of TMF LUN RESET. This change passes libiscsi test SCSI.MultipathIO.Reset. Link: https://lore.kernel.org/r/20220913163602.20597-1-d.bogdanov@yadro.com Signed-off-by: Dmitry Bogdanov Signed-off-by: Martin K. Petersen --- drivers/target/target_core_device.c | 19 +++++++++++++++++++ drivers/target/target_core_internal.h | 1 + drivers/target/target_core_transport.c | 3 +-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c index b7f16ee8aa0e5..cb4f7cc02f8fa 100644 --- a/drivers/target/target_core_device.c +++ b/drivers/target/target_core_device.c @@ -284,6 +284,25 @@ void target_pr_kref_release(struct kref *kref) complete(&deve->pr_comp); } +/* + * Establish UA condition on SCSI device - all LUNs + */ +void target_dev_ua_allocate(struct se_device *dev, u8 asc, u8 ascq) +{ + struct se_dev_entry *se_deve; + struct se_lun *lun; + + spin_lock(&dev->se_port_lock); + list_for_each_entry(lun, &dev->dev_sep_list, lun_dev_link) { + + spin_lock(&lun->lun_deve_lock); + list_for_each_entry(se_deve, &lun->lun_deve_list, lun_link) + core_scsi3_ua_allocate(se_deve, asc, ascq); + spin_unlock(&lun->lun_deve_lock); + } + spin_unlock(&dev->se_port_lock); +} + static void target_luns_data_has_changed(struct se_node_acl *nacl, struct se_dev_entry *new, bool skip_new) diff --git a/drivers/target/target_core_internal.h b/drivers/target/target_core_internal.h index 30fcf69e1a1d5..38a6d08f75b34 100644 --- a/drivers/target/target_core_internal.h +++ b/drivers/target/target_core_internal.h @@ -89,6 +89,7 @@ int target_configure_device(struct se_device *dev); void target_free_device(struct se_device *); int target_for_each_device(int (*fn)(struct se_device *dev, void *data), void *data); +void target_dev_ua_allocate(struct se_device *dev, u8 asc, u8 ascq); /* target_core_configfs.c */ extern struct configfs_item_operations target_core_dev_item_ops; diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 7838dc20f7130..5926316252eb9 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -3531,8 +3531,7 @@ static void target_tmr_work(struct work_struct *work) tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE : TMR_FUNCTION_REJECTED; if (tmr->response == TMR_FUNCTION_COMPLETE) { - target_ua_allocate_lun(cmd->se_sess->se_node_acl, - cmd->orig_fe_lun, 0x29, + target_dev_ua_allocate(dev, 0x29, ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED); } break; -- GitLab From 9b78d8fadeee078ca947a3b44157f42035fdf8b1 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 14 Sep 2022 16:47:59 +0800 Subject: [PATCH 0032/1927] scsi: megaraid_sas: Correct value passed to scsi_device_lookup() The "id" parameter currently passed to scsi_device_lookup() when removing a device is incorrect. It should be "ld_target_id % MEGASAS_MAX_DEV_PER_CHANNEL". Link: https://lore.kernel.org/r/1663145283-4872-2-git-send-email-kanie@linux.alibaba.com Fixes: ae6874ba4b43 ("scsi: megaraid_sas: Early detection of VD deletion through RaidMap update") Acked-by: Sumit Saxena Signed-off-by: Guixin Liu Signed-off-by: Martin K. Petersen --- drivers/scsi/megaraid/megaraid_sas_base.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index ae6b9a570fa92..1772b0be88fe7 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -8924,7 +8924,7 @@ megasas_aen_polling(struct work_struct *work) sdev1 = scsi_device_lookup(instance->host, MEGASAS_MAX_PD_CHANNELS + (ld_target_id / MEGASAS_MAX_DEV_PER_CHANNEL), - (ld_target_id - MEGASAS_MAX_DEV_PER_CHANNEL), + (ld_target_id % MEGASAS_MAX_DEV_PER_CHANNEL), 0); if (sdev1) megasas_remove_scsi_device(sdev1); -- GitLab From 9b201b5dff81f298cebda10d51767cd25b432a1a Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 14 Sep 2022 16:48:00 +0800 Subject: [PATCH 0033/1927] scsi: megaraid_sas: Correct an error message Correct the error message logged when allocation of ioc_init_request fails. Link: https://lore.kernel.org/r/1663145283-4872-3-git-send-email-kanie@linux.alibaba.com Acked-by: Sumit Saxena Signed-off-by: Guixin Liu Signed-off-by: Martin K. Petersen --- drivers/scsi/megaraid/megaraid_sas_base.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index 1772b0be88fe7..22c3e3370403a 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -7226,7 +7226,7 @@ int megasas_alloc_ctrl_dma_buffers(struct megasas_instance *instance) if (!fusion->ioc_init_request) { dev_err(&pdev->dev, - "Failed to allocate PD list buffer\n"); + "Failed to allocate ioc init request\n"); return -ENOMEM; } -- GitLab From 17883cd59f5575ebe5b3cce2fd0f0d91738871bb Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 14 Sep 2022 16:48:01 +0800 Subject: [PATCH 0034/1927] scsi: megaraid_sas: Simplify megasas_update_device_list Remove unnecessary dcmd_ret check and goto statement. Link: https://lore.kernel.org/r/1663145283-4872-4-git-send-email-kanie@linux.alibaba.com Acked-by: Sumit Saxena Signed-off-by: Guixin Liu Signed-off-by: Martin K. Petersen --- drivers/scsi/megaraid/megaraid_sas_base.c | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index 22c3e3370403a..4e8b7042e8b38 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -8768,33 +8768,26 @@ static int megasas_update_device_list(struct megasas_instance *instance, int event_type) { - int dcmd_ret = DCMD_SUCCESS; + int dcmd_ret; if (instance->enable_fw_dev_list) { - dcmd_ret = megasas_host_device_list_query(instance, false); - if (dcmd_ret != DCMD_SUCCESS) - goto out; + return megasas_host_device_list_query(instance, false); } else { if (event_type & SCAN_PD_CHANNEL) { dcmd_ret = megasas_get_pd_list(instance); - if (dcmd_ret != DCMD_SUCCESS) - goto out; + return dcmd_ret; } if (event_type & SCAN_VD_CHANNEL) { if (!instance->requestorId || megasas_get_ld_vf_affiliation(instance, 0)) { - dcmd_ret = megasas_ld_list_query(instance, + return megasas_ld_list_query(instance, MR_LD_QUERY_TYPE_EXPOSED_TO_HOST); - if (dcmd_ret != DCMD_SUCCESS) - goto out; } } } - -out: - return dcmd_ret; + return DCMD_SUCCESS; } /** -- GitLab From ad40d51992392a2336af861f83c17c0b08ca64b6 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 14 Sep 2022 16:48:02 +0800 Subject: [PATCH 0035/1927] scsi: megaraid_sas: Remove unnecessary memset() Remove memset() of pd_list and ld_ids in megasas_get_device_list(). These lists will be cleared by megasas_host_device_list_query(), megasas_get_pd_list(), and megasas_ld_list_query(). Link: https://lore.kernel.org/r/1663145283-4872-5-git-send-email-kanie@linux.alibaba.com Acked-by: Sumit Saxena Signed-off-by: Guixin Liu Signed-off-by: Martin K. Petersen --- drivers/scsi/megaraid/megaraid_sas_base.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index 4e8b7042e8b38..f5e8c7cd0dcaa 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -5876,10 +5876,6 @@ static void megasas_setup_reply_map(struct megasas_instance *instance) static int megasas_get_device_list(struct megasas_instance *instance) { - memset(instance->pd_list, 0, - (MEGASAS_MAX_PD * sizeof(struct megasas_pd_list))); - memset(instance->ld_ids, 0xff, MEGASAS_MAX_LD_IDS); - if (instance->enable_fw_dev_list) { if (megasas_host_device_list_query(instance, true)) return FAILED; -- GitLab From 27b571cc454e5a5939b4940ed0bf20aaf37f5225 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 14 Sep 2022 16:48:03 +0800 Subject: [PATCH 0036/1927] scsi: megaraid_sas: Move megasas_dbg_lvl init to megasas_init() The megasas_dbg_lvl is a driver level parameter. Do not initialize it in the probe path. Otherwise we will miss the debug print when binding a new device to the megaraid driver. Link: https://lore.kernel.org/r/1663145283-4872-6-git-send-email-kanie@linux.alibaba.com Acked-by: Sumit Saxena Signed-off-by: Guixin Liu Signed-off-by: Martin K. Petersen --- drivers/scsi/megaraid/megaraid_sas_base.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index f5e8c7cd0dcaa..465274ba9f1be 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -7441,7 +7441,6 @@ static inline void megasas_init_ctrl_params(struct megasas_instance *instance) (instance->pdev->device == PCI_DEVICE_ID_LSI_SAS0071SKINNY)) instance->flag_ieee = 1; - megasas_dbg_lvl = 0; instance->flag = 0; instance->unload = 1; instance->last_time = 0; @@ -9011,6 +9010,7 @@ static int __init megasas_init(void) */ pr_info("megasas: %s\n", MEGASAS_VERSION); + megasas_dbg_lvl = 0; support_poll_for_event = 2; support_device_change = 1; support_nvme_encapsulation = true; -- GitLab From 07e433614cdb91e6f85cc79d738bb0a3d8c741a2 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Fri, 23 Sep 2022 18:12:17 +0800 Subject: [PATCH 0037/1927] scsi: ufs: qcom: Remove redundant dev_err() call devm_ioremap_resource() already prints an error message. Remove the redundant dev_err() call. Link: https://lore.kernel.org/r/20220923101217.18345-1-shangxiaojing@huawei.com Reviewed-by: Bart Van Assche Signed-off-by: Shang XiaoJing Signed-off-by: Martin K. Petersen --- drivers/ufs/host/ufs-qcom-ice.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/ufs/host/ufs-qcom-ice.c b/drivers/ufs/host/ufs-qcom-ice.c index 745e48ec598f8..62387ccd5b307 100644 --- a/drivers/ufs/host/ufs-qcom-ice.c +++ b/drivers/ufs/host/ufs-qcom-ice.c @@ -118,7 +118,6 @@ int ufs_qcom_ice_init(struct ufs_qcom_host *host) host->ice_mmio = devm_ioremap_resource(dev, res); if (IS_ERR(host->ice_mmio)) { err = PTR_ERR(host->ice_mmio); - dev_err(dev, "Failed to map ICE registers; err=%d\n", err); return err; } -- GitLab From 3ddeabd1536a71abf2b66a577c90df84514a0af2 Mon Sep 17 00:00:00 2001 From: Rafael Mendonca Date: Mon, 26 Sep 2022 20:02:44 -0300 Subject: [PATCH 0038/1927] scsi: qla2xxx: Fix serialization of DCBX TLV data request Commit b6faaaf796d7 ("scsi: qla2xxx: Serialize mailbox request") serialized mailbox requests from userspace using the 'optrom' mutex. However, in the case of DCBX TLV data, if the memory for it is already allocated, then the mailbox request ends up not being serialized because it is done without holding the 'optrom' mutex. Link: https://lore.kernel.org/r/20220926230245.790508-1-rafaelmendsr@gmail.com Fixes: b6faaaf796d7 ("scsi: qla2xxx: Serialize mailbox request") Reviewed-by: Himanshu Madhani Signed-off-by: Rafael Mendonca Signed-off-by: Martin K. Petersen --- drivers/scsi/qla2xxx/qla_attr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index fa1fcbfb946f9..c2bc7f9c728a2 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -951,9 +951,9 @@ qla2x00_sysfs_read_dcbx_tlv(struct file *filp, struct kobject *kobj, if (!capable(CAP_SYS_ADMIN) || off != 0 || count > DCBX_TLV_DATA_SIZE) return 0; + mutex_lock(&vha->hw->optrom_mutex); if (ha->dcbx_tlv) goto do_read; - mutex_lock(&vha->hw->optrom_mutex); if (qla2x00_chip_is_down(vha)) { mutex_unlock(&vha->hw->optrom_mutex); return 0; -- GitLab From f915f58e382e907e2be0b2f5472617dc13f2c390 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 27 Sep 2022 10:22:25 +0200 Subject: [PATCH 0039/1927] scsi: target: iblock: Fold iblock_emulate_read_cap_with_block_size() into iblock_get_blocks() Fold iblock_emulate_read_cap_with_block_size() into its only caller. Link: https://lore.kernel.org/r/20220927082225.271975-1-hch@lst.de Reviewed-by: Mike Christie Signed-off-by: Christoph Hellwig Signed-off-by: Martin K. Petersen --- drivers/target/target_core_iblock.c | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 8351c974cee32..d9266cf558dcb 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -230,14 +230,12 @@ static void iblock_unplug_device(struct se_dev_plug *se_plug) clear_bit(IBD_PLUGF_PLUGGED, &ib_dev_plug->flags); } -static unsigned long long iblock_emulate_read_cap_with_block_size( - struct se_device *dev, - struct block_device *bd, - struct request_queue *q) +static sector_t iblock_get_blocks(struct se_device *dev) { - u32 block_size = bdev_logical_block_size(bd); + struct iblock_dev *ib_dev = IBLOCK_DEV(dev); + u32 block_size = bdev_logical_block_size(ib_dev->ibd_bd); unsigned long long blocks_long = - div_u64(bdev_nr_bytes(bd), block_size) - 1; + div_u64(bdev_nr_bytes(ib_dev->ibd_bd), block_size) - 1; if (block_size == dev->dev_attrib.block_size) return blocks_long; @@ -829,15 +827,6 @@ iblock_execute_rw(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; } -static sector_t iblock_get_blocks(struct se_device *dev) -{ - struct iblock_dev *ib_dev = IBLOCK_DEV(dev); - struct block_device *bd = ib_dev->ibd_bd; - struct request_queue *q = bdev_get_queue(bd); - - return iblock_emulate_read_cap_with_block_size(dev, bd, q); -} - static sector_t iblock_get_alignment_offset_lbas(struct se_device *dev) { struct iblock_dev *ib_dev = IBLOCK_DEV(dev); -- GitLab From 0b863257c17c5f57a41e0a48de140ed026957a63 Mon Sep 17 00:00:00 2001 From: Manish Rangankar Date: Tue, 27 Sep 2022 04:59:46 -0700 Subject: [PATCH 0040/1927] scsi: qla2xxx: Use transport-defined speed mask for supported_speeds One of the sysfs values reported for supported_speeds was not valid (20Gb/s reported instead of 64Gb/s). Instead of driver internal speed mask definition, use speed mask defined in transport_fc for reporting host->supported_speeds. Link: https://lore.kernel.org/r/20220927115946.17559-1-njavali@marvell.com Cc: stable@vger.kernel.org Reviewed-by: Himanshu Madhani Signed-off-by: Manish Rangankar Signed-off-by: Nilesh Javali Signed-off-by: Martin K. Petersen --- drivers/scsi/qla2xxx/qla_attr.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index c2bc7f9c728a2..b67ad30d56e6f 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -3330,11 +3330,34 @@ struct fc_function_template qla2xxx_transport_vport_functions = { .bsg_timeout = qla24xx_bsg_timeout, }; +static uint +qla2x00_get_host_supported_speeds(scsi_qla_host_t *vha, uint speeds) +{ + uint supported_speeds = FC_PORTSPEED_UNKNOWN; + + if (speeds & FDMI_PORT_SPEED_64GB) + supported_speeds |= FC_PORTSPEED_64GBIT; + if (speeds & FDMI_PORT_SPEED_32GB) + supported_speeds |= FC_PORTSPEED_32GBIT; + if (speeds & FDMI_PORT_SPEED_16GB) + supported_speeds |= FC_PORTSPEED_16GBIT; + if (speeds & FDMI_PORT_SPEED_8GB) + supported_speeds |= FC_PORTSPEED_8GBIT; + if (speeds & FDMI_PORT_SPEED_4GB) + supported_speeds |= FC_PORTSPEED_4GBIT; + if (speeds & FDMI_PORT_SPEED_2GB) + supported_speeds |= FC_PORTSPEED_2GBIT; + if (speeds & FDMI_PORT_SPEED_1GB) + supported_speeds |= FC_PORTSPEED_1GBIT; + + return supported_speeds; +} + void qla2x00_init_host_attr(scsi_qla_host_t *vha) { struct qla_hw_data *ha = vha->hw; - u32 speeds = FC_PORTSPEED_UNKNOWN; + u32 speeds = 0, fdmi_speed = 0; fc_host_dev_loss_tmo(vha->host) = ha->port_down_retry_count; fc_host_node_name(vha->host) = wwn_to_u64(vha->node_name); @@ -3344,7 +3367,8 @@ qla2x00_init_host_attr(scsi_qla_host_t *vha) fc_host_max_npiv_vports(vha->host) = ha->max_npiv_vports; fc_host_npiv_vports_inuse(vha->host) = ha->cur_vport_count; - speeds = qla25xx_fdmi_port_speed_capability(ha); + fdmi_speed = qla25xx_fdmi_port_speed_capability(ha); + speeds = qla2x00_get_host_supported_speeds(vha, fdmi_speed); fc_host_supported_speeds(vha->host) = speeds; } -- GitLab From 638eec06c7f4df8eb415a0b33dd18cc6dfc986e6 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 28 Sep 2022 23:21:16 +0100 Subject: [PATCH 0041/1927] scsi: lpfc: Fix spelling mistake "unsolicted" -> "unsolicited" There are spelling mistakes in a log message and two comments. Fix them. Link: https://lore.kernel.org/r/20220928222116.68294-1-colin.i.king@gmail.com Signed-off-by: Colin Ian King Signed-off-by: Martin K. Petersen --- drivers/scsi/lpfc/lpfc_bsg.c | 4 ++-- drivers/scsi/lpfc/lpfc_ct.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/scsi/lpfc/lpfc_bsg.c b/drivers/scsi/lpfc/lpfc_bsg.c index ac0c7ccf2eaee..852b025e2fecf 100644 --- a/drivers/scsi/lpfc/lpfc_bsg.c +++ b/drivers/scsi/lpfc/lpfc_bsg.c @@ -2582,7 +2582,7 @@ static int lpfcdiag_loop_self_unreg(struct lpfc_hba *phba, uint16_t rpi) * * This function obtains the transmit and receive ids required to send * an unsolicited ct command with a payload. A special lpfc FsType and CmdRsp - * flags are used to the unsolicted response handler is able to process + * flags are used to the unsolicited response handler is able to process * the ct command sent on the same port. **/ static int lpfcdiag_loop_get_xri(struct lpfc_hba *phba, uint16_t rpi, @@ -2874,7 +2874,7 @@ diag_cmd_data_alloc(struct lpfc_hba *phba, * @len: Number of data bytes * * This function allocates and posts a data buffer of sufficient size to receive - * an unsolicted CT command. + * an unsolicited CT command. **/ static int lpfcdiag_sli3_loop_post_rxbufs(struct lpfc_hba *phba, uint16_t rxxri, size_t len) diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c index 75fd2bfc212b2..e941a99aa9659 100644 --- a/drivers/scsi/lpfc/lpfc_ct.c +++ b/drivers/scsi/lpfc/lpfc_ct.c @@ -90,7 +90,7 @@ lpfc_ct_ignore_hbq_buffer(struct lpfc_hba *phba, struct lpfc_iocbq *piocbq, get_job_ulpstatus(phba, piocbq)); } lpfc_printf_log(phba, KERN_INFO, LOG_ELS, - "0145 Ignoring unsolicted CT HBQ Size:%d " + "0145 Ignoring unsolicited CT HBQ Size:%d " "status = x%x\n", size, get_job_ulpstatus(phba, piocbq)); } -- GitLab From 67d0a917fb3f9e80c3fb6098ada2080d1b425c94 Mon Sep 17 00:00:00 2001 From: Markus Fuchs Date: Thu, 29 Sep 2022 00:22:42 +0200 Subject: [PATCH 0042/1927] scsi: ufs: core: Remove unneeded casts from void * The end_io_data member of the "struct request" type has type "void *", so no cast is necessary. Link: https://lore.kernel.org/r/20220928222241.131334-1-mklntf@gmail.com Reviewed-by: Bart Van Assche Signed-off-by: Markus Fuchs Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshpb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/ufs/core/ufshpb.c b/drivers/ufs/core/ufshpb.c index a1a7a1175a5a6..0ce5063bedc57 100644 --- a/drivers/ufs/core/ufshpb.c +++ b/drivers/ufs/core/ufshpb.c @@ -615,14 +615,14 @@ static void ufshpb_activate_subregion(struct ufshpb_lu *hpb, static void ufshpb_umap_req_compl_fn(struct request *req, blk_status_t error) { - struct ufshpb_req *umap_req = (struct ufshpb_req *)req->end_io_data; + struct ufshpb_req *umap_req = req->end_io_data; ufshpb_put_req(umap_req->hpb, umap_req); } static void ufshpb_map_req_compl_fn(struct request *req, blk_status_t error) { - struct ufshpb_req *map_req = (struct ufshpb_req *) req->end_io_data; + struct ufshpb_req *map_req = req->end_io_data; struct ufshpb_lu *hpb = map_req->hpb; struct ufshpb_subregion *srgn; unsigned long flags; -- GitLab From 8777dd9dff4020bba66654ec92e4b0ab6367ad30 Mon Sep 17 00:00:00 2001 From: Krishna Yarlagadda Date: Sat, 1 Oct 2022 17:51:44 +0530 Subject: [PATCH 0043/1927] spi: tegra210-quad: Fix combined sequence Return value should be updated to zero in combined sequence routine if transfer is completed successfully. Currently it holds timeout value resulting in errors. Signed-off-by: Krishna Yarlagadda Link: https://lore.kernel.org/r/20221001122148.9158-1-kyarlagadda@nvidia.com Signed-off-by: Mark Brown --- drivers/spi/spi-tegra210-quad.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c index c89592b21ffc5..904972606bd45 100644 --- a/drivers/spi/spi-tegra210-quad.c +++ b/drivers/spi/spi-tegra210-quad.c @@ -1157,6 +1157,11 @@ static int tegra_qspi_combined_seq_xfer(struct tegra_qspi *tqspi, msg->actual_length += xfer->len; transfer_phase++; } + if (!xfer->cs_change) { + tegra_qspi_transfer_end(spi); + spi_transfer_delay_exec(xfer); + } + ret = 0; exit: msg->status = ret; -- GitLab From fdabc3f10e774ddc86ba715b9bc0c861d7e0834c Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Thu, 29 Sep 2022 23:06:50 +0800 Subject: [PATCH 0044/1927] ASoC: wm8997: Fix PM disable depth imbalance in wm8997_probe The pm_runtime_enable will increase power disable depth. Thus a pairing decrement is needed on the error handling path to keep it balanced according to context. We fix it by calling pm_runtime_disable when error returns. Fixes:40843aea5a9bd ("ASoC: wm8997: Initial CODEC driver") Signed-off-by: Zhang Qilong Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20220929150653.63845-2-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm8997.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/codecs/wm8997.c b/sound/soc/codecs/wm8997.c index 77136a5216059..f8993176d5c0b 100644 --- a/sound/soc/codecs/wm8997.c +++ b/sound/soc/codecs/wm8997.c @@ -1187,6 +1187,7 @@ static int wm8997_probe(struct platform_device *pdev) err_spk_irqs: arizona_free_spk_irqs(arizona); err_jack_codec_dev: + pm_runtime_disable(&pdev->dev); arizona_jack_codec_dev_remove(&wm8997->core); return ret; -- GitLab From 6ab646c985b529a32bf162de48d2d4a8bb7c9b64 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Thu, 29 Sep 2022 23:06:51 +0800 Subject: [PATCH 0045/1927] ASoC: wm5110: Fix PM disable depth imbalance in wm5110_probe The pm_runtime_enable will increase power disable depth. Thus a pairing decrement is needed on the error handling path to keep it balanced according to context. We fix it by calling pm_runtime_enable when error returns. Fixes:5c6af635fd772 ("ASoC: wm5110: Add audio CODEC driver") Signed-off-by: Zhang Qilong Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20220929150653.63845-3-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm5110.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c index fc634c995834d..8a61563eae11a 100644 --- a/sound/soc/codecs/wm5110.c +++ b/sound/soc/codecs/wm5110.c @@ -2500,6 +2500,7 @@ static int wm5110_probe(struct platform_device *pdev) arizona_set_irq_wake(arizona, ARIZONA_IRQ_DSP_IRQ1, 0); arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, wm5110); err_jack_codec_dev: + pm_runtime_disable(&pdev->dev); arizona_jack_codec_dev_remove(&wm5110->core); return ret; -- GitLab From 96e4abbd35adb5582573c463ccc554a644ac2434 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Thu, 29 Sep 2022 23:06:52 +0800 Subject: [PATCH 0046/1927] ASoC: wm5102: Fix PM disable depth imbalance in wm5102_probe The pm_runtime_enable will increase power disable depth. Thus a pairing decrement is needed on the error handling path to keep it balanced according to context. We fix it by calling pm_runtime_disable when error returns. Fixes:93e8791dd34ca ("ASoC: wm5102: Initial driver") Signed-off-by: Zhang Qilong Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20220929150653.63845-4-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm5102.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/codecs/wm5102.c b/sound/soc/codecs/wm5102.c index c09c9ac51b3e7..e56e30d59760d 100644 --- a/sound/soc/codecs/wm5102.c +++ b/sound/soc/codecs/wm5102.c @@ -2142,6 +2142,7 @@ static int wm5102_probe(struct platform_device *pdev) arizona_set_irq_wake(arizona, ARIZONA_IRQ_DSP_IRQ1, 0); arizona_free_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, wm5102); err_jack_codec_dev: + pm_runtime_disable(&pdev->dev); arizona_jack_codec_dev_remove(&wm5102->core); return ret; -- GitLab From 033d2d13a25113b6ffd24d72490f0e363dd3eb4c Mon Sep 17 00:00:00 2001 From: "Bird, Tim" Date: Mon, 3 Oct 2022 19:05:18 +0000 Subject: [PATCH 0047/1927] spi: spi-gxp: fix typo in SPDX identifier line Use '-' instead of '=' in identifier: "GPL-2.0-or-later" Signed-off-by: Tim Bird Link: https://lore.kernel.org/r/BYAPR13MB2503FF6412666D29FEAC8DCDFD5B9@BYAPR13MB2503.namprd13.prod.outlook.com Signed-off-by: Mark Brown --- drivers/spi/spi-gxp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-gxp.c b/drivers/spi/spi-gxp.c index 15b1101838390..c900c2f39b578 100644 --- a/drivers/spi/spi-gxp.c +++ b/drivers/spi/spi-gxp.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0=or-later +// SPDX-License-Identifier: GPL-2.0-or-later /* Copyright (C) 2022 Hewlett-Packard Development Company, L.P. */ #include -- GitLab From 0c72dbc96be870e4de8f9707c9a4c6d7a641381c Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Tue, 4 Oct 2022 14:51:21 +0300 Subject: [PATCH 0048/1927] Revert "ASoC: soc-component: using pm_runtime_resume_and_get instead of pm_runtime_get_sync" This reverts commit 08fc2a7448afc1660ec2f1b5c437fcd14155a7ee. The reverted commit causes the following warnigs: Runtime PM usage count underflow! This is due to the fact that the pm_runtime_resume_and_get() is calling pm_runtime_put_noidle() in case of < 0 return value of pm_runtime_get_sync() which includes the -EACCES. The change is wrong as -EACCES is returned in case of 'nested' get_sync() and it is a valid use of PM runtime. Fixes: 08fc2a7448af ("ASoC: soc-component: using pm_runtime_resume_and_get instead of pm_runtime_get_sync") Signed-off-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20221004115121.26180-1-peter.ujfalusi@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/soc-component.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sound/soc/soc-component.c b/sound/soc/soc-component.c index 659b9ade41586..e12f8244242b9 100644 --- a/sound/soc/soc-component.c +++ b/sound/soc/soc-component.c @@ -1213,9 +1213,11 @@ int snd_soc_pcm_component_pm_runtime_get(struct snd_soc_pcm_runtime *rtd, int i; for_each_rtd_components(rtd, i, component) { - int ret = pm_runtime_resume_and_get(component->dev); - if (ret < 0 && ret != -EACCES) + int ret = pm_runtime_get_sync(component->dev); + if (ret < 0 && ret != -EACCES) { + pm_runtime_put_noidle(component->dev); return soc_component_ret(component, ret); + } /* mark stream if succeeded */ soc_component_mark_push(component, stream, pm); } -- GitLab From 68fe1db04443cee58ddbeae9c506f7262b256168 Mon Sep 17 00:00:00 2001 From: Alexander Aring Date: Wed, 5 Oct 2022 22:02:37 -0400 Subject: [PATCH 0049/1927] net: ieee802154: return -EINVAL for unknown addr type This patch adds handling to return -EINVAL for an unknown addr type. The current behaviour is to return 0 as successful but the size of an unknown addr type is not defined and should return an error like -EINVAL. Fixes: 94160108a70c ("net/ieee802154: fix uninit value bug in dgram_sendmsg") Signed-off-by: Alexander Aring Link: https://lore.kernel.org/r/20221006020237.318511-1-aahringo@redhat.com Signed-off-by: Stefan Schmidt --- include/net/ieee802154_netdev.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/net/ieee802154_netdev.h b/include/net/ieee802154_netdev.h index a8994f307fc38..03b64bf876a46 100644 --- a/include/net/ieee802154_netdev.h +++ b/include/net/ieee802154_netdev.h @@ -185,21 +185,27 @@ static inline int ieee802154_sockaddr_check_size(struct sockaddr_ieee802154 *daddr, int len) { struct ieee802154_addr_sa *sa; + int ret = 0; sa = &daddr->addr; if (len < IEEE802154_MIN_NAMELEN) return -EINVAL; switch (sa->addr_type) { + case IEEE802154_ADDR_NONE: + break; case IEEE802154_ADDR_SHORT: if (len < IEEE802154_NAMELEN_SHORT) - return -EINVAL; + ret = -EINVAL; break; case IEEE802154_ADDR_LONG: if (len < IEEE802154_NAMELEN_LONG) - return -EINVAL; + ret = -EINVAL; + break; + default: + ret = -EINVAL; break; } - return 0; + return ret; } static inline void ieee802154_addr_from_sa(struct ieee802154_addr *a, -- GitLab From 444d8ad4916edec8a9fc684e841287db9b1e999f Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Mon, 19 Sep 2022 16:08:30 +0000 Subject: [PATCH 0050/1927] net: ieee802154: fix error return code in dgram_bind() Fix to return error code -EINVAL from the error handling case instead of 0, as done elsewhere in this function. Fixes: 94160108a70c ("net/ieee802154: fix uninit value bug in dgram_sendmsg") Signed-off-by: Wei Yongjun Link: https://lore.kernel.org/r/20220919160830.1436109-1-weiyongjun@huaweicloud.com Signed-off-by: Stefan Schmidt --- net/ieee802154/socket.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c index 6e55fae4c6860..1fa2fe041ec03 100644 --- a/net/ieee802154/socket.c +++ b/net/ieee802154/socket.c @@ -502,8 +502,10 @@ static int dgram_bind(struct sock *sk, struct sockaddr *uaddr, int len) if (err < 0) goto out; - if (addr->family != AF_IEEE802154) + if (addr->family != AF_IEEE802154) { + err = -EINVAL; goto out; + } ieee802154_addr_from_sa(&haddr, &addr->addr); dev = ieee802154_get_dev(sock_net(sk), &haddr); -- GitLab From de71d7567e358effd06dfc3e2a154b25f1331c10 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Mon, 10 Oct 2022 19:48:50 +0800 Subject: [PATCH 0051/1927] ASoC: wm5102: Revert "ASoC: wm5102: Fix PM disable depth imbalance in wm5102_probe" This reverts commit fcbb60820cd3008bb44334a0395e5e57ccb77329. The pm_runtime_disable is redundant when error returns in wm5102_probe, we just revert the old patch to fix it. Signed-off-by: Zhang Qilong Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20221010114852.88127-2-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm5102.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sound/soc/codecs/wm5102.c b/sound/soc/codecs/wm5102.c index e56e30d59760d..adaf886b0a9d2 100644 --- a/sound/soc/codecs/wm5102.c +++ b/sound/soc/codecs/wm5102.c @@ -2099,6 +2099,9 @@ static int wm5102_probe(struct platform_device *pdev) regmap_update_bits(arizona->regmap, wm5102_digital_vu[i], WM5102_DIG_VU, WM5102_DIG_VU); + pm_runtime_enable(&pdev->dev); + pm_runtime_idle(&pdev->dev); + ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, "ADSP2 Compressed IRQ", wm5102_adsp2_irq, wm5102); @@ -2131,9 +2134,6 @@ static int wm5102_probe(struct platform_device *pdev) goto err_spk_irqs; } - pm_runtime_enable(&pdev->dev); - pm_runtime_idle(&pdev->dev); - return ret; err_spk_irqs: -- GitLab From 7d4e966f4cd73ff69bf06934e8e14a33fb7ef447 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Mon, 10 Oct 2022 19:48:51 +0800 Subject: [PATCH 0052/1927] ASoC: wm5110: Revert "ASoC: wm5110: Fix PM disable depth imbalance in wm5110_probe" This reverts commit 86b46bf1feb83898d89a2b4a8d08d21e9ea277a7. The pm_runtime_disable is redundant when error returns in wm5110_probe, we just revert the old patch to fix it. Signed-off-by: Zhang Qilong Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20221010114852.88127-3-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm5110.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c index 8a61563eae11a..e0b971620d0fb 100644 --- a/sound/soc/codecs/wm5110.c +++ b/sound/soc/codecs/wm5110.c @@ -2457,6 +2457,9 @@ static int wm5110_probe(struct platform_device *pdev) regmap_update_bits(arizona->regmap, wm5110_digital_vu[i], WM5110_DIG_VU, WM5110_DIG_VU); + pm_runtime_enable(&pdev->dev); + pm_runtime_idle(&pdev->dev); + ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1, "ADSP2 Compressed IRQ", wm5110_adsp2_irq, wm5110); @@ -2489,9 +2492,6 @@ static int wm5110_probe(struct platform_device *pdev) goto err_spk_irqs; } - pm_runtime_enable(&pdev->dev); - pm_runtime_idle(&pdev->dev); - return ret; err_spk_irqs: -- GitLab From 68ce83e3bb26feba0fcdd59667fde942b3a600a1 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Mon, 10 Oct 2022 19:48:52 +0800 Subject: [PATCH 0053/1927] ASoC: wm8997: Revert "ASoC: wm8997: Fix PM disable depth imbalance in wm8997_probe" This reverts commit 41a736ac20602f64773e80f0f5b32cde1830a44a. The pm_runtime_disable is redundant when error returns in wm8997_probe, we just revert the old patch to fix it. Signed-off-by: Zhang Qilong Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20221010114852.88127-4-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm8997.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sound/soc/codecs/wm8997.c b/sound/soc/codecs/wm8997.c index f8993176d5c0b..c0207e9a7d53a 100644 --- a/sound/soc/codecs/wm8997.c +++ b/sound/soc/codecs/wm8997.c @@ -1161,6 +1161,9 @@ static int wm8997_probe(struct platform_device *pdev) regmap_update_bits(arizona->regmap, wm8997_digital_vu[i], WM8997_DIG_VU, WM8997_DIG_VU); + pm_runtime_enable(&pdev->dev); + pm_runtime_idle(&pdev->dev); + arizona_init_common(arizona); ret = arizona_init_vol_limit(arizona); @@ -1179,9 +1182,6 @@ static int wm8997_probe(struct platform_device *pdev) goto err_spk_irqs; } - pm_runtime_enable(&pdev->dev); - pm_runtime_idle(&pdev->dev); - return ret; err_spk_irqs: -- GitLab From 551f2994b8ccdbe296e239278531e345d6e94d4d Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Thu, 6 Oct 2022 16:58:22 -0700 Subject: [PATCH 0054/1927] ASoC: codec: tlv320adc3xxx: add GPIOLIB dependency Fix build errors when CONFIG_GPIOLIB is not enabled: ../sound/soc/codecs/tlv320adc3xxx.c: In function 'adc3xxx_i2c_probe': ../sound/soc/codecs/tlv320adc3xxx.c:1352:28: error: implicit declaration of function 'devm_gpiod_get'; did you mean 'devm_gpio_free'? [-Werror=implicit-function-declaration] 1352 | adc3xxx->rst_pin = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); ../sound/soc/codecs/tlv320adc3xxx.c:1352:57: error: 'GPIOD_OUT_LOW' undeclared (first use in this function); did you mean 'GPIOF_INIT_LOW'? 1352 | adc3xxx->rst_pin = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); CC lib/dynamic_debug.o ../sound/soc/codecs/tlv320adc3xxx.c:1400:9: error: implicit declaration of function 'gpiod_set_value_cansleep'; did you mean 'gpio_set_value_cansleep'? [-Werror=implicit-function-declaration] 1400 | gpiod_set_value_cansleep(adc3xxx->rst_pin, 1); Fixes: e9a3b57efd28 ("ASoC: codec: tlv320adc3xxx: New codec driver") Signed-off-by: Randy Dunlap Reported-by: kernel test robot Cc: Mark Brown Cc: Liam Girdwood Cc: Ricard Wanderlof Cc: Jaroslav Kysela Cc: Takashi Iwai Cc: alsa-devel@alsa-project.org Link: https://lore.kernel.org/r/20221006235822.30074-1-rdunlap@infradead.org Signed-off-by: Mark Brown --- sound/soc/codecs/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig index e3b90c425fafe..7022e6286e6cb 100644 --- a/sound/soc/codecs/Kconfig +++ b/sound/soc/codecs/Kconfig @@ -1629,6 +1629,7 @@ config SND_SOC_TFA989X config SND_SOC_TLV320ADC3XXX tristate "Texas Instruments TLV320ADC3001/3101 audio ADC" depends on I2C + depends on GPIOLIB help Enable support for Texas Instruments TLV320ADC3001 and TLV320ADC3101 ADCs. -- GitLab From c4ab29b0f3a6f1e167c5a627f7cd036c1d2b7d65 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Sat, 8 Oct 2022 22:05:22 +0800 Subject: [PATCH 0055/1927] ASoC: mt6660: Keep the pm_runtime enables before component stuff in mt6660_i2c_probe It would be better to keep the pm_runtime enables before the IRQ and component stuff. Both of those could start triggering PM runtime events. Signed-off-by: Zhang Qilong Reviewed-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20221008140522.134912-1-zhangqilong3@huawei.com Signed-off-by: Mark Brown --- sound/soc/codecs/mt6660.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sound/soc/codecs/mt6660.c b/sound/soc/codecs/mt6660.c index 45e0df13afb9f..b8369eeccc302 100644 --- a/sound/soc/codecs/mt6660.c +++ b/sound/soc/codecs/mt6660.c @@ -503,14 +503,14 @@ static int mt6660_i2c_probe(struct i2c_client *client) dev_err(chip->dev, "read chip revision fail\n"); goto probe_fail; } + pm_runtime_set_active(chip->dev); + pm_runtime_enable(chip->dev); ret = devm_snd_soc_register_component(chip->dev, &mt6660_component_driver, &mt6660_codec_dai, 1); - if (!ret) { - pm_runtime_set_active(chip->dev); - pm_runtime_enable(chip->dev); - } + if (ret) + pm_runtime_disable(chip->dev); return ret; -- GitLab From 29eb79a9a6283d661ea1f70ab012809fdbf057a7 Mon Sep 17 00:00:00 2001 From: Jiangshan Yi Date: Sun, 9 Oct 2022 15:48:16 +0800 Subject: [PATCH 0056/1927] ASoC: cx2072x: fix spelling typo in comment Fix spelling typo in comment. Reported-by: k2ci Signed-off-by: Jiangshan Yi Link: https://lore.kernel.org/r/20221009074816.2641162-1-13667453960@163.com Signed-off-by: Mark Brown --- sound/soc/codecs/cx2072x.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/codecs/cx2072x.h b/sound/soc/codecs/cx2072x.h index ebdd567fa2251..09e3a92b184f4 100644 --- a/sound/soc/codecs/cx2072x.h +++ b/sound/soc/codecs/cx2072x.h @@ -177,7 +177,7 @@ #define CX2072X_PLBK_DRC_PARM_LEN 9 #define CX2072X_CLASSD_AMP_LEN 6 -/* DAI interfae type */ +/* DAI interface type */ #define CX2072X_DAI_HIFI 1 #define CX2072X_DAI_DSP 2 #define CX2072X_DAI_DSP_PWM 3 /* 4 ch, including mic and AEC */ -- GitLab From b994d8f0773cf3b01129c094d00050710f2c422b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Neusch=C3=A4fer?= Date: Sat, 8 Oct 2022 17:14:59 +0200 Subject: [PATCH 0057/1927] spi: spi-mem: Fix typo (of -> or) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this instance, "or" makes more sense than "of", so I guess that "or" was intended and "of" was a typo. Signed-off-by: Jonathan Neuschäfer Link: https://lore.kernel.org/r/20221008151459.1421406-1-j.neuschaefer@gmx.net Signed-off-by: Mark Brown --- include/linux/spi/spi-mem.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/spi/spi-mem.h b/include/linux/spi/spi-mem.h index 2ba044d0d5e5b..8e984d75f5b6c 100644 --- a/include/linux/spi/spi-mem.h +++ b/include/linux/spi/spi-mem.h @@ -225,7 +225,7 @@ static inline void *spi_mem_get_drvdata(struct spi_mem *mem) /** * struct spi_controller_mem_ops - SPI memory operations * @adjust_op_size: shrink the data xfer of an operation to match controller's - * limitations (can be alignment of max RX/TX size + * limitations (can be alignment or max RX/TX size * limitations) * @supports_op: check if an operation is supported by the controller * @exec_op: execute a SPI memory operation -- GitLab From b25fe93ff705cdd8e98ad8d0435c621ea5462c82 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Thu, 6 Oct 2022 07:35:12 +0200 Subject: [PATCH 0058/1927] spi: mpc52xx: Replace NO_IRQ by 0 NO_IRQ is used to check the return of irq_of_parse_and_map(). On some architecture NO_IRQ is 0, on other architectures it is -1. irq_of_parse_and_map() returns 0 on error, independent of NO_IRQ. So use 0 instead of using NO_IRQ. Signed-off-by: Christophe Leroy Link: https://lore.kernel.org/r/f41e09d710879726eacb98daedf16d0847303b9b.1665034444.git.christophe.leroy@csgroup.eu Signed-off-by: Mark Brown --- drivers/spi/spi-mpc52xx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c index cb075c1acbee3..7b64e64c65cfe 100644 --- a/drivers/spi/spi-mpc52xx.c +++ b/drivers/spi/spi-mpc52xx.c @@ -151,7 +151,7 @@ mpc52xx_spi_fsmstate_idle(int irq, struct mpc52xx_spi *ms, u8 status, u8 data) int spr, sppr; u8 ctrl1; - if (status && (irq != NO_IRQ)) + if (status && irq) dev_err(&ms->master->dev, "spurious irq, status=0x%.2x\n", status); -- GitLab From 5302e1ff315b40dfc9bb3f08911f5a788cc1de01 Mon Sep 17 00:00:00 2001 From: Chin-Ting Kuo Date: Wed, 5 Oct 2022 16:32:09 +0800 Subject: [PATCH 0059/1927] spi: aspeed: Fix typo in mode_bits field for AST2600 platform MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both quad SPI TX and RX modes can be supported on AST2600. Correct typo in mode_bits field in both ast2600_fmc_data and ast2600_spi_data structs. Signed-off-by: Chin-Ting Kuo Reviewed-by: Cédric Le Goater Link: https://lore.kernel.org/r/20221005083209.222272-1-chin-ting_kuo@aspeedtech.com Signed-off-by: Mark Brown --- drivers/spi/spi-aspeed-smc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c index a334e89add869..33cefcf183927 100644 --- a/drivers/spi/spi-aspeed-smc.c +++ b/drivers/spi/spi-aspeed-smc.c @@ -1163,7 +1163,7 @@ static const struct aspeed_spi_data ast2500_spi_data = { static const struct aspeed_spi_data ast2600_fmc_data = { .max_cs = 3, .hastype = false, - .mode_bits = SPI_RX_QUAD | SPI_RX_QUAD, + .mode_bits = SPI_RX_QUAD | SPI_TX_QUAD, .we0 = 16, .ctl0 = CE0_CTRL_REG, .timing = CE0_TIMING_COMPENSATION_REG, @@ -1178,7 +1178,7 @@ static const struct aspeed_spi_data ast2600_fmc_data = { static const struct aspeed_spi_data ast2600_spi_data = { .max_cs = 2, .hastype = false, - .mode_bits = SPI_RX_QUAD | SPI_RX_QUAD, + .mode_bits = SPI_RX_QUAD | SPI_TX_QUAD, .we0 = 16, .ctl0 = CE0_CTRL_REG, .timing = CE0_TIMING_COMPENSATION_REG, -- GitLab From d94bf16e920047c9b4ea2b57f7b53b4ff5039d9f Mon Sep 17 00:00:00 2001 From: Derek Fang Date: Wed, 12 Oct 2022 11:13:20 +0800 Subject: [PATCH 0060/1927] ASoC: rt5682s: Fix the TDM Tx settings Complete the missing and correct the TDM Tx settings. Signed-off-by: Derek Fang Link: https://lore.kernel.org/r/20221012031320.6980-1-derek.fang@realtek.com Signed-off-by: Mark Brown --- sound/soc/codecs/rt5682s.c | 15 +++++++++++++-- sound/soc/codecs/rt5682s.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/sound/soc/codecs/rt5682s.c b/sound/soc/codecs/rt5682s.c index 3b91a3442c682..5199d3bbaf0b9 100644 --- a/sound/soc/codecs/rt5682s.c +++ b/sound/soc/codecs/rt5682s.c @@ -1981,7 +1981,7 @@ static int rt5682s_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, unsigned int rx_mask, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - unsigned int cl, val = 0; + unsigned int cl, val = 0, tx_slotnum; if (tx_mask || rx_mask) snd_soc_component_update_bits(component, @@ -1990,6 +1990,16 @@ static int rt5682s_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, snd_soc_component_update_bits(component, RT5682S_TDM_ADDA_CTRL_2, RT5682S_TDM_EN, 0); + /* Tx slot configuration */ + tx_slotnum = hweight_long(tx_mask); + if (tx_slotnum) { + if (tx_slotnum > slots) { + dev_err(component->dev, "Invalid or oversized Tx slots.\n"); + return -EINVAL; + } + val |= (tx_slotnum - 1) << RT5682S_TDM_ADC_DL_SFT; + } + switch (slots) { case 4: val |= RT5682S_TDM_TX_CH_4; @@ -2010,7 +2020,8 @@ static int rt5682s_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, } snd_soc_component_update_bits(component, RT5682S_TDM_CTRL, - RT5682S_TDM_TX_CH_MASK | RT5682S_TDM_RX_CH_MASK, val); + RT5682S_TDM_TX_CH_MASK | RT5682S_TDM_RX_CH_MASK | + RT5682S_TDM_ADC_DL_MASK, val); switch (slot_width) { case 8: diff --git a/sound/soc/codecs/rt5682s.h b/sound/soc/codecs/rt5682s.h index 824dc6543c182..45464a0417654 100644 --- a/sound/soc/codecs/rt5682s.h +++ b/sound/soc/codecs/rt5682s.h @@ -899,6 +899,7 @@ #define RT5682S_TDM_RX_CH_8 (0x3 << 8) #define RT5682S_TDM_ADC_LCA_MASK (0x7 << 4) #define RT5682S_TDM_ADC_LCA_SFT 4 +#define RT5682S_TDM_ADC_DL_MASK (0x3 << 0) #define RT5682S_TDM_ADC_DL_SFT 0 /* TDM control 2 (0x007a) */ -- GitLab From f2635d45a750182c6d5de15e2d6b059e0c302d7e Mon Sep 17 00:00:00 2001 From: Derek Fang Date: Wed, 12 Oct 2022 11:01:02 +0800 Subject: [PATCH 0061/1927] ASoC: rt1019: Fix the TDM settings Complete the missing and correct the TDM settings. Signed-off-by: Derek Fang Link: https://lore.kernel.org/r/20221012030102.4042-1-derek.fang@realtek.com Signed-off-by: Mark Brown --- sound/soc/codecs/rt1019.c | 20 +++++++++++--------- sound/soc/codecs/rt1019.h | 6 ++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/sound/soc/codecs/rt1019.c b/sound/soc/codecs/rt1019.c index b66bfecbb879b..49f527c61a7ad 100644 --- a/sound/soc/codecs/rt1019.c +++ b/sound/soc/codecs/rt1019.c @@ -391,18 +391,18 @@ static int rt1019_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, unsigned int rx_mask, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - unsigned int val = 0, rx_slotnum; + unsigned int cn = 0, cl = 0, rx_slotnum; int ret = 0, first_bit; switch (slots) { case 4: - val |= RT1019_I2S_TX_4CH; + cn = RT1019_I2S_TX_4CH; break; case 6: - val |= RT1019_I2S_TX_6CH; + cn = RT1019_I2S_TX_6CH; break; case 8: - val |= RT1019_I2S_TX_8CH; + cn = RT1019_I2S_TX_8CH; break; case 2: break; @@ -412,16 +412,16 @@ static int rt1019_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, switch (slot_width) { case 20: - val |= RT1019_I2S_DL_20; + cl = RT1019_TDM_CL_20; break; case 24: - val |= RT1019_I2S_DL_24; + cl = RT1019_TDM_CL_24; break; case 32: - val |= RT1019_I2S_DL_32; + cl = RT1019_TDM_CL_32; break; case 8: - val |= RT1019_I2S_DL_8; + cl = RT1019_TDM_CL_8; break; case 16: break; @@ -470,8 +470,10 @@ static int rt1019_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask, goto _set_tdm_err_; } + snd_soc_component_update_bits(component, RT1019_TDM_1, + RT1019_TDM_CL_MASK, cl); snd_soc_component_update_bits(component, RT1019_TDM_2, - RT1019_I2S_CH_TX_MASK | RT1019_I2S_DF_MASK, val); + RT1019_I2S_CH_TX_MASK, cn); _set_tdm_err_: return ret; diff --git a/sound/soc/codecs/rt1019.h b/sound/soc/codecs/rt1019.h index 64df831eeb720..48ba15efb48dd 100644 --- a/sound/soc/codecs/rt1019.h +++ b/sound/soc/codecs/rt1019.h @@ -95,6 +95,12 @@ #define RT1019_TDM_BCLK_MASK (0x1 << 6) #define RT1019_TDM_BCLK_NORM (0x0 << 6) #define RT1019_TDM_BCLK_INV (0x1 << 6) +#define RT1019_TDM_CL_MASK (0x7) +#define RT1019_TDM_CL_8 (0x4) +#define RT1019_TDM_CL_32 (0x3) +#define RT1019_TDM_CL_24 (0x2) +#define RT1019_TDM_CL_20 (0x1) +#define RT1019_TDM_CL_16 (0x0) /* 0x0401 TDM Control-2 */ #define RT1019_I2S_CH_TX_MASK (0x3 << 6) -- GitLab From ee1aa2ae3eaa96e70229fa61deee87ef4528ffdf Mon Sep 17 00:00:00 2001 From: Xiaolei Wang Date: Mon, 10 Oct 2022 17:20:14 +0800 Subject: [PATCH 0062/1927] ASoC: wm8962: Add an event handler for TEMP_HP and TEMP_SPK In wm8962 driver, the WM8962_ADDITIONAL_CONTROL_4 is used as a volatile register, but this register mixes a bunch of volatile status bits and a bunch of non-volatile control bits. The dapm widgets TEMP_HP and TEMP_SPK leverages the control bits in this register. After the wm8962 probe, the regmap will bet set to cache only mode, then a read error like below would be triggered when trying to read the initial power state of the dapm widgets TEMP_HP and TEMP_SPK. wm8962 0-001a: ASoC: error at soc_component_read_no_lock on wm8962.0-001a: -16 In order to fix this issue, we add event handler to actually power up/down these widgets. With this change, we also need to explicitly power off these widgets in the wm8962 probe since they are enabled by default. Signed-off-by: Xiaolei Wang Tested-by: Adam Ford Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20221010092014.2229246-1-xiaolei.wang@windriver.com Signed-off-by: Mark Brown --- sound/soc/codecs/wm8962.c | 54 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c index 398c448ea8540..6df06fba43778 100644 --- a/sound/soc/codecs/wm8962.c +++ b/sound/soc/codecs/wm8962.c @@ -1840,6 +1840,49 @@ SOC_SINGLE_TLV("SPKOUTR Mixer DACR Volume", WM8962_SPEAKER_MIXER_5, 4, 1, 0, inmix_tlv), }; +static int tp_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *kcontrol, int event) +{ + int ret, reg, val, mask; + struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm); + + ret = pm_runtime_resume_and_get(component->dev); + if (ret < 0) { + dev_err(component->dev, "Failed to resume device: %d\n", ret); + return ret; + } + + reg = WM8962_ADDITIONAL_CONTROL_4; + + if (!strcmp(w->name, "TEMP_HP")) { + mask = WM8962_TEMP_ENA_HP_MASK; + val = WM8962_TEMP_ENA_HP; + } else if (!strcmp(w->name, "TEMP_SPK")) { + mask = WM8962_TEMP_ENA_SPK_MASK; + val = WM8962_TEMP_ENA_SPK; + } else { + pm_runtime_put(component->dev); + return -EINVAL; + } + + switch (event) { + case SND_SOC_DAPM_POST_PMD: + val = 0; + fallthrough; + case SND_SOC_DAPM_POST_PMU: + ret = snd_soc_component_update_bits(component, reg, mask, val); + break; + default: + WARN(1, "Invalid event %d\n", event); + pm_runtime_put(component->dev); + return -EINVAL; + } + + pm_runtime_put(component->dev); + + return 0; +} + static int cp_event(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol, int event) { @@ -2140,8 +2183,10 @@ SND_SOC_DAPM_SUPPLY("TOCLK", WM8962_ADDITIONAL_CONTROL_1, 0, 0, NULL, 0), SND_SOC_DAPM_SUPPLY_S("DSP2", 1, WM8962_DSP2_POWER_MANAGEMENT, WM8962_DSP2_ENA_SHIFT, 0, dsp2_event, SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD), -SND_SOC_DAPM_SUPPLY("TEMP_HP", WM8962_ADDITIONAL_CONTROL_4, 2, 0, NULL, 0), -SND_SOC_DAPM_SUPPLY("TEMP_SPK", WM8962_ADDITIONAL_CONTROL_4, 1, 0, NULL, 0), +SND_SOC_DAPM_SUPPLY("TEMP_HP", SND_SOC_NOPM, 0, 0, tp_event, + SND_SOC_DAPM_POST_PMU|SND_SOC_DAPM_POST_PMD), +SND_SOC_DAPM_SUPPLY("TEMP_SPK", SND_SOC_NOPM, 0, 0, tp_event, + SND_SOC_DAPM_POST_PMU|SND_SOC_DAPM_POST_PMD), SND_SOC_DAPM_MIXER("INPGAL", WM8962_LEFT_INPUT_PGA_CONTROL, 4, 0, inpgal, ARRAY_SIZE(inpgal)), @@ -3763,6 +3808,11 @@ static int wm8962_i2c_probe(struct i2c_client *i2c) if (ret < 0) goto err_pm_runtime; + regmap_update_bits(wm8962->regmap, WM8962_ADDITIONAL_CONTROL_4, + WM8962_TEMP_ENA_HP_MASK, 0); + regmap_update_bits(wm8962->regmap, WM8962_ADDITIONAL_CONTROL_4, + WM8962_TEMP_ENA_SPK_MASK, 0); + regcache_cache_only(wm8962->regmap, true); /* The drivers should power up as needed */ -- GitLab From c9a3545b1d771fb7b06a487796c40288c02c41c5 Mon Sep 17 00:00:00 2001 From: Srinivasa Rao Mandadapu Date: Thu, 13 Oct 2022 10:38:31 +0530 Subject: [PATCH 0063/1927] ASoC: qcom: lpass-cpu: mark HDMI TX registers as volatile Update HDMI volatile registers list as DMA, Channel Selection registers, vbit control registers are being reflected by hardware DP port disconnection. This update is required to fix no display and no sound issue observed after reconnecting TAMA/SANWA DP cables. Once DP cable is unplugged, DMA control registers are being reset by hardware, however at second plugin, new dma control values does not updated to the dma hardware registers since new register value and cached values at the time of first plugin are same. Fixes: 7cb37b7bd0d3 ("ASoC: qcom: Add support for lpass hdmi driver") Signed-off-by: Srinivasa Rao Mandadapu Reported-by: Kuogee Hsieh Link: https://lore.kernel.org/r/1665637711-13300-1-git-send-email-quic_srivasam@quicinc.com Signed-off-by: Mark Brown --- sound/soc/qcom/lpass-cpu.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sound/soc/qcom/lpass-cpu.c b/sound/soc/qcom/lpass-cpu.c index 8a56f38dc7e86..99a3b44285913 100644 --- a/sound/soc/qcom/lpass-cpu.c +++ b/sound/soc/qcom/lpass-cpu.c @@ -782,10 +782,18 @@ static bool lpass_hdmi_regmap_volatile(struct device *dev, unsigned int reg) return true; if (reg == LPASS_HDMI_TX_LEGACY_ADDR(v)) return true; + if (reg == LPASS_HDMI_TX_VBIT_CTL_ADDR(v)) + return true; for (i = 0; i < v->hdmi_rdma_channels; ++i) { if (reg == LPAIF_HDMI_RDMACURR_REG(v, i)) return true; + if (reg == LPASS_HDMI_TX_DMA_ADDR(v, i)) + return true; + if (reg == LPASS_HDMI_TX_CH_LSB_ADDR(v, i)) + return true; + if (reg == LPASS_HDMI_TX_CH_MSB_ADDR(v, i)) + return true; } return false; } -- GitLab From 6a43cd02ddbc597dc9a1f82c1e433f871a2f6f06 Mon Sep 17 00:00:00 2001 From: Mauro Lima Date: Wed, 12 Oct 2022 12:21:35 -0300 Subject: [PATCH 0064/1927] spi: intel: Fix the offset to get the 64K erase opcode According to documentation, the 64K erase opcode is located in VSCC range [16:23] instead of [8:15]. Use the proper value to shift the mask over the correct range. Signed-off-by: Mauro Lima Reviewed-by: Mika Westerberg Link: https://lore.kernel.org/r/20221012152135.28353-1-mauro.lima@eclypsium.com Signed-off-by: Mark Brown --- drivers/spi/spi-intel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c index 55f4ee2db002c..605acb1bf4b08 100644 --- a/drivers/spi/spi-intel.c +++ b/drivers/spi/spi-intel.c @@ -114,7 +114,7 @@ #define ERASE_OPCODE_SHIFT 8 #define ERASE_OPCODE_MASK (0xff << ERASE_OPCODE_SHIFT) #define ERASE_64K_OPCODE_SHIFT 16 -#define ERASE_64K_OPCODE_MASK (0xff << ERASE_OPCODE_SHIFT) +#define ERASE_64K_OPCODE_MASK (0xff << ERASE_64K_OPCODE_SHIFT) /* Flash descriptor fields */ #define FLVALSIG_MAGIC 0x0ff0a55a -- GitLab From c92a7a52243871eb10e0ea2260685def47fb5094 Mon Sep 17 00:00:00 2001 From: David Vernet Date: Wed, 12 Oct 2022 18:20:14 -0500 Subject: [PATCH 0065/1927] bpf: Allow bpf_user_ringbuf_drain() callbacks to return 1 The bpf_user_ringbuf_drain() helper function allows a BPF program to specify a callback that is invoked when draining entries from a BPF_MAP_TYPE_USER_RINGBUF ring buffer map. The API is meant to allow the callback to return 0 if it wants to continue draining samples, and 1 if it's done draining. Unfortunately, bpf_user_ringbuf_drain() landed shortly after commit 1bfe26fb0827 ("bpf: Add verifier support for custom callback return range"), which changed the default behavior of callbacks to only support returning 0. This patch corrects that oversight by allowing bpf_user_ringbuf_drain() callbacks to return 0 or 1. A follow-on patch will update the user_ringbuf selftests to also return 1 from a bpf_user_ringbuf_drain() callback to prevent this from regressing in the future. Fixes: 205715673844 ("bpf: Add bpf_user_ringbuf_drain() helper") Signed-off-by: David Vernet Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20221012232015.1510043-2-void@manifault.com --- kernel/bpf/verifier.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 6f6d2d511c06f..9ab7188d8f681 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6946,6 +6946,7 @@ static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env, __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); callee->in_callback_fn = true; + callee->callback_ret_range = tnum_range(0, 1); return 0; } -- GitLab From 6e44b9f375a3135fc4960d76a9ea6720625cad73 Mon Sep 17 00:00:00 2001 From: David Vernet Date: Wed, 12 Oct 2022 18:20:15 -0500 Subject: [PATCH 0066/1927] selftests/bpf: Make bpf_user_ringbuf_drain() selftest callback return 1 In commit 1bfe26fb0827 ("bpf: Add verifier support for custom callback return range"), the verifier was updated to require callbacks to BPF helpers to explicitly specify the range of values that can be returned. bpf_user_ringbuf_drain() was merged after this in commit 205715673844 ("bpf: Add bpf_user_ringbuf_drain() helper"), and this change in default behavior was missed. This patch updates the BPF_MAP_TYPE_USER_RINGBUF selftests to also return 1 from a bpf_user_ringbuf_drain() callback so as to properly test this going forward. Signed-off-by: David Vernet Signed-off-by: Andrii Nakryiko Link: https://lore.kernel.org/bpf/20221012232015.1510043-3-void@manifault.com --- tools/testing/selftests/bpf/progs/user_ringbuf_success.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_success.c b/tools/testing/selftests/bpf/progs/user_ringbuf_success.c index 099c23d9aa21b..b39093dd57159 100644 --- a/tools/testing/selftests/bpf/progs/user_ringbuf_success.c +++ b/tools/testing/selftests/bpf/progs/user_ringbuf_success.c @@ -47,14 +47,14 @@ record_sample(struct bpf_dynptr *dynptr, void *context) if (status) { bpf_printk("bpf_dynptr_read() failed: %d\n", status); err = 1; - return 0; + return 1; } } else { sample = bpf_dynptr_data(dynptr, 0, sizeof(*sample)); if (!sample) { bpf_printk("Unexpectedly failed to get sample\n"); err = 2; - return 0; + return 1; } stack_sample = *sample; } -- GitLab From 83d18e9d9c0150d98dc24e3642ea93f5e245322c Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Thu, 13 Oct 2022 15:55:13 -0700 Subject: [PATCH 0067/1927] drm/msm/a6xx: Fix kvzalloc vs state_kcalloc usage adreno_show_object() is a trap! It will re-allocate the pointer it is passed on first call, when the data is ascii85 encoded, using kvmalloc/ kvfree(). Which means the data *passed* to it must be kvmalloc'd, ie. we cannot use the state_kcalloc() helper. This partially reverts commit ec8f1813bf8d ("drm/msm/a6xx: Replace kcalloc() with kvzalloc()"), but adds the missing kvfree() to fix the memory leak that was present previously. And adds a warning comment. Fixes: ec8f1813bf8d ("drm/msm/a6xx: Replace kcalloc() with kvzalloc()") Closes: https://gitlab.freedesktop.org/drm/msm/-/issues/20 Signed-off-by: Rob Clark Reviewed-by: Chia-I Wu Reviewed-by: Akhil P Oommen Patchwork: https://patchwork.freedesktop.org/patch/507014/ Link: https://lore.kernel.org/r/20221013225520.371226-2-robdclark@gmail.com --- drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c | 11 ++++++++++- drivers/gpu/drm/msm/adreno/adreno_gpu.c | 7 ++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 3c112a6cc8a2f..730355f9e2d48 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -819,7 +819,7 @@ static struct msm_gpu_state_bo *a6xx_snapshot_gmu_bo( snapshot->iova = bo->iova; snapshot->size = bo->size; - snapshot->data = state_kcalloc(a6xx_state, 1, snapshot->size); + snapshot->data = kvzalloc(snapshot->size, GFP_KERNEL); if (!snapshot->data) return NULL; @@ -1034,6 +1034,15 @@ static void a6xx_gpu_state_destroy(struct kref *kref) struct a6xx_gpu_state *a6xx_state = container_of(state, struct a6xx_gpu_state, base); + if (a6xx_state->gmu_log) + kvfree(a6xx_state->gmu_log->data); + + if (a6xx_state->gmu_hfi) + kvfree(a6xx_state->gmu_hfi->data); + + if (a6xx_state->gmu_debug) + kvfree(a6xx_state->gmu_debug->data); + list_for_each_entry_safe(obj, tmp, &a6xx_state->objs, node) kvfree(obj); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 382fb7f9e4976..5a0e8491cd3a0 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -729,7 +729,12 @@ static char *adreno_gpu_ascii85_encode(u32 *src, size_t len) return buf; } -/* len is expected to be in bytes */ +/* len is expected to be in bytes + * + * WARNING: *ptr should be allocated with kvmalloc or friends. It can be free'd + * with kvfree() and replaced with a newly kvmalloc'd buffer on the first call + * when the unencoded raw data is encoded + */ void adreno_show_object(struct drm_printer *p, void **ptr, int len, bool *encoded) { -- GitLab From fab384c496a313a1083554fb1eb2332a6756be2e Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Thu, 13 Oct 2022 15:55:14 -0700 Subject: [PATCH 0068/1927] drm/msm/a6xx: Skip snapshotting unused GMU buffers Some buffers are unused on certain sub-generations of a6xx. So just skip them. Signed-off-by: Rob Clark Reviewed-by: Chia-I Wu Reviewed-by: Akhil P Oommen Patchwork: https://patchwork.freedesktop.org/patch/507013/ Link: https://lore.kernel.org/r/20221013225520.371226-3-robdclark@gmail.com --- drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 730355f9e2d48..b0124d0f286ca 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -813,6 +813,9 @@ static struct msm_gpu_state_bo *a6xx_snapshot_gmu_bo( { struct msm_gpu_state_bo *snapshot; + if (!bo->size) + return NULL; + snapshot = state_kcalloc(a6xx_state, 1, sizeof(*snapshot)); if (!snapshot) return NULL; -- GitLab From ec4fbd791519423726ce9fd5130de9a95365c8eb Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Thu, 13 Oct 2022 15:55:15 -0700 Subject: [PATCH 0069/1927] drm/msm/a6xx: Remove state objects from list before freeing Technically it worked as it was before, only because it was using the _safe version of the iterator. But it is sloppy practice to leave dangling pointers. Signed-off-by: Rob Clark Reviewed-by: Chia-I Wu Reviewed-by: Akhil P Oommen Patchwork: https://patchwork.freedesktop.org/patch/507017/ Link: https://lore.kernel.org/r/20221013225520.371226-4-robdclark@gmail.com --- drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index b0124d0f286ca..a5c3d1ed255a6 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -1046,8 +1046,10 @@ static void a6xx_gpu_state_destroy(struct kref *kref) if (a6xx_state->gmu_debug) kvfree(a6xx_state->gmu_debug->data); - list_for_each_entry_safe(obj, tmp, &a6xx_state->objs, node) + list_for_each_entry_safe(obj, tmp, &a6xx_state->objs, node) { + list_del(&obj->node); kvfree(obj); + } adreno_gpu_state_destroy(state); kfree(a6xx_state); -- GitLab From ccc40d42bd59ac858b58322775004613f1d805af Mon Sep 17 00:00:00 2001 From: Aashish Sharma Date: Tue, 11 Oct 2022 13:25:19 +0530 Subject: [PATCH 0070/1927] drm/msm: Remove redundant check for 'submit' Rectify the below smatch warning: drivers/gpu/drm/msm/msm_gem_submit.c:963 msm_ioctl_gem_submit() warn: variable dereferenced before check 'submit' 'submit' is normally error pointer or valid, so remove its NULL initializer as it's confusing and also remove a redundant check for it's value. Signed-off-by: Aashish Sharma Reported-by: kernel test robot Reported-by: Dan Carpenter Reviewed-by: Guenter Roeck Patchwork: https://patchwork.freedesktop.org/patch/506653/ Link: https://lore.kernel.org/r/20221011075519.3111928-1-shraash@google.com Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/msm_gem_submit.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c index c670591995e6f..45a3e5cadc7da 100644 --- a/drivers/gpu/drm/msm/msm_gem_submit.c +++ b/drivers/gpu/drm/msm/msm_gem_submit.c @@ -706,7 +706,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, struct msm_drm_private *priv = dev->dev_private; struct drm_msm_gem_submit *args = data; struct msm_file_private *ctx = file->driver_priv; - struct msm_gem_submit *submit = NULL; + struct msm_gem_submit *submit; struct msm_gpu *gpu = priv->gpu; struct msm_gpu_submitqueue *queue; struct msm_ringbuffer *ring; @@ -946,8 +946,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, put_unused_fd(out_fence_fd); mutex_unlock(&queue->lock); out_post_unlock: - if (submit) - msm_gem_submit_put(submit); + msm_gem_submit_put(submit); if (!IS_ERR_OR_NULL(post_deps)) { for (i = 0; i < args->nr_out_syncobjs; ++i) { kfree(post_deps[i].chain); -- GitLab From e0e86f25fd469ca76c1b50091372aed1ff99ca1a Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Fri, 7 Oct 2022 21:23:38 +0100 Subject: [PATCH 0071/1927] drm/msm: Kconfig: Fix spelling mistake "throught" -> "through" There is a spelling mistake in a Kconfig description. Fix it. Signed-off-by: Colin Ian King Patchwork: https://patchwork.freedesktop.org/patch/506301/ Link: https://lore.kernel.org/r/20221007202338.2755731-1-colin.i.king@gmail.com Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig index 4e0cbd682725e..3c9dfdb0b3283 100644 --- a/drivers/gpu/drm/msm/Kconfig +++ b/drivers/gpu/drm/msm/Kconfig @@ -155,7 +155,7 @@ config DRM_MSM_HDMI Compile in support for the HDMI output MSM DRM driver. It can be a primary or a secondary display on device. Note that this is used only for the direct HDMI output. If the device outputs HDMI data - throught some kind of DSI-to-HDMI bridge, this option can be disabled. + through some kind of DSI-to-HDMI bridge, this option can be disabled. config DRM_MSM_HDMI_HDCP bool "Enable HDMI HDCP support in MSM DRM driver" -- GitLab From ac5408991ea6b06e29129b4d4861097c4c3e0d59 Mon Sep 17 00:00:00 2001 From: Aman Dhoot Date: Sat, 15 Oct 2022 20:41:17 -0700 Subject: [PATCH 0072/1927] Input: synaptics - switch touchpad on HP Laptop 15-da3001TU to RMI mode The device works fine in native RMI mode, there is no reason to use legacy PS/2 mode with it. Signed-off-by: Aman Dhoot Signed-off-by: Dmitry Torokhov --- drivers/input/mouse/synaptics.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c index fa021af8506e4..b0f776448a1cd 100644 --- a/drivers/input/mouse/synaptics.c +++ b/drivers/input/mouse/synaptics.c @@ -192,6 +192,7 @@ static const char * const smbus_pnp_ids[] = { "SYN3221", /* HP 15-ay000 */ "SYN323d", /* HP Spectre X360 13-w013dx */ "SYN3257", /* HP Envy 13-ad105ng */ + "SYN3286", /* HP Laptop 15-da3001TU */ NULL }; -- GitLab From a01aca4b05174b6dee2392ec44406f85e0f8bd46 Mon Sep 17 00:00:00 2001 From: Sheng-Liang Pan Date: Sun, 16 Oct 2022 21:47:46 -0700 Subject: [PATCH 0073/1927] dt-bindings: input: touchscreen: Add compatible for Goodix GT7986U chip Add a compatible for Goodix touch screen chip GT7986U which is is expected to be fully compatible with a driver written for GT7375P. Signed-off-by: Sheng-Liang Pan Reviewed-by: Douglas Anderson Acked-by: Rob Herring Link: https://lore.kernel.org/r/20221006185333.v7.3.I52e4b4b20e2eb0ae20f2a9bb198aa6410f04cf16@changeid Signed-off-by: Dmitry Torokhov --- Documentation/devicetree/bindings/input/goodix,gt7375p.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml b/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml index fe1c5016f7f31..1c191bc5a1782 100644 --- a/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml +++ b/Documentation/devicetree/bindings/input/goodix,gt7375p.yaml @@ -16,8 +16,11 @@ description: properties: compatible: - items: + oneOf: - const: goodix,gt7375p + - items: + - const: goodix,gt7986u + - const: goodix,gt7375p reg: enum: -- GitLab From 76845ba539c3693f572377b38563d19a87266a5b Mon Sep 17 00:00:00 2001 From: Kishon Vijay Abraham I Date: Fri, 14 Oct 2022 11:54:24 +0530 Subject: [PATCH 0074/1927] MAINTAINERS: Update Kishon's email address in GENERIC PHY FRAMEWORK Update Kishon's email address in GENERIC PHY FRAMEWORK maintainer entry. Cc: Kishon Vijay Abraham I Signed-off-by: Kishon Vijay Abraham I Link: https://lore.kernel.org/r/20221014062424.3327-1-kishon@ti.com Signed-off-by: Vinod Koul --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..2526755bf9771 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8595,8 +8595,8 @@ F: include/asm-generic/ F: include/uapi/asm-generic/ GENERIC PHY FRAMEWORK -M: Kishon Vijay Abraham I M: Vinod Koul +M: Kishon Vijay Abraham I L: linux-phy@lists.infradead.org S: Supported Q: https://patchwork.kernel.org/project/linux-phy/list/ -- GitLab From ca1c73628f5bd0c1ef6e46073cc3be2450605b06 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 14 Oct 2022 12:25:06 +0300 Subject: [PATCH 0075/1927] phy: stm32: fix an error code in probe If "index > usbphyc->nphys" is true then this returns success but it should return -EINVAL. Fixes: 94c358da3a05 ("phy: stm32: add support for STM32 USB PHY Controller (USBPHYC)") Signed-off-by: Dan Carpenter Reviewed-by: Amelie Delaunay Link: https://lore.kernel.org/r/Y0kq8j6S+5nDdMpr@kili Signed-off-by: Vinod Koul --- drivers/phy/st/phy-stm32-usbphyc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/phy/st/phy-stm32-usbphyc.c b/drivers/phy/st/phy-stm32-usbphyc.c index a98c911cc37ae..5bb9647b078f1 100644 --- a/drivers/phy/st/phy-stm32-usbphyc.c +++ b/drivers/phy/st/phy-stm32-usbphyc.c @@ -710,6 +710,8 @@ static int stm32_usbphyc_probe(struct platform_device *pdev) ret = of_property_read_u32(child, "reg", &index); if (ret || index > usbphyc->nphys) { dev_err(&phy->dev, "invalid reg property: %d\n", ret); + if (!ret) + ret = -EINVAL; goto put_child; } -- GitLab From 2a4ea83bb8e54986703187edce59c9130a75eb26 Mon Sep 17 00:00:00 2001 From: Jon Hunter Date: Mon, 10 Oct 2022 14:51:32 +0100 Subject: [PATCH 0076/1927] phy: tegra: xusb: Fix crash during pad power on/down Commit a88520bfc0ec ("usb: gadget: tegra: Reduce pad power") added calls to tegra_phy_xusb_utmi_pad_power_on/down in the Tegra XUDC driver to control the pad power. This change is causing a kernel panic when powering down the pads on entering suspend with the Jetson TX2 platform. The panic occurs because the 'xudc->curr_utmi_phy' is not configured on this platform and we do not check to see if the pointer is valid before attempting to deference the pointer. Fix this by checking to see if the 'phy' pointer passed to tegra_phy_xusb_utmi_pad_power_on/down is valid. Fixes: a88520bfc0ec ("usb: gadget: tegra: Reduce pad power") Signed-off-by: Jon Hunter Link: https://lore.kernel.org/r/20221010135132.30809-1-jonathanh@nvidia.com Signed-off-by: Vinod Koul --- drivers/phy/tegra/xusb.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/phy/tegra/xusb.c b/drivers/phy/tegra/xusb.c index 95091876c4222..dce45fbbd699c 100644 --- a/drivers/phy/tegra/xusb.c +++ b/drivers/phy/tegra/xusb.c @@ -1461,8 +1461,14 @@ EXPORT_SYMBOL_GPL(tegra_phy_xusb_utmi_port_reset); void tegra_phy_xusb_utmi_pad_power_on(struct phy *phy) { - struct tegra_xusb_lane *lane = phy_get_drvdata(phy); - struct tegra_xusb_padctl *padctl = lane->pad->padctl; + struct tegra_xusb_lane *lane; + struct tegra_xusb_padctl *padctl; + + if (!phy) + return; + + lane = phy_get_drvdata(phy); + padctl = lane->pad->padctl; if (padctl->soc->ops->utmi_pad_power_on) padctl->soc->ops->utmi_pad_power_on(phy); @@ -1471,8 +1477,14 @@ EXPORT_SYMBOL_GPL(tegra_phy_xusb_utmi_pad_power_on); void tegra_phy_xusb_utmi_pad_power_down(struct phy *phy) { - struct tegra_xusb_lane *lane = phy_get_drvdata(phy); - struct tegra_xusb_padctl *padctl = lane->pad->padctl; + struct tegra_xusb_lane *lane; + struct tegra_xusb_padctl *padctl; + + if (!phy) + return; + + lane = phy_get_drvdata(phy); + padctl = lane->pad->padctl; if (padctl->soc->ops->utmi_pad_power_down) padctl->soc->ops->utmi_pad_power_down(phy); -- GitLab From 0dec4d2f2636b9e54d9d29f17afc7687c5407f78 Mon Sep 17 00:00:00 2001 From: Shreeya Patel Date: Fri, 26 Aug 2022 17:53:52 +0530 Subject: [PATCH 0077/1927] iio: light: tsl2583: Fix module unloading tsl2583 probe() uses devm_iio_device_register() and calling iio_device_unregister() causes the unregister to occur twice. s Switch to iio_device_register() instead of devm_iio_device_register() in probe to avoid the device managed cleanup. Fixes: 371894f5d1a0 ("iio: tsl2583: add runtime power management support") Signed-off-by: Shreeya Patel Link: https://lore.kernel.org/r/20220826122352.288438-1-shreeya.patel@collabora.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/light/tsl2583.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c index 0a2ca1a8146da..7bcb5c7189224 100644 --- a/drivers/iio/light/tsl2583.c +++ b/drivers/iio/light/tsl2583.c @@ -858,7 +858,7 @@ static int tsl2583_probe(struct i2c_client *clientp, TSL2583_POWER_OFF_DELAY_MS); pm_runtime_use_autosuspend(&clientp->dev); - ret = devm_iio_device_register(indio_dev->dev.parent, indio_dev); + ret = iio_device_register(indio_dev); if (ret) { dev_err(&clientp->dev, "%s: iio registration failed\n", __func__); -- GitLab From 7578847b5949db3a75163908bd99c46d27e8b19f Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Mon, 26 Sep 2022 12:10:37 +0300 Subject: [PATCH 0078/1927] iio: adc: mcp3911: fix sizeof() vs ARRAY_SIZE() bug This code uses sizeof() instead of ARRAY_SIZE() so it reads beyond the end of the mcp3911_osr_table[] array. Fixes: 6d965885f4ea ("iio: adc: mcp3911: add support for oversampling ratio") Signed-off-by: Dan Carpenter Reviewed-by: Marcus Folkesson Link: https://lore.kernel.org/r/YzFsjY3xLHUQMjVr@kili Signed-off-by: Jonathan Cameron --- drivers/iio/adc/mcp3911.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c index b35fd2c9c3c00..015a9ffdb26a2 100644 --- a/drivers/iio/adc/mcp3911.c +++ b/drivers/iio/adc/mcp3911.c @@ -248,7 +248,7 @@ static int mcp3911_write_raw(struct iio_dev *indio_dev, break; case IIO_CHAN_INFO_OVERSAMPLING_RATIO: - for (int i = 0; i < sizeof(mcp3911_osr_table); i++) { + for (int i = 0; i < ARRAY_SIZE(mcp3911_osr_table); i++) { if (val == mcp3911_osr_table[i]) { val = FIELD_PREP(MCP3911_CONFIG_OSR, i); ret = mcp3911_update(adc, MCP3911_REG_CONFIG, MCP3911_CONFIG_OSR, -- GitLab From a83695a666eb3541873c3c9734ec4e1d10ca2d7f Mon Sep 17 00:00:00 2001 From: Marcus Folkesson Date: Tue, 27 Sep 2022 11:25:37 +0200 Subject: [PATCH 0079/1927] iio: adc: mcp3911: return proper error code on failure to allocate trigger smatch warnings: drivers/iio/adc/mcp3911.c:441 mcp3911_probe() warn: passing zero to 'PTR_ERR' Fixes: 08a65f61db69 ("iio: adc: mcp3911: add support for interrupts") Reported-by: kernel test robot Reported-by: Dan Carpenter Signed-off-by: Marcus Folkesson Link: https://lore.kernel.org/r/20220927092537.94663-1-marcus.folkesson@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/adc/mcp3911.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c index 015a9ffdb26a2..7f0b37994fe5c 100644 --- a/drivers/iio/adc/mcp3911.c +++ b/drivers/iio/adc/mcp3911.c @@ -496,7 +496,7 @@ static int mcp3911_probe(struct spi_device *spi) indio_dev->name, iio_device_id(indio_dev)); if (!adc->trig) - return PTR_ERR(adc->trig); + return -ENOMEM; adc->trig->ops = &mcp3911_trigger_ops; iio_trigger_set_drvdata(adc->trig, adc); -- GitLab From 815f1647a603a822d66630bbe22cab4bc097c8c3 Mon Sep 17 00:00:00 2001 From: Marcus Folkesson Date: Mon, 10 Oct 2022 21:46:41 +0200 Subject: [PATCH 0080/1927] iio: adc: mcp3911: use correct id bits The device ID should be shifted 6 bits to left according to datasheet. Fixes: 3a89b289df5d ("iio: adc: add support for mcp3911") Signed-off-by: Marcus Folkesson Link: https://lore.kernel.org/r/20221010194641.676484-1-marcus.folkesson@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/adc/mcp3911.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c index 7f0b37994fe5c..f57f5eb23d49a 100644 --- a/drivers/iio/adc/mcp3911.c +++ b/drivers/iio/adc/mcp3911.c @@ -55,8 +55,8 @@ /* Internal voltage reference in mV */ #define MCP3911_INT_VREF_MV 1200 -#define MCP3911_REG_READ(reg, id) ((((reg) << 1) | ((id) << 5) | (1 << 0)) & 0xff) -#define MCP3911_REG_WRITE(reg, id) ((((reg) << 1) | ((id) << 5) | (0 << 0)) & 0xff) +#define MCP3911_REG_READ(reg, id) ((((reg) << 1) | ((id) << 6) | (1 << 0)) & 0xff) +#define MCP3911_REG_WRITE(reg, id) ((((reg) << 1) | ((id) << 6) | (0 << 0)) & 0xff) #define MCP3911_NUM_CHANNELS 2 -- GitLab From aa6c77d05eb1c57ee5b95a7b83a39384c37df4d9 Mon Sep 17 00:00:00 2001 From: Marcus Folkesson Date: Mon, 10 Oct 2022 21:46:54 +0200 Subject: [PATCH 0081/1927] iio: adc: mcp3911: mask out device ID in debug prints The Device ID should not be included when printing register. Signed-off-by: Marcus Folkesson Link: https://lore.kernel.org/r/20221010194654.676525-1-marcus.folkesson@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/adc/mcp3911.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/iio/adc/mcp3911.c b/drivers/iio/adc/mcp3911.c index f57f5eb23d49a..76b334f5ac616 100644 --- a/drivers/iio/adc/mcp3911.c +++ b/drivers/iio/adc/mcp3911.c @@ -57,6 +57,7 @@ #define MCP3911_REG_READ(reg, id) ((((reg) << 1) | ((id) << 6) | (1 << 0)) & 0xff) #define MCP3911_REG_WRITE(reg, id) ((((reg) << 1) | ((id) << 6) | (0 << 0)) & 0xff) +#define MCP3911_REG_MASK GENMASK(4, 1) #define MCP3911_NUM_CHANNELS 2 @@ -89,8 +90,8 @@ static int mcp3911_read(struct mcp3911 *adc, u8 reg, u32 *val, u8 len) be32_to_cpus(val); *val >>= ((4 - len) * 8); - dev_dbg(&adc->spi->dev, "reading 0x%x from register 0x%x\n", *val, - reg >> 1); + dev_dbg(&adc->spi->dev, "reading 0x%x from register 0x%lx\n", *val, + FIELD_GET(MCP3911_REG_MASK, reg)); return ret; } -- GitLab From 174dac5dc800e4e2e4552baf6340846a344d01a3 Mon Sep 17 00:00:00 2001 From: Olivier Moysan Date: Wed, 12 Oct 2022 16:21:58 +0200 Subject: [PATCH 0082/1927] iio: adc: stm32-adc: fix channel sampling time init Fix channel init for ADC generic channel bindings. In generic channel initialization, stm32_adc_smpr_init() is called to initialize channel sampling time. The "st,min-sample-time-ns" property is an optional property. If it is not defined, stm32_adc_smpr_init() is currently skipped. However stm32_adc_smpr_init() must always be called, to force a minimum sampling time for the internal channels, as the minimum sampling time is known. Make stm32_adc_smpr_init() call unconditional. Fixes: 796e5d0b1e9b ("iio: adc: stm32-adc: use generic binding for sample-time") Signed-off-by: Olivier Moysan Reviewed-by: Andy Shevchenko Reviewed-by: Fabrice Gasnier Link: https://lore.kernel.org/r/20221012142205.13041-2-olivier.moysan@foss.st.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/adc/stm32-adc.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/iio/adc/stm32-adc.c b/drivers/iio/adc/stm32-adc.c index 6256977eb7f7b..3cda529f081db 100644 --- a/drivers/iio/adc/stm32-adc.c +++ b/drivers/iio/adc/stm32-adc.c @@ -2086,18 +2086,19 @@ static int stm32_adc_generic_chan_init(struct iio_dev *indio_dev, stm32_adc_chan_init_one(indio_dev, &channels[scan_index], val, vin[1], scan_index, differential); + val = 0; ret = fwnode_property_read_u32(child, "st,min-sample-time-ns", &val); /* st,min-sample-time-ns is optional */ - if (!ret) { - stm32_adc_smpr_init(adc, channels[scan_index].channel, val); - if (differential) - stm32_adc_smpr_init(adc, vin[1], val); - } else if (ret != -EINVAL) { + if (ret && ret != -EINVAL) { dev_err(&indio_dev->dev, "Invalid st,min-sample-time-ns property %d\n", ret); goto err; } + stm32_adc_smpr_init(adc, channels[scan_index].channel, val); + if (differential) + stm32_adc_smpr_init(adc, vin[1], val); + scan_index++; } -- GitLab From 72b2aa38191bcba28389b0e20bf6b4f15017ff2b Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Thu, 13 Oct 2022 15:04:04 +0300 Subject: [PATCH 0083/1927] tools: iio: iio_utils: fix digit calculation The iio_utils uses a digit calculation in order to know length of the file name containing a buffer number. The digit calculation does not work for number 0. This leads to allocation of one character too small buffer for the file-name when file name contains value '0'. (Eg. buffer0). Fix digit calculation by returning one digit to be present for number '0'. Fixes: 096f9b862e60 ("tools:iio:iio_utils: implement digit calculation") Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/Y0f+tKCz+ZAIoroQ@dc75zzyyyyyyyyyyyyycy-3.rev.dnainternet.fi Cc: Signed-off-by: Jonathan Cameron --- tools/iio/iio_utils.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/iio/iio_utils.c b/tools/iio/iio_utils.c index aadee6d34c74c..8d35893b2fa85 100644 --- a/tools/iio/iio_utils.c +++ b/tools/iio/iio_utils.c @@ -547,6 +547,10 @@ static int calc_digits(int num) { int count = 0; + /* It takes a digit to represent zero */ + if (!num) + return 1; + while (num != 0) { num /= 10; count++; -- GitLab From 4132f19173211856d35180958d2754f5c56d520a Mon Sep 17 00:00:00 2001 From: Cosmin Tanislav Date: Fri, 14 Oct 2022 15:37:22 +0300 Subject: [PATCH 0084/1927] iio: temperature: ltc2983: allocate iio channels once Currently, every time the device wakes up from sleep, the iio_chan array is reallocated, leaking the previous one until the device is removed (basically never). Move the allocation to the probe function to avoid this. Signed-off-by: Cosmin Tanislav Fixes: f110f3188e563 ("iio: temperature: Add support for LTC2983") Cc: Link: https://lore.kernel.org/r/20221014123724.1401011-2-demonsingur@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/temperature/ltc2983.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/iio/temperature/ltc2983.c b/drivers/iio/temperature/ltc2983.c index b652d2b39bcf2..a60ccf1836872 100644 --- a/drivers/iio/temperature/ltc2983.c +++ b/drivers/iio/temperature/ltc2983.c @@ -1385,13 +1385,6 @@ static int ltc2983_setup(struct ltc2983_data *st, bool assign_iio) return ret; } - st->iio_chan = devm_kzalloc(&st->spi->dev, - st->iio_channels * sizeof(*st->iio_chan), - GFP_KERNEL); - - if (!st->iio_chan) - return -ENOMEM; - ret = regmap_update_bits(st->regmap, LTC2983_GLOBAL_CONFIG_REG, LTC2983_NOTCH_FREQ_MASK, LTC2983_NOTCH_FREQ(st->filter_notch_freq)); @@ -1514,6 +1507,12 @@ static int ltc2983_probe(struct spi_device *spi) gpiod_set_value_cansleep(gpio, 0); } + st->iio_chan = devm_kzalloc(&spi->dev, + st->iio_channels * sizeof(*st->iio_chan), + GFP_KERNEL); + if (!st->iio_chan) + return -ENOMEM; + ret = ltc2983_setup(st, true); if (ret) return ret; -- GitLab From 54246b9034da08087ceb2083478c0d13403e12b4 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Mon, 3 Oct 2022 11:11:33 +0300 Subject: [PATCH 0085/1927] iio: at91-sama5d2_adc: Fix unsafe buffer attributes The iio_triggered_buffer_setup_ext() was changed by commit 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") to silently expect that all attributes given in buffer_attrs array are device-attributes. This expectation was not forced by the API - and some drivers did register attributes created by IIO_CONST_ATTR(). The added attribute "wrapping" does not copy the pointer to stored string constant and when the sysfs file is read the kernel will access to invalid location. Change the IIO_CONST_ATTRs from the driver to IIO_DEVICE_ATTR in order to prevent the invalid memory access. Signed-off-by: Matti Vaittinen Fixes: 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") Tested-by: Claudiu Beznea Link: https://lore.kernel.org/r/be69775aa302159f088b8b91894e6ec449bca65b.1664782676.git.mazziesaccount@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/adc/at91-sama5d2_adc.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c index 4294d6539cdb3..33e2515522149 100644 --- a/drivers/iio/adc/at91-sama5d2_adc.c +++ b/drivers/iio/adc/at91-sama5d2_adc.c @@ -2193,17 +2193,30 @@ static ssize_t at91_adc_get_watermark(struct device *dev, return scnprintf(buf, PAGE_SIZE, "%d\n", st->dma_st.watermark); } +static ssize_t hwfifo_watermark_min_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", "2"); +} + +static ssize_t hwfifo_watermark_max_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", AT91_HWFIFO_MAX_SIZE_STR); +} + static IIO_DEVICE_ATTR(hwfifo_enabled, 0444, at91_adc_get_fifo_state, NULL, 0); static IIO_DEVICE_ATTR(hwfifo_watermark, 0444, at91_adc_get_watermark, NULL, 0); - -static IIO_CONST_ATTR(hwfifo_watermark_min, "2"); -static IIO_CONST_ATTR(hwfifo_watermark_max, AT91_HWFIFO_MAX_SIZE_STR); +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_min, 0); +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_max, 0); static const struct attribute *at91_adc_fifo_attributes[] = { - &iio_const_attr_hwfifo_watermark_min.dev_attr.attr, - &iio_const_attr_hwfifo_watermark_max.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_min.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_max.dev_attr.attr, &iio_dev_attr_hwfifo_watermark.dev_attr.attr, &iio_dev_attr_hwfifo_enabled.dev_attr.attr, NULL, -- GitLab From ab0ee36e90f611f32c3a53afe9dc743de48138e2 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Mon, 3 Oct 2022 11:10:51 +0300 Subject: [PATCH 0086/1927] iio: adxl372: Fix unsafe buffer attributes The iio_triggered_buffer_setup_ext() was changed by commit 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") to silently expect that all attributes given in buffer_attrs array are device-attributes. This expectation was not forced by the API - and some drivers did register attributes created by IIO_CONST_ATTR(). The added attribute "wrapping" does not copy the pointer to stored string constant and when the sysfs file is read the kernel will access to invalid location. Change the IIO_CONST_ATTRs from the driver to IIO_DEVICE_ATTR in order to prevent the invalid memory access. Signed-off-by: Matti Vaittinen Fixes: 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") Cc: Link: https://lore.kernel.org/r/19158499623cdf7f9c5efae1f13c9f1a918ff75f.1664782676.git.mazziesaccount@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/accel/adxl372.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/iio/accel/adxl372.c b/drivers/iio/accel/adxl372.c index e3ecbaee61f70..bc53af809d5de 100644 --- a/drivers/iio/accel/adxl372.c +++ b/drivers/iio/accel/adxl372.c @@ -998,17 +998,30 @@ static ssize_t adxl372_get_fifo_watermark(struct device *dev, return sprintf(buf, "%d\n", st->watermark); } -static IIO_CONST_ATTR(hwfifo_watermark_min, "1"); -static IIO_CONST_ATTR(hwfifo_watermark_max, - __stringify(ADXL372_FIFO_SIZE)); +static ssize_t hwfifo_watermark_min_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", "1"); +} + +static ssize_t hwfifo_watermark_max_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", __stringify(ADXL372_FIFO_SIZE)); +} + +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_min, 0); +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_max, 0); static IIO_DEVICE_ATTR(hwfifo_watermark, 0444, adxl372_get_fifo_watermark, NULL, 0); static IIO_DEVICE_ATTR(hwfifo_enabled, 0444, adxl372_get_fifo_enabled, NULL, 0); static const struct attribute *adxl372_fifo_attributes[] = { - &iio_const_attr_hwfifo_watermark_min.dev_attr.attr, - &iio_const_attr_hwfifo_watermark_max.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_min.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_max.dev_attr.attr, &iio_dev_attr_hwfifo_watermark.dev_attr.attr, &iio_dev_attr_hwfifo_enabled.dev_attr.attr, NULL, -- GitLab From 5e23b33d1e84f04c80da6f1d89cbb3d3a3f81e01 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Mon, 3 Oct 2022 11:10:29 +0300 Subject: [PATCH 0087/1927] iio: adxl367: Fix unsafe buffer attributes The devm_iio_kfifo_buffer_setup_ext() was changed by commit 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") to silently expect that all attributes given in buffer_attrs array are device-attributes. This expectation was not forced by the API - and some drivers did register attributes created by IIO_CONST_ATTR(). The added attribute "wrapping" does not copy the pointer to stored string constant and when the sysfs file is read the kernel will access to invalid location. Change the IIO_CONST_ATTRs from the driver to IIO_DEVICE_ATTR in order to prevent the invalid memory access. Signed-off-by: Matti Vaittinen Fixes: 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") Cc: Link: https://lore.kernel.org/r/2e2d9ec34fb1df8ab8e2749199822db8cc91d302.1664782676.git.mazziesaccount@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/accel/adxl367.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/iio/accel/adxl367.c b/drivers/iio/accel/adxl367.c index 47feb375b70be..7c7d780407937 100644 --- a/drivers/iio/accel/adxl367.c +++ b/drivers/iio/accel/adxl367.c @@ -1185,17 +1185,30 @@ static ssize_t adxl367_get_fifo_watermark(struct device *dev, return sysfs_emit(buf, "%d\n", fifo_watermark); } -static IIO_CONST_ATTR(hwfifo_watermark_min, "1"); -static IIO_CONST_ATTR(hwfifo_watermark_max, - __stringify(ADXL367_FIFO_MAX_WATERMARK)); +static ssize_t hwfifo_watermark_min_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", "1"); +} + +static ssize_t hwfifo_watermark_max_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", __stringify(ADXL367_FIFO_MAX_WATERMARK)); +} + +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_min, 0); +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_max, 0); static IIO_DEVICE_ATTR(hwfifo_watermark, 0444, adxl367_get_fifo_watermark, NULL, 0); static IIO_DEVICE_ATTR(hwfifo_enabled, 0444, adxl367_get_fifo_enabled, NULL, 0); static const struct attribute *adxl367_fifo_attributes[] = { - &iio_const_attr_hwfifo_watermark_min.dev_attr.attr, - &iio_const_attr_hwfifo_watermark_max.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_min.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_max.dev_attr.attr, &iio_dev_attr_hwfifo_watermark.dev_attr.attr, &iio_dev_attr_hwfifo_enabled.dev_attr.attr, NULL, -- GitLab From a10a0f385ab8af08ddb762ac3eca11e1b6d1fe69 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Mon, 3 Oct 2022 11:11:12 +0300 Subject: [PATCH 0088/1927] iio: bmc150-accel-core: Fix unsafe buffer attributes The iio_triggered_buffer_setup_ext() was changed by commit 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") to silently expect that all attributes given in buffer_attrs array are device-attributes. This expectation was not forced by the API - and some drivers did register attributes created by IIO_CONST_ATTR(). The added attribute "wrapping" does not copy the pointer to stored string constant and when the sysfs file is read the kernel will access to invalid location. Change the IIO_CONST_ATTRs from the driver to IIO_DEVICE_ATTR in order to prevent the invalid memory access. Signed-off-by: Matti Vaittinen Fixes: 15097c7a1adc ("iio: buffer: wrap all buffer attributes into iio_dev_attr") Link: https://lore.kernel.org/r/cf8a56658fc38db8bed64f456d898f5ad5a2814f.1664782676.git.mazziesaccount@gmail.com Signed-off-by: Jonathan Cameron --- drivers/iio/accel/bmc150-accel-core.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c index 57e8a8350cd16..92f8b139acce9 100644 --- a/drivers/iio/accel/bmc150-accel-core.c +++ b/drivers/iio/accel/bmc150-accel-core.c @@ -925,17 +925,30 @@ static const struct iio_chan_spec_ext_info bmc150_accel_ext_info[] = { { } }; -static IIO_CONST_ATTR(hwfifo_watermark_min, "1"); -static IIO_CONST_ATTR(hwfifo_watermark_max, - __stringify(BMC150_ACCEL_FIFO_LENGTH)); +static ssize_t hwfifo_watermark_min_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", "1"); +} + +static ssize_t hwfifo_watermark_max_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, "%s\n", __stringify(BMC150_ACCEL_FIFO_LENGTH)); +} + +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_min, 0); +static IIO_DEVICE_ATTR_RO(hwfifo_watermark_max, 0); static IIO_DEVICE_ATTR(hwfifo_enabled, S_IRUGO, bmc150_accel_get_fifo_state, NULL, 0); static IIO_DEVICE_ATTR(hwfifo_watermark, S_IRUGO, bmc150_accel_get_fifo_watermark, NULL, 0); static const struct attribute *bmc150_accel_fifo_attributes[] = { - &iio_const_attr_hwfifo_watermark_min.dev_attr.attr, - &iio_const_attr_hwfifo_watermark_max.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_min.dev_attr.attr, + &iio_dev_attr_hwfifo_watermark_max.dev_attr.attr, &iio_dev_attr_hwfifo_watermark.dev_attr.attr, &iio_dev_attr_hwfifo_enabled.dev_attr.attr, NULL, -- GitLab From 17747577bbcb496e1b1c4096d64c2fc1e7bc0fef Mon Sep 17 00:00:00 2001 From: Siarhei Volkau Date: Sun, 16 Oct 2022 18:35:48 +0300 Subject: [PATCH 0089/1927] pinctrl: Ingenic: JZ4755 bug fixes Fixes UART1 function bits and MMC groups typo. For pins 0x97,0x99 function 0 is designated to PWM3/PWM5 respectively, function is 1 designated to the UART1. Diff from v1: - sent separately - added tag Fixes Cc: stable@vger.kernel.org Fixes: b582b5a434d3 ("pinctrl: Ingenic: Add pinctrl driver for JZ4755.") Tested-by: Siarhei Volkau Signed-off-by: Siarhei Volkau Link: https://lore.kernel.org/r/20221016153548.3024209-1-lis8215@gmail.com Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-ingenic.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c index 7e732076dedf0..9e46d83e5138c 100644 --- a/drivers/pinctrl/pinctrl-ingenic.c +++ b/drivers/pinctrl/pinctrl-ingenic.c @@ -667,7 +667,7 @@ static u8 jz4755_lcd_24bit_funcs[] = { 1, 1, 1, 1, 0, 0, }; static const struct group_desc jz4755_groups[] = { INGENIC_PIN_GROUP("uart0-data", jz4755_uart0_data, 0), INGENIC_PIN_GROUP("uart0-hwflow", jz4755_uart0_hwflow, 0), - INGENIC_PIN_GROUP("uart1-data", jz4755_uart1_data, 0), + INGENIC_PIN_GROUP("uart1-data", jz4755_uart1_data, 1), INGENIC_PIN_GROUP("uart2-data", jz4755_uart2_data, 1), INGENIC_PIN_GROUP("ssi-dt-b", jz4755_ssi_dt_b, 0), INGENIC_PIN_GROUP("ssi-dt-f", jz4755_ssi_dt_f, 0), @@ -721,7 +721,7 @@ static const char *jz4755_ssi_groups[] = { "ssi-ce1-b", "ssi-ce1-f", }; static const char *jz4755_mmc0_groups[] = { "mmc0-1bit", "mmc0-4bit", }; -static const char *jz4755_mmc1_groups[] = { "mmc0-1bit", "mmc0-4bit", }; +static const char *jz4755_mmc1_groups[] = { "mmc1-1bit", "mmc1-4bit", }; static const char *jz4755_i2c_groups[] = { "i2c-data", }; static const char *jz4755_cim_groups[] = { "cim-data", }; static const char *jz4755_lcd_groups[] = { -- GitLab From 406e14808ee695cbae1eafa5fd3ac563c29470ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20L=C3=B6hle?= Date: Thu, 13 Oct 2022 11:16:37 +0000 Subject: [PATCH 0090/1927] mmc: block: Remove error check of hw_reset on reset Before switching back to the right partition in mmc_blk_reset there used to be a check if hw_reset was even supported. This return value was removed, so there is no reason to check. Furthermore ensure part_curr is not falsely set to a valid value on reset or partition switch error. As part of this change the code paths of mmc_blk_reset calls were checked to ensure no commands are issued after a failed mmc_blk_reset directly without going through the block layer. Fixes: fefdd3c91e0a ("mmc: core: Drop superfluous validations in mmc_hw|sw_reset()") Cc: stable@vger.kernel.org Signed-off-by: Christian Loehle Reviewed-by: Adrian Hunter Link: https://lore.kernel.org/r/e91be6199d04414a91e20611c81bfe1d@hyperstone.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/block.c | 44 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/drivers/mmc/core/block.c b/drivers/mmc/core/block.c index 54cd009aee50e..db6d8a0999100 100644 --- a/drivers/mmc/core/block.c +++ b/drivers/mmc/core/block.c @@ -134,6 +134,7 @@ struct mmc_blk_data { * track of the current selected device partition. */ unsigned int part_curr; +#define MMC_BLK_PART_INVALID UINT_MAX /* Unknown partition active */ int area_type; /* debugfs files (only in main mmc_blk_data) */ @@ -987,33 +988,39 @@ static unsigned int mmc_blk_data_timeout_ms(struct mmc_host *host, return ms; } +/* + * Attempts to reset the card and get back to the requested partition. + * Therefore any error here must result in cancelling the block layer + * request, it must not be reattempted without going through the mmc_blk + * partition sanity checks. + */ static int mmc_blk_reset(struct mmc_blk_data *md, struct mmc_host *host, int type) { int err; + struct mmc_blk_data *main_md = dev_get_drvdata(&host->card->dev); if (md->reset_done & type) return -EEXIST; md->reset_done |= type; err = mmc_hw_reset(host->card); + /* + * A successful reset will leave the card in the main partition, but + * upon failure it might not be, so set it to MMC_BLK_PART_INVALID + * in that case. + */ + main_md->part_curr = err ? MMC_BLK_PART_INVALID : main_md->part_type; + if (err) + return err; /* Ensure we switch back to the correct partition */ - if (err) { - struct mmc_blk_data *main_md = - dev_get_drvdata(&host->card->dev); - int part_err; - - main_md->part_curr = main_md->part_type; - part_err = mmc_blk_part_switch(host->card, md->part_type); - if (part_err) { - /* - * We have failed to get back into the correct - * partition, so we need to abort the whole request. - */ - return -ENODEV; - } - } - return err; + if (mmc_blk_part_switch(host->card, md->part_type)) + /* + * We have failed to get back into the correct + * partition, so we need to abort the whole request. + */ + return -ENODEV; + return 0; } static inline void mmc_blk_reset_success(struct mmc_blk_data *md, int type) @@ -1871,8 +1878,9 @@ static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req) return; /* Reset before last retry */ - if (mqrq->retries + 1 == MMC_MAX_RETRIES) - mmc_blk_reset(md, card->host, type); + if (mqrq->retries + 1 == MMC_MAX_RETRIES && + mmc_blk_reset(md, card->host, type)) + return; /* Command errors fail fast, so use all MMC_MAX_RETRIES */ if (brq->sbc.error || brq->cmd.error) -- GitLab From 339e3eb1facd18a98ceb1171d70674780e5014a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20L=C3=B6hle?= Date: Fri, 7 Oct 2022 15:43:52 +0000 Subject: [PATCH 0091/1927] mmc: queue: Cancel recovery work on cleanup To prevent any recovery work running after the queue cleanup cancel it. Any recovery running post-cleanup dereferenced mq->card as NULL and was not meaningful to begin with. Cc: stable@vger.kernel.org Signed-off-by: Christian Loehle Acked-by: Adrian Hunter Link: https://lore.kernel.org/r/c865c0c9789d428494b67b820a78923e@hyperstone.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/queue.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index fefaa901b50f3..86be55d7cf556 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -493,6 +493,13 @@ void mmc_cleanup_queue(struct mmc_queue *mq) if (blk_queue_quiesced(q)) blk_mq_unquiesce_queue(q); + /* + * If the recovery completes the last (and only remaining) request in + * the queue, and the card has been removed, we could end up here with + * the recovery not quite finished yet, so cancel it. + */ + cancel_work_sync(&mq->recovery_work); + blk_mq_free_tag_set(&mq->tag_set); /* -- GitLab From 1ed5c3b22fc78735c539e4767832aea58db6761c Mon Sep 17 00:00:00 2001 From: Sascha Hauer Date: Thu, 13 Oct 2022 11:32:48 +0200 Subject: [PATCH 0092/1927] mmc: sdhci-esdhc-imx: Propagate ESDHC_FLAG_HS400* only on 8bit bus The core issues the warning "drop HS400 support since no 8-bit bus" when one of the ESDHC_FLAG_HS400* flags is set on a non 8bit capable host. To avoid this warning set these flags only on hosts that actually can do 8bit, i.e. have bus-width = <8> set in the device tree. Signed-off-by: Sascha Hauer Reviewed-by: Haibo Chen Fixes: 029e2476f9e6 ("mmc: sdhci-esdhc-imx: add HS400_ES support for i.MX8QXP") Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221013093248.2220802-1-s.hauer@pengutronix.de Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-esdhc-imx.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index 55981b0f0b10c..747df79d90eef 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -1660,6 +1660,10 @@ static int sdhci_esdhc_imx_probe(struct platform_device *pdev) host->mmc_host_ops.execute_tuning = usdhc_execute_tuning; } + err = sdhci_esdhc_imx_probe_dt(pdev, host, imx_data); + if (err) + goto disable_ahb_clk; + if (imx_data->socdata->flags & ESDHC_FLAG_MAN_TUNING) sdhci_esdhc_ops.platform_execute_tuning = esdhc_executing_tuning; @@ -1667,13 +1671,15 @@ static int sdhci_esdhc_imx_probe(struct platform_device *pdev) if (imx_data->socdata->flags & ESDHC_FLAG_ERR004536) host->quirks |= SDHCI_QUIRK_BROKEN_ADMA; - if (imx_data->socdata->flags & ESDHC_FLAG_HS400) + if (host->caps & MMC_CAP_8_BIT_DATA && + imx_data->socdata->flags & ESDHC_FLAG_HS400) host->mmc->caps2 |= MMC_CAP2_HS400; if (imx_data->socdata->flags & ESDHC_FLAG_BROKEN_AUTO_CMD23) host->quirks2 |= SDHCI_QUIRK2_ACMD23_BROKEN; - if (imx_data->socdata->flags & ESDHC_FLAG_HS400_ES) { + if (host->caps & MMC_CAP_8_BIT_DATA && + imx_data->socdata->flags & ESDHC_FLAG_HS400_ES) { host->mmc->caps2 |= MMC_CAP2_HS400_ES; host->mmc_host_ops.hs400_enhanced_strobe = esdhc_hs400_enhanced_strobe; @@ -1695,10 +1701,6 @@ static int sdhci_esdhc_imx_probe(struct platform_device *pdev) goto disable_ahb_clk; } - err = sdhci_esdhc_imx_probe_dt(pdev, host, imx_data); - if (err) - goto disable_ahb_clk; - sdhci_esdhc_imx_hwinit(host); err = sdhci_add_host(host); -- GitLab From 9dc0033e4658d6f9d9952c3c0c6be3ec25bc2985 Mon Sep 17 00:00:00 2001 From: Patrick Thompson Date: Thu, 13 Oct 2022 17:00:17 -0400 Subject: [PATCH 0093/1927] mmc: sdhci-pci-core: Disable ES for ASUS BIOS on Jasper Lake Enhanced Strobe (ES) does not work correctly on the ASUS 1100 series of devices. Jasper Lake eMMCs (pci_id 8086:4dc4) are supposed to support ES. There are also two system families under the series, thus this is being scoped to the ASUS BIOS. The failing ES prevents the installer from writing to disk. Falling back to HS400 without ES fixes the issue. Signed-off-by: Patrick Thompson Fixes: 315e3bd7ac19 ("mmc: sdhci-pci: Add support for Intel JSL") Acked-by: Adrian Hunter Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221013210017.3751025-1-ptf@google.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-pci-core.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c index 169b84761041f..34ea1acbb3ccf 100644 --- a/drivers/mmc/host/sdhci-pci-core.c +++ b/drivers/mmc/host/sdhci-pci-core.c @@ -914,6 +914,12 @@ static bool glk_broken_cqhci(struct sdhci_pci_slot *slot) dmi_match(DMI_SYS_VENDOR, "IRBIS")); } +static bool jsl_broken_hs400es(struct sdhci_pci_slot *slot) +{ + return slot->chip->pdev->device == PCI_DEVICE_ID_INTEL_JSL_EMMC && + dmi_match(DMI_BIOS_VENDOR, "ASUSTeK COMPUTER INC."); +} + static int glk_emmc_probe_slot(struct sdhci_pci_slot *slot) { int ret = byt_emmc_probe_slot(slot); @@ -922,9 +928,11 @@ static int glk_emmc_probe_slot(struct sdhci_pci_slot *slot) slot->host->mmc->caps2 |= MMC_CAP2_CQE; if (slot->chip->pdev->device != PCI_DEVICE_ID_INTEL_GLK_EMMC) { - slot->host->mmc->caps2 |= MMC_CAP2_HS400_ES; - slot->host->mmc_host_ops.hs400_enhanced_strobe = - intel_hs400_enhanced_strobe; + if (!jsl_broken_hs400es(slot)) { + slot->host->mmc->caps2 |= MMC_CAP2_HS400_ES; + slot->host->mmc_host_ops.hs400_enhanced_strobe = + intel_hs400_enhanced_strobe; + } slot->host->mmc->caps2 |= MMC_CAP2_CQE_DCMD; } -- GitLab From 9972e6b404884adae9eec7463e30d9b3c9a70b18 Mon Sep 17 00:00:00 2001 From: Matthew Ma Date: Fri, 14 Oct 2022 11:49:51 +0800 Subject: [PATCH 0094/1927] mmc: core: Fix kernel panic when remove non-standard SDIO card SDIO tuple is only allocated for standard SDIO card, especially it causes memory corruption issues when the non-standard SDIO card has removed, which is because the card device's reference counter does not increase for it at sdio_init_func(), but all SDIO card device reference counter gets decreased at sdio_release_func(). Fixes: 6f51be3d37df ("sdio: allow non-standard SDIO cards") Signed-off-by: Matthew Ma Reviewed-by: Weizhao Ouyang Reviewed-by: John Wang Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221014034951.2300386-1-ouyangweizhao@zeku.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/sdio_bus.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/core/sdio_bus.c b/drivers/mmc/core/sdio_bus.c index c6268c38c69e5..babf21a0adeb6 100644 --- a/drivers/mmc/core/sdio_bus.c +++ b/drivers/mmc/core/sdio_bus.c @@ -291,7 +291,8 @@ static void sdio_release_func(struct device *dev) { struct sdio_func *func = dev_to_sdio_func(dev); - sdio_free_func_cis(func); + if (!(func->card->quirks & MMC_QUIRK_NONSTD_SDIO)) + sdio_free_func_cis(func); kfree(func->info); kfree(func->tmpbuf); -- GitLab From d21f4b7ffc22c009da925046b69b15af08de9d75 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Fri, 14 Oct 2022 10:33:18 -0700 Subject: [PATCH 0095/1927] pinctrl: qcom: Avoid glitching lines when we first mux to output Back in the description of commit e440e30e26dd ("arm64: dts: qcom: sc7180: Avoid glitching SPI CS at bootup on trogdor") we described a problem that we were seeing on trogdor devices. I'll re-summarize here but you can also re-read the original commit. On trogdor devices, the BIOS is setting up the SPI chip select as: - mux special function (SPI chip select) - output enable - output low (unused because we've muxed as special function) In the kernel, however, we've moved away from using the chip select line as special function. Since the kernel wants to fully control the chip select it's far more efficient to treat the line as a GPIO rather than sending packet-like commands to the GENI firmware every time we want the line to toggle. When we transition from how the BIOS had the pin configured to how the kernel has the pin configured we end up glitching the line. That's because we _first_ change the mux of the line and then later set its output. This glitch is bad and can confuse the device on the other end of the line. The old commit e440e30e26dd ("arm64: dts: qcom: sc7180: Avoid glitching SPI CS at bootup on trogdor") fixed the glitch, though the solution was far from elegant. It essentially did the thing that everyone always hates: encoding a sequential program in device tree, even if it's a simple one. It also, unfortunately, got broken by commit b991f8c3622c ("pinctrl: core: Handling pinmux and pinconf separately"). After that commit we did all the muxing _first_ even though the config (set the pin to output high) was listed first. :( I looked at ideas for how to solve this more properly. My first thought was to use the "init" pinctrl state. In theory the "init" pinctrl state is supposed to be exactly for achieving glitch-free transitions. My dream would have been for the "init" pinctrl to do nothing at all. That would let us delay the automatic pin muxing until the driver could set things up and call pinctrl_init_done(). In other words, my dream was: /* Request the GPIO; init it 1 (because DT says GPIO_ACTIVE_LOW) */ devm_gpiod_get_index(dev, "cs", GPIOD_OUT_LOW); /* Output should be right, so we can remux, yay! */ pinctrl_init_done(dev); Unfortunately, it didn't work out. The primary reason is that the MSM GPIO driver implements gpio_request_enable(). As documented in pinmux.h, that function automatically remuxes a line as a GPIO. ...and it does this remuxing _before_ specifying the output of the pin. You can see in gpiod_get_index() that we call gpiod_request() before gpiod_configure_flags(). gpiod_request() isn't passed any flags so it has no idea what the eventual output will be. We could have debates about whether or not the automatic remuxing to GPIO for the MSM pinctrl was a good idea or not, but at this point I think there is a plethora of code that's relying on it and I certainly wouldn't suggest changing it. Alternatively, we could try to come up with a way to pass the initial output state to gpio_request_enable() and plumb all that through. That seems like it would be doable, but we'd have to plumb it through several layers in the stack. This patch implements yet another alternative. Here, we specifically avoid glitching the first time a pin is muxed to GPIO function if the direction of the pin is output. The idea is that we can read the state of the pin before we set the mux and make sure that the re-mux won't change the state. NOTES: - We only do this the first time since later swaps between mux states might want to preserve the old output value. In other words, I wouldn't want to break a driver that did: gpiod_set_value(g, 1); pinctrl_select_state(pinctrl, special_state); pinctrl_select_default_state(); /* We should be driving 1 even if "special_state" made the pin 0 */ - It's safe to do this the first time since the driver _couldn't_ have explicitly set a state. In order to even be able to control the GPIO (at least using gpiod) we have to have requested it which would have counted as the first mux. - In theory, instead of keeping track of the first time a pin was set as a GPIO we could enable the glitch-free behavior only when msm_pinmux_request_gpio() is in the callchain. That works an enables my "dream" implementation above where we use an "init" state to solve this. However, it's nice not to have to do this. By handling just the first transition to GPIO we can simply let the normal "default" remuxing happen and we can be assured that there won't be a glitch. Before this change I could see the glitch reported on the EC console when booting. It would say this when booting the kernel: Unexpected state 1 in CSNRE ISR After this change there is no error reported. Note that I haven't reproduced the original problem described in e440e30e26dd ("arm64: dts: qcom: sc7180: Avoid glitching SPI CS at bootup on trogdor") but I could believe it might happen in certain timing conditions. Fixes: b991f8c3622c ("pinctrl: core: Handling pinmux and pinconf separately") Signed-off-by: Douglas Anderson Reviewed-by: Stephen Boyd Link: https://lore.kernel.org/r/20221014103217.1.I656bb2c976ed626e5d37294eb252c1cf3be769dc@changeid Signed-off-by: Linus Walleij --- drivers/pinctrl/qcom/pinctrl-msm.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c index a2abfe987ab12..8bf8b21954fe4 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm.c +++ b/drivers/pinctrl/qcom/pinctrl-msm.c @@ -51,6 +51,7 @@ * detection. * @skip_wake_irqs: Skip IRQs that are handled by wakeup interrupt controller * @disabled_for_mux: These IRQs were disabled because we muxed away. + * @ever_gpio: This bit is set the first time we mux a pin to gpio_func. * @soc: Reference to soc_data of platform specific data. * @regs: Base addresses for the TLMM tiles. * @phys_base: Physical base address @@ -72,6 +73,7 @@ struct msm_pinctrl { DECLARE_BITMAP(enabled_irqs, MAX_NR_GPIO); DECLARE_BITMAP(skip_wake_irqs, MAX_NR_GPIO); DECLARE_BITMAP(disabled_for_mux, MAX_NR_GPIO); + DECLARE_BITMAP(ever_gpio, MAX_NR_GPIO); const struct msm_pinctrl_soc_data *soc; void __iomem *regs[MAX_NR_TILES]; @@ -218,6 +220,25 @@ static int msm_pinmux_set_mux(struct pinctrl_dev *pctldev, val = msm_readl_ctl(pctrl, g); + /* + * If this is the first time muxing to GPIO and the direction is + * output, make sure that we're not going to be glitching the pin + * by reading the current state of the pin and setting it as the + * output. + */ + if (i == gpio_func && (val & BIT(g->oe_bit)) && + !test_and_set_bit(group, pctrl->ever_gpio)) { + u32 io_val = msm_readl_io(pctrl, g); + + if (io_val & BIT(g->in_bit)) { + if (!(io_val & BIT(g->out_bit))) + msm_writel_io(io_val | BIT(g->out_bit), pctrl, g); + } else { + if (io_val & BIT(g->out_bit)) + msm_writel_io(io_val & ~BIT(g->out_bit), pctrl, g); + } + } + if (egpio_func && i == egpio_func) { if (val & BIT(g->egpio_present)) val &= ~BIT(g->egpio_enable); -- GitLab From a8dfb85095dd8b884ee962e64b16ef52bc54119d Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Fri, 14 Oct 2022 09:36:40 +0800 Subject: [PATCH 0096/1927] ALSA: hda/realtek: simplify the return of comp_bind() After commit 23904f7b2518 ("ALSA: hda: cs35l41: Remove suspend/resume hda hooks"), the return of comp_bind() can be simplified. No functional changed. Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221014013640.1142107-1-yangyingliang@huawei.com Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_realtek.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index e6c4bb5fa041a..7c177426bf303 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -6654,13 +6654,8 @@ static int comp_bind(struct device *dev) { struct hda_codec *cdc = dev_to_hda_codec(dev); struct alc_spec *spec = cdc->spec; - int ret; - ret = component_bind_all(dev, spec->comps); - if (ret) - return ret; - - return 0; + return component_bind_all(dev, spec->comps); } static void comp_unbind(struct device *dev) -- GitLab From b40af6183b685b0cf7870987b858de0d48db9ea0 Mon Sep 17 00:00:00 2001 From: Robert Marko Date: Thu, 6 Oct 2022 21:48:19 +0200 Subject: [PATCH 0097/1927] spi: qup: support using GPIO as chip select line Most of the device with QUP SPI adapter are actually using GPIO-s for chip select. However, this stopped working after ("spi: Retire legacy GPIO handling") as it introduced a check on ->use_gpio_descriptors flag and since spi-qup driver does not set the flag it meant that all of boards using GPIO-s and with QUP adapter SPI devices stopped working. So, to enable using GPIO-s again set ->use_gpio_descriptors to true and populate ->max_native_cs. Fixes: f48dc6b96649 ("spi: Retire legacy GPIO handling") Signed-off-by: Robert Marko Cc: luka.perkov@sartura.hr Reviewed-by: Linus Walleij Link: https://lore.kernel.org/r/20221006194819.1536932-1-robert.marko@sartura.hr Signed-off-by: Mark Brown --- drivers/spi/spi-qup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c index 7d89510dc3f00..678dc51ef0174 100644 --- a/drivers/spi/spi-qup.c +++ b/drivers/spi/spi-qup.c @@ -1057,6 +1057,8 @@ static int spi_qup_probe(struct platform_device *pdev) else master->num_chipselect = num_cs; + master->use_gpio_descriptors = true; + master->max_native_cs = SPI_NUM_CHIPSELECTS; master->bus_num = pdev->id; master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_CS_HIGH | SPI_LOOP; master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32); -- GitLab From 25b72d530e7aa185955196b63f53c38f751f1632 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Wed, 12 Oct 2022 12:18:54 -0700 Subject: [PATCH 0098/1927] fbdev: MIPS supports iomem addresses Add MIPS to fb_* helpers list for iomem addresses. This silences Sparse warnings about lacking __iomem address space casts: drivers/video/fbdev/pvr2fb.c:800:9: sparse: sparse: incorrect type in argument 1 (different address spaces) drivers/video/fbdev/pvr2fb.c:800:9: sparse: expected void const * drivers/video/fbdev/pvr2fb.c:800:9: sparse: got char [noderef] __iomem *screen_base Reported-by: kernel test robot Link: https://lore.kernel.org/lkml/202210100209.tR2Iqbqk-lkp@intel.com/ Cc: Helge Deller Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Signed-off-by: Kees Cook Signed-off-by: Helge Deller --- include/linux/fb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/fb.h b/include/linux/fb.h index 0aff76bcbb00c..bcb8658f5b64d 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -555,7 +555,7 @@ static inline struct apertures_struct *alloc_apertures(unsigned int max_num) { #elif defined(__i386__) || defined(__alpha__) || defined(__x86_64__) || \ defined(__hppa__) || defined(__sh__) || defined(__powerpc__) || \ - defined(__arm__) || defined(__aarch64__) + defined(__arm__) || defined(__aarch64__) || defined(__mips__) #define fb_readb __raw_readb #define fb_readw __raw_readw -- GitLab From 1013999b431b4bcdc1f5ae47dd3338122751db31 Mon Sep 17 00:00:00 2001 From: Siarhei Volkau Date: Sun, 16 Oct 2022 16:26:42 +0300 Subject: [PATCH 0099/1927] ASoC: codecs: jz4725b: add missed Line In power control bit Line In path stayed powered off during capturing or bypass to mixer. Signed-off-by: Siarhei Volkau Link: https://lore.kernel.org/r/20221016132648.3011729-2-lis8215@gmail.com Signed-off-by: Mark Brown --- sound/soc/codecs/jz4725b.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sound/soc/codecs/jz4725b.c b/sound/soc/codecs/jz4725b.c index 5201a8f6d7b63..cc7a48c96aa44 100644 --- a/sound/soc/codecs/jz4725b.c +++ b/sound/soc/codecs/jz4725b.c @@ -236,7 +236,8 @@ static const struct snd_soc_dapm_widget jz4725b_codec_dapm_widgets[] = { SND_SOC_DAPM_MIXER("DAC to Mixer", JZ4725B_CODEC_REG_CR1, REG_CR1_DACSEL_OFFSET, 0, NULL, 0), - SND_SOC_DAPM_MIXER("Line In", SND_SOC_NOPM, 0, 0, NULL, 0), + SND_SOC_DAPM_MIXER("Line In", JZ4725B_CODEC_REG_PMR1, + REG_PMR1_SB_LIN_OFFSET, 1, NULL, 0), SND_SOC_DAPM_MIXER("HP Out", JZ4725B_CODEC_REG_CR1, REG_CR1_HP_DIS_OFFSET, 1, NULL, 0), -- GitLab From 088777bf65b98cfa4b5378119d0a7d49a58ece44 Mon Sep 17 00:00:00 2001 From: Siarhei Volkau Date: Sun, 16 Oct 2022 16:26:43 +0300 Subject: [PATCH 0100/1927] ASoC: codecs: jz4725b: fix reported volume for Master ctl DAC volume control is the Master Playback Volume at the moment and it reports wrong levels in alsamixer and other alsa apps. The patch fixes that, as stated in manual on the jz4725b SoC (16.6.3.4 Programmable attenuation: GOD) the ctl range varies from -22.5dB to 0dB with 1.5dB step. Signed-off-by: Siarhei Volkau Link: https://lore.kernel.org/r/20221016132648.3011729-3-lis8215@gmail.com Signed-off-by: Mark Brown --- sound/soc/codecs/jz4725b.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/codecs/jz4725b.c b/sound/soc/codecs/jz4725b.c index cc7a48c96aa44..72549ee2e7893 100644 --- a/sound/soc/codecs/jz4725b.c +++ b/sound/soc/codecs/jz4725b.c @@ -142,8 +142,8 @@ struct jz_icdc { struct clk *clk; }; -static const SNDRV_CTL_TLVD_DECLARE_DB_LINEAR(jz4725b_dac_tlv, -2250, 0); static const SNDRV_CTL_TLVD_DECLARE_DB_LINEAR(jz4725b_line_tlv, -1500, 600); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(jz4725b_dac_tlv, -2250, 150, 0); static const struct snd_kcontrol_new jz4725b_codec_controls[] = { SOC_DOUBLE_TLV("Master Playback Volume", -- GitLab From 1538e2c8c9b7e7a656effcc6e4e7cfe8c1b405fd Mon Sep 17 00:00:00 2001 From: Siarhei Volkau Date: Sun, 16 Oct 2022 16:26:44 +0300 Subject: [PATCH 0101/1927] ASoC: codecs: jz4725b: use right control for Capture Volume Line In Bypass control is used as Master Capture at the moment this is completely incorrect. Current control routed to Mixer instead of ADC, thus can't affect Capture path. ADC control shall be used instead. ADC volume control parameters are different, so the patch fixes that as well. Manual says (16.6.3.2 Programmable input attenuation amplifier: PGATM) that gain varies in range 0dB..22.5dB with 1.5dB step. Signed-off-by: Siarhei Volkau Link: https://lore.kernel.org/r/20221016132648.3011729-4-lis8215@gmail.com Signed-off-by: Mark Brown --- sound/soc/codecs/jz4725b.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sound/soc/codecs/jz4725b.c b/sound/soc/codecs/jz4725b.c index 72549ee2e7893..4363d898a7d4e 100644 --- a/sound/soc/codecs/jz4725b.c +++ b/sound/soc/codecs/jz4725b.c @@ -136,13 +136,16 @@ enum { #define REG_CGR3_GO1L_OFFSET 0 #define REG_CGR3_GO1L_MASK (0x1f << REG_CGR3_GO1L_OFFSET) +#define REG_CGR10_GIL_OFFSET 0 +#define REG_CGR10_GIR_OFFSET 4 + struct jz_icdc { struct regmap *regmap; void __iomem *base; struct clk *clk; }; -static const SNDRV_CTL_TLVD_DECLARE_DB_LINEAR(jz4725b_line_tlv, -1500, 600); +static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(jz4725b_adc_tlv, 0, 150, 0); static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(jz4725b_dac_tlv, -2250, 150, 0); static const struct snd_kcontrol_new jz4725b_codec_controls[] = { @@ -151,11 +154,11 @@ static const struct snd_kcontrol_new jz4725b_codec_controls[] = { REG_CGR1_GODL_OFFSET, REG_CGR1_GODR_OFFSET, 0xf, 1, jz4725b_dac_tlv), - SOC_DOUBLE_R_TLV("Master Capture Volume", - JZ4725B_CODEC_REG_CGR3, - JZ4725B_CODEC_REG_CGR2, - REG_CGR2_GO1R_OFFSET, - 0x1f, 1, jz4725b_line_tlv), + SOC_DOUBLE_TLV("Master Capture Volume", + JZ4725B_CODEC_REG_CGR10, + REG_CGR10_GIL_OFFSET, + REG_CGR10_GIR_OFFSET, + 0xf, 0, jz4725b_adc_tlv), SOC_SINGLE("Master Playback Switch", JZ4725B_CODEC_REG_CR1, REG_CR1_DAC_MUTE_OFFSET, 1, 1), -- GitLab From 80852f8268769715db335a22305e81a0c4a38a84 Mon Sep 17 00:00:00 2001 From: Siarhei Volkau Date: Sun, 16 Oct 2022 16:26:45 +0300 Subject: [PATCH 0102/1927] ASoC: codecs: jz4725b: fix capture selector naming At the moment Capture source selector appears on Playback tab in the alsamixer and has a senseless name. Let's fix that. Signed-off-by: Siarhei Volkau Link: https://lore.kernel.org/r/20221016132648.3011729-5-lis8215@gmail.com Signed-off-by: Mark Brown --- sound/soc/codecs/jz4725b.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sound/soc/codecs/jz4725b.c b/sound/soc/codecs/jz4725b.c index 4363d898a7d4e..d57c2c6a3addf 100644 --- a/sound/soc/codecs/jz4725b.c +++ b/sound/soc/codecs/jz4725b.c @@ -183,7 +183,7 @@ static SOC_VALUE_ENUM_SINGLE_DECL(jz4725b_codec_adc_src_enum, jz4725b_codec_adc_src_texts, jz4725b_codec_adc_src_values); static const struct snd_kcontrol_new jz4725b_codec_adc_src_ctrl = - SOC_DAPM_ENUM("Route", jz4725b_codec_adc_src_enum); + SOC_DAPM_ENUM("ADC Source Capture Route", jz4725b_codec_adc_src_enum); static const struct snd_kcontrol_new jz4725b_codec_mixer_controls[] = { SOC_DAPM_SINGLE("Line In Bypass", JZ4725B_CODEC_REG_CR1, @@ -228,7 +228,7 @@ static const struct snd_soc_dapm_widget jz4725b_codec_dapm_widgets[] = { SND_SOC_DAPM_ADC("ADC", "Capture", JZ4725B_CODEC_REG_PMR1, REG_PMR1_SB_ADC_OFFSET, 1), - SND_SOC_DAPM_MUX("ADC Source", SND_SOC_NOPM, 0, 0, + SND_SOC_DAPM_MUX("ADC Source Capture Route", SND_SOC_NOPM, 0, 0, &jz4725b_codec_adc_src_ctrl), /* Mixer */ @@ -287,11 +287,11 @@ static const struct snd_soc_dapm_route jz4725b_codec_dapm_routes[] = { {"Mixer", NULL, "DAC to Mixer"}, {"Mixer to ADC", NULL, "Mixer"}, - {"ADC Source", "Mixer", "Mixer to ADC"}, - {"ADC Source", "Line In", "Line In"}, - {"ADC Source", "Mic 1", "Mic 1"}, - {"ADC Source", "Mic 2", "Mic 2"}, - {"ADC", NULL, "ADC Source"}, + {"ADC Source Capture Route", "Mixer", "Mixer to ADC"}, + {"ADC Sourc Capture Routee", "Line In", "Line In"}, + {"ADC Source Capture Route", "Mic 1", "Mic 1"}, + {"ADC Source Capture Route", "Mic 2", "Mic 2"}, + {"ADC", NULL, "ADC Source Capture Route"}, {"Out Stage", NULL, "Mixer"}, {"HP Out", NULL, "Out Stage"}, -- GitLab From 472a1482325b3a285e0bcf82c0b0edc689b7e8cd Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Sun, 2 Oct 2022 08:04:19 -0400 Subject: [PATCH 0103/1927] counter: Reduce DEFINE_COUNTER_ARRAY_POLARITY() to defining counter_array A spare warning was reported for drivers/counter/ti-ecap-capture.c:: sparse warnings: (new ones prefixed by >>) >> drivers/counter/ti-ecap-capture.c:380:8: sparse: sparse: symbol 'ecap_cnt_pol_array' was not declared. Should it be static? vim +/ecap_cnt_pol_array +380 drivers/counter/ti-ecap-capture.c 379 > 380 static DEFINE_COUNTER_ARRAY_POLARITY(ecap_cnt_pol_array, ecap_cnt_pol_avail, ECAP_NB_CEVT); 381 The first argument to the DEFINE_COUNTER_ARRAY_POLARITY() macro is a token serving as the symbol name in the definition of a new struct counter_array structure. However, this macro actually expands to two statements:: #define DEFINE_COUNTER_ARRAY_POLARITY(_name, _enums, _length) \ DEFINE_COUNTER_AVAILABLE(_name##_available, _enums); \ struct counter_array _name = { \ .type = COUNTER_COMP_SIGNAL_POLARITY, \ .avail = &(_name##_available), \ .length = (_length), \ } Because of this, the "static" on line 380 only applies to the first statement. This patch splits out the DEFINE_COUNTER_AVAILABLE() line and leaves DEFINE_COUNTER_ARRAY_POLARITY() as a simple structure definition to avoid issues like this. Reported-by: kernel test robot Link: https://lore.kernel.org/all/202210020619.NQbyomII-lkp@intel.com/ Cc: Julien Panis Signed-off-by: William Breathitt Gray --- drivers/counter/ti-ecap-capture.c | 3 ++- include/linux/counter.h | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/counter/ti-ecap-capture.c b/drivers/counter/ti-ecap-capture.c index af10de30aba51..b8dbf0212a8e7 100644 --- a/drivers/counter/ti-ecap-capture.c +++ b/drivers/counter/ti-ecap-capture.c @@ -377,7 +377,8 @@ static const enum counter_signal_polarity ecap_cnt_pol_avail[] = { COUNTER_SIGNAL_POLARITY_NEGATIVE, }; -static DEFINE_COUNTER_ARRAY_POLARITY(ecap_cnt_pol_array, ecap_cnt_pol_avail, ECAP_NB_CEVT); +static DEFINE_COUNTER_AVAILABLE(ecap_cnt_pol_available, ecap_cnt_pol_avail); +static DEFINE_COUNTER_ARRAY_POLARITY(ecap_cnt_pol_array, ecap_cnt_pol_available, ECAP_NB_CEVT); static struct counter_comp ecap_cnt_signal_ext[] = { COUNTER_COMP_ARRAY_POLARITY(ecap_cnt_pol_read, ecap_cnt_pol_write, ecap_cnt_pol_array), diff --git a/include/linux/counter.h b/include/linux/counter.h index c41fa602ed283..b63746637de2a 100644 --- a/include/linux/counter.h +++ b/include/linux/counter.h @@ -542,11 +542,10 @@ struct counter_array { #define DEFINE_COUNTER_ARRAY_CAPTURE(_name, _length) \ DEFINE_COUNTER_ARRAY_U64(_name, _length) -#define DEFINE_COUNTER_ARRAY_POLARITY(_name, _enums, _length) \ - DEFINE_COUNTER_AVAILABLE(_name##_available, _enums); \ +#define DEFINE_COUNTER_ARRAY_POLARITY(_name, _available, _length) \ struct counter_array _name = { \ .type = COUNTER_COMP_SIGNAL_POLARITY, \ - .avail = &(_name##_available), \ + .avail = &(_available), \ .length = (_length), \ } -- GitLab From ec0286dce78c3bb0e6a665c0baade2f2db56ce00 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 12 Oct 2022 17:51:25 +0300 Subject: [PATCH 0104/1927] counter: ti-ecap-capture: fix IS_ERR() vs NULL check The devm_counter_alloc() function returns NULL on error. It doesn't return error pointers. Fixes: 4e2f42aa00b6 ("counter: ti-ecap-capture: capture driver support for ECAP") Signed-off-by: Dan Carpenter Reviewed-by: Julien Panis Acked-by: Vignesh Raghavendra Link: https://lore.kernel.org/r/Y0bUbZvfDJHBG9C6@kili/ Signed-off-by: William Breathitt Gray --- drivers/counter/ti-ecap-capture.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/counter/ti-ecap-capture.c b/drivers/counter/ti-ecap-capture.c index b8dbf0212a8e7..fb1cb1774674a 100644 --- a/drivers/counter/ti-ecap-capture.c +++ b/drivers/counter/ti-ecap-capture.c @@ -480,8 +480,8 @@ static int ecap_cnt_probe(struct platform_device *pdev) int ret; counter_dev = devm_counter_alloc(dev, sizeof(*ecap_dev)); - if (IS_ERR(counter_dev)) - return PTR_ERR(counter_dev); + if (!counter_dev) + return -ENOMEM; counter_dev->name = ECAP_DRV_NAME; counter_dev->parent = dev; -- GitLab From 10f3f10f202109e3841c0c75add5a743ca197205 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 13 Oct 2022 14:46:39 -0700 Subject: [PATCH 0105/1927] MAINTAINERS: git://github -> https://github.com for broadcom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Github deprecated the git:// links about a year ago, so let's move to the https:// URLs instead. Acked-by: William Zhang Reviewed-by: Philippe Mathieu-Daudé Reported-by: Conor Dooley Link: https://github.blog/2021-09-01-improving-git-protocol-security-github/ Signed-off-by: Palmer Dabbelt Signed-off-by: Florian Fainelli --- MAINTAINERS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..83612d907bed7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3984,7 +3984,7 @@ M: Rafał Miłecki R: Broadcom internal kernel review list L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained -T: git git://github.com/broadcom/stblinux.git +T: git https://github.com/broadcom/stblinux.git F: Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml F: arch/arm64/boot/dts/broadcom/bcmbca/* N: bcmbca @@ -4009,7 +4009,7 @@ R: Broadcom internal kernel review list L: linux-rpi-kernel@lists.infradead.org (moderated for non-subscribers) L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained -T: git git://github.com/broadcom/stblinux.git +T: git https://github.com/broadcom/stblinux.git F: Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml F: drivers/pci/controller/pcie-brcmstb.c F: drivers/staging/vc04_services @@ -4023,7 +4023,7 @@ M: Ray Jui M: Scott Branden R: Broadcom internal kernel review list S: Maintained -T: git git://github.com/broadcom/mach-bcm +T: git https://github.com/broadcom/mach-bcm F: arch/arm/mach-bcm/ N: bcm281* N: bcm113* @@ -4088,7 +4088,7 @@ M: Florian Fainelli R: Broadcom internal kernel review list L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained -T: git git://github.com/broadcom/stblinux.git +T: git https://github.com/broadcom/stblinux.git F: Documentation/devicetree/bindings/pci/brcm,stb-pcie.yaml F: arch/arm/boot/dts/bcm7*.dts* F: arch/arm/include/asm/hardware/cache-b15-rac.h @@ -4120,7 +4120,7 @@ M: Florian Fainelli R: Broadcom internal kernel review list L: linux-mips@vger.kernel.org S: Maintained -T: git git://github.com/broadcom/stblinux.git +T: git https://github.com/broadcom/stblinux.git F: arch/mips/bmips/* F: arch/mips/boot/dts/brcm/bcm*.dts* F: arch/mips/include/asm/mach-bmips/* @@ -4259,7 +4259,7 @@ M: Scott Branden R: Broadcom internal kernel review list L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained -T: git git://github.com/broadcom/stblinux.git +T: git https://github.com/broadcom/stblinux.git F: arch/arm64/boot/dts/broadcom/northstar2/* F: arch/arm64/boot/dts/broadcom/stingray/* F: drivers/clk/bcm/clk-ns* @@ -4329,7 +4329,7 @@ M: Florian Fainelli R: Broadcom internal kernel review list L: linux-pm@vger.kernel.org S: Maintained -T: git git://github.com/broadcom/stblinux.git +T: git https://github.com/broadcom/stblinux.git F: drivers/soc/bcm/bcm63xx/bcm-pmb.c F: include/dt-bindings/soc/bcm-pmb.h -- GitLab From 35cc9d622e8cd45029a1656ab2c6817538bc4180 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 14 Oct 2022 17:24:43 -0700 Subject: [PATCH 0106/1927] selftests/bpf: Add reproducer for decl_tag in func_proto return type It should trigger a WARN_ON_ONCE in btf_type_id_size. btf_func_proto_check kernel/bpf/btf.c:4447 [inline] btf_check_all_types kernel/bpf/btf.c:4723 [inline] btf_parse_type_sec kernel/bpf/btf.c:4752 [inline] btf_parse kernel/bpf/btf.c:5026 [inline] btf_new_fd+0x1926/0x1e70 kernel/bpf/btf.c:6892 bpf_btf_load kernel/bpf/syscall.c:4324 [inline] __sys_bpf+0xb7d/0x4cf0 kernel/bpf/syscall.c:5010 __do_sys_bpf kernel/bpf/syscall.c:5069 [inline] __se_sys_bpf kernel/bpf/syscall.c:5067 [inline] __x64_sys_bpf+0x75/0xb0 kernel/bpf/syscall.c:5067 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd Cc: Yonghong Song Cc: Martin KaFai Lau Signed-off-by: Stanislav Fomichev Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20221015002444.2680969-1-sdf@google.com Signed-off-by: Martin KaFai Lau --- tools/testing/selftests/bpf/prog_tests/btf.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index 127b8caa3dc1e..24dd6214394e0 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -3935,6 +3935,19 @@ static struct btf_raw_test raw_tests[] = { .btf_load_err = true, .err_str = "Invalid type_id", }, +{ + .descr = "decl_tag test #16, func proto, return type", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_VAR_ENC(NAME_TBD, 1, 0), /* [2] */ + BTF_TYPE_ENC(NAME_TBD, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), 2), (-1), /* [3] */ + BTF_FUNC_PROTO_ENC(3, 0), /* [4] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0local\0tag1"), + .btf_load_err = true, + .err_str = "Invalid return type", +}, { .descr = "type_tag test #1", .raw_types = { -- GitLab From ea68376c8bed5cd156900852aada20c3a0874d17 Mon Sep 17 00:00:00 2001 From: Stanislav Fomichev Date: Fri, 14 Oct 2022 17:24:44 -0700 Subject: [PATCH 0107/1927] bpf: prevent decl_tag from being referenced in func_proto Syzkaller was able to hit the following issue: ------------[ cut here ]------------ WARNING: CPU: 0 PID: 3609 at kernel/bpf/btf.c:1946 btf_type_id_size+0x2d5/0x9d0 kernel/bpf/btf.c:1946 Modules linked in: CPU: 0 PID: 3609 Comm: syz-executor361 Not tainted 6.0.0-syzkaller-02734-g0326074ff465 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/22/2022 RIP: 0010:btf_type_id_size+0x2d5/0x9d0 kernel/bpf/btf.c:1946 Code: ef e8 7f 8e e4 ff 41 83 ff 0b 77 28 f6 44 24 10 18 75 3f e8 6d 91 e4 ff 44 89 fe bf 0e 00 00 00 e8 20 8e e4 ff e8 5b 91 e4 ff <0f> 0b 45 31 f6 e9 98 02 00 00 41 83 ff 12 74 18 e8 46 91 e4 ff 44 RSP: 0018:ffffc90003cefb40 EFLAGS: 00010293 RAX: 0000000000000000 RBX: 0000000000000002 RCX: 0000000000000000 RDX: ffff8880259c0000 RSI: ffffffff81968415 RDI: 0000000000000005 RBP: ffff88801270ca00 R08: 0000000000000005 R09: 000000000000000e R10: 0000000000000011 R11: 0000000000000000 R12: 0000000000000000 R13: 0000000000000011 R14: ffff888026ee6424 R15: 0000000000000011 FS: 000055555641b300(0000) GS:ffff8880b9a00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000f2e258 CR3: 000000007110e000 CR4: 00000000003506f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: btf_func_proto_check kernel/bpf/btf.c:4447 [inline] btf_check_all_types kernel/bpf/btf.c:4723 [inline] btf_parse_type_sec kernel/bpf/btf.c:4752 [inline] btf_parse kernel/bpf/btf.c:5026 [inline] btf_new_fd+0x1926/0x1e70 kernel/bpf/btf.c:6892 bpf_btf_load kernel/bpf/syscall.c:4324 [inline] __sys_bpf+0xb7d/0x4cf0 kernel/bpf/syscall.c:5010 __do_sys_bpf kernel/bpf/syscall.c:5069 [inline] __se_sys_bpf kernel/bpf/syscall.c:5067 [inline] __x64_sys_bpf+0x75/0xb0 kernel/bpf/syscall.c:5067 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7f0fbae41c69 Code: 28 c3 e8 2a 14 00 00 66 2e 0f 1f 84 00 00 00 00 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 c0 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007ffc8aeb6228 EFLAGS: 00000246 ORIG_RAX: 0000000000000141 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007f0fbae41c69 RDX: 0000000000000020 RSI: 0000000020000140 RDI: 0000000000000012 RBP: 00007f0fbae05e10 R08: 0000000000000000 R09: 0000000000000000 R10: 00000000ffffffff R11: 0000000000000246 R12: 00007f0fbae05ea0 R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 Looks like it tries to create a func_proto which return type is decl_tag. For the details, see Martin's spot on analysis in [0]. 0: https://lore.kernel.org/bpf/CAKH8qBuQDLva_hHxxBuZzyAcYNO4ejhovz6TQeVSk8HY-2SO6g@mail.gmail.com/T/#mea6524b3fcd6298347432226e81b1e6155efc62c Cc: Yonghong Song Cc: Martin KaFai Lau Fixes: bd16dee66ae4 ("bpf: Add BTF_KIND_DECL_TAG typedef support") Reported-by: syzbot+d8bd751aef7c6b39a344@syzkaller.appspotmail.com Signed-off-by: Stanislav Fomichev Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20221015002444.2680969-2-sdf@google.com Signed-off-by: Martin KaFai Lau --- kernel/bpf/btf.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index eba603cec2c58..35c07afac924e 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -4436,6 +4436,11 @@ static int btf_func_proto_check(struct btf_verifier_env *env, return -EINVAL; } + if (btf_type_is_resolve_source_only(ret_type)) { + btf_verifier_log_type(env, t, "Invalid return type"); + return -EINVAL; + } + if (btf_type_needs_resolve(ret_type) && !env_type_is_resolved(env, ret_type_id)) { err = btf_resolve(env, ret_type, ret_type_id); -- GitLab From 40b21d466a86bd5b10d24f59746ed41283a9b3f6 Mon Sep 17 00:00:00 2001 From: Robert Marko Date: Fri, 19 Aug 2022 00:06:27 +0200 Subject: [PATCH 0108/1927] arm64: dts: qcom: ipq8074: correct APCS register space size APCS DTS addition that was merged, was not supposed to get merged as it was part of patch series that was superseded by 2 more patch series that resolved issues with this one and greatly simplified things. Since it already got merged, start by correcting the register space size as APCS will not be providing regmap for PLL and it will conflict with the standalone A53 PLL node. Fixes: 50ed9fffec3a ("arm64: dts: qcom: ipq8074: add APCS node") Signed-off-by: Robert Marko Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220818220628.339366-8-robimarko@gmail.com --- arch/arm64/boot/dts/qcom/ipq8074.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi index a47acf9bdf24f..a721cdd80489e 100644 --- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi @@ -668,7 +668,7 @@ watchdog: watchdog@b017000 { apcs_glb: mailbox@b111000 { compatible = "qcom,ipq8074-apcs-apps-global"; - reg = <0x0b111000 0x6000>; + reg = <0x0b111000 0x1000>; #clock-cells = <1>; #mbox-cells = <1>; -- GitLab From 20772f506fa4aab4d03035807f30eecee856e274 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Wed, 21 Sep 2022 12:52:58 +0300 Subject: [PATCH 0109/1927] Revert "arm64: dts: qcom: msm8996: add missing TCSR syscon compatible" This reverts commit 8a99e0fc8bd3 ("arm64: dts: qcom: msm8996: add missing TCSR syscon compatible"). This commit marked the saw3 (syscon@9a10000) node as compatible with qcom,tcsr-msm8996. However the mentioned device is not not a TCSR (system registers, hardware mutex). It is a CPU power controller/regulator, which is currently being handled as a syscon. Fixes: 8a99e0fc8bd3 ("arm64: dts: qcom: msm8996: add missing TCSR syscon compatible") Cc: Krzysztof Kozlowski Signed-off-by: Dmitry Baryshkov Reviewed-by: Krzysztof Kozlowski Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220921095258.2332568-1-dmitry.baryshkov@linaro.org --- arch/arm64/boot/dts/qcom/msm8996.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi index c0a2baffa49d5..aba7176443919 100644 --- a/arch/arm64/boot/dts/qcom/msm8996.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi @@ -3504,7 +3504,7 @@ frame@98c0000 { }; saw3: syscon@9a10000 { - compatible = "qcom,tcsr-msm8996", "syscon"; + compatible = "syscon"; reg = <0x09a10000 0x1000>; }; -- GitLab From bd9f3dcf42d943b53190f99bcdbcfe98a56ac4cd Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 29 Aug 2022 09:49:47 -0700 Subject: [PATCH 0110/1927] arm64: dts: qcom: sa8155p-adp: Specify which LDO modes are allowed This board uses RPMH, specifies "regulator-allow-set-load" for LDOs, but doesn't specify any modes with "regulator-allowed-modes". Prior to commit efb0cb50c427 ("regulator: qcom-rpmh: Implement get_optimum_mode(), not set_load()") the above meant that we were able to set either LPM or HPM mode. After that commit (and fixes [1]) we'll be stuck at the initial mode. Discussion of this has resulted in the decision that the old dts files were wrong and should be fixed to fully restore old functionality. Let's re-enable the old functionality by fixing the dts. NOTE: while here, let's also remove the nonsensical "regulator-allow-set-load" on the fixed regulator "vreg_s4a_1p8". [1] https://lore.kernel.org/r/20220824142229.RFT.v2.2.I6f77860e5cd98bf5c67208fa9edda4a08847c304@changeid Fixes: 5b85e8f2225c ("arm64: dts: qcom: sa8155p-adp: Add base dts file") Signed-off-by: Douglas Anderson Reviewed-by: Andrew Halaney Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220829094903.v2.1.Id59c32b560c4662d8b3697de2bd494d08d654806@changeid --- arch/arm64/boot/dts/qcom/sa8155p-adp.dts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts index 87ab0e1ecd16d..4dee790f1049d 100644 --- a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts @@ -43,7 +43,6 @@ vreg_s4a_1p8: smps4 { regulator-always-on; regulator-boot-on; - regulator-allow-set-load; vin-supply = <&vreg_3p3>; }; @@ -137,6 +136,9 @@ vreg_l5a_0p88: ldo5 { regulator-max-microvolt = <880000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l7a_1p8: ldo7 { @@ -152,6 +154,9 @@ vreg_l10a_2p96: ldo10 { regulator-max-microvolt = <2960000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l11a_0p8: ldo11 { @@ -258,6 +263,9 @@ vreg_l5c_1p2: ldo5 { regulator-max-microvolt = <1200000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l7c_1p8: ldo7 { @@ -273,6 +281,9 @@ vreg_l8c_1p2: ldo8 { regulator-max-microvolt = <1200000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l10c_3p3: ldo10 { -- GitLab From 09a1710b3e12e7ac8d871506bc395a9e8a0f63d6 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 29 Aug 2022 09:49:48 -0700 Subject: [PATCH 0111/1927] arm64: dts: qcom: sa8295p-adp: Specify which LDO modes are allowed This board uses RPMH, specifies "regulator-allow-set-load" for LDOs, but doesn't specify any modes with "regulator-allowed-modes". Prior to commit efb0cb50c427 ("regulator: qcom-rpmh: Implement get_optimum_mode(), not set_load()") the above meant that we were able to set either LPM or HPM mode. After that commit (and fixes [1]) we'll be stuck at the initial mode. Discussion of this has resulted in the decision that the old dts files were wrong and should be fixed to fully restore old functionality. Let's re-enable the old functionality by fixing the dts. [1] https://lore.kernel.org/r/20220824142229.RFT.v2.2.I6f77860e5cd98bf5c67208fa9edda4a08847c304@changeid Fixes: 519183af39b2 ("arm64: dts: qcom: add SA8540P and ADP") Signed-off-by: Douglas Anderson Reviewed-by: Andrew Halaney Reviewed-by: Konrad Dybcio Reviewed-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220829094903.v2.2.I430a56702ab0af65244e62667bb7743107de0c96@changeid --- arch/arm64/boot/dts/qcom/sa8295p-adp.dts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts index b608b82dff03c..2c62ba6a49c5b 100644 --- a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts +++ b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts @@ -83,6 +83,9 @@ vreg_l3c: ldo3 { regulator-max-microvolt = <1200000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l4c: ldo4 { @@ -98,6 +101,9 @@ vreg_l6c: ldo6 { regulator-max-microvolt = <1200000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l7c: ldo7 { @@ -113,6 +119,9 @@ vreg_l10c: ldo10 { regulator-max-microvolt = <2504000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l17c: ldo17 { @@ -121,6 +130,9 @@ vreg_l17c: ldo17 { regulator-max-microvolt = <2504000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; }; -- GitLab From a4543e21ae363f4f094fb3c3503d77029e0c5d90 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 29 Aug 2022 09:49:49 -0700 Subject: [PATCH 0112/1927] arm64: dts: qcom: sc8280xp-crd: Specify which LDO modes are allowed This board uses RPMH, specifies "regulator-allow-set-load" for LDOs, but doesn't specify any modes with "regulator-allowed-modes". Prior to commit efb0cb50c427 ("regulator: qcom-rpmh: Implement get_optimum_mode(), not set_load()") the above meant that we were able to set either LPM or HPM mode. After that commit (and fixes [1]) we'll be stuck at the initial mode. Discussion of this has resulted in the decision that the old dts files were wrong and should be fixed to fully restore old functionality. Let's re-enable the old functionality by fixing the dts. [1] https://lore.kernel.org/r/20220824142229.RFT.v2.2.I6f77860e5cd98bf5c67208fa9edda4a08847c304@changeid Fixes: ccd3517faf18 ("arm64: dts: qcom: sc8280xp: Add reference device") Signed-off-by: Douglas Anderson Reviewed-by: Andrew Halaney Reviewed-by: Konrad Dybcio Reviewed-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220829094903.v2.3.Ie7d2c50d2b42ef2d364f3a0c8e300e5ce1875b79@changeid --- arch/arm64/boot/dts/qcom/sc8280xp-crd.dts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts index fea7d8273ccde..5e30349efd204 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts @@ -124,6 +124,9 @@ vreg_l7c: ldo7 { regulator-max-microvolt = <2504000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l13c: ldo13 { @@ -146,6 +149,9 @@ vreg_l3d: ldo3 { regulator-max-microvolt = <1200000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l4d: ldo4 { -- GitLab From aa30e786202e4ed1df980442d305658441f65859 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 29 Aug 2022 09:49:50 -0700 Subject: [PATCH 0113/1927] arm64: dts: qcom: sm8150-xperia-kumano: Specify which LDO modes are allowed This board uses RPMH, specifies "regulator-allow-set-load" for LDOs, but doesn't specify any modes with "regulator-allowed-modes". Prior to commit efb0cb50c427 ("regulator: qcom-rpmh: Implement get_optimum_mode(), not set_load()") the above meant that we were able to set either LPM or HPM mode. After that commit (and fixes [1]) we'll be stuck at the initial mode. Discussion of this has resulted in the decision that the old dts files were wrong and should be fixed to fully restore old functionality. Let's re-enable the old functionality by fixing the dts. [1] https://lore.kernel.org/r/20220824142229.RFT.v2.2.I6f77860e5cd98bf5c67208fa9edda4a08847c304@changeid Fixes: d0a6ce59ea4e ("arm64: dts: qcom: sm8150: Add support for SONY Xperia 1 / 5 (Kumano platform)") Signed-off-by: Douglas Anderson Reviewed-by: Andrew Halaney Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220829094903.v2.4.I51d60414a42ba9e3008e208d60a04c9ffc425fa7@changeid --- arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi index 014fe3a315489..fb6e5a140c9f6 100644 --- a/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150-sony-xperia-kumano.dtsi @@ -348,6 +348,9 @@ vreg_l6c_2p9: ldo6 { regulator-max-microvolt = <2960000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l7c_3p0: ldo7 { @@ -367,6 +370,9 @@ vreg_l9c_2p9: ldo9 { regulator-max-microvolt = <2960000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l10c_3p3: ldo10 { -- GitLab From b7870d460c05ce31e2311036d91de1e2e0b32cea Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 29 Aug 2022 09:49:51 -0700 Subject: [PATCH 0114/1927] arm64: dts: qcom: sm8250-xperia-edo: Specify which LDO modes are allowed This board uses RPMH, specifies "regulator-allow-set-load" for LDOs, but doesn't specify any modes with "regulator-allowed-modes". Prior to commit efb0cb50c427 ("regulator: qcom-rpmh: Implement get_optimum_mode(), not set_load()") the above meant that we were able to set either LPM or HPM mode. After that commit (and fixes [1]) we'll be stuck at the initial mode. Discussion of this has resulted in the decision that the old dts files were wrong and should be fixed to fully restore old functionality. Let's re-enable the old functionality by fixing the dts. [1] https://lore.kernel.org/r/20220824142229.RFT.v2.2.I6f77860e5cd98bf5c67208fa9edda4a08847c304@changeid Fixes: 69cdb97ef652 ("arm64: dts: qcom: sm8250: Add support for SONY Xperia 1 II / 5 II (Edo platform)") Signed-off-by: Douglas Anderson Reviewed-by: Andrew Halaney Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220829094903.v2.5.Ie446d5183d8b1e9ec4e32228ca300e604e3315eb@changeid --- arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi index 549e0a2aa9fe4..5428aab3058dd 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-sony-xperia-edo.dtsi @@ -317,6 +317,9 @@ vreg_l6c_2p9: ldo6 { regulator-max-microvolt = <2960000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l7c_2p85: ldo7 { @@ -339,6 +342,9 @@ vreg_l9c_2p9: ldo9 { regulator-max-microvolt = <2960000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l10c_3p3: ldo10 { -- GitLab From 1ce8aaf6abdc35cde555924418b3d4516b4ec871 Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 29 Aug 2022 09:49:52 -0700 Subject: [PATCH 0115/1927] arm64: dts: qcom: sm8350-hdk: Specify which LDO modes are allowed This board uses RPMH, specifies "regulator-allow-set-load" for LDOs, but doesn't specify any modes with "regulator-allowed-modes". Prior to commit efb0cb50c427 ("regulator: qcom-rpmh: Implement get_optimum_mode(), not set_load()") the above meant that we were able to set either LPM or HPM mode. After that commit (and fixes [1]) we'll be stuck at the initial mode. Discussion of this has resulted in the decision that the old dts files were wrong and should be fixed to fully restore old functionality. Let's re-enable the old functionality by fixing the dts. [1] https://lore.kernel.org/r/20220824142229.RFT.v2.2.I6f77860e5cd98bf5c67208fa9edda4a08847c304@changeid Fixes: 9208c19f2124 ("arm64: dts: qcom: Introduce SM8350 HDK") Signed-off-by: Douglas Anderson Reviewed-by: Andrew Halaney Reviewed-by: Vinod Koul Reviewed-by: Konrad Dybcio Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220829094903.v2.6.I6799be85cf36d3b494f803cba767a569080624f5@changeid --- arch/arm64/boot/dts/qcom/sm8350-hdk.dts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts index 0fcf5bd88fc7d..69ae6503c2f66 100644 --- a/arch/arm64/boot/dts/qcom/sm8350-hdk.dts +++ b/arch/arm64/boot/dts/qcom/sm8350-hdk.dts @@ -107,6 +107,9 @@ vreg_l5b_0p88: ldo5 { regulator-max-microvolt = <888000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l6b_1p2: ldo6 { @@ -115,6 +118,9 @@ vreg_l6b_1p2: ldo6 { regulator-max-microvolt = <1208000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l7b_2p96: ldo7 { @@ -123,6 +129,9 @@ vreg_l7b_2p96: ldo7 { regulator-max-microvolt = <2504000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; vreg_l9b_1p2: ldo9 { @@ -131,6 +140,9 @@ vreg_l9b_1p2: ldo9 { regulator-max-microvolt = <1200000>; regulator-initial-mode = ; regulator-allow-set-load; + regulator-allowed-modes = + ; }; }; -- GitLab From 8d6b458ce6e93286a607e54f787f7a86067f58bd Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Wed, 5 Oct 2022 16:33:05 +0200 Subject: [PATCH 0116/1927] arm64: dts: qcom: sc8280xp: fix ufs_card_phy ref clock The GCC_UFS_REF_CLKREF_CLK must be enabled or the second UFS controller fails to enumerate on sa8295p-adp. Note that the vendor kernel enables both GCC_UFS_REF_CLKREF_CLK and GCC_UFS_1_CARD_CLKREF_CLK and it is possible that the former should be modelled as a parent of the latter. The clock driver also has a GCC_UFS_CARD_CLKREF_CLK clock which the firmware appears to enable on the ADP. The usual lack of documentation for Qualcomm SoCs makes this a highly annoying guessing game, but as the second controller works on the ADP without either card reference clock enabled, only enable GCC_UFS_REF_CLKREF_CLK for now. Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Signed-off-by: Johan Hovold Reviewed-by: Konrad Dybcio Reviewed-by: Brian Masney Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20221005143305.388-1-johan+linaro@kernel.org --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index c32bcded2aef1..4921bbaf5caf7 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -959,7 +959,7 @@ ufs_card_phy: phy@1da7000 { ranges; clock-names = "ref", "ref_aux"; - clocks = <&gcc GCC_UFS_1_CARD_CLKREF_CLK>, + clocks = <&gcc GCC_UFS_REF_CLKREF_CLK>, <&gcc GCC_UFS_CARD_PHY_AUX_CLK>; resets = <&ufs_card_hc 0>; -- GitLab From f3aa975e230e060c07dcfdf3fe92b59809422c13 Mon Sep 17 00:00:00 2001 From: Brian Masney Date: Thu, 6 Oct 2022 10:55:29 -0400 Subject: [PATCH 0117/1927] arm64: dts: qcom: sc8280xp: correct ref clock for ufs_mem_phy The first UFS host controller fails to start on the SA8540P automotive board (QDrive3) due to the following errors: ufshcd-qcom 1d84000.ufs: ufshcd_query_flag: Sending flag query for idn 18 failed, err = 253 ufshcd-qcom 1d84000.ufs: ufshcd_query_flag: Sending flag query for idn 18 failed, err = 253 ufshcd-qcom 1d84000.ufs: ufshcd_query_flag: Sending flag query for idn 18 failed, err = 253 ufshcd-qcom 1d84000.ufs: ufshcd_query_flag_retry: query attribute, opcode 5, idn 18, failed with error 253 after 3 retries The system eventually fails to boot with the warning: gcc_ufs_phy_axi_clk status stuck at 'off' This issue can be worked around by adding clk_ignore_unused to the kernel command line since the system firmware sets up this clock for us. Let's fix this issue by updating the ref clock on ufs_mem_phy. Note that the downstream MSM 5.4 sources list this as ref_clk_parent. With this patch, the SA8540P is able to be booted without clk_ignore_unused. Signed-off-by: Brian Masney Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Tested-by: Johan Hovold Reviewed-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20221006145529.755521-1-bmasney@redhat.com --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index 4921bbaf5caf7..f5824f44e15f4 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -891,7 +891,7 @@ ufs_mem_phy: phy@1d87000 { ranges; clock-names = "ref", "ref_aux"; - clocks = <&rpmhcc RPMH_CXO_CLK>, + clocks = <&gcc GCC_UFS_REF_CLKREF_CLK>, <&gcc GCC_UFS_PHY_PHY_AUX_CLK>; resets = <&ufs_mem_hc 0>; -- GitLab From 31b3b3059791be536e2ec0c6830767b596af340f Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 19 Sep 2022 11:44:51 +0200 Subject: [PATCH 0118/1927] arm64: dts: qcom: sc8280xp: fix USB0 PHY PCS_MISC registers The USB0 SS PHY node had the PCS_MISC register block (0x1200) replaced with PCS_USB (0x1700). Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Signed-off-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220919094454.1574-2-johan+linaro@kernel.org --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index f5824f44e15f4..cab9965fe0f8c 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -1184,7 +1184,7 @@ usb_0_ssphy: usb3-phy@88eb400 { <0 0x088ec400 0 0x1f0>, <0 0x088eba00 0 0x100>, <0 0x088ebc00 0 0x3ec>, - <0 0x088ec700 0 0x64>; + <0 0x088ec200 0 0x18>; #phy-cells = <0>; #clock-cells = <0>; clocks = <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>; -- GitLab From 81cad26c6c3984d01b0612069c70ffd820f62dfa Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 19 Sep 2022 11:44:52 +0200 Subject: [PATCH 0119/1927] arm64: dts: qcom: sc8280xp: fix USB1 PHY RX1 registers The USB1 SS PHY node had the RX1 register block (0x600) replaced with RX2 (0xc00). Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Signed-off-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220919094454.1574-3-johan+linaro@kernel.org --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index cab9965fe0f8c..fce05d69ffbe8 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -1242,7 +1242,7 @@ usb_1_qmpphy: phy-wrapper@8904000 { usb_1_ssphy: usb3-phy@8903400 { reg = <0 0x08903400 0 0x100>, - <0 0x08903c00 0 0x3ec>, + <0 0x08903600 0 0x3ec>, <0 0x08904400 0 0x1f0>, <0 0x08903a00 0 0x100>, <0 0x08903c00 0 0x3ec>, -- GitLab From 8723c3f290c7b798c0cbd89998576e6b365bda3a Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 19 Sep 2022 11:44:53 +0200 Subject: [PATCH 0120/1927] arm64: dts: qcom: sc8280xp: fix USB PHY PCS registers With the current binding, the PCS register block (0x1400) needs to include the PCS_USB registers (0x1700). Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Signed-off-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220919094454.1574-4-johan+linaro@kernel.org --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index fce05d69ffbe8..2cf2b3d836313 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -1181,7 +1181,7 @@ usb_0_qmpphy: phy-wrapper@88ec000 { usb_0_ssphy: usb3-phy@88eb400 { reg = <0 0x088eb400 0 0x100>, <0 0x088eb600 0 0x3ec>, - <0 0x088ec400 0 0x1f0>, + <0 0x088ec400 0 0x364>, <0 0x088eba00 0 0x100>, <0 0x088ebc00 0 0x3ec>, <0 0x088ec200 0 0x18>; @@ -1243,7 +1243,7 @@ usb_1_qmpphy: phy-wrapper@8904000 { usb_1_ssphy: usb3-phy@8903400 { reg = <0 0x08903400 0 0x100>, <0 0x08903600 0 0x3ec>, - <0 0x08904400 0 0x1f0>, + <0 0x08904400 0 0x364>, <0 0x08903a00 0 0x100>, <0 0x08903c00 0 0x3ec>, <0 0x08904200 0 0x18>; -- GitLab From 7cdfb7a54ac88f7cb6d830ebb78bdbcbcb44bb4c Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Mon, 19 Sep 2022 11:44:54 +0200 Subject: [PATCH 0121/1927] arm64: dts: qcom: sc8280xp: drop broken DP PHY nodes The DP PHY register layout of the current binding do not apply to the newer USB4/USB3/DP PHY which uses a different register layout entirely. Drop the DP PHY subnodes until the binding has been updated to prevent the driver from corrupting unrelated registers. Note that this is also needed in order to not break USB with an upcoming PHY driver change that checks for overlapping register regions. Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Signed-off-by: Johan Hovold Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220919094454.1574-5-johan+linaro@kernel.org --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index 2cf2b3d836313..6ed4ab2c08b4b 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -1191,16 +1191,6 @@ usb_0_ssphy: usb3-phy@88eb400 { clock-names = "pipe0"; clock-output-names = "usb0_phy_pipe_clk_src"; }; - - usb_0_dpphy: dp-phy@88ed200 { - reg = <0 0x088ed200 0 0x200>, - <0 0x088ed400 0 0x200>, - <0 0x088eda00 0 0x200>, - <0 0x088ea600 0 0x200>, - <0 0x088ea800 0 0x200>; - #clock-cells = <1>; - #phy-cells = <0>; - }; }; usb_1_hsphy: phy@8902000 { @@ -1253,16 +1243,6 @@ usb_1_ssphy: usb3-phy@8903400 { clock-names = "pipe0"; clock-output-names = "usb1_phy_pipe_clk_src"; }; - - usb_1_dpphy: dp-phy@8904200 { - reg = <0 0x08904200 0 0x200>, - <0 0x08904400 0 0x200>, - <0 0x08904a00 0 0x200>, - <0 0x08904600 0 0x200>, - <0 0x08904800 0 0x200>; - #clock-cells = <1>; - #phy-cells = <0>; - }; }; system-cache-controller@9200000 { -- GitLab From 2ff4ba9e37024735f5cefc5ea2a73fc66addfe0e Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Thu, 29 Sep 2022 21:55:21 +0200 Subject: [PATCH 0122/1927] clk: rs9: Fix I2C accessors Add custom I2C accessors to this driver, since the regular I2C regmap ones do not generate the exact I2C transfers required by the chip. On I2C write, it is mandatory to send transfer length first, on read the chip returns the transfer length in first byte. Instead of always reading back 8 bytes, which is the default and also the size of the entire register file, set BCP register to 1 to read out 1 byte which is less wasteful. Fixes: 892e0ddea1aa ("clk: rs9: Add Renesas 9-series PCIe clock generator driver") Reported-by: Alexander Stein Signed-off-by: Marek Vasut Link: https://lore.kernel.org/r/20220929195521.284497-1-marex@denx.de Reviewed-by: Alexander Stein Signed-off-by: Stephen Boyd --- drivers/clk/clk-renesas-pcie.c | 65 ++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/drivers/clk/clk-renesas-pcie.c b/drivers/clk/clk-renesas-pcie.c index 4f5df1fc74b46..e6247141d0c05 100644 --- a/drivers/clk/clk-renesas-pcie.c +++ b/drivers/clk/clk-renesas-pcie.c @@ -90,13 +90,66 @@ static const struct regmap_access_table rs9_writeable_table = { .n_yes_ranges = ARRAY_SIZE(rs9_writeable_ranges), }; +static int rs9_regmap_i2c_write(void *context, + unsigned int reg, unsigned int val) +{ + struct i2c_client *i2c = context; + const u8 data[3] = { reg, 1, val }; + const int count = ARRAY_SIZE(data); + int ret; + + ret = i2c_master_send(i2c, data, count); + if (ret == count) + return 0; + else if (ret < 0) + return ret; + else + return -EIO; +} + +static int rs9_regmap_i2c_read(void *context, + unsigned int reg, unsigned int *val) +{ + struct i2c_client *i2c = context; + struct i2c_msg xfer[2]; + u8 txdata = reg; + u8 rxdata[2]; + int ret; + + xfer[0].addr = i2c->addr; + xfer[0].flags = 0; + xfer[0].len = 1; + xfer[0].buf = (void *)&txdata; + + xfer[1].addr = i2c->addr; + xfer[1].flags = I2C_M_RD; + xfer[1].len = 2; + xfer[1].buf = (void *)rxdata; + + ret = i2c_transfer(i2c->adapter, xfer, 2); + if (ret < 0) + return ret; + if (ret != 2) + return -EIO; + + /* + * Byte 0 is transfer length, which is always 1 due + * to BCP register programming to 1 in rs9_probe(), + * ignore it and use data from Byte 1. + */ + *val = rxdata[1]; + return 0; +} + static const struct regmap_config rs9_regmap_config = { .reg_bits = 8, .val_bits = 8, - .cache_type = REGCACHE_FLAT, - .max_register = 0x8, + .cache_type = REGCACHE_NONE, + .max_register = RS9_REG_BCP, .rd_table = &rs9_readable_table, .wr_table = &rs9_writeable_table, + .reg_write = rs9_regmap_i2c_write, + .reg_read = rs9_regmap_i2c_read, }; static int rs9_get_output_config(struct rs9_driver_data *rs9, int idx) @@ -242,11 +295,17 @@ static int rs9_probe(struct i2c_client *client) return ret; } - rs9->regmap = devm_regmap_init_i2c(client, &rs9_regmap_config); + rs9->regmap = devm_regmap_init(&client->dev, NULL, + client, &rs9_regmap_config); if (IS_ERR(rs9->regmap)) return dev_err_probe(&client->dev, PTR_ERR(rs9->regmap), "Failed to allocate register map\n"); + /* Always read back 1 Byte via I2C */ + ret = regmap_write(rs9->regmap, RS9_REG_BCP, 1); + if (ret < 0) + return ret; + /* Register clock */ for (i = 0; i < rs9->chip_info->num_clks; i++) { snprintf(name, 5, "DIF%d", i); -- GitLab From 7e3e6e1b75c9643e25e8ca7d6caf1b1faf8f022e Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Wed, 5 Oct 2022 18:13:44 +0100 Subject: [PATCH 0123/1927] clk: sifive: select by default if SOC_SIFIVE With the aim of dropping direct selects of drivers from Kconfig.socs, default the SiFive clock drivers to the value of SOC_SIFIVE. Signed-off-by: Conor Dooley Link: https://lore.kernel.org/r/20221005171348.167476-2-conor@kernel.org Signed-off-by: Stephen Boyd --- drivers/clk/sifive/Kconfig | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/clk/sifive/Kconfig b/drivers/clk/sifive/Kconfig index 9132c3c4aa868..b7fde0aadfcbb 100644 --- a/drivers/clk/sifive/Kconfig +++ b/drivers/clk/sifive/Kconfig @@ -2,7 +2,8 @@ menuconfig CLK_SIFIVE bool "SiFive SoC driver support" - depends on RISCV || COMPILE_TEST + depends on SOC_SIFIVE || COMPILE_TEST + default SOC_SIFIVE help SoC drivers for SiFive Linux-capable SoCs. @@ -10,6 +11,7 @@ if CLK_SIFIVE config CLK_SIFIVE_PRCI bool "PRCI driver for SiFive SoCs" + default SOC_SIFIVE select RESET_CONTROLLER select RESET_SIMPLE select CLK_ANALOGBITS_WRPLL_CLN28HPC -- GitLab From 8fbf8636cd37b821ce3482748340008dbbe2dcb5 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sun, 9 Oct 2022 10:50:56 +0800 Subject: [PATCH 0124/1927] clk: mediatek: clk-mt8195-topckgen: Fix error return code in clk_mt8195_topck_probe() If devm_clk_hw_register_mux() fails in clk_mt8195_topck_probe(), it should return error code. Fixes: deeb2af77cf6 ("clk: mediatek: clk-mt8195-topckgen: Register mfg_ck_fast_ref as generic mux") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221009025056.35311-1-yangyingliang@huawei.com Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Stephen Boyd --- drivers/clk/mediatek/clk-mt8195-topckgen.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/clk/mediatek/clk-mt8195-topckgen.c b/drivers/clk/mediatek/clk-mt8195-topckgen.c index 8cbab5ca2e581..1e016329c1d23 100644 --- a/drivers/clk/mediatek/clk-mt8195-topckgen.c +++ b/drivers/clk/mediatek/clk-mt8195-topckgen.c @@ -1270,8 +1270,10 @@ static int clk_mt8195_topck_probe(struct platform_device *pdev) hw = devm_clk_hw_register_mux(&pdev->dev, "mfg_ck_fast_ref", mfg_fast_parents, ARRAY_SIZE(mfg_fast_parents), CLK_SET_RATE_PARENT, (base + 0x250), 8, 1, 0, &mt8195_clk_lock); - if (IS_ERR(hw)) + if (IS_ERR(hw)) { + r = PTR_ERR(hw); goto unregister_muxes; + } top_clk_data->hws[CLK_TOP_MFG_CK_FAST_REF] = hw; r = clk_mt8195_reg_mfg_mux_notifier(&pdev->dev, -- GitLab From 97074216917b4188f0af3e52cc5b3f2b277bbbca Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Sun, 9 Oct 2022 14:35:20 +0800 Subject: [PATCH 0125/1927] drm/amdgpu: add tmz support for GC 11.0.1 this patch to add tmz support for GC 11.0.1. Signed-off-by: Yifan Zhang Reviewed-by: Tim Huang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 34233a74248c2..9c0d9baab4e2a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -542,6 +542,7 @@ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) case IP_VERSION(10, 3, 1): /* YELLOW_CARP*/ case IP_VERSION(10, 3, 3): + case IP_VERSION(11, 0, 1): /* Don't enable it by default yet. */ if (amdgpu_tmz < 1) { -- GitLab From bfa8cb055fef348c896b70b9ac13d1838665031a Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Sun, 9 Oct 2022 14:37:06 +0800 Subject: [PATCH 0126/1927] drm/amdgpu: allow secure submission on gfx11 and sdma6 This patch to allow secure submission on gfx11 and sdma6. Signed-off-by: Yifan Zhang Reviewed-by: Tim Huang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 1 + drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 251109723ab63..257b2e4de6007 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -6058,6 +6058,7 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { .align_mask = 0xff, .nop = PACKET3(PACKET3_NOP, 0x3FFF), .support_64bit_ptrs = true, + .secure_submission_supported = true, .vmhub = AMDGPU_GFXHUB_0, .get_rptr = gfx_v11_0_ring_get_rptr_gfx, .get_wptr = gfx_v11_0_ring_get_wptr_gfx, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index da3beb0bf2fa2..d9ae69be5ea4a 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -1523,6 +1523,7 @@ static const struct amdgpu_ring_funcs sdma_v6_0_ring_funcs = { .align_mask = 0xf, .nop = SDMA_PKT_NOP_HEADER_OP(SDMA_OP_NOP), .support_64bit_ptrs = true, + .secure_submission_supported = true, .vmhub = AMDGPU_GFXHUB_0, .get_rptr = sdma_v6_0_ring_get_rptr, .get_wptr = sdma_v6_0_ring_get_wptr, -- GitLab From 7a94c8602fbe585fac636dae355cf73b53d50866 Mon Sep 17 00:00:00 2001 From: Hawking Zhang Date: Tue, 17 May 2022 22:10:41 +0800 Subject: [PATCH 0127/1927] drm/amdgpu: extend HWIP_MAX_INSTANCE to 28 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit more ip instances are available Acked-by: Christian König Signed-off-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index ae9371b172e3a..3ce91f660c3f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -678,7 +678,7 @@ enum amd_hw_ip_block_type { MAX_HWIP }; -#define HWIP_MAX_INSTANCE 11 +#define HWIP_MAX_INSTANCE 28 #define HW_ID_MAX 300 #define IP_VERSION(mj, mn, rv) (((mj) << 16) | ((mn) << 8) | (rv)) -- GitLab From e9ff000b5a2a6b2f34828ebcfb0a829ce2dcdc2a Mon Sep 17 00:00:00 2001 From: Hawking Zhang Date: Tue, 31 Aug 2021 15:30:08 +0800 Subject: [PATCH 0128/1927] drm/amdgpu: update psp_fw_type enum in amdgpu_ucode header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To match with the definition in psp firmware Acked-by: Christian König Signed-off-by: Hawking Zhang Reviewed-by: Le Ma Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 1c36235b4539c..552e06929229c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -125,6 +125,7 @@ enum psp_fw_type { PSP_FW_TYPE_PSP_INTF_DRV, PSP_FW_TYPE_PSP_DBG_DRV, PSP_FW_TYPE_PSP_RAS_DRV, + PSP_FW_TYPE_MAX_INDEX, }; /* version_major=2, version_minor=0 */ -- GitLab From 886f1816c2ad9e5bf594549cc4977df7a533dde9 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 3 Oct 2022 15:00:47 -0400 Subject: [PATCH 0129/1927] drm/amdgpu: convert vega20_ih.c to IP version checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consistency with newer asics. Acked-by: Christian König Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/vega20_ih.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c index 59dfca093155c..1706081d054dd 100644 --- a/drivers/gpu/drm/amd/amdgpu/vega20_ih.c +++ b/drivers/gpu/drm/amd/amdgpu/vega20_ih.c @@ -267,7 +267,7 @@ static void vega20_ih_reroute_ih(struct amdgpu_device *adev) /* vega20 ih reroute will go through psp this * function is used for newer asics starting arcturus */ - if (adev->asic_type >= CHIP_ARCTURUS) { + if (adev->ip_versions[OSSSYS_HWIP][0] >= IP_VERSION(4, 2, 1)) { /* Reroute to IH ring 1 for VMC */ WREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_INDEX, 0x12); tmp = RREG32_SOC15(OSSSYS, 0, mmIH_CLIENT_CFG_DATA); @@ -308,7 +308,7 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev) adev->nbio.funcs->ih_control(adev); - if (adev->asic_type == CHIP_ARCTURUS && + if ((adev->ip_versions[OSSSYS_HWIP][0] == IP_VERSION(4, 2, 1)) && adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN); if (adev->irq.ih.use_bus_addr) { @@ -321,7 +321,7 @@ static int vega20_ih_irq_init(struct amdgpu_device *adev) /* psp firmware won't program IH_CHICKEN for aldebaran * driver needs to program it properly according to * MC_SPACE type in IH_RB_CNTL */ - if (adev->asic_type == CHIP_ALDEBARAN) { + if (adev->ip_versions[OSSSYS_HWIP][0] == IP_VERSION(4, 4, 0)) { ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN_ALDEBARAN); if (adev->irq.ih.use_bus_addr) { ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN, -- GitLab From eb1670787eb7b9fe339631605dda1a53603a7699 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Mon, 3 Oct 2022 15:09:57 -0400 Subject: [PATCH 0130/1927] drm/amdgpu: convert amdgpu_amdkfd_gpuvm.c to IP version checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For consistency with the rest of the code. Acked-by: Christian König Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 978d3970b5cc4..8ad01e1ee4c64 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -418,9 +418,9 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; - switch (adev->asic_type) { - case CHIP_ARCTURUS: - case CHIP_ALDEBARAN: + switch (adev->ip_versions[GC_HWIP][0]) { + case IP_VERSION(9, 4, 1): + case IP_VERSION(9, 4, 2): if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { if (bo_adev == adev) { if (uncached) @@ -429,7 +429,7 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) mapping_flags |= AMDGPU_VM_MTYPE_CC; else mapping_flags |= AMDGPU_VM_MTYPE_RW; - if (adev->asic_type == CHIP_ALDEBARAN && + if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) && adev->gmc.xgmi.connected_to_cpu) snoop = true; } else { -- GitLab From 7fe441d8b77a1e4fe09099092945d27607dda69b Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Wed, 12 Oct 2022 15:37:07 +0800 Subject: [PATCH 0131/1927] drm/amdgpu/si_dma: remove unused variable in si_dma_stop() After commit 571c05365892 ("drm/amdgpu: switch sdma buffer function tear down to a helper"), the variable 'ring' is not used anymore, it can be removed. Signed-off-by: Yang Yingliang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/si_dma.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c index 4d5e718540aa9..abca8b529721e 100644 --- a/drivers/gpu/drm/amd/amdgpu/si_dma.c +++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c @@ -112,14 +112,12 @@ static void si_dma_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq, static void si_dma_stop(struct amdgpu_device *adev) { - struct amdgpu_ring *ring; u32 rb_cntl; unsigned i; amdgpu_sdma_unset_buffer_funcs_helper(adev); for (i = 0; i < adev->sdma.num_instances; i++) { - ring = &adev->sdma.instance[i].ring; /* dma0 */ rb_cntl = RREG32(DMA_RB_CNTL + sdma_offsets[i]); rb_cntl &= ~DMA_RB_ENABLE; -- GitLab From a7310d8de3ba60a6ec4294392daf747b8333b3b2 Mon Sep 17 00:00:00 2001 From: Danijel Slivka Date: Tue, 4 Oct 2022 15:39:44 +0200 Subject: [PATCH 0132/1927] drm/amdgpu: set vm_update_mode=0 as default for Sienna Cichlid in SRIOV case For asic with VF MMIO access protection avoid using CPU for VM table updates. CPU pagetable updates have issues with HDP flush as VF MMIO access protection blocks write to mmBIF_BX_DEV0_EPF0_VF0_HDP_MEM_COHERENCY_FLUSH_CNTL register during sriov runtime. v3: introduce virtualization capability flag AMDGPU_VF_MMIO_ACCESS_PROTECT which indicates that VF MMIO write access is not allowed in sriov runtime Signed-off-by: Danijel Slivka Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 6 ++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 4 ++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 6 +++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index e4af40b9a8aac..9c765b04aae3b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -726,6 +726,12 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev) adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE; } + if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) + /* VF MMIO access (except mailbox range) from CPU + * will be blocked during sriov runtime + */ + adev->virt.caps |= AMDGPU_VF_MMIO_ACCESS_PROTECT; + /* we have the ability to check now */ if (amdgpu_sriov_vf(adev)) { switch (adev->asic_type) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h index d94c31e68a147..49c4347d154ce 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h @@ -31,6 +31,7 @@ #define AMDGPU_SRIOV_CAPS_IS_VF (1 << 2) /* this GPU is a virtual function */ #define AMDGPU_PASSTHROUGH_MODE (1 << 3) /* thw whole GPU is pass through for VM */ #define AMDGPU_SRIOV_CAPS_RUNTIME (1 << 4) /* is out of full access mode */ +#define AMDGPU_VF_MMIO_ACCESS_PROTECT (1 << 5) /* MMIO write access is not allowed in sriov runtime */ /* flags for indirect register access path supported by rlcg for sriov */ #define AMDGPU_RLCG_GC_WRITE_LEGACY (0x8 << 28) @@ -297,6 +298,9 @@ struct amdgpu_video_codec_info; #define amdgpu_passthrough(adev) \ ((adev)->virt.caps & AMDGPU_PASSTHROUGH_MODE) +#define amdgpu_sriov_vf_mmio_access_protection(adev) \ +((adev)->virt.caps & AMDGPU_VF_MMIO_ACCESS_PROTECT) + static inline bool is_virtual_machine(void) { #if defined(CONFIG_X86) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 83b0c5d86e480..2291aa14d888c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -2338,7 +2338,11 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) */ #ifdef CONFIG_X86_64 if (amdgpu_vm_update_mode == -1) { - if (amdgpu_gmc_vram_full_visible(&adev->gmc)) + /* For asic with VF MMIO access protection + * avoid using CPU for VM table updates + */ + if (amdgpu_gmc_vram_full_visible(&adev->gmc) && + !amdgpu_sriov_vf_mmio_access_protection(adev)) adev->vm_manager.vm_update_mode = AMDGPU_VM_USE_CPU_FOR_COMPUTE; else -- GitLab From 6aa5893926371ebc6c2b6ca6ad37b2aec3a11d29 Mon Sep 17 00:00:00 2001 From: Victor Zhao Date: Thu, 13 Oct 2022 10:42:04 +0800 Subject: [PATCH 0133/1927] Revert "drm/amdgpu: add debugfs amdgpu_reset_level" This reverts commit 5bd8d53f6fa53eab5433698d1362dae2aa53c1cc. This commit breaks the reset logic for aldebaran, revert it for now. Will move the mask inside the reset handler. Fixes: 5bd8d53f6fa53e ("drm/amdgpu: add debugfs amdgpu_reset_level") Signed-off-by: Victor Zhao Reviewed-by: Lijo Lazar Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 4 ---- drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 2 -- drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 8 -------- drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 3 --- 4 files changed, 17 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 3ce91f660c3f3..0e6ddf05c23c9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -274,9 +274,6 @@ extern int amdgpu_vcnfw_log; #define AMDGPU_RESET_VCE (1 << 13) #define AMDGPU_RESET_VCE1 (1 << 14) -#define AMDGPU_RESET_LEVEL_SOFT_RECOVERY (1 << 0) -#define AMDGPU_RESET_LEVEL_MODE2 (1 << 1) - /* max cursor sizes (in pixels) */ #define CIK_CURSOR_WIDTH 128 #define CIK_CURSOR_HEIGHT 128 @@ -1065,7 +1062,6 @@ struct amdgpu_device { struct work_struct reset_work; - uint32_t amdgpu_reset_level_mask; bool job_hang; }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index 6066aebf491cf..de61a85c4b022 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -1954,8 +1954,6 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev) return PTR_ERR(ent); } - debugfs_create_u32("amdgpu_reset_level", 0600, root, &adev->amdgpu_reset_level_mask); - /* Register debugfs entries for amdgpu_ttm */ amdgpu_ttm_debugfs_init(adev); amdgpu_debugfs_pm_init(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c index 9da5ead50c900..831fb222139c6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c @@ -37,8 +37,6 @@ int amdgpu_reset_init(struct amdgpu_device *adev) { int ret = 0; - adev->amdgpu_reset_level_mask = 0x1; - switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 2): ret = aldebaran_reset_init(adev); @@ -76,9 +74,6 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, { struct amdgpu_reset_handler *reset_handler = NULL; - if (!(adev->amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_MODE2)) - return -ENOSYS; - if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags)) return -ENOSYS; @@ -98,9 +93,6 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev, int ret; struct amdgpu_reset_handler *reset_handler = NULL; - if (!(adev->amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_MODE2)) - return -ENOSYS; - if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags)) return -ENOSYS; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 3e316b013fd95..d3558c34d406c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -405,9 +405,6 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid, { ktime_t deadline = ktime_add_us(ktime_get(), 10000); - if (!(ring->adev->amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_SOFT_RECOVERY)) - return false; - if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence) return false; -- GitLab From b98a1648d6616d288e888c6dc6dcd4fa543585b3 Mon Sep 17 00:00:00 2001 From: Victor Zhao Date: Thu, 13 Oct 2022 11:06:33 +0800 Subject: [PATCH 0134/1927] Revert "drm/amdgpu: let mode2 reset fallback to default when failure" This reverts commit dac6b80818ac2353631c5a33d140d8d5508e2957. This commit reverted the AMDGPU_SKIP_MODE2_RESET as it conflicts with the original design of reset handler. Will redesign it. Fixes: dac6b80818ac23 ("drm/amdgpu: let mode2 reset fallback to default when failure") Signed-off-by: Victor Zhao Reviewed-by: Lijo Lazar Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +------ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 1 - drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c | 6 ------ drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h | 3 +-- drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c | 1 - drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c | 1 - drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c | 1 - 9 files changed, 2 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 03bbfaa51cbcb..0561812aa0a43 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -134,7 +134,6 @@ static void amdgpu_amdkfd_reset_work(struct work_struct *work) reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); amdgpu_device_gpu_recover(adev, NULL, &reset_context); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index ab8f970b28491..bb73fb420ffcd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -5210,7 +5210,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, reset_context->job = job; reset_context->hive = hive; - /* * Build list of devices to reset. * In case we are in XGMI hive mode, resort the device list @@ -5337,11 +5336,8 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, amdgpu_ras_resume(adev); } else { r = amdgpu_do_asic_reset(device_list_handle, reset_context); - if (r && r == -EAGAIN) { - set_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags); - adev->asic_reset_res = 0; + if (r && r == -EAGAIN) goto retry; - } if (!r && gpu_reset_for_dev_remove) goto recover_end; @@ -5777,7 +5773,6 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) reset_context.reset_req_dev = adev; set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); - set_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); adev->no_hw_access = true; r = amdgpu_device_pre_asic_reset(adev, &reset_context); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index 46c99331d7f12..cd968e781077e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -72,7 +72,6 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); r = amdgpu_device_gpu_recover(ring->adev, job, &reset_context); if (r) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 2dad7aa9a03b9..75f1402101f4c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -1950,7 +1950,6 @@ static void amdgpu_ras_do_recovery(struct work_struct *work) reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c index 831fb222139c6..f778466bb9dbd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c @@ -74,9 +74,6 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev, { struct amdgpu_reset_handler *reset_handler = NULL; - if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags)) - return -ENOSYS; - if (adev->reset_cntl && adev->reset_cntl->get_reset_handler) reset_handler = adev->reset_cntl->get_reset_handler( adev->reset_cntl, reset_context); @@ -93,9 +90,6 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev, int ret; struct amdgpu_reset_handler *reset_handler = NULL; - if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags)) - return -ENOSYS; - if (adev->reset_cntl) reset_handler = adev->reset_cntl->get_reset_handler( adev->reset_cntl, reset_context); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h index f5318fedf2f04..f4a501ff87d90 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h @@ -30,8 +30,7 @@ enum AMDGPU_RESET_FLAGS { AMDGPU_NEED_FULL_RESET = 0, AMDGPU_SKIP_HW_RESET = 1, - AMDGPU_SKIP_MODE2_RESET = 2, - AMDGPU_RESET_FOR_DEVICE_REMOVE = 3, + AMDGPU_RESET_FOR_DEVICE_REMOVE = 2, }; struct amdgpu_reset_context { diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c index a2f04b2491329..12906ba74462f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c @@ -290,7 +290,6 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work) reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); amdgpu_device_gpu_recover(adev, NULL, &reset_context); } diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c index a977f0027928d..e07757eea7adf 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c @@ -317,7 +317,6 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work) reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); amdgpu_device_gpu_recover(adev, NULL, &reset_context); } diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c index fd14fa9b9cd7c..288c414babdfa 100644 --- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c +++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c @@ -529,7 +529,6 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work) reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); - clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags); amdgpu_device_gpu_recover(adev, NULL, &reset_context); } -- GitLab From 16e311612456df01308585b89227854b1f495041 Mon Sep 17 00:00:00 2001 From: Victor Zhao Date: Thu, 13 Oct 2022 15:53:19 +0800 Subject: [PATCH 0135/1927] drm/amdgpu: Refactor mode2 reset logic for v11.0.7 - refactor mode2 on v11.0.7 to align with aldebaran - comment out using mode2 reset as default for now, will introduce another controller to replace previous reset_level_mask v2: squash in unused variable removal (Alex) Signed-off-by: Victor Zhao Reviewed-by: Lijo Lazar Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c | 25 ++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c index 7aa570c1ce4a9..81a6d5b94987f 100644 --- a/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c +++ b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c @@ -31,12 +31,23 @@ #include "amdgpu_psp.h" #include "amdgpu_xgmi.h" +static bool sienna_cichlid_is_mode2_default(struct amdgpu_reset_control *reset_ctl) +{ +#if 0 + struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; + + if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(11, 0, 7) && + adev->pm.fw_version >= 0x3a5500 && !amdgpu_sriov_vf(adev)) + return true; +#endif + return false; +} + static struct amdgpu_reset_handler * sienna_cichlid_get_reset_handler(struct amdgpu_reset_control *reset_ctl, struct amdgpu_reset_context *reset_context) { struct amdgpu_reset_handler *handler; - struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle; if (reset_context->method != AMD_RESET_METHOD_NONE) { list_for_each_entry(handler, &reset_ctl->reset_handlers, @@ -44,15 +55,13 @@ sienna_cichlid_get_reset_handler(struct amdgpu_reset_control *reset_ctl, if (handler->reset_method == reset_context->method) return handler; } - } else { - list_for_each_entry(handler, &reset_ctl->reset_handlers, + } + + if (sienna_cichlid_is_mode2_default(reset_ctl)) { + list_for_each_entry (handler, &reset_ctl->reset_handlers, handler_list) { - if (handler->reset_method == AMD_RESET_METHOD_MODE2 && - adev->pm.fw_version >= 0x3a5500 && - !amdgpu_sriov_vf(adev)) { - reset_context->method = AMD_RESET_METHOD_MODE2; + if (handler->reset_method == AMD_RESET_METHOD_MODE2) return handler; - } } } -- GitLab From e5b781c56d46c44c52caa915f1b65064f2f7c1ba Mon Sep 17 00:00:00 2001 From: Asher Song Date: Fri, 14 Oct 2022 11:36:33 +0800 Subject: [PATCH 0136/1927] drm/amdgpu: Revert "drm/amdgpu: getting fan speed pwm for vega10 properly" This reverts commit 16fb4dca95daa9d8e037201166a58de8284f4268. Unfortunately, that commit causes fan monitors can't be read and written properly. Fixes: 16fb4dca95daa9 ("drm/amdgpu: getting fan speed pwm for vega10 properly") Signed-off-by: Asher Song Reviewed-by: Guchun Chen Signed-off-by: Alex Deucher --- .../amd/pm/powerplay/hwmgr/vega10_thermal.c | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c index 190af79f3236f..dad3e3741a4e8 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c @@ -67,21 +67,22 @@ int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { - struct amdgpu_device *adev = hwmgr->adev; - uint32_t duty100, duty; - uint64_t tmp64; + uint32_t current_rpm; + uint32_t percent = 0; - duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), - CG_FDO_CTRL1, FMAX_DUTY100); - duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS), - CG_THERMAL_STATUS, FDO_PWM_DUTY); + if (hwmgr->thermal_controller.fanInfo.bNoFan) + return 0; - if (!duty100) - return -EINVAL; + if (vega10_get_current_rpm(hwmgr, ¤t_rpm)) + return -1; + + if (hwmgr->thermal_controller. + advanceFanControlParameters.usMaxFanRPM != 0) + percent = current_rpm * 255 / + hwmgr->thermal_controller. + advanceFanControlParameters.usMaxFanRPM; - tmp64 = (uint64_t)duty * 255; - do_div(tmp64, duty100); - *speed = MIN((uint32_t)tmp64, 255); + *speed = MIN(percent, 255); return 0; } -- GitLab From 8a7b97672dfdb97691dfb15e350b38339f7c69f3 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Sat, 30 Jul 2022 10:58:37 +0800 Subject: [PATCH 0137/1927] drm/amd/pm: temporarily disable thermal alert on smu_v13_0_10 temporarily disable thermal alert on smu_v13_0_10 due to kfd test fail. will enable it again after confirming the thermal hardware setting. Signed-off-by: Kenneth Feng Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 13c5c7f1ecb9f..3d436e7f6e952 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1312,10 +1312,12 @@ static int smu_smc_hw_setup(struct smu_context *smu) return ret; } - ret = smu_enable_thermal_alert(smu); - if (ret) { - dev_err(adev->dev, "Failed to enable thermal alert!\n"); - return ret; + if (adev->ip_versions[MP1_HWIP][0] != IP_VERSION(13, 0, 10)) { + ret = smu_enable_thermal_alert(smu); + if (ret) { + dev_err(adev->dev, "Failed to enable thermal alert!\n"); + return ret; + } } ret = smu_notify_display_change(smu); -- GitLab From e77422a5d4518109511334e6c2274422fa1559c8 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Wed, 7 Sep 2022 15:40:34 +0800 Subject: [PATCH 0138/1927] drm/amd/pm: remove the pptable id override on smu_v13_0_10 remove the pptable id override on smu_v13_0_10, and the id is fetched from vbios now. Signed-off-by: Kenneth Feng Reviewed-by: Likun Gao Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index 93fffdbab4f07..d9323293179a0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -454,9 +454,6 @@ int smu_v13_0_setup_pptable(struct smu_context *smu) dev_info(adev->dev, "override pptable id %d\n", pptable_id); } else { pptable_id = smu->smu_table.boot_values.pp_table_id; - - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 10)) - pptable_id = 6666; } /* force using vbios pptable in sriov mode */ -- GitLab From 4ecdb30ec49d76ccb803c9b9c1464fcf27d6d041 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Wed, 7 Sep 2022 17:05:34 +0800 Subject: [PATCH 0139/1927] drm/amd/amdgpu: enable gfx clock gating features on smu_v13_0_10 enable gfx clock gating features on smu_v13_0_10 Signed-off-by: Kenneth Feng Reviewed-by: Jack Gui Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 1 + drivers/gpu/drm/amd/amdgpu/soc21.c | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 257b2e4de6007..73106f53246d5 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -5076,6 +5076,7 @@ static int gfx_v11_0_set_clockgating_state(void *handle, case IP_VERSION(11, 0, 0): case IP_VERSION(11, 0, 1): case IP_VERSION(11, 0, 2): + case IP_VERSION(11, 0, 3): gfx_v11_0_update_gfx_clock_gating(adev, state == AMD_CG_STATE_GATE); break; diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index 795706b3b092f..fdd842a3fcb6d 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -636,7 +636,11 @@ static int soc21_common_early_init(void *handle) break; case IP_VERSION(11, 0, 3): adev->cg_flags = AMD_CG_SUPPORT_VCN_MGCG | - AMD_CG_SUPPORT_JPEG_MGCG; + AMD_CG_SUPPORT_JPEG_MGCG | + AMD_CG_SUPPORT_GFX_CGCG | + AMD_CG_SUPPORT_GFX_CGLS | + AMD_CG_SUPPORT_REPEATER_FGCG | + AMD_CG_SUPPORT_GFX_MGCG; adev->pg_flags = AMD_PG_SUPPORT_VCN | AMD_PG_SUPPORT_VCN_DPG | AMD_PG_SUPPORT_JPEG; -- GitLab From 7faf684b15f5533142ebeed5ec09d5ea9d9239c4 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Mon, 26 Sep 2022 17:15:04 +0800 Subject: [PATCH 0140/1927] drm/amd/pm: skip loading pptable from driver on secure board for smu_v13_0_10 skip loading pptable from driver on secure board since it's loaded from psp. Signed-off-by: Kenneth Feng Reviewed-by: Guan Yu Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index d9323293179a0..c4552ade8d441 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -211,7 +211,8 @@ int smu_v13_0_init_pptable_microcode(struct smu_context *smu) return 0; if ((adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 7)) || - (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 0))) + (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 0)) || + (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 10))) return 0; /* override pptable_id from driver parameter */ -- GitLab From bbce8cdb8390c4cae8ebe99f13a82c846995e8d9 Mon Sep 17 00:00:00 2001 From: Likun Gao Date: Mon, 25 Jul 2022 20:02:40 +0800 Subject: [PATCH 0141/1927] drm/amdgpu: skip mes self test for gc 11.0.3 Temporary disable mes self teset for gc 11.0.3. Signed-off-by: Likun Gao Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index 5cec6b259b7f7..133804e6018a7 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -1296,7 +1296,8 @@ static int mes_v11_0_late_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; - if (!amdgpu_in_reset(adev)) + if (!amdgpu_in_reset(adev) && + (adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))) amdgpu_mes_self_test(adev); return 0; -- GitLab From 2e26bf1e461fb934cbd5e34142068705ceba1fc1 Mon Sep 17 00:00:00 2001 From: YiPeng Chai Date: Tue, 27 Sep 2022 13:16:27 +0800 Subject: [PATCH 0142/1927] drm/amdgpu: Enable gmc soft reset on gmc_v11_0_3 Enable gmc soft reset on gmc_v11_0_3. Signed-off-by: YiPeng Chai Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/soc21.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index fdd842a3fcb6d..e08044008186e 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -423,6 +423,7 @@ static bool soc21_need_full_reset(struct amdgpu_device *adev) case IP_VERSION(11, 0, 0): return amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC); case IP_VERSION(11, 0, 2): + case IP_VERSION(11, 0, 3): return false; default: return true; -- GitLab From 073285efde229ae82d3b853c7f4bcca81f97a55f Mon Sep 17 00:00:00 2001 From: YiPeng Chai Date: Tue, 27 Sep 2022 14:06:42 +0800 Subject: [PATCH 0143/1927] drm/amdgpu: Enable ras support for mp0 v13_0_0 and v13_0_10 V1: Enable ras support for CHIP_IP_DISCOVERY asic type. V2: 1. Change commit comment. 2. Enable ras support for mp0 v13_0_0 and v13_0_10. Signed-off-by: YiPeng Chai Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 75f1402101f4c..4a8f73cc4cb5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -2267,6 +2267,16 @@ static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) { + if (adev->asic_type == CHIP_IP_DISCOVERY) { + switch (adev->ip_versions[MP0_HWIP][0]) { + case IP_VERSION(13, 0, 0): + case IP_VERSION(13, 0, 10): + return true; + default: + return false; + } + } + return adev->asic_type == CHIP_VEGA10 || adev->asic_type == CHIP_VEGA20 || adev->asic_type == CHIP_ARCTURUS || -- GitLab From 82835055c62fa49b50dc00736743e8f99ed93638 Mon Sep 17 00:00:00 2001 From: YiPeng Chai Date: Wed, 28 Sep 2022 15:52:02 +0800 Subject: [PATCH 0144/1927] drm/amdgpu: Add sriov vf ras support in amdgpu_ras_asic_supported V2: Add sriov vf ras support in amdgpu_ras_asic_supported. Signed-off-by: YiPeng Chai Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 4a8f73cc4cb5c..a4b47e1bd111d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -2267,6 +2267,15 @@ static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) { + if (amdgpu_sriov_vf(adev)) { + switch (adev->ip_versions[MP0_HWIP][0]) { + case IP_VERSION(13, 0, 2): + return true; + default: + return false; + } + } + if (adev->asic_type == CHIP_IP_DISCOVERY) { switch (adev->ip_versions[MP0_HWIP][0]) { case IP_VERSION(13, 0, 0): @@ -2320,11 +2329,6 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev) !amdgpu_ras_asic_supported(adev)) return; - /* If driver run on sriov guest side, only enable ras for aldebaran */ - if (amdgpu_sriov_vf(adev) && - adev->ip_versions[MP1_HWIP][0] != IP_VERSION(13, 0, 2)) - return; - if (!adev->gmc.xgmi.connected_to_cpu) { if (amdgpu_atomfirmware_mem_ecc_supported(adev)) { dev_info(adev->dev, "MEM ECC is active.\n"); -- GitLab From c6863be231791fa80dca203b9295780488671195 Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Thu, 29 Sep 2022 10:24:51 +0800 Subject: [PATCH 0145/1927] drm/amd/pm: fulfill SMU13.0.0 cstate control interface Fulfill the functionality for cstate control. Signed-off-by: Evan Quan Reviewed-by: Hawking Zhang Reviewed-by: Lijo Lazar Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 1d454485e0d91..29529328152d0 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -119,6 +119,7 @@ static struct cmn2asic_msg_mapping smu_v13_0_0_message_map[SMU_MSG_MAX_COUNT] = MSG_MAP(NotifyPowerSource, PPSMC_MSG_NotifyPowerSource, 0), MSG_MAP(Mode1Reset, PPSMC_MSG_Mode1Reset, 0), MSG_MAP(PrepareMp1ForUnload, PPSMC_MSG_PrepareMp1ForUnload, 0), + MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), }; static struct cmn2asic_mapping smu_v13_0_0_clk_map[SMU_CLK_COUNT] = { @@ -1753,6 +1754,15 @@ static int smu_v13_0_0_set_mp1_state(struct smu_context *smu, return ret; } +static int smu_v13_0_0_set_df_cstate(struct smu_context *smu, + enum pp_df_cstate state) +{ + return smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_DFCstateControl, + state, + NULL); +} + static const struct pptable_funcs smu_v13_0_0_ppt_funcs = { .get_allowed_feature_mask = smu_v13_0_0_get_allowed_feature_mask, .set_default_dpm_table = smu_v13_0_0_set_default_dpm_table, @@ -1822,6 +1832,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = { .mode1_reset_is_support = smu_v13_0_0_is_mode1_reset_supported, .mode1_reset = smu_v13_0_mode1_reset, .set_mp1_state = smu_v13_0_0_set_mp1_state, + .set_df_cstate = smu_v13_0_0_set_df_cstate, }; void smu_v13_0_0_set_ppt_funcs(struct smu_context *smu) -- GitLab From 3cf377ee8df7dc4ae5e543c37833ae5a5b2a78d3 Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Thu, 29 Sep 2022 10:30:01 +0800 Subject: [PATCH 0146/1927] drm/amd/pm: fulfill SMU13.0.7 cstate control interface Fulfill the functionality for cstate control. Signed-off-by: Evan Quan Reviewed-by: Hawking Zhang Reviewed-by: Lijo Lazar Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index c422bf8a09b1d..c4102cfb734c2 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -121,6 +121,7 @@ static struct cmn2asic_msg_mapping smu_v13_0_7_message_map[SMU_MSG_MAX_COUNT] = MSG_MAP(Mode1Reset, PPSMC_MSG_Mode1Reset, 0), MSG_MAP(PrepareMp1ForUnload, PPSMC_MSG_PrepareMp1ForUnload, 0), MSG_MAP(SetMGpuFanBoostLimitRpm, PPSMC_MSG_SetMGpuFanBoostLimitRpm, 0), + MSG_MAP(DFCstateControl, PPSMC_MSG_SetExternalClientDfCstateAllow, 0), }; static struct cmn2asic_mapping smu_v13_0_7_clk_map[SMU_CLK_COUNT] = { @@ -1587,6 +1588,16 @@ static bool smu_v13_0_7_is_mode1_reset_supported(struct smu_context *smu) return true; } + +static int smu_v13_0_7_set_df_cstate(struct smu_context *smu, + enum pp_df_cstate state) +{ + return smu_cmn_send_smc_msg_with_param(smu, + SMU_MSG_DFCstateControl, + state, + NULL); +} + static const struct pptable_funcs smu_v13_0_7_ppt_funcs = { .get_allowed_feature_mask = smu_v13_0_7_get_allowed_feature_mask, .set_default_dpm_table = smu_v13_0_7_set_default_dpm_table, @@ -1649,6 +1660,7 @@ static const struct pptable_funcs smu_v13_0_7_ppt_funcs = { .mode1_reset_is_support = smu_v13_0_7_is_mode1_reset_supported, .mode1_reset = smu_v13_0_mode1_reset, .set_mp1_state = smu_v13_0_7_set_mp1_state, + .set_df_cstate = smu_v13_0_7_set_df_cstate, }; void smu_v13_0_7_set_ppt_funcs(struct smu_context *smu) -- GitLab From b31d6ada8346574ce04656e5ce9676ec763f5144 Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Thu, 29 Sep 2022 10:50:44 +0800 Subject: [PATCH 0147/1927] drm/amd/pm: disable cstate feature for gpu reset scenario Suggested by PMFW team and same as what did for gfxoff feature. This can address some Mode1Reset failures observed on SMU13.0.0. Signed-off-by: Evan Quan Reviewed-by: Hawking Zhang Reviewed-by: Lijo Lazar Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 8 ++++++++ drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c | 8 ++++++++ drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c | 9 +++++++++ 3 files changed, 25 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index bb73fb420ffcd..e0445e8cc3424 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2928,6 +2928,14 @@ static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); + /* + * Per PMFW team's suggestion, driver needs to handle gfxoff + * and df cstate features disablement for gpu reset(e.g. Mode1Reset) + * scenario. Add the missing df cstate disablement here. + */ + if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) + dev_warn(adev->dev, "Failed to disallow df cstate"); + for (i = adev->num_ip_blocks - 1; i >= 0; i--) { if (!adev->ip_blocks[i].status.valid) continue; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index 445005571f76f..9cd005131f566 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -2242,9 +2242,17 @@ static void arcturus_get_unique_id(struct smu_context *smu) static int arcturus_set_df_cstate(struct smu_context *smu, enum pp_df_cstate state) { + struct amdgpu_device *adev = smu->adev; uint32_t smu_version; int ret; + /* + * Arcturus does not need the cstate disablement + * prerequisite for gpu reset. + */ + if (amdgpu_in_reset(adev) || adev->in_suspend) + return 0; + ret = smu_cmn_get_smc_version(smu, NULL, &smu_version); if (ret) { dev_err(smu->adev->dev, "Failed to get smu version!\n"); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c index 619aee51b1238..d30ec3005ea19 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c @@ -1640,6 +1640,15 @@ static bool aldebaran_is_baco_supported(struct smu_context *smu) static int aldebaran_set_df_cstate(struct smu_context *smu, enum pp_df_cstate state) { + struct amdgpu_device *adev = smu->adev; + + /* + * Aldebaran does not need the cstate disablement + * prerequisite for gpu reset. + */ + if (amdgpu_in_reset(adev) || adev->in_suspend) + return 0; + return smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_DFCstateControl, state, NULL); } -- GitLab From 5af392a89bd009aced92e9079589bd82f249010a Mon Sep 17 00:00:00 2001 From: ZhenGuo Yin Date: Wed, 12 Oct 2022 16:54:38 +0800 Subject: [PATCH 0148/1927] drm/amd/pm: Init pm_attr_list when dpm is disabled [Why] In SRIOV multi-vf, dpm is always disabled, and pm_attr_list won't be initialized. There will be a NULL pointer call trace after removing the dpm check condition in amdgpu_pm_sysfs_fini. BUG: kernel NULL pointer dereference, address: 0000000000000000 RIP: 0010:amdgpu_device_attr_remove_groups+0x20/0x90 [amdgpu] Call Trace: amdgpu_pm_sysfs_fini+0x2f/0x40 [amdgpu] amdgpu_device_fini_hw+0xdf/0x290 [amdgpu] [How] List pm_attr_list should be initialized when dpm is disabled. Fixes: a6ad27cec585fe ("drm/amd/pm: Remove redundant check condition") Signed-off-by: ZhenGuo Yin Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 948cc75376f8b..236657eece477 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -3362,11 +3362,11 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev) if (adev->pm.sysfs_initialized) return 0; + INIT_LIST_HEAD(&adev->pm.pm_attr_list); + if (adev->pm.dpm_enabled == 0) return 0; - INIT_LIST_HEAD(&adev->pm.pm_attr_list); - adev->pm.int_hwmon_dev = hwmon_device_register_with_groups(adev->dev, DRIVER_NAME, adev, hwmon_groups); -- GitLab From d1bb3afc0527ab55d118852b398fd0f1d2fe802d Mon Sep 17 00:00:00 2001 From: Tim Huang Date: Thu, 29 Sep 2022 14:39:21 +0800 Subject: [PATCH 0149/1927] drm/amd/pm: update SMU IP v13.0.4 driver interface version Update the SMU driver interface version to V7. Signed-off-by: Tim Huang Reviewed-by: Mario Limonciello Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- .../swsmu/inc/pmfw_if/smu13_driver_if_v13_0_4.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_4.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_4.h index ae2d337158f3b..f77401709d83c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_4.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_4.h @@ -27,7 +27,7 @@ // *** IMPORTANT *** // SMU TEAM: Always increment the interface version if // any structure is changed in this file -#define PMFW_DRIVER_IF_VERSION 5 +#define PMFW_DRIVER_IF_VERSION 7 typedef struct { int32_t value; @@ -163,8 +163,8 @@ typedef struct { uint16_t DclkFrequency; //[MHz] uint16_t MemclkFrequency; //[MHz] uint16_t spare; //[centi] - uint16_t UvdActivity; //[centi] uint16_t GfxActivity; //[centi] + uint16_t UvdActivity; //[centi] uint16_t Voltage[2]; //[mV] indices: VDDCR_VDD, VDDCR_SOC uint16_t Current[2]; //[mA] indices: VDDCR_VDD, VDDCR_SOC @@ -199,6 +199,19 @@ typedef struct { uint16_t DeviceState; uint16_t CurTemp; //[centi-Celsius] uint16_t spare2; + + uint16_t AverageGfxclkFrequency; + uint16_t AverageFclkFrequency; + uint16_t AverageGfxActivity; + uint16_t AverageSocclkFrequency; + uint16_t AverageVclkFrequency; + uint16_t AverageVcnActivity; + uint16_t AverageDRAMReads; //Filtered DF Bandwidth::DRAM Reads + uint16_t AverageDRAMWrites; //Filtered DF Bandwidth::DRAM Writes + uint16_t AverageSocketPower; //Filtered value of CurrentSocketPower + uint16_t AverageCorePower; //Filtered of [sum of CorePower[8]]) + uint16_t AverageCoreC0Residency[8]; //Filtered of [average C0 residency % per core] + uint32_t MetricsCounter; //Counts the # of metrics table parameter reads per update to the metrics table, i.e. if the metrics table update happens every 1 second, this value could be up to 1000 if the smu collected metrics data every cycle, or as low as 0 if the smu was asleep the whole time. Reset to 0 after writing. } SmuMetrics_t; typedef struct { -- GitLab From 027bf0cee89a27325a9a4f2240c21dd5fb81e4fa Mon Sep 17 00:00:00 2001 From: Tim Huang Date: Thu, 29 Sep 2022 15:06:47 +0800 Subject: [PATCH 0150/1927] drm/amd/pm: add SMU IP v13.0.4 IF version define to V7 The pmfw has changed the driver interface version, so keep same with the fw. Signed-off-by: Tim Huang Reviewed-by: Mario Limonciello Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h index 9d62ea2af132c..8f72202aea8e9 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h @@ -28,7 +28,7 @@ #define SMU13_DRIVER_IF_VERSION_INV 0xFFFFFFFF #define SMU13_DRIVER_IF_VERSION_YELLOW_CARP 0x04 #define SMU13_DRIVER_IF_VERSION_ALDE 0x08 -#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_4 0x05 +#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_4 0x07 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_5 0x04 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0 0x30 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_7 0x2C -- GitLab From 6c0ca748205dc815505c6de79ecf565953390b66 Mon Sep 17 00:00:00 2001 From: Hawking Zhang Date: Fri, 14 Oct 2022 15:17:43 +0800 Subject: [PATCH 0151/1927] drm/amdgpu: move convert_error_address out of umc_ras RAS error address translation algorithm is common across dGPU and A + A platform as along as the SOC integrates the same generation of UMC IP. UMC RAS is managed by x86 MCA on A + A platform, umc_ras in GPU driver is not initialized at all on A + A platform. In such case, any umc_ras callback implemented for dGPU config shouldn't be invoked from A + A specific callback. The change moves convert_error_address out of dGPU umc_ras structure and makes it share between A + A and dGPU config. Signed-off-by: Hawking Zhang Reviewed-by: Stanley Yang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 16 ++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h | 3 --- drivers/gpu/drm/amd/amdgpu/umc_v6_7.c | 7 +++---- drivers/gpu/drm/amd/amdgpu/umc_v6_7.h | 4 +++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index a4b47e1bd111d..21a47f2bb87bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -36,6 +36,7 @@ #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" #include "atom.h" #include "amdgpu_reset.h" +#include "umc_v6_7.h" #ifdef CONFIG_X86_MCE_AMD #include @@ -2899,10 +2900,17 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb, /* * Translate UMC channel address to Physical address */ - if (adev->umc.ras && - adev->umc.ras->convert_ras_error_address) - adev->umc.ras->convert_ras_error_address(adev, - &err_data, m->addr, ch_inst, umc_inst); + switch (adev->ip_versions[UMC_HWIP][0]) { + case IP_VERSION(6, 7, 0): + umc_v6_7_convert_error_address(adev, + &err_data, m->addr, ch_inst, umc_inst); + break; + default: + dev_warn(adev->dev, + "UMC address to Physical address translation is not supported\n"); + kfree(err_data.err_addr); + return NOTIFY_DONE; + } if (amdgpu_bad_page_threshold != 0) { amdgpu_ras_add_bad_pages(adev, err_data.err_addr, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h index e46439274f3a0..3629d8f292ef9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h @@ -51,9 +51,6 @@ struct amdgpu_umc_ras { struct amdgpu_ras_block_object ras_block; void (*err_cnt_init)(struct amdgpu_device *adev); bool (*query_ras_poison_mode)(struct amdgpu_device *adev); - void (*convert_ras_error_address)(struct amdgpu_device *adev, - struct ras_err_data *err_data, uint64_t err_addr, - uint32_t ch_inst, uint32_t umc_inst); void (*ecc_info_query_ras_error_count)(struct amdgpu_device *adev, void *ras_error_status); void (*ecc_info_query_ras_error_address)(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c index 5d5d031c9e7d0..72fd963f178bc 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c @@ -187,9 +187,9 @@ static void umc_v6_7_ecc_info_query_ras_error_count(struct amdgpu_device *adev, } } -static void umc_v6_7_convert_error_address(struct amdgpu_device *adev, - struct ras_err_data *err_data, uint64_t err_addr, - uint32_t ch_inst, uint32_t umc_inst) +void umc_v6_7_convert_error_address(struct amdgpu_device *adev, + struct ras_err_data *err_data, uint64_t err_addr, + uint32_t ch_inst, uint32_t umc_inst) { uint32_t channel_index; uint64_t soc_pa, retired_page, column; @@ -553,5 +553,4 @@ struct amdgpu_umc_ras umc_v6_7_ras = { .query_ras_poison_mode = umc_v6_7_query_ras_poison_mode, .ecc_info_query_ras_error_count = umc_v6_7_ecc_info_query_ras_error_count, .ecc_info_query_ras_error_address = umc_v6_7_ecc_info_query_ras_error_address, - .convert_ras_error_address = umc_v6_7_convert_error_address, }; diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h index fe41ed2f59451..105245d5b6e50 100644 --- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h +++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.h @@ -71,5 +71,7 @@ extern const uint32_t umc_v6_7_channel_idx_tbl_second[UMC_V6_7_UMC_INSTANCE_NUM][UMC_V6_7_CHANNEL_INSTANCE_NUM]; extern const uint32_t umc_v6_7_channel_idx_tbl_first[UMC_V6_7_UMC_INSTANCE_NUM][UMC_V6_7_CHANNEL_INSTANCE_NUM]; - +void umc_v6_7_convert_error_address(struct amdgpu_device *adev, + struct ras_err_data *err_data, uint64_t err_addr, + uint32_t ch_inst, uint32_t umc_inst); #endif -- GitLab From 45950d88709ce1cd77756aec2e78b2b8dfc58894 Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Thu, 13 Oct 2022 11:25:23 -0700 Subject: [PATCH 0152/1927] drm/amd/display: Increase frame size limit for display_mode_vba_util_32.o MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building 32-bit images may fail with the following error. drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dcn32/display_mode_vba_util_32.c: In function ‘dml32_UseMinimumDCFCLK’: drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dcn32/display_mode_vba_util_32.c:3142:1: error: the frame size of 1096 bytes is larger than 1024 bytes This is seen when building i386:allmodconfig with any of the following compilers. gcc (Debian 12.2.0-3) 12.2.0 gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0 The problem is not seen if the compiler supports GCC_PLUGIN_LATENT_ENTROPY because in that case CONFIG_FRAME_WARN is already set to 2048 even for 32-bit builds. dml32_UseMinimumDCFCLK() was introduced with commit dda4fb85e433 ("drm/amd/display: DML changes for DCN32/321"). It declares a large number of local variables. Increase the frame size for the affected file to 2048, similar to other files in the same directory, to enable 32-bit build tests with affected compilers. Fixes: dda4fb85e433 ("drm/amd/display: DML changes for DCN32/321") Cc: Aurabindo Pillai Reported-by: Łukasz Bartosik Signed-off-by: Guenter Roeck Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index d70838edba801..ca7d240006213 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -77,7 +77,7 @@ CFLAGS_$(AMDDALPATH)/dc/dml/dcn30/dcn30_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn32/dcn32_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn32/display_mode_vba_32.o := $(dml_ccflags) $(frame_warn_flag) CFLAGS_$(AMDDALPATH)/dc/dml/dcn32/display_rq_dlg_calc_32.o := $(dml_ccflags) -CFLAGS_$(AMDDALPATH)/dc/dml/dcn32/display_mode_vba_util_32.o := $(dml_ccflags) +CFLAGS_$(AMDDALPATH)/dc/dml/dcn32/display_mode_vba_util_32.o := $(dml_ccflags) $(frame_warn_flag) CFLAGS_$(AMDDALPATH)/dc/dml/dcn321/dcn321_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn31/dcn31_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn301/dcn301_fpu.o := $(dml_ccflags) -- GitLab From cdabbde1ed262090bff45929e1f5c1153ba5003e Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Thu, 13 Oct 2022 23:07:14 +0200 Subject: [PATCH 0153/1927] drm/radeon: Replace kmap() with kmap_local_page() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The use of kmap() is being deprecated in favor of kmap_local_page(). There are two main problems with kmap(): (1) It comes with an overhead as the mapping space is restricted and protected by a global lock for synchronization and (2) it also requires global TLB invalidation when the kmap’s pool wraps and it might block when the mapping space is fully utilized until a slot becomes available. With kmap_local_page() the mappings are per thread, CPU local, can take page faults, and can be called from any context (including interrupts). It is faster than kmap() in kernels with HIGHMEM enabled. Furthermore, the tasks can be preempted and, when they are scheduled to run again, the kernel virtual addresses are restored and still valid. Therefore, replace kmap() with kmap_local_page() in radeon_ttm_gtt_read(). Cc: "Venkataramanan, Anirudh" Reviewed-by: Kees Cook Reviewed-by: Christian König Suggested-by: Ira Weiny Signed-off-by: Fabio M. De Francesco Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_ttm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c index d33fec488713b..bdb4c0e0736ba 100644 --- a/drivers/gpu/drm/radeon/radeon_ttm.c +++ b/drivers/gpu/drm/radeon/radeon_ttm.c @@ -869,11 +869,11 @@ static ssize_t radeon_ttm_gtt_read(struct file *f, char __user *buf, page = rdev->gart.pages[p]; if (page) { - ptr = kmap(page); + ptr = kmap_local_page(page); ptr += off; r = copy_to_user(buf, ptr, cur_size); - kunmap(rdev->gart.pages[p]); + kunmap_local(ptr); } else r = clear_user(buf, cur_size); -- GitLab From a2c554262d39f81be7422fd8bee2f2fe3779f7f5 Mon Sep 17 00:00:00 2001 From: "Fabio M. De Francesco" Date: Sun, 16 Oct 2022 19:41:58 +0200 Subject: [PATCH 0154/1927] drm/amd/amdgpu: Replace kmap() with kmap_local_page() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit kmap() is being deprecated in favor of kmap_local_page(). There are two main problems with kmap(): (1) It comes with an overhead as mapping space is restricted and protected by a global lock for synchronization and (2) it also requires global TLB invalidation when the kmap’s pool wraps and it might block when the mapping space is fully utilized until a slot becomes available. With kmap_local_page() the mappings are per thread, CPU local, can take page faults, and can be called from any context (including interrupts). It is faster than kmap() in kernels with HIGHMEM enabled. Furthermore, the tasks can be preempted and, when they are scheduled to run again, the kernel virtual addresses are restored and are still valid. Since its use in amdgpu/amdgpu_ttm.c is safe, it should be preferred. Therefore, replace kmap() with kmap_local_page() in amdgpu/amdgpu_ttm.c. Suggested-by: Ira Weiny Acked-by: Christian König Signed-off-by: Fabio M. De Francesco Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index dc262d2c2925e..8782fc6bcd2c1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -2292,9 +2292,9 @@ static ssize_t amdgpu_iomem_read(struct file *f, char __user *buf, if (p->mapping != adev->mman.bdev.dev_mapping) return -EPERM; - ptr = kmap(p); + ptr = kmap_local_page(p); r = copy_to_user(buf, ptr + off, bytes); - kunmap(p); + kunmap_local(ptr); if (r) return -EFAULT; @@ -2343,9 +2343,9 @@ static ssize_t amdgpu_iomem_write(struct file *f, const char __user *buf, if (p->mapping != adev->mman.bdev.dev_mapping) return -EPERM; - ptr = kmap(p); + ptr = kmap_local_page(p); r = copy_from_user(ptr + off, buf, bytes); - kunmap(p); + kunmap_local(ptr); if (r) return -EFAULT; -- GitLab From e299b00adf3d4505132e624894f549422ad05eeb Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 17 Oct 2022 09:28:38 -0700 Subject: [PATCH 0155/1927] drm/amdkfd: Fix type of reset_type parameter in hqd_destroy() callback When booting a kernel compiled with CONFIG_CFI_CLANG on a machine with an RX 6700 XT, there is a CFI failure in kfd_destroy_mqd_cp(): [ 12.894543] CFI failure at kfd_destroy_mqd_cp+0x2a/0x40 [amdgpu] (target: hqd_destroy_v10_3+0x0/0x260 [amdgpu]; expected type: 0x8594d794) Clang's kernel Control Flow Integrity (kCFI) makes sure that all indirect call targets have a type that exactly matches the function pointer prototype. In this case, hqd_destroy()'s third parameter, reset_type, should have a type of 'uint32_t' but every implementation of this callback has a third parameter type of 'enum kfd_preempt_type'. Update the function pointer prototype to match reality so that there is no more CFI violation. Link: https://github.com/ClangBuiltLinux/linux/issues/1738 Signed-off-by: Nathan Chancellor Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/include/kgd_kfd_interface.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index e85364dff4e04..5cb3e8634739d 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -262,8 +262,9 @@ struct kfd2kgd_calls { uint32_t queue_id); int (*hqd_destroy)(struct amdgpu_device *adev, void *mqd, - uint32_t reset_type, unsigned int timeout, - uint32_t pipe_id, uint32_t queue_id); + enum kfd_preempt_type reset_type, + unsigned int timeout, uint32_t pipe_id, + uint32_t queue_id); bool (*hqd_sdma_is_occupied)(struct amdgpu_device *adev, void *mqd); -- GitLab From b3372fa74d2a7f840bea706607ee2224dfd24039 Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Fri, 14 Oct 2022 10:29:46 -0400 Subject: [PATCH 0156/1927] drm/amd/display: add an ASSERT() to irq service functions Currently, if we encounter unimplemented functions, it is difficult to tell what caused them just by looking at dmesg and that is compounded by the fact that it is often hard to reproduce said issues, for instance we have had reports of this condition being triggered when removing a secondary display that is setup in mirror mode and is connected using usb-c. So, to have access to more detailed debugging information, add an ASSERT() to dal_irq_service_ack() and dal_irq_service_set() that only triggers when we encounter an unimplemented function. Reviewed-by: Harry Wentland Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/irq/irq_service.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c index 7bad39bba86b6..d100edaedbbb8 100644 --- a/drivers/gpu/drm/amd/display/dc/irq/irq_service.c +++ b/drivers/gpu/drm/amd/display/dc/irq/irq_service.c @@ -112,8 +112,15 @@ bool dal_irq_service_set( dal_irq_service_ack(irq_service, source); - if (info->funcs && info->funcs->set) + if (info->funcs && info->funcs->set) { + if (info->funcs->set == dal_irq_service_dummy_set) { + DC_LOG_WARNING("%s: src: %d, st: %d\n", __func__, + source, enable); + ASSERT(0); + } + return info->funcs->set(irq_service, info, enable); + } dal_irq_service_set_generic(irq_service, info, enable); @@ -146,8 +153,14 @@ bool dal_irq_service_ack( return false; } - if (info->funcs && info->funcs->ack) + if (info->funcs && info->funcs->ack) { + if (info->funcs->ack == dal_irq_service_dummy_ack) { + DC_LOG_WARNING("%s: src: %d\n", __func__, source); + ASSERT(0); + } + return info->funcs->ack(irq_service, info); + } dal_irq_service_ack_generic(irq_service, info); -- GitLab From 8703d55bd5eac642275fe91b34ac62ad0ad312b5 Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Thu, 15 Sep 2022 16:16:01 +0200 Subject: [PATCH 0157/1927] arm64: dts: qcom: sc8280xp: fix UFS PHY serdes size The size of the UFS PHY serdes register region is 0x1c8 and the corresponding 'reg' property should specifically not include the adjacent regions that are defined in the child node (e.g. tx and rx). Fixes: 152d1faf1e2f ("arm64: dts: qcom: add SC8280XP platform") Signed-off-by: Johan Hovold Tested-by: Andrew Halaney #Qdrive3/sa8540p-adp-ride Reviewed-by: Brian Masney Reviewed-by: Bjorn Andersson Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220915141601.18435-1-johan+linaro@kernel.org --- arch/arm64/boot/dts/qcom/sc8280xp.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index 6ed4ab2c08b4b..212d63d5cbf28 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -885,7 +885,7 @@ ufs_mem_hc: ufs@1d84000 { ufs_mem_phy: phy@1d87000 { compatible = "qcom,sc8280xp-qmp-ufs-phy"; - reg = <0 0x01d87000 0 0xe10>; + reg = <0 0x01d87000 0 0x1c8>; #address-cells = <2>; #size-cells = <2>; ranges; @@ -953,7 +953,7 @@ ufs_card_hc: ufs@1da4000 { ufs_card_phy: phy@1da7000 { compatible = "qcom,sc8280xp-qmp-ufs-phy"; - reg = <0 0x01da7000 0 0xe10>; + reg = <0 0x01da7000 0 0x1c8>; #address-cells = <2>; #size-cells = <2>; ranges; -- GitLab From cb1d0aaa674e99957b85af570cb2730145af01df Mon Sep 17 00:00:00 2001 From: Satya Priya Date: Tue, 20 Sep 2022 17:18:15 +0530 Subject: [PATCH 0158/1927] arm64: dts: qcom: sc7280: Add the reset reg for lpass audiocc on SC7280 Add the reset register offset for clock gating. Fixes: 9499240d15f2 ("arm64: dts: qcom: sc7280: Add lpasscore & lpassaudio clock controllers") Signed-off-by: Satya Priya Reviewed-by: Neil Armstrong Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/1663674495-25748-1-git-send-email-quic_c_skakit@quicinc.com --- arch/arm64/boot/dts/qcom/sc7280.dtsi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi index 212580316d3e6..4cdc88d339445 100644 --- a/arch/arm64/boot/dts/qcom/sc7280.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi @@ -2296,7 +2296,8 @@ swr1: soundwire@3230000 { lpass_audiocc: clock-controller@3300000 { compatible = "qcom,sc7280-lpassaudiocc"; - reg = <0 0x03300000 0 0x30000>; + reg = <0 0x03300000 0 0x30000>, + <0 0x032a9000 0 0x1000>; clocks = <&rpmhcc RPMH_CXO_CLK>, <&lpass_aon LPASS_AON_CC_MAIN_RCG_CLK_SRC>; clock-names = "bi_tcxo", "lpass_aon_cc_main_rcg_clk_src"; -- GitLab From c29f446108e10bb791220ffbba54d82722d60d45 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Tue, 20 Sep 2022 17:04:12 +0200 Subject: [PATCH 0159/1927] dt-bindings: hwlock: qcom-hwspinlock: add syscon to MSM8974 The TCSR_MUTEX region contains two set of registers: mutex and halt. Add syscon, so the TCSR mutex device (hwspinlock) can use MMIO based method and in the same time share regmap with other devices for the halt regs. Signed-off-by: Krzysztof Kozlowski Acked-by: Rob Herring Tested-by: Luca Weiss # fairphone-fp2 Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20220920150414.637634-2-krzysztof.kozlowski@linaro.org --- .../devicetree/bindings/hwlock/qcom-hwspinlock.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml b/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml index 1a3adf75934bc..ee2726149cf35 100644 --- a/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml +++ b/Documentation/devicetree/bindings/hwlock/qcom-hwspinlock.yaml @@ -24,9 +24,13 @@ properties: - qcom,apq8084-tcsr-mutex - qcom,ipq6018-tcsr-mutex - qcom,msm8226-tcsr-mutex - - qcom,msm8974-tcsr-mutex - qcom,msm8994-tcsr-mutex - const: qcom,tcsr-mutex + - items: + - enum: + - qcom,msm8974-tcsr-mutex + - const: qcom,tcsr-mutex + - const: syscon reg: maxItems: 1 -- GitLab From 63d1dfd067f07c11eafe05ebadc5896491416f86 Mon Sep 17 00:00:00 2001 From: Jilin Yuan Date: Sun, 8 May 2022 11:02:24 +0800 Subject: [PATCH 0160/1927] ARC: Fix comment typo - Remove one of the repeated 'call' in comment line 396. - Delete the redundant word 'to', 'since' Signed-off-by: Jilin Yuan Signed-off-by: Vineet Gupta --- arch/arc/kernel/smp.c | 2 +- arch/arc/mm/cache.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arc/kernel/smp.c b/arch/arc/kernel/smp.c index ab9e75e90f729..ad93fe6e4b77d 100644 --- a/arch/arc/kernel/smp.c +++ b/arch/arc/kernel/smp.c @@ -385,7 +385,7 @@ irqreturn_t do_IPI(int irq, void *dev_id) * API called by platform code to hookup arch-common ISR to their IPI IRQ * * Note: If IPI is provided by platform (vs. say ARC MCIP), their intc setup/map - * function needs to call call irq_set_percpu_devid() for IPI IRQ, otherwise + * function needs to call irq_set_percpu_devid() for IPI IRQ, otherwise * request_percpu_irq() below will fail */ static DEFINE_PER_CPU(int, ipi_dev); diff --git a/arch/arc/mm/cache.c b/arch/arc/mm/cache.c index 5446967ea98d3..55c6de138eae0 100644 --- a/arch/arc/mm/cache.c +++ b/arch/arc/mm/cache.c @@ -750,7 +750,7 @@ static inline void arc_slc_enable(void) * -In SMP, if hardware caches are coherent * * There's a corollary case, where kernel READs from a userspace mapped page. - * If the U-mapping is not congruent to to K-mapping, former needs flushing. + * If the U-mapping is not congruent to K-mapping, former needs flushing. */ void flush_dcache_page(struct page *page) { @@ -910,7 +910,7 @@ EXPORT_SYMBOL(flush_icache_range); * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) * However in one instance, when called by kprobe (for a breakpt in * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will - * use a paddr to index the cache (despite VIPT). This is fine since since a + * use a paddr to index the cache (despite VIPT). This is fine since a * builtin kernel page will not have any virtual mappings. * kprobe on loadable module will be kernel vaddr. */ -- GitLab From 6e32c89c0f67b481ec17de69e556907d6445f91e Mon Sep 17 00:00:00 2001 From: Zhang Jiaming Date: Thu, 23 Jun 2022 15:46:23 +0800 Subject: [PATCH 0161/1927] ARC: Fix comment typo Change 'seperate' to 'separate'. Signed-off-by: Zhang Jiaming Signed-off-by: Vineet Gupta --- arch/arc/include/asm/entry-compact.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arc/include/asm/entry-compact.h b/arch/arc/include/asm/entry-compact.h index 5aab4f93ab8aa..67ff06e15ceaf 100644 --- a/arch/arc/include/asm/entry-compact.h +++ b/arch/arc/include/asm/entry-compact.h @@ -21,7 +21,7 @@ * r25 contains the kernel current task ptr * - Defined Stack Switching Macro to be reused in all intr/excp hdlrs * - Shaved off 11 instructions from RESTORE_ALL_INT1 by using the - * address Write back load ld.ab instead of seperate ld/add instn + * address Write back load ld.ab instead of separate ld/add instn * * Amit Bhor, Sameer Dhavale: Codito Technologies 2004 */ -- GitLab From a1db7ad3120e787350c83712c6b1087c7894c6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amadeusz=20S=C5=82awi=C5=84ski?= Date: Fri, 27 May 2022 13:53:43 +0200 Subject: [PATCH 0162/1927] ARC: bitops: Change __fls to return unsigned long MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per asm-generic definition and other architectures __fls should return unsigned long. No functional change is expected as return value should fit in unsigned long. Reviewed-by: Cezary Rojewski Signed-off-by: Amadeusz Sławiński Signed-off-by: Vineet Gupta --- arch/arc/include/asm/bitops.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arc/include/asm/bitops.h b/arch/arc/include/asm/bitops.h index bdb7e190a294e..f5a936496f060 100644 --- a/arch/arc/include/asm/bitops.h +++ b/arch/arc/include/asm/bitops.h @@ -82,7 +82,7 @@ static inline __attribute__ ((const)) int fls(unsigned int x) /* * __fls: Similar to fls, but zero based (0-31) */ -static inline __attribute__ ((const)) int __fls(unsigned long x) +static inline __attribute__ ((const)) unsigned long __fls(unsigned long x) { if (!x) return 0; @@ -131,7 +131,7 @@ static inline __attribute__ ((const)) int fls(unsigned int x) /* * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set */ -static inline __attribute__ ((const)) int __fls(unsigned long x) +static inline __attribute__ ((const)) unsigned long __fls(unsigned long x) { /* FLS insn has exactly same semantics as the API */ return __builtin_arc_fls(x); -- GitLab From c8f878582838f57bc0984f47da2a8d275731240f Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Fri, 24 Jun 2022 17:16:17 +0300 Subject: [PATCH 0163/1927] arc: dts: Harmonize EHCI/OHCI DT nodes name In accordance with the Generic EHCI/OHCI bindings the corresponding node name is suppose to comply with the Generic USB HCD DT schema, which requires the USB nodes to have the name acceptable by the regexp: "^usb(@.*)?" . Make sure the "generic-ehci" and "generic-ohci"-compatible nodes are correctly named. Signed-off-by: Serge Semin Acked-by: Alexey Brodkin Acked-by: Krzysztof Kozlowski Signed-off-by: Vineet Gupta --- arch/arc/boot/dts/axc003.dtsi | 4 ++-- arch/arc/boot/dts/axc003_idu.dtsi | 4 ++-- arch/arc/boot/dts/axs10x_mb.dtsi | 4 ++-- arch/arc/boot/dts/hsdk.dts | 4 ++-- arch/arc/boot/dts/vdk_axs10x_mb.dtsi | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arc/boot/dts/axc003.dtsi b/arch/arc/boot/dts/axc003.dtsi index cd1edcf4f95ef..3434c8131ecd5 100644 --- a/arch/arc/boot/dts/axc003.dtsi +++ b/arch/arc/boot/dts/axc003.dtsi @@ -103,11 +103,11 @@ ethernet@18000 { dma-coherent; }; - ehci@40000 { + usb@40000 { dma-coherent; }; - ohci@60000 { + usb@60000 { dma-coherent; }; diff --git a/arch/arc/boot/dts/axc003_idu.dtsi b/arch/arc/boot/dts/axc003_idu.dtsi index 70779386ca796..67556f4b70574 100644 --- a/arch/arc/boot/dts/axc003_idu.dtsi +++ b/arch/arc/boot/dts/axc003_idu.dtsi @@ -110,11 +110,11 @@ ethernet@18000 { dma-coherent; }; - ehci@40000 { + usb@40000 { dma-coherent; }; - ohci@60000 { + usb@60000 { dma-coherent; }; diff --git a/arch/arc/boot/dts/axs10x_mb.dtsi b/arch/arc/boot/dts/axs10x_mb.dtsi index 99d3e7175bf70..b644353853049 100644 --- a/arch/arc/boot/dts/axs10x_mb.dtsi +++ b/arch/arc/boot/dts/axs10x_mb.dtsi @@ -87,13 +87,13 @@ gmac: ethernet@18000 { mac-address = [00 00 00 00 00 00]; /* Filled in by U-Boot */ }; - ehci@40000 { + usb@40000 { compatible = "generic-ehci"; reg = < 0x40000 0x100 >; interrupts = < 8 >; }; - ohci@60000 { + usb@60000 { compatible = "generic-ohci"; reg = < 0x60000 0x100 >; interrupts = < 8 >; diff --git a/arch/arc/boot/dts/hsdk.dts b/arch/arc/boot/dts/hsdk.dts index f48ba03e9b5e7..6691f42550778 100644 --- a/arch/arc/boot/dts/hsdk.dts +++ b/arch/arc/boot/dts/hsdk.dts @@ -234,7 +234,7 @@ phy0: ethernet-phy@0 { /* Micrel KSZ9031 */ }; }; - ohci@60000 { + usb@60000 { compatible = "snps,hsdk-v1.0-ohci", "generic-ohci"; reg = <0x60000 0x100>; interrupts = <15>; @@ -242,7 +242,7 @@ ohci@60000 { dma-coherent; }; - ehci@40000 { + usb@40000 { compatible = "snps,hsdk-v1.0-ehci", "generic-ehci"; reg = <0x40000 0x100>; interrupts = <15>; diff --git a/arch/arc/boot/dts/vdk_axs10x_mb.dtsi b/arch/arc/boot/dts/vdk_axs10x_mb.dtsi index cbb179770293e..90a412026e643 100644 --- a/arch/arc/boot/dts/vdk_axs10x_mb.dtsi +++ b/arch/arc/boot/dts/vdk_axs10x_mb.dtsi @@ -46,7 +46,7 @@ ethernet@18000 { clock-names = "stmmaceth"; }; - ehci@40000 { + usb@40000 { compatible = "generic-ehci"; reg = < 0x40000 0x100 >; interrupts = < 8 >; -- GitLab From c44f15c1c09481d50fd33478ebb5b8284f8f5edb Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sun, 9 Oct 2022 19:28:46 -0700 Subject: [PATCH 0164/1927] arc: iounmap() arg is volatile Add 'volatile' to iounmap()'s argument to prevent build warnings. This make it the same as other major architectures. Placates these warnings: (12 such warnings) ../drivers/video/fbdev/riva/fbdev.c: In function 'rivafb_probe': ../drivers/video/fbdev/riva/fbdev.c:2067:42: error: passing argument 1 of 'iounmap' discards 'volatile' qualifier from pointer target type [-Werror=discarded-qualifiers] 2067 | iounmap(default_par->riva.PRAMIN); Fixes: 1162b0701b14b ("ARC: I/O and DMA Mappings") Signed-off-by: Randy Dunlap Cc: Vineet Gupta Cc: linux-snps-arc@lists.infradead.org Cc: Arnd Bergmann Signed-off-by: Vineet Gupta --- arch/arc/include/asm/io.h | 2 +- arch/arc/mm/ioremap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arc/include/asm/io.h b/arch/arc/include/asm/io.h index 8f777d6441a5d..80347382a3800 100644 --- a/arch/arc/include/asm/io.h +++ b/arch/arc/include/asm/io.h @@ -32,7 +32,7 @@ static inline void ioport_unmap(void __iomem *addr) { } -extern void iounmap(const void __iomem *addr); +extern void iounmap(const volatile void __iomem *addr); /* * io{read,write}{16,32}be() macros diff --git a/arch/arc/mm/ioremap.c b/arch/arc/mm/ioremap.c index 0ee75aca6e109..712c2311daefb 100644 --- a/arch/arc/mm/ioremap.c +++ b/arch/arc/mm/ioremap.c @@ -94,7 +94,7 @@ void __iomem *ioremap_prot(phys_addr_t paddr, unsigned long size, EXPORT_SYMBOL(ioremap_prot); -void iounmap(const void __iomem *addr) +void iounmap(const volatile void __iomem *addr) { /* weird double cast to handle phys_addr_t > 32 bits */ if (arc_uncached_addr_space((phys_addr_t)(u32)addr)) -- GitLab From 2df1f4a77bc0e94e1a0cc7485d09a26855461dd6 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Thu, 29 Sep 2022 12:14:21 +0200 Subject: [PATCH 0165/1927] arc: update config files Clean up config files by: - removing configs that were deleted in the past - removing configs not in tree and without recently pending patches - adding new configs that are replacements for old configs in the file For some detailed information, see Link. Link: https://lore.kernel.org/kernel-janitors/20220929090645.1389-1-lukas.bulwahn@gmail.com/ Signed-off-by: Lukas Bulwahn Signed-off-by: Vineet Gupta --- arch/arc/configs/axs101_defconfig | 4 ---- arch/arc/configs/axs103_defconfig | 4 ---- arch/arc/configs/axs103_smp_defconfig | 4 ---- arch/arc/configs/haps_hs_defconfig | 1 - arch/arc/configs/haps_hs_smp_defconfig | 1 - arch/arc/configs/hsdk_defconfig | 1 - arch/arc/configs/nsim_700_defconfig | 1 - arch/arc/configs/nsimosci_defconfig | 1 - arch/arc/configs/nsimosci_hs_defconfig | 1 - arch/arc/configs/nsimosci_hs_smp_defconfig | 6 +----- arch/arc/configs/tb10x_defconfig | 7 +------ arch/arc/configs/vdk_hs38_defconfig | 3 --- arch/arc/configs/vdk_hs38_smp_defconfig | 1 - 13 files changed, 2 insertions(+), 33 deletions(-) diff --git a/arch/arc/configs/axs101_defconfig b/arch/arc/configs/axs101_defconfig index e31a8ebc3eccb..81764160451f7 100644 --- a/arch/arc/configs/axs101_defconfig +++ b/arch/arc/configs/axs101_defconfig @@ -35,9 +35,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_DEVTMPFS=y # CONFIG_STANDALONE is not set @@ -99,7 +96,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=10 diff --git a/arch/arc/configs/axs103_defconfig b/arch/arc/configs/axs103_defconfig index e0e8567f0d758..d5181275490ed 100644 --- a/arch/arc/configs/axs103_defconfig +++ b/arch/arc/configs/axs103_defconfig @@ -34,9 +34,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_DEVTMPFS=y # CONFIG_STANDALONE is not set @@ -97,7 +94,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=10 diff --git a/arch/arc/configs/axs103_smp_defconfig b/arch/arc/configs/axs103_smp_defconfig index fcbc952bc75bb..2f336d99a8cf3 100644 --- a/arch/arc/configs/axs103_smp_defconfig +++ b/arch/arc/configs/axs103_smp_defconfig @@ -35,9 +35,6 @@ CONFIG_IP_PNP=y CONFIG_IP_PNP_DHCP=y CONFIG_IP_PNP_BOOTP=y CONFIG_IP_PNP_RARP=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set CONFIG_DEVTMPFS=y # CONFIG_STANDALONE is not set @@ -100,7 +97,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=10 diff --git a/arch/arc/configs/haps_hs_defconfig b/arch/arc/configs/haps_hs_defconfig index d87ad7e88d624..899b2fd5c71d1 100644 --- a/arch/arc/configs/haps_hs_defconfig +++ b/arch/arc/configs/haps_hs_defconfig @@ -59,6 +59,5 @@ CONFIG_EXT2_FS_XATTR=y CONFIG_TMPFS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_DEBUG_MEMORY_INIT=y # CONFIG_DEBUG_PREEMPT is not set diff --git a/arch/arc/configs/haps_hs_smp_defconfig b/arch/arc/configs/haps_hs_smp_defconfig index 8d82cdb7f86a6..0d32aac8069f2 100644 --- a/arch/arc/configs/haps_hs_smp_defconfig +++ b/arch/arc/configs/haps_hs_smp_defconfig @@ -59,6 +59,5 @@ CONFIG_EXT2_FS_XATTR=y CONFIG_TMPFS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_SOFTLOCKUP_DETECTOR=y # CONFIG_DEBUG_PREEMPT is not set diff --git a/arch/arc/configs/hsdk_defconfig b/arch/arc/configs/hsdk_defconfig index f856b03e0fb5c..d18378d2c2a63 100644 --- a/arch/arc/configs/hsdk_defconfig +++ b/arch/arc/configs/hsdk_defconfig @@ -85,7 +85,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=10 diff --git a/arch/arc/configs/nsim_700_defconfig b/arch/arc/configs/nsim_700_defconfig index a1ce12bf5b165..3e98297759925 100644 --- a/arch/arc/configs/nsim_700_defconfig +++ b/arch/arc/configs/nsim_700_defconfig @@ -56,5 +56,4 @@ CONFIG_EXT2_FS_XATTR=y CONFIG_TMPFS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y -# CONFIG_ENABLE_MUST_CHECK is not set # CONFIG_DEBUG_PREEMPT is not set diff --git a/arch/arc/configs/nsimosci_defconfig b/arch/arc/configs/nsimosci_defconfig index ca10f4a2c823f..502c87f351c87 100644 --- a/arch/arc/configs/nsimosci_defconfig +++ b/arch/arc/configs/nsimosci_defconfig @@ -65,4 +65,3 @@ CONFIG_TMPFS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y -# CONFIG_ENABLE_MUST_CHECK is not set diff --git a/arch/arc/configs/nsimosci_hs_defconfig b/arch/arc/configs/nsimosci_hs_defconfig index 31b6ec3683c65..f721cc3997d02 100644 --- a/arch/arc/configs/nsimosci_hs_defconfig +++ b/arch/arc/configs/nsimosci_hs_defconfig @@ -63,4 +63,3 @@ CONFIG_TMPFS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y -# CONFIG_ENABLE_MUST_CHECK is not set diff --git a/arch/arc/configs/nsimosci_hs_smp_defconfig b/arch/arc/configs/nsimosci_hs_smp_defconfig index 41a0037f48a58..1419fc946a083 100644 --- a/arch/arc/configs/nsimosci_hs_smp_defconfig +++ b/arch/arc/configs/nsimosci_hs_smp_defconfig @@ -26,9 +26,6 @@ CONFIG_UNIX=y CONFIG_UNIX_DIAG=y CONFIG_NET_KEY=y CONFIG_INET=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y @@ -37,7 +34,6 @@ CONFIG_DEVTMPFS=y # CONFIG_BLK_DEV is not set CONFIG_NETDEVICES=y # CONFIG_NET_VENDOR_ARC is not set -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set CONFIG_EZCHIP_NPS_MANAGEMENT_ENET=y # CONFIG_NET_VENDOR_INTEL is not set @@ -74,5 +70,5 @@ CONFIG_TMPFS=y # CONFIG_MISC_FILESYSTEMS is not set CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_FTRACE=y +# CONFIG_NET_VENDOR_CADENCE is not set diff --git a/arch/arc/configs/tb10x_defconfig b/arch/arc/configs/tb10x_defconfig index 4a94d1684ed60..6f0d2be9d926c 100644 --- a/arch/arc/configs/tb10x_defconfig +++ b/arch/arc/configs/tb10x_defconfig @@ -35,15 +35,11 @@ CONFIG_PACKET=y CONFIG_UNIX=y CONFIG_INET=y CONFIG_IP_MULTICAST=y -# CONFIG_INET_XFRM_MODE_TRANSPORT is not set -# CONFIG_INET_XFRM_MODE_TUNNEL is not set -# CONFIG_INET_XFRM_MODE_BEET is not set # CONFIG_INET_DIAG is not set # CONFIG_IPV6 is not set # CONFIG_WIRELESS is not set CONFIG_DEVTMPFS=y CONFIG_NETDEVICES=y -# CONFIG_NET_CADENCE is not set # CONFIG_NET_VENDOR_BROADCOM is not set # CONFIG_NET_VENDOR_INTEL is not set # CONFIG_NET_VENDOR_MARVELL is not set @@ -94,12 +90,11 @@ CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_FS=y CONFIG_HEADERS_INSTALL=y -CONFIG_HEADERS_CHECK=y CONFIG_DEBUG_SECTION_MISMATCH=y CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_MEMORY_INIT=y CONFIG_DEBUG_STACKOVERFLOW=y CONFIG_DETECT_HUNG_TASK=y CONFIG_SCHEDSTATS=y -CONFIG_TIMER_STATS=y # CONFIG_CRYPTO_HW is not set +# CONFIG_NET_VENDOR_CADENCE is not set diff --git a/arch/arc/configs/vdk_hs38_defconfig b/arch/arc/configs/vdk_hs38_defconfig index 0c3b214168197..d3ef189c75f8b 100644 --- a/arch/arc/configs/vdk_hs38_defconfig +++ b/arch/arc/configs/vdk_hs38_defconfig @@ -58,8 +58,6 @@ CONFIG_SERIAL_OF_PLATFORM=y # CONFIG_HW_RANDOM is not set # CONFIG_HWMON is not set CONFIG_FB=y -CONFIG_ARCPGU_RGB888=y -CONFIG_ARCPGU_DISPTYPE=0 # CONFIG_VGA_CONSOLE is not set CONFIG_FRAMEBUFFER_CONSOLE=y CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y @@ -87,7 +85,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_SHIRQ=y CONFIG_SOFTLOCKUP_DETECTOR=y diff --git a/arch/arc/configs/vdk_hs38_smp_defconfig b/arch/arc/configs/vdk_hs38_smp_defconfig index f9ad9d3ee702d..944b347025fd1 100644 --- a/arch/arc/configs/vdk_hs38_smp_defconfig +++ b/arch/arc/configs/vdk_hs38_smp_defconfig @@ -91,7 +91,6 @@ CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y -# CONFIG_ENABLE_MUST_CHECK is not set CONFIG_STRIP_ASM_SYMS=y CONFIG_DEBUG_SHIRQ=y CONFIG_SOFTLOCKUP_DETECTOR=y -- GitLab From 4fd9df10cb7a9289fbd22d669f9f98164d95a1ce Mon Sep 17 00:00:00 2001 From: Pavel Kozlov Date: Mon, 17 Oct 2022 20:11:27 +0400 Subject: [PATCH 0166/1927] ARC: mm: fix leakage of memory allocated for PTE Since commit d9820ff ("ARC: mm: switch pgtable_t back to struct page *") a memory leakage problem occurs. Memory allocated for page table entries not released during process termination. This issue can be reproduced by a small program that allocates a large amount of memory. After several runs, you'll see that the amount of free memory has reduced and will continue to reduce after each run. All ARC CPUs are effected by this issue. The issue was introduced since the kernel stable release v5.15-rc1. As described in commit d9820ff after switch pgtable_t back to struct page *, a pointer to "struct page" and appropriate functions are used to allocate and free a memory page for PTEs, but the pmd_pgtable macro hasn't changed and returns the direct virtual address from the PMD (PGD) entry. Than this address used as a parameter in the __pte_free() and as a result this function couldn't release memory page allocated for PTEs. Fix this issue by changing the pmd_pgtable macro and returning pointer to struct page. Fixes: d9820ff76f95 ("ARC: mm: switch pgtable_t back to struct page *") Cc: Mike Rapoport Cc: # 5.15.x Signed-off-by: Pavel Kozlov Signed-off-by: Vineet Gupta --- arch/arc/include/asm/pgtable-levels.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arc/include/asm/pgtable-levels.h b/arch/arc/include/asm/pgtable-levels.h index 64ca25d199bea..ef68758b69f7e 100644 --- a/arch/arc/include/asm/pgtable-levels.h +++ b/arch/arc/include/asm/pgtable-levels.h @@ -161,7 +161,7 @@ #define pmd_pfn(pmd) ((pmd_val(pmd) & PAGE_MASK) >> PAGE_SHIFT) #define pmd_page(pmd) virt_to_page(pmd_page_vaddr(pmd)) #define set_pmd(pmdp, pmd) (*(pmdp) = pmd) -#define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd)) +#define pmd_pgtable(pmd) ((pgtable_t) pmd_page(pmd)) /* * 4th level paging: pte -- GitLab From ba5284ebe497044f37c9bb9c7b1564932f4b6610 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 7 Oct 2022 15:10:00 +0200 Subject: [PATCH 0167/1927] clk: renesas: r8a779g0: Add SASYNCPER clocks On R-Car V4H, all PLLs except PLL5 support Spread Spectrum and/or Fractional Multiplication to reduce electromagnetic interference. Add the SASYNCPER and SASYNCPERD[124] clocks, which are used as clock sources for modules that must not be affected by Spread Spectrum and/or Fractional Multiplication. Signed-off-by: Geert Uytterhoeven Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/d0f35c35e1f96c5a649ab477e7ba5d8025957cd0.1665147497.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779g0-cpg-mssr.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/clk/renesas/r8a779g0-cpg-mssr.c b/drivers/clk/renesas/r8a779g0-cpg-mssr.c index 9641122133b54..3e8c93facfa14 100644 --- a/drivers/clk/renesas/r8a779g0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779g0-cpg-mssr.c @@ -47,6 +47,7 @@ enum clk_ids { CLK_S0_VIO, CLK_S0_VC, CLK_S0_HSC, + CLK_SASYNCPER, CLK_SV_VIP, CLK_SV_IR, CLK_SDSRC, @@ -84,6 +85,7 @@ static const struct cpg_core_clk r8a779g0_core_clks[] __initconst = { DEF_FIXED(".s0_vio", CLK_S0_VIO, CLK_PLL1_DIV2, 2, 1), DEF_FIXED(".s0_vc", CLK_S0_VC, CLK_PLL1_DIV2, 2, 1), DEF_FIXED(".s0_hsc", CLK_S0_HSC, CLK_PLL1_DIV2, 2, 1), + DEF_FIXED(".sasyncper", CLK_SASYNCPER, CLK_PLL5_DIV4, 3, 1), DEF_FIXED(".sv_vip", CLK_SV_VIP, CLK_PLL1, 5, 1), DEF_FIXED(".sv_ir", CLK_SV_IR, CLK_PLL1, 5, 1), DEF_BASE(".sdsrc", CLK_SDSRC, CLK_TYPE_GEN4_SDSRC, CLK_PLL5), @@ -128,6 +130,9 @@ static const struct cpg_core_clk r8a779g0_core_clks[] __initconst = { DEF_FIXED("s0d4_hsc", R8A779G0_CLK_S0D4_HSC, CLK_S0_HSC, 4, 1), DEF_FIXED("cl16m_hsc", R8A779G0_CLK_CL16M_HSC, CLK_S0_HSC, 48, 1), DEF_FIXED("s0d2_cc", R8A779G0_CLK_S0D2_CC, CLK_S0, 2, 1), + DEF_FIXED("sasyncperd1",R8A779G0_CLK_SASYNCPERD1, CLK_SASYNCPER,1, 1), + DEF_FIXED("sasyncperd2",R8A779G0_CLK_SASYNCPERD2, CLK_SASYNCPER,2, 1), + DEF_FIXED("sasyncperd4",R8A779G0_CLK_SASYNCPERD4, CLK_SASYNCPER,4, 1), DEF_FIXED("svd1_ir", R8A779G0_CLK_SVD1_IR, CLK_SV_IR, 1, 1), DEF_FIXED("svd2_ir", R8A779G0_CLK_SVD2_IR, CLK_SV_IR, 2, 1), DEF_FIXED("svd1_vip", R8A779G0_CLK_SVD1_VIP, CLK_SV_VIP, 1, 1), -- GitLab From 9989bc33c4894e0751679b91fc6eb585772487b9 Mon Sep 17 00:00:00 2001 From: Sai Krishna Potthuri Date: Mon, 17 Oct 2022 18:33:02 +0530 Subject: [PATCH 0168/1927] Revert "pinctrl: pinctrl-zynqmp: Add support for output-enable and bias-high-impedance" This reverts commit ad2bea79ef0144043721d4893eef719c907e2e63. On systems with older PMUFW (Xilinx ZynqMP Platform Management Firmware) using these pinctrl properties can cause system hang because there is missing feature autodetection. When this feature is implemented in the PMUFW, support for these two properties should bring back. Cc: stable@vger.kernel.org Signed-off-by: Sai Krishna Potthuri Acked-by: Michal Simek Link: https://lore.kernel.org/r/20221017130303.21746-2-sai.krishna.potthuri@amd.com Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-zynqmp.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/drivers/pinctrl/pinctrl-zynqmp.c b/drivers/pinctrl/pinctrl-zynqmp.c index 7d2fbf8a02cd6..c98f35ad89217 100644 --- a/drivers/pinctrl/pinctrl-zynqmp.c +++ b/drivers/pinctrl/pinctrl-zynqmp.c @@ -412,10 +412,6 @@ static int zynqmp_pinconf_cfg_set(struct pinctrl_dev *pctldev, break; case PIN_CONFIG_BIAS_HIGH_IMPEDANCE: - param = PM_PINCTRL_CONFIG_TRI_STATE; - arg = PM_PINCTRL_TRI_STATE_ENABLE; - ret = zynqmp_pm_pinctrl_set_config(pin, param, arg); - break; case PIN_CONFIG_MODE_LOW_POWER: /* * These cases are mentioned in dts but configurable @@ -424,11 +420,6 @@ static int zynqmp_pinconf_cfg_set(struct pinctrl_dev *pctldev, */ ret = 0; break; - case PIN_CONFIG_OUTPUT_ENABLE: - param = PM_PINCTRL_CONFIG_TRI_STATE; - arg = PM_PINCTRL_TRI_STATE_DISABLE; - ret = zynqmp_pm_pinctrl_set_config(pin, param, arg); - break; default: dev_warn(pctldev->dev, "unsupported configuration parameter '%u'\n", -- GitLab From ff8356060e3a5e126abb5e1f6b6e9931c220dec2 Mon Sep 17 00:00:00 2001 From: Sai Krishna Potthuri Date: Mon, 17 Oct 2022 18:33:03 +0530 Subject: [PATCH 0169/1927] Revert "dt-bindings: pinctrl-zynqmp: Add output-enable configuration" This reverts commit 133ad0d9af99bdca90705dadd8d31c20bfc9919f. On systems with older PMUFW (Xilinx ZynqMP Platform Management Firmware) using these pinctrl properties can cause system hang because there is missing feature autodetection. When this feature is implemented, support for these two properties should bring back. Cc: stable@vger.kernel.org Signed-off-by: Sai Krishna Potthuri Acked-by: Michal Simek Link: https://lore.kernel.org/r/20221017130303.21746-3-sai.krishna.potthuri@amd.com Signed-off-by: Linus Walleij --- .../devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml index 1e2b9b627b126..2722dc7bb03da 100644 --- a/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml +++ b/Documentation/devicetree/bindings/pinctrl/xlnx,zynqmp-pinctrl.yaml @@ -274,10 +274,6 @@ patternProperties: slew-rate: enum: [0, 1] - output-enable: - description: - This will internally disable the tri-state for MIO pins. - drive-strength: description: Selects the drive strength for MIO pins, in mA. -- GitLab From 69d04ca999499bccb6ca849fa2bfc5e6448f7233 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Fri, 23 Sep 2022 12:34:41 +0300 Subject: [PATCH 0170/1927] mtd: spi-nor: core: Ignore -ENOTSUPP in spi_nor_init() The Intel SPI-NOR controller does not support the 4-byte address opcode so ->set_4byte_addr_mode() ends up returning -ENOTSUPP and the SPI flash chip probe fail like this: [ 12.291082] spi-nor: probe of spi0.0 failed with error -524 Whereas previously before commit 08412e72afba ("mtd: spi-nor: core: Return error code from set_4byte_addr_mode()") it worked just fine. Fix this by ignoring -ENOTSUPP in spi_nor_init(). Fixes: 08412e72afba ("mtd: spi-nor: core: Return error code from set_4byte_addr_mode()") Cc: stable@vger.kernel.org Reported-by: Hongyu Ning Signed-off-by: Mika Westerberg Reviewed-by: Michael Walle Acked-by: Tudor Ambarus Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20220923093441.3178-1-mika.westerberg@linux.intel.com --- drivers/mtd/spi-nor/core.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c index f2c64006f8d75..bee8fc4c9f078 100644 --- a/drivers/mtd/spi-nor/core.c +++ b/drivers/mtd/spi-nor/core.c @@ -2724,7 +2724,9 @@ static int spi_nor_init(struct spi_nor *nor) */ WARN_ONCE(nor->flags & SNOR_F_BROKEN_RESET, "enabling reset hack; may not recover from unexpected reboots\n"); - return nor->params->set_4byte_addr_mode(nor, true); + err = nor->params->set_4byte_addr_mode(nor, true); + if (err && err != -ENOTSUPP) + return err; } return 0; -- GitLab From 12b58961de0bd88b3c7dfa5d21f6d67f4678b780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Tue, 18 Oct 2022 07:18:22 +0200 Subject: [PATCH 0171/1927] mtd: core: add missing of_node_get() in dynamic partitions code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes unbalanced of_node_put(): [ 1.078910] 6 cmdlinepart partitions found on MTD device gpmi-nand [ 1.085116] Creating 6 MTD partitions on "gpmi-nand": [ 1.090181] 0x000000000000-0x000008000000 : "nandboot" [ 1.096952] 0x000008000000-0x000009000000 : "nandfit" [ 1.103547] 0x000009000000-0x00000b000000 : "nandkernel" [ 1.110317] 0x00000b000000-0x00000c000000 : "nanddtb" [ 1.115525] ------------[ cut here ]------------ [ 1.120141] refcount_t: addition on 0; use-after-free. [ 1.125328] WARNING: CPU: 0 PID: 1 at lib/refcount.c:25 refcount_warn_saturate+0xdc/0x148 [ 1.133528] Modules linked in: [ 1.136589] CPU: 0 PID: 1 Comm: swapper/0 Not tainted 6.0.0-rc7-next-20220930-04543-g8cf3f7 [ 1.146342] Hardware name: Freescale i.MX8DXL DDR3L EVK (DT) [ 1.151999] pstate: 600000c5 (nZCv daIF -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 1.158965] pc : refcount_warn_saturate+0xdc/0x148 [ 1.163760] lr : refcount_warn_saturate+0xdc/0x148 [ 1.168556] sp : ffff800009ddb080 [ 1.171866] x29: ffff800009ddb080 x28: ffff800009ddb35a x27: 0000000000000002 [ 1.179015] x26: ffff8000098b06ad x25: ffffffffffffffff x24: ffff0a00ffffff05 [ 1.186165] x23: ffff00001fdf6470 x22: ffff800009ddb367 x21: 0000000000000000 [ 1.193314] x20: ffff00001fdfebe8 x19: ffff00001fdfec50 x18: ffffffffffffffff [ 1.200464] x17: 0000000000000000 x16: 0000000000000118 x15: 0000000000000004 [ 1.207614] x14: 0000000000000fff x13: ffff800009bca248 x12: 0000000000000003 [ 1.214764] x11: 00000000ffffefff x10: c0000000ffffefff x9 : 4762cb2ccb52de00 [ 1.221914] x8 : 4762cb2ccb52de00 x7 : 205d313431303231 x6 : 312e31202020205b [ 1.229063] x5 : ffff800009d55c1f x4 : 0000000000000001 x3 : 0000000000000000 [ 1.236213] x2 : 0000000000000000 x1 : ffff800009954be6 x0 : 000000000000002a [ 1.243365] Call trace: [ 1.245806] refcount_warn_saturate+0xdc/0x148 [ 1.250253] kobject_get+0x98/0x9c [ 1.253658] of_node_get+0x20/0x34 [ 1.257072] of_fwnode_get+0x3c/0x54 [ 1.260652] fwnode_get_nth_parent+0xd8/0xf4 [ 1.264926] fwnode_full_name_string+0x3c/0xb4 [ 1.269373] device_node_string+0x498/0x5b4 [ 1.273561] pointer+0x41c/0x5d0 [ 1.276793] vsnprintf+0x4d8/0x694 [ 1.280198] vprintk_store+0x164/0x528 [ 1.283951] vprintk_emit+0x98/0x164 [ 1.287530] vprintk_default+0x44/0x6c [ 1.291284] vprintk+0xf0/0x134 [ 1.294428] _printk+0x54/0x7c [ 1.297486] of_node_release+0xe8/0x128 [ 1.301326] kobject_put+0x98/0xfc [ 1.304732] of_node_put+0x1c/0x28 [ 1.308137] add_mtd_device+0x484/0x6d4 [ 1.311977] add_mtd_partitions+0xf0/0x1d0 [ 1.316078] parse_mtd_partitions+0x45c/0x518 [ 1.320439] mtd_device_parse_register+0xb0/0x274 [ 1.325147] gpmi_nand_probe+0x51c/0x650 [ 1.329074] platform_probe+0xa8/0xd0 [ 1.332740] really_probe+0x130/0x334 [ 1.336406] __driver_probe_device+0xb4/0xe0 [ 1.340681] driver_probe_device+0x3c/0x1f8 [ 1.344869] __driver_attach+0xdc/0x1a4 [ 1.348708] bus_for_each_dev+0x80/0xcc [ 1.352548] driver_attach+0x24/0x30 [ 1.356127] bus_add_driver+0x108/0x1f4 [ 1.359967] driver_register+0x78/0x114 [ 1.363807] __platform_driver_register+0x24/0x30 [ 1.368515] gpmi_nand_driver_init+0x1c/0x28 [ 1.372798] do_one_initcall+0xbc/0x238 [ 1.376638] do_initcall_level+0x94/0xb4 [ 1.380565] do_initcalls+0x54/0x94 [ 1.384058] do_basic_setup+0x1c/0x28 [ 1.387724] kernel_init_freeable+0x110/0x188 [ 1.392084] kernel_init+0x20/0x1a0 [ 1.395578] ret_from_fork+0x10/0x20 [ 1.399157] ---[ end trace 0000000000000000 ]--- [ 1.403782] ------------[ cut here ]------------ Reported-by: Han Xu Fixes: ad9b10d1eaada169 ("mtd: core: introduce of support for dynamic partitions") Signed-off-by: Rafał Miłecki Tested-by: Han Xu Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20221018051822.28685-1-zajec5@gmail.com --- drivers/mtd/mtdcore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 18aa54460d36c..0b4ca0aa41321 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -562,7 +562,7 @@ static void mtd_check_of_node(struct mtd_info *mtd) if (!mtd_is_partition(mtd)) return; parent = mtd->parent; - parent_dn = dev_of_node(&parent->dev); + parent_dn = of_node_get(dev_of_node(&parent->dev)); if (!parent_dn) return; -- GitLab From 3c6174f9ffcb63ac8b54809c8043d7800b185bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Mon, 17 Oct 2022 21:52:50 +0200 Subject: [PATCH 0172/1927] fbdev: da8xx-fb: Fix error handling in .remove() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Even in the presence of problems (here: regulator_disable() might fail), it's important to unregister all resources acquired during .probe() and disable the device (i.e. DMA activity) because even if .remove() returns an error code, the device is removed and the .remove() callback is never called again later to catch up. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Helge Deller Fixes: 611097d5daea ("fbdev: da8xx: add support for a regulator") --- drivers/video/fbdev/da8xx-fb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/video/fbdev/da8xx-fb.c b/drivers/video/fbdev/da8xx-fb.c index ae76a2111c774..11922b009ed7c 100644 --- a/drivers/video/fbdev/da8xx-fb.c +++ b/drivers/video/fbdev/da8xx-fb.c @@ -1076,7 +1076,8 @@ static int fb_remove(struct platform_device *dev) if (par->lcd_supply) { ret = regulator_disable(par->lcd_supply); if (ret) - return ret; + dev_warn(&dev->dev, "Failed to disable regulator (%pe)\n", + ERR_PTR(ret)); } lcd_disable_raster(DA8XX_FRAME_WAIT); -- GitLab From 776d875fd4cbb3884860ea7f63c3958f02b0c80e Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Fri, 14 Oct 2022 20:01:17 +0200 Subject: [PATCH 0173/1927] fbdev: stifb: Fall back to cfb_fillrect() on 32-bit HCRX cards When the text console is scrolling text upwards it calls the fillrect() function to empty the new line. The current implementation doesn't seem to work correctly on HCRX cards in 32-bit mode and leave garbage in that line instead. Fix it by falling back to standard cfb_fillrect() in that case. Signed-off-by: Helge Deller Cc: --- drivers/video/fbdev/stifb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/video/fbdev/stifb.c b/drivers/video/fbdev/stifb.c index 7753e586e65a0..3feb6e40d56d8 100644 --- a/drivers/video/fbdev/stifb.c +++ b/drivers/video/fbdev/stifb.c @@ -1055,7 +1055,8 @@ stifb_fillrect(struct fb_info *info, const struct fb_fillrect *rect) { struct stifb_info *fb = container_of(info, struct stifb_info, info); - if (rect->rop != ROP_COPY) + if (rect->rop != ROP_COPY || + (fb->id == S9000_ID_HCRX && fb->info.var.bits_per_pixel == 32)) return cfb_fillrect(info, rect); SETUP_HW(fb); -- GitLab From 9750737130dc7b2e4c6f4f33e7e2381e49014299 Mon Sep 17 00:00:00 2001 From: Xuezhi Zhang Date: Tue, 18 Oct 2022 14:25:48 +0800 Subject: [PATCH 0174/1927] fbdev: sm501fb: Convert sysfs snprintf to sysfs_emit Follow the advice of the Documentation/filesystems/sysfs.rst and show() should only use sysfs_emit() or sysfs_emit_at() when formatting the value to be returned to user space. Signed-off-by: Xuezhi Zhang Signed-off-by: Helge Deller --- drivers/video/fbdev/sm501fb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/video/fbdev/sm501fb.c b/drivers/video/fbdev/sm501fb.c index fce6cfbadfd60..f743bfbde2a6c 100644 --- a/drivers/video/fbdev/sm501fb.c +++ b/drivers/video/fbdev/sm501fb.c @@ -1166,7 +1166,7 @@ static ssize_t sm501fb_crtsrc_show(struct device *dev, ctrl = smc501_readl(info->regs + SM501_DC_CRT_CONTROL); ctrl &= SM501_DC_CRT_CONTROL_SEL; - return snprintf(buf, PAGE_SIZE, "%s\n", ctrl ? "crt" : "panel"); + return sysfs_emit(buf, "%s\n", ctrl ? "crt" : "panel"); } /* sm501fb_crtsrc_show -- GitLab From 1f3b494d1fc18ebb37aaa47107e9b84bf5b54ff7 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sat, 24 Sep 2022 21:10:10 +0800 Subject: [PATCH 0175/1927] mtd: rawnand: intel: Add missing of_node_put() in ebu_nand_probe() The 'chip_np' returned by of_get_next_child() with refcount decremented, of_node_put() need be called in error path to decrease the refcount. Fixes: bfc618fcc3f1 ("mtd: rawnand: intel: Read the chip-select line from the correct OF node") Signed-off-by: Yang Yingliang Reviewed-by: Martin Blumenstingl Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20220924131010.957117-1-yangyingliang@huawei.com --- drivers/mtd/nand/raw/intel-nand-controller.c | 23 +++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/drivers/mtd/nand/raw/intel-nand-controller.c b/drivers/mtd/nand/raw/intel-nand-controller.c index d4a0987e93ace..6f4cea81f97c0 100644 --- a/drivers/mtd/nand/raw/intel-nand-controller.c +++ b/drivers/mtd/nand/raw/intel-nand-controller.c @@ -608,11 +608,12 @@ static int ebu_nand_probe(struct platform_device *pdev) ret = of_property_read_u32(chip_np, "reg", &cs); if (ret) { dev_err(dev, "failed to get chip select: %d\n", ret); - return ret; + goto err_of_node_put; } if (cs >= MAX_CS) { dev_err(dev, "got invalid chip select: %d\n", cs); - return -EINVAL; + ret = -EINVAL; + goto err_of_node_put; } ebu_host->cs_num = cs; @@ -620,18 +621,22 @@ static int ebu_nand_probe(struct platform_device *pdev) resname = devm_kasprintf(dev, GFP_KERNEL, "nand_cs%d", cs); ebu_host->cs[cs].chipaddr = devm_platform_ioremap_resource_byname(pdev, resname); - if (IS_ERR(ebu_host->cs[cs].chipaddr)) - return PTR_ERR(ebu_host->cs[cs].chipaddr); + if (IS_ERR(ebu_host->cs[cs].chipaddr)) { + ret = PTR_ERR(ebu_host->cs[cs].chipaddr); + goto err_of_node_put; + } ebu_host->clk = devm_clk_get(dev, NULL); - if (IS_ERR(ebu_host->clk)) - return dev_err_probe(dev, PTR_ERR(ebu_host->clk), - "failed to get clock\n"); + if (IS_ERR(ebu_host->clk)) { + ret = dev_err_probe(dev, PTR_ERR(ebu_host->clk), + "failed to get clock\n"); + goto err_of_node_put; + } ret = clk_prepare_enable(ebu_host->clk); if (ret) { dev_err(dev, "failed to enable clock: %d\n", ret); - return ret; + goto err_of_node_put; } ebu_host->dma_tx = dma_request_chan(dev, "tx"); @@ -695,6 +700,8 @@ static int ebu_nand_probe(struct platform_device *pdev) ebu_dma_cleanup(ebu_host); err_disable_unprepare_clk: clk_disable_unprepare(ebu_host->clk); +err_of_node_put: + of_node_put(chip_np); return ret; } -- GitLab From 0a974e6ae43b3a6aac63dfdfdf171be205fa370c Mon Sep 17 00:00:00 2001 From: Xuezhi Zhang Date: Tue, 18 Oct 2022 15:51:18 +0800 Subject: [PATCH 0176/1927] fbdev: gbefb: Convert sysfs snprintf to sysfs_emit Follow the advice of the Documentation/filesystems/sysfs.rst and show() should only use sysfs_emit() or sysfs_emit_at() when formatting the value to be returned to user space. Signed-off-by: Xuezhi Zhang Signed-off-by: Helge Deller --- drivers/video/fbdev/gbefb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/video/fbdev/gbefb.c b/drivers/video/fbdev/gbefb.c index 1582c718329c7..000b4aa442415 100644 --- a/drivers/video/fbdev/gbefb.c +++ b/drivers/video/fbdev/gbefb.c @@ -1060,14 +1060,14 @@ static const struct fb_ops gbefb_ops = { static ssize_t gbefb_show_memsize(struct device *dev, struct device_attribute *attr, char *buf) { - return snprintf(buf, PAGE_SIZE, "%u\n", gbe_mem_size); + return sysfs_emit(buf, "%u\n", gbe_mem_size); } static DEVICE_ATTR(size, S_IRUGO, gbefb_show_memsize, NULL); static ssize_t gbefb_show_rev(struct device *device, struct device_attribute *attr, char *buf) { - return snprintf(buf, PAGE_SIZE, "%d\n", gbe_revision); + return sysfs_emit(buf, "%d\n", gbe_revision); } static DEVICE_ATTR(revision, S_IRUGO, gbefb_show_rev, NULL); -- GitLab From 3ada71310d2c68eebb57772df6bb1f5f033ae802 Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Mon, 26 Sep 2022 16:44:56 +0800 Subject: [PATCH 0177/1927] mtd: rawnand: tegra: Fix PM disable depth imbalance in probe The pm_runtime_enable will increase power disable depth. Thus a pairing decrement is needed on the error handling path to keep it balanced according to context. Cc: stable@vger.kernel.org Fixes: d7d9f8ec77fe9 ("mtd: rawnand: add NVIDIA Tegra NAND Flash controller driver") Signed-off-by: Zhang Qilong Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20220926084456.98160-1-zhangqilong3@huawei.com --- drivers/mtd/nand/raw/tegra_nand.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/mtd/nand/raw/tegra_nand.c b/drivers/mtd/nand/raw/tegra_nand.c index e12f9f580a15f..a9b9031ce6167 100644 --- a/drivers/mtd/nand/raw/tegra_nand.c +++ b/drivers/mtd/nand/raw/tegra_nand.c @@ -1181,7 +1181,7 @@ static int tegra_nand_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); err = pm_runtime_resume_and_get(&pdev->dev); if (err) - return err; + goto err_dis_pm; err = reset_control_reset(rst); if (err) { @@ -1215,6 +1215,8 @@ static int tegra_nand_probe(struct platform_device *pdev) err_put_pm: pm_runtime_put_sync_suspend(ctrl->dev); pm_runtime_force_suspend(ctrl->dev); +err_dis_pm: + pm_runtime_disable(&pdev->dev); return err; } -- GitLab From ce107713b722af57c4b7f2477594d445b496420e Mon Sep 17 00:00:00 2001 From: Tony O'Brien Date: Tue, 27 Sep 2022 15:47:28 +1300 Subject: [PATCH 0178/1927] mtd: rawnand: marvell: Use correct logic for nand-keep-config Originally the absence of the marvell,nand-keep-config property caused the setup_data_interface function to be provided. However when setup_data_interface was moved into nand_controller_ops the logic was unintentionally inverted. Update the logic so that only if the marvell,nand-keep-config property is present the bootloader NAND config kept. Cc: stable@vger.kernel.org Fixes: 7a08dbaedd36 ("mtd: rawnand: Move ->setup_data_interface() to nand_controller_ops") Signed-off-by: Tony O'Brien Signed-off-by: Chris Packham Reviewed-by: Boris Brezillon Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20220927024728.28447-1-chris.packham@alliedtelesis.co.nz --- drivers/mtd/nand/raw/marvell_nand.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/mtd/nand/raw/marvell_nand.c b/drivers/mtd/nand/raw/marvell_nand.c index d9f2f1d0b5efc..b9d1e96e33345 100644 --- a/drivers/mtd/nand/raw/marvell_nand.c +++ b/drivers/mtd/nand/raw/marvell_nand.c @@ -2678,7 +2678,7 @@ static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc, chip->controller = &nfc->controller; nand_set_flash_node(chip, np); - if (!of_property_read_bool(np, "marvell,nand-keep-config")) + if (of_property_read_bool(np, "marvell,nand-keep-config")) chip->options |= NAND_KEEP_TIMINGS; mtd = nand_to_mtd(chip); -- GitLab From e9945b2633deccda74a769d94060df49c53ff181 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Tue, 18 Oct 2022 09:09:59 +0200 Subject: [PATCH 0179/1927] pinctrl: ocelot: Fix incorrect trigger of the interrupt. The interrupt controller can detect only link changes. So in case an external device generated a level based interrupt, then the interrupt controller detected correctly the first edge. But the problem was that the interrupt controller was detecting also the edge when the interrupt was cleared. So it would generate another interrupt. The fix for this is to clear the second interrupt but still check the interrupt line status. Fixes: c297561bc98a ("pinctrl: ocelot: Fix interrupt controller") Signed-off-by: Horatiu Vultur Tested-by: Michael Walle Link: https://lore.kernel.org/r/20221018070959.1322606-1-horatiu.vultur@microchip.com Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-ocelot.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/pinctrl/pinctrl-ocelot.c b/drivers/pinctrl/pinctrl-ocelot.c index 62ce3957abe4e..687aaa6015555 100644 --- a/drivers/pinctrl/pinctrl-ocelot.c +++ b/drivers/pinctrl/pinctrl-ocelot.c @@ -1864,19 +1864,28 @@ static void ocelot_irq_unmask_level(struct irq_data *data) if (val & bit) ack = true; + /* Try to clear any rising edges */ + if (!active && ack) + regmap_write_bits(info->map, REG(OCELOT_GPIO_INTR, info, gpio), + bit, bit); + /* Enable the interrupt now */ gpiochip_enable_irq(chip, gpio); regmap_update_bits(info->map, REG(OCELOT_GPIO_INTR_ENA, info, gpio), bit, bit); /* - * In case the interrupt line is still active and the interrupt - * controller has not seen any changes in the interrupt line, then it - * means that there happen another interrupt while the line was active. + * In case the interrupt line is still active then it means that + * there happen another interrupt while the line was active. * So we missed that one, so we need to kick the interrupt again * handler. */ - if (active && !ack) { + regmap_read(info->map, REG(OCELOT_GPIO_IN, info, gpio), &val); + if ((!(val & bit) && trigger_level == IRQ_TYPE_LEVEL_LOW) || + (val & bit && trigger_level == IRQ_TYPE_LEVEL_HIGH)) + active = true; + + if (active) { struct ocelot_irq_work *work; work = kmalloc(sizeof(*work), GFP_ATOMIC); -- GitLab From 05e258c6ec669d6d18c494ea03d35962d6f5b545 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Tue, 18 Oct 2022 11:11:29 +0200 Subject: [PATCH 0180/1927] mtd: parsers: bcm47xxpart: Fix halfblock reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is some code in the parser that tries to read 0x8000 bytes into a block to "read in the middle" of the block. Well that only works if the block is also 0x10000 bytes all the time, else we get these parse errors as we reach the end of the flash: spi-nor spi0.0: mx25l1606e (2048 Kbytes) mtd_read error while parsing (offset: 0x200000): -22 mtd_read error while parsing (offset: 0x201000): -22 (...) Fix the code to do what I think was intended. Cc: stable@vger.kernel.org Fixes: f0501e81fbaa ("mtd: bcm47xxpart: alternative MAGIC for board_data partition") Cc: Rafał Miłecki Cc: Florian Fainelli Signed-off-by: Linus Walleij Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20221018091129.280026-1-linus.walleij@linaro.org --- drivers/mtd/parsers/bcm47xxpart.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mtd/parsers/bcm47xxpart.c b/drivers/mtd/parsers/bcm47xxpart.c index 50fcf4c2174ba..13daf9bffd081 100644 --- a/drivers/mtd/parsers/bcm47xxpart.c +++ b/drivers/mtd/parsers/bcm47xxpart.c @@ -233,11 +233,11 @@ static int bcm47xxpart_parse(struct mtd_info *master, } /* Read middle of the block */ - err = mtd_read(master, offset + 0x8000, 0x4, &bytes_read, + err = mtd_read(master, offset + (blocksize / 2), 0x4, &bytes_read, (uint8_t *)buf); if (err && !mtd_is_bitflip(err)) { pr_err("mtd_read error while parsing (offset: 0x%X): %d\n", - offset + 0x8000, err); + offset + (blocksize / 2), err); continue; } -- GitLab From 4e8ff35878685291978b93543d6b9e9290be770a Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Sun, 16 Oct 2022 10:33:50 +0200 Subject: [PATCH 0181/1927] ASoC: codecs: tlv320adc3xxx: Wrap adc3xxx_i2c_remove() in __exit_p() If CONFIG_SND_SOC_TLV320ADC3XXX=y: `.exit.text' referenced in section `.data' of sound/soc/codecs/tlv320adc3xxx.o: defined in discarded section `.exit.text' of sound/soc/codecs/tlv320adc3xxx.o Fix this by wrapping the adc3xxx_i2c_remove() pointer in __exit_p(). Fixes: e9a3b57efd28fe88 ("ASoC: codec: tlv320adc3xxx: New codec driver") Signed-off-by: Geert Uytterhoeven Link: https://lore.kernel.org/r/3225ba4cfe558d9380155e75385954dd21d4e7eb.1665909132.git.geert@linux-m68k.org Signed-off-by: Mark Brown --- sound/soc/codecs/tlv320adc3xxx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/codecs/tlv320adc3xxx.c b/sound/soc/codecs/tlv320adc3xxx.c index 748998e48af97..8a0965cd3e667 100644 --- a/sound/soc/codecs/tlv320adc3xxx.c +++ b/sound/soc/codecs/tlv320adc3xxx.c @@ -1450,7 +1450,7 @@ static struct i2c_driver adc3xxx_i2c_driver = { .of_match_table = tlv320adc3xxx_of_match, }, .probe_new = adc3xxx_i2c_probe, - .remove = adc3xxx_i2c_remove, + .remove = __exit_p(adc3xxx_i2c_remove), .id_table = adc3xxx_i2c_id, }; -- GitLab From 9a7f2c9e7a19b16b4409f372cf2e16e4334cdca2 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 14 Oct 2022 17:12:28 -0700 Subject: [PATCH 0182/1927] ASoC: qcom: SND_SOC_SC7180 optionally depends on SOUNDWIRE If SOUNDWIRE is enabled, then SND_SOC_SC7180 should depend on SOUNDWIRE to prevent SOUNDWIRE=m and SND_SOC_SC7180=y, which causes build errors: s390-linux-ld: sound/soc/qcom/common.o: in function `qcom_snd_sdw_prepare': common.c:(.text+0x140): undefined reference to `sdw_disable_stream' s390-linux-ld: common.c:(.text+0x14a): undefined reference to `sdw_deprepare_stream' s390-linux-ld: common.c:(.text+0x158): undefined reference to `sdw_prepare_stream' s390-linux-ld: common.c:(.text+0x16a): undefined reference to `sdw_enable_stream' s390-linux-ld: common.c:(.text+0x17c): undefined reference to `sdw_deprepare_stream' s390-linux-ld: sound/soc/qcom/common.o: in function `qcom_snd_sdw_hw_free': common.c:(.text+0x344): undefined reference to `sdw_disable_stream' s390-linux-ld: common.c:(.text+0x34e): undefined reference to `sdw_deprepare_stream' Fixes: 3bd975f3ae0a ("ASoC: qcom: sm8250: move some code to common") Fixes: 9e3ecb5b1681 ("ASoC: qcom: sc7180: Add machine driver for sound card registration") Signed-off-by: Randy Dunlap Reported-by: kernel test robot Cc: Srinivas Kandagatla Cc: Banajit Goswami Cc: Mark Brown Cc: Liam Girdwood Cc: Ajit Pandey Cc: Cheng-Yi Chiang Cc: Jaroslav Kysela Cc: Takashi Iwai Cc: stable@vger.kernel.org Cc: alsa-devel@alsa-project.org Link: https://lore.kernel.org/r/20221015001228.18990-1-rdunlap@infradead.org Signed-off-by: Mark Brown --- sound/soc/qcom/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/soc/qcom/Kconfig b/sound/soc/qcom/Kconfig index d0e59e07b1fc2..8c7398bc1ca89 100644 --- a/sound/soc/qcom/Kconfig +++ b/sound/soc/qcom/Kconfig @@ -187,6 +187,7 @@ config SND_SOC_SC8280XP config SND_SOC_SC7180 tristate "SoC Machine driver for SC7180 boards" depends on I2C && GPIOLIB + depends on SOUNDWIRE || SOUNDWIRE=n select SND_SOC_QCOM_COMMON select SND_SOC_LPASS_SC7180 select SND_SOC_MAX98357A -- GitLab From 89ed0b769d6adf30364f60e6b1566961821a9893 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Sun, 9 Oct 2022 20:41:25 -0700 Subject: [PATCH 0183/1927] powerpc/pseries/vas: Add VAS IRQ primary handler irq_default_primary_handler() can be used only with IRQF_ONESHOT flag, but the flag disables IRQ before executing the thread handler and enables it after the interrupt is handled. But this IRQ disable sets the VAS IRQ OFF state in the hypervisor. In case if NX faults during this window, the hypervisor will not deliver the fault interrupt to the partition and the user space may wait continuously for the CSB update. So use VAS specific IRQ handler instead of calling the default primary handler. Increment pending_faults counter in IRQ handler and the bottom thread handler will process all faults based on this counter. In case if the another interrupt is received while the thread is running, it will be processed using this counter. The synchronization of top and bottom handlers will be done with IRQTF_RUNTHREAD flag and will re-enter to bottom half if this flag is set. Signed-off-by: Haren Myneni Reviewed-by: Frederic Barrat Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/aaad8813b4762a6753cfcd0b605a7574a5192ec7.camel@linux.ibm.com --- arch/powerpc/platforms/pseries/vas.c | 40 +++++++++++++++++++++++----- arch/powerpc/platforms/pseries/vas.h | 1 + 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c index 0e0524cbe20c9..53b36381489ed 100644 --- a/arch/powerpc/platforms/pseries/vas.c +++ b/arch/powerpc/platforms/pseries/vas.c @@ -200,16 +200,41 @@ static irqreturn_t pseries_vas_fault_thread_fn(int irq, void *data) struct vas_user_win_ref *tsk_ref; int rc; - rc = h_get_nx_fault(txwin->vas_win.winid, (u64)virt_to_phys(&crb)); - if (!rc) { - tsk_ref = &txwin->vas_win.task_ref; - vas_dump_crb(&crb); - vas_update_csb(&crb, tsk_ref); + while (atomic_read(&txwin->pending_faults)) { + rc = h_get_nx_fault(txwin->vas_win.winid, (u64)virt_to_phys(&crb)); + if (!rc) { + tsk_ref = &txwin->vas_win.task_ref; + vas_dump_crb(&crb); + vas_update_csb(&crb, tsk_ref); + } + atomic_dec(&txwin->pending_faults); } return IRQ_HANDLED; } +/* + * irq_default_primary_handler() can be used only with IRQF_ONESHOT + * which disables IRQ before executing the thread handler and enables + * it after. But this disabling interrupt sets the VAS IRQ OFF + * state in the hypervisor. If the NX generates fault interrupt + * during this window, the hypervisor will not deliver this + * interrupt to the LPAR. So use VAS specific IRQ handler instead + * of calling the default primary handler. + */ +static irqreturn_t pseries_vas_irq_handler(int irq, void *data) +{ + struct pseries_vas_window *txwin = data; + + /* + * The thread hanlder will process this interrupt if it is + * already running. + */ + atomic_inc(&txwin->pending_faults); + + return IRQ_WAKE_THREAD; +} + /* * Allocate window and setup IRQ mapping. */ @@ -240,8 +265,9 @@ static int allocate_setup_window(struct pseries_vas_window *txwin, goto out_irq; } - rc = request_threaded_irq(txwin->fault_virq, NULL, - pseries_vas_fault_thread_fn, IRQF_ONESHOT, + rc = request_threaded_irq(txwin->fault_virq, + pseries_vas_irq_handler, + pseries_vas_fault_thread_fn, 0, txwin->name, txwin); if (rc) { pr_err("VAS-Window[%d]: Request IRQ(%u) failed with %d\n", diff --git a/arch/powerpc/platforms/pseries/vas.h b/arch/powerpc/platforms/pseries/vas.h index 333ffa2f9f426..a2cb12a31c178 100644 --- a/arch/powerpc/platforms/pseries/vas.h +++ b/arch/powerpc/platforms/pseries/vas.h @@ -132,6 +132,7 @@ struct pseries_vas_window { u64 flags; char *name; int fault_virq; + atomic_t pending_faults; /* Number of pending faults */ }; int sysfs_add_vas_caps(struct vas_cop_feat_caps *caps); -- GitLab From 2147783d6bf0b7ca14c72a25527dc5135bd17f65 Mon Sep 17 00:00:00 2001 From: Haren Myneni Date: Thu, 6 Oct 2022 22:29:59 -0700 Subject: [PATCH 0184/1927] powerpc/pseries: Use lparcfg to reconfig VAS windows for DLPAR CPU The hypervisor assigns VAS (Virtual Accelerator Switchboard) windows depends on cores configured in LPAR. The kernel uses OF reconfig notifier to reconfig VAS windows for DLPAR CPU event. In the case of shared CPU mode partition, the hypervisor assigns VAS windows depends on CPU entitled capacity, not based on vcpus. When the user changes CPU entitled capacity for the partition, drmgr uses /proc/ppc64/lparcfg interface to notify the kernel. This patch adds the following changes to update VAS resources for shared mode: - Call vas reconfig windows from lparcfg_write() - Ignore reconfig changes in the VAS notifier Signed-off-by: Haren Myneni [mpe: Rework error handling, report any errors as EIO] Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/efa9c16e4a78dda4567a16f13dabfd73cb4674a2.camel@linux.ibm.com --- arch/powerpc/platforms/pseries/lparcfg.c | 11 ++++++ arch/powerpc/platforms/pseries/vas.c | 43 ++++++++++++++++-------- arch/powerpc/platforms/pseries/vas.h | 5 +++ 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c index 507dc0b5987d0..63fd925ccbb83 100644 --- a/arch/powerpc/platforms/pseries/lparcfg.c +++ b/arch/powerpc/platforms/pseries/lparcfg.c @@ -35,6 +35,7 @@ #include #include "pseries.h" +#include "vas.h" /* pseries_vas_dlpar_cpu() */ /* * This isn't a module but we expose that to userspace @@ -748,6 +749,16 @@ static ssize_t lparcfg_write(struct file *file, const char __user * buf, return -EINVAL; retval = update_ppp(new_entitled_ptr, NULL); + + if (retval == H_SUCCESS || retval == H_CONSTRAINED) { + /* + * The hypervisor assigns VAS resources based + * on entitled capacity for shared mode. + * Reconfig VAS windows based on DLPAR CPU events. + */ + if (pseries_vas_dlpar_cpu() != 0) + retval = H_HARDWARE; + } } else if (!strcmp(kbuf, "capacity_weight")) { char *endp; *new_weight_ptr = (u8) simple_strtoul(tmp, &endp, 10); diff --git a/arch/powerpc/platforms/pseries/vas.c b/arch/powerpc/platforms/pseries/vas.c index 53b36381489ed..4ad6e510d405f 100644 --- a/arch/powerpc/platforms/pseries/vas.c +++ b/arch/powerpc/platforms/pseries/vas.c @@ -852,6 +852,25 @@ int vas_reconfig_capabilties(u8 type, int new_nr_creds) mutex_unlock(&vas_pseries_mutex); return rc; } + +int pseries_vas_dlpar_cpu(void) +{ + int new_nr_creds, rc; + + rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, + vascaps[VAS_GZIP_DEF_FEAT_TYPE].feat, + (u64)virt_to_phys(&hv_cop_caps)); + if (!rc) { + new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds); + rc = vas_reconfig_capabilties(VAS_GZIP_DEF_FEAT_TYPE, new_nr_creds); + } + + if (rc) + pr_err("Failed reconfig VAS capabilities with DLPAR\n"); + + return rc; +} + /* * Total number of default credits available (target_credits) * in LPAR depends on number of cores configured. It varies based on @@ -866,7 +885,15 @@ static int pseries_vas_notifier(struct notifier_block *nb, struct of_reconfig_data *rd = data; struct device_node *dn = rd->dn; const __be32 *intserv = NULL; - int new_nr_creds, len, rc = 0; + int len; + + /* + * For shared CPU partition, the hypervisor assigns total credits + * based on entitled core capacity. So updating VAS windows will + * be called from lparcfg_write(). + */ + if (is_shared_processor()) + return NOTIFY_OK; if ((action == OF_RECONFIG_ATTACH_NODE) || (action == OF_RECONFIG_DETACH_NODE)) @@ -878,19 +905,7 @@ static int pseries_vas_notifier(struct notifier_block *nb, if (!intserv) return NOTIFY_OK; - rc = h_query_vas_capabilities(H_QUERY_VAS_CAPABILITIES, - vascaps[VAS_GZIP_DEF_FEAT_TYPE].feat, - (u64)virt_to_phys(&hv_cop_caps)); - if (!rc) { - new_nr_creds = be16_to_cpu(hv_cop_caps.target_lpar_creds); - rc = vas_reconfig_capabilties(VAS_GZIP_DEF_FEAT_TYPE, - new_nr_creds); - } - - if (rc) - pr_err("Failed reconfig VAS capabilities with DLPAR\n"); - - return rc; + return pseries_vas_dlpar_cpu(); } static struct notifier_block pseries_vas_nb = { diff --git a/arch/powerpc/platforms/pseries/vas.h b/arch/powerpc/platforms/pseries/vas.h index a2cb12a31c178..7115043ec4883 100644 --- a/arch/powerpc/platforms/pseries/vas.h +++ b/arch/powerpc/platforms/pseries/vas.h @@ -141,10 +141,15 @@ int __init sysfs_pseries_vas_init(struct vas_all_caps *vas_caps); #ifdef CONFIG_PPC_VAS int vas_migration_handler(int action); +int pseries_vas_dlpar_cpu(void); #else static inline int vas_migration_handler(int action) { return 0; } +static inline int pseries_vas_dlpar_cpu(void) +{ + return 0; +} #endif #endif /* _VAS_H */ -- GitLab From be83d5485da549d934ec65463ea831709f2827b1 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 09:07:08 +1000 Subject: [PATCH 0185/1927] powerpc/64s: Add lockdep for HPTE lock Add lockdep annotation for the HPTE bit-spinlock. Modern systems don't take the tlbie lock, so this shows up some of the same lockdep warnings that were being reported by the ppc970. And they're not taken in exactly the same places so this is nice to have in its own right. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221013230710.1987253-1-npiggin@gmail.com --- arch/powerpc/mm/book3s64/hash_native.c | 42 +++++++++++++++++++++----- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/mm/book3s64/hash_native.c b/arch/powerpc/mm/book3s64/hash_native.c index 623a7b7ab38b1..7a640e610ea35 100644 --- a/arch/powerpc/mm/book3s64/hash_native.c +++ b/arch/powerpc/mm/book3s64/hash_native.c @@ -43,6 +43,29 @@ static DEFINE_RAW_SPINLOCK(native_tlbie_lock); +#ifdef CONFIG_LOCKDEP +static struct lockdep_map hpte_lock_map = + STATIC_LOCKDEP_MAP_INIT("hpte_lock", &hpte_lock_map); + +static void acquire_hpte_lock(void) +{ + lock_map_acquire(&hpte_lock_map); +} + +static void release_hpte_lock(void) +{ + lock_map_release(&hpte_lock_map); +} +#else +static void acquire_hpte_lock(void) +{ +} + +static void release_hpte_lock(void) +{ +} +#endif + static inline unsigned long ___tlbie(unsigned long vpn, int psize, int apsize, int ssize) { @@ -220,6 +243,7 @@ static inline void native_lock_hpte(struct hash_pte *hptep) { unsigned long *word = (unsigned long *)&hptep->v; + acquire_hpte_lock(); while (1) { if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word)) break; @@ -234,6 +258,7 @@ static inline void native_unlock_hpte(struct hash_pte *hptep) { unsigned long *word = (unsigned long *)&hptep->v; + release_hpte_lock(); clear_bit_unlock(HPTE_LOCK_BIT, word); } @@ -279,6 +304,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, hpte_v = hpte_old_to_new_v(hpte_v); } + release_hpte_lock(); hptep->r = cpu_to_be64(hpte_r); /* Guarantee the second dword is visible before the valid bit */ eieio(); @@ -327,6 +353,7 @@ static long native_hpte_remove(unsigned long hpte_group) return -1; /* Invalidate the hpte. NOTE: this also unlocks it */ + release_hpte_lock(); hptep->v = 0; return i; @@ -517,10 +544,11 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, /* recheck with locks held */ hpte_v = hpte_get_old_v(hptep); - if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { /* Invalidate the hpte. NOTE: this also unlocks it */ + release_hpte_lock(); hptep->v = 0; - else + } else native_unlock_hpte(hptep); } /* @@ -580,10 +608,8 @@ static void native_hugepage_invalidate(unsigned long vsid, hpte_v = hpte_get_old_v(hptep); if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { - /* - * Invalidate the hpte. NOTE: this also unlocks it - */ - + /* Invalidate the hpte. NOTE: this also unlocks it */ + release_hpte_lock(); hptep->v = 0; } else native_unlock_hpte(hptep); @@ -765,8 +791,10 @@ static void native_flush_hash_range(unsigned long number, int local) if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) native_unlock_hpte(hptep); - else + else { + release_hpte_lock(); hptep->v = 0; + } } pte_iterate_hashed_end(); } -- GitLab From 35159b5717fa9c6031fdd6a2193c7a3dc717ce33 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 09:07:09 +1000 Subject: [PATCH 0186/1927] powerpc/64s: make HPTE lock and native_tlbie_lock irq-safe With kfence enabled, there are several cases where HPTE and TLBIE locks are called from softirq context, for example: WARNING: inconsistent lock state 6.0.0-11845-g0cbbc95b12ac #1 Tainted: G N -------------------------------- inconsistent {IN-SOFTIRQ-W} -> {SOFTIRQ-ON-W} usage. swapper/0/1 [HC0[0]:SC0[0]:HE1:SE1] takes: c000000002734de8 (native_tlbie_lock){+.?.}-{2:2}, at: .native_hpte_updateboltedpp+0x1a4/0x600 {IN-SOFTIRQ-W} state was registered at: .lock_acquire+0x20c/0x520 ._raw_spin_lock+0x4c/0x70 .native_hpte_invalidate+0x62c/0x840 .hash__kernel_map_pages+0x450/0x640 .kfence_protect+0x58/0xc0 .kfence_guarded_free+0x374/0x5a0 .__slab_free+0x3d0/0x630 .put_cred_rcu+0xcc/0x120 .rcu_core+0x3c4/0x14e0 .__do_softirq+0x1dc/0x7dc .do_softirq_own_stack+0x40/0x60 Fix this by consistently disabling irqs while taking either of these locks. Don't just disable bh because several of the more common cases already disable irqs, so this just makes the locks always irq-safe. Reported-by: Guenter Roeck Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221013230710.1987253-2-npiggin@gmail.com --- arch/powerpc/mm/book3s64/hash_native.c | 27 ++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/mm/book3s64/hash_native.c b/arch/powerpc/mm/book3s64/hash_native.c index 7a640e610ea35..9342e79870dfd 100644 --- a/arch/powerpc/mm/book3s64/hash_native.c +++ b/arch/powerpc/mm/book3s64/hash_native.c @@ -268,8 +268,11 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, { struct hash_pte *hptep = htab_address + hpte_group; unsigned long hpte_v, hpte_r; + unsigned long flags; int i; + local_irq_save(flags); + if (!(vflags & HPTE_V_BOLTED)) { DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx," " rflags=%lx, vflags=%lx, psize=%d)\n", @@ -288,8 +291,10 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, hptep++; } - if (i == HPTES_PER_GROUP) + if (i == HPTES_PER_GROUP) { + local_irq_restore(flags); return -1; + } hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; @@ -304,7 +309,6 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, hpte_v = hpte_old_to_new_v(hpte_v); } - release_hpte_lock(); hptep->r = cpu_to_be64(hpte_r); /* Guarantee the second dword is visible before the valid bit */ eieio(); @@ -312,10 +316,13 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, * Now set the first dword including the valid bit * NOTE: this also unlocks the hpte */ + release_hpte_lock(); hptep->v = cpu_to_be64(hpte_v); __asm__ __volatile__ ("ptesync" : : : "memory"); + local_irq_restore(flags); + return i | (!!(vflags & HPTE_V_SECONDARY) << 3); } @@ -366,6 +373,9 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, struct hash_pte *hptep = htab_address + slot; unsigned long hpte_v, want_v; int ret = 0, local = 0; + unsigned long irqflags; + + local_irq_save(irqflags); want_v = hpte_encode_avpn(vpn, bpsize, ssize); @@ -409,6 +419,8 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, if (!(flags & HPTE_NOHPTE_UPDATE)) tlbie(vpn, bpsize, apsize, ssize, local); + local_irq_restore(irqflags); + return ret; } @@ -472,6 +484,9 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, unsigned long vsid; long slot; struct hash_pte *hptep; + unsigned long flags; + + local_irq_save(flags); vsid = get_kernel_vsid(ea, ssize); vpn = hpt_vpn(ea, vsid, ssize); @@ -490,6 +505,8 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, * actual page size will be same. */ tlbie(vpn, psize, psize, ssize, 0); + + local_irq_restore(flags); } /* @@ -503,6 +520,9 @@ static int native_hpte_removebolted(unsigned long ea, int psize, int ssize) unsigned long vsid; long slot; struct hash_pte *hptep; + unsigned long flags; + + local_irq_save(flags); vsid = get_kernel_vsid(ea, ssize); vpn = hpt_vpn(ea, vsid, ssize); @@ -520,6 +540,9 @@ static int native_hpte_removebolted(unsigned long ea, int psize, int ssize) /* Invalidate the TLB */ tlbie(vpn, psize, psize, ssize, 0); + + local_irq_restore(flags); + return 0; } -- GitLab From b12eb279ff552bd67c167b0fe701ae602aa7311e Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 09:07:10 +1000 Subject: [PATCH 0187/1927] powerpc/64s: make linear_map_hash_lock a raw spinlock This lock is taken while the raw kfence_freelist_lock is held, so it must also be a raw spinlock, as reported by lockdep when raw lock nesting checking is enabled. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221013230710.1987253-3-npiggin@gmail.com --- arch/powerpc/mm/book3s64/hash_utils.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c index df008edf7be09..6df4c6d38b660 100644 --- a/arch/powerpc/mm/book3s64/hash_utils.c +++ b/arch/powerpc/mm/book3s64/hash_utils.c @@ -1981,7 +1981,7 @@ long hpte_insert_repeating(unsigned long hash, unsigned long vpn, } #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_KFENCE) -static DEFINE_SPINLOCK(linear_map_hash_lock); +static DEFINE_RAW_SPINLOCK(linear_map_hash_lock); static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) { @@ -2005,10 +2005,10 @@ static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) mmu_linear_psize, mmu_kernel_ssize); BUG_ON (ret < 0); - spin_lock(&linear_map_hash_lock); + raw_spin_lock(&linear_map_hash_lock); BUG_ON(linear_map_hash_slots[lmi] & 0x80); linear_map_hash_slots[lmi] = ret | 0x80; - spin_unlock(&linear_map_hash_lock); + raw_spin_unlock(&linear_map_hash_lock); } static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) @@ -2018,14 +2018,14 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) unsigned long vpn = hpt_vpn(vaddr, vsid, mmu_kernel_ssize); hash = hpt_hash(vpn, PAGE_SHIFT, mmu_kernel_ssize); - spin_lock(&linear_map_hash_lock); + raw_spin_lock(&linear_map_hash_lock); if (!(linear_map_hash_slots[lmi] & 0x80)) { - spin_unlock(&linear_map_hash_lock); + raw_spin_unlock(&linear_map_hash_lock); return; } hidx = linear_map_hash_slots[lmi] & 0x7f; linear_map_hash_slots[lmi] = 0; - spin_unlock(&linear_map_hash_lock); + raw_spin_unlock(&linear_map_hash_lock); if (hidx & _PTEIDX_SECONDARY) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; -- GitLab From b9ef323ea1682f9837bf63ba10c5e3750f71a20a Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 01:16:45 +1000 Subject: [PATCH 0188/1927] powerpc/64s: Disable preemption in hash lazy mmu mode apply_to_page_range on kernel pages does not disable preemption, which is a requirement for hash's lazy mmu mode, which keeps track of the TLBs to flush with a per-cpu array. Reported-by: Guenter Roeck Signed-off-by: Nicholas Piggin Tested-by: Guenter Roeck Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221013151647.1857994-1-npiggin@gmail.com --- arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h index fab8332fe1add..751921f6db461 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -32,6 +32,11 @@ static inline void arch_enter_lazy_mmu_mode(void) if (radix_enabled()) return; + /* + * apply_to_page_range can call us this preempt enabled when + * operating on kernel page tables. + */ + preempt_disable(); batch = this_cpu_ptr(&ppc64_tlb_batch); batch->active = 1; } @@ -47,6 +52,7 @@ static inline void arch_leave_lazy_mmu_mode(void) if (batch->index) __flush_tlb_pending(batch); batch->active = 0; + preempt_enable(); } #define arch_flush_lazy_mmu_mode() do {} while (0) -- GitLab From 2b2095f3a6b43ec36ff890febc588df1ec32e826 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 01:16:46 +1000 Subject: [PATCH 0189/1927] powerpc/64s: Fix hash__change_memory_range preemption warning stop_machine_cpuslocked takes a mutex so it must be called in a preemptible context, so it can't simply be fixed by disabling preemption. This is not a bug, because CPU hotplug is locked, so this processor will call in to the stop machine function. So raw_smp_processor_id() could be used. This leaves a small chance that this thread will be migrated to another CPU, so the master work would be done by a CPU from a different context. Better for test coverage to make that a common case by just having the first CPU to call in become the master. Signed-off-by: Nicholas Piggin Tested-by: Guenter Roeck Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221013151647.1857994-2-npiggin@gmail.com --- arch/powerpc/mm/book3s64/hash_pgtable.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/mm/book3s64/hash_pgtable.c b/arch/powerpc/mm/book3s64/hash_pgtable.c index 747492edb75ab..51f48984abca9 100644 --- a/arch/powerpc/mm/book3s64/hash_pgtable.c +++ b/arch/powerpc/mm/book3s64/hash_pgtable.c @@ -404,7 +404,8 @@ EXPORT_SYMBOL_GPL(hash__has_transparent_hugepage); struct change_memory_parms { unsigned long start, end, newpp; - unsigned int step, nr_cpus, master_cpu; + unsigned int step, nr_cpus; + atomic_t master_cpu; atomic_t cpu_counter; }; @@ -478,7 +479,8 @@ static int change_memory_range_fn(void *data) { struct change_memory_parms *parms = data; - if (parms->master_cpu != smp_processor_id()) + // First CPU goes through, all others wait. + if (atomic_xchg(&parms->master_cpu, 1) == 1) return chmem_secondary_loop(parms); // Wait for all but one CPU (this one) to call-in @@ -516,7 +518,7 @@ static bool hash__change_memory_range(unsigned long start, unsigned long end, chmem_parms.end = end; chmem_parms.step = step; chmem_parms.newpp = newpp; - chmem_parms.master_cpu = smp_processor_id(); + atomic_set(&chmem_parms.master_cpu, 0); cpus_read_lock(); -- GitLab From 00ff1eaac129a24516a3f6d75adfb9df1efb55dd Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 01:16:47 +1000 Subject: [PATCH 0190/1927] powerpc: Fix reschedule bug in KUAP-unlocked user copy schedule must not be explicitly called while KUAP is unlocked, because the AMR register will not be saved across the context switch on 64s (preemption is allowed because that is driven by interrupts which do save the AMR). exit_vmx_usercopy() runs inside an unlocked user access region, and it calls preempt_enable() which will call schedule() if need_resched() was set while non-preemptible. This can cause tasks to run unprotected when the should not, and can cause the user copy to be improperly blocked when scheduling back to it. Fix this by avoiding the explicit resched for preempt kernels by generating an interrupt to reschedule the context if need_resched() got set. Reported-by: Samuel Holland Signed-off-by: Nicholas Piggin Tested-by: Guenter Roeck Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221013151647.1857994-3-npiggin@gmail.com --- arch/powerpc/lib/vmx-helper.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c index f76a50291fd75..d491da8d18389 100644 --- a/arch/powerpc/lib/vmx-helper.c +++ b/arch/powerpc/lib/vmx-helper.c @@ -36,7 +36,17 @@ int exit_vmx_usercopy(void) { disable_kernel_altivec(); pagefault_enable(); - preempt_enable(); + preempt_enable_no_resched(); + /* + * Must never explicitly call schedule (including preempt_enable()) + * while in a kuap-unlocked user copy, because the AMR register will + * not be saved and restored across context switch. However preempt + * kernels need to be preempted as soon as possible if need_resched is + * set and we are preemptible. The hack here is to schedule a + * decrementer to fire here and reschedule for us if necessary. + */ + if (IS_ENABLED(CONFIG_PREEMPT) && need_resched()) + set_dec(1); return 0; } -- GitLab From e59b3399fde5e173b026d4952b215043e77b4521 Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 13:07:28 +1000 Subject: [PATCH 0191/1927] KVM: PPC: BookS PR-KVM and BookE do not support context tracking The context tracking code in PR-KVM and BookE implementations is not complete, and can cause host crashes if context tracking is enabled. Make these implementations depend on !CONTEXT_TRACKING_USER. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221014030729.2077151-2-npiggin@gmail.com --- arch/powerpc/kvm/Kconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/powerpc/kvm/Kconfig b/arch/powerpc/kvm/Kconfig index 61cdd782d3c5e..a9f57dad6d916 100644 --- a/arch/powerpc/kvm/Kconfig +++ b/arch/powerpc/kvm/Kconfig @@ -51,6 +51,7 @@ config KVM_BOOK3S_HV_POSSIBLE config KVM_BOOK3S_32 tristate "KVM support for PowerPC book3s_32 processors" depends on PPC_BOOK3S_32 && !SMP && !PTE_64BIT + depends on !CONTEXT_TRACKING_USER select KVM select KVM_BOOK3S_32_HANDLER select KVM_BOOK3S_PR_POSSIBLE @@ -105,6 +106,7 @@ config KVM_BOOK3S_64_HV config KVM_BOOK3S_64_PR tristate "KVM support without using hypervisor mode in host" depends on KVM_BOOK3S_64 + depends on !CONTEXT_TRACKING_USER select KVM_BOOK3S_PR_POSSIBLE help Support running guest kernels in virtual machines on processors @@ -190,6 +192,7 @@ config KVM_EXIT_TIMING config KVM_E500V2 bool "KVM support for PowerPC E500v2 processors" depends on PPC_E500 && !PPC_E500MC + depends on !CONTEXT_TRACKING_USER select KVM select KVM_MMIO select MMU_NOTIFIER @@ -205,6 +208,7 @@ config KVM_E500V2 config KVM_E500MC bool "KVM support for PowerPC E500MC/E5500/E6500 processors" depends on PPC_E500MC + depends on !CONTEXT_TRACKING_USER select KVM select KVM_MMIO select KVM_BOOKE_HV -- GitLab From a073672eb09670540e95a2a4aa1c46f5da74159f Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 14 Oct 2022 13:07:29 +1000 Subject: [PATCH 0192/1927] powerpc/64/interrupt: Prevent NMI PMI causing a dangerous warning NMI PMIs really should not return using the normal interrupt_return function. If such a PMI hits in code returning to user with the context switched to user mode, this warning can fire. This was enough to cause crashes when reproducing on 64s, because another perf interrupt would hit while reporting bug, and that would cause another bug, and so on until smashing the stack. Work around that particular crash for now by just disabling that context warning for PMIs. This is a hack and not a complete fix, there could be other such problems lurking in corners. But it does fix the known crash. Signed-off-by: Nicholas Piggin Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221014030729.2077151-3-npiggin@gmail.com --- arch/powerpc/kernel/exceptions-64e.S | 7 +++++++ arch/powerpc/kernel/interrupt.c | 12 +++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 930e360990152..2f68fb2ee4fc3 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -813,6 +813,13 @@ kernel_dbg_exc: EXCEPTION_COMMON(0x260) CHECK_NAPPING() addi r3,r1,STACK_FRAME_OVERHEAD + /* + * XXX: Returning from performance_monitor_exception taken as a + * soft-NMI (Linux irqs disabled) may be risky to use interrupt_return + * and could cause bugs in return or elsewhere. That case should just + * restore registers and return. There is a workaround for one known + * problem in interrupt_exit_kernel_prepare(). + */ bl performance_monitor_exception b interrupt_return diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index f9db0a172401a..7bc93367de68e 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -374,10 +374,16 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) if (regs_is_unrecoverable(regs)) unrecoverable_exception(regs); /* - * CT_WARN_ON comes here via program_check_exception, - * so avoid recursion. + * CT_WARN_ON comes here via program_check_exception, so avoid + * recursion. + * + * Skip the assertion on PMIs to work around a problem caused by NMI + * PMIs incorrectly taking this interrupt return path, it's possible + * for this to hit after interrupt exit to user switches context to + * user. See also the comment in the performance monitor handler in + * exceptions-64e/s.S */ - if (TRAP(regs) != INTERRUPT_PROGRAM) + if (TRAP(regs) != INTERRUPT_PROGRAM && TRAP(regs) != INTERRUPT_PERFMON) CT_WARN_ON(ct_state() == CONTEXT_USER); kuap = kuap_get_and_assert_locked(); -- GitLab From dc398a084d459f065658855454e09f2778f8c5cc Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Fri, 7 Oct 2022 00:04:11 +1000 Subject: [PATCH 0193/1927] powerpc/64s/interrupt: Perf NMI should not take normal exit path NMI interrupts should exit with EXCEPTION_RESTORE_REGS not with interrupt_return_srr, which is what the perf NMI handler currently does. This breaks if a PMI hits after interrupt_exit_user_prepare_main() has switched the context tracking to user mode, then the CT_WARN_ON() in interrupt_exit_kernel_prepare() fires because it returns to kernel with context set to user. This could possibly be solved by soft-disabling PMIs in the exit path, but that reduces our ability to profile that code. The warning could be removed, but it's potentially useful. All other NMIs and soft-NMIs return using EXCEPTION_RESTORE_REGS, so this makes perf interrupts consistent with that and seems like the best fix. Signed-off-by: Nicholas Piggin [mpe: Squash in fixups from Nick] Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221006140413.126443-3-npiggin@gmail.com --- arch/powerpc/kernel/exceptions-64s.S | 14 +++++++++++++- arch/powerpc/kernel/interrupt.c | 14 ++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 5381a43e50fef..651c36b056bde 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -2357,9 +2357,21 @@ EXC_VIRT_END(performance_monitor, 0x4f00, 0x20) EXC_COMMON_BEGIN(performance_monitor_common) GEN_COMMON performance_monitor addi r3,r1,STACK_FRAME_OVERHEAD - bl performance_monitor_exception + lbz r4,PACAIRQSOFTMASK(r13) + cmpdi r4,IRQS_ENABLED + bne 1f + bl performance_monitor_exception_async b interrupt_return_srr +1: + bl performance_monitor_exception_nmi + /* Clear MSR_RI before setting SRR0 and SRR1. */ + li r9,0 + mtmsrd r9,1 + kuap_kernel_restore r9, r10 + + EXCEPTION_RESTORE_REGS hsrr=0 + RFI_TO_KERNEL /** * Interrupt 0xf20 - Vector Unavailable Interrupt. diff --git a/arch/powerpc/kernel/interrupt.c b/arch/powerpc/kernel/interrupt.c index 7bc93367de68e..fc6631a805272 100644 --- a/arch/powerpc/kernel/interrupt.c +++ b/arch/powerpc/kernel/interrupt.c @@ -377,13 +377,15 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs) * CT_WARN_ON comes here via program_check_exception, so avoid * recursion. * - * Skip the assertion on PMIs to work around a problem caused by NMI - * PMIs incorrectly taking this interrupt return path, it's possible - * for this to hit after interrupt exit to user switches context to - * user. See also the comment in the performance monitor handler in - * exceptions-64e/s.S + * Skip the assertion on PMIs on 64e to work around a problem caused + * by NMI PMIs incorrectly taking this interrupt return path, it's + * possible for this to hit after interrupt exit to user switches + * context to user. See also the comment in the performance monitor + * handler in exceptions-64e.S */ - if (TRAP(regs) != INTERRUPT_PROGRAM && TRAP(regs) != INTERRUPT_PERFMON) + if (!IS_ENABLED(CONFIG_PPC_BOOK3E_64) && + TRAP(regs) != INTERRUPT_PROGRAM && + TRAP(regs) != INTERRUPT_PERFMON) CT_WARN_ON(ct_state() == CONTEXT_USER); kuap = kuap_get_and_assert_locked(); -- GitLab From 491a4ccd8a0258392900c80c6b2b622c7115fc23 Mon Sep 17 00:00:00 2001 From: Stefan Binding Date: Tue, 18 Oct 2022 13:15:06 +0100 Subject: [PATCH 0194/1927] ALSA: hda/realtek: Add quirk for ASUS Zenbook using CS35L41 This Asus Zenbook laptop use Realtek HDA codec combined with 2xCS35L41 Amplifiers using SPI with External Boost. Signed-off-by: Stefan Binding Cc: Link: https://lore.kernel.org/r/20221018121506.2561397-1-sbinding@opensource.cirrus.com Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_realtek.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 7c177426bf303..79acd2a2caf20 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -9395,6 +9395,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1ccd, "ASUS X555UB", ALC256_FIXUP_ASUS_MIC), SND_PCI_QUIRK(0x1043, 0x1d42, "ASUS Zephyrus G14 2022", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1d4e, "ASUS TM420", ALC256_FIXUP_ASUS_HPE), + SND_PCI_QUIRK(0x1043, 0x1e02, "ASUS UX3402", ALC245_FIXUP_CS35L41_SPI_2), SND_PCI_QUIRK(0x1043, 0x1e11, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA502), SND_PCI_QUIRK(0x1043, 0x1e51, "ASUS Zephyrus M15", ALC294_FIXUP_ASUS_GU502_PINS), SND_PCI_QUIRK(0x1043, 0x1e5e, "ASUS ROG Strix G513", ALC294_FIXUP_ASUS_G513_PINS), -- GitLab From 0782b66ed2fbb035dda76111df0954515e417b24 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Tue, 18 Oct 2022 18:09:31 +0200 Subject: [PATCH 0195/1927] rtc: cmos: Fix wake alarm breakage Commit 4919d3eb2ec0 ("rtc: cmos: Fix event handler registration ordering issue") overlooked the fact that cmos_do_probe() depended on the preparations carried out by cmos_wake_setup() and the wake alarm stopped working after the ordering of them had been changed. Address this by partially reverting commit 4919d3eb2ec0 so that cmos_wake_setup() is called before cmos_do_probe() again and moving the rtc_wake_setup() invocation from cmos_wake_setup() directly to the callers of cmos_do_probe() where it will happen after a successful completion of the latter. Fixes: 4919d3eb2ec0 ("rtc: cmos: Fix event handler registration ordering issue") Reported-by: Zhang Rui Reported-by: Todd Brandt Signed-off-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/5887691.lOV4Wx5bFT@kreacher Signed-off-by: Alexandre Belloni --- drivers/rtc/rtc-cmos.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index 610413b4e9ca7..01fb31f8e5341 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -1233,6 +1233,9 @@ static u32 rtc_handler(void *context) static inline void rtc_wake_setup(struct device *dev) { + if (acpi_disabled) + return; + acpi_install_fixed_event_handler(ACPI_EVENT_RTC, rtc_handler, dev); /* * After the RTC handler is installed, the Fixed_RTC event should @@ -1286,7 +1289,6 @@ static void cmos_wake_setup(struct device *dev) use_acpi_alarm_quirks(); - rtc_wake_setup(dev); acpi_rtc_info.wake_on = rtc_wake_on; acpi_rtc_info.wake_off = rtc_wake_off; @@ -1354,6 +1356,8 @@ static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) { int irq, ret; + cmos_wake_setup(&pnp->dev); + if (pnp_port_start(pnp, 0) == 0x70 && !pnp_irq_valid(pnp, 0)) { irq = 0; #ifdef CONFIG_X86 @@ -1372,7 +1376,7 @@ static int cmos_pnp_probe(struct pnp_dev *pnp, const struct pnp_device_id *id) if (ret) return ret; - cmos_wake_setup(&pnp->dev); + rtc_wake_setup(&pnp->dev); return 0; } @@ -1461,6 +1465,7 @@ static int __init cmos_platform_probe(struct platform_device *pdev) int irq, ret; cmos_of_init(pdev); + cmos_wake_setup(&pdev->dev); if (RTC_IOMAPPED) resource = platform_get_resource(pdev, IORESOURCE_IO, 0); @@ -1474,7 +1479,7 @@ static int __init cmos_platform_probe(struct platform_device *pdev) if (ret) return ret; - cmos_wake_setup(&pdev->dev); + rtc_wake_setup(&pdev->dev); return 0; } -- GitLab From 41deb2db64997d01110faaf763bd911d490dfde7 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Mon, 17 Oct 2022 15:40:54 -0500 Subject: [PATCH 0196/1927] ASoC: Intel: sof_sdw: add quirk variant for LAPBC710 NUC15 Some NUC15 LAPBC710 devices don't expose the same DMI information as the Intel reference, add additional entry in the match table. BugLink: https://github.com/thesofproject/linux/issues/3885 Signed-off-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Reviewed-by: Bard Liao Link: https://lore.kernel.org/r/20221017204054.207512-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/intel/boards/sof_sdw.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index a49bfaab6b21f..5223089c3426f 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -202,6 +202,17 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { SOF_SDW_PCH_DMIC | RT711_JD1), }, + { + /* NUC15 LAPBC710 skews */ + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Intel Corporation"), + DMI_MATCH(DMI_BOARD_NAME, "LAPBC710"), + }, + .driver_data = (void *)(SOF_SDW_TGL_HDMI | + SOF_SDW_PCH_DMIC | + RT711_JD1), + }, /* TigerLake-SDCA devices */ { .callback = sof_sdw_quirk_cb, -- GitLab From 73189c064e11137c8b78a825800a374924ebb7b7 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Mon, 17 Oct 2022 15:40:04 -0500 Subject: [PATCH 0197/1927] ASoC: SOF: Intel: pci-mtl: fix firmware name Initial IPC4 tests used the same conventions as previous reference closed-source firmware, but for MeteorLake the convention is the same as previous SOF releases (sof-.ri). Only the prefix changes to avoid confusions between IPC types. This change has no impact on users since the firmware has not yet been released. Fixes: 064520e8aeaa2 ("ASoC: SOF: Intel: Add support for MeteorLake (MTL)") Signed-off-by: Pierre-Louis Bossart Reviewed-by: Chao Song Reviewed-by: Kai Vehmanen Reviewed-by: Ranjani Sridharan Link: https://lore.kernel.org/r/20221017204004.207446-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/sof/intel/pci-mtl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/sof/intel/pci-mtl.c b/sound/soc/sof/intel/pci-mtl.c index 899b00d53d64f..9f39da984e9fa 100644 --- a/sound/soc/sof/intel/pci-mtl.c +++ b/sound/soc/sof/intel/pci-mtl.c @@ -38,7 +38,7 @@ static const struct sof_dev_desc mtl_desc = { [SOF_INTEL_IPC4] = "intel/sof-ace-tplg", }, .default_fw_filename = { - [SOF_INTEL_IPC4] = "dsp_basefw.bin", + [SOF_INTEL_IPC4] = "sof-mtl.ri", }, .nocodec_tplg_filename = "sof-mtl-nocodec.tplg", .ops = &sof_mtl_ops, -- GitLab From 7e2c58320ed59c9326e82cf9c00090f6a912fb64 Mon Sep 17 00:00:00 2001 From: Yifan Zha Date: Wed, 7 Sep 2022 14:13:02 +0800 Subject: [PATCH 0198/1927] drm/amdgpu: Program GC registers through RLCG interface in gfx_v11/gmc_v11 [Why] L1 blocks most of GC registers accessing by MMIO. [How] Use RLCG interface to program GC registers under SRIOV VF in full access time. Signed-off-by: Yifan Zha Reviewed-by: Hawking Zhang Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c | 2 +- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 2 +- drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 18 +++++++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c index 0b0a72ca56956..7e80caa05060b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v11.c @@ -111,7 +111,7 @@ static int init_interrupts_v11(struct amdgpu_device *adev, uint32_t pipe_id) lock_srbm(adev, mec, pipe, 0, 0); - WREG32(SOC15_REG_OFFSET(GC, 0, regCPC_INT_CNTL), + WREG32_SOC15(GC, 0, regCPC_INT_CNTL, CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK | CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 73106f53246d5..e3842dc100d66 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -1571,7 +1571,7 @@ static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev) WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); /* Enable trap for each kfd vmid. */ - data = RREG32(SOC15_REG_OFFSET(GC, 0, regSPI_GDBG_PER_VMID_CNTL)); + data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); } soc21_grbm_select(adev, 0, 0, 0, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 846ccb6cf07d9..66dfb574cc7d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -186,6 +186,10 @@ static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, /* Use register 17 for GART */ const unsigned eng = 17; unsigned int i; + unsigned char hub_ip = 0; + + hub_ip = (vmhub == AMDGPU_GFXHUB_0) ? + GC_HWIP : MMHUB_HWIP; spin_lock(&adev->gmc.invalidate_lock); /* @@ -199,8 +203,8 @@ static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, if (use_semaphore) { for (i = 0; i < adev->usec_timeout; i++) { /* a read return value of 1 means semaphore acuqire */ - tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + - hub->eng_distance * eng); + tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + + hub->eng_distance * eng, hub_ip); if (tmp & 0x1) break; udelay(1); @@ -210,12 +214,12 @@ static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); } - WREG32_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req); + WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip); /* Wait for ACK with a delay.*/ for (i = 0; i < adev->usec_timeout; i++) { - tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + - hub->eng_distance * eng); + tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack + + hub->eng_distance * eng, hub_ip); tmp &= 1 << vmid; if (tmp) break; @@ -229,8 +233,8 @@ static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, * add semaphore release after invalidation, * write with 0 means semaphore release */ - WREG32_NO_KIQ(hub->vm_inv_eng0_sem + - hub->eng_distance * eng, 0); + WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + + hub->eng_distance * eng, 0, hub_ip); /* Issue additional private vm invalidation to MMHUB */ if ((vmhub != AMDGPU_GFXHUB_0) && -- GitLab From c520ba3fad335c76f4fad2ab49485eb9edf3de54 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Fri, 14 Oct 2022 15:19:51 +0800 Subject: [PATCH 0199/1927] drm/amd/pm: enable thermal alert on smu_v13_0_10 enable thermal alert on smu_v13_0_10 Signed-off-by: Kenneth Feng Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 3d436e7f6e952..4fe75dd2b329d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1312,12 +1312,10 @@ static int smu_smc_hw_setup(struct smu_context *smu) return ret; } - if (adev->ip_versions[MP1_HWIP][0] != IP_VERSION(13, 0, 10)) { - ret = smu_enable_thermal_alert(smu); - if (ret) { - dev_err(adev->dev, "Failed to enable thermal alert!\n"); - return ret; - } + ret = smu_enable_thermal_alert(smu); + if (ret) { + dev_err(adev->dev, "Failed to enable thermal alert!\n"); + return ret; } ret = smu_notify_display_change(smu); -- GitLab From c4dfad81e444ba27ea9944c9af061a789ab91201 Mon Sep 17 00:00:00 2001 From: YuBiao Wang Date: Thu, 13 Oct 2022 11:31:55 +0800 Subject: [PATCH 0200/1927] drm/amdgpu: dequeue mes scheduler during fini [Why] If mes is not dequeued during fini, mes will be in an uncleaned state during reload, then mes couldn't receive some commands which leads to reload failure. [How] Perform MES dequeue via MMIO after all the unmap jobs are done by mes and before kiq fini. v2: Move the dequeue operation inside kiq_hw_fini. Signed-off-by: YuBiao Wang Reviewed-by: Jack Xiao Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 42 ++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index 133804e6018a7..fef7d020bc5f3 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -1156,6 +1156,42 @@ static int mes_v11_0_sw_fini(void *handle) return 0; } +static void mes_v11_0_kiq_dequeue_sched(struct amdgpu_device *adev) +{ + uint32_t data; + int i; + + mutex_lock(&adev->srbm_mutex); + soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0); + + /* disable the queue if it's active */ + if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { + WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); + for (i = 0; i < adev->usec_timeout; i++) { + if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) + break; + udelay(1); + } + } + data = RREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL); + data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL, + DOORBELL_EN, 0); + data = REG_SET_FIELD(data, CP_HQD_PQ_DOORBELL_CONTROL, + DOORBELL_HIT, 1); + WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, data); + + WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 0); + + WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 0); + WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0); + WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0); + + soc21_grbm_select(adev, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); + + adev->mes.ring.sched.ready = false; +} + static void mes_v11_0_kiq_setting(struct amdgpu_ring *ring) { uint32_t tmp; @@ -1207,6 +1243,9 @@ static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev) static int mes_v11_0_kiq_hw_fini(struct amdgpu_device *adev) { + if (adev->mes.ring.sched.ready) + mes_v11_0_kiq_dequeue_sched(adev); + mes_v11_0_enable(adev, false); return 0; } @@ -1262,9 +1301,6 @@ static int mes_v11_0_hw_init(void *handle) static int mes_v11_0_hw_fini(void *handle) { - struct amdgpu_device *adev = (struct amdgpu_device *)handle; - - adev->mes.ring.sched.ready = false; return 0; } -- GitLab From df768a9770271b0d9faab25f42dfc7bdec87b21c Mon Sep 17 00:00:00 2001 From: Arunpravin Paneer Selvam Date: Tue, 18 Oct 2022 07:08:38 -0700 Subject: [PATCH 0201/1927] drm/amdgpu: Fix for BO move issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A user reported a bug on CAPE VERDE system where uvd_v3_1 IP component failed to initialize as there is an issue with BO move code from one memory to other. In function amdgpu_mem_visible() called by amdgpu_bo_move(), when there are no blocks to compare or if we have a single block then break the loop. Fixes: 312b4dc11d4f ("drm/amdgpu: Fix VRAM BO swap issue") Signed-off-by: Arunpravin Paneer Selvam Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 8782fc6bcd2c1..0de9bc12793d5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -439,6 +439,9 @@ static bool amdgpu_mem_visible(struct amdgpu_device *adev, while (cursor.remaining) { amdgpu_res_next(&cursor, cursor.size); + if (!cursor.remaining) + break; + /* ttm_resource_ioremap only supports contiguous memory */ if (end != cursor.start) return false; -- GitLab From 8f8033d5663b18e6efb33feb61f2287a04605ab5 Mon Sep 17 00:00:00 2001 From: Rafael Mendonca Date: Mon, 17 Oct 2022 21:53:57 -0300 Subject: [PATCH 0202/1927] drm/amdgpu/powerplay/psm: Fix memory leak in power state init Commit 902bc65de0b3 ("drm/amdgpu/powerplay/psm: return an error in power state init") made the power state init function return early in case of failure to get an entry from the powerplay table, but it missed to clean up the allocated memory for the current power state before returning. Fixes: 902bc65de0b3 ("drm/amdgpu/powerplay/psm: return an error in power state init") Reviewed-by: Evan Quan Signed-off-by: Rafael Mendonca Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c index 67d7da0b6fed5..1d829402cd2e2 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/pp_psm.c @@ -75,8 +75,10 @@ int psm_init_power_state_table(struct pp_hwmgr *hwmgr) for (i = 0; i < table_entries; i++) { result = hwmgr->hwmgr_func->get_pp_table_entry(hwmgr, i, state); if (result) { + kfree(hwmgr->current_ps); kfree(hwmgr->request_ps); kfree(hwmgr->ps); + hwmgr->current_ps = NULL; hwmgr->request_ps = NULL; hwmgr->ps = NULL; return -EINVAL; -- GitLab From 03cab65a07e083b6c1010fbc8f9b817e9aca75d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Ca=C3=B1uelo?= Date: Mon, 10 Oct 2022 08:37:02 +0200 Subject: [PATCH 0203/1927] selftests/futex: fix build for clang MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't use the test-specific header files as source files to force a target dependency, as clang will complain if more than one source file is used for a compile command with a single '-o' flag. Use the proper Makefile variables instead as defined in tools/testing/selftests/lib.mk. Signed-off-by: Ricardo Cañuelo Reviewed-by: André Almeida Signed-off-by: Shuah Khan --- tools/testing/selftests/futex/functional/Makefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile index 7321490116925..5a0e0df8de9b3 100644 --- a/tools/testing/selftests/futex/functional/Makefile +++ b/tools/testing/selftests/futex/functional/Makefile @@ -3,11 +3,11 @@ INCLUDES := -I../include -I../../ -I../../../../../usr/include/ CFLAGS := $(CFLAGS) -g -O2 -Wall -D_GNU_SOURCE -pthread $(INCLUDES) $(KHDR_INCLUDES) LDLIBS := -lpthread -lrt -HEADERS := \ +LOCAL_HDRS := \ ../include/futextest.h \ ../include/atomic.h \ ../include/logging.h -TEST_GEN_FILES := \ +TEST_GEN_PROGS := \ futex_wait_timeout \ futex_wait_wouldblock \ futex_requeue_pi \ @@ -24,5 +24,3 @@ TEST_PROGS := run.sh top_srcdir = ../../../../.. DEFAULT_INSTALL_HDR_PATH := 1 include ../../lib.mk - -$(TEST_GEN_FILES): $(HEADERS) -- GitLab From beb7d862ed4ac6aa14625418970f22a7d55b8615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Ca=C3=B1uelo?= Date: Mon, 10 Oct 2022 08:38:11 +0200 Subject: [PATCH 0204/1927] selftests/intel_pstate: fix build for ARCH=x86_64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle the scenario where the build is launched with the ARCH envvar defined as x86_64. Signed-off-by: Ricardo Cañuelo Signed-off-by: Shuah Khan --- tools/testing/selftests/intel_pstate/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/intel_pstate/Makefile b/tools/testing/selftests/intel_pstate/Makefile index 39f0fa2a8fd63..05d66ef50c977 100644 --- a/tools/testing/selftests/intel_pstate/Makefile +++ b/tools/testing/selftests/intel_pstate/Makefile @@ -2,10 +2,10 @@ CFLAGS := $(CFLAGS) -Wall -D_GNU_SOURCE LDLIBS += -lm -uname_M := $(shell uname -m 2>/dev/null || echo not) -ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/) +ARCH ?= $(shell uname -m 2>/dev/null || echo not) +ARCH_PROCESSED := $(shell echo $(ARCH) | sed -e s/i.86/x86/ -e s/x86_64/x86/) -ifeq (x86,$(ARCH)) +ifeq (x86,$(ARCH_PROCESSED)) TEST_GEN_FILES := msr aperf endif -- GitLab From 2a8e366b23fea29a5308f71ba49555e3c8c664f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Ca=C3=B1uelo?= Date: Mon, 10 Oct 2022 08:39:27 +0200 Subject: [PATCH 0205/1927] selftests/kexec: fix build for ARCH=x86_64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handle the scenario where the build is launched with the ARCH envvar defined as x86_64. Signed-off-by: Ricardo Cañuelo Signed-off-by: Shuah Khan --- tools/testing/selftests/kexec/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kexec/Makefile b/tools/testing/selftests/kexec/Makefile index 806a150648c36..67fe7a46cb624 100644 --- a/tools/testing/selftests/kexec/Makefile +++ b/tools/testing/selftests/kexec/Makefile @@ -1,10 +1,10 @@ # SPDX-License-Identifier: GPL-2.0-only # Makefile for kexec tests -uname_M := $(shell uname -m 2>/dev/null || echo not) -ARCH ?= $(shell echo $(uname_M) | sed -e s/i.86/x86/ -e s/x86_64/x86/) +ARCH ?= $(shell uname -m 2>/dev/null || echo not) +ARCH_PROCESSED := $(shell echo $(ARCH) | sed -e s/i.86/x86/ -e s/x86_64/x86/) -ifeq ($(ARCH),$(filter $(ARCH),x86 ppc64le)) +ifeq ($(ARCH_PROCESSED),$(filter $(ARCH_PROCESSED),x86 ppc64le)) TEST_PROGS := test_kexec_load.sh test_kexec_file_load.sh TEST_FILES := kexec_common_lib.sh -- GitLab From b4dd2e3758709aa8a2abd1ac34c56bd09b980039 Mon Sep 17 00:00:00 2001 From: Yong Zhi Date: Mon, 17 Oct 2022 15:57:28 -0500 Subject: [PATCH 0206/1927] ASoC: Intel: sof_rt5682: Add quirk for Rex board Add mtl_mx98357_rt5682 driver data for Chrome Rex board support. Reviewed-by: Bard Liao Reviewed-by: Curtis Malainey Signed-off-by: Yong Zhi Signed-off-by: Pierre-Louis Bossart Link: https://lore.kernel.org/r/20221017205728.210813-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/intel/boards/sof_rt5682.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sound/soc/intel/boards/sof_rt5682.c b/sound/soc/intel/boards/sof_rt5682.c index 2d0986824b3d7..2358be208c1fd 100644 --- a/sound/soc/intel/boards/sof_rt5682.c +++ b/sound/soc/intel/boards/sof_rt5682.c @@ -223,6 +223,18 @@ static const struct dmi_system_id sof_rt5682_quirk_table[] = { SOF_RT5682_SSP_AMP(2) | SOF_RT5682_NUM_HDMIDEV(4)), }, + { + .callback = sof_rt5682_quirk_cb, + .matches = { + DMI_MATCH(DMI_PRODUCT_FAMILY, "Google_Rex"), + }, + .driver_data = (void *)(SOF_RT5682_MCLK_EN | + SOF_RT5682_SSP_CODEC(2) | + SOF_SPEAKER_AMP_PRESENT | + SOF_RT5682_SSP_AMP(0) | + SOF_RT5682_NUM_HDMIDEV(4) + ), + }, {} }; -- GitLab From af6514f2f3828dc39c96cd4686ef5c9d8368626f Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 18 Oct 2022 15:13:32 +0300 Subject: [PATCH 0207/1927] ASoC: SOF: ipc4-mtrace: protect per-core nodes against multiple open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add protection against multiple open of the mtrace/coreN debugfs nodes. This is not supported in the implementation, and this will show up as unexpected behaviour of the interface, and potential use of already freed memory. Fixes: f4ea22f7aa75 ("ASoC: SOF: ipc4: Add support for mtrace log extraction") Signed-off-by: Kai Vehmanen Reviewed-by: Péter Ujfalusi Reviewed-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Link: https://lore.kernel.org/r/20221018121332.20802-1-peter.ujfalusi@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/sof/ipc4-mtrace.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/sound/soc/sof/ipc4-mtrace.c b/sound/soc/sof/ipc4-mtrace.c index 9c7080041d082..70dea8ae706e9 100644 --- a/sound/soc/sof/ipc4-mtrace.c +++ b/sound/soc/sof/ipc4-mtrace.c @@ -108,6 +108,7 @@ struct sof_mtrace_core_data { int id; u32 slot_offset; void *log_buffer; + struct mutex buffer_lock; /* for log_buffer alloc/free */ u32 host_read_ptr; u32 dsp_write_ptr; /* pos update IPC arrived before the slot offset is known, queried */ @@ -128,14 +129,22 @@ static int sof_ipc4_mtrace_dfs_open(struct inode *inode, struct file *file) struct sof_mtrace_core_data *core_data = inode->i_private; int ret; + mutex_lock(&core_data->buffer_lock); + + if (core_data->log_buffer) { + ret = -EBUSY; + goto out; + } + ret = debugfs_file_get(file->f_path.dentry); if (unlikely(ret)) - return ret; + goto out; core_data->log_buffer = kmalloc(SOF_MTRACE_SLOT_SIZE, GFP_KERNEL); if (!core_data->log_buffer) { debugfs_file_put(file->f_path.dentry); - return -ENOMEM; + ret = -ENOMEM; + goto out; } ret = simple_open(inode, file); @@ -144,6 +153,9 @@ static int sof_ipc4_mtrace_dfs_open(struct inode *inode, struct file *file) debugfs_file_put(file->f_path.dentry); } +out: + mutex_unlock(&core_data->buffer_lock); + return ret; } @@ -280,7 +292,10 @@ static int sof_ipc4_mtrace_dfs_release(struct inode *inode, struct file *file) debugfs_file_put(file->f_path.dentry); + mutex_lock(&core_data->buffer_lock); kfree(core_data->log_buffer); + core_data->log_buffer = NULL; + mutex_unlock(&core_data->buffer_lock); return 0; } @@ -563,6 +578,7 @@ static int ipc4_mtrace_init(struct snd_sof_dev *sdev) struct sof_mtrace_core_data *core_data = &priv->cores[i]; init_waitqueue_head(&core_data->trace_sleep); + mutex_init(&core_data->buffer_lock); core_data->sdev = sdev; core_data->id = i; } -- GitLab From eb6789b0c3424f84e8441c4796083db2f095c391 Mon Sep 17 00:00:00 2001 From: Zhao Gongyi Date: Tue, 11 Oct 2022 09:39:26 +0800 Subject: [PATCH 0208/1927] selftests/memory-hotplug: Remove the redundant warning information Remove the redundant warning information of online_all_offline_memory() since there is a warning in online_memory_expect_success(). Signed-off-by: Zhao Gongyi Reviewed-by: David Hildenbrand Signed-off-by: Shuah Khan --- tools/testing/selftests/memory-hotplug/mem-on-off-test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh b/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh index 74ee5067a8ce2..611be86eaf3de 100755 --- a/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh +++ b/tools/testing/selftests/memory-hotplug/mem-on-off-test.sh @@ -138,7 +138,6 @@ online_all_offline_memory() { for memory in `hotpluggable_offline_memory`; do if ! online_memory_expect_success $memory; then - echo "$FUNCNAME $memory: unexpected fail" >&2 retval=1 fi done -- GitLab From cb05c81ada76a30a25a5f79b249375e33473af33 Mon Sep 17 00:00:00 2001 From: Sven Schnelle Date: Mon, 10 Oct 2022 09:42:07 +0200 Subject: [PATCH 0209/1927] selftests/ftrace: fix dynamic_events dependency check commit 95c104c378dc ("tracing: Auto generate event name when creating a group of events") changed the syntax in the ftrace README file which is used by the selftests to check what features are support. Adjust the string to make test_duplicates.tc and trigger-synthetic-eprobe.tc work again. Fixes: 95c104c378dc ("tracing: Auto generate event name when creating a group of events") Signed-off-by: Sven Schnelle Acked-by: Steven Rostedt (Google) Acked-by: Masami Hiramatsu (Google) Signed-off-by: Shuah Khan --- .../testing/selftests/ftrace/test.d/dynevent/test_duplicates.tc | 2 +- .../test.d/trigger/inter-event/trigger-synthetic-eprobe.tc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/test_duplicates.tc b/tools/testing/selftests/ftrace/test.d/dynevent/test_duplicates.tc index db522577ff787..d3a79da215c8b 100644 --- a/tools/testing/selftests/ftrace/test.d/dynevent/test_duplicates.tc +++ b/tools/testing/selftests/ftrace/test.d/dynevent/test_duplicates.tc @@ -1,7 +1,7 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 # description: Generic dynamic event - check if duplicate events are caught -# requires: dynamic_events "e[:[/]] . []":README +# requires: dynamic_events "e[:[/][]] . []":README echo 0 > events/enable diff --git a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-eprobe.tc b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-eprobe.tc index 914fe2e5d0309..6461c375694f0 100644 --- a/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-eprobe.tc +++ b/tools/testing/selftests/ftrace/test.d/trigger/inter-event/trigger-synthetic-eprobe.tc @@ -1,7 +1,7 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 # description: event trigger - test inter-event histogram trigger eprobe on synthetic event -# requires: dynamic_events synthetic_events events/syscalls/sys_enter_openat/hist "e[:[/]] . []":README +# requires: dynamic_events synthetic_events events/syscalls/sys_enter_openat/hist "e[:[/][]] . []":README echo 0 > events/enable -- GitLab From db4e955ae333567dea02822624106c0b96a2f84f Mon Sep 17 00:00:00 2001 From: Alexandre Belloni Date: Tue, 18 Oct 2022 22:35:11 +0200 Subject: [PATCH 0210/1927] rtc: cmos: fix build on non-ACPI platforms Now that rtc_wake_setup is called outside of cmos_wake_setup, it also need to be defined on non-ACPI platforms. Reported-by: kernel test robot Link: https://lore.kernel.org/r/20221018203512.2532407-1-alexandre.belloni@bootlin.com Signed-off-by: Alexandre Belloni --- drivers/rtc/rtc-cmos.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index 01fb31f8e5341..58cc2bae2f8a0 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -1346,6 +1346,9 @@ static void cmos_check_acpi_rtc_status(struct device *dev, { } +static void rtc_wake_setup(struct device *dev) +{ +} #endif #ifdef CONFIG_PNP -- GitLab From 618887768bb71f0a475334fa5a4fba7dc98d7ab5 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 14 Oct 2022 12:37:25 +0300 Subject: [PATCH 0211/1927] kunit: update NULL vs IS_ERR() tests The alloc_string_stream() functions were changed from returning NULL on error to returning error pointers so these caller needs to be updated as well. Fixes: 78b1c6584fce ("kunit: string-stream: Simplify resource use") Signed-off-by: Dan Carpenter Reviewed-by: Daniel Latypov Reviewed-by: David Gow Signed-off-by: Shuah Khan --- lib/kunit/string-stream.c | 4 ++-- lib/kunit/test.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/kunit/string-stream.c b/lib/kunit/string-stream.c index f5ae79c374003..a608746020a99 100644 --- a/lib/kunit/string-stream.c +++ b/lib/kunit/string-stream.c @@ -56,8 +56,8 @@ int string_stream_vadd(struct string_stream *stream, frag_container = alloc_string_stream_fragment(stream->test, len, stream->gfp); - if (!frag_container) - return -ENOMEM; + if (IS_ERR(frag_container)) + return PTR_ERR(frag_container); len = vsnprintf(frag_container->fragment, len, fmt, args); spin_lock(&stream->lock); diff --git a/lib/kunit/test.c b/lib/kunit/test.c index 90640a43cf623..2a6992fe7c3e4 100644 --- a/lib/kunit/test.c +++ b/lib/kunit/test.c @@ -265,7 +265,7 @@ static void kunit_fail(struct kunit *test, const struct kunit_loc *loc, kunit_set_failure(test); stream = alloc_string_stream(test, GFP_KERNEL); - if (!stream) { + if (IS_ERR(stream)) { WARN(true, "Could not allocate stream to print failed assertion in %s:%d\n", loc->file, -- GitLab From 97cf79677ecb50a38517253ae2fd705849a7e51a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Sun, 16 Oct 2022 17:54:40 -0700 Subject: [PATCH 0212/1927] xfs: avoid a UAF when log intent item recovery fails KASAN reported a UAF bug when I was running xfs/235: BUG: KASAN: use-after-free in xlog_recover_process_intents+0xa77/0xae0 [xfs] Read of size 8 at addr ffff88804391b360 by task mount/5680 CPU: 2 PID: 5680 Comm: mount Not tainted 6.0.0-xfsx #6.0.0 77e7b52a4943a975441e5ac90a5ad7748b7867f6 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.15.0-1 04/01/2014 Call Trace: dump_stack_lvl+0x34/0x44 print_report.cold+0x2cc/0x682 kasan_report+0xa3/0x120 xlog_recover_process_intents+0xa77/0xae0 [xfs fb841c7180aad3f8359438576e27867f5795667e] xlog_recover_finish+0x7d/0x970 [xfs fb841c7180aad3f8359438576e27867f5795667e] xfs_log_mount_finish+0x2d7/0x5d0 [xfs fb841c7180aad3f8359438576e27867f5795667e] xfs_mountfs+0x11d4/0x1d10 [xfs fb841c7180aad3f8359438576e27867f5795667e] xfs_fs_fill_super+0x13d5/0x1a80 [xfs fb841c7180aad3f8359438576e27867f5795667e] get_tree_bdev+0x3da/0x6e0 vfs_get_tree+0x7d/0x240 path_mount+0xdd3/0x17d0 __x64_sys_mount+0x1fa/0x270 do_syscall_64+0x2b/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 RIP: 0033:0x7ff5bc069eae Code: 48 8b 0d 85 1f 0f 00 f7 d8 64 89 01 48 83 c8 ff c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa 49 89 ca b8 a5 00 00 00 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 52 1f 0f 00 f7 d8 64 89 01 48 RSP: 002b:00007ffe433fd448 EFLAGS: 00000246 ORIG_RAX: 00000000000000a5 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007ff5bc069eae RDX: 00005575d7213290 RSI: 00005575d72132d0 RDI: 00005575d72132b0 RBP: 00005575d7212fd0 R08: 00005575d7213230 R09: 00005575d7213fe0 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 R13: 00005575d7213290 R14: 00005575d72132b0 R15: 00005575d7212fd0 Allocated by task 5680: kasan_save_stack+0x1e/0x40 __kasan_slab_alloc+0x66/0x80 kmem_cache_alloc+0x152/0x320 xfs_rui_init+0x17a/0x1b0 [xfs] xlog_recover_rui_commit_pass2+0xb9/0x2e0 [xfs] xlog_recover_items_pass2+0xe9/0x220 [xfs] xlog_recover_commit_trans+0x673/0x900 [xfs] xlog_recovery_process_trans+0xbe/0x130 [xfs] xlog_recover_process_data+0x103/0x2a0 [xfs] xlog_do_recovery_pass+0x548/0xc60 [xfs] xlog_do_log_recovery+0x62/0xc0 [xfs] xlog_do_recover+0x73/0x480 [xfs] xlog_recover+0x229/0x460 [xfs] xfs_log_mount+0x284/0x640 [xfs] xfs_mountfs+0xf8b/0x1d10 [xfs] xfs_fs_fill_super+0x13d5/0x1a80 [xfs] get_tree_bdev+0x3da/0x6e0 vfs_get_tree+0x7d/0x240 path_mount+0xdd3/0x17d0 __x64_sys_mount+0x1fa/0x270 do_syscall_64+0x2b/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Freed by task 5680: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x30 ____kasan_slab_free+0x144/0x1b0 slab_free_freelist_hook+0xab/0x180 kmem_cache_free+0x1f1/0x410 xfs_rud_item_release+0x33/0x80 [xfs] xfs_trans_free_items+0xc3/0x220 [xfs] xfs_trans_cancel+0x1fa/0x590 [xfs] xfs_rui_item_recover+0x913/0xd60 [xfs] xlog_recover_process_intents+0x24e/0xae0 [xfs] xlog_recover_finish+0x7d/0x970 [xfs] xfs_log_mount_finish+0x2d7/0x5d0 [xfs] xfs_mountfs+0x11d4/0x1d10 [xfs] xfs_fs_fill_super+0x13d5/0x1a80 [xfs] get_tree_bdev+0x3da/0x6e0 vfs_get_tree+0x7d/0x240 path_mount+0xdd3/0x17d0 __x64_sys_mount+0x1fa/0x270 do_syscall_64+0x2b/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 The buggy address belongs to the object at ffff88804391b300 which belongs to the cache xfs_rui_item of size 688 The buggy address is located 96 bytes inside of 688-byte region [ffff88804391b300, ffff88804391b5b0) The buggy address belongs to the physical page: page:ffffea00010e4600 refcount:1 mapcount:0 mapping:0000000000000000 index:0xffff888043919320 pfn:0x43918 head:ffffea00010e4600 order:2 compound_mapcount:0 compound_pincount:0 flags: 0x4fff80000010200(slab|head|node=1|zone=1|lastcpupid=0xfff) raw: 04fff80000010200 0000000000000000 dead000000000122 ffff88807f0eadc0 raw: ffff888043919320 0000000080140010 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff88804391b200: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff88804391b280: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc >ffff88804391b300: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ^ ffff88804391b380: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ffff88804391b400: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb ================================================================== The test fuzzes an rmap btree block and starts writer threads to induce a filesystem shutdown on the corrupt block. When the filesystem is remounted, recovery will try to replay the committed rmap intent item, but the corruption problem causes the recovery transaction to fail. Cancelling the transaction frees the RUD, which frees the RUI that we recovered. When we return to xlog_recover_process_intents, @lip is now a dangling pointer, and we cannot use it to find the iop_recover method for the tracepoint. Hence we must store the item ops before calling ->iop_recover if we want to give it to the tracepoint so that the trace data will tell us exactly which intent item failed. Signed-off-by: Darrick J. Wong Reviewed-by: Christoph Hellwig --- fs/xfs/xfs_log_recover.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c index 17e923b9c5fa2..322eb2ee6c550 100644 --- a/fs/xfs/xfs_log_recover.c +++ b/fs/xfs/xfs_log_recover.c @@ -2552,6 +2552,8 @@ xlog_recover_process_intents( for (lip = xfs_trans_ail_cursor_first(ailp, &cur, 0); lip != NULL; lip = xfs_trans_ail_cursor_next(ailp, &cur)) { + const struct xfs_item_ops *ops; + if (!xlog_item_is_intent(lip)) break; @@ -2567,13 +2569,17 @@ xlog_recover_process_intents( * deferred ops, you /must/ attach them to the capture list in * the recover routine or else those subsequent intents will be * replayed in the wrong order! + * + * The recovery function can free the log item, so we must not + * access lip after it returns. */ spin_unlock(&ailp->ail_lock); - error = lip->li_ops->iop_recover(lip, &capture_list); + ops = lip->li_ops; + error = ops->iop_recover(lip, &capture_list); spin_lock(&ailp->ail_lock); if (error) { trace_xlog_intent_recovery_failed(log->l_mp, error, - lip->li_ops->iop_recover); + ops->iop_recover); break; } } -- GitLab From a635beeacc6d56d2b71c39e6c0103f85b53d108e Mon Sep 17 00:00:00 2001 From: Zheng Yejian Date: Mon, 17 Oct 2022 10:38:06 +0000 Subject: [PATCH 0213/1927] tracing/histogram: Update document for KEYS_MAX size After commit 4f36c2d85ced ("tracing: Increase tracing map KEYS_MAX size"), 'keys' supports up to three fields. Signed-off-by: Zheng Yejian Cc: stable@vger.kernel.org Acked-by: Masami Hiramatsu (Google) Link: https://lore.kernel.org/r/20221017103806.2479139-1-zhengyejian1@huawei.com Signed-off-by: Jonathan Corbet --- Documentation/trace/histogram.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/trace/histogram.rst b/Documentation/trace/histogram.rst index c1b685a38f6b4..87bd772836c0c 100644 --- a/Documentation/trace/histogram.rst +++ b/Documentation/trace/histogram.rst @@ -39,7 +39,7 @@ Documentation written by Tom Zanussi will use the event's kernel stacktrace as the key. The keywords 'keys' or 'key' can be used to specify keys, and the keywords 'values', 'vals', or 'val' can be used to specify values. Compound - keys consisting of up to two fields can be specified by the 'keys' + keys consisting of up to three fields can be specified by the 'keys' keyword. Hashing a compound key produces a unique entry in the table for each unique combination of component keys, and can be useful for providing more fine-grained summaries of event data. -- GitLab From eb83f502adb036cd56c27e13b9ca3b2aabfa790b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20Bugge?= Date: Wed, 12 Oct 2022 16:15:42 +0200 Subject: [PATCH 0214/1927] RDMA/cma: Use output interface for net_dev check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 27cfde795a96 ("RDMA/cma: Fix arguments order in net device validation") swapped the src and dst addresses in the call to validate_net_dev(). As a consequence, the test in validate_ipv4_net_dev() to see if the net_dev is the right one, is incorrect for port 1 <-> 2 communication when the ports are on the same sub-net. This is fixed by denoting the flowi4_oif as the device instead of the incoming one. The bug has not been observed using IPv6 addresses. Fixes: 27cfde795a96 ("RDMA/cma: Fix arguments order in net device validation") Signed-off-by: Håkon Bugge Link: https://lore.kernel.org/r/20221012141542.16925-1-haakon.bugge@oracle.com Reviewed-by: Leon Romanovsky Signed-off-by: Leon Romanovsky --- drivers/infiniband/core/cma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index cc2222b85c881..26d1772179b8f 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -1556,7 +1556,7 @@ static bool validate_ipv4_net_dev(struct net_device *net_dev, return false; memset(&fl4, 0, sizeof(fl4)); - fl4.flowi4_iif = net_dev->ifindex; + fl4.flowi4_oif = net_dev->ifindex; fl4.daddr = daddr; fl4.saddr = saddr; -- GitLab From 1afac08b39d85437187bb2a92d89a741b1078f55 Mon Sep 17 00:00:00 2001 From: Dean Luick Date: Tue, 18 Oct 2022 10:27:50 -0400 Subject: [PATCH 0215/1927] IB/hfi1: Correctly move list in sc_disable() Commit 13bac861952a ("IB/hfi1: Fix abba locking issue with sc_disable()") incorrectly tries to move a list from one list head to another. The result is a kernel crash. The crash is triggered when a link goes down and there are waiters for a send to complete. The following signature is seen: BUG: kernel NULL pointer dereference, address: 0000000000000030 [...] Call Trace: sc_disable+0x1ba/0x240 [hfi1] pio_freeze+0x3d/0x60 [hfi1] handle_freeze+0x27/0x1b0 [hfi1] process_one_work+0x1b0/0x380 ? process_one_work+0x380/0x380 worker_thread+0x30/0x360 ? process_one_work+0x380/0x380 kthread+0xd7/0x100 ? kthread_complete_and_exit+0x20/0x20 ret_from_fork+0x1f/0x30 The fix is to use the correct call to move the list. Fixes: 13bac861952a ("IB/hfi1: Fix abba locking issue with sc_disable()") Signed-off-by: Dean Luick Signed-off-by: Dennis Dalessandro Link: https://lore.kernel.org/r/166610327042.674422.6146908799669288976.stgit@awfm-02.cornelisnetworks.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hfi1/pio.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c index 3d42bd2b36bd4..51ae58c02b15c 100644 --- a/drivers/infiniband/hw/hfi1/pio.c +++ b/drivers/infiniband/hw/hfi1/pio.c @@ -913,8 +913,7 @@ void sc_disable(struct send_context *sc) spin_unlock(&sc->release_lock); write_seqlock(&sc->waitlock); - if (!list_empty(&sc->piowait)) - list_move(&sc->piowait, &wake_list); + list_splice_init(&sc->piowait, &wake_list); write_sequnlock(&sc->waitlock); while (!list_empty(&wake_list)) { struct iowait *wait; -- GitLab From 00aaf8bfe0ee2b807b452df806d725e080d85404 Mon Sep 17 00:00:00 2001 From: Shuming Fan Date: Wed, 19 Oct 2022 17:57:31 +0800 Subject: [PATCH 0216/1927] ASoC: rt1308-sdw: update the preset settings This patch updates the pad control and checks the hardware version to set the different preset settings. Signed-off-by: Shuming Fan Link: https://lore.kernel.org/r/20221019095731.31101-1-shumingf@realtek.com Signed-off-by: Mark Brown --- sound/soc/codecs/rt1308-sdw.c | 17 ++++++++++++++--- sound/soc/codecs/rt1308-sdw.h | 1 + sound/soc/codecs/rt1308.h | 5 +++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/sound/soc/codecs/rt1308-sdw.c b/sound/soc/codecs/rt1308-sdw.c index 5c29416aa781c..f99aed353f10c 100644 --- a/sound/soc/codecs/rt1308-sdw.c +++ b/sound/soc/codecs/rt1308-sdw.c @@ -50,6 +50,7 @@ static bool rt1308_volatile_register(struct device *dev, unsigned int reg) case 0x3008: case 0x300a: case 0xc000: + case 0xc710: case 0xc860 ... 0xc863: case 0xc870 ... 0xc873: return true; @@ -200,6 +201,7 @@ static int rt1308_io_init(struct device *dev, struct sdw_slave *slave) { struct rt1308_sdw_priv *rt1308 = dev_get_drvdata(dev); int ret = 0; + unsigned int tmp; if (rt1308->hw_init) return 0; @@ -231,6 +233,10 @@ static int rt1308_io_init(struct device *dev, struct sdw_slave *slave) /* sw reset */ regmap_write(rt1308->regmap, RT1308_SDW_RESET, 0); + regmap_read(rt1308->regmap, 0xc710, &tmp); + rt1308->hw_ver = tmp; + dev_dbg(dev, "%s, hw_ver=0x%x\n", __func__, rt1308->hw_ver); + /* initial settings */ regmap_write(rt1308->regmap, 0xc103, 0xc0); regmap_write(rt1308->regmap, 0xc030, 0x17); @@ -246,8 +252,14 @@ static int rt1308_io_init(struct device *dev, struct sdw_slave *slave) regmap_write(rt1308->regmap, 0xc062, 0x05); regmap_write(rt1308->regmap, 0xc171, 0x07); regmap_write(rt1308->regmap, 0xc173, 0x0d); - regmap_write(rt1308->regmap, 0xc311, 0x7f); - regmap_write(rt1308->regmap, 0xc900, 0x90); + if (rt1308->hw_ver == RT1308_VER_C) { + regmap_write(rt1308->regmap, 0xc311, 0x7f); + regmap_write(rt1308->regmap, 0xc300, 0x09); + } else { + regmap_write(rt1308->regmap, 0xc311, 0x4f); + regmap_write(rt1308->regmap, 0xc300, 0x0b); + } + regmap_write(rt1308->regmap, 0xc900, 0x5a); regmap_write(rt1308->regmap, 0xc1a0, 0x84); regmap_write(rt1308->regmap, 0xc1a1, 0x01); regmap_write(rt1308->regmap, 0xc360, 0x78); @@ -257,7 +269,6 @@ static int rt1308_io_init(struct device *dev, struct sdw_slave *slave) regmap_write(rt1308->regmap, 0xc070, 0x00); regmap_write(rt1308->regmap, 0xc100, 0xd7); regmap_write(rt1308->regmap, 0xc101, 0xd7); - regmap_write(rt1308->regmap, 0xc300, 0x09); if (rt1308->first_hw_init) { regcache_cache_bypass(rt1308->regmap, false); diff --git a/sound/soc/codecs/rt1308-sdw.h b/sound/soc/codecs/rt1308-sdw.h index 6668e19d85d46..f88f52e8917e6 100644 --- a/sound/soc/codecs/rt1308-sdw.h +++ b/sound/soc/codecs/rt1308-sdw.h @@ -163,6 +163,7 @@ struct rt1308_sdw_priv { bool first_hw_init; int rx_mask; int slots; + int hw_ver; }; struct sdw_stream_data { diff --git a/sound/soc/codecs/rt1308.h b/sound/soc/codecs/rt1308.h index ff7c423e879e4..d3a0f91630ca4 100644 --- a/sound/soc/codecs/rt1308.h +++ b/sound/soc/codecs/rt1308.h @@ -286,4 +286,9 @@ enum { RT1308_AIFS }; +enum rt1308_hw_ver { + RT1308_VER_C = 2, + RT1308_VER_D +}; + #endif /* end of _RT1308_H_ */ -- GitLab From 75d8b1662ca5c20cf8365575222abaef18ff1f50 Mon Sep 17 00:00:00 2001 From: Shuming Fan Date: Wed, 19 Oct 2022 17:57:15 +0800 Subject: [PATCH 0217/1927] ASoC: rt1308-sdw: add the default value of some registers The driver missed the default value of register 0xc070/0xc360. This patch adds that default value to avoid invalid register access when the device doesn't be enumerated yet. BugLink: https://github.com/thesofproject/linux/issues/3924 Signed-off-by: Shuming Fan Link: https://lore.kernel.org/r/20221019095715.31082-1-shumingf@realtek.com Signed-off-by: Mark Brown --- sound/soc/codecs/rt1308-sdw.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/soc/codecs/rt1308-sdw.h b/sound/soc/codecs/rt1308-sdw.h index f88f52e8917e6..62ce277993075 100644 --- a/sound/soc/codecs/rt1308-sdw.h +++ b/sound/soc/codecs/rt1308-sdw.h @@ -139,10 +139,12 @@ static const struct reg_default rt1308_reg_defaults[] = { { 0x3005, 0x23 }, { 0x3008, 0x02 }, { 0x300a, 0x00 }, + { 0xc000 | (RT1308_DATA_PATH << 4), 0x00 }, { 0xc003 | (RT1308_DAC_SET << 4), 0x00 }, { 0xc000 | (RT1308_POWER << 4), 0x00 }, { 0xc001 | (RT1308_POWER << 4), 0x00 }, { 0xc002 | (RT1308_POWER << 4), 0x00 }, + { 0xc000 | (RT1308_POWER_STATUS << 4), 0x00 }, }; #define RT1308_SDW_OFFSET 0xc000 -- GitLab From 32def55d237e8507d4eb8442628fc2e59a899ea0 Mon Sep 17 00:00:00 2001 From: Aidan MacDonald Date: Wed, 19 Oct 2022 02:23:02 +0100 Subject: [PATCH 0218/1927] ASoC: simple-card: Fix up checks for HW param fixups The "convert-xxx" properties only have an effect for DPCM DAI links. A DAI link is only created as DPCM if the device tree requires it; part of this involves checking for the use of "convert-xxx" properties. When the convert-sample-format property was added, the checks got out of sync. A DAI link that specified only convert-sample-format but did not pass any of the other DPCM checks would not go into DPCM mode and the convert-sample-format property would be silently ignored. Fix this by adding a function to do the "convert-xxx" property checks, instead of open-coding it in simple-card and audio-graph-card. And add "convert-sample-format" to the check function so that DAI links using it will be initialized correctly. Fixes: 047a05366f4b ("ASoC: simple-card-utils: Fixup DAI sample format") Acked-by: Kuninori Morimoto Signed-off-by: Aidan MacDonald Acked-by: Sameer Pujar Link: https://lore.kernel.org/r/20221019012302.633830-1-aidanmacdonald.0x0@gmail.com Signed-off-by: Mark Brown --- include/sound/simple_card_utils.h | 1 + sound/soc/generic/audio-graph-card.c | 2 +- sound/soc/generic/simple-card-utils.c | 15 +++++++++++++++ sound/soc/generic/simple-card.c | 3 +-- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h index a0b827f0c2f60..25e049f441784 100644 --- a/include/sound/simple_card_utils.h +++ b/include/sound/simple_card_utils.h @@ -177,6 +177,7 @@ void asoc_simple_convert_fixup(struct asoc_simple_data *data, struct snd_pcm_hw_params *params); void asoc_simple_parse_convert(struct device_node *np, char *prefix, struct asoc_simple_data *data); +bool asoc_simple_is_convert_required(const struct asoc_simple_data *data); int asoc_simple_parse_routing(struct snd_soc_card *card, char *prefix); diff --git a/sound/soc/generic/audio-graph-card.c b/sound/soc/generic/audio-graph-card.c index b327372f2e4ae..fe7cf972d44ce 100644 --- a/sound/soc/generic/audio-graph-card.c +++ b/sound/soc/generic/audio-graph-card.c @@ -417,7 +417,7 @@ static inline bool parse_as_dpcm_link(struct asoc_simple_priv *priv, * or has convert-xxx property */ if ((of_get_child_count(codec_port) > 1) || - (adata->convert_rate || adata->convert_channels)) + asoc_simple_is_convert_required(adata)) return true; return false; diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c index bef16833c487e..be69bbc47f813 100644 --- a/sound/soc/generic/simple-card-utils.c +++ b/sound/soc/generic/simple-card-utils.c @@ -85,6 +85,21 @@ void asoc_simple_parse_convert(struct device_node *np, } EXPORT_SYMBOL_GPL(asoc_simple_parse_convert); +/** + * asoc_simple_is_convert_required() - Query if HW param conversion was requested + * @data: Link data. + * + * Returns true if any HW param conversion was requested for this DAI link with + * any "convert-xxx" properties. + */ +bool asoc_simple_is_convert_required(const struct asoc_simple_data *data) +{ + return data->convert_rate || + data->convert_channels || + data->convert_sample_format; +} +EXPORT_SYMBOL_GPL(asoc_simple_is_convert_required); + int asoc_simple_parse_daifmt(struct device *dev, struct device_node *node, struct device_node *codec, diff --git a/sound/soc/generic/simple-card.c b/sound/soc/generic/simple-card.c index 78419e18717d6..feb55b66239b8 100644 --- a/sound/soc/generic/simple-card.c +++ b/sound/soc/generic/simple-card.c @@ -393,8 +393,7 @@ static int __simple_for_each_link(struct asoc_simple_priv *priv, * or has convert-xxx property */ if (dpcm_selectable && - (num > 2 || - adata.convert_rate || adata.convert_channels)) { + (num > 2 || asoc_simple_is_convert_required(&adata))) { /* * np * |1(CPU)|0(Codec) li->cpu -- GitLab From df496157a5afa1b6d1f4c46ad6549c2c346d1e59 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Wed, 19 Oct 2022 08:16:39 +0100 Subject: [PATCH 0219/1927] ASoC: codecs: jz4725b: Fix spelling mistake "Sourc" -> "Source", "Routee" -> "Route" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are two spelling mistakes in codec routing description. Fix it. Signed-off-by: Colin Ian King Reviewed-by: Philippe Mathieu-Daudé Acked-by: Paul Cercueil Link: https://lore.kernel.org/r/20221019071639.1003730-1-colin.i.king@gmail.com Signed-off-by: Mark Brown --- sound/soc/codecs/jz4725b.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/codecs/jz4725b.c b/sound/soc/codecs/jz4725b.c index d57c2c6a3addf..71ea576f7e67a 100644 --- a/sound/soc/codecs/jz4725b.c +++ b/sound/soc/codecs/jz4725b.c @@ -288,7 +288,7 @@ static const struct snd_soc_dapm_route jz4725b_codec_dapm_routes[] = { {"Mixer to ADC", NULL, "Mixer"}, {"ADC Source Capture Route", "Mixer", "Mixer to ADC"}, - {"ADC Sourc Capture Routee", "Line In", "Line In"}, + {"ADC Source Capture Route", "Line In", "Line In"}, {"ADC Source Capture Route", "Mic 1", "Mic 1"}, {"ADC Source Capture Route", "Mic 2", "Mic 2"}, {"ADC", NULL, "ADC Source Capture Route"}, -- GitLab From 5a637f83d96fc9f27e93385924e7253a5f5187bc Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Mon, 17 Oct 2022 15:14:13 +0200 Subject: [PATCH 0220/1927] dmaengine: stm32-mdma: memset stm32_mdma_chan_config struct before using it New bool m2m_hw has been added at the end of stm32_mdma_chan_config struct to support the STM32 DMA MDMA chaining. m2m_hw is set true in stm32_mdma_slave_config() if peripheral_size is set, but m2m_hw is never initialized false. To ensure this case, and any further new update of the structure, memset it to 0 before using it. Fixes: 696874322771 ("dmaengine: stm32-mdma: add support to be triggered by STM32 DMA") Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20221017131413.202567-1-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/dma/stm32-mdma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/stm32-mdma.c b/drivers/dma/stm32-mdma.c index e28acbcb53f4c..b9d4c843635fc 100644 --- a/drivers/dma/stm32-mdma.c +++ b/drivers/dma/stm32-mdma.c @@ -1539,6 +1539,7 @@ static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec, return NULL; } + memset(&config, 0, sizeof(config)); config.request = dma_spec->args[0]; config.priority_level = dma_spec->args[1]; config.transfer_config = dma_spec->args[2]; -- GitLab From 0ec8ce07394442d722806fe61b901a5b2b17249d Mon Sep 17 00:00:00 2001 From: Fenghua Yu Date: Fri, 14 Oct 2022 15:25:41 -0700 Subject: [PATCH 0221/1927] dmaengine: idxd: Do not enable user type Work Queue without Shared Virtual Addressing When the idxd_user_drv driver is bound to a Work Queue (WQ) device without IOMMU or with IOMMU Passthrough without Shared Virtual Addressing (SVA), the application gains direct access to physical memory via the device by programming physical address to a submitted descriptor. This allows direct userspace read and write access to arbitrary physical memory. This is inconsistent with the security goals of a good kernel API. Unlike vfio_pci driver, the IDXD char device driver does not provide any ways to pin user pages and translate the address from user VA to IOVA or PA without IOMMU SVA. Therefore the application has no way to instruct the device to perform DMA function. This makes the char device not usable for normal application usage. Since user type WQ without SVA cannot be used for normal application usage and presents the security issue, bind idxd_user_drv driver and enable user type WQ only when SVA is enabled (i.e. user PASID is enabled). Fixes: 448c3de8ac83 ("dmaengine: idxd: create user driver for wq 'device'") Cc: stable@vger.kernel.org Suggested-by: Arjan Van De Ven Signed-off-by: Fenghua Yu Reviewed-by: Dave Jiang Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20221014222541.3912195-1-fenghua.yu@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/cdev.c | 18 ++++++++++++++++++ include/uapi/linux/idxd.h | 1 + 2 files changed, 19 insertions(+) diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index c2808fd081d65..a9b96b18772f3 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -312,6 +312,24 @@ static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) if (idxd->state != IDXD_DEV_ENABLED) return -ENXIO; + /* + * User type WQ is enabled only when SVA is enabled for two reasons: + * - If no IOMMU or IOMMU Passthrough without SVA, userspace + * can directly access physical address through the WQ. + * - The IDXD cdev driver does not provide any ways to pin + * user pages and translate the address from user VA to IOVA or + * PA without IOMMU SVA. Therefore the application has no way + * to instruct the device to perform DMA function. This makes + * the cdev not usable for normal application usage. + */ + if (!device_user_pasid_enabled(idxd)) { + idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; + dev_dbg(&idxd->pdev->dev, + "User type WQ cannot be enabled without SVA.\n"); + + return -EOPNOTSUPP; + } + mutex_lock(&wq->wq_lock); wq->type = IDXD_WQT_USER; rc = drv_enable_wq(wq); diff --git a/include/uapi/linux/idxd.h b/include/uapi/linux/idxd.h index 095299c75828c..2b9e7feba3f32 100644 --- a/include/uapi/linux/idxd.h +++ b/include/uapi/linux/idxd.h @@ -29,6 +29,7 @@ enum idxd_scmd_stat { IDXD_SCMD_WQ_NO_SIZE = 0x800e0000, IDXD_SCMD_WQ_NO_PRIV = 0x800f0000, IDXD_SCMD_WQ_IRQ_ERR = 0x80100000, + IDXD_SCMD_WQ_USER_NO_IOMMU = 0x80110000, }; #define IDXD_SCMD_SOFTERR_MASK 0x80000000 -- GitLab From f8aa6c895d482847c9b799dcdac8bbdb56cb8e04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Le=20Goater?= Date: Sun, 16 Oct 2022 17:57:22 +0200 Subject: [PATCH 0222/1927] spi: aspeed: Fix window offset of CE1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The offset value of the mapping window in the kernel structure is calculated using the value of the previous window offset. This doesn't reflect how the HW is configured and can lead to erroneous setting of the second flash device (CE1). Cc: Chin-Ting Kuo Fixes: e3228ed92893 ("spi: spi-mem: Convert Aspeed SMC driver to spi-mem") Signed-off-by: Cédric Le Goater Reviewed-by: Joel Stanley Link: https://lore.kernel.org/r/20221016155722.3520802-1-clg@kaod.org Signed-off-by: Mark Brown --- drivers/spi/spi-aspeed-smc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c index 33cefcf183927..b90571396a604 100644 --- a/drivers/spi/spi-aspeed-smc.c +++ b/drivers/spi/spi-aspeed-smc.c @@ -398,7 +398,7 @@ static void aspeed_spi_get_windows(struct aspeed_spi *aspi, windows[cs].cs = cs; windows[cs].size = data->segment_end(aspi, reg_val) - data->segment_start(aspi, reg_val); - windows[cs].offset = cs ? windows[cs - 1].offset + windows[cs - 1].size : 0; + windows[cs].offset = data->segment_start(aspi, reg_val) - aspi->ahb_base_phy; dev_vdbg(aspi->dev, "CE%d offset=0x%.8x size=0x%x\n", cs, windows[cs].offset, windows[cs].size); } -- GitLab From a450b5c8739248069e11f72129fca61a56125577 Mon Sep 17 00:00:00 2001 From: linkt Date: Tue, 11 Oct 2022 10:51:36 +0800 Subject: [PATCH 0223/1927] ASoC: amd: yc: Adding Lenovo ThinkBook 14 Gen 4+ ARA and Lenovo ThinkBook 16 Gen 4+ ARA to the Quirks List Lenovo ThinkBook 14 Gen 4+ ARA and ThinkBook 16 Gen 4+ ARA need to be added to the list of quirks for the microphone to work properly. Signed-off-by: linkt Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/MEYPR01MB8397A3C27DE6206FA3EF834DB6239@MEYPR01MB8397.ausprd01.prod.outlook.com Signed-off-by: Mark Brown --- sound/soc/amd/yc/acp6x-mach.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 2cb50d5cf1a9a..09a8aceff22f1 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -45,6 +45,20 @@ static struct snd_soc_card acp6x_card = { }; static const struct dmi_system_id yc_acp_quirk_table[] = { + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D0"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D1"), + } + }, { .driver_data = &acp6x_card, .matches = { -- GitLab From 1dd5166102e7ca91e8c5d833110333835e147ddb Mon Sep 17 00:00:00 2001 From: Srinivasa Rao Mandadapu Date: Sat, 15 Oct 2022 14:48:50 +0530 Subject: [PATCH 0224/1927] ASoC: qcom: lpass-cpu: Mark HDMI TX parity register as volatile Update LPASS_HDMI_TX_PARITY_ADDR register as volatile, to fix dp audio failures observed with some of external monitors. Fixes: 7cb37b7bd0d3 ("ASoC: qcom: Add support for lpass hdmi driver") Signed-off-by: Srinivasa Rao Mandadapu Reviewed-by: Stephen Boyd Link: https://lore.kernel.org/r/1665825530-7593-1-git-send-email-quic_srivasam@quicinc.com Signed-off-by: Mark Brown --- sound/soc/qcom/lpass-cpu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/soc/qcom/lpass-cpu.c b/sound/soc/qcom/lpass-cpu.c index 99a3b44285913..54353842dc07f 100644 --- a/sound/soc/qcom/lpass-cpu.c +++ b/sound/soc/qcom/lpass-cpu.c @@ -784,6 +784,8 @@ static bool lpass_hdmi_regmap_volatile(struct device *dev, unsigned int reg) return true; if (reg == LPASS_HDMI_TX_VBIT_CTL_ADDR(v)) return true; + if (reg == LPASS_HDMI_TX_PARITY_ADDR(v)) + return true; for (i = 0; i < v->hdmi_rdma_channels; ++i) { if (reg == LPAIF_HDMI_RDMACURR_REG(v, i)) -- GitLab From 05de5cf6fb7d73d2bf0a0c882433f31db5c93f63 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Wed, 19 Oct 2022 10:49:26 -0500 Subject: [PATCH 0225/1927] ASoC: SOF: Intel: pci-tgl: fix ADL-N descriptor ADL-N uses a different signing key, which means we can't reuse the regular ADL descriptor used for ADL-P/M/S. Fixes: cd57eb3c403cb ("ASoC: SOF: Intel: pci-tgl: add ADL-N support") Signed-off-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Reviewed-by: Chao Song Reviewed-by: Bard Liao Link: https://lore.kernel.org/r/20221019154926.163539-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/sof/intel/pci-tgl.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/sound/soc/sof/intel/pci-tgl.c b/sound/soc/sof/intel/pci-tgl.c index 2d63cc236a68e..4cfe4f242fc5e 100644 --- a/sound/soc/sof/intel/pci-tgl.c +++ b/sound/soc/sof/intel/pci-tgl.c @@ -159,6 +159,34 @@ static const struct sof_dev_desc adl_desc = { .ops_init = sof_tgl_ops_init, }; +static const struct sof_dev_desc adl_n_desc = { + .machines = snd_soc_acpi_intel_adl_machines, + .alt_machines = snd_soc_acpi_intel_adl_sdw_machines, + .use_acpi_target_states = true, + .resindex_lpe_base = 0, + .resindex_pcicfg_base = -1, + .resindex_imr_base = -1, + .irqindex_host_ipc = -1, + .chip_info = &tgl_chip_info, + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/adl-n", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-adl-n.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, + .nocodec_tplg_filename = "sof-adl-nocodec.tplg", + .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, +}; + static const struct sof_dev_desc rpls_desc = { .machines = snd_soc_acpi_intel_rpl_machines, .alt_machines = snd_soc_acpi_intel_rpl_sdw_machines, @@ -246,7 +274,7 @@ static const struct pci_device_id sof_pci_ids[] = { { PCI_DEVICE(0x8086, 0x51cf), /* RPL-PX */ .driver_data = (unsigned long)&rpl_desc}, { PCI_DEVICE(0x8086, 0x54c8), /* ADL-N */ - .driver_data = (unsigned long)&adl_desc}, + .driver_data = (unsigned long)&adl_n_desc}, { 0, } }; MODULE_DEVICE_TABLE(pci, sof_pci_ids); -- GitLab From 097a4a1612389c31d2c4b95dfa816b91212d7f54 Mon Sep 17 00:00:00 2001 From: Jiangshan Yi Date: Sun, 9 Oct 2022 16:39:44 +0800 Subject: [PATCH 0226/1927] watchdog: sp805_wdt: fix spelling typo in comment Fix spelling typo in comment. Reported-by: k2ci Signed-off-by: Jiangshan Yi Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20221009083944.2988237-1-13667453960@163.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/sp805_wdt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/watchdog/sp805_wdt.c b/drivers/watchdog/sp805_wdt.c index 78ba36689eeca..2756ed54ca3d5 100644 --- a/drivers/watchdog/sp805_wdt.c +++ b/drivers/watchdog/sp805_wdt.c @@ -88,7 +88,7 @@ static bool wdt_is_running(struct watchdog_device *wdd) return (wdtcontrol & ENABLE_MASK) == ENABLE_MASK; } -/* This routine finds load value that will reset system in required timout */ +/* This routine finds load value that will reset system in required timeout */ static int wdt_setload(struct watchdog_device *wdd, unsigned int timeout) { struct sp805_wdt *wdt = watchdog_get_drvdata(wdd); -- GitLab From 82ebbe65d781064cfb0a6a8af221a9cebcaaac9e Mon Sep 17 00:00:00 2001 From: Manank Patel Date: Thu, 13 Oct 2022 15:22:58 +0530 Subject: [PATCH 0227/1927] drivers: watchdog: exar_wdt.c fix use after free fix use after free by storing the result of PTR_ERR(n->pdev) to a local variable before returning. Signed-off-by: Manank Patel Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20221013095258.1424967-1-pmanank200502@gmail.com Signed-off-by: Guenter Roeck Signed-off-by: Wim Van Sebroeck --- drivers/watchdog/exar_wdt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/watchdog/exar_wdt.c b/drivers/watchdog/exar_wdt.c index 35058d8b21bc7..7c61ff3432711 100644 --- a/drivers/watchdog/exar_wdt.c +++ b/drivers/watchdog/exar_wdt.c @@ -355,8 +355,10 @@ static int __init exar_wdt_register(struct wdt_priv *priv, const int idx) &priv->wdt_res, 1, priv, sizeof(*priv)); if (IS_ERR(n->pdev)) { + int err = PTR_ERR(n->pdev); + kfree(n); - return PTR_ERR(n->pdev); + return err; } list_add_tail(&n->list, &pdev_list); -- GitLab From 091873e47ef700e935aa80079b63929af599a0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Sala=C3=BCn?= Date: Wed, 19 Oct 2022 22:05:36 +0200 Subject: [PATCH 0228/1927] selftests/landlock: Build without static libraries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only (forced) static test binary doesn't depend on libcap. Because using -lcap on systems that don't have such static library would fail (e.g. on Arch Linux), let's be more specific and require only dynamic libcap linking. Fixes: a52540522c95 ("selftests/landlock: Fix out-of-tree builds") Cc: Anders Roxell Cc: Guillaume Tucker Cc: Mark Brown Cc: Shuah Khan Cc: stable@vger.kernel.org Signed-off-by: Mickaël Salaün Link: https://lore.kernel.org/r/20221019200536.2771316-1-mic@digikod.net --- tools/testing/selftests/landlock/Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/landlock/Makefile b/tools/testing/selftests/landlock/Makefile index 6632bfff486b8..348e2dbdb4e0b 100644 --- a/tools/testing/selftests/landlock/Makefile +++ b/tools/testing/selftests/landlock/Makefile @@ -3,7 +3,6 @@ # First run: make -C ../../../.. headers_install CFLAGS += -Wall -O2 $(KHDR_INCLUDES) -LDLIBS += -lcap LOCAL_HDRS += common.h @@ -13,10 +12,12 @@ TEST_GEN_PROGS := $(src_test:.c=) TEST_GEN_PROGS_EXTENDED := true -# Static linking for short targets: +# Short targets: +$(TEST_GEN_PROGS): LDLIBS += -lcap $(TEST_GEN_PROGS_EXTENDED): LDFLAGS += -static include ../lib.mk -# Static linking for targets with $(OUTPUT)/ prefix: +# Targets with $(OUTPUT)/ prefix: +$(TEST_GEN_PROGS): LDLIBS += -lcap $(TEST_GEN_PROGS_EXTENDED): LDFLAGS += -static -- GitLab From ccd30a476f8e864732de220bd50e6f372f5ebcab Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Tue, 11 Oct 2022 14:38:38 -0700 Subject: [PATCH 0229/1927] fscrypt: fix keyring memory leak on mount failure Commit d7e7b9af104c ("fscrypt: stop using keyrings subsystem for fscrypt_master_key") moved the keyring destruction from __put_super() to generic_shutdown_super() so that the filesystem's block device(s) are still available. Unfortunately, this causes a memory leak in the case where a mount is attempted with the test_dummy_encryption mount option, but the mount fails after the option has already been processed. To fix this, attempt the keyring destruction in both places. Reported-by: syzbot+104c2a89561289cec13e@syzkaller.appspotmail.com Fixes: d7e7b9af104c ("fscrypt: stop using keyrings subsystem for fscrypt_master_key") Signed-off-by: Eric Biggers Reviewed-by: Christian Brauner (Microsoft) Link: https://lore.kernel.org/r/20221011213838.209879-1-ebiggers@kernel.org --- fs/crypto/keyring.c | 17 +++++++++++------ fs/super.c | 3 ++- include/linux/fscrypt.h | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c index 1cca09aa43f8b..2a24b1f0ae688 100644 --- a/fs/crypto/keyring.c +++ b/fs/crypto/keyring.c @@ -205,14 +205,19 @@ static int allocate_filesystem_keyring(struct super_block *sb) } /* - * This is called at unmount time to release all encryption keys that have been - * added to the filesystem, along with the keyring that contains them. + * Release all encryption keys that have been added to the filesystem, along + * with the keyring that contains them. * - * Note that besides clearing and freeing memory, this might need to evict keys - * from the keyslots of an inline crypto engine. Therefore, this must be called - * while the filesystem's underlying block device(s) are still available. + * This is called at unmount time. The filesystem's underlying block device(s) + * are still available at this time; this is important because after user file + * accesses have been allowed, this function may need to evict keys from the + * keyslots of an inline crypto engine, which requires the block device(s). + * + * This is also called when the super_block is being freed. This is needed to + * avoid a memory leak if mounting fails after the "test_dummy_encryption" + * option was processed, as in that case the unmount-time call isn't made. */ -void fscrypt_sb_delete(struct super_block *sb) +void fscrypt_destroy_keyring(struct super_block *sb) { struct fscrypt_keyring *keyring = sb->s_master_keys; size_t i; diff --git a/fs/super.c b/fs/super.c index 6a82660e1adba..8d39e4f11cfa3 100644 --- a/fs/super.c +++ b/fs/super.c @@ -291,6 +291,7 @@ static void __put_super(struct super_block *s) WARN_ON(s->s_inode_lru.node); WARN_ON(!list_empty(&s->s_mounts)); security_sb_free(s); + fscrypt_destroy_keyring(s); put_user_ns(s->s_user_ns); kfree(s->s_subtype); call_rcu(&s->rcu, destroy_super_rcu); @@ -479,7 +480,7 @@ void generic_shutdown_super(struct super_block *sb) evict_inodes(sb); /* only nonzero refcount inodes can have marks */ fsnotify_sb_delete(sb); - fscrypt_sb_delete(sb); + fscrypt_destroy_keyring(sb); security_sb_delete(sb); if (sb->s_dio_done_wq) { diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index cad78b569c7ef..4f5f8a6512132 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -307,7 +307,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy) } /* keyring.c */ -void fscrypt_sb_delete(struct super_block *sb); +void fscrypt_destroy_keyring(struct super_block *sb); int fscrypt_ioctl_add_key(struct file *filp, void __user *arg); int fscrypt_add_test_dummy_key(struct super_block *sb, const struct fscrypt_dummy_policy *dummy_policy); @@ -521,7 +521,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy) } /* keyring.c */ -static inline void fscrypt_sb_delete(struct super_block *sb) +static inline void fscrypt_destroy_keyring(struct super_block *sb) { } -- GitLab From a4f7fcd7023ba63bdfe82a054c4ceb636a55d155 Mon Sep 17 00:00:00 2001 From: Jilin Yuan Date: Wed, 19 Oct 2022 20:57:38 +0800 Subject: [PATCH 0230/1927] fbdev: sisfb: fix repeated word in comment Signed-off-by: Jilin Yuan Signed-off-by: Helge Deller --- drivers/video/fbdev/sis/sis_accel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/video/fbdev/sis/sis_accel.c b/drivers/video/fbdev/sis/sis_accel.c index 1914ab5a5a91b..5850e4325f071 100644 --- a/drivers/video/fbdev/sis/sis_accel.c +++ b/drivers/video/fbdev/sis/sis_accel.c @@ -202,7 +202,7 @@ SiS310SubsequentScreenToScreenCopy(struct sis_video_info *ivideo, int src_x, int * and destination blitting areas overlap and * adapt the bitmap addresses synchronously * if the coordinates exceed the valid range. - * The the areas do not overlap, we do our + * The areas do not overlap, we do our * normal check. */ if((mymax - mymin) < height) { -- GitLab From 70281592bf3fb7a2a193dced4d4e58a9ee96aa6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig?= Date: Wed, 19 Oct 2022 15:24:22 +0200 Subject: [PATCH 0231/1927] fbdev: xilinxfb: Make xilinxfb_release() return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function xilinxfb_release() returns zero unconditionally. Make it return void. There is no semantic change, the only effect is that it becomes obvious that the driver's .remove() callback always returns zero. This is a preparation for making platform remove callbacks return void. Signed-off-by: Uwe Kleine-König Signed-off-by: Helge Deller --- drivers/video/fbdev/xilinxfb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/video/fbdev/xilinxfb.c b/drivers/video/fbdev/xilinxfb.c index 438e2c78142f9..1ac83900a21cc 100644 --- a/drivers/video/fbdev/xilinxfb.c +++ b/drivers/video/fbdev/xilinxfb.c @@ -376,7 +376,7 @@ static int xilinxfb_assign(struct platform_device *pdev, return rc; } -static int xilinxfb_release(struct device *dev) +static void xilinxfb_release(struct device *dev) { struct xilinxfb_drvdata *drvdata = dev_get_drvdata(dev); @@ -402,8 +402,6 @@ static int xilinxfb_release(struct device *dev) if (!(drvdata->flags & BUS_ACCESS_FLAG)) dcr_unmap(drvdata->dcr_host, drvdata->dcr_len); #endif - - return 0; } /* --------------------------------------------------------------------- @@ -480,7 +478,9 @@ static int xilinxfb_of_probe(struct platform_device *pdev) static int xilinxfb_of_remove(struct platform_device *op) { - return xilinxfb_release(&op->dev); + xilinxfb_release(&op->dev); + + return 0; } /* Match table for of_platform binding */ -- GitLab From 7d1aa08aff0621a595c1b42efb493c475eefeeb3 Mon Sep 17 00:00:00 2001 From: Svyatoslav Ryhel Date: Wed, 19 Oct 2022 09:12:01 +0300 Subject: [PATCH 0232/1927] gpio: tegra: Convert to immutable irq chip Update the driver to use an immutable IRQ chip to fix this warning: "not an immutable chip, please consider fixing it!" Preserve per-chip labels by adding an ->irq_print_chip() callback. Tested-by: Svyatoslav Ryhel # TF201 T30 Tested-by: Robert Eckelmann # TF101 T20 Signed-off-by: Svyatoslav Ryhel Reviewed-by: Dmitry Osipenko Signed-off-by: Bartosz Golaszewski --- drivers/gpio/gpio-tegra.c | 60 ++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/drivers/gpio/gpio-tegra.c b/drivers/gpio/gpio-tegra.c index e4fb4cb38a0f4..5b265a6fd3c18 100644 --- a/drivers/gpio/gpio-tegra.c +++ b/drivers/gpio/gpio-tegra.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -94,7 +95,6 @@ struct tegra_gpio_info { struct tegra_gpio_bank *bank_info; const struct tegra_gpio_soc_config *soc; struct gpio_chip gc; - struct irq_chip ic; u32 bank_count; unsigned int *irqs; }; @@ -288,6 +288,7 @@ static void tegra_gpio_irq_mask(struct irq_data *d) unsigned int gpio = d->hwirq; tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 0); + gpiochip_disable_irq(chip, gpio); } static void tegra_gpio_irq_unmask(struct irq_data *d) @@ -296,6 +297,7 @@ static void tegra_gpio_irq_unmask(struct irq_data *d) struct tegra_gpio_info *tgi = gpiochip_get_data(chip); unsigned int gpio = d->hwirq; + gpiochip_enable_irq(chip, gpio); tegra_gpio_mask_write(tgi, GPIO_MSK_INT_ENB(tgi, gpio), gpio, 1); } @@ -598,10 +600,47 @@ static void tegra_gpio_irq_release_resources(struct irq_data *d) tegra_gpio_enable(tgi, d->hwirq); } +static void tegra_gpio_irq_print_chip(struct irq_data *d, struct seq_file *s) +{ + struct gpio_chip *chip = irq_data_get_irq_chip_data(d); + + seq_printf(s, dev_name(chip->parent)); +} + +static const struct irq_chip tegra_gpio_irq_chip = { + .irq_shutdown = tegra_gpio_irq_shutdown, + .irq_ack = tegra_gpio_irq_ack, + .irq_mask = tegra_gpio_irq_mask, + .irq_unmask = tegra_gpio_irq_unmask, + .irq_set_type = tegra_gpio_irq_set_type, +#ifdef CONFIG_PM_SLEEP + .irq_set_wake = tegra_gpio_irq_set_wake, +#endif + .irq_print_chip = tegra_gpio_irq_print_chip, + .irq_request_resources = tegra_gpio_irq_request_resources, + .irq_release_resources = tegra_gpio_irq_release_resources, + .flags = IRQCHIP_IMMUTABLE, +}; + +static const struct irq_chip tegra210_gpio_irq_chip = { + .irq_shutdown = tegra_gpio_irq_shutdown, + .irq_ack = tegra_gpio_irq_ack, + .irq_mask = tegra_gpio_irq_mask, + .irq_unmask = tegra_gpio_irq_unmask, + .irq_set_affinity = tegra_gpio_irq_set_affinity, + .irq_set_type = tegra_gpio_irq_set_type, +#ifdef CONFIG_PM_SLEEP + .irq_set_wake = tegra_gpio_irq_set_wake, +#endif + .irq_print_chip = tegra_gpio_irq_print_chip, + .irq_request_resources = tegra_gpio_irq_request_resources, + .irq_release_resources = tegra_gpio_irq_release_resources, + .flags = IRQCHIP_IMMUTABLE, +}; + #ifdef CONFIG_DEBUG_FS #include -#include static int tegra_dbg_gpio_show(struct seq_file *s, void *unused) { @@ -689,18 +728,6 @@ static int tegra_gpio_probe(struct platform_device *pdev) tgi->gc.ngpio = tgi->bank_count * 32; tgi->gc.parent = &pdev->dev; - tgi->ic.name = "GPIO"; - tgi->ic.irq_ack = tegra_gpio_irq_ack; - tgi->ic.irq_mask = tegra_gpio_irq_mask; - tgi->ic.irq_unmask = tegra_gpio_irq_unmask; - tgi->ic.irq_set_type = tegra_gpio_irq_set_type; - tgi->ic.irq_shutdown = tegra_gpio_irq_shutdown; -#ifdef CONFIG_PM_SLEEP - tgi->ic.irq_set_wake = tegra_gpio_irq_set_wake; -#endif - tgi->ic.irq_request_resources = tegra_gpio_irq_request_resources; - tgi->ic.irq_release_resources = tegra_gpio_irq_release_resources; - platform_set_drvdata(pdev, tgi); if (tgi->soc->debounce_supported) @@ -733,7 +760,6 @@ static int tegra_gpio_probe(struct platform_device *pdev) } irq = &tgi->gc.irq; - irq->chip = &tgi->ic; irq->fwnode = of_node_to_fwnode(pdev->dev.of_node); irq->child_to_parent_hwirq = tegra_gpio_child_to_parent_hwirq; irq->populate_parent_alloc_arg = tegra_gpio_populate_parent_fwspec; @@ -752,7 +778,9 @@ static int tegra_gpio_probe(struct platform_device *pdev) if (!irq->parent_domain) return -EPROBE_DEFER; - tgi->ic.irq_set_affinity = tegra_gpio_irq_set_affinity; + gpio_irq_chip_set_chip(irq, &tegra210_gpio_irq_chip); + } else { + gpio_irq_chip_set_chip(irq, &tegra_gpio_irq_chip); } tgi->regs = devm_platform_ioremap_resource(pdev, 0); -- GitLab From 4881bda5ea05c8c240fc8afeaa928e2bc43f61fa Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Wed, 19 Oct 2022 17:30:25 +0800 Subject: [PATCH 0233/1927] ALSA: ac97: fix possible memory leak in snd_ac97_dev_register() If device_register() fails in snd_ac97_dev_register(), it should call put_device() to give up reference, or the name allocated in dev_set_name() is leaked. Fixes: 0ca06a00e206 ("[ALSA] AC97 bus interface for ad-hoc drivers") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221019093025.1179475-1-yangyingliang@huawei.com Signed-off-by: Takashi Iwai --- sound/pci/ac97/ac97_codec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/ac97/ac97_codec.c b/sound/pci/ac97/ac97_codec.c index cb60a07d39a8e..ceead55f13ab1 100644 --- a/sound/pci/ac97/ac97_codec.c +++ b/sound/pci/ac97/ac97_codec.c @@ -2009,6 +2009,7 @@ static int snd_ac97_dev_register(struct snd_device *device) err = device_register(&ac97->dev); if (err < 0) { ac97_err(ac97, "Can't register ac97 bus\n"); + put_device(&ac97->dev); ac97->dev.bus = NULL; return err; } -- GitLab From 9fa248c65bdbf5af0a2f74dd38575acfc8dfd2bf Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Thu, 20 Oct 2022 17:18:58 +0200 Subject: [PATCH 0234/1927] fuse: fix readdir cache race There's a race in fuse's readdir cache that can result in an uninitilized page being read. The page lock is supposed to prevent this from happening but in the following case it doesn't: Two fuse_add_dirent_to_cache() start out and get the same parameters (size=0,offset=0). One of them wins the race to create and lock the page, after which it fills in data, sets rdc.size and unlocks the page. In the meantime the page gets evicted from the cache before the other instance gets to run. That one also creates the page, but finds the size to be mismatched, bails out and leaves the uninitialized page in the cache. Fix by marking a filled page uptodate and ignoring non-uptodate pages. Reported-by: Frank Sorenson Fixes: 5d7bc7e8680c ("fuse: allow using readdir cache") Cc: # v4.20 Signed-off-by: Miklos Szeredi --- fs/fuse/readdir.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c index b4e5657110457..e8deaacf1832a 100644 --- a/fs/fuse/readdir.c +++ b/fs/fuse/readdir.c @@ -77,8 +77,10 @@ static void fuse_add_dirent_to_cache(struct file *file, goto unlock; addr = kmap_local_page(page); - if (!offset) + if (!offset) { clear_page(addr); + SetPageUptodate(page); + } memcpy(addr + offset, dirent, reclen); kunmap_local(addr); fi->rdc.size = (index << PAGE_SHIFT) + offset + reclen; @@ -516,6 +518,12 @@ static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) page = find_get_page_flags(file->f_mapping, index, FGP_ACCESSED | FGP_LOCK); + /* Page gone missing, then re-added to cache, but not initialized? */ + if (page && !PageUptodate(page)) { + unlock_page(page); + put_page(page); + page = NULL; + } spin_lock(&fi->rdc.lock); if (!page) { /* -- GitLab From 54de93cd8740d52a83728802b4270f953d1a636f Mon Sep 17 00:00:00 2001 From: Petr Vorel Date: Thu, 20 Oct 2022 17:06:45 +0200 Subject: [PATCH 0235/1927] kernel/utsname_sysctl.c: Add missing enum uts_proc value bfca3dd3d068 added new struct ctl_table uts_kern_table[], but not new enum uts_proc value. It broke the notification mechanism between the sethostname syscall and the pollers of /proc/sys/kernel/hostname. The table uts_kern_table is addressed within uts_proc_notify by the enum value, that's why new enum value is needed. Fixes: bfca3dd3d068 ("kernel/utsname_sysctl.c: print kernel arch") Reported-by: Torsten Hilbrich Signed-off-by: Petr Vorel Link: https://lore.kernel.org/r/20221020150645.11719-1-pvorel@suse.cz Signed-off-by: Greg Kroah-Hartman --- include/linux/utsname.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/utsname.h b/include/linux/utsname.h index 2b1737c9b244d..bf7613ba412bf 100644 --- a/include/linux/utsname.h +++ b/include/linux/utsname.h @@ -10,6 +10,7 @@ #include enum uts_proc { + UTS_PROC_ARCH, UTS_PROC_OSTYPE, UTS_PROC_OSRELEASE, UTS_PROC_VERSION, -- GitLab From 13cf24e00665c9751951a422756d975812b71173 Mon Sep 17 00:00:00 2001 From: Guo Xuenan Date: Tue, 18 Oct 2022 14:32:35 -0700 Subject: [PATCH 0236/1927] xfs: fix exception caused by unexpected illegal bestcount in leaf dir For leaf dir, In most cases, there should be as many bestfree slots as the dir data blocks that can fit under i_size (except for [1]). Root cause is we don't examin the number bestfree slots, when the slots number less than dir data blocks, if we need to allocate new dir data block and update the bestfree array, we will use the dir block number as index to assign bestfree array, while we did not check the leaf buf boundary which may cause UAF or other memory access problem. This issue can also triggered with test cases xfs/473 from fstests. According to Dave Chinner & Darrick's suggestion, adding buffer verifier to detect this abnormal situation in time. Simplify the testcase for fstest xfs/554 [1] The error log is shown as follows: ================================================================== BUG: KASAN: use-after-free in xfs_dir2_leaf_addname+0x1995/0x1ac0 Write of size 2 at addr ffff88810168b000 by task touch/1552 CPU: 5 PID: 1552 Comm: touch Not tainted 6.0.0-rc3+ #101 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014 Call Trace: dump_stack_lvl+0x4d/0x66 print_report.cold+0xf6/0x691 kasan_report+0xa8/0x120 xfs_dir2_leaf_addname+0x1995/0x1ac0 xfs_dir_createname+0x58c/0x7f0 xfs_create+0x7af/0x1010 xfs_generic_create+0x270/0x5e0 path_openat+0x270b/0x3450 do_filp_open+0x1cf/0x2b0 do_sys_openat2+0x46b/0x7a0 do_sys_open+0xb7/0x130 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7fe4d9e9312b Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 91 00 00 00 48 8b 4c 24 28 64 48 33 0c 25 RSP: 002b:00007ffda4c16c20 EFLAGS: 00000246 ORIG_RAX: 0000000000000101 RAX: ffffffffffffffda RBX: 0000000000000001 RCX: 00007fe4d9e9312b RDX: 0000000000000941 RSI: 00007ffda4c17f33 RDI: 00000000ffffff9c RBP: 00007ffda4c17f33 R08: 0000000000000000 R09: 0000000000000000 R10: 00000000000001b6 R11: 0000000000000246 R12: 0000000000000941 R13: 00007fe4d9f631a4 R14: 00007ffda4c17f33 R15: 0000000000000000 The buggy address belongs to the physical page: page:ffffea000405a2c0 refcount:0 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x10168b flags: 0x2fffff80000000(node=0|zone=2|lastcpupid=0x1fffff) raw: 002fffff80000000 ffffea0004057788 ffffea000402dbc8 0000000000000000 raw: 0000000000000000 0000000000170000 00000000ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff88810168af00: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ffff88810168af80: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 >ffff88810168b000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ^ ffff88810168b080: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ffff88810168b100: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ================================================================== Disabling lock debugging due to kernel taint 00000000: 58 44 44 33 5b 53 35 c2 00 00 00 00 00 00 00 78 XDD3[S5........x XFS (sdb): Internal error xfs_dir2_data_use_free at line 1200 of file fs/xfs/libxfs/xfs_dir2_data.c. Caller xfs_dir2_data_use_free+0x28a/0xeb0 CPU: 5 PID: 1552 Comm: touch Tainted: G B 6.0.0-rc3+ Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014 Call Trace: dump_stack_lvl+0x4d/0x66 xfs_corruption_error+0x132/0x150 xfs_dir2_data_use_free+0x198/0xeb0 xfs_dir2_leaf_addname+0xa59/0x1ac0 xfs_dir_createname+0x58c/0x7f0 xfs_create+0x7af/0x1010 xfs_generic_create+0x270/0x5e0 path_openat+0x270b/0x3450 do_filp_open+0x1cf/0x2b0 do_sys_openat2+0x46b/0x7a0 do_sys_open+0xb7/0x130 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7fe4d9e9312b Code: 25 00 00 41 00 3d 00 00 41 00 74 4b 64 8b 04 25 18 00 00 00 85 c0 75 67 44 89 e2 48 89 ee bf 9c ff ff ff b8 01 01 00 00 0f 05 <48> 3d 00 f0 ff ff 0f 87 91 00 00 00 48 8b 4c 24 28 64 48 33 0c 25 RSP: 002b:00007ffda4c16c20 EFLAGS: 00000246 ORIG_RAX: 0000000000000101 RAX: ffffffffffffffda RBX: 0000000000000001 RCX: 00007fe4d9e9312b RDX: 0000000000000941 RSI: 00007ffda4c17f46 RDI: 00000000ffffff9c RBP: 00007ffda4c17f46 R08: 0000000000000000 R09: 0000000000000001 R10: 00000000000001b6 R11: 0000000000000246 R12: 0000000000000941 R13: 00007fe4d9f631a4 R14: 00007ffda4c17f46 R15: 0000000000000000 XFS (sdb): Corruption detected. Unmount and run xfs_repair [1] https://lore.kernel.org/all/20220928095355.2074025-1-guoxuenan@huawei.com/ Reviewed-by: Hou Tao Signed-off-by: Guo Xuenan Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_dir2_leaf.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c index d9b66306a9a77..cb9e950a911d8 100644 --- a/fs/xfs/libxfs/xfs_dir2_leaf.c +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -146,6 +146,8 @@ xfs_dir3_leaf_check_int( xfs_dir2_leaf_tail_t *ltp; int stale; int i; + bool isleaf1 = (hdr->magic == XFS_DIR2_LEAF1_MAGIC || + hdr->magic == XFS_DIR3_LEAF1_MAGIC); ltp = xfs_dir2_leaf_tail_p(geo, leaf); @@ -158,8 +160,7 @@ xfs_dir3_leaf_check_int( return __this_address; /* Leaves and bests don't overlap in leaf format. */ - if ((hdr->magic == XFS_DIR2_LEAF1_MAGIC || - hdr->magic == XFS_DIR3_LEAF1_MAGIC) && + if (isleaf1 && (char *)&hdr->ents[hdr->count] > (char *)xfs_dir2_leaf_bests_p(ltp)) return __this_address; @@ -175,6 +176,10 @@ xfs_dir3_leaf_check_int( } if (hdr->ents[i].address == cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) stale++; + if (isleaf1 && xfs_dir2_dataptr_to_db(geo, + be32_to_cpu(hdr->ents[i].address)) >= + be32_to_cpu(ltp->bestcount)) + return __this_address; } if (hdr->stale != stale) return __this_address; -- GitLab From fc93812c725068e6a491ce574f058a4530130c00 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Tue, 18 Oct 2022 14:37:39 -0700 Subject: [PATCH 0237/1927] xfs: remove redundant pointer lip The assignment to pointer lip is not really required, the pointer lip is redundant and can be removed. Cleans up clang-scan warning: warning: Although the value stored to 'lip' is used in the enclosing expression, the value is never actually read from 'lip' [deadcode.DeadStores] Signed-off-by: Colin Ian King Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_trans_ail.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c index 16fbf2a1144c1..f51df7d94ef74 100644 --- a/fs/xfs/xfs_trans_ail.c +++ b/fs/xfs/xfs_trans_ail.c @@ -730,11 +730,10 @@ void xfs_ail_push_all_sync( struct xfs_ail *ailp) { - struct xfs_log_item *lip; DEFINE_WAIT(wait); spin_lock(&ailp->ail_lock); - while ((lip = xfs_ail_max(ailp)) != NULL) { + while (xfs_ail_max(ailp) != NULL) { prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE); wake_up_process(ailp->ail_task); spin_unlock(&ailp->ail_lock); -- GitLab From cf4f4c12dea7a977a143c8fe5af1740b7f9876f8 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Tue, 18 Oct 2022 14:38:14 -0700 Subject: [PATCH 0238/1927] xfs: fix memory leak in xfs_errortag_init When `xfs_sysfs_init` returns failed, `mp->m_errortag` needs to free. Otherwise kmemleak would report memory leak after mounting xfs image: unreferenced object 0xffff888101364900 (size 192): comm "mount", pid 13099, jiffies 4294915218 (age 335.207s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<00000000f08ad25c>] __kmalloc+0x41/0x1b0 [<00000000dca9aeb6>] kmem_alloc+0xfd/0x430 [<0000000040361882>] xfs_errortag_init+0x20/0x110 [<00000000b384a0f6>] xfs_mountfs+0x6ea/0x1a30 [<000000003774395d>] xfs_fs_fill_super+0xe10/0x1a80 [<000000009cf07b6c>] get_tree_bdev+0x3e7/0x700 [<00000000046b5426>] vfs_get_tree+0x8e/0x2e0 [<00000000952ec082>] path_mount+0xf8c/0x1990 [<00000000beb1f838>] do_mount+0xee/0x110 [<000000000e9c41bb>] __x64_sys_mount+0x14b/0x1f0 [<00000000f7bb938e>] do_syscall_64+0x3b/0x90 [<000000003fcd67a9>] entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: c68401011522 ("xfs: expose errortag knobs via sysfs") Signed-off-by: Zeng Heng Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_error.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c index 7db588ed0be59..c6b2aabd6f187 100644 --- a/fs/xfs/xfs_error.c +++ b/fs/xfs/xfs_error.c @@ -234,13 +234,18 @@ int xfs_errortag_init( struct xfs_mount *mp) { + int ret; + mp->m_errortag = kmem_zalloc(sizeof(unsigned int) * XFS_ERRTAG_MAX, KM_MAYFAIL); if (!mp->m_errortag) return -ENOMEM; - return xfs_sysfs_init(&mp->m_errortag_kobj, &xfs_errortag_ktype, - &mp->m_kobj, "errortag"); + ret = xfs_sysfs_init(&mp->m_errortag_kobj, &xfs_errortag_ktype, + &mp->m_kobj, "errortag"); + if (ret) + kmem_free(mp->m_errortag); + return ret; } void -- GitLab From d08af40340cad0e025d643c3982781a8f99d5032 Mon Sep 17 00:00:00 2001 From: Li Zetao Date: Tue, 18 Oct 2022 14:38:29 -0700 Subject: [PATCH 0239/1927] xfs: Fix unreferenced object reported by kmemleak in xfs_sysfs_init() kmemleak reported a sequence of memory leaks, and one of them indicated we failed to free a pointer: comm "mount", pid 19610, jiffies 4297086464 (age 60.635s) hex dump (first 8 bytes): 73 64 61 00 81 88 ff ff sda..... backtrace: [<00000000d77f3e04>] kstrdup_const+0x46/0x70 [<00000000e51fa804>] kobject_set_name_vargs+0x2f/0xb0 [<00000000247cd595>] kobject_init_and_add+0xb0/0x120 [<00000000f9139aaf>] xfs_mountfs+0x367/0xfc0 [<00000000250d3caf>] xfs_fs_fill_super+0xa16/0xdc0 [<000000008d873d38>] get_tree_bdev+0x256/0x390 [<000000004881f3fa>] vfs_get_tree+0x41/0xf0 [<000000008291ab52>] path_mount+0x9b3/0xdd0 [<0000000022ba8f2d>] __x64_sys_mount+0x190/0x1d0 As mentioned in kobject_init_and_add() comment, if this function returns an error, kobject_put() must be called to properly clean up the memory associated with the object. Apparently, xfs_sysfs_init() does not follow such a requirement. When kobject_init_and_add() returns an error, the space of kobj->kobject.name alloced by kstrdup_const() is unfree, which will cause the above stack. Fix it by adding kobject_put() when kobject_init_and_add returns an error. Fixes: a31b1d3d89e4 ("xfs: add xfs_mount sysfs kobject") Signed-off-by: Li Zetao Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/xfs_sysfs.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/fs/xfs/xfs_sysfs.h b/fs/xfs/xfs_sysfs.h index 43585850f1546..513095e353a5b 100644 --- a/fs/xfs/xfs_sysfs.h +++ b/fs/xfs/xfs_sysfs.h @@ -33,10 +33,15 @@ xfs_sysfs_init( const char *name) { struct kobject *parent; + int err; parent = parent_kobj ? &parent_kobj->kobject : NULL; init_completion(&kobj->complete); - return kobject_init_and_add(&kobj->kobject, ktype, parent, "%s", name); + err = kobject_init_and_add(&kobj->kobject, ktype, parent, "%s", name); + if (err) + kobject_put(&kobj->kobject); + + return err; } static inline void -- GitLab From 23722fb46725da42b80bc55a91a9bac69e35188a Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Thu, 21 Jul 2022 14:03:29 +0100 Subject: [PATCH 0240/1927] coresight: Fix possible deadlock with lock dependency With lockdeps enabled, we get the following warning: ====================================================== WARNING: possible circular locking dependency detected ------------------------------------------------------ kworker/u12:1/53 is trying to acquire lock: ffff80000adce220 (coresight_mutex){+.+.}-{4:4}, at: coresight_set_assoc_ectdev_mutex+0x3c/0x5c but task is already holding lock: ffff80000add1f60 (ect_mutex){+.+.}-{4:4}, at: cti_probe+0x318/0x394 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #1 (ect_mutex){+.+.}-{4:4}: __mutex_lock_common+0xd8/0xe60 mutex_lock_nested+0x44/0x50 cti_add_assoc_to_csdev+0x4c/0x184 coresight_register+0x2f0/0x314 tmc_probe+0x33c/0x414 -> #0 (coresight_mutex){+.+.}-{4:4}: __lock_acquire+0x1a20/0x32d0 lock_acquire+0x160/0x308 __mutex_lock_common+0xd8/0xe60 mutex_lock_nested+0x44/0x50 coresight_set_assoc_ectdev_mutex+0x3c/0x5c cti_update_conn_xrefs+0x6c/0xf8 cti_probe+0x33c/0x394 other info that might help us debug this: Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(ect_mutex); lock(coresight_mutex); lock(ect_mutex); lock(coresight_mutex); *** DEADLOCK *** 4 locks held by kworker/u12:1/53: #0: ((wq_completion)events_unbound){+.+.}-{0:0}, at: process_one_work+0x1fc/0x63c #1: (deferred_probe_work){+.+.}-{0:0}, at: process_one_work+0x228/0x63c #2: (&dev->mutex){....}-{4:4}, at: __device_attach+0x48/0x1a8 #3: (ect_mutex){+.+.}-{4:4}, at: cti_probe+0x318/0x394 To fix the same, call cti_add_assoc_to_csdev without the holding coresight_mutex and confine the locking while setting the associated ect / cti device using coresight_set_assoc_ectdev_mutex(). Fixes: 177af8285b59 ("coresight: cti: Enable CTI associated with devices") Cc: Mathieu Poirier Cc: Suzuki K Poulose Cc: Mike Leach Cc: Leo Yan Signed-off-by: Sudeep Holla Reviewed-by: Mike Leach Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20220721130329.3787211-1-sudeep.holla@arm.com --- drivers/hwtracing/coresight/coresight-core.c | 7 ++++--- drivers/hwtracing/coresight/coresight-cti-core.c | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index d5dbc67bacb44..f3068175ca9d9 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -1687,14 +1687,15 @@ struct coresight_device *coresight_register(struct coresight_desc *desc) ret = coresight_fixup_device_conns(csdev); if (!ret) ret = coresight_fixup_orphan_conns(csdev); - if (!ret && cti_assoc_ops && cti_assoc_ops->add) - cti_assoc_ops->add(csdev); out_unlock: mutex_unlock(&coresight_mutex); /* Success */ - if (!ret) + if (!ret) { + if (cti_assoc_ops && cti_assoc_ops->add) + cti_assoc_ops->add(csdev); return csdev; + } /* Unregister the device if needed */ if (registered) { diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 8988b2ed2ea6f..1be92342b5b98 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -541,7 +541,7 @@ cti_match_fixup_csdev(struct cti_device *ctidev, const char *node_name, /* * Search the cti list to add an associated CTI into the supplied CS device * This will set the association if CTI declared before the CS device. - * (called from coresight_register() with coresight_mutex locked). + * (called from coresight_register() without coresight_mutex locked). */ static void cti_add_assoc_to_csdev(struct coresight_device *csdev) { @@ -569,7 +569,8 @@ static void cti_add_assoc_to_csdev(struct coresight_device *csdev) * if we found a matching csdev then update the ECT * association pointer for the device with this CTI. */ - csdev->ect_dev = ect_item->csdev; + coresight_set_assoc_ectdev_mutex(csdev->ect_dev, + ect_item->csdev); break; } } -- GitLab From 665c157e0204176023860b51a46528ba0ba62c33 Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 5 Oct 2022 14:14:52 +0100 Subject: [PATCH 0241/1927] coresight: cti: Fix hang in cti_disable_hw() cti_enable_hw() and cti_disable_hw() are called from an atomic context so shouldn't use runtime PM because it can result in a sleep when communicating with firmware. Since commit 3c6656337852 ("Revert "firmware: arm_scmi: Add clock management to the SCMI power domain""), this causes a hang on Juno when running the Perf Coresight tests or running this command: perf record -e cs_etm//u -- ls This was also missed until the revert commit because pm_runtime_put() was called with the wrong device until commit 692c9a499b28 ("coresight: cti: Correct the parameter for pm_runtime_put") With lock and scheduler debugging enabled the following is output: coresight cti_sys0: cti_enable_hw -- dev:cti_sys0 parent: 20020000.cti BUG: sleeping function called from invalid context at drivers/base/power/runtime.c:1151 in_atomic(): 1, irqs_disabled(): 128, non_block: 0, pid: 330, name: perf-exec preempt_count: 2, expected: 0 RCU nest depth: 0, expected: 0 INFO: lockdep is turned off. irq event stamp: 0 hardirqs last enabled at (0): [<0000000000000000>] 0x0 hardirqs last disabled at (0): [] copy_process+0xa0c/0x1948 softirqs last enabled at (0): [] copy_process+0xa0c/0x1948 softirqs last disabled at (0): [<0000000000000000>] 0x0 CPU: 3 PID: 330 Comm: perf-exec Not tainted 6.0.0-00053-g042116d99298 #7 Hardware name: ARM LTD ARM Juno Development Platform/ARM Juno Development Platform, BIOS EDK II Sep 13 2022 Call trace: dump_backtrace+0x134/0x140 show_stack+0x20/0x58 dump_stack_lvl+0x8c/0xb8 dump_stack+0x18/0x34 __might_resched+0x180/0x228 __might_sleep+0x50/0x88 __pm_runtime_resume+0xac/0xb0 cti_enable+0x44/0x120 coresight_control_assoc_ectdev+0xc0/0x150 coresight_enable_path+0xb4/0x288 etm_event_start+0x138/0x170 etm_event_add+0x48/0x70 event_sched_in.isra.122+0xb4/0x280 merge_sched_in+0x1fc/0x3d0 visit_groups_merge.constprop.137+0x16c/0x4b0 ctx_sched_in+0x114/0x1f0 perf_event_sched_in+0x60/0x90 ctx_resched+0x68/0xb0 perf_event_exec+0x138/0x508 begin_new_exec+0x52c/0xd40 load_elf_binary+0x6b8/0x17d0 bprm_execve+0x360/0x7f8 do_execveat_common.isra.47+0x218/0x238 __arm64_sys_execve+0x48/0x60 invoke_syscall+0x4c/0x110 el0_svc_common.constprop.4+0xfc/0x120 do_el0_svc+0x34/0xc0 el0_svc+0x40/0x98 el0t_64_sync_handler+0x98/0xc0 el0t_64_sync+0x170/0x174 Fix the issue by removing the runtime PM calls completely. They are not needed here because it must have already been done when building the path for a trace. Fixes: 835d722ba10a ("coresight: cti: Initial CoreSight CTI Driver") Reported-by: Aishwarya TCV Reported-by: Cristian Marussi Suggested-by: Suzuki Poulose Signed-off-by: James Clark Reviewed-by: Mike Leach Tested-by: Mike Leach Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20221005131452.1506328-1-james.clark@arm.com --- drivers/hwtracing/coresight/coresight-cti-core.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 1be92342b5b98..4a02ae23d3a06 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -94,7 +94,6 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) unsigned long flags; int rc = 0; - pm_runtime_get_sync(dev->parent); spin_lock_irqsave(&drvdata->spinlock, flags); /* no need to do anything if enabled or unpowered*/ @@ -119,7 +118,6 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) /* cannot enable due to error */ cti_err_not_enabled: spin_unlock_irqrestore(&drvdata->spinlock, flags); - pm_runtime_put(dev->parent); return rc; } @@ -175,7 +173,6 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); - pm_runtime_put(dev->parent); return 0; /* not disabled this call */ -- GitLab From 31d8aaa87fcef1be5932f3813ea369e21bd3b11d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 20 Oct 2022 10:58:14 -0700 Subject: [PATCH 0242/1927] rcu: Keep synchronize_rcu() from enabling irqs in early boot Making polled RCU grace periods account for expedited grace periods required acquiring the leaf rcu_node structure's lock during early boot, but after rcu_init() was called. This lock is irq-disabled, but the code incorrectly assumes that irqs are always disabled when invoking synchronize_rcu(). The exception is early boot before the scheduler has started, which means that upon return from synchronize_rcu(), irqs will be incorrectly enabled. This commit fixes this bug by using irqsave/irqrestore locking primitives. Fixes: bf95b2bc3e42 ("rcu: Switch polled grace-period APIs to ->gp_seq_polled") Reported-by: Steven Rostedt Signed-off-by: Paul E. McKenney --- kernel/rcu/tree.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 6bb8e72bc8151..93416afebd59c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1403,30 +1403,32 @@ static void rcu_poll_gp_seq_end(unsigned long *snap) // where caller does not hold the root rcu_node structure's lock. static void rcu_poll_gp_seq_start_unlocked(unsigned long *snap) { + unsigned long flags; struct rcu_node *rnp = rcu_get_root(); if (rcu_init_invoked()) { lockdep_assert_irqs_enabled(); - raw_spin_lock_irq_rcu_node(rnp); + raw_spin_lock_irqsave_rcu_node(rnp, flags); } rcu_poll_gp_seq_start(snap); if (rcu_init_invoked()) - raw_spin_unlock_irq_rcu_node(rnp); + raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } // Make the polled API aware of the end of a grace period, but where // caller does not hold the root rcu_node structure's lock. static void rcu_poll_gp_seq_end_unlocked(unsigned long *snap) { + unsigned long flags; struct rcu_node *rnp = rcu_get_root(); if (rcu_init_invoked()) { lockdep_assert_irqs_enabled(); - raw_spin_lock_irq_rcu_node(rnp); + raw_spin_lock_irqsave_rcu_node(rnp, flags); } rcu_poll_gp_seq_end(snap); if (rcu_init_invoked()) - raw_spin_unlock_irq_rcu_node(rnp); + raw_spin_unlock_irqrestore_rcu_node(rnp, flags); } /* -- GitLab From cf00b33058b196b4db928419dde68993b15a975b Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 15 Aug 2022 16:40:43 +0100 Subject: [PATCH 0243/1927] cxl/mbox: Add a check on input payload size A bug in the LSA code resulted in transfers slightly larger than the mailbox size. Let us make it easier to catch similar issues in future by adding a low level check. Signed-off-by: Jonathan Cameron Link: https://lore.kernel.org/r/20220815154044.24733-2-Jonathan.Cameron@huawei.com Signed-off-by: Dan Williams --- drivers/cxl/core/mbox.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 16176b9278b4e..0c90f13870a43 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -174,7 +174,7 @@ int cxl_mbox_send_cmd(struct cxl_dev_state *cxlds, u16 opcode, void *in, }; int rc; - if (out_size > cxlds->payload_size) + if (in_size > cxlds->payload_size || out_size > cxlds->payload_size) return -E2BIG; rc = cxlds->mbox_send(cxlds, &mbox_cmd); -- GitLab From 2816e24b0510e0c185c0c46acff1ce7aa4c4443f Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Thu, 18 Aug 2022 17:42:10 +0100 Subject: [PATCH 0244/1927] cxl/region: Fix null pointer dereference due to pass through decoder commit Not all decoders have a commit callback. The CXL specification allows a host bridge with a single root port to have no explicit HDM decoders. Currently the region driver assumes there are none. As such the CXL core creates a special pass through decoder instance without a commit callback. Prior to this patch, the ->commit() callback was called unconditionally. Thus a configuration with 1 Host Bridge, 1 Root Port, 1 switch with multiple downstream ports below which there are multiple CXL type 3 devices results in a situation where committing the region causes a null pointer dereference. Reported-by: Bobo WL Fixes: 176baefb2eb5 ("cxl/hdm: Commit decoder state to hardware") Signed-off-by: Jonathan Cameron Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/20220818164210.2084-1-Jonathan.Cameron@huawei.com Signed-off-by: Dan Williams --- drivers/cxl/core/region.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 4011480169784..c49d9a5f1091c 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -174,7 +174,8 @@ static int cxl_region_decode_commit(struct cxl_region *cxlr) iter = to_cxl_port(iter->dev.parent)) { cxl_rr = cxl_rr_load(iter, cxlr); cxld = cxl_rr->decoder; - rc = cxld->commit(cxld); + if (cxld->commit) + rc = cxld->commit(cxld); if (rc) break; } -- GitLab From f010c75c05299ecd65adfd31a7841eea3476ce1f Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Mon, 15 Aug 2022 16:40:44 +0100 Subject: [PATCH 0245/1927] cxl/pmem: Fix failure to account for 8 byte header for writes to the device LSA. Writes to the device must include an offset and size as defined in CXL 2.0 8.2.9.5.2.4 Set LSA (Opcode 4103h) Fixes tag is non obvious as this code has been through several reworks and variable names + wasn't in use until the addition of the region code. Due to a bug in QEMU CXL emulation this overrun resulted in QEMU crashing. Reported-by: Bobo WL Signed-off-by: Jonathan Cameron Fixes: 60b8f17215de ("cxl/pmem: Translate NVDIMM label commands to CXL label commands") Link: https://lore.kernel.org/r/20220815154044.24733-3-Jonathan.Cameron@huawei.com Signed-off-by: Dan Williams --- drivers/cxl/pmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 7dc0a2fa1a6b6..115a7b79f3439 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -107,7 +107,7 @@ static int cxl_pmem_get_config_size(struct cxl_dev_state *cxlds, *cmd = (struct nd_cmd_get_config_size) { .config_size = cxlds->lsa_size, - .max_xfer = cxlds->payload_size, + .max_xfer = cxlds->payload_size - sizeof(struct cxl_mbox_set_lsa), }; return 0; -- GitLab From dbe69b29988465b011f198f2797b1c2b6980b50e Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 18 Oct 2022 09:59:34 +0200 Subject: [PATCH 0246/1927] bpf: Fix dispatcher patchable function entry to 5 bytes nop The patchable_function_entry(5) might output 5 single nop instructions (depends on toolchain), which will clash with bpf_arch_text_poke check for 5 bytes nop instruction. Adding early init call for dispatcher that checks and change the patchable entry into expected 5 nop instruction if needed. There's no need to take text_mutex, because we are using it in early init call which is called at pre-smp time. Fixes: ceea991a019c ("bpf: Move bpf_dispatcher function out of ftrace locations") Signed-off-by: Jiri Olsa Acked-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/20221018075934.574415-1-jolsa@kernel.org Signed-off-by: Alexei Starovoitov --- arch/x86/net/bpf_jit_comp.c | 13 +++++++++++++ include/linux/bpf.h | 14 +++++++++++++- kernel/bpf/dispatcher.c | 6 ++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 99620428ad785..00127abd89ee5 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -388,6 +389,18 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, return ret; } +int __init bpf_arch_init_dispatcher_early(void *ip) +{ + const u8 *nop_insn = x86_nops[5]; + + if (is_endbr(*(u32 *)ip)) + ip += ENDBR_INSN_SIZE; + + if (memcmp(ip, nop_insn, X86_PATCH_SIZE)) + text_poke_early(ip, nop_insn, X86_PATCH_SIZE); + return 0; +} + int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, void *old_addr, void *new_addr) { diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 9e7d46d16032f..0566705c1d4ea 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -27,6 +27,7 @@ #include #include #include +#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -970,6 +971,8 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); +int __init bpf_arch_init_dispatcher_early(void *ip); + #define BPF_DISPATCHER_INIT(_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ .func = &_name##_func, \ @@ -983,6 +986,13 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func }, \ } +#define BPF_DISPATCHER_INIT_CALL(_name) \ + static int __init _name##_init(void) \ + { \ + return bpf_arch_init_dispatcher_early(_name##_func); \ + } \ + early_initcall(_name##_init) + #ifdef CONFIG_X86_64 #define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5))) #else @@ -1000,7 +1010,9 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func } \ EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ struct bpf_dispatcher bpf_dispatcher_##name = \ - BPF_DISPATCHER_INIT(bpf_dispatcher_##name); + BPF_DISPATCHER_INIT(bpf_dispatcher_##name); \ + BPF_DISPATCHER_INIT_CALL(bpf_dispatcher_##name); + #define DECLARE_BPF_DISPATCHER(name) \ unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index fa64b80b8bcab..04f0a045dcaa6 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -4,6 +4,7 @@ #include #include #include +#include /* The BPF dispatcher is a multiway branch code generator. The * dispatcher is a mechanism to avoid the performance penalty of an @@ -90,6 +91,11 @@ int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int n return -ENOTSUPP; } +int __weak __init bpf_arch_init_dispatcher_early(void *ip) +{ + return -ENOTSUPP; +} + static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf) { s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; -- GitLab From 82cb4e4612c633a9ce320e1773114875604a3cce Mon Sep 17 00:00:00 2001 From: Xin Long Date: Tue, 18 Oct 2022 15:19:50 -0400 Subject: [PATCH 0247/1927] tipc: fix a null-ptr-deref in tipc_topsrv_accept syzbot found a crash in tipc_topsrv_accept: KASAN: null-ptr-deref in range [0x0000000000000008-0x000000000000000f] Workqueue: tipc_rcv tipc_topsrv_accept RIP: 0010:kernel_accept+0x22d/0x350 net/socket.c:3487 Call Trace: tipc_topsrv_accept+0x197/0x280 net/tipc/topsrv.c:460 process_one_work+0x991/0x1610 kernel/workqueue.c:2289 worker_thread+0x665/0x1080 kernel/workqueue.c:2436 kthread+0x2e4/0x3a0 kernel/kthread.c:376 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:306 It was caused by srv->listener that might be set to null by tipc_topsrv_stop() in net .exit whereas it's still used in tipc_topsrv_accept() worker. srv->listener is protected by srv->idr_lock in tipc_topsrv_stop(), so add a check for srv->listener under srv->idr_lock in tipc_topsrv_accept() to avoid the null-ptr-deref. To ensure the lsock is not released during the tipc_topsrv_accept(), move sock_release() after tipc_topsrv_work_stop() where it's waiting until the tipc_topsrv_accept worker to be done. Note that sk_callback_lock is used to protect sk->sk_user_data instead of srv->listener, and it should check srv in tipc_topsrv_listener_data_ready() instead. This also ensures that no more tipc_topsrv_accept worker will be started after tipc_conn_close() is called in tipc_topsrv_stop() where it sets sk->sk_user_data to null. Fixes: 0ef897be12b8 ("tipc: separate topology server listener socket from subcsriber sockets") Reported-by: syzbot+c5ce866a8d30f4be0651@syzkaller.appspotmail.com Signed-off-by: Xin Long Acked-by: Jon Maloy Link: https://lore.kernel.org/r/4eee264380c409c61c6451af1059b7fb271a7e7b.1666120790.git.lucien.xin@gmail.com Signed-off-by: Jakub Kicinski --- net/tipc/topsrv.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/net/tipc/topsrv.c b/net/tipc/topsrv.c index 14fd05fd6107d..d92ec92f0b71d 100644 --- a/net/tipc/topsrv.c +++ b/net/tipc/topsrv.c @@ -450,12 +450,19 @@ static void tipc_conn_data_ready(struct sock *sk) static void tipc_topsrv_accept(struct work_struct *work) { struct tipc_topsrv *srv = container_of(work, struct tipc_topsrv, awork); - struct socket *lsock = srv->listener; - struct socket *newsock; + struct socket *newsock, *lsock; struct tipc_conn *con; struct sock *newsk; int ret; + spin_lock_bh(&srv->idr_lock); + if (!srv->listener) { + spin_unlock_bh(&srv->idr_lock); + return; + } + lsock = srv->listener; + spin_unlock_bh(&srv->idr_lock); + while (1) { ret = kernel_accept(lsock, &newsock, O_NONBLOCK); if (ret < 0) @@ -489,7 +496,7 @@ static void tipc_topsrv_listener_data_ready(struct sock *sk) read_lock_bh(&sk->sk_callback_lock); srv = sk->sk_user_data; - if (srv->listener) + if (srv) queue_work(srv->rcv_wq, &srv->awork); read_unlock_bh(&sk->sk_callback_lock); } @@ -699,8 +706,9 @@ static void tipc_topsrv_stop(struct net *net) __module_get(lsock->sk->sk_prot_creator->owner); srv->listener = NULL; spin_unlock_bh(&srv->idr_lock); - sock_release(lsock); + tipc_topsrv_work_stop(srv); + sock_release(lsock); idr_destroy(&srv->conn_idr); kfree(srv); } -- GitLab From 94423589689124e8cd145b38a1034be7f25835b2 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Wed, 19 Oct 2022 14:41:04 +0800 Subject: [PATCH 0248/1927] net: netsec: fix error handling in netsec_register_mdio() If phy_device_register() fails, phy_device_free() need be called to put refcount, so memory of phy device and device name can be freed in callback function. If get_phy_device() fails, mdiobus_unregister() need be called, or it will cause warning in mdiobus_free() and kobject is leaked. Fixes: 533dd11a12f6 ("net: socionext: Add Synquacer NetSec driver") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221019064104.3228892-1-yangyingliang@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/socionext/netsec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index 2240f6d0b89ba..9b46579b5a103 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -1961,11 +1961,13 @@ static int netsec_register_mdio(struct netsec_priv *priv, u32 phy_addr) ret = PTR_ERR(priv->phydev); dev_err(priv->dev, "get_phy_device err(%d)\n", ret); priv->phydev = NULL; + mdiobus_unregister(bus); return -ENODEV; } ret = phy_device_register(priv->phydev); if (ret) { + phy_device_free(priv->phydev); mdiobus_unregister(bus); dev_err(priv->dev, "phy_device_register err(%d)\n", ret); -- GitLab From f8c1c66b99a570c08b9d26e4347276f00e49bba7 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Wed, 19 Oct 2022 10:30:56 +0200 Subject: [PATCH 0249/1927] net: lan966x: Fix the rx drop counter Currently the rx drop is calculated as the sum of multiple HW drop counters. The issue is that not all the HW drop counters were added for the rx drop counter. So if for example you have a police that drops frames, they were not see in the rx drop counter. Fix this by updating how the rx drop counter is calculated. It is required to add also RX_RED_PRIO_* HW counters. Fixes: 12c2d0a5b8e2 ("net: lan966x: add ethtool configuration and statistics") Signed-off-by: Horatiu Vultur Link: https://lore.kernel.org/r/20221019083056.2744282-1-horatiu.vultur@microchip.com Signed-off-by: Jakub Kicinski --- .../net/ethernet/microchip/lan966x/lan966x_ethtool.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c index e58a27fd8b508..fea42542be280 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c @@ -656,7 +656,15 @@ void lan966x_stats_get(struct net_device *dev, stats->rx_dropped = dev->stats.rx_dropped + lan966x->stats[idx + SYS_COUNT_RX_LONG] + lan966x->stats[idx + SYS_COUNT_DR_LOCAL] + - lan966x->stats[idx + SYS_COUNT_DR_TAIL]; + lan966x->stats[idx + SYS_COUNT_DR_TAIL] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_0] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_1] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_2] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_3] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_4] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_5] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_6] + + lan966x->stats[idx + SYS_COUNT_RX_RED_PRIO_7]; for (i = 0; i < LAN966X_NUM_TC; i++) { stats->rx_dropped += -- GitLab From ae108c48b5d2b34bcef3c4fb5076f42c922c426a Mon Sep 17 00:00:00 2001 From: Benjamin Poirier Date: Wed, 19 Oct 2022 18:10:41 +0900 Subject: [PATCH 0250/1927] selftests: net: Fix cross-tree inclusion of scripts When exporting and running a subset of selftests via kselftest, files from parts of the source tree which were not exported are not available. A few tests are trying to source such files. Address the problem by using symlinks. The problem can be reproduced by running: make -C tools/testing/selftests gen_tar TARGETS="drivers/net/bonding" [... extract archive ...] ./run_kselftest.sh or: make kselftest KBUILD_OUTPUT=/tmp/kselftests TARGETS="drivers/net/bonding" Fixes: bbb774d921e2 ("net: Add tests for bonding and team address list management") Fixes: eccd0a80dc7f ("selftests: net: dsa: add a stress test for unlocked FDB operations") Link: https://lore.kernel.org/netdev/40f04ded-0c86-8669-24b1-9a313ca21076@redhat.com/ Reported-by: Jonathan Toppins Signed-off-by: Benjamin Poirier Reviewed-by: Jonathan Toppins Signed-off-by: Jakub Kicinski --- tools/testing/selftests/drivers/net/bonding/Makefile | 4 +++- tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh | 2 +- .../selftests/drivers/net/bonding/net_forwarding_lib.sh | 1 + .../selftests/drivers/net/dsa/test_bridge_fdb_stress.sh | 4 ++-- tools/testing/selftests/drivers/net/team/Makefile | 4 ++++ tools/testing/selftests/drivers/net/team/dev_addr_lists.sh | 4 ++-- tools/testing/selftests/drivers/net/team/lag_lib.sh | 1 + .../testing/selftests/drivers/net/team/net_forwarding_lib.sh | 1 + tools/testing/selftests/lib.mk | 4 ++-- 9 files changed, 17 insertions(+), 8 deletions(-) create mode 120000 tools/testing/selftests/drivers/net/bonding/net_forwarding_lib.sh create mode 120000 tools/testing/selftests/drivers/net/team/lag_lib.sh create mode 120000 tools/testing/selftests/drivers/net/team/net_forwarding_lib.sh diff --git a/tools/testing/selftests/drivers/net/bonding/Makefile b/tools/testing/selftests/drivers/net/bonding/Makefile index e9dab5f9d7738..6b8d2e2f23c2a 100644 --- a/tools/testing/selftests/drivers/net/bonding/Makefile +++ b/tools/testing/selftests/drivers/net/bonding/Makefile @@ -7,6 +7,8 @@ TEST_PROGS := \ bond-lladdr-target.sh \ dev_addr_lists.sh -TEST_FILES := lag_lib.sh +TEST_FILES := \ + lag_lib.sh \ + net_forwarding_lib.sh include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh b/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh index e6fa24eded5b8..5cfe7d8ebc256 100755 --- a/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh +++ b/tools/testing/selftests/drivers/net/bonding/dev_addr_lists.sh @@ -14,7 +14,7 @@ ALL_TESTS=" REQUIRE_MZ=no NUM_NETIFS=0 lib_dir=$(dirname "$0") -source "$lib_dir"/../../../net/forwarding/lib.sh +source "$lib_dir"/net_forwarding_lib.sh source "$lib_dir"/lag_lib.sh diff --git a/tools/testing/selftests/drivers/net/bonding/net_forwarding_lib.sh b/tools/testing/selftests/drivers/net/bonding/net_forwarding_lib.sh new file mode 120000 index 0000000000000..39c96828c5eff --- /dev/null +++ b/tools/testing/selftests/drivers/net/bonding/net_forwarding_lib.sh @@ -0,0 +1 @@ +../../../net/forwarding/lib.sh \ No newline at end of file diff --git a/tools/testing/selftests/drivers/net/dsa/test_bridge_fdb_stress.sh b/tools/testing/selftests/drivers/net/dsa/test_bridge_fdb_stress.sh index dca8be6092b92..a1f269ee84dac 100755 --- a/tools/testing/selftests/drivers/net/dsa/test_bridge_fdb_stress.sh +++ b/tools/testing/selftests/drivers/net/dsa/test_bridge_fdb_stress.sh @@ -18,8 +18,8 @@ NUM_NETIFS=1 REQUIRE_JQ="no" REQUIRE_MZ="no" NETIF_CREATE="no" -lib_dir=$(dirname $0)/../../../net/forwarding -source $lib_dir/lib.sh +lib_dir=$(dirname "$0") +source "$lib_dir"/lib.sh cleanup() { echo "Cleaning up" diff --git a/tools/testing/selftests/drivers/net/team/Makefile b/tools/testing/selftests/drivers/net/team/Makefile index 642d8df1c137b..6a86e61e8bfe5 100644 --- a/tools/testing/selftests/drivers/net/team/Makefile +++ b/tools/testing/selftests/drivers/net/team/Makefile @@ -3,4 +3,8 @@ TEST_PROGS := dev_addr_lists.sh +TEST_FILES := \ + lag_lib.sh \ + net_forwarding_lib.sh + include ../../../lib.mk diff --git a/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh b/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh index debda72629564..9684163949f08 100755 --- a/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh +++ b/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh @@ -11,9 +11,9 @@ ALL_TESTS=" REQUIRE_MZ=no NUM_NETIFS=0 lib_dir=$(dirname "$0") -source "$lib_dir"/../../../net/forwarding/lib.sh +source "$lib_dir"/net_forwarding_lib.sh -source "$lib_dir"/../bonding/lag_lib.sh +source "$lib_dir"/lag_lib.sh destroy() diff --git a/tools/testing/selftests/drivers/net/team/lag_lib.sh b/tools/testing/selftests/drivers/net/team/lag_lib.sh new file mode 120000 index 0000000000000..e1347a10afde6 --- /dev/null +++ b/tools/testing/selftests/drivers/net/team/lag_lib.sh @@ -0,0 +1 @@ +../bonding/lag_lib.sh \ No newline at end of file diff --git a/tools/testing/selftests/drivers/net/team/net_forwarding_lib.sh b/tools/testing/selftests/drivers/net/team/net_forwarding_lib.sh new file mode 120000 index 0000000000000..39c96828c5eff --- /dev/null +++ b/tools/testing/selftests/drivers/net/team/net_forwarding_lib.sh @@ -0,0 +1 @@ +../../../net/forwarding/lib.sh \ No newline at end of file diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index 9d4cb94cf4374..a3ea3d4a206d0 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -70,7 +70,7 @@ endef run_tests: all ifdef building_out_of_srctree @if [ "X$(TEST_PROGS)$(TEST_PROGS_EXTENDED)$(TEST_FILES)" != "X" ]; then \ - rsync -aq $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES) $(OUTPUT); \ + rsync -aLq $(TEST_PROGS) $(TEST_PROGS_EXTENDED) $(TEST_FILES) $(OUTPUT); \ fi @if [ "X$(TEST_PROGS)" != "X" ]; then \ $(call RUN_TESTS, $(TEST_GEN_PROGS) $(TEST_CUSTOM_PROGS) \ @@ -84,7 +84,7 @@ endif define INSTALL_SINGLE_RULE $(if $(INSTALL_LIST),@mkdir -p $(INSTALL_PATH)) - $(if $(INSTALL_LIST),rsync -a $(INSTALL_LIST) $(INSTALL_PATH)/) + $(if $(INSTALL_LIST),rsync -aL $(INSTALL_LIST) $(INSTALL_PATH)/) endef define INSTALL_RULE -- GitLab From b2c0921b926ca69cc399eb356162f35340598112 Mon Sep 17 00:00:00 2001 From: Benjamin Poirier Date: Wed, 19 Oct 2022 18:10:42 +0900 Subject: [PATCH 0251/1927] selftests: net: Fix netdev name mismatch in cleanup lag_lib.sh creates the interfaces dummy1 and dummy2 whereas dev_addr_lists.sh:destroy() deletes the interfaces dummy0 and dummy1. Fix the mismatch in names. Fixes: bbb774d921e2 ("net: Add tests for bonding and team address list management") Signed-off-by: Benjamin Poirier Reviewed-by: Jonathan Toppins Signed-off-by: Jakub Kicinski --- tools/testing/selftests/drivers/net/team/dev_addr_lists.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh b/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh index 9684163949f08..33913112d5ca0 100755 --- a/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh +++ b/tools/testing/selftests/drivers/net/team/dev_addr_lists.sh @@ -18,7 +18,7 @@ source "$lib_dir"/lag_lib.sh destroy() { - local ifnames=(dummy0 dummy1 team0 mv0) + local ifnames=(dummy1 dummy2 team0 mv0) local ifname for ifname in "${ifnames[@]}"; do -- GitLab From c0605cd6750f2db9890c43a91ea4d77be8fb4908 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 19 Oct 2022 17:57:51 +0800 Subject: [PATCH 0252/1927] net: hinic: fix incorrect assignment issue in hinic_set_interrupt_cfg() The value of lli_credit_cnt is incorrectly assigned, fix it. Fixes: a0337c0dee68 ("hinic: add support to set and get irq coalesce") Signed-off-by: Zhengchao Shao Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c index 94f470556295b..27795288c5861 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c @@ -877,7 +877,7 @@ int hinic_set_interrupt_cfg(struct hinic_hwdev *hwdev, if (err) return -EINVAL; - interrupt_info->lli_credit_cnt = temp_info.lli_timer_cnt; + interrupt_info->lli_credit_cnt = temp_info.lli_credit_cnt; interrupt_info->lli_timer_cnt = temp_info.lli_timer_cnt; err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM, -- GitLab From 4c1f602df8956bc0decdafd7e4fc7eef50c550b1 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 19 Oct 2022 17:57:52 +0800 Subject: [PATCH 0253/1927] net: hinic: fix memory leak when reading function table When the input parameter idx meets the expected case option in hinic_dbg_get_func_table(), read_data is not released. Fix it. Fixes: 5215e16244ee ("hinic: add support to query function table") Signed-off-by: Zhengchao Shao Signed-off-by: Jakub Kicinski --- .../net/ethernet/huawei/hinic/hinic_debugfs.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_debugfs.c b/drivers/net/ethernet/huawei/hinic/hinic_debugfs.c index 19eb839177ec2..061952c6c21a4 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_debugfs.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_debugfs.c @@ -85,6 +85,7 @@ static int hinic_dbg_get_func_table(struct hinic_dev *nic_dev, int idx) struct tag_sml_funcfg_tbl *funcfg_table_elem; struct hinic_cmd_lt_rd *read_data; u16 out_size = sizeof(*read_data); + int ret = ~0; int err; read_data = kzalloc(sizeof(*read_data), GFP_KERNEL); @@ -111,20 +112,25 @@ static int hinic_dbg_get_func_table(struct hinic_dev *nic_dev, int idx) switch (idx) { case VALID: - return funcfg_table_elem->dw0.bs.valid; + ret = funcfg_table_elem->dw0.bs.valid; + break; case RX_MODE: - return funcfg_table_elem->dw0.bs.nic_rx_mode; + ret = funcfg_table_elem->dw0.bs.nic_rx_mode; + break; case MTU: - return funcfg_table_elem->dw1.bs.mtu; + ret = funcfg_table_elem->dw1.bs.mtu; + break; case RQ_DEPTH: - return funcfg_table_elem->dw13.bs.cfg_rq_depth; + ret = funcfg_table_elem->dw13.bs.cfg_rq_depth; + break; case QUEUE_NUM: - return funcfg_table_elem->dw13.bs.cfg_q_num; + ret = funcfg_table_elem->dw13.bs.cfg_q_num; + break; } kfree(read_data); - return ~0; + return ret; } static ssize_t hinic_dbg_cmd_read(struct file *filp, char __user *buffer, size_t count, -- GitLab From 363cc87767f6ddcfb9158ad2e2afa2f8d5c4b94e Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 19 Oct 2022 17:57:53 +0800 Subject: [PATCH 0254/1927] net: hinic: fix the issue of CMDQ memory leaks When hinic_set_cmdq_depth() fails in hinic_init_cmdqs(), the cmdq memory is not released correctly. Fix it. Fixes: 72ef908bb3ff ("hinic: add three net_device_ops of vf") Signed-off-by: Zhengchao Shao Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c index 78190e88cd75f..d39eec9c62bf5 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c @@ -924,7 +924,7 @@ int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, err_set_cmdq_depth: hinic_ceq_unregister_cb(&func_to_io->ceqs, HINIC_CEQ_CMDQ); - + free_cmdq(&cmdqs->cmdq[HINIC_CMDQ_SYNC]); err_cmdq_ctxt: hinic_wqs_cmdq_free(&cmdqs->cmdq_pages, cmdqs->saved_wqs, HINIC_MAX_CMDQ_TYPES); -- GitLab From 8ec2f4c6b2e11a4249bba77460f0cfe6d95a82f8 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 19 Oct 2022 17:57:54 +0800 Subject: [PATCH 0255/1927] net: hinic: fix the issue of double release MBOX callback of VF In hinic_vf_func_init(), if VF fails to register information with PF through the MBOX, the MBOX callback function of VF is released once. But it is released again in hinic_init_hwdev(). Remove one. Fixes: 7dd29ee12865 ("hinic: add sriov feature support") Signed-off-by: Zhengchao Shao Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/huawei/hinic/hinic_sriov.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_sriov.c b/drivers/net/ethernet/huawei/hinic/hinic_sriov.c index a5f08b969e3f6..f7e05b41385bf 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_sriov.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_sriov.c @@ -1174,7 +1174,6 @@ int hinic_vf_func_init(struct hinic_hwdev *hwdev) dev_err(&hwdev->hwif->pdev->dev, "Failed to register VF, err: %d, status: 0x%x, out size: 0x%x\n", err, register_info.status, out_size); - hinic_unregister_vf_mbox_cb(hwdev, HINIC_MOD_L2NIC); return -EIO; } } else { -- GitLab From 15a9dbec631cd69dfbbfc4e2cbf90c9dd8432a8f Mon Sep 17 00:00:00 2001 From: Sergiu Moga Date: Wed, 19 Oct 2022 15:09:32 +0300 Subject: [PATCH 0256/1927] net: macb: Specify PHY PM management done by MAC The `macb_resume`/`macb_suspend` methods already call the `phylink_start`/`phylink_stop` methods during their execution so explicitly say that the PM of the PHY is done by MAC by using the `mac_managed_pm` flag of the `struct phylink_config`. This also fixes the warning message issued during resume: WARNING: CPU: 0 PID: 237 at drivers/net/phy/phy_device.c:323 mdio_bus_phy_resume+0x144/0x148 Depends-on: 96de900ae78e ("net: phylink: add mac_managed_pm in phylink_config structure") Fixes: 744d23c71af3 ("net: phy: Warn about incorrect mdio_bus_phy_resume() state") Signed-off-by: Sergiu Moga Reviewed-by: Florian Fainelli Reviewed-by: Claudiu Beznea Link: https://lore.kernel.org/r/20221019120929.63098-1-sergiu.moga@microchip.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/cadence/macb_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 51c9fd6f68a42..4f63f1ba3161c 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -806,6 +806,7 @@ static int macb_mii_probe(struct net_device *dev) bp->phylink_config.dev = &dev->dev; bp->phylink_config.type = PHYLINK_NETDEV; + bp->phylink_config.mac_managed_pm = true; if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { bp->phylink_config.poll_fixed_state = true; -- GitLab From e840d8f4a1b323973052a1af5ad4edafcde8ae3d Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Thu, 20 Oct 2022 11:05:05 +0800 Subject: [PATCH 0257/1927] nfc: virtual_ncidev: Fix memory leak in virtual_nci_send() skb should be free in virtual_nci_send(), otherwise kmemleak will report memleak. Steps for reproduction (simulated in qemu): cd tools/testing/selftests/nci make ./nci_dev BUG: memory leak unreferenced object 0xffff888107588000 (size 208): comm "nci_dev", pid 206, jiffies 4294945376 (age 368.248s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<000000008d94c8fd>] __alloc_skb+0x1da/0x290 [<00000000278bc7f8>] nci_send_cmd+0xa3/0x350 [<0000000081256a22>] nci_reset_req+0x6b/0xa0 [<000000009e721112>] __nci_request+0x90/0x250 [<000000005d556e59>] nci_dev_up+0x217/0x5b0 [<00000000e618ce62>] nfc_dev_up+0x114/0x220 [<00000000981e226b>] nfc_genl_dev_up+0x94/0xe0 [<000000009bb03517>] genl_family_rcv_msg_doit.isra.14+0x228/0x2d0 [<00000000b7f8c101>] genl_rcv_msg+0x35c/0x640 [<00000000c94075ff>] netlink_rcv_skb+0x11e/0x350 [<00000000440cfb1e>] genl_rcv+0x24/0x40 [<0000000062593b40>] netlink_unicast+0x43f/0x640 [<000000001d0b13cc>] netlink_sendmsg+0x73a/0xbf0 [<000000003272487f>] __sys_sendto+0x324/0x370 [<00000000ef9f1747>] __x64_sys_sendto+0xdd/0x1b0 [<000000001e437841>] do_syscall_64+0x3f/0x90 Fixes: e624e6c3e777 ("nfc: Add a virtual nci device driver") Signed-off-by: Shang XiaoJing Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20221020030505.15572-1-shangxiaojing@huawei.com Signed-off-by: Jakub Kicinski --- drivers/nfc/virtual_ncidev.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/nfc/virtual_ncidev.c b/drivers/nfc/virtual_ncidev.c index f577449e49350..85c06dbb2c449 100644 --- a/drivers/nfc/virtual_ncidev.c +++ b/drivers/nfc/virtual_ncidev.c @@ -54,16 +54,19 @@ static int virtual_nci_send(struct nci_dev *ndev, struct sk_buff *skb) mutex_lock(&nci_mutex); if (state != virtual_ncidev_enabled) { mutex_unlock(&nci_mutex); + kfree_skb(skb); return 0; } if (send_buff) { mutex_unlock(&nci_mutex); + kfree_skb(skb); return -1; } send_buff = skb_copy(skb, GFP_KERNEL); mutex_unlock(&nci_mutex); wake_up_interruptible(&wq); + consume_skb(skb); return 0; } -- GitLab From cc67482c9e5f2c80d62f623bcc347c29f9f648e1 Mon Sep 17 00:00:00 2001 From: Hyunwoo Kim Date: Thu, 20 Oct 2022 18:15:44 -0700 Subject: [PATCH 0258/1927] fbdev: smscufx: Fix several use-after-free bugs Several types of UAFs can occur when physically removing a USB device. Adds ufx_ops_destroy() function to .fb_destroy of fb_ops, and in this function, there is kref_put() that finally calls ufx_free(). This fix prevents multiple UAFs. Signed-off-by: Hyunwoo Kim Link: https://lore.kernel.org/linux-fbdev/20221011153436.GA4446@ubuntu/ Cc: Signed-off-by: Helge Deller --- drivers/video/fbdev/smscufx.c | 55 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/drivers/video/fbdev/smscufx.c b/drivers/video/fbdev/smscufx.c index e65bdc499c236..9343b7a4ac899 100644 --- a/drivers/video/fbdev/smscufx.c +++ b/drivers/video/fbdev/smscufx.c @@ -97,7 +97,6 @@ struct ufx_data { struct kref kref; int fb_count; bool virtualized; /* true when physical usb device not present */ - struct delayed_work free_framebuffer_work; atomic_t usb_active; /* 0 = update virtual buffer, but no usb traffic */ atomic_t lost_pixels; /* 1 = a render op failed. Need screen refresh */ u8 *edid; /* null until we read edid from hw or get from sysfs */ @@ -1117,15 +1116,24 @@ static void ufx_free(struct kref *kref) { struct ufx_data *dev = container_of(kref, struct ufx_data, kref); - /* this function will wait for all in-flight urbs to complete */ - if (dev->urbs.count > 0) - ufx_free_urb_list(dev); + kfree(dev); +} - pr_debug("freeing ufx_data %p", dev); +static void ufx_ops_destory(struct fb_info *info) +{ + struct ufx_data *dev = info->par; + int node = info->node; - kfree(dev); + /* Assume info structure is freed after this point */ + framebuffer_release(info); + + pr_debug("fb_info for /dev/fb%d has been freed", node); + + /* release reference taken by kref_init in probe() */ + kref_put(&dev->kref, ufx_free); } + static void ufx_release_urb_work(struct work_struct *work) { struct urb_node *unode = container_of(work, struct urb_node, @@ -1134,14 +1142,9 @@ static void ufx_release_urb_work(struct work_struct *work) up(&unode->dev->urbs.limit_sem); } -static void ufx_free_framebuffer_work(struct work_struct *work) +static void ufx_free_framebuffer(struct ufx_data *dev) { - struct ufx_data *dev = container_of(work, struct ufx_data, - free_framebuffer_work.work); struct fb_info *info = dev->info; - int node = info->node; - - unregister_framebuffer(info); if (info->cmap.len != 0) fb_dealloc_cmap(&info->cmap); @@ -1153,11 +1156,6 @@ static void ufx_free_framebuffer_work(struct work_struct *work) dev->info = NULL; - /* Assume info structure is freed after this point */ - framebuffer_release(info); - - pr_debug("fb_info for /dev/fb%d has been freed", node); - /* ref taken in probe() as part of registering framebfufer */ kref_put(&dev->kref, ufx_free); } @@ -1169,11 +1167,13 @@ static int ufx_ops_release(struct fb_info *info, int user) { struct ufx_data *dev = info->par; + mutex_lock(&disconnect_mutex); + dev->fb_count--; /* We can't free fb_info here - fbmem will touch it when we return */ if (dev->virtualized && (dev->fb_count == 0)) - schedule_delayed_work(&dev->free_framebuffer_work, HZ); + ufx_free_framebuffer(dev); if ((dev->fb_count == 0) && (info->fbdefio)) { fb_deferred_io_cleanup(info); @@ -1186,6 +1186,8 @@ static int ufx_ops_release(struct fb_info *info, int user) kref_put(&dev->kref, ufx_free); + mutex_unlock(&disconnect_mutex); + return 0; } @@ -1292,6 +1294,7 @@ static const struct fb_ops ufx_ops = { .fb_blank = ufx_ops_blank, .fb_check_var = ufx_ops_check_var, .fb_set_par = ufx_ops_set_par, + .fb_destroy = ufx_ops_destory, }; /* Assumes &info->lock held by caller @@ -1673,9 +1676,6 @@ static int ufx_usb_probe(struct usb_interface *interface, goto destroy_modedb; } - INIT_DELAYED_WORK(&dev->free_framebuffer_work, - ufx_free_framebuffer_work); - retval = ufx_reg_read(dev, 0x3000, &id_rev); check_warn_goto_error(retval, "error %d reading 0x3000 register from device", retval); dev_dbg(dev->gdev, "ID_REV register value 0x%08x", id_rev); @@ -1748,10 +1748,12 @@ static int ufx_usb_probe(struct usb_interface *interface, static void ufx_usb_disconnect(struct usb_interface *interface) { struct ufx_data *dev; + struct fb_info *info; mutex_lock(&disconnect_mutex); dev = usb_get_intfdata(interface); + info = dev->info; pr_debug("USB disconnect starting\n"); @@ -1765,12 +1767,15 @@ static void ufx_usb_disconnect(struct usb_interface *interface) /* if clients still have us open, will be freed on last close */ if (dev->fb_count == 0) - schedule_delayed_work(&dev->free_framebuffer_work, 0); + ufx_free_framebuffer(dev); - /* release reference taken by kref_init in probe() */ - kref_put(&dev->kref, ufx_free); + /* this function will wait for all in-flight urbs to complete */ + if (dev->urbs.count > 0) + ufx_free_urb_list(dev); - /* consider ufx_data freed */ + pr_debug("freeing ufx_data %p", dev); + + unregister_framebuffer(info); mutex_unlock(&disconnect_mutex); } -- GitLab From e8a18e3f00f3ee8d07c17ab1ea3ad4df4a3b6fe0 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Fri, 21 Oct 2022 07:44:49 +0200 Subject: [PATCH 0259/1927] parisc: Make 8250_gsc driver dependend on CONFIG_PARISC Although the name of the driver 8250_gsc.c suggests that it handles only serial ports on the GSC bus, it does handle serial ports listed in the parisc machine inventory as well, e.g. the serial ports in a C8000 PCI-only workstation. Change the dependency to CONFIG_PARISC, so that the driver gets included in the kernel even if CONFIG_GSC isn't set. Reported-by: Mikulas Patocka Cc: Signed-off-by: Helge Deller --- drivers/tty/serial/8250/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig index d0b49e15fbf5e..7b3ffb7f50aba 100644 --- a/drivers/tty/serial/8250/Kconfig +++ b/drivers/tty/serial/8250/Kconfig @@ -118,7 +118,7 @@ config SERIAL_8250_CONSOLE config SERIAL_8250_GSC tristate - depends on SERIAL_8250 && GSC + depends on SERIAL_8250 && PARISC default SERIAL_8250 config SERIAL_8250_DMA -- GitLab From 9e4e2ce1a78ed92ed91135e90c85f27d75388129 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Fri, 21 Oct 2022 08:02:31 +0200 Subject: [PATCH 0260/1927] parisc/serial: Rename 8250_gsc.c to 8250_parisc.c The file name of this driver is misleading - it handles various serial ports on parisc machines, not just such on the GSC bus. Rename the file to make this clearer. Suggested-by: Mikulas Patocka Signed-off-by: Helge Deller --- drivers/tty/serial/8250/{8250_gsc.c => 8250_parisc.c} | 0 drivers/tty/serial/8250/Kconfig | 2 +- drivers/tty/serial/8250/Makefile | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename drivers/tty/serial/8250/{8250_gsc.c => 8250_parisc.c} (100%) diff --git a/drivers/tty/serial/8250/8250_gsc.c b/drivers/tty/serial/8250/8250_parisc.c similarity index 100% rename from drivers/tty/serial/8250/8250_gsc.c rename to drivers/tty/serial/8250/8250_parisc.c diff --git a/drivers/tty/serial/8250/Kconfig b/drivers/tty/serial/8250/Kconfig index 7b3ffb7f50aba..b0f62345bc846 100644 --- a/drivers/tty/serial/8250/Kconfig +++ b/drivers/tty/serial/8250/Kconfig @@ -116,7 +116,7 @@ config SERIAL_8250_CONSOLE If unsure, say N. -config SERIAL_8250_GSC +config SERIAL_8250_PARISC tristate depends on SERIAL_8250 && PARISC default SERIAL_8250 diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile index bee908f99ea0e..1615bfdde2a07 100644 --- a/drivers/tty/serial/8250/Makefile +++ b/drivers/tty/serial/8250/Makefile @@ -12,7 +12,7 @@ obj-$(CONFIG_SERIAL_8250) += 8250.o 8250_base.o 8250_base-$(CONFIG_SERIAL_8250_DMA) += 8250_dma.o 8250_base-$(CONFIG_SERIAL_8250_DWLIB) += 8250_dwlib.o 8250_base-$(CONFIG_SERIAL_8250_FINTEK) += 8250_fintek.o -obj-$(CONFIG_SERIAL_8250_GSC) += 8250_gsc.o +obj-$(CONFIG_SERIAL_8250_PARISC) += 8250_parisc.o obj-$(CONFIG_SERIAL_8250_PCI) += 8250_pci.o obj-$(CONFIG_SERIAL_8250_EXAR) += 8250_exar.o obj-$(CONFIG_SERIAL_8250_HP300) += 8250_hp300.o -- GitLab From 966f015fe4329199cc49084ee2886cfb626b34d3 Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Thu, 20 Oct 2022 22:46:21 +0200 Subject: [PATCH 0261/1927] ALSA: control: add snd_ctl_rename() Add a snd_ctl_rename() function that takes care of updating the control hash entries for callers that already have the relevant struct snd_kcontrol at hand and hold the control write lock (or simply haven't registered the card yet). Fixes: c27e1efb61c5 ("ALSA: control: Use xarray for faster lookups") Cc: stable@vger.kernel.org Signed-off-by: Maciej S. Szmigiero Link: https://lore.kernel.org/r/4170b71117ea81357a4f7eb8410f7cde20836c70.1666296963.git.maciej.szmigiero@oracle.com Signed-off-by: Takashi Iwai --- include/sound/control.h | 1 + sound/core/control.c | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/include/sound/control.h b/include/sound/control.h index eae443ba79ba5..cc3dcc6cfb0f2 100644 --- a/include/sound/control.h +++ b/include/sound/control.h @@ -138,6 +138,7 @@ int snd_ctl_remove(struct snd_card * card, struct snd_kcontrol * kcontrol); int snd_ctl_replace(struct snd_card *card, struct snd_kcontrol *kcontrol, bool add_on_replace); int snd_ctl_remove_id(struct snd_card * card, struct snd_ctl_elem_id *id); int snd_ctl_rename_id(struct snd_card * card, struct snd_ctl_elem_id *src_id, struct snd_ctl_elem_id *dst_id); +void snd_ctl_rename(struct snd_card *card, struct snd_kcontrol *kctl, const char *name); int snd_ctl_activate_id(struct snd_card *card, struct snd_ctl_elem_id *id, int active); struct snd_kcontrol *snd_ctl_find_numid(struct snd_card * card, unsigned int numid); struct snd_kcontrol *snd_ctl_find_id(struct snd_card * card, struct snd_ctl_elem_id *id); diff --git a/sound/core/control.c b/sound/core/control.c index a7271927d875f..50e7ba66f1876 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -753,6 +753,29 @@ int snd_ctl_rename_id(struct snd_card *card, struct snd_ctl_elem_id *src_id, } EXPORT_SYMBOL(snd_ctl_rename_id); +/** + * snd_ctl_rename - rename the control on the card + * @card: the card instance + * @kctl: the control to rename + * @name: the new name + * + * Renames the specified control on the card to the new name. + * + * Make sure to take the control write lock - down_write(&card->controls_rwsem). + */ +void snd_ctl_rename(struct snd_card *card, struct snd_kcontrol *kctl, + const char *name) +{ + remove_hash_entries(card, kctl); + + if (strscpy(kctl->id.name, name, sizeof(kctl->id.name)) < 0) + pr_warn("ALSA: Renamed control new name '%s' truncated to '%s'\n", + name, kctl->id.name); + + add_hash_entries(card, kctl); +} +EXPORT_SYMBOL(snd_ctl_rename); + #ifndef CONFIG_SND_CTL_FAST_LOOKUP static struct snd_kcontrol * snd_ctl_find_numid_slow(struct snd_card *card, unsigned int numid) -- GitLab From 0b4f0debb34754002cee295441c9ca89ba8cdfcc Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Thu, 20 Oct 2022 22:46:22 +0200 Subject: [PATCH 0262/1927] ALSA: usb-audio: Use snd_ctl_rename() to rename a control With the recent addition of hashed controls lookup it's not enough to just update the control name field, the hash entries for the modified control have to be updated too. snd_ctl_rename() takes care of that, so use it instead of directly modifying the control name. Fixes: c27e1efb61c5 ("ALSA: control: Use xarray for faster lookups") Cc: stable@vger.kernel.org Signed-off-by: Maciej S. Szmigiero Link: https://lore.kernel.org/r/723877882e3a56bb42a2a2214cfc85f347d36e19.1666296963.git.maciej.szmigiero@oracle.com Signed-off-by: Takashi Iwai --- sound/usb/mixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index a5641956ef102..9105ec623120a 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1631,7 +1631,7 @@ static void check_no_speaker_on_headset(struct snd_kcontrol *kctl, if (!found) return; - strscpy(kctl->id.name, "Headphone", sizeof(kctl->id.name)); + snd_ctl_rename(card, kctl, "Headphone"); } static const struct usb_feature_control_info *get_feature_control_info(int control) -- GitLab From b51c225376a684d02fb58b49cf0ce3d693b6f14b Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Thu, 20 Oct 2022 22:46:23 +0200 Subject: [PATCH 0263/1927] ALSA: hda/realtek: Use snd_ctl_rename() to rename a control With the recent addition of hashed controls lookup it's not enough to just update the control name field, the hash entries for the modified control have to be updated too. snd_ctl_rename() takes care of that, so use it instead of directly modifying the control name. Fixes: c27e1efb61c5 ("ALSA: control: Use xarray for faster lookups") Cc: stable@vger.kernel.org Signed-off-by: Maciej S. Szmigiero Link: https://lore.kernel.org/r/37496bd80f91f373268148f877fd735917d97287.1666296963.git.maciej.szmigiero@oracle.com Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_realtek.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 79acd2a2caf20..9945861f02efa 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -2142,7 +2142,7 @@ static void rename_ctl(struct hda_codec *codec, const char *oldname, kctl = snd_hda_find_mixer_ctl(codec, oldname); if (kctl) - strcpy(kctl->id.name, newname); + snd_ctl_rename(codec->card, kctl, newname); } static void alc1220_fixup_gb_dual_codecs(struct hda_codec *codec, -- GitLab From 36476b81b2b5db1de5adb8ced1f71b8972a9d4dd Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Thu, 20 Oct 2022 22:46:24 +0200 Subject: [PATCH 0264/1927] ALSA: emu10k1: Use snd_ctl_rename() to rename a control With the recent addition of hashed controls lookup it's not enough to just update the control name field, the hash entries for the modified control have to be updated too. snd_ctl_rename() takes care of that, so use it instead of directly modifying the control name. Fixes: c27e1efb61c5 ("ALSA: control: Use xarray for faster lookups") Cc: stable@vger.kernel.org Signed-off-by: Maciej S. Szmigiero Link: https://lore.kernel.org/r/38b19f019f95ee78a6e4e59d39afb9e2c3379413.1666296963.git.maciej.szmigiero@oracle.com Signed-off-by: Takashi Iwai --- sound/pci/emu10k1/emumixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/pci/emu10k1/emumixer.c b/sound/pci/emu10k1/emumixer.c index e9c0fe3b84461..3c115f8ab96c0 100644 --- a/sound/pci/emu10k1/emumixer.c +++ b/sound/pci/emu10k1/emumixer.c @@ -1767,7 +1767,7 @@ static int rename_ctl(struct snd_card *card, const char *src, const char *dst) { struct snd_kcontrol *kctl = ctl_find(card, src); if (kctl) { - strcpy(kctl->id.name, dst); + snd_ctl_rename(card, kctl, dst); return 0; } return -ENOENT; -- GitLab From 957ccc434c398a88a332ae92d70790c186a18a1c Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Thu, 20 Oct 2022 22:46:25 +0200 Subject: [PATCH 0265/1927] ALSA: ca0106: Use snd_ctl_rename() to rename a control With the recent addition of hashed controls lookup it's not enough to just update the control name field, the hash entries for the modified control have to be updated too. snd_ctl_rename() takes care of that, so use it instead of directly modifying the control name. Fixes: c27e1efb61c5 ("ALSA: control: Use xarray for faster lookups") Cc: stable@vger.kernel.org Signed-off-by: Maciej S. Szmigiero Link: https://lore.kernel.org/r/bffee980a420f9b0eee5681d2f48d34a70cec0ce.1666296963.git.maciej.szmigiero@oracle.com Signed-off-by: Takashi Iwai --- sound/pci/ca0106/ca0106_mixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/pci/ca0106/ca0106_mixer.c b/sound/pci/ca0106/ca0106_mixer.c index 05f56015ddd87..f6381c098d4f6 100644 --- a/sound/pci/ca0106/ca0106_mixer.c +++ b/sound/pci/ca0106/ca0106_mixer.c @@ -720,7 +720,7 @@ static int rename_ctl(struct snd_card *card, const char *src, const char *dst) { struct snd_kcontrol *kctl = ctl_find(card, src); if (kctl) { - strcpy(kctl->id.name, dst); + snd_ctl_rename(card, kctl, dst); return 0; } return -ENOENT; -- GitLab From 52d256cc71f546f67037100c64eb4fa3ae5e4704 Mon Sep 17 00:00:00 2001 From: "Maciej S. Szmigiero" Date: Thu, 20 Oct 2022 22:46:26 +0200 Subject: [PATCH 0266/1927] ALSA: ac97: Use snd_ctl_rename() to rename a control With the recent addition of hashed controls lookup it's not enough to just update the control name field, the hash entries for the modified control have to be updated too. snd_ctl_rename() takes care of that, so use it instead of directly modifying the control name. While we are at it, check also that the new control name doesn't accidentally overwrite the available buffer space. Fixes: c27e1efb61c5 ("ALSA: control: Use xarray for faster lookups") Cc: stable@vger.kernel.org Signed-off-by: Maciej S. Szmigiero Link: https://lore.kernel.org/r/adb68bfa0885ba4a2583794b828f8e20d23f67c7.1666296963.git.maciej.szmigiero@oracle.com Signed-off-by: Takashi Iwai --- sound/pci/ac97/ac97_codec.c | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/sound/pci/ac97/ac97_codec.c b/sound/pci/ac97/ac97_codec.c index ceead55f13ab1..ff685321f1a11 100644 --- a/sound/pci/ac97/ac97_codec.c +++ b/sound/pci/ac97/ac97_codec.c @@ -2656,11 +2656,18 @@ EXPORT_SYMBOL(snd_ac97_resume); */ static void set_ctl_name(char *dst, const char *src, const char *suffix) { - if (suffix) - sprintf(dst, "%s %s", src, suffix); - else - strcpy(dst, src); -} + const size_t msize = SNDRV_CTL_ELEM_ID_NAME_MAXLEN; + + if (suffix) { + if (snprintf(dst, msize, "%s %s", src, suffix) >= msize) + pr_warn("ALSA: AC97 control name '%s %s' truncated to '%s'\n", + src, suffix, dst); + } else { + if (strscpy(dst, src, msize) < 0) + pr_warn("ALSA: AC97 control name '%s' truncated to '%s'\n", + src, dst); + } +} /* remove the control with the given name and optional suffix */ static int snd_ac97_remove_ctl(struct snd_ac97 *ac97, const char *name, @@ -2687,8 +2694,11 @@ static int snd_ac97_rename_ctl(struct snd_ac97 *ac97, const char *src, const char *dst, const char *suffix) { struct snd_kcontrol *kctl = ctl_find(ac97, src, suffix); + char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; + if (kctl) { - set_ctl_name(kctl->id.name, dst, suffix); + set_ctl_name(name, dst, suffix); + snd_ctl_rename(ac97->bus->card, kctl, name); return 0; } return -ENOENT; @@ -2707,11 +2717,17 @@ static int snd_ac97_swap_ctl(struct snd_ac97 *ac97, const char *s1, const char *s2, const char *suffix) { struct snd_kcontrol *kctl1, *kctl2; + char name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN]; + kctl1 = ctl_find(ac97, s1, suffix); kctl2 = ctl_find(ac97, s2, suffix); if (kctl1 && kctl2) { - set_ctl_name(kctl1->id.name, s2, suffix); - set_ctl_name(kctl2->id.name, s1, suffix); + set_ctl_name(name, s2, suffix); + snd_ctl_rename(ac97->bus->card, kctl1, name); + + set_ctl_name(name, s1, suffix); + snd_ctl_rename(ac97->bus->card, kctl2, name); + return 0; } return -ENOENT; -- GitLab From 66063033f77e10b985258126a97573f84bb8d3b4 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 19 Oct 2022 09:55:41 -0600 Subject: [PATCH 0267/1927] wifi: rt2x00: use explicitly signed or unsigned types On some platforms, `char` is unsigned, but this driver, for the most part, assumed it was signed. In other places, it uses `char` to mean an unsigned number, but only in cases when the values are small. And in still other places, `char` is used as a boolean. Put an end to this confusion by declaring explicit types, depending on the context. Cc: Andrew Morton Cc: Andy Shevchenko Cc: Stanislaw Gruszka Cc: Helmut Schaa Cc: Kalle Valo Signed-off-by: Jason A. Donenfeld Acked-by: Stanislaw Gruszka Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20221019155541.3410813-1-Jason@zx2c4.com --- .../net/wireless/ralink/rt2x00/rt2400pci.c | 8 +-- .../net/wireless/ralink/rt2x00/rt2400pci.h | 2 +- .../net/wireless/ralink/rt2x00/rt2500pci.c | 8 +-- .../net/wireless/ralink/rt2x00/rt2500pci.h | 2 +- .../net/wireless/ralink/rt2x00/rt2500usb.c | 8 +-- .../net/wireless/ralink/rt2x00/rt2500usb.h | 2 +- .../net/wireless/ralink/rt2x00/rt2800lib.c | 60 +++++++++---------- .../net/wireless/ralink/rt2x00/rt2800lib.h | 8 +-- .../net/wireless/ralink/rt2x00/rt2x00usb.c | 6 +- drivers/net/wireless/ralink/rt2x00/rt61pci.c | 4 +- drivers/net/wireless/ralink/rt2x00/rt61pci.h | 2 +- drivers/net/wireless/ralink/rt2x00/rt73usb.c | 4 +- drivers/net/wireless/ralink/rt2x00/rt73usb.h | 2 +- 13 files changed, 58 insertions(+), 58 deletions(-) diff --git a/drivers/net/wireless/ralink/rt2x00/rt2400pci.c b/drivers/net/wireless/ralink/rt2x00/rt2400pci.c index 273c5eac33620..ddfc16de1b26f 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2400pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2400pci.c @@ -1023,9 +1023,9 @@ static int rt2400pci_set_state(struct rt2x00_dev *rt2x00dev, { u32 reg, reg2; unsigned int i; - char put_to_sleep; - char bbp_state; - char rf_state; + bool put_to_sleep; + u8 bbp_state; + u8 rf_state; put_to_sleep = (state != STATE_AWAKE); @@ -1561,7 +1561,7 @@ static int rt2400pci_probe_hw_mode(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; struct channel_info *info; - char *tx_power; + u8 *tx_power; unsigned int i; /* diff --git a/drivers/net/wireless/ralink/rt2x00/rt2400pci.h b/drivers/net/wireless/ralink/rt2x00/rt2400pci.h index b8187b6de1439..979d5fd8babf7 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2400pci.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2400pci.h @@ -939,7 +939,7 @@ #define DEFAULT_TXPOWER 39 #define __CLAMP_TX(__txpower) \ - clamp_t(char, (__txpower), MIN_TXPOWER, MAX_TXPOWER) + clamp_t(u8, (__txpower), MIN_TXPOWER, MAX_TXPOWER) #define TXPOWER_FROM_DEV(__txpower) \ ((__CLAMP_TX(__txpower) - MAX_TXPOWER) + MIN_TXPOWER) diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500pci.c b/drivers/net/wireless/ralink/rt2x00/rt2500pci.c index 8faa0a80e73a6..cd6371e25062b 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2500pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2500pci.c @@ -1176,9 +1176,9 @@ static int rt2500pci_set_state(struct rt2x00_dev *rt2x00dev, { u32 reg, reg2; unsigned int i; - char put_to_sleep; - char bbp_state; - char rf_state; + bool put_to_sleep; + u8 bbp_state; + u8 rf_state; put_to_sleep = (state != STATE_AWAKE); @@ -1856,7 +1856,7 @@ static int rt2500pci_probe_hw_mode(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; struct channel_info *info; - char *tx_power; + u8 *tx_power; unsigned int i; /* diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500pci.h b/drivers/net/wireless/ralink/rt2x00/rt2500pci.h index 7e64aee2a172e..ba362675c52c0 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2500pci.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2500pci.h @@ -1219,6 +1219,6 @@ (((u8)(__txpower)) > MAX_TXPOWER) ? DEFAULT_TXPOWER : (__txpower) #define TXPOWER_TO_DEV(__txpower) \ - clamp_t(char, __txpower, MIN_TXPOWER, MAX_TXPOWER) + clamp_t(u8, __txpower, MIN_TXPOWER, MAX_TXPOWER) #endif /* RT2500PCI_H */ diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500usb.c b/drivers/net/wireless/ralink/rt2x00/rt2500usb.c index bb5ed66306458..4f3b0e6c6256c 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2500usb.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2500usb.c @@ -984,9 +984,9 @@ static int rt2500usb_set_state(struct rt2x00_dev *rt2x00dev, u16 reg; u16 reg2; unsigned int i; - char put_to_sleep; - char bbp_state; - char rf_state; + bool put_to_sleep; + u8 bbp_state; + u8 rf_state; put_to_sleep = (state != STATE_AWAKE); @@ -1663,7 +1663,7 @@ static int rt2500usb_probe_hw_mode(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; struct channel_info *info; - char *tx_power; + u8 *tx_power; unsigned int i; /* diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500usb.h b/drivers/net/wireless/ralink/rt2x00/rt2500usb.h index 0c070288a140f..746f0e950b76e 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2500usb.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2500usb.h @@ -839,6 +839,6 @@ (((u8)(__txpower)) > MAX_TXPOWER) ? DEFAULT_TXPOWER : (__txpower) #define TXPOWER_TO_DEV(__txpower) \ - clamp_t(char, __txpower, MIN_TXPOWER, MAX_TXPOWER) + clamp_t(u8, __txpower, MIN_TXPOWER, MAX_TXPOWER) #endif /* RT2500USB_H */ diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c index cbbb1a4849cff..12b700c7b9c3b 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c @@ -3372,10 +3372,10 @@ static void rt2800_config_channel_rf53xx(struct rt2x00_dev *rt2x00dev, if (rt2x00_has_cap_bt_coexist(rt2x00dev)) { if (rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5390F)) { /* r55/r59 value array of channel 1~14 */ - static const char r55_bt_rev[] = {0x83, 0x83, + static const u8 r55_bt_rev[] = {0x83, 0x83, 0x83, 0x73, 0x73, 0x63, 0x53, 0x53, 0x53, 0x43, 0x43, 0x43, 0x43, 0x43}; - static const char r59_bt_rev[] = {0x0e, 0x0e, + static const u8 r59_bt_rev[] = {0x0e, 0x0e, 0x0e, 0x0e, 0x0e, 0x0b, 0x0a, 0x09, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07}; @@ -3384,7 +3384,7 @@ static void rt2800_config_channel_rf53xx(struct rt2x00_dev *rt2x00dev, rt2800_rfcsr_write(rt2x00dev, 59, r59_bt_rev[idx]); } else { - static const char r59_bt[] = {0x8b, 0x8b, 0x8b, + static const u8 r59_bt[] = {0x8b, 0x8b, 0x8b, 0x8b, 0x8b, 0x8b, 0x8b, 0x8a, 0x89, 0x88, 0x88, 0x86, 0x85, 0x84}; @@ -3392,10 +3392,10 @@ static void rt2800_config_channel_rf53xx(struct rt2x00_dev *rt2x00dev, } } else { if (rt2x00_rt_rev_gte(rt2x00dev, RT5390, REV_RT5390F)) { - static const char r55_nonbt_rev[] = {0x23, 0x23, + static const u8 r55_nonbt_rev[] = {0x23, 0x23, 0x23, 0x23, 0x13, 0x13, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03}; - static const char r59_nonbt_rev[] = {0x07, 0x07, + static const u8 r59_nonbt_rev[] = {0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x06, 0x05, 0x04, 0x04}; @@ -3406,14 +3406,14 @@ static void rt2800_config_channel_rf53xx(struct rt2x00_dev *rt2x00dev, } else if (rt2x00_rt(rt2x00dev, RT5390) || rt2x00_rt(rt2x00dev, RT5392) || rt2x00_rt(rt2x00dev, RT6352)) { - static const char r59_non_bt[] = {0x8f, 0x8f, + static const u8 r59_non_bt[] = {0x8f, 0x8f, 0x8f, 0x8f, 0x8f, 0x8f, 0x8f, 0x8d, 0x8a, 0x88, 0x88, 0x87, 0x87, 0x86}; rt2800_rfcsr_write(rt2x00dev, 59, r59_non_bt[idx]); } else if (rt2x00_rt(rt2x00dev, RT5350)) { - static const char r59_non_bt[] = {0x0b, 0x0b, + static const u8 r59_non_bt[] = {0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0a, 0x0a, 0x09, 0x08, 0x07, 0x07, 0x06}; @@ -4035,23 +4035,23 @@ static void rt2800_iq_calibrate(struct rt2x00_dev *rt2x00dev, int channel) rt2800_bbp_write(rt2x00dev, 159, cal != 0xff ? cal : 0); } -static char rt2800_txpower_to_dev(struct rt2x00_dev *rt2x00dev, +static s8 rt2800_txpower_to_dev(struct rt2x00_dev *rt2x00dev, unsigned int channel, - char txpower) + s8 txpower) { if (rt2x00_rt(rt2x00dev, RT3593) || rt2x00_rt(rt2x00dev, RT3883)) txpower = rt2x00_get_field8(txpower, EEPROM_TXPOWER_ALC); if (channel <= 14) - return clamp_t(char, txpower, MIN_G_TXPOWER, MAX_G_TXPOWER); + return clamp_t(s8, txpower, MIN_G_TXPOWER, MAX_G_TXPOWER); if (rt2x00_rt(rt2x00dev, RT3593) || rt2x00_rt(rt2x00dev, RT3883)) - return clamp_t(char, txpower, MIN_A_TXPOWER_3593, + return clamp_t(s8, txpower, MIN_A_TXPOWER_3593, MAX_A_TXPOWER_3593); else - return clamp_t(char, txpower, MIN_A_TXPOWER, MAX_A_TXPOWER); + return clamp_t(s8, txpower, MIN_A_TXPOWER, MAX_A_TXPOWER); } static void rt3883_bbp_adjust(struct rt2x00_dev *rt2x00dev, @@ -8530,7 +8530,7 @@ static void rt2800_r_calibration(struct rt2x00_dev *rt2x00dev) u8 bytevalue = 0; int rcalcode; u8 r_cal_code = 0; - char d1 = 0, d2 = 0; + s8 d1 = 0, d2 = 0; u8 rfvalue; u32 MAC_RF_BYPASS0, MAC_RF_CONTROL0, MAC_PWR_PIN_CFG; u32 maccfg; @@ -8591,7 +8591,7 @@ static void rt2800_r_calibration(struct rt2x00_dev *rt2x00dev) if (bytevalue > 128) d1 = bytevalue - 256; else - d1 = (char)bytevalue; + d1 = (s8)bytevalue; rt2800_bbp_write(rt2x00dev, 22, 0x0); rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x01); @@ -8601,7 +8601,7 @@ static void rt2800_r_calibration(struct rt2x00_dev *rt2x00dev) if (bytevalue > 128) d2 = bytevalue - 256; else - d2 = (char)bytevalue; + d2 = (s8)bytevalue; rt2800_bbp_write(rt2x00dev, 22, 0x0); rcalcode = rt2800_calcrcalibrationcode(rt2x00dev, d1, d2); @@ -8703,7 +8703,7 @@ static void rt2800_rxdcoc_calibration(struct rt2x00_dev *rt2x00dev) static u32 rt2800_do_sqrt_accumulation(u32 si) { u32 root, root_pre, bit; - char i; + s8 i; bit = 1 << 15; root = 0; @@ -9330,11 +9330,11 @@ static void rt2800_loft_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx, u8 alc_idx, u8 dc_result[][RF_ALC_NUM][2]) { u32 p0 = 0, p1 = 0, pf = 0; - char idx0 = 0, idx1 = 0; + s8 idx0 = 0, idx1 = 0; u8 idxf[] = {0x00, 0x00}; u8 ibit = 0x20; u8 iorq; - char bidx; + s8 bidx; rt2800_bbp_write(rt2x00dev, 158, 0xb0); rt2800_bbp_write(rt2x00dev, 159, 0x80); @@ -9384,17 +9384,17 @@ static void rt2800_loft_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx, static void rt2800_iq_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx, u8 *ges, u8 *pes) { u32 p0 = 0, p1 = 0, pf = 0; - char perr = 0, gerr = 0, iq_err = 0; - char pef = 0, gef = 0; - char psta, pend; - char gsta, gend; + s8 perr = 0, gerr = 0, iq_err = 0; + s8 pef = 0, gef = 0; + s8 psta, pend; + s8 gsta, gend; u8 ibit = 0x20; u8 first_search = 0x00, touch_neg_max = 0x00; - char idx0 = 0, idx1 = 0; + s8 idx0 = 0, idx1 = 0; u8 gop; u8 bbp = 0; - char bidx; + s8 bidx; for (bidx = 5; bidx >= 1; bidx--) { for (gop = 0; gop < 2; gop++) { @@ -10043,11 +10043,11 @@ static int rt2800_rf_lp_config(struct rt2x00_dev *rt2x00dev, bool btxcal) return 0; } -static char rt2800_lp_tx_filter_bw_cal(struct rt2x00_dev *rt2x00dev) +static s8 rt2800_lp_tx_filter_bw_cal(struct rt2x00_dev *rt2x00dev) { unsigned int cnt; u8 bbp_val; - char cal_val; + s8 cal_val; rt2800_bbp_dcoc_write(rt2x00dev, 0, 0x82); @@ -10079,7 +10079,7 @@ static void rt2800_bw_filter_calibration(struct rt2x00_dev *rt2x00dev, u8 rx_filter_target_20m = 0x27, rx_filter_target_40m = 0x31; int loop = 0, is_ht40, cnt; u8 bbp_val, rf_val; - char cal_r32_init, cal_r32_val, cal_diff; + s8 cal_r32_init, cal_r32_val, cal_diff; u8 saverfb5r00, saverfb5r01, saverfb5r03, saverfb5r04, saverfb5r05; u8 saverfb5r06, saverfb5r07; u8 saverfb5r08, saverfb5r17, saverfb5r18, saverfb5r19, saverfb5r20; @@ -11550,9 +11550,9 @@ static int rt2800_probe_hw_mode(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; struct channel_info *info; - char *default_power1; - char *default_power2; - char *default_power3; + s8 *default_power1; + s8 *default_power2; + s8 *default_power3; unsigned int i, tx_chains, rx_chains; u32 reg; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h index 3cbef77b4bd30..194de676df8fd 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h +++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h @@ -32,10 +32,10 @@ struct rf_reg_pair { struct rt2800_drv_data { u8 calibration_bw20; u8 calibration_bw40; - char rx_calibration_bw20; - char rx_calibration_bw40; - char tx_calibration_bw20; - char tx_calibration_bw40; + s8 rx_calibration_bw20; + s8 rx_calibration_bw40; + s8 tx_calibration_bw20; + s8 tx_calibration_bw40; u8 bbp25; u8 bbp26; u8 txmixer_gain_24g; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c index 0827bc860bf86..8fd22c69855fa 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c @@ -117,12 +117,12 @@ int rt2x00usb_vendor_request_buff(struct rt2x00_dev *rt2x00dev, const u16 buffer_length) { int status = 0; - unsigned char *tb; + u8 *tb; u16 off, len, bsize; mutex_lock(&rt2x00dev->csr_mutex); - tb = (char *)buffer; + tb = (u8 *)buffer; off = offset; len = buffer_length; while (len && !status) { @@ -215,7 +215,7 @@ void rt2x00usb_register_read_async(struct rt2x00_dev *rt2x00dev, rd->cr.wLength = cpu_to_le16(sizeof(u32)); usb_fill_control_urb(urb, usb_dev, usb_rcvctrlpipe(usb_dev, 0), - (unsigned char *)(&rd->cr), &rd->reg, sizeof(rd->reg), + (u8 *)(&rd->cr), &rd->reg, sizeof(rd->reg), rt2x00usb_register_read_async_cb, rd); usb_anchor_urb(urb, rt2x00dev->anchor); if (usb_submit_urb(urb, GFP_ATOMIC) < 0) { diff --git a/drivers/net/wireless/ralink/rt2x00/rt61pci.c b/drivers/net/wireless/ralink/rt2x00/rt61pci.c index d92f9eb07dc99..81db7f57c7e42 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt61pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt61pci.c @@ -1709,7 +1709,7 @@ static int rt61pci_set_state(struct rt2x00_dev *rt2x00dev, enum dev_state state) { u32 reg, reg2; unsigned int i; - char put_to_sleep; + bool put_to_sleep; put_to_sleep = (state != STATE_AWAKE); @@ -2656,7 +2656,7 @@ static int rt61pci_probe_hw_mode(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; struct channel_info *info; - char *tx_power; + u8 *tx_power; unsigned int i; /* diff --git a/drivers/net/wireless/ralink/rt2x00/rt61pci.h b/drivers/net/wireless/ralink/rt2x00/rt61pci.h index 5f208ad509bd4..d72d0ffd11275 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt61pci.h +++ b/drivers/net/wireless/ralink/rt2x00/rt61pci.h @@ -1484,6 +1484,6 @@ struct hw_pairwise_ta_entry { (((u8)(__txpower)) > MAX_TXPOWER) ? DEFAULT_TXPOWER : (__txpower) #define TXPOWER_TO_DEV(__txpower) \ - clamp_t(char, __txpower, MIN_TXPOWER, MAX_TXPOWER) + clamp_t(u8, __txpower, MIN_TXPOWER, MAX_TXPOWER) #endif /* RT61PCI_H */ diff --git a/drivers/net/wireless/ralink/rt2x00/rt73usb.c b/drivers/net/wireless/ralink/rt2x00/rt73usb.c index e3269fd7c59e3..8610354443740 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt73usb.c +++ b/drivers/net/wireless/ralink/rt2x00/rt73usb.c @@ -1378,7 +1378,7 @@ static int rt73usb_set_state(struct rt2x00_dev *rt2x00dev, enum dev_state state) { u32 reg, reg2; unsigned int i; - char put_to_sleep; + bool put_to_sleep; put_to_sleep = (state != STATE_AWAKE); @@ -2090,7 +2090,7 @@ static int rt73usb_probe_hw_mode(struct rt2x00_dev *rt2x00dev) { struct hw_mode_spec *spec = &rt2x00dev->spec; struct channel_info *info; - char *tx_power; + u8 *tx_power; unsigned int i; /* diff --git a/drivers/net/wireless/ralink/rt2x00/rt73usb.h b/drivers/net/wireless/ralink/rt2x00/rt73usb.h index 1b56d285c34b4..bb0a68516c08b 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt73usb.h +++ b/drivers/net/wireless/ralink/rt2x00/rt73usb.h @@ -1063,6 +1063,6 @@ struct hw_pairwise_ta_entry { (((u8)(__txpower)) > MAX_TXPOWER) ? DEFAULT_TXPOWER : (__txpower) #define TXPOWER_TO_DEV(__txpower) \ - clamp_t(char, __txpower, MIN_TXPOWER, MAX_TXPOWER) + clamp_t(u8, __txpower, MIN_TXPOWER, MAX_TXPOWER) #endif /* RT73USB_H */ -- GitLab From 50f19697dd768d8b072cf7f12c0c99c7d31b67d8 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Fri, 21 Oct 2022 09:15:20 +0200 Subject: [PATCH 0268/1927] parisc: Use signed char for hardware path in pdc.h Clean up the struct for hardware_path and drop the struct device_path with a proper assignment of bc[] and mod members as signed chars. This patch prepares for the kbuild change from Jason A. Donenfeld to treat char as always unsigned. Signed-off-by: Helge Deller Cc: Jason A. Donenfeld --- arch/parisc/include/uapi/asm/pdc.h | 36 +++++++++++------------------- drivers/parisc/pdc_stable.c | 34 ++++++++++++++-------------- 2 files changed, 30 insertions(+), 40 deletions(-) diff --git a/arch/parisc/include/uapi/asm/pdc.h b/arch/parisc/include/uapi/asm/pdc.h index e794e143ec5f8..7a90070136e82 100644 --- a/arch/parisc/include/uapi/asm/pdc.h +++ b/arch/parisc/include/uapi/asm/pdc.h @@ -363,20 +363,25 @@ #if !defined(__ASSEMBLY__) -/* flags of the device_path */ +/* flags for hardware_path */ #define PF_AUTOBOOT 0x80 #define PF_AUTOSEARCH 0x40 #define PF_TIMER 0x0F -struct device_path { /* page 1-69 */ - unsigned char flags; /* flags see above! */ - unsigned char bc[6]; /* bus converter routing info */ - unsigned char mod; - unsigned int layers[6];/* device-specific layer-info */ -} __attribute__((aligned(8))) ; +struct hardware_path { + unsigned char flags; /* see bit definitions below */ + signed char bc[6]; /* Bus Converter routing info to a specific */ + /* I/O adaptor (< 0 means none, > 63 resvd) */ + signed char mod; /* fixed field of specified module */ +}; + +struct pdc_module_path { /* page 1-69 */ + struct hardware_path path; + unsigned int layers[6]; /* device-specific info (ctlr #, unit # ...) */ +} __attribute__((aligned(8))); struct pz_device { - struct device_path dp; /* see above */ + struct pdc_module_path dp; /* see above */ /* struct iomod *hpa; */ unsigned int hpa; /* HPA base address */ /* char *spa; */ @@ -611,21 +616,6 @@ struct pdc_initiator { /* PDC_INITIATOR */ int mode; }; -struct hardware_path { - char flags; /* see bit definitions below */ - char bc[6]; /* Bus Converter routing info to a specific */ - /* I/O adaptor (< 0 means none, > 63 resvd) */ - char mod; /* fixed field of specified module */ -}; - -/* - * Device path specifications used by PDC. - */ -struct pdc_module_path { - struct hardware_path path; - unsigned int layers[6]; /* device-specific info (ctlr #, unit # ...) */ -}; - /* Only used on some pre-PA2.0 boxes */ struct pdc_memory_map { /* PDC_MEMORY_MAP */ unsigned long hpa; /* mod's register set address */ diff --git a/drivers/parisc/pdc_stable.c b/drivers/parisc/pdc_stable.c index d9e51036a4fac..d6af5726ddf35 100644 --- a/drivers/parisc/pdc_stable.c +++ b/drivers/parisc/pdc_stable.c @@ -14,7 +14,7 @@ * all) PA-RISC machines should have them. Anyway, for safety reasons, the * following code can deal with just 96 bytes of Stable Storage, and all * sizes between 96 and 192 bytes (provided they are multiple of struct - * device_path size, eg: 128, 160 and 192) to provide full information. + * pdc_module_path size, eg: 128, 160 and 192) to provide full information. * One last word: there's one path we can always count on: the primary path. * Anything above 224 bytes is used for 'osdep2' OS-dependent storage area. * @@ -88,7 +88,7 @@ struct pdcspath_entry { short ready; /* entry record is valid if != 0 */ unsigned long addr; /* entry address in stable storage */ char *name; /* entry name */ - struct device_path devpath; /* device path in parisc representation */ + struct pdc_module_path devpath; /* device path in parisc representation */ struct device *dev; /* corresponding device */ struct kobject kobj; }; @@ -138,7 +138,7 @@ struct pdcspath_attribute paths_attr_##_name = { \ static int pdcspath_fetch(struct pdcspath_entry *entry) { - struct device_path *devpath; + struct pdc_module_path *devpath; if (!entry) return -EINVAL; @@ -153,7 +153,7 @@ pdcspath_fetch(struct pdcspath_entry *entry) return -EIO; /* Find the matching device. - NOTE: hardware_path overlays with device_path, so the nice cast can + NOTE: hardware_path overlays with pdc_module_path, so the nice cast can be used */ entry->dev = hwpath_to_device((struct hardware_path *)devpath); @@ -179,7 +179,7 @@ pdcspath_fetch(struct pdcspath_entry *entry) static void pdcspath_store(struct pdcspath_entry *entry) { - struct device_path *devpath; + struct pdc_module_path *devpath; BUG_ON(!entry); @@ -221,7 +221,7 @@ static ssize_t pdcspath_hwpath_read(struct pdcspath_entry *entry, char *buf) { char *out = buf; - struct device_path *devpath; + struct pdc_module_path *devpath; short i; if (!entry || !buf) @@ -236,11 +236,11 @@ pdcspath_hwpath_read(struct pdcspath_entry *entry, char *buf) return -ENODATA; for (i = 0; i < 6; i++) { - if (devpath->bc[i] >= 128) + if (devpath->path.bc[i] < 0) continue; - out += sprintf(out, "%u/", (unsigned char)devpath->bc[i]); + out += sprintf(out, "%d/", devpath->path.bc[i]); } - out += sprintf(out, "%u\n", (unsigned char)devpath->mod); + out += sprintf(out, "%u\n", (unsigned char)devpath->path.mod); return out - buf; } @@ -296,12 +296,12 @@ pdcspath_hwpath_write(struct pdcspath_entry *entry, const char *buf, size_t coun for (i=5; ((temp = strrchr(in, '/'))) && (temp-in > 0) && (likely(i)); i--) { hwpath.bc[i] = simple_strtoul(temp+1, NULL, 10); in[temp-in] = '\0'; - DPRINTK("%s: bc[%d]: %d\n", __func__, i, hwpath.bc[i]); + DPRINTK("%s: bc[%d]: %d\n", __func__, i, hwpath.path.bc[i]); } /* Store the final field */ hwpath.bc[i] = simple_strtoul(in, NULL, 10); - DPRINTK("%s: bc[%d]: %d\n", __func__, i, hwpath.bc[i]); + DPRINTK("%s: bc[%d]: %d\n", __func__, i, hwpath.path.bc[i]); /* Now we check that the user isn't trying to lure us */ if (!(dev = hwpath_to_device((struct hardware_path *)&hwpath))) { @@ -342,7 +342,7 @@ static ssize_t pdcspath_layer_read(struct pdcspath_entry *entry, char *buf) { char *out = buf; - struct device_path *devpath; + struct pdc_module_path *devpath; short i; if (!entry || !buf) @@ -547,7 +547,7 @@ static ssize_t pdcs_auto_read(struct kobject *kobj, pathentry = &pdcspath_entry_primary; read_lock(&pathentry->rw_lock); - out += sprintf(out, "%s\n", (pathentry->devpath.flags & knob) ? + out += sprintf(out, "%s\n", (pathentry->devpath.path.flags & knob) ? "On" : "Off"); read_unlock(&pathentry->rw_lock); @@ -594,8 +594,8 @@ static ssize_t pdcs_timer_read(struct kobject *kobj, /* print the timer value in seconds */ read_lock(&pathentry->rw_lock); - out += sprintf(out, "%u\n", (pathentry->devpath.flags & PF_TIMER) ? - (1 << (pathentry->devpath.flags & PF_TIMER)) : 0); + out += sprintf(out, "%u\n", (pathentry->devpath.path.flags & PF_TIMER) ? + (1 << (pathentry->devpath.path.flags & PF_TIMER)) : 0); read_unlock(&pathentry->rw_lock); return out - buf; @@ -764,7 +764,7 @@ static ssize_t pdcs_auto_write(struct kobject *kobj, /* Be nice to the existing flag record */ read_lock(&pathentry->rw_lock); - flags = pathentry->devpath.flags; + flags = pathentry->devpath.path.flags; read_unlock(&pathentry->rw_lock); DPRINTK("%s: flags before: 0x%X\n", __func__, flags); @@ -785,7 +785,7 @@ static ssize_t pdcs_auto_write(struct kobject *kobj, write_lock(&pathentry->rw_lock); /* Change the path entry flags first */ - pathentry->devpath.flags = flags; + pathentry->devpath.path.flags = flags; /* Now, dive in. Write back to the hardware */ pdcspath_store(pathentry); -- GitLab From 03c0ad4b06c3566de624b4f4b78ac1a5d1e4c8e7 Mon Sep 17 00:00:00 2001 From: Johannes Berg Date: Thu, 13 Oct 2022 19:41:51 +0200 Subject: [PATCH 0269/1927] wifi: cfg80211: silence a sparse RCU warning All we're going to do with this pointer is assign it to another __rcu pointer, but sparse can't see that, so use rcu_access_pointer() to silence the warning here. Fixes: c90b93b5b782 ("wifi: cfg80211: update hidden BSSes to avoid WARN_ON") Signed-off-by: Johannes Berg --- net/wireless/scan.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 806a5f1330ff5..da752b0cc7520 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -1674,7 +1674,9 @@ cfg80211_update_known_bss(struct cfg80211_registered_device *rdev, if (old == rcu_access_pointer(known->pub.ies)) rcu_assign_pointer(known->pub.ies, new->pub.beacon_ies); - cfg80211_update_hidden_bsses(known, new->pub.beacon_ies, old); + cfg80211_update_hidden_bsses(known, + rcu_access_pointer(new->pub.beacon_ies), + old); if (old) kfree_rcu((struct cfg80211_bss_ies *)old, rcu_head); -- GitLab From 50b2e8711462409cd368c41067405aa446dfa2af Mon Sep 17 00:00:00 2001 From: taozhang Date: Sat, 15 Oct 2022 17:38:31 +0800 Subject: [PATCH 0270/1927] wifi: mac80211: fix memory free error when registering wiphy fail ieee80211_register_hw free the allocated cipher suites when registering wiphy fail, and ieee80211_free_hw will re-free it. set wiphy_ciphers_allocated to false after freeing allocated cipher suites. Signed-off-by: taozhang Signed-off-by: Johannes Berg --- net/mac80211/main.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 46f3eddc23887..02b5abc7326bc 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -1439,8 +1439,10 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) ieee80211_led_exit(local); destroy_workqueue(local->workqueue); fail_workqueue: - if (local->wiphy_ciphers_allocated) + if (local->wiphy_ciphers_allocated) { kfree(local->hw.wiphy->cipher_suites); + local->wiphy_ciphers_allocated = false; + } kfree(local->int_scan_req); return result; } @@ -1508,8 +1510,10 @@ void ieee80211_free_hw(struct ieee80211_hw *hw) mutex_destroy(&local->iflist_mtx); mutex_destroy(&local->mtx); - if (local->wiphy_ciphers_allocated) + if (local->wiphy_ciphers_allocated) { kfree(local->hw.wiphy->cipher_suites); + local->wiphy_ciphers_allocated = false; + } idr_for_each(&local->ack_status_frames, ieee80211_free_ack_frame, NULL); -- GitLab From 57b962e627ec0ae53d4d16d7bd1033e27e67677a Mon Sep 17 00:00:00 2001 From: Arend van Spriel Date: Thu, 20 Oct 2022 13:40:40 +0200 Subject: [PATCH 0271/1927] wifi: cfg80211: fix memory leak in query_regdb_file() In the function query_regdb_file() the alpha2 parameter is duplicated using kmemdup() and subsequently freed in regdb_fw_cb(). However, request_firmware_nowait() can fail without calling regdb_fw_cb() and thus leak memory. Fixes: 007f6c5e6eb4 ("cfg80211: support loading regulatory database as firmware file") Signed-off-by: Arend van Spriel Signed-off-by: Johannes Berg --- net/wireless/reg.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/net/wireless/reg.c b/net/wireless/reg.c index d5c7a5aa68532..c3d950d294329 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -1084,6 +1084,8 @@ MODULE_FIRMWARE("regulatory.db"); static int query_regdb_file(const char *alpha2) { + int err; + ASSERT_RTNL(); if (regdb) @@ -1093,9 +1095,13 @@ static int query_regdb_file(const char *alpha2) if (!alpha2) return -ENOMEM; - return request_firmware_nowait(THIS_MODULE, true, "regulatory.db", - ®_pdev->dev, GFP_KERNEL, - (void *)alpha2, regdb_fw_cb); + err = request_firmware_nowait(THIS_MODULE, true, "regulatory.db", + ®_pdev->dev, GFP_KERNEL, + (void *)alpha2, regdb_fw_cb); + if (err) + kfree(alpha2); + + return err; } int reg_reload_regdb(void) -- GitLab From 18429c51c7ff6e6bfd627316c54670230967a7e5 Mon Sep 17 00:00:00 2001 From: Paul Zhang Date: Tue, 11 Oct 2022 21:04:28 +0800 Subject: [PATCH 0272/1927] wifi: cfg80211: Fix bitrates overflow issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When invoking function cfg80211_calculate_bitrate_eht about (320 MHz, EHT-MCS 13, EHT-NSS 2, EHT-GI 0), which means the parameters as flags: 0x80, bw: 7, mcs: 13, eht_gi: 0, nss: 2, this formula (result * rate->nss) will overflow and causes the returned bitrate to be 3959 when it should be 57646. Here is the explanation: u64 tmp; u32 result; … /* tmp = result = 4 * rates_996[0] * = 4 * 480388888 = 0x72889c60 */ tmp = result; /* tmp = 0x72889c60 * 6144 = 0xabccea90000 */ tmp *= SCALE; /* tmp = 0xabccea90000 / mcs_divisors[13] * = 0xabccea90000 / 5120 = 0x8970bba6 */ do_div(tmp, mcs_divisors[rate->mcs]); /* result = 0x8970bba6 */ result = tmp; /* normally (result * rate->nss) = 0x8970bba6 * 2 = 0x112e1774c, * but since result is u32, (result * rate->nss) = 0x12e1774c, * overflow happens and it loses the highest bit. * Then result = 0x12e1774c / 8 = 39595753, */ result = (result * rate->nss) / 8; Signed-off-by: Paul Zhang Signed-off-by: Johannes Berg --- net/wireless/util.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/net/wireless/util.c b/net/wireless/util.c index 1f285b5150286..39680e7bad45a 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -1557,10 +1557,12 @@ static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) tmp = result; tmp *= SCALE; do_div(tmp, mcs_divisors[rate->mcs]); - result = tmp; /* and take NSS */ - result = (result * rate->nss) / 8; + tmp *= rate->nss; + do_div(tmp, 8); + + result = tmp; return result / 10000; } -- GitLab From 69188df5f6e4cecc6b76b958979ba363cd5240e8 Mon Sep 17 00:00:00 2001 From: Jonas Jelonek Date: Fri, 14 Oct 2022 16:54:39 +0200 Subject: [PATCH 0273/1927] wifi: mac80211_hwsim: fix debugfs attribute ps with rc table support Fixes a warning that occurs when rc table support is enabled (IEEE80211_HW_SUPPORTS_RC_TABLE) in mac80211_hwsim and the PS mode is changed via the exported debugfs attribute. When the PS mode is changed, a packet is broadcasted via hwsim_send_nullfunc by creating and transmitting a plain skb with only header initialized. The ieee80211 rate array in the control buffer is zero-initialized. When ratetbl support is enabled, ieee80211_get_tx_rates is called for the skb with sta parameter set to NULL and thus no ratetbl can be used. The final rate array then looks like [-1,0; 0,0; 0,0; 0,0] which causes the warning in ieee80211_get_tx_rate. The issue is fixed by setting the count of the first rate with idx '0' to 1 and hence ieee80211_get_tx_rates won't overwrite it with idx '-1'. Signed-off-by: Jonas Jelonek Signed-off-by: Johannes Berg --- drivers/net/wireless/mac80211_hwsim.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c index a40636c90ec36..0d81098c7b45c 100644 --- a/drivers/net/wireless/mac80211_hwsim.c +++ b/drivers/net/wireless/mac80211_hwsim.c @@ -910,6 +910,7 @@ static void hwsim_send_nullfunc(struct mac80211_hwsim_data *data, u8 *mac, struct hwsim_vif_priv *vp = (void *)vif->drv_priv; struct sk_buff *skb; struct ieee80211_hdr *hdr; + struct ieee80211_tx_info *cb; if (!vp->assoc) return; @@ -931,6 +932,10 @@ static void hwsim_send_nullfunc(struct mac80211_hwsim_data *data, u8 *mac, memcpy(hdr->addr2, mac, ETH_ALEN); memcpy(hdr->addr3, vp->bssid, ETH_ALEN); + cb = IEEE80211_SKB_CB(skb); + cb->control.rates[0].count = 1; + cb->control.rates[1].idx = -1; + rcu_read_lock(); mac80211_hwsim_tx_frame(data->hw, skb, rcu_dereference(vif->bss_conf.chanctx_conf)->def.chan); -- GitLab From 9f6035af06b526e678808d492fc0830aef6cfbd8 Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Tue, 18 Oct 2022 16:04:12 -0700 Subject: [PATCH 0274/1927] crypto: x86/polyval - Fix crashes when keys are not 16-byte aligned crypto_tfm::__crt_ctx is not guaranteed to be 16-byte aligned on x86-64. This causes crashes due to movaps instructions in clmul_polyval_update. Add logic to align polyval_tfm_ctx to 16 bytes. Cc: Fixes: 34f7f6c30112 ("crypto: x86/polyval - Add PCLMULQDQ accelerated implementation of POLYVAL") Reported-by: Bruno Goncalves Signed-off-by: Nathan Huckleberry Reviewed-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/polyval-clmulni_glue.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/arch/x86/crypto/polyval-clmulni_glue.c b/arch/x86/crypto/polyval-clmulni_glue.c index b7664d0188510..8fa58b0f3cb3d 100644 --- a/arch/x86/crypto/polyval-clmulni_glue.c +++ b/arch/x86/crypto/polyval-clmulni_glue.c @@ -27,13 +27,17 @@ #include #include +#define POLYVAL_ALIGN 16 +#define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN) +#define POLYVAL_ALIGN_EXTRA ((POLYVAL_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) +#define POLYVAL_CTX_SIZE (sizeof(struct polyval_tfm_ctx) + POLYVAL_ALIGN_EXTRA) #define NUM_KEY_POWERS 8 struct polyval_tfm_ctx { /* * These powers must be in the order h^8, ..., h^1. */ - u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE]; + u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE] POLYVAL_ALIGN_ATTR; }; struct polyval_desc_ctx { @@ -45,6 +49,11 @@ asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys, const u8 *in, size_t nblocks, u8 *accumulator); asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2); +static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm) +{ + return PTR_ALIGN(crypto_shash_ctx(tfm), POLYVAL_ALIGN); +} + static void internal_polyval_update(const struct polyval_tfm_ctx *keys, const u8 *in, size_t nblocks, u8 *accumulator) { @@ -72,7 +81,7 @@ static void internal_polyval_mul(u8 *op1, const u8 *op2) static int polyval_x86_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { - struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm); + struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm); int i; if (keylen != POLYVAL_BLOCK_SIZE) @@ -102,7 +111,7 @@ static int polyval_x86_update(struct shash_desc *desc, const u8 *src, unsigned int srclen) { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); - const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm); u8 *pos; unsigned int nblocks; unsigned int n; @@ -143,7 +152,7 @@ static int polyval_x86_update(struct shash_desc *desc, static int polyval_x86_final(struct shash_desc *desc, u8 *dst) { struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); - const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(desc->tfm); if (dctx->bytes) { internal_polyval_mul(dctx->buffer, @@ -167,7 +176,7 @@ static struct shash_alg polyval_alg = { .cra_driver_name = "polyval-clmulni", .cra_priority = 200, .cra_blocksize = POLYVAL_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct polyval_tfm_ctx), + .cra_ctxsize = POLYVAL_CTX_SIZE, .cra_module = THIS_MODULE, }, }; -- GitLab From 0e213813df02da048ffd22a2c4fac041768ca327 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 20 Oct 2022 18:59:37 +0800 Subject: [PATCH 0275/1927] ASoC: Intel: Skylake: fix possible memory leak in skl_codec_device_init() If snd_hdac_device_register() fails, 'codec' and name allocated in dev_set_name() called in snd_hdac_device_init() are leaked. Fix this by calling put_device(), so they can be freed in snd_hda_codec_dev_release() and kobject_cleanup(). Fixes: e4746d94d00c ("ASoC: Intel: Skylake: Introduce HDA codec init and exit routines") Signed-off-by: Yang Yingliang Suggested-by: Cezary Rojewski Link: https://lore.kernel.org/r/20221020105937.1448951-1-yangyingliang@huawei.com Signed-off-by: Mark Brown --- sound/soc/intel/skylake/skl.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/sound/soc/intel/skylake/skl.c b/sound/soc/intel/skylake/skl.c index bbba2df33aaf0..3312b57e3c0cb 100644 --- a/sound/soc/intel/skylake/skl.c +++ b/sound/soc/intel/skylake/skl.c @@ -689,11 +689,6 @@ static void load_codec_module(struct hda_codec *codec) #endif /* CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC */ -static void skl_codec_device_exit(struct device *dev) -{ - snd_hdac_device_exit(dev_to_hdac_dev(dev)); -} - static struct hda_codec *skl_codec_device_init(struct hdac_bus *bus, int addr) { struct hda_codec *codec; @@ -706,12 +701,11 @@ static struct hda_codec *skl_codec_device_init(struct hdac_bus *bus, int addr) } codec->core.type = HDA_DEV_ASOC; - codec->core.dev.release = skl_codec_device_exit; ret = snd_hdac_device_register(&codec->core); if (ret) { dev_err(bus->dev, "failed to register hdac device\n"); - snd_hdac_device_exit(&codec->core); + put_device(&codec->core.dev); return ERR_PTR(ret); } -- GitLab From a75481fa00cc06a8763e1795b93140407948c03a Mon Sep 17 00:00:00 2001 From: Leohearts Date: Fri, 21 Oct 2022 14:34:32 +0800 Subject: [PATCH 0276/1927] ASoC: amd: yc: Add Lenovo Thinkbook 14+ 2022 21D0 to quirks table Lenovo Thinkbook 14+ 2022 (ThinkBook 14 G4+ ARA) uses Ryzen 6000 processor, and has the same microphone problem as other ThinkPads with AMD Ryzen 6000 series CPUs, which has been listed in https://bugzilla.kernel.org/show_bug.cgi?id=216267. Adding 21D0 to quirks table solves this microphone problem for ThinkBook 14 G4+ ARA. Signed-off-by: Taroe Leohearts Link: https://lore.kernel.org/r/26B141B486BEF706+313d1732-e00c-ea41-3123-0d048d40ebb6@leohearts.com Signed-off-by: Mark Brown --- sound/soc/amd/yc/acp6x-mach.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 09a8aceff22f1..6c0f1de10429a 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -52,6 +52,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "21D0"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "21D0"), + } + }, { .driver_data = &acp6x_card, .matches = { -- GitLab From e9441675edc1bb8dbfadacf68aafacca60d65a25 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 20 Oct 2022 19:01:57 +0800 Subject: [PATCH 0277/1927] ASoC: SOF: Intel: hda-codec: fix possible memory leak in hda_codec_device_init() If snd_hdac_device_register() fails, 'codec' and name allocated in dev_set_name() called in snd_hdac_device_init() are leaked. Fix this by calling put_device(), so they can be freed in snd_hda_codec_dev_release() and kobject_cleanup(). Fixes: 829c67319806 ("ASoC: SOF: Intel: Introduce HDA codec init and exit routines") Fixes: dfe66a18780d ("ALSA: hdac_ext: add extended HDA bus") Signed-off-by: Yang Yingliang Reviewed-by: Kai Vehmanen Link: https://lore.kernel.org/r/20221020110157.1450191-1-yangyingliang@huawei.com Signed-off-by: Mark Brown --- sound/soc/sof/intel/hda-codec.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/sound/soc/sof/intel/hda-codec.c b/sound/soc/sof/intel/hda-codec.c index 1e9afc48394c7..f2ec2a6c2e0f3 100644 --- a/sound/soc/sof/intel/hda-codec.c +++ b/sound/soc/sof/intel/hda-codec.c @@ -109,11 +109,6 @@ EXPORT_SYMBOL_NS(hda_codec_jack_check, SND_SOC_SOF_HDA_AUDIO_CODEC); #define is_generic_config(x) 0 #endif -static void hda_codec_device_exit(struct device *dev) -{ - snd_hdac_device_exit(dev_to_hdac_dev(dev)); -} - static struct hda_codec *hda_codec_device_init(struct hdac_bus *bus, int addr, int type) { struct hda_codec *codec; @@ -126,12 +121,11 @@ static struct hda_codec *hda_codec_device_init(struct hdac_bus *bus, int addr, i } codec->core.type = type; - codec->core.dev.release = hda_codec_device_exit; ret = snd_hdac_device_register(&codec->core); if (ret) { dev_err(bus->dev, "failed to register hdac device\n"); - snd_hdac_device_exit(&codec->core); + put_device(&codec->core.dev); return ERR_PTR(ret); } -- GitLab From 46cdedf2a0fa20a99ca8be40bccde7487e13b77a Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 19 Oct 2022 15:35:51 -0700 Subject: [PATCH 0278/1927] ethtool: pse-pd: fix null-deref on genl_info in dump ethnl_default_dump_one() passes NULL as info. It's correct not to set extack during dump, as we should just silently skip interfaces which can't provide the information. Reported-by: syzbot+81c4b4bbba6eea2cfcae@syzkaller.appspotmail.com Fixes: 18ff0bcda6d1 ("ethtool: add interface to interact with Ethernet Power Equipment") Signed-off-by: Jakub Kicinski Reviewed-by: Oleksij Rempel Signed-off-by: David S. Miller --- net/ethtool/pse-pd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/ethtool/pse-pd.c b/net/ethtool/pse-pd.c index 5a471e115b66a..e8683e485dc91 100644 --- a/net/ethtool/pse-pd.c +++ b/net/ethtool/pse-pd.c @@ -64,7 +64,7 @@ static int pse_prepare_data(const struct ethnl_req_info *req_base, if (ret < 0) return ret; - ret = pse_get_pse_attributes(dev, info->extack, data); + ret = pse_get_pse_attributes(dev, info ? info->extack : NULL, data); ethnl_ops_complete(dev); -- GitLab From 4d814b329a4d54cd10eee4bd2ce5a8175646cc16 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 19 Oct 2022 19:19:13 -0700 Subject: [PATCH 0279/1927] MAINTAINERS: add keyword match on PTP Most of PTP drivers live under ethernet and we have to keep telling people to CC the PTP maintainers. Let's try a keyword match, we can refine as we go if it causes false positives. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 8b73135ddb213..57ad33aa0eb84 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16669,6 +16669,7 @@ F: Documentation/driver-api/ptp.rst F: drivers/net/phy/dp83640* F: drivers/ptp/* F: include/linux/ptp_cl* +K: (?:\b|_)ptp(?:\b|_) PTP VIRTUAL CLOCK SUPPORT M: Yangbo Lu -- GitLab From 794814529384721ce8f4d34228dc599cc010353d Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Fri, 21 Oct 2022 14:27:22 +0200 Subject: [PATCH 0280/1927] ALSA: usb-audio: Add quirks for M-Audio Fast Track C400/600 M-Audio Fast Track C400 and C600 devices (0763:2030 and 0763:2031, respectively) seem requiring the explicit setup for the implicit feedback mode. This patch adds the quirk entries for those. BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=214817 Cc: Link: https://lore.kernel.org/r/20221021122722.24784-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/usb/implicit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/usb/implicit.c b/sound/usb/implicit.c index e1bf1b5da423c..f3e8484b3d9cb 100644 --- a/sound/usb/implicit.c +++ b/sound/usb/implicit.c @@ -47,6 +47,8 @@ struct snd_usb_implicit_fb_match { static const struct snd_usb_implicit_fb_match playback_implicit_fb_quirks[] = { /* Fixed EP */ /* FIXME: check the availability of generic matching */ + IMPLICIT_FB_FIXED_DEV(0x0763, 0x2030, 0x81, 3), /* M-Audio Fast Track C400 */ + IMPLICIT_FB_FIXED_DEV(0x0763, 0x2031, 0x81, 3), /* M-Audio Fast Track C600 */ IMPLICIT_FB_FIXED_DEV(0x0763, 0x2080, 0x81, 2), /* M-Audio FastTrack Ultra */ IMPLICIT_FB_FIXED_DEV(0x0763, 0x2081, 0x81, 2), /* M-Audio FastTrack Ultra */ IMPLICIT_FB_FIXED_DEV(0x2466, 0x8010, 0x81, 2), /* Fractal Audio Axe-Fx III */ -- GitLab From cb8e30ddb7e345867f6f2da8a08291d7d9e037db Mon Sep 17 00:00:00 2001 From: Douglas Anderson Date: Mon, 17 Oct 2022 12:18:51 -0700 Subject: [PATCH 0281/1927] drm/bridge: ps8640: Add back the 50 ms mystery delay after HPD Back in commit 826cff3f7ebb ("drm/bridge: parade-ps8640: Enable runtime power management") we removed a mysterious 50 ms delay because "Parade's support [couldn't] explain what the delay [was] for". While I'm always a fan of removing mysterious delays, I suspect that we need this mysterious delay to avoid some problems. Specifically, what I found recently is that on sc7180-trogdor-homestar sometimes the AUX backlight wasn't initializing properly. Some debugging showed that the drm_dp_dpcd_read() function that the AUX backlight driver was calling was returning bogus data about 1% of the time when I booted up. This confused drm_panel_dp_aux_backlight(). From continued debugging: - If I retried the read then the read worked just fine. - If I added a loop to perform the same read that drm_panel_dp_aux_backlight() was doing 30 times at bootup I could see that some percentage of the time the first read would give bogus data but all 29 additional reads would always be fine. - If I added a large delay _after_ powering on the panel but before powering on PS8640 I could still reproduce the problem. - If I added a delay after PS8640 powered on then I couldn't reproduce the problem. - I couldn't reproduce the problem on a board with the same panel but the ti-sn65dsi86 bridge chip. To me, the above indicated that there was a problem with PS8640 and not the panel. I don't really have any insight into what's going on in the MCU, but my best guess is that when the MCU itself sees the HPD go high that it does some AUX transfers itself and this is confusing things. Let's go back and add back in the mysterious 50 ms delay. We only want to do this the first time we see HPD go high after booting the MCU, not every time we double-check HPD. With this, the backlight initializes reliably on homestar. Fixes: 826cff3f7ebb ("drm/bridge: parade-ps8640: Enable runtime power management") Reviewed-by: Stephen Boyd Signed-off-by: Douglas Anderson Link: https://patchwork.freedesktop.org/patch/msgid/20221017121813.1.I59700c745fbc31559a5d5c8e2a960279c751dbd5@changeid --- drivers/gpu/drm/bridge/parade-ps8640.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/bridge/parade-ps8640.c b/drivers/gpu/drm/bridge/parade-ps8640.c index d7483c13c569b..083337a279665 100644 --- a/drivers/gpu/drm/bridge/parade-ps8640.c +++ b/drivers/gpu/drm/bridge/parade-ps8640.c @@ -105,6 +105,7 @@ struct ps8640 { struct gpio_desc *gpio_powerdown; struct device_link *link; bool pre_enabled; + bool need_post_hpd_delay; }; static const struct regmap_config ps8640_regmap_config[] = { @@ -173,14 +174,31 @@ static int _ps8640_wait_hpd_asserted(struct ps8640 *ps_bridge, unsigned long wai { struct regmap *map = ps_bridge->regmap[PAGE2_TOP_CNTL]; int status; + int ret; /* * Apparently something about the firmware in the chip signals that * HPD goes high by reporting GPIO9 as high (even though HPD isn't * actually connected to GPIO9). */ - return regmap_read_poll_timeout(map, PAGE2_GPIO_H, status, - status & PS_GPIO9, wait_us / 10, wait_us); + ret = regmap_read_poll_timeout(map, PAGE2_GPIO_H, status, + status & PS_GPIO9, wait_us / 10, wait_us); + + /* + * The first time we see HPD go high after a reset we delay an extra + * 50 ms. The best guess is that the MCU is doing "stuff" during this + * time (maybe talking to the panel) and we don't want to interrupt it. + * + * No locking is done around "need_post_hpd_delay". If we're here we + * know we're holding a PM Runtime reference and the only other place + * that touches this is PM Runtime resume. + */ + if (!ret && ps_bridge->need_post_hpd_delay) { + ps_bridge->need_post_hpd_delay = false; + msleep(50); + } + + return ret; } static int ps8640_wait_hpd_asserted(struct drm_dp_aux *aux, unsigned long wait_us) @@ -381,6 +399,9 @@ static int __maybe_unused ps8640_resume(struct device *dev) msleep(50); gpiod_set_value(ps_bridge->gpio_reset, 0); + /* We just reset things, so we need a delay after the first HPD */ + ps_bridge->need_post_hpd_delay = true; + /* * Mystery 200 ms delay for the "MCU to be ready". It's unclear if * this is truly necessary since the MCU will already signal that -- GitLab From 8a7a5b5f23377a1eb79b8cc47ba79d805767da47 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Tue, 18 Oct 2022 17:31:38 +0800 Subject: [PATCH 0282/1927] drm/amdgpu: fix pstate setting issue [WHY] 0, original pstate X 1, ctx_A_create -> ctx_A->stable_pstate = X 2, ctx_A_set_pstate (Y) -> current pstate is Y (PEAK or STANDARD) 3, ctx_B_create -> ctx_B->stable_pstate = Y 4, ctx_A_destroy -> restore pstate to X 5, ctx_B_destroy -> restore pstate to Y Above sequence will cause final pstate is wrong (Y), should be original X. [HOW] When ctx_B create, if ctx_A touched pstate setting (not auto, stable_pstate_ctx != NULL), set ctx_B->stable_pstate the same value as ctx_A saved, if stable_pstate_ctx == NULL, fetch current pstate to fill ctx_B->stable_pstate. Signed-off-by: Chengming Gui Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index f6d9d5da53cd2..d2139ac121595 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -326,7 +326,10 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, if (r) return r; - ctx->stable_pstate = current_stable_pstate; + if (mgr->adev->pm.stable_pstate_ctx) + ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate; + else + ctx->stable_pstate = current_stable_pstate; return 0; } -- GitLab From 7e5632cdf68b2d2d53ab9d39daf6a545c2ac2045 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Wed, 19 Oct 2022 11:24:05 +0800 Subject: [PATCH 0283/1927] drm/amd/pm: update driver-if header for smu_v13_0_10 update driver-if header for smu_v13_0_10 and merge with smu_v13_0_0 Signed-off-by: Kenneth Feng Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- .../inc/pmfw_if/smu13_driver_if_v13_0_0.h | 111 +++++++++++++----- drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h | 2 +- .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c | 6 +- 3 files changed, 84 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h index 063f4a7376056..b76f0f7e42998 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h @@ -25,7 +25,7 @@ #define SMU13_DRIVER_IF_V13_0_0_H //Increment this version if SkuTable_t or BoardTable_t change -#define PPTABLE_VERSION 0x24 +#define PPTABLE_VERSION 0x26 #define NUM_GFXCLK_DPM_LEVELS 16 #define NUM_SOCCLK_DPM_LEVELS 8 @@ -109,6 +109,22 @@ #define FEATURE_SPARE_63_BIT 63 #define NUM_FEATURES 64 +#define ALLOWED_FEATURE_CTRL_DEFAULT 0xFFFFFFFFFFFFFFFFULL +#define ALLOWED_FEATURE_CTRL_SCPM ((1 << FEATURE_DPM_GFXCLK_BIT) | \ + (1 << FEATURE_DPM_GFX_POWER_OPTIMIZER_BIT) | \ + (1 << FEATURE_DPM_UCLK_BIT) | \ + (1 << FEATURE_DPM_FCLK_BIT) | \ + (1 << FEATURE_DPM_SOCCLK_BIT) | \ + (1 << FEATURE_DPM_MP0CLK_BIT) | \ + (1 << FEATURE_DPM_LINK_BIT) | \ + (1 << FEATURE_DPM_DCN_BIT) | \ + (1 << FEATURE_DS_GFXCLK_BIT) | \ + (1 << FEATURE_DS_SOCCLK_BIT) | \ + (1 << FEATURE_DS_FCLK_BIT) | \ + (1 << FEATURE_DS_LCLK_BIT) | \ + (1 << FEATURE_DS_DCFCLK_BIT) | \ + (1 << FEATURE_DS_UCLK_BIT)) + //For use with feature control messages typedef enum { FEATURE_PWR_ALL, @@ -133,6 +149,7 @@ typedef enum { #define DEBUG_OVERRIDE_DISABLE_DFLL 0x00000200 #define DEBUG_OVERRIDE_ENABLE_RLC_VF_BRINGUP_MODE 0x00000400 #define DEBUG_OVERRIDE_DFLL_MASTER_MODE 0x00000800 +#define DEBUG_OVERRIDE_ENABLE_PROFILING_MODE 0x00001000 // VR Mapping Bit Defines #define VR_MAPPING_VR_SELECT_MASK 0x01 @@ -262,15 +279,15 @@ typedef enum { } I2cControllerPort_e; typedef enum { - I2C_CONTROLLER_NAME_VR_GFX = 0, - I2C_CONTROLLER_NAME_VR_SOC, - I2C_CONTROLLER_NAME_VR_VMEMP, - I2C_CONTROLLER_NAME_VR_VDDIO, - I2C_CONTROLLER_NAME_LIQUID0, - I2C_CONTROLLER_NAME_LIQUID1, - I2C_CONTROLLER_NAME_PLX, - I2C_CONTROLLER_NAME_OTHER, - I2C_CONTROLLER_NAME_COUNT, + I2C_CONTROLLER_NAME_VR_GFX = 0, + I2C_CONTROLLER_NAME_VR_SOC, + I2C_CONTROLLER_NAME_VR_VMEMP, + I2C_CONTROLLER_NAME_VR_VDDIO, + I2C_CONTROLLER_NAME_LIQUID0, + I2C_CONTROLLER_NAME_LIQUID1, + I2C_CONTROLLER_NAME_PLX, + I2C_CONTROLLER_NAME_FAN_INTAKE, + I2C_CONTROLLER_NAME_COUNT, } I2cControllerName_e; typedef enum { @@ -282,16 +299,17 @@ typedef enum { I2C_CONTROLLER_THROTTLER_LIQUID0, I2C_CONTROLLER_THROTTLER_LIQUID1, I2C_CONTROLLER_THROTTLER_PLX, + I2C_CONTROLLER_THROTTLER_FAN_INTAKE, I2C_CONTROLLER_THROTTLER_INA3221, I2C_CONTROLLER_THROTTLER_COUNT, } I2cControllerThrottler_e; typedef enum { - I2C_CONTROLLER_PROTOCOL_VR_XPDE132G5, - I2C_CONTROLLER_PROTOCOL_VR_IR35217, - I2C_CONTROLLER_PROTOCOL_TMP_TMP102A, - I2C_CONTROLLER_PROTOCOL_INA3221, - I2C_CONTROLLER_PROTOCOL_COUNT, + I2C_CONTROLLER_PROTOCOL_VR_XPDE132G5, + I2C_CONTROLLER_PROTOCOL_VR_IR35217, + I2C_CONTROLLER_PROTOCOL_TMP_MAX31875, + I2C_CONTROLLER_PROTOCOL_INA3221, + I2C_CONTROLLER_PROTOCOL_COUNT, } I2cControllerProtocol_e; typedef struct { @@ -658,13 +676,20 @@ typedef struct { #define PP_NUM_OD_VF_CURVE_POINTS PP_NUM_RTAVFS_PWL_ZONES + 1 +typedef enum { + FAN_MODE_AUTO = 0, + FAN_MODE_MANUAL_LINEAR, +} FanMode_e; typedef struct { uint32_t FeatureCtrlMask; //Voltage control int16_t VoltageOffsetPerZoneBoundary[PP_NUM_OD_VF_CURVE_POINTS]; - uint16_t reserved[2]; + uint16_t VddGfxVmax; // in mV + + uint8_t IdlePwrSavingFeaturesCtrl; + uint8_t RuntimePwrSavingFeaturesCtrl; //Frequency changes int16_t GfxclkFmin; // MHz @@ -674,7 +699,7 @@ typedef struct { //PPT int16_t Ppt; // % - int16_t reserved1; + int16_t Tdc; //Fan control uint8_t FanLinearPwmPoints[NUM_OD_FAN_MAX_POINTS]; @@ -701,16 +726,19 @@ typedef struct { uint32_t FeatureCtrlMask; int16_t VoltageOffsetPerZoneBoundary; - uint16_t reserved[2]; + uint16_t VddGfxVmax; // in mV + + uint8_t IdlePwrSavingFeaturesCtrl; + uint8_t RuntimePwrSavingFeaturesCtrl; - uint16_t GfxclkFmin; // MHz - uint16_t GfxclkFmax; // MHz + int16_t GfxclkFmin; // MHz + int16_t GfxclkFmax; // MHz uint16_t UclkFmin; // MHz uint16_t UclkFmax; // MHz //PPT int16_t Ppt; // % - int16_t reserved1; + int16_t Tdc; uint8_t FanLinearPwmPoints; uint8_t FanLinearTempPoints; @@ -857,7 +885,8 @@ typedef struct { uint16_t FanStartTempMin; uint16_t FanStartTempMax; - uint32_t Spare[12]; + uint16_t PowerMinPpt0[POWER_SOURCE_COUNT]; + uint32_t Spare[11]; } MsgLimits_t; @@ -1041,7 +1070,17 @@ typedef struct { uint32_t GfxoffSpare[15]; // GFX GPO - uint32_t GfxGpoSpare[16]; + uint32_t DfllBtcMasterScalerM; + int32_t DfllBtcMasterScalerB; + uint32_t DfllBtcSlaveScalerM; + int32_t DfllBtcSlaveScalerB; + + uint32_t DfllPccAsWaitCtrl; //GDFLL_AS_WAIT_CTRL_PCC register value to be passed to RLC msg + uint32_t DfllPccAsStepCtrl; //GDFLL_AS_STEP_CTRL_PCC register value to be passed to RLC msg + + uint32_t DfllL2FrequencyBoostM; //Unitless (float) + uint32_t DfllL2FrequencyBoostB; //In MHz (integer) + uint32_t GfxGpoSpare[8]; // GFX DCS @@ -1114,12 +1153,14 @@ typedef struct { uint16_t IntakeTempHighIntakeAcousticLimit; uint16_t IntakeTempAcouticLimitReleaseRate; - uint16_t FanStalledTempLimitOffset; + int16_t FanAbnormalTempLimitOffset; uint16_t FanStalledTriggerRpm; - uint16_t FanAbnormalTriggerRpm; - uint16_t FanPadding; + uint16_t FanAbnormalTriggerRpmCoeff; + uint16_t FanAbnormalDetectionEnable; - uint32_t FanSpare[14]; + uint8_t FanIntakeSensorSupport; + uint8_t FanIntakePadding[3]; + uint32_t FanSpare[13]; // SECTION: VDD_GFX AVFS @@ -1198,8 +1239,13 @@ typedef struct { int16_t TotalBoardPowerM; int16_t TotalBoardPowerB; + //PMFW-11158 + QuadraticInt_t qFeffCoeffGameClock[POWER_SOURCE_COUNT]; + QuadraticInt_t qFeffCoeffBaseClock[POWER_SOURCE_COUNT]; + QuadraticInt_t qFeffCoeffBoostClock[POWER_SOURCE_COUNT]; + // SECTION: Sku Reserved - uint32_t Spare[61]; + uint32_t Spare[43]; // Padding for MMHUB - do not modify this uint32_t MmHubPadding[8]; @@ -1288,8 +1334,11 @@ typedef struct { uint32_t PostVoltageSetBacoDelay; // in microseconds. Amount of time FW will wait after power good is established or PSI0 command is issued uint32_t BacoEntryDelay; // in milliseconds. Amount of time FW will wait to trigger BACO entry after receiving entry notification from OS + uint8_t FuseWritePowerMuxPresent; + uint8_t FuseWritePadding[3]; + // SECTION: Board Reserved - uint32_t BoardSpare[64]; + uint32_t BoardSpare[63]; // SECTION: Structure Padding @@ -1381,7 +1430,7 @@ typedef struct { uint16_t AverageTotalBoardPower; uint16_t AvgTemperature[TEMP_COUNT]; - uint16_t TempPadding; + uint16_t AvgTemperatureFanIntake; uint8_t PcieRate ; uint8_t PcieWidth ; @@ -1550,5 +1599,7 @@ typedef struct { #define IH_INTERRUPT_CONTEXT_ID_AUDIO_D0 0x5 #define IH_INTERRUPT_CONTEXT_ID_AUDIO_D3 0x6 #define IH_INTERRUPT_CONTEXT_ID_THERMAL_THROTTLING 0x7 +#define IH_INTERRUPT_CONTEXT_ID_FAN_ABNORMAL 0x8 +#define IH_INTERRUPT_CONTEXT_ID_FAN_RECOVERY 0x9 #endif diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h index 8f72202aea8e9..80fb583b18d93 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h @@ -30,7 +30,7 @@ #define SMU13_DRIVER_IF_VERSION_ALDE 0x08 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_4 0x07 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_5 0x04 -#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0 0x30 +#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0_10 0x32 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_7 0x2C #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_10 0x1D diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index c4552ade8d441..feb4d68f3fd9a 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -289,7 +289,8 @@ int smu_v13_0_check_fw_version(struct smu_context *smu) smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_ALDE; break; case IP_VERSION(13, 0, 0): - smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_0; + case IP_VERSION(13, 0, 10): + smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_0_10; break; case IP_VERSION(13, 0, 7): smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_7; @@ -305,9 +306,6 @@ int smu_v13_0_check_fw_version(struct smu_context *smu) case IP_VERSION(13, 0, 5): smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_5; break; - case IP_VERSION(13, 0, 10): - smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_10; - break; default: dev_err(adev->dev, "smu unsupported IP version: 0x%x.\n", adev->ip_versions[MP1_HWIP][0]); -- GitLab From bb3c846ad26e1c8551f68078278cbd437a20fb64 Mon Sep 17 00:00:00 2001 From: Yiqing Yao Date: Tue, 18 Oct 2022 15:17:59 +0800 Subject: [PATCH 0284/1927] drm/amdgpu: Adjust MES polling timeout for sriov [why] MES response time in sriov may be longer than default value due to reset or init in other VF. A timeout value specific to sriov is needed. [how] When in sriov, adjust the timeout value to calculated worst case scenario. Signed-off-by: Yiqing Yao Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index fef7d020bc5f3..f141fadd2d86f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -98,7 +98,14 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, struct amdgpu_device *adev = mes->adev; struct amdgpu_ring *ring = &mes->ring; unsigned long flags; + signed long timeout = adev->usec_timeout; + if (amdgpu_emu_mode) { + timeout *= 100; + } else if (amdgpu_sriov_vf(adev)) { + /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */ + timeout = 15 * 600 * 1000; + } BUG_ON(size % 4 != 0); spin_lock_irqsave(&mes->ring_lock, flags); @@ -118,7 +125,7 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode); r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, - adev->usec_timeout * (amdgpu_emu_mode ? 100 : 1)); + timeout); if (r < 1) { DRM_ERROR("MES failed to response msg=%d\n", x_pkt->header.opcode); -- GitLab From 79610d3041338dc1ef554d6fd8b3b3e23be527f5 Mon Sep 17 00:00:00 2001 From: Chengming Gui Date: Tue, 18 Oct 2022 17:31:38 +0800 Subject: [PATCH 0285/1927] drm/amdgpu: fix pstate setting issue [WHY] 0, original pstate X 1, ctx_A_create -> ctx_A->stable_pstate = X 2, ctx_A_set_pstate (Y) -> current pstate is Y (PEAK or STANDARD) 3, ctx_B_create -> ctx_B->stable_pstate = Y 4, ctx_A_destroy -> restore pstate to X 5, ctx_B_destroy -> restore pstate to Y Above sequence will cause final pstate is wrong (Y), should be original X. [HOW] When ctx_B create, if ctx_A touched pstate setting (not auto, stable_pstate_ctx != NULL), set ctx_B->stable_pstate the same value as ctx_A saved, if stable_pstate_ctx == NULL, fetch current pstate to fill ctx_B->stable_pstate. Signed-off-by: Chengming Gui Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index f6d9d5da53cd2..d2139ac121595 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -326,7 +326,10 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, if (r) return r; - ctx->stable_pstate = current_stable_pstate; + if (mgr->adev->pm.stable_pstate_ctx) + ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate; + else + ctx->stable_pstate = current_stable_pstate; return 0; } -- GitLab From 09aef0258a327409bb2279a5ba8f82ad2ca099ca Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Wed, 19 Oct 2022 11:24:05 +0800 Subject: [PATCH 0286/1927] drm/amd/pm: update driver-if header for smu_v13_0_10 update driver-if header for smu_v13_0_10 and merge with smu_v13_0_0 Signed-off-by: Kenneth Feng Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- .../inc/pmfw_if/smu13_driver_if_v13_0_0.h | 111 +++++++++++++----- drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h | 2 +- .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c | 6 +- 3 files changed, 84 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h index 063f4a7376056..b76f0f7e42998 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu13_driver_if_v13_0_0.h @@ -25,7 +25,7 @@ #define SMU13_DRIVER_IF_V13_0_0_H //Increment this version if SkuTable_t or BoardTable_t change -#define PPTABLE_VERSION 0x24 +#define PPTABLE_VERSION 0x26 #define NUM_GFXCLK_DPM_LEVELS 16 #define NUM_SOCCLK_DPM_LEVELS 8 @@ -109,6 +109,22 @@ #define FEATURE_SPARE_63_BIT 63 #define NUM_FEATURES 64 +#define ALLOWED_FEATURE_CTRL_DEFAULT 0xFFFFFFFFFFFFFFFFULL +#define ALLOWED_FEATURE_CTRL_SCPM ((1 << FEATURE_DPM_GFXCLK_BIT) | \ + (1 << FEATURE_DPM_GFX_POWER_OPTIMIZER_BIT) | \ + (1 << FEATURE_DPM_UCLK_BIT) | \ + (1 << FEATURE_DPM_FCLK_BIT) | \ + (1 << FEATURE_DPM_SOCCLK_BIT) | \ + (1 << FEATURE_DPM_MP0CLK_BIT) | \ + (1 << FEATURE_DPM_LINK_BIT) | \ + (1 << FEATURE_DPM_DCN_BIT) | \ + (1 << FEATURE_DS_GFXCLK_BIT) | \ + (1 << FEATURE_DS_SOCCLK_BIT) | \ + (1 << FEATURE_DS_FCLK_BIT) | \ + (1 << FEATURE_DS_LCLK_BIT) | \ + (1 << FEATURE_DS_DCFCLK_BIT) | \ + (1 << FEATURE_DS_UCLK_BIT)) + //For use with feature control messages typedef enum { FEATURE_PWR_ALL, @@ -133,6 +149,7 @@ typedef enum { #define DEBUG_OVERRIDE_DISABLE_DFLL 0x00000200 #define DEBUG_OVERRIDE_ENABLE_RLC_VF_BRINGUP_MODE 0x00000400 #define DEBUG_OVERRIDE_DFLL_MASTER_MODE 0x00000800 +#define DEBUG_OVERRIDE_ENABLE_PROFILING_MODE 0x00001000 // VR Mapping Bit Defines #define VR_MAPPING_VR_SELECT_MASK 0x01 @@ -262,15 +279,15 @@ typedef enum { } I2cControllerPort_e; typedef enum { - I2C_CONTROLLER_NAME_VR_GFX = 0, - I2C_CONTROLLER_NAME_VR_SOC, - I2C_CONTROLLER_NAME_VR_VMEMP, - I2C_CONTROLLER_NAME_VR_VDDIO, - I2C_CONTROLLER_NAME_LIQUID0, - I2C_CONTROLLER_NAME_LIQUID1, - I2C_CONTROLLER_NAME_PLX, - I2C_CONTROLLER_NAME_OTHER, - I2C_CONTROLLER_NAME_COUNT, + I2C_CONTROLLER_NAME_VR_GFX = 0, + I2C_CONTROLLER_NAME_VR_SOC, + I2C_CONTROLLER_NAME_VR_VMEMP, + I2C_CONTROLLER_NAME_VR_VDDIO, + I2C_CONTROLLER_NAME_LIQUID0, + I2C_CONTROLLER_NAME_LIQUID1, + I2C_CONTROLLER_NAME_PLX, + I2C_CONTROLLER_NAME_FAN_INTAKE, + I2C_CONTROLLER_NAME_COUNT, } I2cControllerName_e; typedef enum { @@ -282,16 +299,17 @@ typedef enum { I2C_CONTROLLER_THROTTLER_LIQUID0, I2C_CONTROLLER_THROTTLER_LIQUID1, I2C_CONTROLLER_THROTTLER_PLX, + I2C_CONTROLLER_THROTTLER_FAN_INTAKE, I2C_CONTROLLER_THROTTLER_INA3221, I2C_CONTROLLER_THROTTLER_COUNT, } I2cControllerThrottler_e; typedef enum { - I2C_CONTROLLER_PROTOCOL_VR_XPDE132G5, - I2C_CONTROLLER_PROTOCOL_VR_IR35217, - I2C_CONTROLLER_PROTOCOL_TMP_TMP102A, - I2C_CONTROLLER_PROTOCOL_INA3221, - I2C_CONTROLLER_PROTOCOL_COUNT, + I2C_CONTROLLER_PROTOCOL_VR_XPDE132G5, + I2C_CONTROLLER_PROTOCOL_VR_IR35217, + I2C_CONTROLLER_PROTOCOL_TMP_MAX31875, + I2C_CONTROLLER_PROTOCOL_INA3221, + I2C_CONTROLLER_PROTOCOL_COUNT, } I2cControllerProtocol_e; typedef struct { @@ -658,13 +676,20 @@ typedef struct { #define PP_NUM_OD_VF_CURVE_POINTS PP_NUM_RTAVFS_PWL_ZONES + 1 +typedef enum { + FAN_MODE_AUTO = 0, + FAN_MODE_MANUAL_LINEAR, +} FanMode_e; typedef struct { uint32_t FeatureCtrlMask; //Voltage control int16_t VoltageOffsetPerZoneBoundary[PP_NUM_OD_VF_CURVE_POINTS]; - uint16_t reserved[2]; + uint16_t VddGfxVmax; // in mV + + uint8_t IdlePwrSavingFeaturesCtrl; + uint8_t RuntimePwrSavingFeaturesCtrl; //Frequency changes int16_t GfxclkFmin; // MHz @@ -674,7 +699,7 @@ typedef struct { //PPT int16_t Ppt; // % - int16_t reserved1; + int16_t Tdc; //Fan control uint8_t FanLinearPwmPoints[NUM_OD_FAN_MAX_POINTS]; @@ -701,16 +726,19 @@ typedef struct { uint32_t FeatureCtrlMask; int16_t VoltageOffsetPerZoneBoundary; - uint16_t reserved[2]; + uint16_t VddGfxVmax; // in mV + + uint8_t IdlePwrSavingFeaturesCtrl; + uint8_t RuntimePwrSavingFeaturesCtrl; - uint16_t GfxclkFmin; // MHz - uint16_t GfxclkFmax; // MHz + int16_t GfxclkFmin; // MHz + int16_t GfxclkFmax; // MHz uint16_t UclkFmin; // MHz uint16_t UclkFmax; // MHz //PPT int16_t Ppt; // % - int16_t reserved1; + int16_t Tdc; uint8_t FanLinearPwmPoints; uint8_t FanLinearTempPoints; @@ -857,7 +885,8 @@ typedef struct { uint16_t FanStartTempMin; uint16_t FanStartTempMax; - uint32_t Spare[12]; + uint16_t PowerMinPpt0[POWER_SOURCE_COUNT]; + uint32_t Spare[11]; } MsgLimits_t; @@ -1041,7 +1070,17 @@ typedef struct { uint32_t GfxoffSpare[15]; // GFX GPO - uint32_t GfxGpoSpare[16]; + uint32_t DfllBtcMasterScalerM; + int32_t DfllBtcMasterScalerB; + uint32_t DfllBtcSlaveScalerM; + int32_t DfllBtcSlaveScalerB; + + uint32_t DfllPccAsWaitCtrl; //GDFLL_AS_WAIT_CTRL_PCC register value to be passed to RLC msg + uint32_t DfllPccAsStepCtrl; //GDFLL_AS_STEP_CTRL_PCC register value to be passed to RLC msg + + uint32_t DfllL2FrequencyBoostM; //Unitless (float) + uint32_t DfllL2FrequencyBoostB; //In MHz (integer) + uint32_t GfxGpoSpare[8]; // GFX DCS @@ -1114,12 +1153,14 @@ typedef struct { uint16_t IntakeTempHighIntakeAcousticLimit; uint16_t IntakeTempAcouticLimitReleaseRate; - uint16_t FanStalledTempLimitOffset; + int16_t FanAbnormalTempLimitOffset; uint16_t FanStalledTriggerRpm; - uint16_t FanAbnormalTriggerRpm; - uint16_t FanPadding; + uint16_t FanAbnormalTriggerRpmCoeff; + uint16_t FanAbnormalDetectionEnable; - uint32_t FanSpare[14]; + uint8_t FanIntakeSensorSupport; + uint8_t FanIntakePadding[3]; + uint32_t FanSpare[13]; // SECTION: VDD_GFX AVFS @@ -1198,8 +1239,13 @@ typedef struct { int16_t TotalBoardPowerM; int16_t TotalBoardPowerB; + //PMFW-11158 + QuadraticInt_t qFeffCoeffGameClock[POWER_SOURCE_COUNT]; + QuadraticInt_t qFeffCoeffBaseClock[POWER_SOURCE_COUNT]; + QuadraticInt_t qFeffCoeffBoostClock[POWER_SOURCE_COUNT]; + // SECTION: Sku Reserved - uint32_t Spare[61]; + uint32_t Spare[43]; // Padding for MMHUB - do not modify this uint32_t MmHubPadding[8]; @@ -1288,8 +1334,11 @@ typedef struct { uint32_t PostVoltageSetBacoDelay; // in microseconds. Amount of time FW will wait after power good is established or PSI0 command is issued uint32_t BacoEntryDelay; // in milliseconds. Amount of time FW will wait to trigger BACO entry after receiving entry notification from OS + uint8_t FuseWritePowerMuxPresent; + uint8_t FuseWritePadding[3]; + // SECTION: Board Reserved - uint32_t BoardSpare[64]; + uint32_t BoardSpare[63]; // SECTION: Structure Padding @@ -1381,7 +1430,7 @@ typedef struct { uint16_t AverageTotalBoardPower; uint16_t AvgTemperature[TEMP_COUNT]; - uint16_t TempPadding; + uint16_t AvgTemperatureFanIntake; uint8_t PcieRate ; uint8_t PcieWidth ; @@ -1550,5 +1599,7 @@ typedef struct { #define IH_INTERRUPT_CONTEXT_ID_AUDIO_D0 0x5 #define IH_INTERRUPT_CONTEXT_ID_AUDIO_D3 0x6 #define IH_INTERRUPT_CONTEXT_ID_THERMAL_THROTTLING 0x7 +#define IH_INTERRUPT_CONTEXT_ID_FAN_ABNORMAL 0x8 +#define IH_INTERRUPT_CONTEXT_ID_FAN_RECOVERY 0x9 #endif diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h index 8f72202aea8e9..80fb583b18d93 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h @@ -30,7 +30,7 @@ #define SMU13_DRIVER_IF_VERSION_ALDE 0x08 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_4 0x07 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_5 0x04 -#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0 0x30 +#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0_10 0x32 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_7 0x2C #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_10 0x1D diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index c4552ade8d441..feb4d68f3fd9a 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -289,7 +289,8 @@ int smu_v13_0_check_fw_version(struct smu_context *smu) smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_ALDE; break; case IP_VERSION(13, 0, 0): - smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_0; + case IP_VERSION(13, 0, 10): + smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_0_10; break; case IP_VERSION(13, 0, 7): smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_7; @@ -305,9 +306,6 @@ int smu_v13_0_check_fw_version(struct smu_context *smu) case IP_VERSION(13, 0, 5): smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_5; break; - case IP_VERSION(13, 0, 10): - smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_10; - break; default: dev_err(adev->dev, "smu unsupported IP version: 0x%x.\n", adev->ip_versions[MP1_HWIP][0]); -- GitLab From 226dcfad349f23f7744d02b24f8ec3bc4f6198ac Mon Sep 17 00:00:00 2001 From: Yiqing Yao Date: Tue, 18 Oct 2022 15:17:59 +0800 Subject: [PATCH 0287/1927] drm/amdgpu: Adjust MES polling timeout for sriov [why] MES response time in sriov may be longer than default value due to reset or init in other VF. A timeout value specific to sriov is needed. [how] When in sriov, adjust the timeout value to calculated worst case scenario. Signed-off-by: Yiqing Yao Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index fef7d020bc5f3..f141fadd2d86f 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -98,7 +98,14 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, struct amdgpu_device *adev = mes->adev; struct amdgpu_ring *ring = &mes->ring; unsigned long flags; + signed long timeout = adev->usec_timeout; + if (amdgpu_emu_mode) { + timeout *= 100; + } else if (amdgpu_sriov_vf(adev)) { + /* Worst case in sriov where all other 15 VF timeout, each VF needs about 600ms */ + timeout = 15 * 600 * 1000; + } BUG_ON(size % 4 != 0); spin_lock_irqsave(&mes->ring_lock, flags); @@ -118,7 +125,7 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, DRM_DEBUG("MES msg=%d was emitted\n", x_pkt->header.opcode); r = amdgpu_fence_wait_polling(ring, ring->fence_drv.sync_seq, - adev->usec_timeout * (amdgpu_emu_mode ? 100 : 1)); + timeout); if (r < 1) { DRM_ERROR("MES failed to response msg=%d\n", x_pkt->header.opcode); -- GitLab From 24f0692bfd41fd207d99c993a5785c3426762046 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 20 Oct 2022 16:54:55 -0700 Subject: [PATCH 0288/1927] ACPI: NUMA: Add CXL CFMWS 'nodes' to the possible nodes set The ACPI CEDT.CFMWS indicates a range of possible address where new CXL regions can appear. Each range is associated with a QTG id (QoS Throttling Group id). For each range + QTG pair that is not covered by a proximity domain in the SRAT, Linux creates a new NUMA node. However, the commit that added the new ranges missed updating the node_possible mask which causes memory_group_register() to fail. Add the new nodes to the nodes_possible mask. Cc: Fixes: fd49f99c1809 ("ACPI: NUMA: Add a node and memblk for each CFMWS not in SRAT") Cc: Alison Schofield Cc: Rafael J. Wysocki Reported-by: Vishal Verma Tested-by: Vishal Verma Acked-by: Rafael J. Wysocki Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166631003537.1167078.9373680312035292395.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/acpi/numa/srat.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/acpi/numa/srat.c b/drivers/acpi/numa/srat.c index 3b818ab186be8..1f4fc5f8a819d 100644 --- a/drivers/acpi/numa/srat.c +++ b/drivers/acpi/numa/srat.c @@ -327,6 +327,7 @@ static int __init acpi_parse_cfmws(union acpi_subtable_headers *header, pr_warn("ACPI NUMA: Failed to add memblk for CFMWS node %d [mem %#llx-%#llx]\n", node, start, end); } + node_set(node, numa_nodes_parsed); /* Set the next available fake_pxm value */ (*fake_pxm)++; -- GitLab From 3d05818707bb2cf133ccdcd29f2d5585b5bc1298 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Fri, 21 Oct 2022 19:49:12 +0800 Subject: [PATCH 0289/1927] bpf: Wait for busy refill_work when destroying bpf memory allocator A busy irq work is an unfinished irq work and it can be either in the pending state or in the running state. When destroying bpf memory allocator, refill_work may be busy for PREEMPT_RT kernel in which irq work is invoked in a per-CPU RT-kthread. It is also possible for kernel with arch_irq_work_has_interrupt() being false (e.g. 1-cpu arm32 host or mips) and irq work is inovked in timer interrupt. The busy refill_work leads to various issues. The obvious one is that there will be concurrent operations on free_by_rcu and free_list between irq work and memory draining. Another one is call_rcu_in_progress will not be reliable for the checking of pending RCU callback because do_call_rcu() may have not been invoked by irq work yet. The other is there will be use-after-free if irq work is freed before the callback of irq work is invoked as shown below: BUG: kernel NULL pointer dereference, address: 0000000000000000 #PF: supervisor instruction fetch in kernel mode #PF: error_code(0x0010) - not-present page PGD 12ab94067 P4D 12ab94067 PUD 1796b4067 PMD 0 Oops: 0010 [#1] PREEMPT_RT SMP CPU: 5 PID: 64 Comm: irq_work/5 Not tainted 6.0.0-rt11+ #1 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996) RIP: 0010:0x0 Code: Unable to access opcode bytes at 0xffffffffffffffd6. RSP: 0018:ffffadc080293e78 EFLAGS: 00010286 RAX: 0000000000000000 RBX: ffffcdc07fb6a388 RCX: ffffa05000a2e000 RDX: ffffa05000a2e000 RSI: ffffffff96cc9827 RDI: ffffcdc07fb6a388 ...... Call Trace: irq_work_single+0x24/0x60 irq_work_run_list+0x24/0x30 run_irq_workd+0x23/0x30 smpboot_thread_fn+0x203/0x300 kthread+0x126/0x150 ret_from_fork+0x1f/0x30 Considering the ease of concurrency handling, no overhead for irq_work_sync() under non-PREEMPT_RT kernel and has-irq-work-interrupt kernel and the short wait time used for irq_work_sync() under PREEMPT_RT (When running two test_maps on PREEMPT_RT kernel and 72-cpus host, the max wait time is about 8ms and the 99th percentile is 10us), just using irq_work_sync() to wait for busy refill_work to complete before memory draining and memory freeing. Fixes: 7c8199e24fa0 ("bpf: Introduce any context BPF specific memory allocator.") Acked-by: Stanislav Fomichev Signed-off-by: Hou Tao Link: https://lore.kernel.org/r/20221021114913.60508-2-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/memalloc.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 5f83be1d20181..1cb24323826f5 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -493,6 +493,16 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) rcu_in_progress = 0; for_each_possible_cpu(cpu) { c = per_cpu_ptr(ma->cache, cpu); + /* + * refill_work may be unfinished for PREEMPT_RT kernel + * in which irq work is invoked in a per-CPU RT thread. + * It is also possible for kernel with + * arch_irq_work_has_interrupt() being false and irq + * work is invoked in timer interrupt. So waiting for + * the completion of irq work to ease the handling of + * concurrency. + */ + irq_work_sync(&c->refill_work); drain_mem_cache(c); rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } @@ -507,6 +517,7 @@ void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma) cc = per_cpu_ptr(ma->caches, cpu); for (i = 0; i < NUM_CACHES; i++) { c = &cc->cache[i]; + irq_work_sync(&c->refill_work); drain_mem_cache(c); rcu_in_progress += atomic_read(&c->call_rcu_in_progress); } -- GitLab From fa4447cb73b2bfe7175f1b7ffdc70580fcfcb991 Mon Sep 17 00:00:00 2001 From: Hou Tao Date: Fri, 21 Oct 2022 19:49:13 +0800 Subject: [PATCH 0290/1927] bpf: Use __llist_del_all() whenever possbile during memory draining Except for waiting_for_gp list, there are no concurrent operations on free_by_rcu, free_llist and free_llist_extra lists, so use __llist_del_all() instead of llist_del_all(). waiting_for_gp list can be deleted by RCU callback concurrently, so still use llist_del_all(). Acked-by: Stanislav Fomichev Signed-off-by: Hou Tao Link: https://lore.kernel.org/r/20221021114913.60508-3-houtao@huaweicloud.com Signed-off-by: Alexei Starovoitov --- kernel/bpf/memalloc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index 1cb24323826f5..4901fa1048cd7 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -418,14 +418,17 @@ static void drain_mem_cache(struct bpf_mem_cache *c) /* No progs are using this bpf_mem_cache, but htab_map_free() called * bpf_mem_cache_free() for all remaining elements and they can be in * free_by_rcu or in waiting_for_gp lists, so drain those lists now. + * + * Except for waiting_for_gp list, there are no concurrent operations + * on these lists, so it is safe to use __llist_del_all(). */ llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu)) free_one(c, llnode); llist_for_each_safe(llnode, t, llist_del_all(&c->waiting_for_gp)) free_one(c, llnode); - llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist)) + llist_for_each_safe(llnode, t, __llist_del_all(&c->free_llist)) free_one(c, llnode); - llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra)) + llist_for_each_safe(llnode, t, __llist_del_all(&c->free_llist_extra)) free_one(c, llnode); } -- GitLab From 48ee7952808183201b0601d85b89d2d8ccca95ff Mon Sep 17 00:00:00 2001 From: Dukhyun Kwon Date: Tue, 18 Oct 2022 16:30:03 +0900 Subject: [PATCH 0291/1927] scsi: ufs: core: Fix the error log in ufshcd_query_flag_retry() In ufshcd_query_flag_retry() failed log is incorrectly output as "ufs attibute". Signed-off-by: Dukhyun Kwon Link: https://lore.kernel.org/r/1891546521.01666080182092.JavaMail.epsvc@epcpadp4 Reviewed-by: Bean Huo Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 7256e6c43ca68..54ba80cc71bc3 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -3098,7 +3098,7 @@ static int ufshcd_query_flag_retry(struct ufs_hba *hba, if (ret) dev_err(hba->dev, - "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retries\n", + "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n", __func__, opcode, idn, ret, retries); return ret; } -- GitLab From 181dfce9b63b80adbb861b219550ec9b27fe63d5 Mon Sep 17 00:00:00 2001 From: Igor Pylypiv Date: Fri, 7 Oct 2022 16:06:51 -0700 Subject: [PATCH 0292/1927] scsi: pm80xx: Display proc_name in sysfs The proc_name entry in sysfs for pm80xx is "(null)" because it is not initialized in scsi_host_template: Before: host:~# cat /sys/class/scsi_host/host6/proc_name (null) After: host:~# cat /sys/class/scsi_host/host6/proc_name pm80xx Signed-off-by: Igor Pylypiv Link: https://lore.kernel.org/r/20221007230651.308969-1-ipylypiv@google.com Reviewed-by: Jolly Shah Acked-by: Jack Wang Signed-off-by: Martin K. Petersen --- drivers/scsi/pm8001/pm8001_init.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index 2ff2fac1e403d..7a7d63aa90e21 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -99,6 +99,7 @@ static void pm8001_map_queues(struct Scsi_Host *shost) static struct scsi_host_template pm8001_sht = { .module = THIS_MODULE, .name = DRV_NAME, + .proc_name = DRV_NAME, .queuecommand = sas_queuecommand, .dma_need_drain = ata_scsi_dma_need_drain, .target_alloc = sas_target_alloc, -- GitLab From 4652b58fe3bb177a9b208bb7a8b7a3fb64184a00 Mon Sep 17 00:00:00 2001 From: Keoseong Park Date: Mon, 17 Oct 2022 18:58:15 +0900 Subject: [PATCH 0293/1927] scsi: ufs: core: Fix typo for register name in comments Change "UTRMLCLR" to "UTMRLCLR". The meaning is "UTP Task Management Request List CLear Register" Signed-off-by: Keoseong Park Link: https://lore.kernel.org/r/20221017095815epcms2p110e3421b99bb9a937620b4d065d0ed12@epcms2p1 Reviewed-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshcd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 54ba80cc71bc3..b1f59a5fe6327 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -772,7 +772,7 @@ static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask) } /** - * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register + * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register * @hba: per adapter instance * @pos: position of the bit to be cleared */ -- GitLab From 65244389b1b347447a7eed866d9d458963e851e8 Mon Sep 17 00:00:00 2001 From: Michal Kubecek Date: Mon, 17 Oct 2022 16:55:17 +0200 Subject: [PATCH 0294/1927] scsi: mpi3mr: Select CONFIG_SCSI_SAS_ATTRS Starting with commit 42fc9fee116f ("scsi: mpi3mr: Add helper functions to manage device's port"), kernel configured with CONFIG_SCSI_MPI3MR=m and CONFIG_SCSI_SAS_ATTRS=n fails to build because modpost cannot find symbols used in mpi3mr_transport.c: ERROR: modpost: "sas_port_alloc_num" [drivers/scsi/mpi3mr/mpi3mr.ko] undefined! ERROR: modpost: "sas_remove_host" [drivers/scsi/mpi3mr/mpi3mr.ko] undefined! ERROR: modpost: "sas_phy_alloc" [drivers/scsi/mpi3mr/mpi3mr.ko] undefined! ERROR: modpost: "sas_phy_free" [drivers/scsi/mpi3mr/mpi3mr.ko] undefined! ... Select CONFIG_SCSI_SAS_ATTRS when CONFIG_SCSI_MPI3MR is enabled to prevent inconsistent configs. Link: https://lore.kernel.org/r/20221017145517.93BCB6043B@lion.mk-sys.cz Fixes: 42fc9fee116f ("scsi: mpi3mr: Add helper functions to manage device's port") Acked-by: Sathya Prakash Veerichetty Signed-off-by: Michal Kubecek Signed-off-by: Martin K. Petersen --- drivers/scsi/mpi3mr/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/scsi/mpi3mr/Kconfig b/drivers/scsi/mpi3mr/Kconfig index 8997531940c25..f48740cd5b957 100644 --- a/drivers/scsi/mpi3mr/Kconfig +++ b/drivers/scsi/mpi3mr/Kconfig @@ -4,5 +4,6 @@ config SCSI_MPI3MR tristate "Broadcom MPI3 Storage Controller Device Driver" depends on PCI && SCSI select BLK_DEV_BSGLIB + select SCSI_SAS_ATTRS help MPI3 based Storage & RAID Controllers Driver. -- GitLab From 307539eed46395d27e0ecc0ae4d9d6e99eb15fcd Mon Sep 17 00:00:00 2001 From: Keoseong Park Date: Wed, 19 Oct 2022 12:45:30 +0900 Subject: [PATCH 0295/1927] scsi: ufs: core: Fix typo in comment Change "drity" to "dirty". Signed-off-by: Keoseong Park Link: https://lore.kernel.org/r/20221019034530epcms2p2b10e072bb66b3fd6cdbe0e2423c11735@epcms2p2 Signed-off-by: Martin K. Petersen --- drivers/ufs/core/ufshpb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ufs/core/ufshpb.c b/drivers/ufs/core/ufshpb.c index 7d56c9b4f7a80..b7f412d0f3019 100644 --- a/drivers/ufs/core/ufshpb.c +++ b/drivers/ufs/core/ufshpb.c @@ -383,7 +383,7 @@ int ufshpb_prep(struct ufs_hba *hba, struct ufshcd_lrb *lrbp) rgn = hpb->rgn_tbl + rgn_idx; srgn = rgn->srgn_tbl + srgn_idx; - /* If command type is WRITE or DISCARD, set bitmap as drity */ + /* If command type is WRITE or DISCARD, set bitmap as dirty */ if (ufshpb_is_write_or_discard(cmd)) { ufshpb_iterate_rgn(hpb, rgn_idx, srgn_idx, srgn_offset, transfer_len, true); -- GitLab From f97fc7ef414603189d5ba6f529407c5341c03c2a Mon Sep 17 00:00:00 2001 From: Raju Rangoju Date: Thu, 20 Oct 2022 12:12:11 +0530 Subject: [PATCH 0296/1927] amd-xgbe: Yellow carp devices do not need rrc Link stability issues are noticed on Yellow carp platforms when Receiver Reset Cycle is issued. Since the CDR workaround is disabled on these platforms, the Receiver Reset Cycle is not needed. So, avoid issuing rrc on Yellow carp platforms. Fixes: dbb6c58b5a61 ("net: amd-xgbe: Add Support for Yellow Carp Ethernet device") Signed-off-by: Raju Rangoju Acked-by: Tom Lendacky Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amd/xgbe/xgbe-pci.c | 5 +++++ drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c | 2 +- drivers/net/ethernet/amd/xgbe/xgbe.h | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c index 2af3da4b2d053..f409d7bd1f1ee 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c @@ -285,6 +285,9 @@ static int xgbe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) /* Yellow Carp devices do not need cdr workaround */ pdata->vdata->an_cdr_workaround = 0; + + /* Yellow Carp devices do not need rrc */ + pdata->vdata->enable_rrc = 0; } else { pdata->xpcs_window_def_reg = PCS_V2_WINDOW_DEF; pdata->xpcs_window_sel_reg = PCS_V2_WINDOW_SELECT; @@ -483,6 +486,7 @@ static struct xgbe_version_data xgbe_v2a = { .tx_desc_prefetch = 5, .rx_desc_prefetch = 5, .an_cdr_workaround = 1, + .enable_rrc = 1, }; static struct xgbe_version_data xgbe_v2b = { @@ -498,6 +502,7 @@ static struct xgbe_version_data xgbe_v2b = { .tx_desc_prefetch = 5, .rx_desc_prefetch = 5, .an_cdr_workaround = 1, + .enable_rrc = 1, }; static const struct pci_device_id xgbe_pci_table[] = { diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index 2156600641b6c..19b943eba560e 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -2640,7 +2640,7 @@ static int xgbe_phy_link_status(struct xgbe_prv_data *pdata, int *an_restart) } /* No link, attempt a receiver reset cycle */ - if (phy_data->rrc_count++ > XGBE_RRC_FREQUENCY) { + if (pdata->vdata->enable_rrc && phy_data->rrc_count++ > XGBE_RRC_FREQUENCY) { phy_data->rrc_count = 0; xgbe_phy_rrc(pdata); } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index b875c430222e5..49d23abce73d2 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -1013,6 +1013,7 @@ struct xgbe_version_data { unsigned int tx_desc_prefetch; unsigned int rx_desc_prefetch; unsigned int an_cdr_workaround; + unsigned int enable_rrc; }; struct xgbe_prv_data { -- GitLab From 1246d0862349f36b65db1043d27c466af930047d Mon Sep 17 00:00:00 2001 From: Raju Rangoju Date: Thu, 20 Oct 2022 12:12:12 +0530 Subject: [PATCH 0297/1927] amd-xgbe: use enums for mailbox cmd and sub_cmds Instead of using hardcoded values, use enumerations for mailbox command and sub commands. Signed-off-by: Raju Rangoju Acked-by: Tom Lendacky Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c | 29 ++++++++++++--------- drivers/net/ethernet/amd/xgbe/xgbe.h | 25 ++++++++++++++++++ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index 19b943eba560e..8cf5d81fca363 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -1989,7 +1989,7 @@ static void xgbe_phy_pll_ctrl(struct xgbe_prv_data *pdata, bool enable) } static void xgbe_phy_perform_ratechange(struct xgbe_prv_data *pdata, - unsigned int cmd, unsigned int sub_cmd) + enum xgbe_mb_cmd cmd, enum xgbe_mb_subcmd sub_cmd) { unsigned int s0 = 0; unsigned int wait; @@ -2036,7 +2036,7 @@ static void xgbe_phy_perform_ratechange(struct xgbe_prv_data *pdata, static void xgbe_phy_rrc(struct xgbe_prv_data *pdata) { /* Receiver Reset Cycle */ - xgbe_phy_perform_ratechange(pdata, 5, 0); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_RRC, XGBE_MB_SUBCMD_NONE); netif_dbg(pdata, link, pdata->netdev, "receiver reset complete\n"); } @@ -2046,7 +2046,7 @@ static void xgbe_phy_power_off(struct xgbe_prv_data *pdata) struct xgbe_phy_data *phy_data = pdata->phy_data; /* Power off */ - xgbe_phy_perform_ratechange(pdata, 0, 0); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_POWER_OFF, XGBE_MB_SUBCMD_NONE); phy_data->cur_mode = XGBE_MODE_UNKNOWN; @@ -2061,14 +2061,17 @@ static void xgbe_phy_sfi_mode(struct xgbe_prv_data *pdata) /* 10G/SFI */ if (phy_data->sfp_cable != XGBE_SFP_CABLE_PASSIVE) { - xgbe_phy_perform_ratechange(pdata, 3, 0); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_10G_SFI, XGBE_MB_SUBCMD_ACTIVE); } else { if (phy_data->sfp_cable_len <= 1) - xgbe_phy_perform_ratechange(pdata, 3, 1); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_10G_SFI, + XGBE_MB_SUBCMD_PASSIVE_1M); else if (phy_data->sfp_cable_len <= 3) - xgbe_phy_perform_ratechange(pdata, 3, 2); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_10G_SFI, + XGBE_MB_SUBCMD_PASSIVE_3M); else - xgbe_phy_perform_ratechange(pdata, 3, 3); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_10G_SFI, + XGBE_MB_SUBCMD_PASSIVE_OTHER); } phy_data->cur_mode = XGBE_MODE_SFI; @@ -2083,7 +2086,7 @@ static void xgbe_phy_x_mode(struct xgbe_prv_data *pdata) xgbe_phy_set_redrv_mode(pdata); /* 1G/X */ - xgbe_phy_perform_ratechange(pdata, 1, 3); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_1G, XGBE_MB_SUBCMD_1G_KX); phy_data->cur_mode = XGBE_MODE_X; @@ -2097,7 +2100,7 @@ static void xgbe_phy_sgmii_1000_mode(struct xgbe_prv_data *pdata) xgbe_phy_set_redrv_mode(pdata); /* 1G/SGMII */ - xgbe_phy_perform_ratechange(pdata, 1, 2); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_1G, XGBE_MB_SUBCMD_1G_SGMII); phy_data->cur_mode = XGBE_MODE_SGMII_1000; @@ -2111,7 +2114,7 @@ static void xgbe_phy_sgmii_100_mode(struct xgbe_prv_data *pdata) xgbe_phy_set_redrv_mode(pdata); /* 100M/SGMII */ - xgbe_phy_perform_ratechange(pdata, 1, 1); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_1G, XGBE_MB_SUBCMD_100MBITS); phy_data->cur_mode = XGBE_MODE_SGMII_100; @@ -2125,7 +2128,7 @@ static void xgbe_phy_kr_mode(struct xgbe_prv_data *pdata) xgbe_phy_set_redrv_mode(pdata); /* 10G/KR */ - xgbe_phy_perform_ratechange(pdata, 4, 0); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_10G_KR, XGBE_MB_SUBCMD_NONE); phy_data->cur_mode = XGBE_MODE_KR; @@ -2139,7 +2142,7 @@ static void xgbe_phy_kx_2500_mode(struct xgbe_prv_data *pdata) xgbe_phy_set_redrv_mode(pdata); /* 2.5G/KX */ - xgbe_phy_perform_ratechange(pdata, 2, 0); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G, XGBE_MB_SUBCMD_NONE); phy_data->cur_mode = XGBE_MODE_KX_2500; @@ -2153,7 +2156,7 @@ static void xgbe_phy_kx_1000_mode(struct xgbe_prv_data *pdata) xgbe_phy_set_redrv_mode(pdata); /* 1G/KX */ - xgbe_phy_perform_ratechange(pdata, 1, 3); + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_1G, XGBE_MB_SUBCMD_1G_KX); phy_data->cur_mode = XGBE_MODE_KX_1000; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 49d23abce73d2..71f24cb479355 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -611,6 +611,31 @@ enum xgbe_mdio_mode { XGBE_MDIO_MODE_CL45, }; +enum xgbe_mb_cmd { + XGBE_MB_CMD_POWER_OFF = 0, + XGBE_MB_CMD_SET_1G, + XGBE_MB_CMD_SET_2_5G, + XGBE_MB_CMD_SET_10G_SFI, + XGBE_MB_CMD_SET_10G_KR, + XGBE_MB_CMD_RRC +}; + +enum xgbe_mb_subcmd { + XGBE_MB_SUBCMD_NONE = 0, + + /* 10GbE SFP subcommands */ + XGBE_MB_SUBCMD_ACTIVE = 0, + XGBE_MB_SUBCMD_PASSIVE_1M, + XGBE_MB_SUBCMD_PASSIVE_3M, + XGBE_MB_SUBCMD_PASSIVE_OTHER, + + /* 1GbE Mode subcommands */ + XGBE_MB_SUBCMD_10MBITS = 0, + XGBE_MB_SUBCMD_100MBITS, + XGBE_MB_SUBCMD_1G_SGMII, + XGBE_MB_SUBCMD_1G_KX +}; + struct xgbe_phy { struct ethtool_link_ksettings lks; -- GitLab From fc75c032aee63e60170d80f44c8567ea45fc59da Mon Sep 17 00:00:00 2001 From: Raju Rangoju Date: Thu, 20 Oct 2022 12:12:13 +0530 Subject: [PATCH 0298/1927] amd-xgbe: enable PLL_CTL for fixed PHY modes only PLL control setting(RRC) is needed only in fixed PHY configuration to fix the peer-peer issues. Without the PLL control setting, the link up takes longer time in a fixed phy configuration. Driver implements SW RRC for Autoneg On configuration, hence PLL control setting (RRC) is not needed for AN On configuration, and can be skipped. Also, PLL re-initialization is not needed for PHY Power Off and RRC commands. Otherwise, they lead to mailbox errors. Added the changes accordingly. Fixes: daf182d360e5 ("net: amd-xgbe: Toggle PLL settings during rate change") Signed-off-by: Raju Rangoju Acked-by: Tom Lendacky Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index 8cf5d81fca363..349ba0dc1fa2c 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -1979,6 +1979,10 @@ static void xgbe_phy_rx_reset(struct xgbe_prv_data *pdata) static void xgbe_phy_pll_ctrl(struct xgbe_prv_data *pdata, bool enable) { + /* PLL_CTRL feature needs to be enabled for fixed PHY modes (Non-Autoneg) only */ + if (pdata->phy.autoneg != AUTONEG_DISABLE) + return; + XMDIO_WRITE_BITS(pdata, MDIO_MMD_PMAPMD, MDIO_VEND2_PMA_MISC_CTRL0, XGBE_PMA_PLL_CTRL_MASK, enable ? XGBE_PMA_PLL_CTRL_ENABLE @@ -2029,8 +2033,10 @@ static void xgbe_phy_perform_ratechange(struct xgbe_prv_data *pdata, xgbe_phy_rx_reset(pdata); reenable_pll: - /* Enable PLL re-initialization */ - xgbe_phy_pll_ctrl(pdata, true); + /* Enable PLL re-initialization, not needed for PHY Power Off and RRC cmds */ + if (cmd != XGBE_MB_CMD_POWER_OFF && + cmd != XGBE_MB_CMD_RRC) + xgbe_phy_pll_ctrl(pdata, true); } static void xgbe_phy_rrc(struct xgbe_prv_data *pdata) -- GitLab From 09c5f6bf11ac98874339e55f4f5f79a9dbc9b375 Mon Sep 17 00:00:00 2001 From: Raju Rangoju Date: Thu, 20 Oct 2022 12:12:14 +0530 Subject: [PATCH 0299/1927] amd-xgbe: fix the SFP compliance codes check for DAC cables The current XGBE code assumes that offset 6 of EEPROM SFP DAC (passive) cables is NULL. However, some cables (the 5 meter and 7 meter Molex passive cables) have non-zero data at offset 6. Fix the logic by moving the passive cable check above the active checks, so as not to be improperly identified as an active cable. This will fix the issue for any passive cable that advertises 1000Base-CX in offset 6. Fixes: abf0a1c2b26a ("amd-xgbe: Add support for SFP+ modules") Signed-off-by: Raju Rangoju Acked-by: Tom Lendacky Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index 349ba0dc1fa2c..8c41ac5676d65 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -1151,7 +1151,10 @@ static void xgbe_phy_sfp_parse_eeprom(struct xgbe_prv_data *pdata) } /* Determine the type of SFP */ - if (sfp_base[XGBE_SFP_BASE_10GBE_CC] & XGBE_SFP_BASE_10GBE_CC_SR) + if (phy_data->sfp_cable == XGBE_SFP_CABLE_PASSIVE && + xgbe_phy_sfp_bit_rate(sfp_eeprom, XGBE_SFP_SPEED_10000)) + phy_data->sfp_base = XGBE_SFP_BASE_10000_CR; + else if (sfp_base[XGBE_SFP_BASE_10GBE_CC] & XGBE_SFP_BASE_10GBE_CC_SR) phy_data->sfp_base = XGBE_SFP_BASE_10000_SR; else if (sfp_base[XGBE_SFP_BASE_10GBE_CC] & XGBE_SFP_BASE_10GBE_CC_LR) phy_data->sfp_base = XGBE_SFP_BASE_10000_LR; @@ -1167,9 +1170,6 @@ static void xgbe_phy_sfp_parse_eeprom(struct xgbe_prv_data *pdata) phy_data->sfp_base = XGBE_SFP_BASE_1000_CX; else if (sfp_base[XGBE_SFP_BASE_1GBE_CC] & XGBE_SFP_BASE_1GBE_CC_T) phy_data->sfp_base = XGBE_SFP_BASE_1000_T; - else if ((phy_data->sfp_cable == XGBE_SFP_CABLE_PASSIVE) && - xgbe_phy_sfp_bit_rate(sfp_eeprom, XGBE_SFP_SPEED_10000)) - phy_data->sfp_base = XGBE_SFP_BASE_10000_CR; switch (phy_data->sfp_base) { case XGBE_SFP_BASE_1000_T: -- GitLab From 170a9e341a3b02c0b2ea0df16ef14a33a4f41de8 Mon Sep 17 00:00:00 2001 From: Raju Rangoju Date: Thu, 20 Oct 2022 12:12:15 +0530 Subject: [PATCH 0300/1927] amd-xgbe: add the bit rate quirk for Molex cables The offset 12 (bit-rate) of EEPROM SFP DAC (passive) cables is expected to be in the range 0x64 to 0x68. However, the 5 meter and 7 meter Molex passive cables have the rate ceiling 0x78 at offset 12. Add a quirk for Molex passive cables to extend the rate ceiling to 0x78. Fixes: abf0a1c2b26a ("amd-xgbe: Add support for SFP+ modules") Signed-off-by: Raju Rangoju Acked-by: Tom Lendacky Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index 8c41ac5676d65..4064c3e3dd492 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -239,6 +239,7 @@ enum xgbe_sfp_speed { #define XGBE_SFP_BASE_BR_1GBE_MAX 0x0d #define XGBE_SFP_BASE_BR_10GBE_MIN 0x64 #define XGBE_SFP_BASE_BR_10GBE_MAX 0x68 +#define XGBE_MOLEX_SFP_BASE_BR_10GBE_MAX 0x78 #define XGBE_SFP_BASE_CU_CABLE_LEN 18 @@ -284,6 +285,8 @@ struct xgbe_sfp_eeprom { #define XGBE_BEL_FUSE_VENDOR "BEL-FUSE " #define XGBE_BEL_FUSE_PARTNO "1GBT-SFP06 " +#define XGBE_MOLEX_VENDOR "Molex Inc. " + struct xgbe_sfp_ascii { union { char vendor[XGBE_SFP_BASE_VENDOR_NAME_LEN + 1]; @@ -834,7 +837,11 @@ static bool xgbe_phy_sfp_bit_rate(struct xgbe_sfp_eeprom *sfp_eeprom, break; case XGBE_SFP_SPEED_10000: min = XGBE_SFP_BASE_BR_10GBE_MIN; - max = XGBE_SFP_BASE_BR_10GBE_MAX; + if (memcmp(&sfp_eeprom->base[XGBE_SFP_BASE_VENDOR_NAME], + XGBE_MOLEX_VENDOR, XGBE_SFP_BASE_VENDOR_NAME_LEN) == 0) + max = XGBE_MOLEX_SFP_BASE_BR_10GBE_MAX; + else + max = XGBE_SFP_BASE_BR_10GBE_MAX; break; default: return false; -- GitLab From 0bda03623e6b8b9052da5ba0145608941bcc2eb0 Mon Sep 17 00:00:00 2001 From: Yinjun Zhang Date: Thu, 20 Oct 2022 09:14:11 +0100 Subject: [PATCH 0301/1927] nfp: only clean `sp_indiff` when application firmware is unloaded Currently `sp_indiff` is cleaned when driver is removed. This will cause problem in multi-PF/multi-host case, considering one PF is removed while another is still in use. Since `sp_indiff` is the application firmware property, it should only be cleaned when the firmware is unloaded. Now let management firmware to clean it when necessary, driver only set it. Fixes: b1e4f11e426d ("nfp: refine the ABI of getting `sp_indiff` info") Signed-off-by: Yinjun Zhang Signed-off-by: Simon Horman Link: https://lore.kernel.org/r/20221020081411.80186-1-simon.horman@corigine.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/netronome/nfp/nfp_main.c | 38 ++++++++----------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c index e66e548919d48..71301dbd8fb5e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c @@ -716,16 +716,26 @@ static u64 nfp_net_pf_get_app_cap(struct nfp_pf *pf) return val; } -static int nfp_pf_cfg_hwinfo(struct nfp_pf *pf, bool sp_indiff) +static void nfp_pf_cfg_hwinfo(struct nfp_pf *pf) { struct nfp_nsp *nsp; char hwinfo[32]; + bool sp_indiff; int err; nsp = nfp_nsp_open(pf->cpp); if (IS_ERR(nsp)) - return PTR_ERR(nsp); + return; + + if (!nfp_nsp_has_hwinfo_set(nsp)) + goto end; + sp_indiff = (nfp_net_pf_get_app_id(pf) == NFP_APP_FLOWER_NIC) || + (nfp_net_pf_get_app_cap(pf) & NFP_NET_APP_CAP_SP_INDIFF); + + /* No need to clean `sp_indiff` in driver, management firmware + * will do it when application firmware is unloaded. + */ snprintf(hwinfo, sizeof(hwinfo), "sp_indiff=%d", sp_indiff); err = nfp_nsp_hwinfo_set(nsp, hwinfo, sizeof(hwinfo)); /* Not a fatal error, no need to return error to stop driver from loading */ @@ -739,21 +749,8 @@ static int nfp_pf_cfg_hwinfo(struct nfp_pf *pf, bool sp_indiff) pf->eth_tbl = __nfp_eth_read_ports(pf->cpp, nsp); } +end: nfp_nsp_close(nsp); - return 0; -} - -static int nfp_pf_nsp_cfg(struct nfp_pf *pf) -{ - bool sp_indiff = (nfp_net_pf_get_app_id(pf) == NFP_APP_FLOWER_NIC) || - (nfp_net_pf_get_app_cap(pf) & NFP_NET_APP_CAP_SP_INDIFF); - - return nfp_pf_cfg_hwinfo(pf, sp_indiff); -} - -static void nfp_pf_nsp_clean(struct nfp_pf *pf) -{ - nfp_pf_cfg_hwinfo(pf, false); } static int nfp_pci_probe(struct pci_dev *pdev, @@ -856,13 +853,11 @@ static int nfp_pci_probe(struct pci_dev *pdev, goto err_fw_unload; } - err = nfp_pf_nsp_cfg(pf); - if (err) - goto err_fw_unload; + nfp_pf_cfg_hwinfo(pf); err = nfp_net_pci_probe(pf); if (err) - goto err_nsp_clean; + goto err_fw_unload; err = nfp_hwmon_register(pf); if (err) { @@ -874,8 +869,6 @@ static int nfp_pci_probe(struct pci_dev *pdev, err_net_remove: nfp_net_pci_remove(pf); -err_nsp_clean: - nfp_pf_nsp_clean(pf); err_fw_unload: kfree(pf->rtbl); nfp_mip_close(pf->mip); @@ -915,7 +908,6 @@ static void __nfp_pci_shutdown(struct pci_dev *pdev, bool unload_fw) nfp_net_pci_remove(pf); - nfp_pf_nsp_clean(pf); vfree(pf->dumpspec); kfree(pf->rtbl); nfp_mip_close(pf->mip); -- GitLab From f86bfeb689f2c4ebe12782ef0578ef778fb1a050 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Sat, 22 Oct 2022 09:21:07 +0200 Subject: [PATCH 0302/1927] ALSA: hda/realtek: Add another HP ZBook G9 model quirks HP ZBook Firefly 16 G9 (103c:896d) and HP ZBook Power 15.6 G9 (103c:89c0) require the same quirk for enabling CS35L41 speaker amps. Signed-off-by: Takashi Iwai Cc: Link: https://lore.kernel.org/r/20221022072107.3401-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_realtek.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 9945861f02efa..701a72ec5629a 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -9323,6 +9323,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8898, "HP EliteBook 845 G8 Notebook PC", ALC285_FIXUP_HP_LIMIT_INT_MIC_BOOST), SND_PCI_QUIRK(0x103c, 0x88d0, "HP Pavilion 15-eh1xxx (mainboard 88D0)", ALC287_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), + SND_PCI_QUIRK(0x103c, 0x896d, "HP ZBook Firefly 16 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x896e, "HP EliteBook x360 830 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8971, "HP EliteBook 830 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8972, "HP EliteBook 840 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), @@ -9341,6 +9342,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x89aa, "HP EliteBook 630 G9", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89ac, "HP EliteBook 640 G9", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89ae, "HP EliteBook 650 G9", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x89c0, "HP ZBook Power 15.6 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89c3, "Zbook Studio G9", ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89c6, "Zbook Fury 17 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x89ca, "HP", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), -- GitLab From 0eafdcfea6bc82f7f8c9a9af2d4add6745beefc5 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Fri, 30 Sep 2022 13:40:41 +0200 Subject: [PATCH 0303/1927] MAINTAINERS: move USB gadget and phy entries under the main USB entry Felipe has done a wonderful job over the years, but now it makes sense to just maintain all of the USB stack in one tree. Do so by removing the current USB gadget and phy entries so that all portions of the stack are now covered by the main USB maintainer entry. Acked-by: Felipe Balbi Link: https://lore.kernel.org/r/20220930114041.1306711-1-gregkh@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..503e3f41e6954 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -21181,15 +21181,6 @@ S: Maintained F: Documentation/usb/ehci.rst F: drivers/usb/host/ehci* -USB GADGET/PERIPHERAL SUBSYSTEM -M: Felipe Balbi -L: linux-usb@vger.kernel.org -S: Maintained -W: http://www.linux-usb.org/gadget -T: git git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb.git -F: drivers/usb/gadget/ -F: include/linux/usb/gadget* - USB HID/HIDBP DRIVERS (USB KEYBOARDS, MICE, REMOTE CONTROLS, ...) M: Jiri Kosina M: Benjamin Tissoires @@ -21296,13 +21287,6 @@ W: https://github.com/petkan/pegasus T: git git://github.com/petkan/pegasus.git F: drivers/net/usb/pegasus.* -USB PHY LAYER -M: Felipe Balbi -L: linux-usb@vger.kernel.org -S: Maintained -T: git git://git.kernel.org/pub/scm/linux/kernel/git/balbi/usb.git -F: drivers/usb/phy/ - USB PRINTER DRIVER (usblp) M: Pete Zaitcev L: linux-usb@vger.kernel.org -- GitLab From 593c5ba288e118ad80b41e8339f0d0dcad65eb04 Mon Sep 17 00:00:00 2001 From: Justin Chen Date: Fri, 7 Oct 2022 11:32:09 -0700 Subject: [PATCH 0304/1927] MAINTAINERS: Update maintainers for broadcom USB Al Cooper is no longer the internal broadcom maintainer for broadcom USB. I will be taking his place as the internal maintainer and as an additional upstream maintainer. Signed-off-by: Justin Chen Acked-by: Florian Fainelli Link: https://lore.kernel.org/r/1665167529-9840-1-git-send-email-justinpopo6@gmail.com Signed-off-by: Greg Kroah-Hartman --- MAINTAINERS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 503e3f41e6954..b5cb696999fe3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4101,6 +4101,7 @@ N: bcm7038 N: bcm7120 BROADCOM BDC DRIVER +M: Justin Chen M: Al Cooper L: linux-usb@vger.kernel.org R: Broadcom internal kernel review list @@ -4207,6 +4208,7 @@ F: Documentation/devicetree/bindings/serial/brcm,bcm7271-uart.yaml F: drivers/tty/serial/8250/8250_bcm7271.c BROADCOM BRCMSTB USB EHCI DRIVER +M: Justin Chen M: Al Cooper R: Broadcom internal kernel review list L: linux-usb@vger.kernel.org @@ -4223,6 +4225,7 @@ F: Documentation/devicetree/bindings/usb/brcm,usb-pinmap.yaml F: drivers/usb/misc/brcmstb-usb-pinmap.c BROADCOM BRCMSTB USB2 and USB3 PHY DRIVER +M: Justin Chen M: Al Cooper R: Broadcom internal kernel review list L: linux-kernel@vger.kernel.org -- GitLab From d182c2e1bc92084c038b44c618f29589a4de9f66 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Tue, 18 Oct 2022 02:35:10 +0300 Subject: [PATCH 0305/1927] usb: dwc3: Don't switch OTG -> peripheral if extcon is present If the extcon device exists, get the mode from the extcon device. If the controller is DRD and the driver is unable to determine the mode, only then default the dr_mode to USB_DR_MODE_PERIPHERAL. Reported-by: Steev Klimaszewski Fixes: 7a84e7353e23 ("Revert "usb: dwc3: Don't switch OTG -> peripheral if extcon is present"") Cc: stable Reviewed-by: Thinh Nguyen Signed-off-by: Andrey Smirnov Signed-off-by: Andy Shevchenko Tested-by: Steev Klimaszewski Acked-by: Thinh Nguyen Link: https://lore.kernel.org/r/20221017233510.53336-1-andriy.shevchenko@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/core.c | 49 +++++++++++++++++++++++++++++++++++++++- drivers/usb/dwc3/drd.c | 50 ----------------------------------------- 2 files changed, 48 insertions(+), 51 deletions(-) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index ea51624461b5b..c0e7c76dc5c8b 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -85,7 +86,7 @@ static int dwc3_get_dr_mode(struct dwc3 *dwc) * mode. If the controller supports DRD but the dr_mode is not * specified or set to OTG, then set the mode to peripheral. */ - if (mode == USB_DR_MODE_OTG && + if (mode == USB_DR_MODE_OTG && !dwc->edev && (!IS_ENABLED(CONFIG_USB_ROLE_SWITCH) || !device_property_read_bool(dwc->dev, "usb-role-switch")) && !DWC3_VER_IS_PRIOR(DWC3, 330A)) @@ -1690,6 +1691,46 @@ static void dwc3_check_params(struct dwc3 *dwc) } } +static struct extcon_dev *dwc3_get_extcon(struct dwc3 *dwc) +{ + struct device *dev = dwc->dev; + struct device_node *np_phy; + struct extcon_dev *edev = NULL; + const char *name; + + if (device_property_read_bool(dev, "extcon")) + return extcon_get_edev_by_phandle(dev, 0); + + /* + * Device tree platforms should get extcon via phandle. + * On ACPI platforms, we get the name from a device property. + * This device property is for kernel internal use only and + * is expected to be set by the glue code. + */ + if (device_property_read_string(dev, "linux,extcon-name", &name) == 0) + return extcon_get_extcon_dev(name); + + /* + * Try to get an extcon device from the USB PHY controller's "port" + * node. Check if it has the "port" node first, to avoid printing the + * error message from underlying code, as it's a valid case: extcon + * device (and "port" node) may be missing in case of "usb-role-switch" + * or OTG mode. + */ + np_phy = of_parse_phandle(dev->of_node, "phys", 0); + if (of_graph_is_present(np_phy)) { + struct device_node *np_conn; + + np_conn = of_graph_get_remote_node(np_phy, -1, -1); + if (np_conn) + edev = extcon_find_edev_by_node(np_conn); + of_node_put(np_conn); + } + of_node_put(np_phy); + + return edev; +} + static int dwc3_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1840,6 +1881,12 @@ static int dwc3_probe(struct platform_device *pdev) goto err2; } + dwc->edev = dwc3_get_extcon(dwc); + if (IS_ERR(dwc->edev)) { + ret = dev_err_probe(dwc->dev, PTR_ERR(dwc->edev), "failed to get extcon\n"); + goto err3; + } + ret = dwc3_get_dr_mode(dwc); if (ret) goto err3; diff --git a/drivers/usb/dwc3/drd.c b/drivers/usb/dwc3/drd.c index 8cad9e7d33687..039bf241769af 100644 --- a/drivers/usb/dwc3/drd.c +++ b/drivers/usb/dwc3/drd.c @@ -8,7 +8,6 @@ */ #include -#include #include #include #include @@ -439,51 +438,6 @@ static int dwc3_drd_notifier(struct notifier_block *nb, return NOTIFY_DONE; } -static struct extcon_dev *dwc3_get_extcon(struct dwc3 *dwc) -{ - struct device *dev = dwc->dev; - struct device_node *np_phy; - struct extcon_dev *edev = NULL; - const char *name; - - if (device_property_read_bool(dev, "extcon")) - return extcon_get_edev_by_phandle(dev, 0); - - /* - * Device tree platforms should get extcon via phandle. - * On ACPI platforms, we get the name from a device property. - * This device property is for kernel internal use only and - * is expected to be set by the glue code. - */ - if (device_property_read_string(dev, "linux,extcon-name", &name) == 0) { - edev = extcon_get_extcon_dev(name); - if (!edev) - return ERR_PTR(-EPROBE_DEFER); - - return edev; - } - - /* - * Try to get an extcon device from the USB PHY controller's "port" - * node. Check if it has the "port" node first, to avoid printing the - * error message from underlying code, as it's a valid case: extcon - * device (and "port" node) may be missing in case of "usb-role-switch" - * or OTG mode. - */ - np_phy = of_parse_phandle(dev->of_node, "phys", 0); - if (of_graph_is_present(np_phy)) { - struct device_node *np_conn; - - np_conn = of_graph_get_remote_node(np_phy, -1, -1); - if (np_conn) - edev = extcon_find_edev_by_node(np_conn); - of_node_put(np_conn); - } - of_node_put(np_phy); - - return edev; -} - #if IS_ENABLED(CONFIG_USB_ROLE_SWITCH) #define ROLE_SWITCH 1 static int dwc3_usb_role_switch_set(struct usb_role_switch *sw, @@ -588,10 +542,6 @@ int dwc3_drd_init(struct dwc3 *dwc) device_property_read_bool(dwc->dev, "usb-role-switch")) return dwc3_setup_role_switch(dwc); - dwc->edev = dwc3_get_extcon(dwc); - if (IS_ERR(dwc->edev)) - return PTR_ERR(dwc->edev); - if (dwc->edev) { dwc->edev_nb.notifier_call = dwc3_drd_notifier; ret = extcon_register_notifier(dwc->edev, EXTCON_USB_HOST, -- GitLab From 4db0fbb601361767144e712beb96704b966339f5 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Tue, 18 Oct 2022 19:39:01 -0700 Subject: [PATCH 0306/1927] usb: dwc3: gadget: Don't delay End Transfer on delayed_status The gadget driver may wait on the request completion when it sets the USB_GADGET_DELAYED_STATUS. Make sure that the End Transfer command can go through if the dwc->delayed_status is set so that the request can complete. When the delayed_status is set, the Setup packet is already processed, and the next phase should be either Data or Status. It's unlikely that the host would cancel the control transfer and send a new Setup packet during End Transfer command. But if that's the case, we can try again when ep0state returns to EP0_SETUP_PHASE. Fixes: e1ee843488d5 ("usb: dwc3: gadget: Force sending delayed status during soft disconnect") Cc: stable@vger.kernel.org Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/3f9f59e5d74efcbaee444cf4b30ef639cc7b124e.1666146954.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/gadget.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 079cd333632e1..dd8ecbe61bec1 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1698,6 +1698,16 @@ static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool int cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); memset(¶ms, 0, sizeof(params)); ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); + /* + * If the End Transfer command was timed out while the device is + * not in SETUP phase, it's possible that an incoming Setup packet + * may prevent the command's completion. Let's retry when the + * ep0state returns to EP0_SETUP_PHASE. + */ + if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { + dep->flags |= DWC3_EP_DELAY_STOP; + return 0; + } WARN_ON_ONCE(ret); dep->resource_index = 0; @@ -3719,7 +3729,7 @@ void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, * timeout. Delay issuing the End Transfer command until the Setup TRB is * prepared. */ - if (dwc->ep0state != EP0_SETUP_PHASE) { + if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { dep->flags |= DWC3_EP_DELAY_STOP; return; } -- GitLab From 8e8e923a49967b798e7d69f1ce9eff1dd2533547 Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Tue, 18 Oct 2022 16:50:37 -0500 Subject: [PATCH 0307/1927] usb: gadget: uvc: fix dropped frame after missed isoc With the re-use of the previous completion status in 0d1c407b1a749 ("usb: dwc3: gadget: Return proper request status") it could be possible that the next frame would also get dropped if the current frame has a missed isoc error. Ensure that an interrupt is requested for the start of a new frame. Fixes: fc78941d8169 ("usb: gadget: uvc: decrease the interrupt load to a quarter") Cc: Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20221018215044.765044-2-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/function/uvc_video.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index bb037fcc90e69..323977716f5a4 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -431,7 +431,8 @@ static void uvcg_video_pump(struct work_struct *work) /* Endpoint now owns the request */ req = NULL; - video->req_int_count++; + if (buf->state != UVC_BUF_STATE_DONE) + video->req_int_count++; } if (!req) -- GitLab From 0a0a2760b04814428800d48281a447a7522470ad Mon Sep 17 00:00:00 2001 From: Dan Vacura Date: Tue, 18 Oct 2022 16:50:39 -0500 Subject: [PATCH 0308/1927] usb: gadget: uvc: fix sg handling in error case If there is a transmission error the buffer will be returned too early, causing a memory fault as subsequent requests for that buffer are still queued up to be sent. Refactor the error handling to wait for the final request to come in before reporting back the buffer to userspace for all transfer types (bulk/isoc/isoc_sg). This ensures userspace knows if the frame was successfully sent. Fixes: e81e7f9a0eb9 ("usb: gadget: uvc: add scatter gather support") Cc: Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20221018215044.765044-4-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/function/uvc_queue.c | 8 +++++--- drivers/usb/gadget/function/uvc_video.c | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index ec500ee499eed..0aa3d7e1f3cc3 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -304,6 +304,7 @@ int uvcg_queue_enable(struct uvc_video_queue *queue, int enable) queue->sequence = 0; queue->buf_used = 0; + queue->flags &= ~UVC_QUEUE_DROP_INCOMPLETE; } else { ret = vb2_streamoff(&queue->queue, queue->queue.type); if (ret < 0) @@ -329,10 +330,11 @@ int uvcg_queue_enable(struct uvc_video_queue *queue, int enable) void uvcg_complete_buffer(struct uvc_video_queue *queue, struct uvc_buffer *buf) { - if ((queue->flags & UVC_QUEUE_DROP_INCOMPLETE) && - buf->length != buf->bytesused) { - buf->state = UVC_BUF_STATE_QUEUED; + if (queue->flags & UVC_QUEUE_DROP_INCOMPLETE) { + queue->flags &= ~UVC_QUEUE_DROP_INCOMPLETE; + buf->state = UVC_BUF_STATE_ERROR; vb2_set_plane_payload(&buf->buf.vb2_buf, 0, 0); + vb2_buffer_done(&buf->buf.vb2_buf, VB2_BUF_STATE_ERROR); return; } diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 323977716f5a4..5993e083819cb 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -88,6 +88,7 @@ uvc_video_encode_bulk(struct usb_request *req, struct uvc_video *video, struct uvc_buffer *buf) { void *mem = req->buf; + struct uvc_request *ureq = req->context; int len = video->req_size; int ret; @@ -113,13 +114,14 @@ uvc_video_encode_bulk(struct usb_request *req, struct uvc_video *video, video->queue.buf_used = 0; buf->state = UVC_BUF_STATE_DONE; list_del(&buf->queue); - uvcg_complete_buffer(&video->queue, buf); video->fid ^= UVC_STREAM_FID; + ureq->last_buf = buf; video->payload_size = 0; } if (video->payload_size == video->max_payload_size || + video->queue.flags & UVC_QUEUE_DROP_INCOMPLETE || buf->bytesused == video->queue.buf_used) video->payload_size = 0; } @@ -180,7 +182,8 @@ uvc_video_encode_isoc_sg(struct usb_request *req, struct uvc_video *video, req->length -= len; video->queue.buf_used += req->length - header_len; - if (buf->bytesused == video->queue.buf_used || !buf->sg) { + if (buf->bytesused == video->queue.buf_used || !buf->sg || + video->queue.flags & UVC_QUEUE_DROP_INCOMPLETE) { video->queue.buf_used = 0; buf->state = UVC_BUF_STATE_DONE; buf->offset = 0; @@ -195,6 +198,7 @@ uvc_video_encode_isoc(struct usb_request *req, struct uvc_video *video, struct uvc_buffer *buf) { void *mem = req->buf; + struct uvc_request *ureq = req->context; int len = video->req_size; int ret; @@ -209,12 +213,13 @@ uvc_video_encode_isoc(struct usb_request *req, struct uvc_video *video, req->length = video->req_size - len; - if (buf->bytesused == video->queue.buf_used) { + if (buf->bytesused == video->queue.buf_used || + video->queue.flags & UVC_QUEUE_DROP_INCOMPLETE) { video->queue.buf_used = 0; buf->state = UVC_BUF_STATE_DONE; list_del(&buf->queue); - uvcg_complete_buffer(&video->queue, buf); video->fid ^= UVC_STREAM_FID; + ureq->last_buf = buf; } } @@ -255,6 +260,11 @@ uvc_video_complete(struct usb_ep *ep, struct usb_request *req) case 0: break; + case -EXDEV: + uvcg_dbg(&video->uvc->func, "VS request missed xfer.\n"); + queue->flags |= UVC_QUEUE_DROP_INCOMPLETE; + break; + case -ESHUTDOWN: /* disconnect from host. */ uvcg_dbg(&video->uvc->func, "VS request cancelled.\n"); uvcg_queue_cancel(queue, 1); -- GitLab From b57b08e6f431348363adffa5b6643fe3ec9dc7fe Mon Sep 17 00:00:00 2001 From: Jeff Vanhoof Date: Tue, 18 Oct 2022 16:50:40 -0500 Subject: [PATCH 0309/1927] usb: gadget: uvc: fix sg handling during video encode In uvc_video_encode_isoc_sg, the uvc_request's sg list is incorrectly being populated leading to corrupt video being received by the remote end. When building the sg list the usage of buf->sg's 'dma_length' field is not correct and instead its 'length' field should be used. Fixes: e81e7f9a0eb9 ("usb: gadget: uvc: add scatter gather support") Cc: Signed-off-by: Jeff Vanhoof Signed-off-by: Dan Vacura Link: https://lore.kernel.org/r/20221018215044.765044-5-w36195@motorola.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/function/uvc_video.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index 5993e083819cb..dd1c6b2ca7c6f 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -157,10 +157,10 @@ uvc_video_encode_isoc_sg(struct usb_request *req, struct uvc_video *video, sg = sg_next(sg); for_each_sg(sg, iter, ureq->sgt.nents - 1, i) { - if (!len || !buf->sg || !sg_dma_len(buf->sg)) + if (!len || !buf->sg || !buf->sg->length) break; - sg_left = sg_dma_len(buf->sg) - buf->offset; + sg_left = buf->sg->length - buf->offset; part = min_t(unsigned int, len, sg_left); sg_set_page(iter, sg_page(buf->sg), part, buf->offset); -- GitLab From 48ed32482c4100069d0c0eebdc6b198c6ae5f71f Mon Sep 17 00:00:00 2001 From: Joel Stanley Date: Mon, 17 Oct 2022 16:00:06 +1030 Subject: [PATCH 0310/1927] usb: gadget: aspeed: Fix probe regression Since commit fc274c1e9973 ("USB: gadget: Add a new bus for gadgets"), the gadget devices are proper driver core devices, which caused each device to request pinmux settings: aspeed_vhub 1e6a0000.usb-vhub: Initialized virtual hub in USB2 mode aspeed-g5-pinctrl 1e6e2080.pinctrl: pin A7 already requested by 1e6a0000.usb-vhub; cannot claim for gadget.0 aspeed-g5-pinctrl 1e6e2080.pinctrl: pin-232 (gadget.0) status -22 aspeed-g5-pinctrl 1e6e2080.pinctrl: could not request pin 232 (A7) from group USB2AD on device aspeed-g5-pinctrl g_mass_storage gadget.0: Error applying setting, reverse things back The vhub driver has already claimed the pins, so prevent the gadgets from requesting them too by setting the magic of_node_reused flag. This causes the driver core to skip the mux request. Reported-by: Zev Weiss Reported-by: Jae Hyun Yoo Fixes: fc274c1e9973 ("USB: gadget: Add a new bus for gadgets") Cc: stable@vger.kernel.org Signed-off-by: Joel Stanley Tested-by: Zev Weiss Tested-by: Jae Hyun Yoo Link: https://lore.kernel.org/r/20221017053006.358520-1-joel@jms.id.au Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/udc/aspeed-vhub/dev.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/gadget/udc/aspeed-vhub/dev.c b/drivers/usb/gadget/udc/aspeed-vhub/dev.c index b0dfca43fbdce..4f3bc27c1c628 100644 --- a/drivers/usb/gadget/udc/aspeed-vhub/dev.c +++ b/drivers/usb/gadget/udc/aspeed-vhub/dev.c @@ -591,6 +591,7 @@ int ast_vhub_init_dev(struct ast_vhub *vhub, unsigned int idx) d->gadget.max_speed = USB_SPEED_HIGH; d->gadget.speed = USB_SPEED_UNKNOWN; d->gadget.dev.of_node = vhub->pdev->dev.of_node; + d->gadget.dev.of_node_reused = true; rc = usb_add_gadget_udc(d->port_dev, &d->gadget); if (rc != 0) -- GitLab From 99f6d43611135bd6f211dec9e88bb41e4167e304 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Fri, 7 Oct 2022 13:09:50 +0300 Subject: [PATCH 0311/1927] usb: typec: ucsi: Check the connection on resume Checking the connection status of every port on resume. This fixes an issue where the partner device is not unregistered properly after resume if it was unplugged while the system was suspended. The function ucsi_check_connection() is also modified so that it can be used also for registering the connection on top of unregistering it. Link: https://bugzilla.kernel.org/show_bug.cgi?id=210425 Fixes: a94ecde41f7e ("usb: typec: ucsi: ccg: enable runtime pm support") Cc: Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20221007100951.43798-2-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/ucsi/ucsi.c | 42 ++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index 74fb5a4c6f21b..a7987fc764cc6 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -183,16 +183,6 @@ int ucsi_send_command(struct ucsi *ucsi, u64 command, } EXPORT_SYMBOL_GPL(ucsi_send_command); -int ucsi_resume(struct ucsi *ucsi) -{ - u64 command; - - /* Restore UCSI notification enable mask after system resume */ - command = UCSI_SET_NOTIFICATION_ENABLE | ucsi->ntfy; - - return ucsi_send_command(ucsi, command, NULL, 0); -} -EXPORT_SYMBOL_GPL(ucsi_resume); /* -------------------------------------------------------------------------- */ struct ucsi_work { @@ -744,6 +734,7 @@ static void ucsi_partner_change(struct ucsi_connector *con) static int ucsi_check_connection(struct ucsi_connector *con) { + u8 prev_flags = con->status.flags; u64 command; int ret; @@ -754,10 +745,13 @@ static int ucsi_check_connection(struct ucsi_connector *con) return ret; } + if (con->status.flags == prev_flags) + return 0; + if (con->status.flags & UCSI_CONSTAT_CONNECTED) { - if (UCSI_CONSTAT_PWR_OPMODE(con->status.flags) == - UCSI_CONSTAT_PWR_OPMODE_PD) - ucsi_partner_task(con, ucsi_check_altmodes, 30, 0); + ucsi_register_partner(con); + ucsi_pwr_opmode_change(con); + ucsi_partner_change(con); } else { ucsi_partner_change(con); ucsi_port_psy_changed(con); @@ -1276,6 +1270,28 @@ static int ucsi_init(struct ucsi *ucsi) return ret; } +int ucsi_resume(struct ucsi *ucsi) +{ + struct ucsi_connector *con; + u64 command; + int ret; + + /* Restore UCSI notification enable mask after system resume */ + command = UCSI_SET_NOTIFICATION_ENABLE | ucsi->ntfy; + ret = ucsi_send_command(ucsi, command, NULL, 0); + if (ret < 0) + return ret; + + for (con = ucsi->connector; con->port; con++) { + mutex_lock(&con->lock); + ucsi_check_connection(con); + mutex_unlock(&con->lock); + } + + return 0; +} +EXPORT_SYMBOL_GPL(ucsi_resume); + static void ucsi_init_work(struct work_struct *work) { struct ucsi *ucsi = container_of(work, struct ucsi, work.work); -- GitLab From 4e3a50293c2b21961f02e1afa2f17d3a1a90c7c8 Mon Sep 17 00:00:00 2001 From: Heikki Krogerus Date: Fri, 7 Oct 2022 13:09:51 +0300 Subject: [PATCH 0312/1927] usb: typec: ucsi: acpi: Implement resume callback The ACPI driver needs to resume the interface by calling ucsi_resume(). Otherwise we may fail to detect connections and disconnections that happen while the system is suspended. Link: https://bugzilla.kernel.org/show_bug.cgi?id=210425 Fixes: a94ecde41f7e ("usb: typec: ucsi: ccg: enable runtime pm support") Cc: Signed-off-by: Heikki Krogerus Link: https://lore.kernel.org/r/20221007100951.43798-3-heikki.krogerus@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/ucsi/ucsi_acpi.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/usb/typec/ucsi/ucsi_acpi.c b/drivers/usb/typec/ucsi/ucsi_acpi.c index 8873c1644a295..ce0c8ef80c043 100644 --- a/drivers/usb/typec/ucsi/ucsi_acpi.c +++ b/drivers/usb/typec/ucsi/ucsi_acpi.c @@ -185,6 +185,15 @@ static int ucsi_acpi_remove(struct platform_device *pdev) return 0; } +static int ucsi_acpi_resume(struct device *dev) +{ + struct ucsi_acpi *ua = dev_get_drvdata(dev); + + return ucsi_resume(ua->ucsi); +} + +static DEFINE_SIMPLE_DEV_PM_OPS(ucsi_acpi_pm_ops, NULL, ucsi_acpi_resume); + static const struct acpi_device_id ucsi_acpi_match[] = { { "PNP0CA0", 0 }, { }, @@ -194,6 +203,7 @@ MODULE_DEVICE_TABLE(acpi, ucsi_acpi_match); static struct platform_driver ucsi_acpi_platform_driver = { .driver = { .name = "ucsi_acpi", + .pm = pm_ptr(&ucsi_acpi_pm_ops), .acpi_match_table = ACPI_PTR(ucsi_acpi_match), }, .probe = ucsi_acpi_probe, -- GitLab From fb8f60dd1b67520e0e0d7978ef17d015690acfc1 Mon Sep 17 00:00:00 2001 From: Justin Chen Date: Wed, 5 Oct 2022 12:13:55 -0700 Subject: [PATCH 0313/1927] usb: bdc: change state when port disconnected When port is connected and then disconnected, the state stays as configured. Which is incorrect as the port is no longer configured, but in a not attached state. Signed-off-by: Justin Chen Acked-by: Florian Fainelli Fixes: efed421a94e6 ("usb: gadget: Add UDC driver for Broadcom USB3.0 device controller IP BDC") Cc: stable Link: https://lore.kernel.org/r/1664997235-18198-1-git-send-email-justinpopo6@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/udc/bdc/bdc_udc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/usb/gadget/udc/bdc/bdc_udc.c b/drivers/usb/gadget/udc/bdc/bdc_udc.c index 5ac0ef88334eb..53ffaf4e2e376 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_udc.c +++ b/drivers/usb/gadget/udc/bdc/bdc_udc.c @@ -151,6 +151,7 @@ static void bdc_uspc_disconnected(struct bdc *bdc, bool reinit) bdc->delayed_status = false; bdc->reinit = reinit; bdc->test_mode = false; + usb_gadget_set_state(&bdc->gadget, USB_STATE_NOTATTACHED); } /* TNotify wkaeup timer */ -- GitLab From 0469e56a14bf8cfb80507e51b7aeec0332cdbc13 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Fri, 30 Sep 2022 00:51:58 +0200 Subject: [PATCH 0314/1927] KVM: x86: Mask off reserved bits in CPUID.80000001H KVM_GET_SUPPORTED_CPUID should only enumerate features that KVM actually supports. CPUID.80000001:EBX[27:16] are reserved bits and should be masked off. Fixes: 0771671749b5 ("KVM: Enhance guest cpuid management") Signed-off-by: Jim Mattson Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 7065462378e29..834feeb0a828d 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1133,6 +1133,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->eax = max(entry->eax, 0x80000021); break; case 0x80000001: + entry->ebx &= ~GENMASK(27, 16); cpuid_entry_override(entry, CPUID_8000_0001_EDX); cpuid_entry_override(entry, CPUID_8000_0001_ECX); break; -- GitLab From eeb69eab57c6604ac90b3fd8e5ac43f24a5535b1 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Thu, 29 Sep 2022 15:51:59 -0700 Subject: [PATCH 0315/1927] KVM: x86: Mask off reserved bits in CPUID.80000006H KVM_GET_SUPPORTED_CPUID should only enumerate features that KVM actually supports. CPUID.80000006H:EDX[17:16] are reserved bits and should be masked off. Fixes: 43d05de2bee7 ("KVM: pass through CPUID(0x80000006)") Signed-off-by: Jim Mattson Message-Id: <20220929225203.2234702-2-jmattson@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 834feeb0a828d..8325a01cb1f17 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1138,7 +1138,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) cpuid_entry_override(entry, CPUID_8000_0001_ECX); break; case 0x80000006: - /* L2 cache and TLB: pass through host info. */ + /* Drop reserved bits, pass host L2 cache and TLB info. */ + entry->edx &= ~GENMASK(17, 16); break; case 0x80000007: /* Advanced power management */ /* invariant TSC is CPUID.80000007H:EDX[8] */ -- GitLab From 7030d8530e533844e2f4b0e7476498afcd324634 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Thu, 29 Sep 2022 15:52:00 -0700 Subject: [PATCH 0316/1927] KVM: x86: Mask off reserved bits in CPUID.80000008H KVM_GET_SUPPORTED_CPUID should only enumerate features that KVM actually supports. The following ranges of CPUID.80000008H are reserved and should be masked off: ECX[31:18] ECX[11:8] In addition, the PerfTscSize field at ECX[17:16] should also be zero because KVM does not set the PERFTSC bit at CPUID.80000001H.ECX[27]. Fixes: 24c82e576b78 ("KVM: Sanitize cpuid") Signed-off-by: Jim Mattson Message-Id: <20220929225203.2234702-3-jmattson@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 8325a01cb1f17..489c028859e19 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1169,6 +1169,7 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) g_phys_as = phys_as; entry->eax = g_phys_as | (virt_as << 8); + entry->ecx &= ~(GENMASK(31, 16) | GENMASK(11, 8)); entry->edx = 0; cpuid_entry_override(entry, CPUID_8000_0008_EBX); break; -- GitLab From 079f6889818dd07903fb36c252532ab47ebb6d48 Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Thu, 29 Sep 2022 15:52:01 -0700 Subject: [PATCH 0317/1927] KVM: x86: Mask off reserved bits in CPUID.8000001AH KVM_GET_SUPPORTED_CPUID should only enumerate features that KVM actually supports. In the case of CPUID.8000001AH, only three bits are currently defined. The 125 reserved bits should be masked off. Fixes: 24c82e576b78 ("KVM: Sanitize cpuid") Signed-off-by: Jim Mattson Message-Id: <20220929225203.2234702-4-jmattson@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 489c028859e19..a0292ba650df3 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1189,6 +1189,9 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->ecx = entry->edx = 0; break; case 0x8000001a: + entry->eax &= GENMASK(2, 0); + entry->ebx = entry->ecx = entry->edx = 0; + break; case 0x8000001e: break; case 0x8000001F: -- GitLab From 02341a08c9dec5a88527981b0bdf0fb6f7499cbf Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Sat, 22 Oct 2022 10:16:15 +0800 Subject: [PATCH 0318/1927] block: fix memory leak for elevator on add_disk failure The default elevator is allocated in the beginning of device_add_disk(), however, it's not freed in the following error path. Fixes: 50e34d78815e ("block: disable the elevator int del_gendisk") Signed-off-by: Yu Kuai Reviewed-by: Christoph Hellwig Reviewed-by: Jason Yan Link: https://lore.kernel.org/r/20221022021615.2756171-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- block/genhd.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/block/genhd.c b/block/genhd.c index 17b33c62423df..fee90eb98b4a5 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -410,9 +410,10 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, * Otherwise just allocate the device numbers for both the whole device * and all partitions from the extended dev_t space. */ + ret = -EINVAL; if (disk->major) { if (WARN_ON(!disk->minors)) - return -EINVAL; + goto out_exit_elevator; if (disk->minors > DISK_MAX_PARTS) { pr_err("block: can't allocate more than %d partitions\n", @@ -420,14 +421,14 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, disk->minors = DISK_MAX_PARTS; } if (disk->first_minor + disk->minors > MINORMASK + 1) - return -EINVAL; + goto out_exit_elevator; } else { if (WARN_ON(disk->minors)) - return -EINVAL; + goto out_exit_elevator; ret = blk_alloc_ext_minor(); if (ret < 0) - return ret; + goto out_exit_elevator; disk->major = BLOCK_EXT_MAJOR; disk->first_minor = ret; } @@ -540,6 +541,9 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, out_free_ext_minor: if (disk->major == BLOCK_EXT_MAJOR) blk_free_ext_minor(disk->first_minor); +out_exit_elevator: + if (disk->queue->elevator) + elevator_exit(disk->queue); return ret; } EXPORT_SYMBOL(device_add_disk); -- GitLab From f23f1a1e8437e38014fe34a2f12e37e861e5bcc7 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 21 Sep 2022 03:10:08 +0200 Subject: [PATCH 0319/1927] arm64: dts: imx8mm: Enable CPLD_Dn pull down resistor on MX8Menlo Enable CPLD_Dn pull down resistor instead of pull up to avoid intefering with CPLD power off functionality. Fixes: 510c527b4ff57 ("arm64: dts: imx8mm: Add i.MX8M Mini Toradex Verdin based Menlo board") Signed-off-by: Marek Vasut Reviewed-by: Fabio Estevam Signed-off-by: Shawn Guo --- .../arm64/boot/dts/freescale/imx8mm-mx8menlo.dts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts b/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts index 32f6f2f50c10c..43e89859c0445 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-mx8menlo.dts @@ -250,21 +250,21 @@ MX8MM_IOMUXC_SAI1_RXD1_GPIO4_IO3 0x1c4 /* SODIMM 96 */ MX8MM_IOMUXC_SAI1_RXD2_GPIO4_IO4 0x1c4 /* CPLD_D[7] */ - MX8MM_IOMUXC_SAI1_RXD3_GPIO4_IO5 0x1c4 + MX8MM_IOMUXC_SAI1_RXD3_GPIO4_IO5 0x184 /* CPLD_D[6] */ - MX8MM_IOMUXC_SAI1_RXFS_GPIO4_IO0 0x1c4 + MX8MM_IOMUXC_SAI1_RXFS_GPIO4_IO0 0x184 /* CPLD_D[5] */ - MX8MM_IOMUXC_SAI1_TXC_GPIO4_IO11 0x1c4 + MX8MM_IOMUXC_SAI1_TXC_GPIO4_IO11 0x184 /* CPLD_D[4] */ - MX8MM_IOMUXC_SAI1_TXD0_GPIO4_IO12 0x1c4 + MX8MM_IOMUXC_SAI1_TXD0_GPIO4_IO12 0x184 /* CPLD_D[3] */ - MX8MM_IOMUXC_SAI1_TXD1_GPIO4_IO13 0x1c4 + MX8MM_IOMUXC_SAI1_TXD1_GPIO4_IO13 0x184 /* CPLD_D[2] */ - MX8MM_IOMUXC_SAI1_TXD2_GPIO4_IO14 0x1c4 + MX8MM_IOMUXC_SAI1_TXD2_GPIO4_IO14 0x184 /* CPLD_D[1] */ - MX8MM_IOMUXC_SAI1_TXD3_GPIO4_IO15 0x1c4 + MX8MM_IOMUXC_SAI1_TXD3_GPIO4_IO15 0x184 /* CPLD_D[0] */ - MX8MM_IOMUXC_SAI1_TXD4_GPIO4_IO16 0x1c4 + MX8MM_IOMUXC_SAI1_TXD4_GPIO4_IO16 0x184 /* KBD_intK */ MX8MM_IOMUXC_SAI2_MCLK_GPIO4_IO27 0x1c4 /* DISP_reset */ -- GitLab From 4153d789e299b29cbc57276d687c92f3a098e59b Mon Sep 17 00:00:00 2001 From: Zhang Xiaoxu Date: Sat, 22 Oct 2022 15:35:21 +0800 Subject: [PATCH 0320/1927] cifs: Fix pages array leak when writedata alloc failed in cifs_writedata_alloc() There is a memory leak when writedata alloc failed: unreferenced object 0xffff888192364000 (size 8192): comm "sync", pid 22839, jiffies 4297313967 (age 60.230s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<0000000027de0814>] __kmalloc+0x4d/0x150 [<00000000b21e81ab>] cifs_writepages+0x35f/0x14a0 [<0000000076f7d20e>] do_writepages+0x10a/0x360 [<00000000d6a36edc>] filemap_fdatawrite_wbc+0x95/0xc0 [<000000005751a323>] __filemap_fdatawrite_range+0xa7/0xe0 [<0000000088afb0ca>] file_write_and_wait_range+0x66/0xb0 [<0000000063dbc443>] cifs_strict_fsync+0x80/0x5f0 [<00000000c4624754>] __x64_sys_fsync+0x40/0x70 [<000000002c0dc744>] do_syscall_64+0x35/0x80 [<0000000052f46bee>] entry_SYSCALL_64_after_hwframe+0x46/0xb0 cifs_writepages+0x35f/0x14a0 is: kmalloc_array at include/linux/slab.h:628 (inlined by) kcalloc at include/linux/slab.h:659 (inlined by) cifs_writedata_alloc at fs/cifs/file.c:2438 (inlined by) wdata_alloc_and_fillpages at fs/cifs/file.c:2527 (inlined by) cifs_writepages at fs/cifs/file.c:2705 If writedata alloc failed in cifs_writedata_alloc(), the pages array should be freed. Fixes: 8e7360f67e75 ("CIFS: Add support for direct pages in wdata") Signed-off-by: Zhang Xiaoxu Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/file.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 5b3b308e115c8..d9fbf1ec6029a 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -2434,12 +2434,16 @@ cifs_writev_complete(struct work_struct *work) struct cifs_writedata * cifs_writedata_alloc(unsigned int nr_pages, work_func_t complete) { + struct cifs_writedata *writedata = NULL; struct page **pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); - if (pages) - return cifs_writedata_direct_alloc(pages, complete); + if (pages) { + writedata = cifs_writedata_direct_alloc(pages, complete); + if (!writedata) + kvfree(pages); + } - return NULL; + return writedata; } struct cifs_writedata * -- GitLab From f950c85e782f90702468bba8243cc97a8d0d04b0 Mon Sep 17 00:00:00 2001 From: Zhang Xiaoxu Date: Sat, 22 Oct 2022 15:35:20 +0800 Subject: [PATCH 0321/1927] cifs: Fix pages leak when writedata alloc failed in cifs_write_from_iter() There is a kmemleak when writedata alloc failed: unreferenced object 0xffff888175ae4000 (size 4096): comm "dd", pid 19419, jiffies 4296028749 (age 739.396s) hex dump (first 32 bytes): 80 02 b0 04 00 ea ff ff c0 02 b0 04 00 ea ff ff ................ 80 22 4c 04 00 ea ff ff c0 22 4c 04 00 ea ff ff ."L......"L..... backtrace: [<0000000072fdbb86>] __kmalloc_node+0x50/0x150 [<0000000039faf56f>] __iov_iter_get_pages_alloc+0x605/0xdd0 [<00000000f862a9d4>] iov_iter_get_pages_alloc2+0x3b/0x80 [<000000008f226067>] cifs_write_from_iter+0x2ae/0xe40 [<000000001f78f2f1>] __cifs_writev+0x337/0x5c0 [<00000000257fcef5>] vfs_write+0x503/0x690 [<000000008778a238>] ksys_write+0xb9/0x150 [<00000000ed82047c>] do_syscall_64+0x35/0x80 [<000000003365551d>] entry_SYSCALL_64_after_hwframe+0x46/0xb0 __iov_iter_get_pages_alloc+0x605/0xdd0 is: want_pages_array at lib/iov_iter.c:1304 (inlined by) __iov_iter_get_pages_alloc at lib/iov_iter.c:1457 If writedata allocate failed, the pages and pagevec should be cleanup. Fixes: 8c5f9c1ab7cb ("CIFS: Add support for direct I/O write") Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Zhang Xiaoxu Signed-off-by: Steve French --- fs/cifs/file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/cifs/file.c b/fs/cifs/file.c index d9fbf1ec6029a..cd96982099309 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -3303,6 +3303,9 @@ cifs_write_from_iter(loff_t offset, size_t len, struct iov_iter *from, cifs_uncached_writev_complete); if (!wdata) { rc = -ENOMEM; + for (i = 0; i < nr_pages; i++) + put_page(pagevec[i]); + kvfree(pagevec); add_credits_and_wake_if(server, credits, 0); break; } -- GitLab From d917a62af81b133f35f627e7936e193c842a7947 Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Tue, 18 Oct 2022 08:10:14 -0400 Subject: [PATCH 0322/1927] counter: microchip-tcb-capture: Handle Signal1 read and Synapse The signal_read(), action_read(), and action_write() callbacks have been assuming Signal0 is requested without checking. This results in requests for Signal1 returning data for Signal0. This patch fixes these oversights by properly checking for the Signal's id in the respective callbacks and handling accordingly based on the particular Signal requested. The trig_inverted member of the mchp_tc_data is removed as superfluous. Fixes: 106b104137fd ("counter: Add microchip TCB capture counter") Cc: stable@vger.kernel.org Reviewed-by: Kamel Bouhara Link: https://lore.kernel.org/r/20221018121014.7368-1-william.gray@linaro.org/ Signed-off-by: William Breathitt Gray --- drivers/counter/microchip-tcb-capture.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/counter/microchip-tcb-capture.c b/drivers/counter/microchip-tcb-capture.c index f9dee15d97776..e2d1dc6ca6682 100644 --- a/drivers/counter/microchip-tcb-capture.c +++ b/drivers/counter/microchip-tcb-capture.c @@ -28,7 +28,6 @@ struct mchp_tc_data { int qdec_mode; int num_channels; int channel[2]; - bool trig_inverted; }; static const enum counter_function mchp_tc_count_functions[] = { @@ -153,7 +152,7 @@ static int mchp_tc_count_signal_read(struct counter_device *counter, regmap_read(priv->regmap, ATMEL_TC_REG(priv->channel[0], SR), &sr); - if (priv->trig_inverted) + if (signal->id == 1) sigstatus = (sr & ATMEL_TC_MTIOB); else sigstatus = (sr & ATMEL_TC_MTIOA); @@ -171,6 +170,17 @@ static int mchp_tc_count_action_read(struct counter_device *counter, struct mchp_tc_data *const priv = counter_priv(counter); u32 cmr; + if (priv->qdec_mode) { + *action = COUNTER_SYNAPSE_ACTION_BOTH_EDGES; + return 0; + } + + /* Only TIOA signal is evaluated in non-QDEC mode */ + if (synapse->signal->id != 0) { + *action = COUNTER_SYNAPSE_ACTION_NONE; + return 0; + } + regmap_read(priv->regmap, ATMEL_TC_REG(priv->channel[0], CMR), &cmr); switch (cmr & ATMEL_TC_ETRGEDG) { @@ -199,8 +209,8 @@ static int mchp_tc_count_action_write(struct counter_device *counter, struct mchp_tc_data *const priv = counter_priv(counter); u32 edge = ATMEL_TC_ETRGEDG_NONE; - /* QDEC mode is rising edge only */ - if (priv->qdec_mode) + /* QDEC mode is rising edge only; only TIOA handled in non-QDEC mode */ + if (priv->qdec_mode || synapse->signal->id != 0) return -EINVAL; switch (action) { -- GitLab From d501d37841d3b7f18402d71a9ef057eb9dde127e Mon Sep 17 00:00:00 2001 From: William Breathitt Gray Date: Thu, 20 Oct 2022 10:11:21 -0400 Subject: [PATCH 0323/1927] counter: 104-quad-8: Fix race getting function mode and direction The quad8_action_read() function checks the Count function mode and Count direction without first acquiring a lock. This is a race condition because the function mode could change by the time the direction is checked. Because the quad8_function_read() already acquires a lock internally, the quad8_function_read() is refactored to spin out the no-lock code to a new quad8_function_get() function. To resolve the race condition in quad8_action_read(), a lock is acquired before calling quad8_function_get() and quad8_direction_read() in order to get both function mode and direction atomically. Fixes: f1d8a071d45b ("counter: 104-quad-8: Add Generic Counter interface support") Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221020141121.15434-1-william.gray@linaro.org/ Signed-off-by: William Breathitt Gray --- drivers/counter/104-quad-8.c | 64 +++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/drivers/counter/104-quad-8.c b/drivers/counter/104-quad-8.c index 77a863b7eefea..deed4afadb298 100644 --- a/drivers/counter/104-quad-8.c +++ b/drivers/counter/104-quad-8.c @@ -232,34 +232,45 @@ static const enum counter_function quad8_count_functions_list[] = { COUNTER_FUNCTION_QUADRATURE_X4, }; +static int quad8_function_get(const struct quad8 *const priv, const size_t id, + enum counter_function *const function) +{ + if (!priv->quadrature_mode[id]) { + *function = COUNTER_FUNCTION_PULSE_DIRECTION; + return 0; + } + + switch (priv->quadrature_scale[id]) { + case 0: + *function = COUNTER_FUNCTION_QUADRATURE_X1_A; + return 0; + case 1: + *function = COUNTER_FUNCTION_QUADRATURE_X2_A; + return 0; + case 2: + *function = COUNTER_FUNCTION_QUADRATURE_X4; + return 0; + default: + /* should never reach this path */ + return -EINVAL; + } +} + static int quad8_function_read(struct counter_device *counter, struct counter_count *count, enum counter_function *function) { struct quad8 *const priv = counter_priv(counter); - const int id = count->id; unsigned long irqflags; + int retval; spin_lock_irqsave(&priv->lock, irqflags); - if (priv->quadrature_mode[id]) - switch (priv->quadrature_scale[id]) { - case 0: - *function = COUNTER_FUNCTION_QUADRATURE_X1_A; - break; - case 1: - *function = COUNTER_FUNCTION_QUADRATURE_X2_A; - break; - case 2: - *function = COUNTER_FUNCTION_QUADRATURE_X4; - break; - } - else - *function = COUNTER_FUNCTION_PULSE_DIRECTION; + retval = quad8_function_get(priv, count->id, function); spin_unlock_irqrestore(&priv->lock, irqflags); - return 0; + return retval; } static int quad8_function_write(struct counter_device *counter, @@ -359,6 +370,7 @@ static int quad8_action_read(struct counter_device *counter, enum counter_synapse_action *action) { struct quad8 *const priv = counter_priv(counter); + unsigned long irqflags; int err; enum counter_function function; const size_t signal_a_id = count->synapses[0].signal->id; @@ -374,9 +386,21 @@ static int quad8_action_read(struct counter_device *counter, return 0; } - err = quad8_function_read(counter, count, &function); - if (err) + spin_lock_irqsave(&priv->lock, irqflags); + + /* Get Count function and direction atomically */ + err = quad8_function_get(priv, count->id, &function); + if (err) { + spin_unlock_irqrestore(&priv->lock, irqflags); + return err; + } + err = quad8_direction_read(counter, count, &direction); + if (err) { + spin_unlock_irqrestore(&priv->lock, irqflags); return err; + } + + spin_unlock_irqrestore(&priv->lock, irqflags); /* Default action mode */ *action = COUNTER_SYNAPSE_ACTION_NONE; @@ -389,10 +413,6 @@ static int quad8_action_read(struct counter_device *counter, return 0; case COUNTER_FUNCTION_QUADRATURE_X1_A: if (synapse->signal->id == signal_a_id) { - err = quad8_direction_read(counter, count, &direction); - if (err) - return err; - if (direction == COUNTER_COUNT_DIRECTION_FORWARD) *action = COUNTER_SYNAPSE_ACTION_RISING_EDGE; else -- GitLab From f4cd18c5b2000df0c382f6530eeca9141ea41faf Mon Sep 17 00:00:00 2001 From: Jerry Snitselaar Date: Sat, 22 Oct 2022 08:23:52 -0700 Subject: [PATCH 0324/1927] efi/tpm: Pass correct address to memblock_reserve memblock_reserve() expects a physical address, but the address being passed for the TPM final events log is what was returned from early_memremap(). This results in something like the following: [ 0.000000] memblock_reserve: [0xffffffffff2c0000-0xffffffffff2c00e4] efi_tpm_eventlog_init+0x324/0x370 Pass the address from efi like what is done for the TPM events log. Fixes: c46f3405692d ("tpm: Reserve the TPM final events table") Cc: Matthew Garrett Cc: Jarkko Sakkinen Cc: Bartosz Szczepanek Cc: Ard Biesheuvel Signed-off-by: Jerry Snitselaar Acked-by: Jarkko Sakkinen Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/tpm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/firmware/efi/tpm.c b/drivers/firmware/efi/tpm.c index 8f665678e9e39..e8d69bd548f3f 100644 --- a/drivers/firmware/efi/tpm.c +++ b/drivers/firmware/efi/tpm.c @@ -97,7 +97,7 @@ int __init efi_tpm_eventlog_init(void) goto out_calc; } - memblock_reserve((unsigned long)final_tbl, + memblock_reserve(efi.tpm_final_log, tbl_size + sizeof(*final_tbl)); efi_tpm_final_log_size = tbl_size; -- GitLab From d76308f03ee1574b0deffde45604252a51c77f6d Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 24 Oct 2022 08:32:39 +0200 Subject: [PATCH 0325/1927] Revert "coresight: cti: Fix hang in cti_disable_hw()" This reverts commit 665c157e0204176023860b51a46528ba0ba62c33. It causes reported build warnings: drivers/hwtracing/coresight/coresight-cti-core.c: In functio n 'cti_enable_hw': drivers/hwtracing/coresight/coresight-cti-core.c:93:24: warning: unused variable 'dev' [-Wunused-variable] 93 | struct device *dev = &drvdata->csdev->dev; | ^~~ drivers/hwtracing/coresight/coresight-cti-core.c: In function 'cti_disable_hw': drivers/hwtracing/coresight/coresight-cti-core.c:154:24: warning: unused variable 'dev' [-Wunused-variable] 154 | struct device *dev = &drvdata->csdev->dev; | ^~~ Reported-by: Stephen Rothwell Cc: Aishwarya TCV Cc: Cristian Marussi Cc: Suzuki Poulose Cc: James Clark Cc: Mike Leach Cc: Mike Leach Cc: Suzuki K Poulose Fixes: 665c157e0204 ("coresight: cti: Fix hang in cti_disable_hw()") Link: https://lore.kernel.org/r/20221024135752.2b83af97@canb.auug.org.au Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-cti-core.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 4a02ae23d3a06..1be92342b5b98 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -94,6 +94,7 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) unsigned long flags; int rc = 0; + pm_runtime_get_sync(dev->parent); spin_lock_irqsave(&drvdata->spinlock, flags); /* no need to do anything if enabled or unpowered*/ @@ -118,6 +119,7 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) /* cannot enable due to error */ cti_err_not_enabled: spin_unlock_irqrestore(&drvdata->spinlock, flags); + pm_runtime_put(dev->parent); return rc; } @@ -173,6 +175,7 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); + pm_runtime_put(dev->parent); return 0; /* not disabled this call */ -- GitLab From 161a438d730dade2ba2b1bf8785f0759aba4ca5f Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 20 Oct 2022 10:39:08 +0200 Subject: [PATCH 0326/1927] efi: random: reduce seed size to 32 bytes We no longer need at least 64 bytes of random seed to permit the early crng init to complete. The RNG is now based on Blake2s, so reduce the EFI seed size to the Blake2s hash size, which is sufficient for our purposes. While at it, drop the READ_ONCE(), which was supposed to prevent size from being evaluated after seed was unmapped. However, this cannot actually happen, so READ_ONCE() is unnecessary here. Cc: # v4.14+ Signed-off-by: Ard Biesheuvel Reviewed-by: Jason A. Donenfeld Acked-by: Ilias Apalodimas --- drivers/firmware/efi/efi.c | 2 +- include/linux/efi.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 3ecdc43a3f2bb..a46df5d1d0942 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -611,7 +611,7 @@ int __init efi_config_parse_tables(const efi_config_table_t *config_tables, seed = early_memremap(efi_rng_seed, sizeof(*seed)); if (seed != NULL) { - size = READ_ONCE(seed->size); + size = min(seed->size, EFI_RANDOM_SEED_SIZE); early_memunmap(seed, sizeof(*seed)); } else { pr_err("Could not map UEFI random seed!\n"); diff --git a/include/linux/efi.h b/include/linux/efi.h index 80f3c1c7827dd..929d559ad41d2 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1222,7 +1222,7 @@ efi_status_t efi_random_get_seed(void); arch_efi_call_virt_teardown(); \ }) -#define EFI_RANDOM_SEED_SIZE 64U +#define EFI_RANDOM_SEED_SIZE 32U // BLAKE2S_HASH_SIZE struct linux_efi_random_seed { u32 size; -- GitLab From 7d866e38c7e9ece8a096d0d098fa9d92b9d4f97e Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 20 Oct 2022 10:39:09 +0200 Subject: [PATCH 0327/1927] efi: random: Use 'ACPI reclaim' memory for random seed EFI runtime services data is guaranteed to be preserved by the OS, making it a suitable candidate for the EFI random seed table, which may be passed to kexec kernels as well (after refreshing the seed), and so we need to ensure that the memory is preserved without support from the OS itself. However, runtime services data is intended for allocations that are relevant to the implementations of the runtime services themselves, and so they are unmapped from the kernel linear map, and mapped into the EFI page tables that are active while runtime service invocations are in progress. None of this is needed for the RNG seed. So let's switch to EFI 'ACPI reclaim' memory: in spite of the name, there is nothing exclusively ACPI about it, it is simply a type of allocation that carries firmware provided data which may or may not be relevant to the OS, and it is left up to the OS to decide whether to reclaim it after having consumed its contents. Given that in Linux, we never reclaim these allocations, it is a good choice for the EFI RNG seed, as the allocation is guaranteed to survive kexec reboots. One additional reason for changing this now is to align it with the upcoming recommendation for EFI bootloader provided RNG seeds, which must not use EFI runtime services code/data allocations. Cc: # v4.14+ Signed-off-by: Ard Biesheuvel Reviewed-by: Ilias Apalodimas --- drivers/firmware/efi/libstub/random.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/firmware/efi/libstub/random.c b/drivers/firmware/efi/libstub/random.c index 24aa375353724..33ab567695951 100644 --- a/drivers/firmware/efi/libstub/random.c +++ b/drivers/firmware/efi/libstub/random.c @@ -75,7 +75,12 @@ efi_status_t efi_random_get_seed(void) if (status != EFI_SUCCESS) return status; - status = efi_bs_call(allocate_pool, EFI_RUNTIME_SERVICES_DATA, + /* + * Use EFI_ACPI_RECLAIM_MEMORY here so that it is guaranteed that the + * allocation will survive a kexec reboot (although we refresh the seed + * beforehand) + */ + status = efi_bs_call(allocate_pool, EFI_ACPI_RECLAIM_MEMORY, sizeof(*seed) + EFI_RANDOM_SEED_SIZE, (void **)&seed); if (status != EFI_SUCCESS) -- GitLab From 5a5c4e06fd03b595542d5590f2bc05a6b7fc5c2b Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Thu, 20 Oct 2022 16:25:35 +0200 Subject: [PATCH 0328/1927] mac802154: Fix LQI recording Back in 2014, the LQI was saved in the skb control buffer (skb->cb, or mac_cb(skb)) without any actual reset of this area prior to its use. As part of a useful rework of the use of this region, 32edc40ae65c ("ieee802154: change _cb handling slightly") introduced mac_cb_init() to basically memset the cb field to 0. In particular, this new function got called at the beginning of mac802154_parse_frame_start(), right before the location where the buffer got actually filled. What went through unnoticed however, is the fact that the very first helper called by device drivers in the receive path already used this area to save the LQI value for later extraction. Resetting the cb field "so late" led to systematically zeroing the LQI. If we consider the reset of the cb field needed, we can make it as soon as we get an skb from a device driver, right before storing the LQI, as is the very first time we need to write something there. Cc: stable@vger.kernel.org Fixes: 32edc40ae65c ("ieee802154: change _cb handling slightly") Signed-off-by: Miquel Raynal Acked-by: Alexander Aring Link: https://lore.kernel.org/r/20221020142535.1038885-1-miquel.raynal@bootlin.com Signed-off-by: Stefan Schmidt --- net/mac802154/rx.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/net/mac802154/rx.c b/net/mac802154/rx.c index c439125ef2b91..726b47a4611b5 100644 --- a/net/mac802154/rx.c +++ b/net/mac802154/rx.c @@ -132,7 +132,7 @@ static int ieee802154_parse_frame_start(struct sk_buff *skb, struct ieee802154_hdr *hdr) { int hlen; - struct ieee802154_mac_cb *cb = mac_cb_init(skb); + struct ieee802154_mac_cb *cb = mac_cb(skb); skb_reset_mac_header(skb); @@ -294,8 +294,9 @@ void ieee802154_rx_irqsafe(struct ieee802154_hw *hw, struct sk_buff *skb, u8 lqi) { struct ieee802154_local *local = hw_to_local(hw); + struct ieee802154_mac_cb *cb = mac_cb_init(skb); - mac_cb(skb)->lqi = lqi; + cb->lqi = lqi; skb->pkt_type = IEEE802154_RX_MSG; skb_queue_tail(&local->skb_queue, skb); tasklet_schedule(&local->tasklet); -- GitLab From 1a3abd12a394f5c66943fee75cef533069e831fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Roberto=20de=20Souza?= Date: Mon, 17 Oct 2022 06:24:32 -0700 Subject: [PATCH 0329/1927] drm/i915: Extend Wa_1607297627 to Alderlake-P MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workaround 1607297627 was missed for Alderlake-P, so here extending it to it and adding the fixes tag so this WA is backported to all stable kernels. v2: - fixed subject - added Fixes tag BSpec: 54369 Cc: # v5.17+ Fixes: dfb924e33927 ("drm/i915/adlp: Remove require_force_probe protection") Reviewed-by: Lucas De Marchi Cc: Tvrtko Ursulin Signed-off-by: José Roberto de Souza Link: https://patchwork.freedesktop.org/patch/msgid/20221017132432.112850-1-jose.souza@intel.com (cherry picked from commit 847eec69f01a28ca44f5ac7e1d71d3a60263d680) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/gt/intel_workarounds.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 6d2003d598e6a..a821e3d405dbe 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -2293,11 +2293,11 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal) } if (IS_DG1_GRAPHICS_STEP(i915, STEP_A0, STEP_B0) || - IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915)) { + IS_ROCKETLAKE(i915) || IS_TIGERLAKE(i915) || IS_ALDERLAKE_P(i915)) { /* * Wa_1607030317:tgl * Wa_1607186500:tgl - * Wa_1607297627:tgl,rkl,dg1[a0] + * Wa_1607297627:tgl,rkl,dg1[a0],adlp * * On TGL and RKL there are multiple entries for this WA in the * BSpec; some indicate this is an A0-only WA, others indicate -- GitLab From 62c52eac1ad680fc68ef6d75955127dca46e2740 Mon Sep 17 00:00:00 2001 From: Anshuman Gupta Date: Fri, 14 Oct 2022 17:02:58 +0530 Subject: [PATCH 0330/1927] drm/i915/dgfx: Keep PCI autosuspend control 'on' by default on all dGPU DGFX platforms has lmem and cpu can access the lmem objects via mmap and i915 internal i915_gem_object_pin_map() for i915 own usages. Both of these methods has pre-requisite requirement to keep GFX PCI endpoint in D0 for a supported iomem transaction over PCI link. (Refer PCIe specs 5.3.1.4.1) Both DG1/DG2 have a known hardware bug that violates the PCIe specs and support the iomem read write transaction over PCIe bus despite endpoint is D3 state. Due to above H/W bug, we had never observed any issue with i915 runtime PM versus lmem access. But this issue becomes visible when PCIe gfx endpoint's upstream bridge enters to D3, at this point any lmem read/write access will be returned as unsupported request. But again this issue is not observed on every platform because it has been observed on few host machines DG1/DG2 endpoint's upstream bridge does not bind with pcieport driver. which really disables the PCIe power savings and leaves the bridge at D0 state. We need a unique interface to read/write from lmem with runtime PM wakeref protection something similar to intel_uncore_{read, write}, keep autosuspend control to 'on' on all discrete platforms, until we have a unique interface to read/write from lmem. This just change the default autosuspend setting of i915 on dGPU, user can still change it to 'auto'. v2: - Modified the commit message and subject with more information. - Changed the Fixes tag to LMEM support commit. [Joonas] - Changed !HAS_LMEM() Cond to !IS_DGFX(). [Rodrigo] Fixes: b908be543e44 ("drm/i915: support creating LMEM objects") Suggested-by: Rodrigo Vivi Reviewed-by: Rodrigo Vivi Signed-off-by: Anshuman Gupta Link: https://patchwork.freedesktop.org/patch/msgid/20221014113258.1284226-1-anshuman.gupta@intel.com (cherry picked from commit 66eb93e71a7a6695b7c5eb682e3ca1c980cf9d58) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/intel_runtime_pm.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c index 6ed5786bcd299..744cca507946b 100644 --- a/drivers/gpu/drm/i915/intel_runtime_pm.c +++ b/drivers/gpu/drm/i915/intel_runtime_pm.c @@ -591,8 +591,15 @@ void intel_runtime_pm_enable(struct intel_runtime_pm *rpm) pm_runtime_use_autosuspend(kdev); } - /* Enable by default */ - pm_runtime_allow(kdev); + /* + * FIXME: Temp hammer to keep autosupend disable on lmem supported platforms. + * As per PCIe specs 5.3.1.4.1, all iomem read write request over a PCIe + * function will be unsupported in case PCIe endpoint function is in D3. + * Let's keep i915 autosuspend control 'on' till we fix all known issue + * with lmem access in D3. + */ + if (!IS_DGFX(i915)) + pm_runtime_allow(kdev); /* * The core calls the driver load handler with an RPM reference held. -- GitLab From 63720a561b3c98199adf0c73e152807f15cc3b7f Mon Sep 17 00:00:00 2001 From: Ankit Nautiyal Date: Tue, 11 Oct 2022 12:04:40 +0530 Subject: [PATCH 0331/1927] drm/i915/dp: Reset frl trained flag before restarting FRL training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For cases where DP has HDMI2.1 sink and FRL Link issues are detected, reset the flag to state FRL trained status before restarting FRL training. Fixes: 9488a030ac91 ("drm/i915: Add support for enabling link status and recovery") Cc: Swati Sharma Cc: Ankit Nautiyal Cc: Uma Shankar (v2) Cc: Jani Nikula Signed-off-by: Ankit Nautiyal Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221011063447.904649-2-ankit.k.nautiyal@intel.com (cherry picked from commit 47e1a59e60c688c5f95b67277202f05b7e84c189) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/display/intel_dp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index c9be61d2348e0..47419d162f302 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -3957,6 +3957,8 @@ intel_dp_handle_hdmi_link_status_change(struct intel_dp *intel_dp) drm_dp_pcon_hdmi_frl_link_error_count(&intel_dp->aux, &intel_dp->attached_connector->base); + intel_dp->frl.is_trained = false; + /* Restart FRL training or fall back to TMDS mode */ intel_dp_check_frl_training(intel_dp); } -- GitLab From 36abde8d24ad740371422a7678ca92b06cc8a3d5 Mon Sep 17 00:00:00 2001 From: "Luke D. Jones" Date: Mon, 10 Oct 2022 19:30:09 +1300 Subject: [PATCH 0332/1927] platform/x86: asus-wmi: Add support for ROG X16 tablet mode Add quirk for ASUS ROG X16 Flow 2-in-1 to enable tablet mode with lid flip (all screen rotations). Signed-off-by: Luke D. Jones Link: https://lore.kernel.org/r/20221010063009.32293-1-luke@ljones.dev Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/asus-nb-wmi.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c index 613c45c9fbe36..c685a705b73dd 100644 --- a/drivers/platform/x86/asus-nb-wmi.c +++ b/drivers/platform/x86/asus-nb-wmi.c @@ -464,6 +464,15 @@ static const struct dmi_system_id asus_quirks[] = { }, .driver_data = &quirk_asus_tablet_mode, }, + { + .callback = dmi_matched, + .ident = "ASUS ROG FLOW X16", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "GV601R"), + }, + .driver_data = &quirk_asus_tablet_mode, + }, {}, }; -- GitLab From a10d50983f7befe85acf95ea7dbf6ba9187c2d70 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Wed, 19 Oct 2022 21:47:51 +0200 Subject: [PATCH 0333/1927] platform/x86: thinkpad_acpi: Fix reporting a non present second fan on some models thinkpad_acpi was reporting 2 fans on a ThinkPad T14s gen 1, even though the laptop has only 1 fan. The second, not present fan always reads 65535 (-1 in 16 bit signed), ignore fans which report 65535 to avoid reporting the non present fan. Signed-off-by: Jelle van der Waa Link: https://lore.kernel.org/r/20221019194751.5392-1-jvanderwaa@redhat.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/thinkpad_acpi.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c index 6a823b850a778..20e5c043a8e87 100644 --- a/drivers/platform/x86/thinkpad_acpi.c +++ b/drivers/platform/x86/thinkpad_acpi.c @@ -263,6 +263,8 @@ enum tpacpi_hkey_event_t { #define TPACPI_DBG_BRGHT 0x0020 #define TPACPI_DBG_MIXER 0x0040 +#define FAN_NOT_PRESENT 65535 + #define strlencmp(a, b) (strncmp((a), (b), strlen(b))) @@ -8876,7 +8878,7 @@ static int __init fan_init(struct ibm_init_struct *iibm) /* Try and probe the 2nd fan */ tp_features.second_fan = 1; /* needed for get_speed to work */ res = fan2_get_speed(&speed); - if (res >= 0) { + if (res >= 0 && speed != FAN_NOT_PRESENT) { /* It responded - so let's assume it's there */ tp_features.second_fan = 1; tp_features.second_fan_ctl = 1; -- GitLab From 6960d133f66ecddcd3af2b1cbd0c7dcd104268b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Huguet?= Date: Thu, 20 Oct 2022 09:53:10 +0200 Subject: [PATCH 0334/1927] atlantic: fix deadlock at aq_nic_stop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NIC is stopped with rtnl_lock held, and during the stop it cancels the 'service_task' work and free irqs. However, if CONFIG_MACSEC is set, rtnl_lock is acquired both from aq_nic_service_task and aq_linkstate_threaded_isr. Then a deadlock happens if aq_nic_stop tries to cancel/disable them when they've already started their execution. As the deadlock is caused by rtnl_lock, it causes many other processes to stall, not only atlantic related stuff. Fix it by introducing a mutex that protects each NIC's macsec related data, and locking it instead of the rtnl_lock from the service task and the threaded IRQ. Before this patch, all macsec data was protected with rtnl_lock, but maybe not all of it needs to be protected. With this new mutex, further efforts can be made to limit the protected data only to that which requires it. However, probably it doesn't worth it because all macsec's data accesses are infrequent, and almost all are done from macsec_ops or ethtool callbacks, called holding rtnl_lock, so macsec_mutex won't never be much contended. The issue appeared repeteadly attaching and deattaching the NIC to a bond interface. Doing that after this patch I cannot reproduce the bug. Fixes: 62c1c2e606f6 ("net: atlantic: MACSec offload skeleton") Reported-by: Li Liang Suggested-by: Andrew Lunn Signed-off-by: Íñigo Huguet Reviewed-by: Igor Russkikh Signed-off-by: David S. Miller --- .../ethernet/aquantia/atlantic/aq_macsec.c | 96 ++++++++++++++----- .../net/ethernet/aquantia/atlantic/aq_nic.h | 2 + 2 files changed, 74 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c index 3d0e16791e1c9..a0180811305dc 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c @@ -1394,26 +1394,57 @@ static void aq_check_txsa_expiration(struct aq_nic_s *nic) egress_sa_threshold_expired); } +#define AQ_LOCKED_MDO_DEF(mdo) \ +static int aq_locked_mdo_##mdo(struct macsec_context *ctx) \ +{ \ + struct aq_nic_s *nic = netdev_priv(ctx->netdev); \ + int ret; \ + mutex_lock(&nic->macsec_mutex); \ + ret = aq_mdo_##mdo(ctx); \ + mutex_unlock(&nic->macsec_mutex); \ + return ret; \ +} + +AQ_LOCKED_MDO_DEF(dev_open) +AQ_LOCKED_MDO_DEF(dev_stop) +AQ_LOCKED_MDO_DEF(add_secy) +AQ_LOCKED_MDO_DEF(upd_secy) +AQ_LOCKED_MDO_DEF(del_secy) +AQ_LOCKED_MDO_DEF(add_rxsc) +AQ_LOCKED_MDO_DEF(upd_rxsc) +AQ_LOCKED_MDO_DEF(del_rxsc) +AQ_LOCKED_MDO_DEF(add_rxsa) +AQ_LOCKED_MDO_DEF(upd_rxsa) +AQ_LOCKED_MDO_DEF(del_rxsa) +AQ_LOCKED_MDO_DEF(add_txsa) +AQ_LOCKED_MDO_DEF(upd_txsa) +AQ_LOCKED_MDO_DEF(del_txsa) +AQ_LOCKED_MDO_DEF(get_dev_stats) +AQ_LOCKED_MDO_DEF(get_tx_sc_stats) +AQ_LOCKED_MDO_DEF(get_tx_sa_stats) +AQ_LOCKED_MDO_DEF(get_rx_sc_stats) +AQ_LOCKED_MDO_DEF(get_rx_sa_stats) + const struct macsec_ops aq_macsec_ops = { - .mdo_dev_open = aq_mdo_dev_open, - .mdo_dev_stop = aq_mdo_dev_stop, - .mdo_add_secy = aq_mdo_add_secy, - .mdo_upd_secy = aq_mdo_upd_secy, - .mdo_del_secy = aq_mdo_del_secy, - .mdo_add_rxsc = aq_mdo_add_rxsc, - .mdo_upd_rxsc = aq_mdo_upd_rxsc, - .mdo_del_rxsc = aq_mdo_del_rxsc, - .mdo_add_rxsa = aq_mdo_add_rxsa, - .mdo_upd_rxsa = aq_mdo_upd_rxsa, - .mdo_del_rxsa = aq_mdo_del_rxsa, - .mdo_add_txsa = aq_mdo_add_txsa, - .mdo_upd_txsa = aq_mdo_upd_txsa, - .mdo_del_txsa = aq_mdo_del_txsa, - .mdo_get_dev_stats = aq_mdo_get_dev_stats, - .mdo_get_tx_sc_stats = aq_mdo_get_tx_sc_stats, - .mdo_get_tx_sa_stats = aq_mdo_get_tx_sa_stats, - .mdo_get_rx_sc_stats = aq_mdo_get_rx_sc_stats, - .mdo_get_rx_sa_stats = aq_mdo_get_rx_sa_stats, + .mdo_dev_open = aq_locked_mdo_dev_open, + .mdo_dev_stop = aq_locked_mdo_dev_stop, + .mdo_add_secy = aq_locked_mdo_add_secy, + .mdo_upd_secy = aq_locked_mdo_upd_secy, + .mdo_del_secy = aq_locked_mdo_del_secy, + .mdo_add_rxsc = aq_locked_mdo_add_rxsc, + .mdo_upd_rxsc = aq_locked_mdo_upd_rxsc, + .mdo_del_rxsc = aq_locked_mdo_del_rxsc, + .mdo_add_rxsa = aq_locked_mdo_add_rxsa, + .mdo_upd_rxsa = aq_locked_mdo_upd_rxsa, + .mdo_del_rxsa = aq_locked_mdo_del_rxsa, + .mdo_add_txsa = aq_locked_mdo_add_txsa, + .mdo_upd_txsa = aq_locked_mdo_upd_txsa, + .mdo_del_txsa = aq_locked_mdo_del_txsa, + .mdo_get_dev_stats = aq_locked_mdo_get_dev_stats, + .mdo_get_tx_sc_stats = aq_locked_mdo_get_tx_sc_stats, + .mdo_get_tx_sa_stats = aq_locked_mdo_get_tx_sa_stats, + .mdo_get_rx_sc_stats = aq_locked_mdo_get_rx_sc_stats, + .mdo_get_rx_sa_stats = aq_locked_mdo_get_rx_sa_stats, }; int aq_macsec_init(struct aq_nic_s *nic) @@ -1435,6 +1466,7 @@ int aq_macsec_init(struct aq_nic_s *nic) nic->ndev->features |= NETIF_F_HW_MACSEC; nic->ndev->macsec_ops = &aq_macsec_ops; + mutex_init(&nic->macsec_mutex); return 0; } @@ -1458,7 +1490,7 @@ int aq_macsec_enable(struct aq_nic_s *nic) if (!nic->macsec_cfg) return 0; - rtnl_lock(); + mutex_lock(&nic->macsec_mutex); if (nic->aq_fw_ops->send_macsec_req) { struct macsec_cfg_request cfg = { 0 }; @@ -1507,7 +1539,7 @@ int aq_macsec_enable(struct aq_nic_s *nic) ret = aq_apply_macsec_cfg(nic); unlock: - rtnl_unlock(); + mutex_unlock(&nic->macsec_mutex); return ret; } @@ -1519,9 +1551,9 @@ void aq_macsec_work(struct aq_nic_s *nic) if (!netif_carrier_ok(nic->ndev)) return; - rtnl_lock(); + mutex_lock(&nic->macsec_mutex); aq_check_txsa_expiration(nic); - rtnl_unlock(); + mutex_unlock(&nic->macsec_mutex); } int aq_macsec_rx_sa_cnt(struct aq_nic_s *nic) @@ -1532,21 +1564,30 @@ int aq_macsec_rx_sa_cnt(struct aq_nic_s *nic) if (!cfg) return 0; + mutex_lock(&nic->macsec_mutex); + for (i = 0; i < AQ_MACSEC_MAX_SC; i++) { if (!test_bit(i, &cfg->rxsc_idx_busy)) continue; cnt += hweight_long(cfg->aq_rxsc[i].rx_sa_idx_busy); } + mutex_unlock(&nic->macsec_mutex); return cnt; } int aq_macsec_tx_sc_cnt(struct aq_nic_s *nic) { + int cnt; + if (!nic->macsec_cfg) return 0; - return hweight_long(nic->macsec_cfg->txsc_idx_busy); + mutex_lock(&nic->macsec_mutex); + cnt = hweight_long(nic->macsec_cfg->txsc_idx_busy); + mutex_unlock(&nic->macsec_mutex); + + return cnt; } int aq_macsec_tx_sa_cnt(struct aq_nic_s *nic) @@ -1557,12 +1598,15 @@ int aq_macsec_tx_sa_cnt(struct aq_nic_s *nic) if (!cfg) return 0; + mutex_lock(&nic->macsec_mutex); + for (i = 0; i < AQ_MACSEC_MAX_SC; i++) { if (!test_bit(i, &cfg->txsc_idx_busy)) continue; cnt += hweight_long(cfg->aq_txsc[i].tx_sa_idx_busy); } + mutex_unlock(&nic->macsec_mutex); return cnt; } @@ -1634,6 +1678,8 @@ u64 *aq_macsec_get_stats(struct aq_nic_s *nic, u64 *data) if (!cfg) return data; + mutex_lock(&nic->macsec_mutex); + aq_macsec_update_stats(nic); common_stats = &cfg->stats; @@ -1716,5 +1762,7 @@ u64 *aq_macsec_get_stats(struct aq_nic_s *nic, u64 *data) data += i; + mutex_unlock(&nic->macsec_mutex); + return data; } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_nic.h b/drivers/net/ethernet/aquantia/atlantic/aq_nic.h index 935ba889bd9a8..ad33f8586532b 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_nic.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_nic.h @@ -157,6 +157,8 @@ struct aq_nic_s { struct mutex fwreq_mutex; #if IS_ENABLED(CONFIG_MACSEC) struct aq_macsec_cfg *macsec_cfg; + /* mutex to protect data in macsec_cfg */ + struct mutex macsec_mutex; #endif /* PTP support */ struct aq_ptp_s *aq_ptp; -- GitLab From 0b6e6e149c136677f1cc859d4185b5a2db50ffbf Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 20 Oct 2022 06:37:49 -0500 Subject: [PATCH 0335/1927] platform/x86/amd: pmc: Read SMU version during suspend on Cezanne systems commit b0c07116c894 ("platform/x86: amd-pmc: Avoid reading SMU version at probe time") adjusted the behavior for amd-pmc to avoid reading the SMU version at startup but rather on first use to improve boot time. However the SMU version is also used to decide whether to place a timer based wakeup in the OS_HINT message. If the idlemask hasn't been read before this message was sent then the SMU version will not have been cached. Ensure the SMU version has been read before deciding whether or not to run this codepath. Cc: stable@vger.kernel.org # 6.0 Reported-by: You-Sheng Yang Tested-by: Anson Tsao Fixes: b0c07116c894 ("platform/x86: amd-pmc: Avoid reading SMU version at probe time") Signed-off-by: Mario Limonciello Link: https://lore.kernel.org/r/20221020113749.6621-2-mario.limonciello@amd.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/amd/pmc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/platform/x86/amd/pmc.c b/drivers/platform/x86/amd/pmc.c index ce859b300712b..96e790e639a21 100644 --- a/drivers/platform/x86/amd/pmc.c +++ b/drivers/platform/x86/amd/pmc.c @@ -663,6 +663,13 @@ static int amd_pmc_verify_czn_rtc(struct amd_pmc_dev *pdev, u32 *arg) struct rtc_time tm; int rc; + /* we haven't yet read SMU version */ + if (!pdev->major) { + rc = amd_pmc_get_smu_version(pdev); + if (rc) + return rc; + } + if (pdev->major < 64 || (pdev->major == 64 && pdev->minor < 53)) return 0; -- GitLab From f8127476930b98fc9e9aa5de0bbf9eeaf45db219 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Thu, 20 Oct 2022 08:28:28 +0300 Subject: [PATCH 0336/1927] net/mlx5e: Cleanup MACsec uninitialization routine The mlx5e_macsec_cleanup() routine has NULL pointer dereferencing if mlx5 device doesn't support MACsec (priv->macsec will be NULL). While at it delete comment line, assignment and extra blank lines, so fix everything in one patch. Fixes: 1f53da676439 ("net/mlx5e: Create advanced steering operation (ASO) object for MACsec") Reported-by: Dan Carpenter Signed-off-by: Leon Romanovsky Signed-off-by: David S. Miller --- .../net/ethernet/mellanox/mlx5/core/en_accel/macsec.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 41970067917bf..4331235b21ee8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -1846,25 +1846,16 @@ int mlx5e_macsec_init(struct mlx5e_priv *priv) void mlx5e_macsec_cleanup(struct mlx5e_priv *priv) { struct mlx5e_macsec *macsec = priv->macsec; - struct mlx5_core_dev *mdev = macsec->mdev; + struct mlx5_core_dev *mdev = priv->mdev; if (!macsec) return; mlx5_notifier_unregister(mdev, &macsec->nb); - mlx5e_macsec_fs_cleanup(macsec->macsec_fs); - - /* Cleanup workqueue */ destroy_workqueue(macsec->wq); - mlx5e_macsec_aso_cleanup(&macsec->aso, mdev); - - priv->macsec = NULL; - rhashtable_destroy(&macsec->sci_hash); - mutex_destroy(&macsec->lock); - kfree(macsec); } -- GitLab From ee24395f91b9cddccae5f6c11c37ee4ed78ff354 Mon Sep 17 00:00:00 2001 From: Henning Schild Date: Mon, 24 Oct 2022 11:20:27 +0200 Subject: [PATCH 0337/1927] leds: simatic-ipc-leds-gpio: fix incorrect LED to GPIO mapping For apollolake the mapping between LEDs and GPIO pins was off because of a refactoring when we introduced a new device model. In addition to the reordering the indices in the lookup table need to be updated as well. Fixes: a97126265dfe ("leds: simatic-ipc-leds-gpio: add new model 227G") Signed-off-by: Henning Schild Link: https://lore.kernel.org/r/20221024092027.4529-1-henning.schild@siemens.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/leds/simple/simatic-ipc-leds-gpio.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/leds/simple/simatic-ipc-leds-gpio.c b/drivers/leds/simple/simatic-ipc-leds-gpio.c index b9eeb8702df0c..07f0d79d604d4 100644 --- a/drivers/leds/simple/simatic-ipc-leds-gpio.c +++ b/drivers/leds/simple/simatic-ipc-leds-gpio.c @@ -20,12 +20,12 @@ static struct gpiod_lookup_table *simatic_ipc_led_gpio_table; static struct gpiod_lookup_table simatic_ipc_led_gpio_table_127e = { .dev_id = "leds-gpio", .table = { - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 52, NULL, 1, GPIO_ACTIVE_LOW), - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 53, NULL, 2, GPIO_ACTIVE_LOW), - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 57, NULL, 3, GPIO_ACTIVE_LOW), - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 58, NULL, 4, GPIO_ACTIVE_LOW), - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 60, NULL, 5, GPIO_ACTIVE_LOW), - GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 51, NULL, 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 52, NULL, 0, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 53, NULL, 1, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 57, NULL, 2, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 58, NULL, 3, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 60, NULL, 4, GPIO_ACTIVE_LOW), + GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 51, NULL, 5, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 56, NULL, 6, GPIO_ACTIVE_LOW), GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 59, NULL, 7, GPIO_ACTIVE_HIGH), }, -- GitLab From 555a68dd681b7437a2708001d465c85f6dfa6955 Mon Sep 17 00:00:00 2001 From: Gayatri Kammela Date: Mon, 12 Sep 2022 16:33:07 -0700 Subject: [PATCH 0338/1927] platform/x86/intel: pmc/core: Add Raptor Lake support to pmc core driver Add Raptor Lake client parts (both RPL and RPL_S) support to pmc core driver. Raptor Lake client parts reuse all the Alder Lake PCH IPs. Cc: Srinivas Pandruvada Cc: Andy Shevchenko Cc: David Box Acked-by: Rajneesh Bhardwaj Signed-off-by: Gayatri Kammela Link: https://lore.kernel.org/r/20220912233307.409954-2-gayatri.kammela@linux.intel.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/intel/pmc/core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c index a1fe1e0dcf4a5..17ec5825d13d7 100644 --- a/drivers/platform/x86/intel/pmc/core.c +++ b/drivers/platform/x86/intel/pmc/core.c @@ -1914,6 +1914,8 @@ static const struct x86_cpu_id intel_pmc_core_ids[] = { X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &tgl_reg_map), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &adl_reg_map), X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &tgl_reg_map), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &adl_reg_map), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &adl_reg_map), {} }; -- GitLab From c99f0f7e68376dda5df8db7950cd6b67e73c6d3c Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Thu, 20 Oct 2022 11:50:41 -0400 Subject: [PATCH 0339/1927] net: fman: Use physical address for userspace interfaces Before 262f2b782e25 ("net: fman: Map the base address once"), the physical address of the MAC was exposed to userspace in two places: via sysfs and via SIOCGIFMAP. While this is not best practice, it is an external ABI which is in use by userspace software. The aforementioned commit inadvertently modified these addresses and made them virtual. This constitutes and ABI break. Additionally, it leaks the kernel's memory layout to userspace. Partially revert that commit, reintroducing the resource back into struct mac_device, while keeping the intended changes (the rework of the address mapping). Fixes: 262f2b782e25 ("net: fman: Map the base address once") Reported-by: Geert Uytterhoeven Signed-off-by: Sean Anderson Acked-by: Madalin Bucur Signed-off-by: David S. Miller --- drivers/net/ethernet/freescale/dpaa/dpaa_eth.c | 4 ++-- drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c | 2 +- drivers/net/ethernet/freescale/fman/mac.c | 12 ++++++------ drivers/net/ethernet/freescale/fman/mac.h | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 31cfa121333df..fc68a32ce2f71 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -221,8 +221,8 @@ static int dpaa_netdev_init(struct net_device *net_dev, net_dev->netdev_ops = dpaa_ops; mac_addr = mac_dev->addr; - net_dev->mem_start = (unsigned long)mac_dev->vaddr; - net_dev->mem_end = (unsigned long)mac_dev->vaddr_end; + net_dev->mem_start = (unsigned long)priv->mac_dev->res->start; + net_dev->mem_end = (unsigned long)priv->mac_dev->res->end; net_dev->min_mtu = ETH_MIN_MTU; net_dev->max_mtu = dpaa_get_max_mtu(); diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c index 258eb6c8f4c06..4fee74c024bd7 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c @@ -18,7 +18,7 @@ static ssize_t dpaa_eth_show_addr(struct device *dev, if (mac_dev) return sprintf(buf, "%llx", - (unsigned long long)mac_dev->vaddr); + (unsigned long long)mac_dev->res->start); else return sprintf(buf, "none"); } diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c index 7b7526fd7da32..65df308bad97e 100644 --- a/drivers/net/ethernet/freescale/fman/mac.c +++ b/drivers/net/ethernet/freescale/fman/mac.c @@ -279,7 +279,6 @@ static int mac_probe(struct platform_device *_of_dev) struct device_node *mac_node, *dev_node; struct mac_device *mac_dev; struct platform_device *of_dev; - struct resource *res; struct mac_priv_s *priv; struct fman_mac_params params; u32 val; @@ -338,24 +337,25 @@ static int mac_probe(struct platform_device *_of_dev) of_node_put(dev_node); /* Get the address of the memory mapped registers */ - res = platform_get_mem_or_io(_of_dev, 0); - if (!res) { + mac_dev->res = platform_get_mem_or_io(_of_dev, 0); + if (!mac_dev->res) { dev_err(dev, "could not get registers\n"); return -EINVAL; } - err = devm_request_resource(dev, fman_get_mem_region(priv->fman), res); + err = devm_request_resource(dev, fman_get_mem_region(priv->fman), + mac_dev->res); if (err) { dev_err_probe(dev, err, "could not request resource\n"); return err; } - mac_dev->vaddr = devm_ioremap(dev, res->start, resource_size(res)); + mac_dev->vaddr = devm_ioremap(dev, mac_dev->res->start, + resource_size(mac_dev->res)); if (!mac_dev->vaddr) { dev_err(dev, "devm_ioremap() failed\n"); return -EIO; } - mac_dev->vaddr_end = mac_dev->vaddr + resource_size(res); if (!of_device_is_available(mac_node)) return -ENODEV; diff --git a/drivers/net/ethernet/freescale/fman/mac.h b/drivers/net/ethernet/freescale/fman/mac.h index b95d384271bd6..13b69ca5f00c2 100644 --- a/drivers/net/ethernet/freescale/fman/mac.h +++ b/drivers/net/ethernet/freescale/fman/mac.h @@ -20,8 +20,8 @@ struct mac_priv_s; struct mac_device { void __iomem *vaddr; - void __iomem *vaddr_end; struct device *dev; + struct resource *res; u8 addr[ETH_ALEN]; struct fman_port *port[2]; u32 if_support; -- GitLab From 15e4dabda11b0fa31d510a915d1a580f47dfc92e Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 20 Oct 2022 22:45:11 +0000 Subject: [PATCH 0340/1927] kcm: annotate data-races around kcm->rx_psock kcm->rx_psock can be read locklessly in kcm_rfree(). Annotate the read and writes accordingly. We do the same for kcm->rx_wait in the following patch. syzbot reported: BUG: KCSAN: data-race in kcm_rfree / unreserve_rx_kcm write to 0xffff888123d827b8 of 8 bytes by task 2758 on cpu 1: unreserve_rx_kcm+0x72/0x1f0 net/kcm/kcmsock.c:313 kcm_rcv_strparser+0x2b5/0x3a0 net/kcm/kcmsock.c:373 __strp_recv+0x64c/0xd20 net/strparser/strparser.c:301 strp_recv+0x6d/0x80 net/strparser/strparser.c:335 tcp_read_sock+0x13e/0x5a0 net/ipv4/tcp.c:1703 strp_read_sock net/strparser/strparser.c:358 [inline] do_strp_work net/strparser/strparser.c:406 [inline] strp_work+0xe8/0x180 net/strparser/strparser.c:415 process_one_work+0x3d3/0x720 kernel/workqueue.c:2289 worker_thread+0x618/0xa70 kernel/workqueue.c:2436 kthread+0x1a9/0x1e0 kernel/kthread.c:376 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:306 read to 0xffff888123d827b8 of 8 bytes by task 5859 on cpu 0: kcm_rfree+0x14c/0x220 net/kcm/kcmsock.c:181 skb_release_head_state+0x8e/0x160 net/core/skbuff.c:841 skb_release_all net/core/skbuff.c:852 [inline] __kfree_skb net/core/skbuff.c:868 [inline] kfree_skb_reason+0x5c/0x260 net/core/skbuff.c:891 kfree_skb include/linux/skbuff.h:1216 [inline] kcm_recvmsg+0x226/0x2b0 net/kcm/kcmsock.c:1161 ____sys_recvmsg+0x16c/0x2e0 ___sys_recvmsg net/socket.c:2743 [inline] do_recvmmsg+0x2f1/0x710 net/socket.c:2837 __sys_recvmmsg net/socket.c:2916 [inline] __do_sys_recvmmsg net/socket.c:2939 [inline] __se_sys_recvmmsg net/socket.c:2932 [inline] __x64_sys_recvmmsg+0xde/0x160 net/socket.c:2932 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x2b/0x70 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd value changed: 0xffff88812971ce00 -> 0x0000000000000000 Reported by Kernel Concurrency Sanitizer on: CPU: 0 PID: 5859 Comm: syz-executor.3 Not tainted 6.0.0-syzkaller-12189-g19d17ab7c68b-dirty #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/22/2022 Fixes: ab7ac4eb9832 ("kcm: Kernel Connection Multiplexor module") Reported-by: syzbot Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- net/kcm/kcmsock.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 27725464ec08f..0109ef6ddf9ae 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -178,7 +178,7 @@ static void kcm_rfree(struct sk_buff *skb) /* For reading rx_wait and rx_psock without holding lock */ smp_mb__after_atomic(); - if (!kcm->rx_wait && !kcm->rx_psock && + if (!kcm->rx_wait && !READ_ONCE(kcm->rx_psock) && sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) { spin_lock_bh(&mux->rx_lock); kcm_rcv_ready(kcm); @@ -283,7 +283,8 @@ static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, kcm->rx_wait = false; psock->rx_kcm = kcm; - kcm->rx_psock = psock; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_psock, psock); spin_unlock_bh(&mux->rx_lock); @@ -310,7 +311,8 @@ static void unreserve_rx_kcm(struct kcm_psock *psock, spin_lock_bh(&mux->rx_lock); psock->rx_kcm = NULL; - kcm->rx_psock = NULL; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_psock, NULL); /* Commit kcm->rx_psock before sk_rmem_alloc_get to sync with * kcm_rfree -- GitLab From 0c745b5141a45a076f1cb9772a399f7ebcb0948a Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 20 Oct 2022 22:45:12 +0000 Subject: [PATCH 0341/1927] kcm: annotate data-races around kcm->rx_wait kcm->rx_psock can be read locklessly in kcm_rfree(). Annotate the read and writes accordingly. syzbot reported: BUG: KCSAN: data-race in kcm_rcv_strparser / kcm_rfree write to 0xffff88810784e3d0 of 1 bytes by task 1823 on cpu 1: reserve_rx_kcm net/kcm/kcmsock.c:283 [inline] kcm_rcv_strparser+0x250/0x3a0 net/kcm/kcmsock.c:363 __strp_recv+0x64c/0xd20 net/strparser/strparser.c:301 strp_recv+0x6d/0x80 net/strparser/strparser.c:335 tcp_read_sock+0x13e/0x5a0 net/ipv4/tcp.c:1703 strp_read_sock net/strparser/strparser.c:358 [inline] do_strp_work net/strparser/strparser.c:406 [inline] strp_work+0xe8/0x180 net/strparser/strparser.c:415 process_one_work+0x3d3/0x720 kernel/workqueue.c:2289 worker_thread+0x618/0xa70 kernel/workqueue.c:2436 kthread+0x1a9/0x1e0 kernel/kthread.c:376 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:306 read to 0xffff88810784e3d0 of 1 bytes by task 17869 on cpu 0: kcm_rfree+0x121/0x220 net/kcm/kcmsock.c:181 skb_release_head_state+0x8e/0x160 net/core/skbuff.c:841 skb_release_all net/core/skbuff.c:852 [inline] __kfree_skb net/core/skbuff.c:868 [inline] kfree_skb_reason+0x5c/0x260 net/core/skbuff.c:891 kfree_skb include/linux/skbuff.h:1216 [inline] kcm_recvmsg+0x226/0x2b0 net/kcm/kcmsock.c:1161 ____sys_recvmsg+0x16c/0x2e0 ___sys_recvmsg net/socket.c:2743 [inline] do_recvmmsg+0x2f1/0x710 net/socket.c:2837 __sys_recvmmsg net/socket.c:2916 [inline] __do_sys_recvmmsg net/socket.c:2939 [inline] __se_sys_recvmmsg net/socket.c:2932 [inline] __x64_sys_recvmmsg+0xde/0x160 net/socket.c:2932 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x2b/0x70 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd value changed: 0x01 -> 0x00 Reported by Kernel Concurrency Sanitizer on: CPU: 0 PID: 17869 Comm: syz-executor.2 Not tainted 6.1.0-rc1-syzkaller-00010-gbb1a1146467a-dirty #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/22/2022 Fixes: ab7ac4eb9832 ("kcm: Kernel Connection Multiplexor module") Reported-by: syzbot Signed-off-by: Eric Dumazet Signed-off-by: David S. Miller --- net/kcm/kcmsock.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 0109ef6ddf9ae..63e32f181f435 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -162,7 +162,8 @@ static void kcm_rcv_ready(struct kcm_sock *kcm) /* Buffer limit is okay now, add to ready list */ list_add_tail(&kcm->wait_rx_list, &kcm->mux->kcm_rx_waiters); - kcm->rx_wait = true; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_wait, true); } static void kcm_rfree(struct sk_buff *skb) @@ -178,7 +179,7 @@ static void kcm_rfree(struct sk_buff *skb) /* For reading rx_wait and rx_psock without holding lock */ smp_mb__after_atomic(); - if (!kcm->rx_wait && !READ_ONCE(kcm->rx_psock) && + if (!READ_ONCE(kcm->rx_wait) && !READ_ONCE(kcm->rx_psock) && sk_rmem_alloc_get(sk) < sk->sk_rcvlowat) { spin_lock_bh(&mux->rx_lock); kcm_rcv_ready(kcm); @@ -237,7 +238,8 @@ static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head) if (kcm_queue_rcv_skb(&kcm->sk, skb)) { /* Should mean socket buffer full */ list_del(&kcm->wait_rx_list); - kcm->rx_wait = false; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_wait, false); /* Commit rx_wait to read in kcm_free */ smp_wmb(); @@ -280,7 +282,8 @@ static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, kcm = list_first_entry(&mux->kcm_rx_waiters, struct kcm_sock, wait_rx_list); list_del(&kcm->wait_rx_list); - kcm->rx_wait = false; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_wait, false); psock->rx_kcm = kcm; /* paired with lockless reads in kcm_rfree() */ @@ -1242,7 +1245,8 @@ static void kcm_recv_disable(struct kcm_sock *kcm) if (!kcm->rx_psock) { if (kcm->rx_wait) { list_del(&kcm->wait_rx_list); - kcm->rx_wait = false; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_wait, false); } requeue_rx_msgs(mux, &kcm->sk.sk_receive_queue); @@ -1795,7 +1799,8 @@ static void kcm_done(struct kcm_sock *kcm) if (kcm->rx_wait) { list_del(&kcm->wait_rx_list); - kcm->rx_wait = false; + /* paired with lockless reads in kcm_rfree() */ + WRITE_ONCE(kcm->rx_wait, false); } /* Move any pending receive messages to other kcm sockets */ requeue_rx_msgs(mux, &sk->sk_receive_queue); -- GitLab From c5884ef477b405aadf49419a417d62dc8137e7f3 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 20 Oct 2022 11:30:31 -0700 Subject: [PATCH 0342/1927] docs: netdev: offer performance feedback to contributors Some of us gotten used to producing large quantities of peer feedback at work, every 3 or 6 months. Extending the same courtesy to community members seems like a logical step. It may be hard for some folks to get validation of how important their work is internally, especially at smaller companies which don't employ many kernel experts. The concept of "peer feedback" may be a hyperscaler / silicon valley thing so YMMV. Hopefully we can build more context as we go. Signed-off-by: Jakub Kicinski Signed-off-by: David S. Miller --- Documentation/process/maintainer-netdev.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/process/maintainer-netdev.rst b/Documentation/process/maintainer-netdev.rst index d140070815955..1fa5ab8754d35 100644 --- a/Documentation/process/maintainer-netdev.rst +++ b/Documentation/process/maintainer-netdev.rst @@ -319,3 +319,13 @@ unpatched tree to confirm infrastructure didn't mangle it. Finally, go back and read :ref:`Documentation/process/submitting-patches.rst ` to be sure you are not repeating some common mistake documented there. + +My company uses peer feedback in employee performance reviews. Can I ask netdev maintainers for feedback? +--------------------------------------------------------------------------------------------------------- + +Yes, especially if you spend significant amount of time reviewing code +and go out of your way to improve shared infrastructure. + +The feedback must be requested by you, the contributor, and will always +be shared with you (even if you request for it to be submitted to your +manager). -- GitLab From b75927cff13e0b3b652a12da7eb9a012911799e8 Mon Sep 17 00:00:00 2001 From: Michael Margolin Date: Thu, 20 Oct 2022 18:19:49 +0300 Subject: [PATCH 0343/1927] RDMA/efa: Add EFA 0xefa2 PCI ID Add support for 0xefa2 devices. Reviewed-by: Firas Jahjah Reviewed-by: Yossi Leybovich Signed-off-by: Michael Margolin Link: https://lore.kernel.org/r/20221020151949.1768-1-mrgolin@amazon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/efa/efa_main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index 94b94cca48709..15ee920811187 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause /* - * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved. */ #include @@ -14,10 +14,12 @@ #define PCI_DEV_ID_EFA0_VF 0xefa0 #define PCI_DEV_ID_EFA1_VF 0xefa1 +#define PCI_DEV_ID_EFA2_VF 0xefa2 static const struct pci_device_id efa_pci_tbl[] = { { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) }, { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) }, + { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA2_VF) }, { } }; -- GitLab From d266935ac43d57586e311a087510fe6a084af742 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Thu, 20 Oct 2022 10:42:13 +0800 Subject: [PATCH 0344/1927] net: fix UAF issue in nfqnl_nf_hook_drop() when ops_init() failed When the ops_init() interface is invoked to initialize the net, but ops->init() fails, data is released. However, the ptr pointer in net->gen is invalid. In this case, when nfqnl_nf_hook_drop() is invoked to release the net, invalid address access occurs. The process is as follows: setup_net() ops_init() data = kzalloc(...) ---> alloc "data" net_assign_generic() ---> assign "date" to ptr in net->gen ... ops->init() ---> failed ... kfree(data); ---> ptr in net->gen is invalid ... ops_exit_list() ... nfqnl_nf_hook_drop() *q = nfnl_queue_pernet(net) ---> q is invalid The following is the Call Trace information: BUG: KASAN: use-after-free in nfqnl_nf_hook_drop+0x264/0x280 Read of size 8 at addr ffff88810396b240 by task ip/15855 Call Trace: dump_stack_lvl+0x8e/0xd1 print_report+0x155/0x454 kasan_report+0xba/0x1f0 nfqnl_nf_hook_drop+0x264/0x280 nf_queue_nf_hook_drop+0x8b/0x1b0 __nf_unregister_net_hook+0x1ae/0x5a0 nf_unregister_net_hooks+0xde/0x130 ops_exit_list+0xb0/0x170 setup_net+0x7ac/0xbd0 copy_net_ns+0x2e6/0x6b0 create_new_namespaces+0x382/0xa50 unshare_nsproxy_namespaces+0xa6/0x1c0 ksys_unshare+0x3a4/0x7e0 __x64_sys_unshare+0x2d/0x40 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Allocated by task 15855: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 __kasan_kmalloc+0xa1/0xb0 __kmalloc+0x49/0xb0 ops_init+0xe7/0x410 setup_net+0x5aa/0xbd0 copy_net_ns+0x2e6/0x6b0 create_new_namespaces+0x382/0xa50 unshare_nsproxy_namespaces+0xa6/0x1c0 ksys_unshare+0x3a4/0x7e0 __x64_sys_unshare+0x2d/0x40 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Freed by task 15855: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_save_free_info+0x2a/0x40 ____kasan_slab_free+0x155/0x1b0 slab_free_freelist_hook+0x11b/0x220 __kmem_cache_free+0xa4/0x360 ops_init+0xb9/0x410 setup_net+0x5aa/0xbd0 copy_net_ns+0x2e6/0x6b0 create_new_namespaces+0x382/0xa50 unshare_nsproxy_namespaces+0xa6/0x1c0 ksys_unshare+0x3a4/0x7e0 __x64_sys_unshare+0x2d/0x40 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Fixes: f875bae06533 ("net: Automatically allocate per namespace data.") Signed-off-by: Zhengchao Shao Signed-off-by: David S. Miller --- net/core/net_namespace.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 0ec2f5906a27c..f64654df71a29 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -117,6 +117,7 @@ static int net_assign_generic(struct net *net, unsigned int id, void *data) static int ops_init(const struct pernet_operations *ops, struct net *net) { + struct net_generic *ng; int err = -ENOMEM; void *data = NULL; @@ -135,7 +136,13 @@ static int ops_init(const struct pernet_operations *ops, struct net *net) if (!err) return 0; + if (ops->id && ops->size) { cleanup: + ng = rcu_dereference_protected(net->gen, + lockdep_is_held(&pernet_ops_rwsem)); + ng->ptr[*ops->id] = NULL; + } + kfree(data); out: -- GitLab From 195583504be28df5d608a4677dd796117aea875f Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Tue, 18 Oct 2022 20:35:13 +0200 Subject: [PATCH 0345/1927] spi: stm32: Print summary 'callbacks suppressed' message The original fix "spi: stm32: Rate-limit the 'Communication suspended' message" still leads to "stm32h7_spi_irq_thread: 1696 callbacks suppressed" spew in the kernel log. Since this 'Communication suspended' message is a debug print, add RATELIMIT_MSG_ON_RELEASE flag to inhibit the "callbacks suspended" part during normal operation and only print summary at the end. Fixes: ea8be08cc9358 ("spi: stm32: Rate-limit the 'Communication suspended' message") Signed-off-by: Marek Vasut Link: https://lore.kernel.org/r/20221018183513.206706-1-marex@denx.de Signed-off-by: Mark Brown --- drivers/spi/spi-stm32.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 6fe617b445a59..3c2fa2e2f94a3 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -886,6 +886,7 @@ static irqreturn_t stm32h7_spi_irq_thread(int irq, void *dev_id) static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL * 10, 1); + ratelimit_set_flags(&rs, RATELIMIT_MSG_ON_RELEASE); if (__ratelimit(&rs)) dev_dbg_ratelimited(spi->dev, "Communication suspended\n"); if (!spi->cur_usedma && (spi->rx_buf && (spi->rx_len > 0))) -- GitLab From 9e272ed69ad6f6952fafd0599d6993575512408e Mon Sep 17 00:00:00 2001 From: Yangyang Li Date: Mon, 24 Oct 2022 16:38:13 +0800 Subject: [PATCH 0346/1927] RDMA/hns: Disable local invalidate operation When function reset and local invalidate are mixed, HNS RoCEE may hang. Before introducing the cause of the problem, two hardware internal concepts need to be introduced: 1. Execution queue: The queue of hardware execution instructions, function reset and local invalidate are queued for execution in this queue. 2.Local queue: A queue that stores local operation instructions. The instructions in the local queue will be sent to the execution queue for execution. The instructions in the local queue will not be removed until the execution is completed. The reason for the problem is as follows: 1. There is a function reset instruction in the execution queue, which is currently being executed. A necessary condition for the successful execution of function reset is: the hardware pipeline needs to empty the instructions that were not completed before; 2. A local invalidate instruction at the head of the local queue is sent to the execution queue. Now there are two instructions in the execution queue, the first is the function reset instruction, and the second is the local invalidate instruction, which will be executed in se quence; 3. The user has issued many local invalidate operations, causing the local queue to be filled up. 4. The user still has a new local operation command and is queuing to enter the local queue. But the local queue is full and cannot receive new instructions, this instruction is temporarily stored at the hardware pipeline. 5. The function reset has been waiting for the instruction before the hardware pipeline stage is drained. The hardware pipeline stage also caches a local invalidate instruction, so the function reset cannot be completed, and the instructions after it cannot be executed. These factors together cause the execution logic deadlock of the hardware, and the consequence is that RoCEE will not have any response. Considering that the local operation command may potentially cause RoCEE to hang, this feature is no longer supported. Fixes: e93df0108579 ("RDMA/hns: Support local invalidate for hip08 in kernel space") Signed-off-by: Yangyang Li Signed-off-by: Wenpeng Liang Signed-off-by: Haoyue Xu Link: https://lore.kernel.org/r/20221024083814.1089722-2-xuhaoyue1@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 11 ----------- drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 2 -- 2 files changed, 13 deletions(-) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 1ead35fb031b0..7f5a4769cee03 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -118,7 +118,6 @@ static const u32 hns_roce_op_code[] = { HR_OPC_MAP(ATOMIC_CMP_AND_SWP, ATOM_CMP_AND_SWAP), HR_OPC_MAP(ATOMIC_FETCH_AND_ADD, ATOM_FETCH_AND_ADD), HR_OPC_MAP(SEND_WITH_INV, SEND_WITH_INV), - HR_OPC_MAP(LOCAL_INV, LOCAL_INV), HR_OPC_MAP(MASKED_ATOMIC_CMP_AND_SWP, ATOM_MSK_CMP_AND_SWAP), HR_OPC_MAP(MASKED_ATOMIC_FETCH_AND_ADD, ATOM_MSK_FETCH_AND_ADD), HR_OPC_MAP(REG_MR, FAST_REG_PMR), @@ -559,9 +558,6 @@ static int set_rc_opcode(struct hns_roce_dev *hr_dev, else ret = -EOPNOTSUPP; break; - case IB_WR_LOCAL_INV: - hr_reg_enable(rc_sq_wqe, RC_SEND_WQE_SO); - fallthrough; case IB_WR_SEND_WITH_INV: rc_sq_wqe->inv_key = cpu_to_le32(wr->ex.invalidate_rkey); break; @@ -3222,7 +3218,6 @@ static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev, hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID); hr_reg_write(mpt_entry, MPT_PD, mr->pd); - hr_reg_enable(mpt_entry, MPT_L_INV_EN); hr_reg_write_bool(mpt_entry, MPT_BIND_EN, mr->access & IB_ACCESS_MW_BIND); @@ -3313,7 +3308,6 @@ static int hns_roce_v2_frmr_write_mtpt(struct hns_roce_dev *hr_dev, hr_reg_enable(mpt_entry, MPT_RA_EN); hr_reg_enable(mpt_entry, MPT_R_INV_EN); - hr_reg_enable(mpt_entry, MPT_L_INV_EN); hr_reg_enable(mpt_entry, MPT_FRE); hr_reg_clear(mpt_entry, MPT_MR_MW); @@ -3345,7 +3339,6 @@ static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw) hr_reg_write(mpt_entry, MPT_PD, mw->pdn); hr_reg_enable(mpt_entry, MPT_R_INV_EN); - hr_reg_enable(mpt_entry, MPT_L_INV_EN); hr_reg_enable(mpt_entry, MPT_LW_EN); hr_reg_enable(mpt_entry, MPT_MR_MW); @@ -3794,7 +3787,6 @@ static const u32 wc_send_op_map[] = { HR_WC_OP_MAP(RDMA_READ, RDMA_READ), HR_WC_OP_MAP(RDMA_WRITE, RDMA_WRITE), HR_WC_OP_MAP(RDMA_WRITE_WITH_IMM, RDMA_WRITE), - HR_WC_OP_MAP(LOCAL_INV, LOCAL_INV), HR_WC_OP_MAP(ATOM_CMP_AND_SWAP, COMP_SWAP), HR_WC_OP_MAP(ATOM_FETCH_AND_ADD, FETCH_ADD), HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP), @@ -3844,9 +3836,6 @@ static void fill_send_wc(struct ib_wc *wc, struct hns_roce_v2_cqe *cqe) case HNS_ROCE_V2_WQE_OP_RDMA_WRITE_WITH_IMM: wc->wc_flags |= IB_WC_WITH_IMM; break; - case HNS_ROCE_V2_WQE_OP_LOCAL_INV: - wc->wc_flags |= IB_WC_WITH_INVALIDATE; - break; case HNS_ROCE_V2_WQE_OP_ATOM_CMP_AND_SWAP: case HNS_ROCE_V2_WQE_OP_ATOM_FETCH_AND_ADD: case HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP: diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index b11579027e827..c7bf2d52c1cdb 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -179,7 +179,6 @@ enum { HNS_ROCE_V2_WQE_OP_ATOM_MSK_CMP_AND_SWAP = 0x8, HNS_ROCE_V2_WQE_OP_ATOM_MSK_FETCH_AND_ADD = 0x9, HNS_ROCE_V2_WQE_OP_FAST_REG_PMR = 0xa, - HNS_ROCE_V2_WQE_OP_LOCAL_INV = 0xb, HNS_ROCE_V2_WQE_OP_BIND_MW = 0xc, HNS_ROCE_V2_WQE_OP_MASK = 0x1f, }; @@ -915,7 +914,6 @@ struct hns_roce_v2_rc_send_wqe { #define RC_SEND_WQE_OWNER RC_SEND_WQE_FIELD_LOC(7, 7) #define RC_SEND_WQE_CQE RC_SEND_WQE_FIELD_LOC(8, 8) #define RC_SEND_WQE_FENCE RC_SEND_WQE_FIELD_LOC(9, 9) -#define RC_SEND_WQE_SO RC_SEND_WQE_FIELD_LOC(10, 10) #define RC_SEND_WQE_SE RC_SEND_WQE_FIELD_LOC(11, 11) #define RC_SEND_WQE_INLINE RC_SEND_WQE_FIELD_LOC(12, 12) #define RC_SEND_WQE_WQE_INDEX RC_SEND_WQE_FIELD_LOC(30, 15) -- GitLab From 12bcaf87d8b66d8cd812479c8a6349dcb245375c Mon Sep 17 00:00:00 2001 From: Yixing Liu Date: Mon, 24 Oct 2022 16:38:14 +0800 Subject: [PATCH 0347/1927] RDMA/hns: Fix NULL pointer problem in free_mr_init() Lock grab occurs in a concurrent scenario, resulting in stepping on a NULL pointer. It should be init mutex_init() first before use the lock. Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 Call trace: __mutex_lock.constprop.0+0xd0/0x5c0 __mutex_lock_slowpath+0x1c/0x2c mutex_lock+0x44/0x50 free_mr_send_cmd_to_hw+0x7c/0x1c0 [hns_roce_hw_v2] hns_roce_v2_dereg_mr+0x30/0x40 [hns_roce_hw_v2] hns_roce_dereg_mr+0x4c/0x130 [hns_roce_hw_v2] ib_dereg_mr_user+0x54/0x124 uverbs_free_mr+0x24/0x30 destroy_hw_idr_uobject+0x38/0x74 uverbs_destroy_uobject+0x48/0x1c4 uobj_destroy+0x74/0xcc ib_uverbs_cmd_verbs+0x368/0xbb0 ib_uverbs_ioctl+0xec/0x1a4 __arm64_sys_ioctl+0xb4/0x100 invoke_syscall+0x50/0x120 el0_svc_common.constprop.0+0x58/0x190 do_el0_svc+0x30/0x90 el0_svc+0x2c/0xb4 el0t_64_sync_handler+0x1a4/0x1b0 el0t_64_sync+0x19c/0x1a0 Fixes: 70f92521584f ("RDMA/hns: Use the reserved loopback QPs to free MR before destroying MPT") Signed-off-by: Yixing Liu Signed-off-by: Haoyue Xu Link: https://lore.kernel.org/r/20221024083814.1089722-3-xuhaoyue1@hisilicon.com Signed-off-by: Leon Romanovsky --- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index 7f5a4769cee03..1435fe2ea176f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -2801,8 +2801,12 @@ static int free_mr_modify_qp(struct hns_roce_dev *hr_dev) static int free_mr_init(struct hns_roce_dev *hr_dev) { + struct hns_roce_v2_priv *priv = hr_dev->priv; + struct hns_roce_v2_free_mr *free_mr = &priv->free_mr; int ret; + mutex_init(&free_mr->mutex); + ret = free_mr_alloc_res(hr_dev); if (ret) return ret; -- GitLab From 9c1eaa27ec599fcc25ed4970c0b73c247d147a2b Mon Sep 17 00:00:00 2001 From: Zhang Changzhong Date: Fri, 21 Oct 2022 09:32:24 +0800 Subject: [PATCH 0348/1927] net: lantiq_etop: don't free skb when returning NETDEV_TX_BUSY The ndo_start_xmit() method must not free skb when returning NETDEV_TX_BUSY, since caller is going to requeue freed skb. Fixes: 504d4721ee8e ("MIPS: Lantiq: Add ethernet driver") Signed-off-by: Zhang Changzhong Signed-off-by: David S. Miller --- drivers/net/ethernet/lantiq_etop.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/ethernet/lantiq_etop.c b/drivers/net/ethernet/lantiq_etop.c index 59aab4086dcce..f5961bdcc4809 100644 --- a/drivers/net/ethernet/lantiq_etop.c +++ b/drivers/net/ethernet/lantiq_etop.c @@ -485,7 +485,6 @@ ltq_etop_tx(struct sk_buff *skb, struct net_device *dev) len = skb->len < ETH_ZLEN ? ETH_ZLEN : skb->len; if ((desc->ctl & (LTQ_DMA_OWN | LTQ_DMA_C)) || ch->skb[ch->dma.desc]) { - dev_kfree_skb_any(skb); netdev_err(dev, "tx ring full\n"); netif_tx_stop_queue(txq); return NETDEV_TX_BUSY; -- GitLab From ec791d8149ff60c40ad2074af3b92a39c916a03f Mon Sep 17 00:00:00 2001 From: Lu Wei Date: Fri, 21 Oct 2022 12:06:22 +0800 Subject: [PATCH 0349/1927] tcp: fix a signed-integer-overflow bug in tcp_add_backlog() The type of sk_rcvbuf and sk_sndbuf in struct sock is int, and in tcp_add_backlog(), the variable limit is caculated by adding sk_rcvbuf, sk_sndbuf and 64 * 1024, it may exceed the max value of int and overflow. This patch reduces the limit budget by halving the sndbuf to solve this issue since ACK packets are much smaller than the payload. Fixes: c9c3321257e1 ("tcp: add tcp_add_backlog()") Signed-off-by: Lu Wei Reviewed-by: Eric Dumazet Acked-by: Kuniyuki Iwashima Signed-off-by: David S. Miller --- net/ipv4/tcp_ipv4.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 7a250ef9d1b7b..87d440f47a703 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1874,11 +1874,13 @@ bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb, __skb_push(skb, hdrlen); no_coalesce: + limit = (u32)READ_ONCE(sk->sk_rcvbuf) + (u32)(READ_ONCE(sk->sk_sndbuf) >> 1); + /* Only socket owner can try to collapse/prune rx queues * to reduce memory overhead, so add a little headroom here. * Few sockets backlog are possibly concurrently non empty. */ - limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf) + 64*1024; + limit += 64 * 1024; if (unlikely(sk_add_backlog(sk, skb, limit))) { bh_unlock_sock(sk); -- GitLab From f15fb2cd979a07fbfc666e2f04b8b30ec9233b2a Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 10 Oct 2022 18:36:06 +0800 Subject: [PATCH 0350/1927] btrfs: raid56: properly handle the error when unable to find the missing stripe In raid56_alloc_missing_rbio(), if we can not determine where the missing device is inside the full stripe, we just BUG_ON(). This is not necessary especially the only caller inside scrub.c is already properly checking the return value, and will treat it as a memory allocation failure. Fix the error handling by: - Add an extra warning for the reason Although personally speaking it may be better to be an ASSERT(). - Properly free the allocated rbio Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index f6395e8288d69..892005f756cfc 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -2742,8 +2742,10 @@ raid56_alloc_missing_rbio(struct bio *bio, struct btrfs_io_context *bioc) rbio->faila = find_logical_bio_stripe(rbio, bio); if (rbio->faila == -1) { - BUG(); - kfree(rbio); + btrfs_warn_rl(fs_info, + "can not determine the failed stripe number for full stripe %llu", + bioc->raid_map[0]); + __free_raid_bio(rbio); return NULL; } -- GitLab From ab4c54c643a01067669df8332b64e3f31b69e071 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 10 Oct 2022 18:36:07 +0800 Subject: [PATCH 0351/1927] btrfs: raid56: avoid double freeing for rbio if full_stripe_write() failed Currently if full_stripe_write() failed to allocate the pages for parity, it will call __free_raid_bio() first, then return -ENOMEM. But some caller of full_stripe_write() will also call __free_raid_bio() again, this would cause double freeing. And it's not a logically sound either, normally we should either free the memory at the same level where we allocated it, or let endio to handle everything. So this patch will solve the double freeing by make raid56_parity_write() to handle the error and free the rbio. Just like what we do in raid56_parity_recover(). Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/raid56.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 892005f756cfc..82c8e991300e3 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1632,10 +1632,8 @@ static int full_stripe_write(struct btrfs_raid_bio *rbio) int ret; ret = alloc_rbio_parity_pages(rbio); - if (ret) { - __free_raid_bio(rbio); + if (ret) return ret; - } ret = lock_stripe_add(rbio); if (ret == 0) @@ -1823,8 +1821,10 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) */ if (rbio_is_full(rbio)) { ret = full_stripe_write(rbio); - if (ret) + if (ret) { + __free_raid_bio(rbio); goto fail; + } return; } @@ -1838,8 +1838,10 @@ void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) list_add_tail(&rbio->plug_list, &plug->rbio_list); } else { ret = __raid56_parity_write(rbio); - if (ret) + if (ret) { + __free_raid_bio(rbio); goto fail; + } } return; -- GitLab From ae0e5df4d1a4a2694c9c203cc25334aaaf9f2dfa Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 11 Oct 2022 12:02:31 +0200 Subject: [PATCH 0352/1927] btrfs: reorder btrfs_bio for better packing After changes in commit 917f32a23501 ("btrfs: give struct btrfs_bio a real end_io handler") the layout of btrfs_bio can be improved. There are two holes and the structure size is 264 bytes on release build. By reordering the iterator we can get rid of the holes and the size is 256 bytes which fits to slabs much better. Final layout: struct btrfs_bio { unsigned int mirror_num; /* 0 4 */ struct bvec_iter iter; /* 4 20 */ u64 file_offset; /* 24 8 */ struct btrfs_device * device; /* 32 8 */ u8 * csum; /* 40 8 */ u8 csum_inline[64]; /* 48 64 */ /* --- cacheline 1 boundary (64 bytes) was 48 bytes ago --- */ btrfs_bio_end_io_t end_io; /* 112 8 */ void * private; /* 120 8 */ /* --- cacheline 2 boundary (128 bytes) --- */ struct work_struct end_io_work; /* 128 32 */ struct bio bio; /* 160 96 */ /* size: 256, cachelines: 4, members: 10 */ }; Fixes: 917f32a23501 ("btrfs: give struct btrfs_bio a real end_io handler") Signed-off-by: David Sterba --- fs/btrfs/volumes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 599b9d5af349f..f8b668dc8bf81 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -395,6 +395,7 @@ typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); */ struct btrfs_bio { unsigned int mirror_num; + struct bvec_iter iter; /* for direct I/O */ u64 file_offset; @@ -403,7 +404,6 @@ struct btrfs_bio { struct btrfs_device *device; u8 *csum; u8 csum_inline[BTRFS_BIO_INLINE_CSUM_SIZE]; - struct bvec_iter iter; /* End I/O information supplied to btrfs_bio_alloc */ btrfs_bio_end_io_t end_io; -- GitLab From 968b71583130b6104c9f33ba60446d598e327a8b Mon Sep 17 00:00:00 2001 From: Josef Bacik Date: Fri, 14 Oct 2022 08:52:46 -0400 Subject: [PATCH 0353/1927] btrfs: fix tree mod log mishandling of reallocated nodes We have been seeing the following panic in production kernel BUG at fs/btrfs/tree-mod-log.c:677! invalid opcode: 0000 [#1] SMP RIP: 0010:tree_mod_log_rewind+0x1b4/0x200 RSP: 0000:ffffc9002c02f890 EFLAGS: 00010293 RAX: 0000000000000003 RBX: ffff8882b448c700 RCX: 0000000000000000 RDX: 0000000000008000 RSI: 00000000000000a7 RDI: ffff88877d831c00 RBP: 0000000000000002 R08: 000000000000009f R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000100c40 R12: 0000000000000001 R13: ffff8886c26d6a00 R14: ffff88829f5424f8 R15: ffff88877d831a00 FS: 00007fee1d80c780(0000) GS:ffff8890400c0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00007fee1963a020 CR3: 0000000434f33002 CR4: 00000000007706e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Call Trace: btrfs_get_old_root+0x12b/0x420 btrfs_search_old_slot+0x64/0x2f0 ? tree_mod_log_oldest_root+0x3d/0xf0 resolve_indirect_ref+0xfd/0x660 ? ulist_alloc+0x31/0x60 ? kmem_cache_alloc_trace+0x114/0x2c0 find_parent_nodes+0x97a/0x17e0 ? ulist_alloc+0x30/0x60 btrfs_find_all_roots_safe+0x97/0x150 iterate_extent_inodes+0x154/0x370 ? btrfs_search_path_in_tree+0x240/0x240 iterate_inodes_from_logical+0x98/0xd0 ? btrfs_search_path_in_tree+0x240/0x240 btrfs_ioctl_logical_to_ino+0xd9/0x180 btrfs_ioctl+0xe2/0x2ec0 ? __mod_memcg_lruvec_state+0x3d/0x280 ? do_sys_openat2+0x6d/0x140 ? kretprobe_dispatcher+0x47/0x70 ? kretprobe_rethook_handler+0x38/0x50 ? rethook_trampoline_handler+0x82/0x140 ? arch_rethook_trampoline_callback+0x3b/0x50 ? kmem_cache_free+0xfb/0x270 ? do_sys_openat2+0xd5/0x140 __x64_sys_ioctl+0x71/0xb0 do_syscall_64+0x2d/0x40 Which is this code in tree_mod_log_rewind() switch (tm->op) { case BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING: BUG_ON(tm->slot < n); This occurs because we replay the nodes in order that they happened, and when we do a REPLACE we will log a REMOVE_WHILE_FREEING for every slot, starting at 0. 'n' here is the number of items in this block, which in this case was 1, but we had 2 REMOVE_WHILE_FREEING operations. The actual root cause of this was that we were replaying operations for a block that shouldn't have been replayed. Consider the following sequence of events 1. We have an already modified root, and we do a btrfs_get_tree_mod_seq(). 2. We begin removing items from this root, triggering KEY_REPLACE for it's child slots. 3. We remove one of the 2 children this root node points to, thus triggering the root node promotion of the remaining child, and freeing this node. 4. We modify a new root, and re-allocate the above node to the root node of this other root. The tree mod log looks something like this logical 0 op KEY_REPLACE (slot 1) seq 2 logical 0 op KEY_REMOVE (slot 1) seq 3 logical 0 op KEY_REMOVE_WHILE_FREEING (slot 0) seq 4 logical 4096 op LOG_ROOT_REPLACE (old logical 0) seq 5 logical 8192 op KEY_REMOVE_WHILE_FREEING (slot 1) seq 6 logical 8192 op KEY_REMOVE_WHILE_FREEING (slot 0) seq 7 logical 0 op LOG_ROOT_REPLACE (old logical 8192) seq 8 >From here the bug is triggered by the following steps 1. Call btrfs_get_old_root() on the new_root. 2. We call tree_mod_log_oldest_root(btrfs_root_node(new_root)), which is currently logical 0. 3. tree_mod_log_oldest_root() calls tree_mod_log_search_oldest(), which gives us the KEY_REPLACE seq 2, and since that's not a LOG_ROOT_REPLACE we incorrectly believe that we don't have an old root, because we expect that the most recent change should be a LOG_ROOT_REPLACE. 4. Back in tree_mod_log_oldest_root() we don't have a LOG_ROOT_REPLACE, so we don't set old_root, we simply use our existing extent buffer. 5. Since we're using our existing extent buffer (logical 0) we call tree_mod_log_search(0) in order to get the newest change to start the rewind from, which ends up being the LOG_ROOT_REPLACE at seq 8. 6. Again since we didn't find an old_root we simply clone logical 0 at it's current state. 7. We call tree_mod_log_rewind() with the cloned extent buffer. 8. Set n = btrfs_header_nritems(logical 0), which would be whatever the original nritems was when we COWed the original root, say for this example it's 2. 9. We start from the newest operation and work our way forward, so we see LOG_ROOT_REPLACE which we ignore. 10. Next we see KEY_REMOVE_WHILE_FREEING for slot 0, which triggers the BUG_ON(tm->slot < n), because it expects if we've done this we have a completely empty extent buffer to replay completely. The correct thing would be to find the first LOG_ROOT_REPLACE, and then get the old_root set to logical 8192. In fact making that change fixes this particular problem. However consider the much more complicated case. We have a child node in this tree and the above situation. In the above case we freed one of the child blocks at the seq 3 operation. If this block was also re-allocated and got new tree mod log operations we would have a different problem. btrfs_search_old_slot(orig root) would get down to the logical 0 root that still pointed at that node. However in btrfs_search_old_slot() we call tree_mod_log_rewind(buf) directly. This is not context aware enough to know which operations we should be replaying. If the block was re-allocated multiple times we may only want to replay a range of operations, and determining what that range is isn't possible to determine. We could maybe solve this by keeping track of which root the node belonged to at every tree mod log operation, and then passing this around to make sure we're only replaying operations that relate to the root we're trying to rewind. However there's a simpler way to solve this problem, simply disallow reallocations if we have currently running tree mod log users. We already do this for leaf's, so we're simply expanding this to nodes as well. This is a relatively uncommon occurrence, and the problem is complicated enough I'm worried that we will still have corner cases in the reallocation case. So fix this in the most straightforward way possible. Fixes: bd989ba359f2 ("Btrfs: add tree modification log functions") CC: stable@vger.kernel.org # 3.3+ Reviewed-by: Filipe Manana Signed-off-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/extent-tree.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index cd2d36580f1ac..2801c991814f5 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -3295,21 +3295,22 @@ void btrfs_free_tree_block(struct btrfs_trans_handle *trans, } /* - * If this is a leaf and there are tree mod log users, we may - * have recorded mod log operations that point to this leaf. - * So we must make sure no one reuses this leaf's extent before - * mod log operations are applied to a node, otherwise after - * rewinding a node using the mod log operations we get an - * inconsistent btree, as the leaf's extent may now be used as - * a node or leaf for another different btree. + * If there are tree mod log users we may have recorded mod log + * operations for this node. If we re-allocate this node we + * could replay operations on this node that happened when it + * existed in a completely different root. For example if it + * was part of root A, then was reallocated to root B, and we + * are doing a btrfs_old_search_slot(root b), we could replay + * operations that happened when the block was part of root A, + * giving us an inconsistent view of the btree. + * * We are safe from races here because at this point no other * node or root points to this extent buffer, so if after this - * check a new tree mod log user joins, it will not be able to - * find a node pointing to this leaf and record operations that - * point to this leaf. + * check a new tree mod log user joins we will not have an + * existing log of operations on this node that we have to + * contend with. */ - if (btrfs_header_level(buf) == 0 && - test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags)) + if (test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags)) must_pin = true; if (must_pin || btrfs_is_zoned(fs_info)) { -- GitLab From 3d17adea74a56a4965f7a603d8ed8c66bb9356d9 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Tue, 18 Oct 2022 09:56:38 +0800 Subject: [PATCH 0354/1927] btrfs: make thaw time super block check to also verify checksum Previous commit a05d3c915314 ("btrfs: check superblock to ensure the fs was not modified at thaw time") only checks the content of the super block, but it doesn't really check if the on-disk super block has a matching checksum. This patch will add the checksum verification to thaw time superblock verification. This involves the following extra changes: - Export btrfs_check_super_csum() As we need to call it in super.c. - Change the argument list of btrfs_check_super_csum() Instead of passing a char *, directly pass struct btrfs_super_block * pointer. - Verify that our checksum type didn't change before checking the checksum value, like it's done at mount time Fixes: a05d3c915314 ("btrfs: check superblock to ensure the fs was not modified at thaw time") Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 10 ++++------ fs/btrfs/disk-io.h | 2 ++ fs/btrfs/super.c | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index a2da9313c6947..4b28263c3d329 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -166,11 +166,9 @@ static bool btrfs_supported_super_csum(u16 csum_type) * Return 0 if the superblock checksum type matches the checksum value of that * algorithm. Pass the raw disk superblock data. */ -static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, - char *raw_disk_sb) +int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *disk_sb) { - struct btrfs_super_block *disk_sb = - (struct btrfs_super_block *)raw_disk_sb; char result[BTRFS_CSUM_SIZE]; SHASH_DESC_ON_STACK(shash, fs_info->csum_shash); @@ -181,7 +179,7 @@ static int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is * filled with zeros and is included in the checksum. */ - crypto_shash_digest(shash, raw_disk_sb + BTRFS_CSUM_SIZE, + crypto_shash_digest(shash, (const u8 *)disk_sb + BTRFS_CSUM_SIZE, BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result); if (memcmp(disk_sb->csum, result, fs_info->csum_size)) @@ -3479,7 +3477,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device * We want to check superblock checksum, the type is stored inside. * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k). */ - if (btrfs_check_super_csum(fs_info, (u8 *)disk_super)) { + if (btrfs_check_super_csum(fs_info, disk_super)) { btrfs_err(fs_info, "superblock checksum mismatch"); err = -EINVAL; btrfs_release_disk_super(disk_super); diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h index c67c15d4d20be..9fa923e005a3a 100644 --- a/fs/btrfs/disk-io.h +++ b/fs/btrfs/disk-io.h @@ -42,6 +42,8 @@ struct extent_buffer *btrfs_find_create_tree_block( void btrfs_clean_tree_block(struct extent_buffer *buf); void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info); int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info); +int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *disk_sb); int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices, char *options); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 9be4fd2db0f44..5942b93840884 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -2555,6 +2555,7 @@ static int check_dev_super(struct btrfs_device *dev) { struct btrfs_fs_info *fs_info = dev->fs_info; struct btrfs_super_block *sb; + u16 csum_type; int ret = 0; /* This should be called with fs still frozen. */ @@ -2569,6 +2570,21 @@ static int check_dev_super(struct btrfs_device *dev) if (IS_ERR(sb)) return PTR_ERR(sb); + /* Verify the checksum. */ + csum_type = btrfs_super_csum_type(sb); + if (csum_type != btrfs_super_csum_type(fs_info->super_copy)) { + btrfs_err(fs_info, "csum type changed, has %u expect %u", + csum_type, btrfs_super_csum_type(fs_info->super_copy)); + ret = -EUCLEAN; + goto out; + } + + if (btrfs_check_super_csum(fs_info, sb)) { + btrfs_err(fs_info, "csum for on-disk super block no longer matches"); + ret = -EUCLEAN; + goto out; + } + /* Btrfs_validate_super() includes fsid check against super->fsid. */ ret = btrfs_validate_super(fs_info, sb, 0); if (ret < 0) -- GitLab From 9b8be45f1ef29081c4b614aa559f934526e70d16 Mon Sep 17 00:00:00 2001 From: BingJing Chang Date: Sun, 16 Oct 2022 23:33:46 +0800 Subject: [PATCH 0355/1927] btrfs: send: fix send failure of a subcase of orphan inodes Commit 9ed0a72e5b35 ("btrfs: send: fix failures when processing inodes with no links") tries to fix all incremental send cases of orphan inodes the send operation will meet. However, there's still a bug causing the corner subcase fails with a ENOENT error. Here's shortened steps of that subcase: $ btrfs subvolume create vol $ touch vol/foo $ btrfs subvolume snapshot -r vol snap1 $ btrfs subvolume snapshot -r vol snap2 # Turn the second snapshot to RW mode and delete the file while # holding an open file descriptor on it $ btrfs property set snap2 ro false $ exec 73 /dev/null At subvol snap1 ERROR: send ioctl failed with -2: No such file or directory It's subcase 3 of BTRFS_COMPARE_TREE_CHANGED in the commit 9ed0a72e5b35 ("btrfs: send: fix failures when processing inodes with no links"). And it's not a common case. We still have not met it in the real world. Theoretically, this case can happen in a batch cascading snapshot backup. In cascading backups, the receive operation in the middle may cause orphan inodes to appear because of the open file descriptors on the snapshot files during receiving. And if we don't do the batch snapshot backups in their creation order, then we can have an inode, which is an orphan in the parent snapshot but refers to a file in the send snapshot. Since an orphan inode has no paths, the send operation will fail with a ENOENT error if it tries to generate a path for it. In that patch, this subcase will be treated as an inode with a new generation. However, when the routine tries to delete the old paths in the parent snapshot, the function process_all_refs() doesn't check whether there are paths recorded or not before it calls the function process_recorded_refs(). And the function process_recorded_refs() try to get the first path in the parent snapshot in the beginning. Since it has no paths in the parent snapshot, the send operation fails. To fix this, we can easily put a link count check to avoid entering the deletion routine like what we do a link count check to avoid creating a new one. Moreover, we can assume that the function process_all_refs() can always collect references to process because we know it has a positive link count. Fixes: 9ed0a72e5b35 ("btrfs: send: fix failures when processing inodes with no links") Reviewed-by: Filipe Manana Signed-off-by: BingJing Chang Signed-off-by: David Sterba --- fs/btrfs/send.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index ec6e1752af2ca..145c84b44fd0b 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -6668,17 +6668,19 @@ static int changed_inode(struct send_ctx *sctx, /* * First, process the inode as if it was deleted. */ - sctx->cur_inode_gen = right_gen; - sctx->cur_inode_new = false; - sctx->cur_inode_deleted = true; - sctx->cur_inode_size = btrfs_inode_size( - sctx->right_path->nodes[0], right_ii); - sctx->cur_inode_mode = btrfs_inode_mode( - sctx->right_path->nodes[0], right_ii); - ret = process_all_refs(sctx, - BTRFS_COMPARE_TREE_DELETED); - if (ret < 0) - goto out; + if (old_nlinks > 0) { + sctx->cur_inode_gen = right_gen; + sctx->cur_inode_new = false; + sctx->cur_inode_deleted = true; + sctx->cur_inode_size = btrfs_inode_size( + sctx->right_path->nodes[0], right_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->right_path->nodes[0], right_ii); + ret = process_all_refs(sctx, + BTRFS_COMPARE_TREE_DELETED); + if (ret < 0) + goto out; + } /* * Now process the inode as if it was new. -- GitLab From 2398091f9c2c8e0040f4f9928666787a3e8108a7 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Tue, 18 Oct 2022 16:05:52 +0200 Subject: [PATCH 0356/1927] btrfs: fix type of parameter generation in btrfs_get_dentry The type of parameter generation has been u32 since the beginning, however all callers pass a u64 generation, so unify the types to prevent potential loss. CC: stable@vger.kernel.org # 4.9+ Reviewed-by: Josef Bacik Signed-off-by: David Sterba --- fs/btrfs/export.c | 2 +- fs/btrfs/export.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c index 1d4c2397d0d62..fab7eb76e53b2 100644 --- a/fs/btrfs/export.c +++ b/fs/btrfs/export.c @@ -58,7 +58,7 @@ static int btrfs_encode_fh(struct inode *inode, u32 *fh, int *max_len, } struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, - u64 root_objectid, u32 generation, + u64 root_objectid, u64 generation, int check_generation) { struct btrfs_fs_info *fs_info = btrfs_sb(sb); diff --git a/fs/btrfs/export.h b/fs/btrfs/export.h index f32f4113c976a..5afb7ca428289 100644 --- a/fs/btrfs/export.h +++ b/fs/btrfs/export.h @@ -19,7 +19,7 @@ struct btrfs_fid { } __attribute__ ((packed)); struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, - u64 root_objectid, u32 generation, + u64 root_objectid, u64 generation, int check_generation); struct dentry *btrfs_get_parent(struct dentry *child); -- GitLab From e9cf4d9b9a6fdb1df6401a59f5ac5d24006bfeae Mon Sep 17 00:00:00 2001 From: Dmitry Osipenko Date: Mon, 24 Oct 2022 17:12:10 +0300 Subject: [PATCH 0357/1927] ACPI: video: Fix missing native backlight on Chromebooks Chromebooks don't have backlight in ACPI table, they suppose to use native backlight in this case. Check presence of the CrOS embedded controller ACPI device and prefer the native backlight if EC found. Suggested-by: Hans de Goede Fixes: 2600bfa3df99 ("ACPI: video: Add acpi_video_backlight_use_native() helper") Signed-off-by: Dmitry Osipenko Reviewed-by: Hans de Goede Acked-by: Rafael J. Wysocki Link: https://lore.kernel.org/r/20221024141210.67784-1-dmitry.osipenko@collabora.com Signed-off-by: Hans de Goede --- drivers/acpi/video_detect.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 0d9064a9804cf..9cd8797d12bb4 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -668,6 +668,11 @@ static const struct dmi_system_id video_detect_dmi_table[] = { { }, }; +static bool google_cros_ec_present(void) +{ + return acpi_dev_found("GOOG0004"); +} + /* * Determine which type of backlight interface to use on this system, * First check cmdline, then dmi quirks, then do autodetect. @@ -730,6 +735,13 @@ static enum acpi_backlight_type __acpi_video_get_backlight_type(bool native) return acpi_backlight_video; } + /* + * Chromebooks that don't have backlight handle in ACPI table + * are supposed to use native backlight if it's available. + */ + if (google_cros_ec_present() && native_available) + return acpi_backlight_native; + /* No ACPI video (old hw), use vendor specific fw methods. */ return acpi_backlight_vendor; } -- GitLab From 9440c42941606af4c379afa3cf8624f0dc43a629 Mon Sep 17 00:00:00 2001 From: Jiri Olsa Date: Tue, 18 Oct 2022 14:27:08 +0200 Subject: [PATCH 0358/1927] x86/syscall: Include asm/ptrace.h in syscall_wrapper header With just the forward declaration of the 'struct pt_regs' in syscall_wrapper.h, the syscall stub functions: __[x64|ia32]_sys_*(struct pt_regs *regs) will have different definition of 'regs' argument in BTF data based on which object file they are defined in. If the syscall's object includes 'struct pt_regs' definition, the BTF argument data will point to a 'struct pt_regs' record, like: [226] STRUCT 'pt_regs' size=168 vlen=21 'r15' type_id=1 bits_offset=0 'r14' type_id=1 bits_offset=64 'r13' type_id=1 bits_offset=128 ... If not, it will point to a fwd declaration record: [15439] FWD 'pt_regs' fwd_kind=struct and make bpf tracing program hooking on those functions unable to access fields from 'struct pt_regs'. Include asm/ptrace.h directly in syscall_wrapper.h to make sure all syscalls see 'struct pt_regs' definition. This then results in BTF for '__*_sys_*(struct pt_regs *regs)' functions to point to the actual struct, not just the forward declaration. [ bp: No Fixes tag as this is not really a bug fix but "adjustment" so that BTF is happy. ] Reported-by: Akihiro HARAI Signed-off-by: Jiri Olsa Signed-off-by: Borislav Petkov Acked-by: Andrii Nakryiko Cc: # this is needed only for BTF so kernels >= 5.15 Link: https://lore.kernel.org/r/20221018122708.823792-1-jolsa@kernel.org --- arch/x86/include/asm/syscall_wrapper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/syscall_wrapper.h b/arch/x86/include/asm/syscall_wrapper.h index 59358d1bf8800..fd2669b1cb2d9 100644 --- a/arch/x86/include/asm/syscall_wrapper.h +++ b/arch/x86/include/asm/syscall_wrapper.h @@ -6,7 +6,7 @@ #ifndef _ASM_X86_SYSCALL_WRAPPER_H #define _ASM_X86_SYSCALL_WRAPPER_H -struct pt_regs; +#include extern long __x64_sys_ni_syscall(const struct pt_regs *regs); extern long __ia32_sys_ni_syscall(const struct pt_regs *regs); -- GitLab From 2b4337c8409b4e9e5aed15c597e4031dd567bdd8 Mon Sep 17 00:00:00 2001 From: Mihai Sain Date: Mon, 17 Oct 2022 11:31:19 +0300 Subject: [PATCH 0359/1927] ARM: dts: at91: sama7g5: fix signal name of pin PB2 The signal name of pin PB2 with function F is FLEXCOM11_IO1 as it is defined in the datasheet. Fixes: 7540629e2fc7 ("ARM: dts: at91: add sama7g5 SoC DT and sama7g5-ek") Signed-off-by: Mihai Sain Reviewed-by: Tudor Ambarus Acked-by: Nicolas Ferre Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20221017083119.1643-1-mihai.sain@microchip.com --- arch/arm/boot/dts/sama7g5-pinfunc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/boot/dts/sama7g5-pinfunc.h b/arch/arm/boot/dts/sama7g5-pinfunc.h index 4eb30445d2057..6e87f0d4b8fce 100644 --- a/arch/arm/boot/dts/sama7g5-pinfunc.h +++ b/arch/arm/boot/dts/sama7g5-pinfunc.h @@ -261,7 +261,7 @@ #define PIN_PB2__FLEXCOM6_IO0 PINMUX_PIN(PIN_PB2, 2, 1) #define PIN_PB2__ADTRG PINMUX_PIN(PIN_PB2, 3, 1) #define PIN_PB2__A20 PINMUX_PIN(PIN_PB2, 4, 1) -#define PIN_PB2__FLEXCOM11_IO0 PINMUX_PIN(PIN_PB2, 6, 3) +#define PIN_PB2__FLEXCOM11_IO1 PINMUX_PIN(PIN_PB2, 6, 3) #define PIN_PB3 35 #define PIN_PB3__GPIO PINMUX_PIN(PIN_PB3, 0, 0) #define PIN_PB3__RF1 PINMUX_PIN(PIN_PB3, 1, 1) -- GitLab From 028822b714bd3a159d65416c53f1549345b53d9e Mon Sep 17 00:00:00 2001 From: Vincent Whitchurch Date: Thu, 20 Oct 2022 15:01:23 +0200 Subject: [PATCH 0360/1927] mmc: core: Fix WRITE_ZEROES CQE handling WRITE_ZEROES requests use TRIM, so mark them as needing to be issued synchronously even when a CQE is being used. Without this, mmc_blk_mq_issue_rq() triggers a WARN_ON_ONCE() and fails the request since we don't have any handling for issuing this asynchronously. Fixes: f7b6fc327327 ("mmc: core: Support zeroout using TRIM for eMMC") Reported-by: Jon Hunter Tested-by: Jon Hunter Signed-off-by: Vincent Whitchurch Reviewed-by: Avri Altman Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221020130123.4033218-1-vincent.whitchurch@axis.com Signed-off-by: Ulf Hansson --- drivers/mmc/core/queue.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index 86be55d7cf556..b396e39007177 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -48,6 +48,7 @@ static enum mmc_issue_type mmc_cqe_issue_type(struct mmc_host *host, case REQ_OP_DRV_OUT: case REQ_OP_DISCARD: case REQ_OP_SECURE_ERASE: + case REQ_OP_WRITE_ZEROES: return MMC_ISSUE_SYNC; case REQ_OP_FLUSH: return mmc_cqe_can_dcmd(host) ? MMC_ISSUE_DCMD : MMC_ISSUE_SYNC; -- GitLab From ea522496afa1dd4ed295466e9c813b88ebda3284 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Fri, 14 Oct 2022 10:10:40 -0700 Subject: [PATCH 0361/1927] Documentation: process: replace outdated LTS table w/ link The existing table was a bit outdated. 3.16 was EOL in 2020. 4.4 was EOL in 2022. 5.10 is new in 2020. 5.15 is new in 2021. We'll see if 6.1 becomes LTS in 2022. Rather than keep this table updated, it does duplicate information from multiple kernel.org pages. Make one less duplication site that needs to be updated and simply refer to the kernel.org page on releases. Suggested-by: Tyler Hicks Suggested-by: Bagas Sanjaya Signed-off-by: Nick Desaulniers Signed-off-by: Greg Kroah-Hartman Link: https://lore.kernel.org/stable/20221014171040.849726-1-ndesaulniers%40google.com Reviewed-by: Tyler Hicks (Microsoft) Link: https://lore.kernel.org/r/20221014171040.849726-1-ndesaulniers@google.com Signed-off-by: Jonathan Corbet --- Documentation/process/2.Process.rst | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Documentation/process/2.Process.rst b/Documentation/process/2.Process.rst index e05fb1b8f8b6b..6a919cffcbfd7 100644 --- a/Documentation/process/2.Process.rst +++ b/Documentation/process/2.Process.rst @@ -126,17 +126,10 @@ than one development cycle past their initial release. So, for example, the 5.2.21 was the final stable update of the 5.2 release. Some kernels are designated "long term" kernels; they will receive support -for a longer period. As of this writing, the current long term kernels -and their maintainers are: - - ====== ================================ ======================= - 3.16 Ben Hutchings (very long-term kernel) - 4.4 Greg Kroah-Hartman & Sasha Levin (very long-term kernel) - 4.9 Greg Kroah-Hartman & Sasha Levin - 4.14 Greg Kroah-Hartman & Sasha Levin - 4.19 Greg Kroah-Hartman & Sasha Levin - 5.4 Greg Kroah-Hartman & Sasha Levin - ====== ================================ ======================= +for a longer period. Please refer to the following link for the list of active +long term kernel versions and their maintainers: + + https://www.kernel.org/category/releases.html The selection of a kernel for long-term support is purely a matter of a maintainer having the need and the time to maintain that release. There -- GitLab From e648174b53f1e29ee72ef33756a97ffb8241b6a5 Mon Sep 17 00:00:00 2001 From: Mushahid Hussain Date: Mon, 17 Oct 2022 16:20:26 +0500 Subject: [PATCH 0362/1927] Documentation: Fix spelling mistake in hacking.rst Fix `botton half locks` to `bottom half locks`. Signed-off-by: Mushahid Hussain Link: https://lore.kernel.org/r/20221017112026.88324-1-mushi.shar@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/kernel-hacking/hacking.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst index 9a1f020c84498..1717348a4404e 100644 --- a/Documentation/kernel-hacking/hacking.rst +++ b/Documentation/kernel-hacking/hacking.rst @@ -120,7 +120,7 @@ You can tell you are in a softirq (or tasklet) using the .. warning:: Beware that this will return a false positive if a - :ref:`botton half lock ` is held. + :ref:`bottom half lock ` is held. Some Basic Rules ================ -- GitLab From 2f3f53d62307262f0086804ea7cea99b0e085450 Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Sat, 15 Oct 2022 18:22:01 +0900 Subject: [PATCH 0363/1927] docs/process/howto: Replace C89 with C11 Commit e8c07082a810 ("Kbuild: move to -std=gnu11") updated process/programming-language.rst, but failed to update process/howto.rst. Update howto.rst and resolve the inconsistency. Fixes: e8c07082a810 ("Kbuild: move to -std=gnu11") Signed-off-by: Akira Yokosawa Cc: Arnd Bergmann Cc: Federico Vaga Cc: Alex Shi Cc: Hu Haowen Cc: Tsugikazu Shibata Link: https://lore.kernel.org/r/20221015092201.32099-1-akiyks@gmail.com Signed-off-by: Jonathan Corbet --- Documentation/process/howto.rst | 2 +- Documentation/translations/it_IT/process/howto.rst | 2 +- Documentation/translations/ja_JP/howto.rst | 2 +- Documentation/translations/ko_KR/howto.rst | 2 +- Documentation/translations/zh_CN/process/howto.rst | 2 +- Documentation/translations/zh_TW/process/howto.rst | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/process/howto.rst b/Documentation/process/howto.rst index bd15c393ba3cd..cb6abcb2b6d07 100644 --- a/Documentation/process/howto.rst +++ b/Documentation/process/howto.rst @@ -36,7 +36,7 @@ experience, the following books are good for, if anything, reference: - "C: A Reference Manual" by Harbison and Steele [Prentice Hall] The kernel is written using GNU C and the GNU toolchain. While it -adheres to the ISO C89 standard, it uses a number of extensions that are +adheres to the ISO C11 standard, it uses a number of extensions that are not featured in the standard. The kernel is a freestanding C environment, with no reliance on the standard C library, so some portions of the C standard are not supported. Arbitrary long long diff --git a/Documentation/translations/it_IT/process/howto.rst b/Documentation/translations/it_IT/process/howto.rst index 15c08aea1dfea..052f1b3610cb1 100644 --- a/Documentation/translations/it_IT/process/howto.rst +++ b/Documentation/translations/it_IT/process/howto.rst @@ -44,7 +44,7 @@ altro, utili riferimenti: - "C: A Reference Manual" di Harbison and Steele [Prentice Hall] Il kernel è stato scritto usando GNU C e la toolchain GNU. -Sebbene si attenga allo standard ISO C89, esso utilizza una serie di +Sebbene si attenga allo standard ISO C11, esso utilizza una serie di estensioni che non sono previste in questo standard. Il kernel è un ambiente C indipendente, che non ha alcuna dipendenza dalle librerie C standard, così alcune parti del C standard non sono supportate. diff --git a/Documentation/translations/ja_JP/howto.rst b/Documentation/translations/ja_JP/howto.rst index b47a682d8dedc..b8eeb45a02d46 100644 --- a/Documentation/translations/ja_JP/howto.rst +++ b/Documentation/translations/ja_JP/howto.rst @@ -65,7 +65,7 @@ Linux カーネル開発のやり方 - 『新・詳説 C 言語 H&S リファレンス』 (サミュエル P ハービソン/ガイ L スティール共著 斉藤 信男監訳)[ソフトバンク] カーネルは GNU C と GNU ツールチェインを使って書かれています。カーネル -は ISO C89 仕様に準拠して書く一方で、標準には無い言語拡張を多く使って +は ISO C11 仕様に準拠して書く一方で、標準には無い言語拡張を多く使って います。カーネルは標準 C ライブラリに依存しない、C 言語非依存環境です。 そのため、C の標準の中で使えないものもあります。特に任意の long long の除算や浮動小数点は使えません。カーネルがツールチェインや C 言語拡張 diff --git a/Documentation/translations/ko_KR/howto.rst b/Documentation/translations/ko_KR/howto.rst index df53fafd1b10a..969e91a95bb0c 100644 --- a/Documentation/translations/ko_KR/howto.rst +++ b/Documentation/translations/ko_KR/howto.rst @@ -62,7 +62,7 @@ Documentation/process/howto.rst - "Practical C Programming" by Steve Oualline [O'Reilly] - "C: A Reference Manual" by Harbison and Steele [Prentice Hall] -커널은 GNU C와 GNU 툴체인을 사용하여 작성되었다. 이 툴들은 ISO C89 표준을 +커널은 GNU C와 GNU 툴체인을 사용하여 작성되었다. 이 툴들은 ISO C11 표준을 따르는 반면 표준에 있지 않은 많은 확장기능도 가지고 있다. 커널은 표준 C 라이브러리와는 관계없이 freestanding C 환경이어서 C 표준의 일부는 지원되지 않는다. 임의의 long long 나누기나 floating point는 지원되지 않는다. diff --git a/Documentation/translations/zh_CN/process/howto.rst b/Documentation/translations/zh_CN/process/howto.rst index 5bf953146929f..888978a62db3b 100644 --- a/Documentation/translations/zh_CN/process/howto.rst +++ b/Documentation/translations/zh_CN/process/howto.rst @@ -45,7 +45,7 @@ Linux内核大部分是由C语言写成的,一些体系结构相关的代码 - "C: A Reference Manual" by Harbison and Steele [Prentice Hall] 《C语言参考手册(原书第5版)》(邱仲潘 等译)[机械工业出版社] -Linux内核使用GNU C和GNU工具链开发。虽然它遵循ISO C89标准,但也用到了一些 +Linux内核使用GNU C和GNU工具链开发。虽然它遵循ISO C11标准,但也用到了一些 标准中没有定义的扩展。内核是自给自足的C环境,不依赖于标准C库的支持,所以 并不支持C标准中的部分定义。比如long long类型的大数除法和浮点运算就不允许 使用。有时候确实很难弄清楚内核对工具链的要求和它所使用的扩展,不幸的是目 diff --git a/Documentation/translations/zh_TW/process/howto.rst b/Documentation/translations/zh_TW/process/howto.rst index 86b0d4c6d6f97..8fb8edcaee665 100644 --- a/Documentation/translations/zh_TW/process/howto.rst +++ b/Documentation/translations/zh_TW/process/howto.rst @@ -48,7 +48,7 @@ Linux內核大部分是由C語言寫成的,一些體系結構相關的代碼 - "C: A Reference Manual" by Harbison and Steele [Prentice Hall] 《C語言參考手冊(原書第5版)》(邱仲潘 等譯)[機械工業出版社] -Linux內核使用GNU C和GNU工具鏈開發。雖然它遵循ISO C89標準,但也用到了一些 +Linux內核使用GNU C和GNU工具鏈開發。雖然它遵循ISO C11標準,但也用到了一些 標準中沒有定義的擴展。內核是自給自足的C環境,不依賴於標準C庫的支持,所以 並不支持C標準中的部分定義。比如long long類型的大數除法和浮點運算就不允許 使用。有時候確實很難弄清楚內核對工具鏈的要求和它所使用的擴展,不幸的是目 -- GitLab From 3d2af9cce3133b3bc596a9d065c6f9d93419ccfb Mon Sep 17 00:00:00 2001 From: Neal Cardwell Date: Fri, 21 Oct 2022 17:08:21 +0000 Subject: [PATCH 0364/1927] tcp: fix indefinite deferral of RTO with SACK reneging This commit fixes a bug that can cause a TCP data sender to repeatedly defer RTOs when encountering SACK reneging. The bug is that when we're in fast recovery in a scenario with SACK reneging, every time we get an ACK we call tcp_check_sack_reneging() and it can note the apparent SACK reneging and rearm the RTO timer for srtt/2 into the future. In some SACK reneging scenarios that can happen repeatedly until the receive window fills up, at which point the sender can't send any more, the ACKs stop arriving, and the RTO fires at srtt/2 after the last ACK. But that can take far too long (O(10 secs)), since the connection is stuck in fast recovery with a low cwnd that cannot grow beyond ssthresh, even if more bandwidth is available. This fix changes the logic in tcp_check_sack_reneging() to only rearm the RTO timer if data is cumulatively ACKed, indicating forward progress. This avoids this kind of nearly infinite loop of RTO timer re-arming. In addition, this meets the goals of tcp_check_sack_reneging() in handling Windows TCP behavior that looks temporarily like SACK reneging but is not really. Many thanks to Jakub Kicinski and Neil Spring, who reported this issue and provided critical packet traces that enabled root-causing this issue. Also, many thanks to Jakub Kicinski for testing this fix. Fixes: 5ae344c949e7 ("tcp: reduce spurious retransmits due to transient SACK reneging") Reported-by: Jakub Kicinski Reported-by: Neil Spring Signed-off-by: Neal Cardwell Reviewed-by: Eric Dumazet Cc: Yuchung Cheng Tested-by: Jakub Kicinski Link: https://lore.kernel.org/r/20221021170821.1093930-1-ncardwell.kernel@gmail.com Signed-off-by: Jakub Kicinski --- net/ipv4/tcp_input.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index bc2ea12221f95..0640453fce54b 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -2192,7 +2192,8 @@ void tcp_enter_loss(struct sock *sk) */ static bool tcp_check_sack_reneging(struct sock *sk, int flag) { - if (flag & FLAG_SACK_RENEGING) { + if (flag & FLAG_SACK_RENEGING && + flag & FLAG_SND_UNA_ADVANCED) { struct tcp_sock *tp = tcp_sk(sk); unsigned long delay = max(usecs_to_jiffies(tp->srtt_us >> 4), msecs_to_jiffies(10)); -- GitLab From 720ca52bcef225b967a339e0fffb6d0c7e962240 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 21 Oct 2022 09:03:04 -0700 Subject: [PATCH 0365/1927] net-memcg: avoid stalls when under memory pressure As Shakeel explains the commit under Fixes had the unintended side-effect of no longer pre-loading the cached memory allowance. Even tho we previously dropped the first packet received when over memory limit - the consecutive ones would get thru by using the cache. The charging was happening in batches of 128kB, so we'd let in 128kB (truesize) worth of packets per one drop. After the change we no longer force charge, there will be no cache filling side effects. This causes significant drops and connection stalls for workloads which use a lot of page cache, since we can't reclaim page cache under GFP_NOWAIT. Some of the latency can be recovered by improving SACK reneg handling but nowhere near enough to get back to the pre-5.15 performance (the application I'm experimenting with still sees 5-10x worst latency). Apply the suggested workaround of using GFP_ATOMIC. We will now be more permissive than previously as we'll drop _no_ packets in softirq when under pressure. But I can't think of any good and simple way to address that within networking. Link: https://lore.kernel.org/all/20221012163300.795e7b86@kernel.org/ Suggested-by: Shakeel Butt Fixes: 4b1327be9fe5 ("net-memcg: pass in gfp_t mask to mem_cgroup_charge_skmem()") Acked-by: Shakeel Butt Acked-by: Roman Gushchin Link: https://lore.kernel.org/r/20221021160304.1362511-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- include/net/sock.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/net/sock.h b/include/net/sock.h index 9e464f6409a71..22f8bab583ddd 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -2585,7 +2585,7 @@ static inline gfp_t gfp_any(void) static inline gfp_t gfp_memcg_charge(void) { - return in_softirq() ? GFP_NOWAIT : GFP_KERNEL; + return in_softirq() ? GFP_ATOMIC : GFP_KERNEL; } static inline long sock_rcvtimeo(const struct sock *sk, bool noblock) -- GitLab From efa17e90e1711bdb084e3954fa44afb6647331c0 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sat, 22 Oct 2022 15:42:12 +0800 Subject: [PATCH 0366/1927] iio: trigger: sysfs: fix possible memory leak in iio_sysfs_trig_init() dev_set_name() allocates memory for name, it need be freed when device_add() fails, call put_device() to give up the reference that hold in device_initialize(), so that it can be freed in kobject_cleanup() when the refcount hit to 0. Fault injection test can trigger this: unreferenced object 0xffff8e8340a7b4c0 (size 32): comm "modprobe", pid 243, jiffies 4294678145 (age 48.845s) hex dump (first 32 bytes): 69 69 6f 5f 73 79 73 66 73 5f 74 72 69 67 67 65 iio_sysfs_trigge 72 00 a7 40 83 8e ff ff 00 86 13 c4 f6 ee ff ff r..@............ backtrace: [<0000000074999de8>] __kmem_cache_alloc_node+0x1e9/0x360 [<00000000497fd30b>] __kmalloc_node_track_caller+0x44/0x1a0 [<000000003636c520>] kstrdup+0x2d/0x60 [<0000000032f84da2>] kobject_set_name_vargs+0x1e/0x90 [<0000000092efe493>] dev_set_name+0x4e/0x70 Fixes: 1f785681a870 ("staging:iio:trigger sysfs userspace trigger rework.") Signed-off-by: Yang Yingliang Cc: Link: https://lore.kernel.org/r/20221022074212.1386424-1-yangyingliang@huawei.com Signed-off-by: Jonathan Cameron --- drivers/iio/trigger/iio-trig-sysfs.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/iio/trigger/iio-trig-sysfs.c b/drivers/iio/trigger/iio-trig-sysfs.c index d6c5e96447383..6b05eed41612b 100644 --- a/drivers/iio/trigger/iio-trig-sysfs.c +++ b/drivers/iio/trigger/iio-trig-sysfs.c @@ -203,9 +203,13 @@ static int iio_sysfs_trigger_remove(int id) static int __init iio_sysfs_trig_init(void) { + int ret; device_initialize(&iio_sysfs_trig_dev); dev_set_name(&iio_sysfs_trig_dev, "iio_sysfs_trigger"); - return device_add(&iio_sysfs_trig_dev); + ret = device_add(&iio_sysfs_trig_dev); + if (ret) + put_device(&iio_sysfs_trig_dev); + return ret; } module_init(iio_sysfs_trig_init); -- GitLab From 207777dc306a732cff76ab63bf19a7e0851410b3 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Thu, 20 Oct 2022 13:27:05 +0300 Subject: [PATCH 0367/1927] iio: adc: at91-sama5d2_adc: get rid of 5 degrees Celsius adjustment On SAMA7G5 final chip version there is no need for 5 degrees Celsius adjustment when computing junction temperature, thus, remove it. Fixes: 5ab38b81895c ("iio: adc: at91-sama5d2_adc: add support for temperature sensor") Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20221020102705.3639204-1-claudiu.beznea@microchip.com Signed-off-by: Jonathan Cameron --- drivers/iio/adc/at91-sama5d2_adc.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/iio/adc/at91-sama5d2_adc.c b/drivers/iio/adc/at91-sama5d2_adc.c index 33e2515522149..870f4cb609237 100644 --- a/drivers/iio/adc/at91-sama5d2_adc.c +++ b/drivers/iio/adc/at91-sama5d2_adc.c @@ -2307,11 +2307,9 @@ static int at91_adc_temp_sensor_init(struct at91_adc_state *st, clb->p6 = buf[AT91_ADC_TS_CLB_IDX_P6]; /* - * We prepare here the conversion to milli and also add constant - * factor (5 degrees Celsius) to p1 here to avoid doing it on - * hotpath. + * We prepare here the conversion to milli to avoid doing it on hotpath. */ - clb->p1 = clb->p1 * 1000 + 5000; + clb->p1 = clb->p1 * 1000; free_buf: kfree(buf); -- GitLab From 30bbe38b28fbc7415e0585f2b05e6f6f95febbaa Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 24 Oct 2022 15:22:23 +0200 Subject: [PATCH 0368/1927] MAINTAINERS: adjust entry after renaming parisc serial driver Commit 9e4e2ce1a78e ("parisc/serial: Rename 8250_gsc.c to 8250_parisc.c") renames the parisc serial driver file, but does not adjust the entry in MAINTAINERS. Hence, ./scripts/get_maintainer.pl --self-test=patterns complains about a broken reference. Repair this file reference in PARISC ARCHITECTURE. Signed-off-by: Lukas Bulwahn Fixes: 9e4e2ce1a78e ("parisc/serial: Rename 8250_gsc.c to 8250_parisc.c") Signed-off-by: Helge Deller --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..107359c46f6c1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15617,7 +15617,7 @@ F: drivers/input/serio/gscps2.c F: drivers/input/serio/hp_sdc* F: drivers/parisc/ F: drivers/parport/parport_gsc.* -F: drivers/tty/serial/8250/8250_gsc.c +F: drivers/tty/serial/8250/8250_parisc.c F: drivers/video/console/sti* F: drivers/video/fbdev/sti* F: drivers/video/logo/logo_parisc* -- GitLab From 835bed1b83952bdbbe874f8ee41d665d52e991de Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 24 Oct 2022 18:29:01 +0200 Subject: [PATCH 0369/1927] fbdev: sisfb: use explicitly signed char With char becoming unsigned by default, and with `char` alone being ambiguous and based on architecture, signed chars need to be marked explicitly as such. This fixes warnings like: drivers/video/fbdev/sis/init301.c:3549 SiS_GetCRT2Data301() warn: 'SiS_Pr->SiS_EModeIDTable[ModeIdIndex]->ROMMODEIDX661' is unsigned Cc: Thomas Winischhofer Cc: Greg Kroah-Hartman Cc: Helge Deller Cc: linux-usb@vger.kernel.org Cc: linux-fbdev@vger.kernel.org Cc: dri-devel@lists.freedesktop.org Signed-off-by: Jason A. Donenfeld Signed-off-by: Helge Deller --- drivers/usb/misc/sisusbvga/sisusb_struct.h | 2 +- drivers/video/fbdev/sis/vstruct.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/misc/sisusbvga/sisusb_struct.h b/drivers/usb/misc/sisusbvga/sisusb_struct.h index 3df64d2a9d43d..a86032a26d36d 100644 --- a/drivers/usb/misc/sisusbvga/sisusb_struct.h +++ b/drivers/usb/misc/sisusbvga/sisusb_struct.h @@ -91,7 +91,7 @@ struct SiS_Ext { unsigned char VB_ExtTVYFilterIndex; unsigned char VB_ExtTVYFilterIndexROM661; unsigned char REFindex; - char ROMMODEIDX661; + signed char ROMMODEIDX661; }; struct SiS_Ext2 { diff --git a/drivers/video/fbdev/sis/vstruct.h b/drivers/video/fbdev/sis/vstruct.h index ea94d214dcff5..d7a14e63ba5ab 100644 --- a/drivers/video/fbdev/sis/vstruct.h +++ b/drivers/video/fbdev/sis/vstruct.h @@ -148,7 +148,7 @@ struct SiS_Ext { unsigned char VB_ExtTVYFilterIndex; unsigned char VB_ExtTVYFilterIndexROM661; unsigned char REFindex; - char ROMMODEIDX661; + signed char ROMMODEIDX661; }; struct SiS_Ext2 { -- GitLab From 20293269d81779a0d0c0865f5877b240c3335c97 Mon Sep 17 00:00:00 2001 From: Lijo Lazar Date: Fri, 30 Sep 2022 10:43:08 +0530 Subject: [PATCH 0370/1927] drm/amdgpu: Remove ATC L2 access for MMHUB 2.1.x MMHUB 2.1.x versions don't have ATCL2. Remove accesses to ATCL2 registers. Since they are non-existing registers, read access will cause a 'Completer Abort' and gets reported when AER is enabled with the below patch. Tagging with the patch so that this is backported along with it. v2: squash in uninitialized warning fix (Nathan Chancellor) Fixes: 8795e182b02d ("PCI/portdrv: Don't disable AER reporting in get_port_device_capability()") Signed-off-by: Lijo Lazar Reviewed-by: Guchun Chen Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c | 28 +++++++------------------ 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c index 4d304f22889e4..998b5d17b271b 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c @@ -32,8 +32,6 @@ #include "gc/gc_10_1_0_offset.h" #include "soc15_common.h" -#define mmMM_ATC_L2_MISC_CG_Sienna_Cichlid 0x064d -#define mmMM_ATC_L2_MISC_CG_Sienna_Cichlid_BASE_IDX 0 #define mmDAGB0_CNTL_MISC2_Sienna_Cichlid 0x0070 #define mmDAGB0_CNTL_MISC2_Sienna_Cichlid_BASE_IDX 0 @@ -574,7 +572,6 @@ static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *ad case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid); def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_Sienna_Cichlid); break; default: @@ -608,8 +605,6 @@ static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *ad case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - if (def != data) - WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid, data); if (def1 != data1) WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_Sienna_Cichlid, data1); break; @@ -634,8 +629,8 @@ static void mmhub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *ade case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid); - break; + /* There is no ATCL2 in MMHUB for 2.1.x */ + return; default: def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG); break; @@ -646,18 +641,8 @@ static void mmhub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *ade else data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK; - if (def != data) { - switch (adev->ip_versions[MMHUB_HWIP][0]) { - case IP_VERSION(2, 1, 0): - case IP_VERSION(2, 1, 1): - case IP_VERSION(2, 1, 2): - WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid, data); - break; - default: - WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG, data); - break; - } - } + if (def != data) + WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG, data); } static int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, @@ -695,7 +680,10 @@ static void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u64 *flags) case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid); + /* There is no ATCL2 in MMHUB for 2.1.x. Keep the status + * based on DAGB + */ + data = MM_ATC_L2_MISC_CG__ENABLE_MASK; data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_Sienna_Cichlid); break; default: -- GitLab From 7b5a4d7b9e2952a15c8d2b2391dfacd7ce841a1a Mon Sep 17 00:00:00 2001 From: Rafael Mendonca Date: Mon, 17 Oct 2022 22:27:50 -0300 Subject: [PATCH 0371/1927] drm/amdkfd: Fix memory leak in kfd_mem_dmamap_userptr() If the number of pages from the userptr BO differs from the SG BO then the allocated memory for the SG table doesn't get freed before returning -EINVAL, which may lead to a memory leak in some error paths. Fix this by checking the number of pages before allocating memory for the SG table. Fixes: 264fb4d332f5 ("drm/amdgpu: Add multi-GPU DMA mapping helpers") Signed-off-by: Rafael Mendonca Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 8ad01e1ee4c64..da9d475d7ef2d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -510,13 +510,13 @@ kfd_mem_dmamap_userptr(struct kgd_mem *mem, struct ttm_tt *ttm = bo->tbo.ttm; int ret; + if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) + return -EINVAL; + ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); if (unlikely(!ttm->sg)) return -ENOMEM; - if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) - return -EINVAL; - /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, ttm->num_pages, 0, -- GitLab From 03ce7b387e8b0f4a1dc97a878545bdf7c7f23251 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:15 -0400 Subject: [PATCH 0372/1927] drm/amd/display: Check if link state is valid The link state is set to false if there is no link and local sink. Even though the stream state may not change, it is desirable to commit the new stream when HPD goes low to high. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 997ab031f816d..7b576bd24cb57 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1500,6 +1500,8 @@ static bool context_changed( for (i = 0; i < dc->current_state->stream_count; i++) { if (dc->current_state->streams[i] != context->streams[i]) return true; + if (!context->streams[i]->link->link_state_valid) + return true; } return false; -- GitLab From 10fdb0a11c555e0d6f7698d2874581d06e99ee71 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:16 -0400 Subject: [PATCH 0373/1927] drm/amd/display: Rework context change check Context change is all about streams; for this reason, this commit renames context_changed to streams_changed. Additionally, to make this function more flexible, this commit changes the function signature to receive the stream array and the stream count as a parameter. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 7b576bd24cb57..9219d1efde2a5 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1488,19 +1488,19 @@ static void program_timing_sync( } } -static bool context_changed( - struct dc *dc, - struct dc_state *context) +static bool streams_changed(struct dc *dc, + struct dc_stream_state *streams[], + uint8_t stream_count) { uint8_t i; - if (context->stream_count != dc->current_state->stream_count) + if (stream_count != dc->current_state->stream_count) return true; for (i = 0; i < dc->current_state->stream_count; i++) { - if (dc->current_state->streams[i] != context->streams[i]) + if (dc->current_state->streams[i] != streams[i]) return true; - if (!context->streams[i]->link->link_state_valid) + if (!streams[i]->link->link_state_valid) return true; } @@ -1895,7 +1895,7 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context) enum dc_status result = DC_ERROR_UNEXPECTED; int i; - if (!context_changed(dc, context)) + if (!streams_changed(dc, context->streams, context->stream_count)) return DC_OK; DC_LOG_DC("%s: %d streams\n", -- GitLab From 7b36f4d18e3e4941d12fe027ad6ad6123c257027 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:17 -0400 Subject: [PATCH 0374/1927] drm/amd/display: Enable new commit sequence only for DCN32x Change commit sequence will impact all ASICs. It is prudent to run this update in small steps to keep issues under control and avoid any potential regression. With this idea in mind, this commit is preparation work for the complete transition to the new commit sequence. To maintain this change manageable across multiple ASICs, this commit adds a new function named dc_commit_streams which is a copy of the dc_commit_state with some minor changes. Finally, inside the dc_commit_state, we check if we are using DCN32x or above and enable the new sequence only for those devices. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 46 ++++++++++++++++++++++++ drivers/gpu/drm/amd/display/dc/dc.h | 13 +++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 9219d1efde2a5..5d9e9e8f87d3a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1890,11 +1890,57 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c return result; } +enum dc_status dc_commit_streams(struct dc *dc, struct dc_state *context) +{ + enum dc_status res = DC_OK; + int i; + + if (!streams_changed(dc, context->streams, context->stream_count)) + return res; + + DC_LOG_DC("%s: %d streams\n", + __func__, context->stream_count); + + for (i = 0; i < context->stream_count; i++) { + struct dc_stream_state *stream = context->streams[i]; + + dc_stream_log(dc, stream); + } + + /* + * Previous validation was perfomred with fast_validation = true and + * the full DML state required for hardware programming was skipped. + * + * Re-validate here to calculate these parameters / watermarks. + */ + res = dc_validate_global_state(dc, context, false); + if (res != DC_OK) { + DC_LOG_ERROR("DC commit global validation failure: %s (%d)", + dc_status_to_str(res), res); + return res; + } + + res = dc_commit_state_no_check(dc, context); + + return (res == DC_OK); +} + +/* TODO: When the transition to the new commit sequence is done, remove this + * function in favor of dc_commit_streams. */ bool dc_commit_state(struct dc *dc, struct dc_state *context) { enum dc_status result = DC_ERROR_UNEXPECTED; int i; + /* TODO: Since change commit sequence can have a huge impact, + * we decided to only enable it for DCN3x. However, as soon as + * we get more confident about this change we'll need to enable + * the new sequence for all ASICs. */ + if (dc->ctx->dce_version >= DCN_VERSION_3_2) { + result = dc_commit_streams(dc, context); + return result == DC_OK; + } + if (!streams_changed(dc, context->streams, context->stream_count)) return DC_OK; diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index bfc5474c0f4c9..2e71781f155b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1321,15 +1321,10 @@ void dc_resource_state_destruct(struct dc_state *context); bool dc_resource_is_dsc_encoding_supported(const struct dc *dc); -/* - * TODO update to make it about validation sets - * Set up streams and links associated to drive sinks - * The streams parameter is an absolute set of all active streams. - * - * After this call: - * Phy, Encoder, Timing Generator are programmed and enabled. - * New streams are enabled with blank stream; no memory read. - */ +enum dc_status dc_commit_streams(struct dc *dc, struct dc_state *context); + +/* TODO: When the transition to the new commit sequence is done, remove this + * function in favor of dc_commit_streams. */ bool dc_commit_state(struct dc *dc, struct dc_state *context); struct dc_state *dc_create_state(struct dc *dc); -- GitLab From 0e986cea0347902b2c72b09c8fe9c0f30d7decb4 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:18 -0400 Subject: [PATCH 0375/1927] drm/amd/display: Copy DC context in the commit streams DC adds an instance of DML (which contains VBA) to each context, and multiple threads might write back to the global VBA resulting in data overwriting. To keep the consistency with other parts of the DC code, this commit changes dc_commit_streams to copy the current DC state, and as a result, it also changes the function signature to expect streams instead of a context. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 41 +++++++++++++++++++----- drivers/gpu/drm/amd/display/dc/dc.h | 4 ++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 5d9e9e8f87d3a..af81c28357380 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1890,23 +1890,44 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c return result; } -enum dc_status dc_commit_streams(struct dc *dc, struct dc_state *context) +/** + * dc_commit_streams - Commit current stream state + * + * @dc: DC object with the commit state to be configured in the hardware + * @streams: Array with a list of stream state + * @stream_count: Total of streams + * + * Function responsible for commit streams change to the hardware. + * + * Return: + * Return DC_OK if everything work as expected, otherwise, return a dc_status + * code. + */ +enum dc_status dc_commit_streams(struct dc *dc, + struct dc_stream_state *streams[], + uint8_t stream_count) { - enum dc_status res = DC_OK; int i; + struct dc_state *context; + enum dc_status res = DC_OK; - if (!streams_changed(dc, context->streams, context->stream_count)) + if (!streams_changed(dc, streams, stream_count)) return res; - DC_LOG_DC("%s: %d streams\n", - __func__, context->stream_count); + DC_LOG_DC("%s: %d streams\n", __func__, stream_count); - for (i = 0; i < context->stream_count; i++) { - struct dc_stream_state *stream = context->streams[i]; + for (i = 0; i < stream_count; i++) { + struct dc_stream_state *stream = streams[i]; dc_stream_log(dc, stream); } + context = dc_create_state(dc); + if (!context) + goto context_alloc_fail; + + dc_resource_state_copy_construct_current(dc, context); + /* * Previous validation was perfomred with fast_validation = true and * the full DML state required for hardware programming was skipped. @@ -1922,6 +1943,10 @@ enum dc_status dc_commit_streams(struct dc *dc, struct dc_state *context) res = dc_commit_state_no_check(dc, context); +context_alloc_fail: + + DC_LOG_DC("%s Finished.\n", __func__); + return (res == DC_OK); } @@ -1937,7 +1962,7 @@ bool dc_commit_state(struct dc *dc, struct dc_state *context) * we get more confident about this change we'll need to enable * the new sequence for all ASICs. */ if (dc->ctx->dce_version >= DCN_VERSION_3_2) { - result = dc_commit_streams(dc, context); + result = dc_commit_streams(dc, context->streams, context->stream_count); return result == DC_OK; } diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 2e71781f155b1..6adecb62e534e 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1321,7 +1321,9 @@ void dc_resource_state_destruct(struct dc_state *context); bool dc_resource_is_dsc_encoding_supported(const struct dc *dc); -enum dc_status dc_commit_streams(struct dc *dc, struct dc_state *context); +enum dc_status dc_commit_streams(struct dc *dc, + struct dc_stream_state *streams[], + uint8_t stream_count); /* TODO: When the transition to the new commit sequence is done, remove this * function in favor of dc_commit_streams. */ -- GitLab From f6ae69f49fcf697b6ffa93d58eb3746897f61cf8 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:19 -0400 Subject: [PATCH 0376/1927] drm/amd/display: Include surface of unaffected streams The commit stream function does not include surfaces of unaffected streams, which may lead to some blank screens during mode change in some edge cases. This commit adds surfaces of unaffected streams followed by kernel-doc for documenting some of the fields that participate in this change. v2: squash in kerneldoc warning fix (Alex) Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 12 +++++++++++- drivers/gpu/drm/amd/display/dc/dc.h | 15 +++++++++++++-- drivers/gpu/drm/amd/display/dc/dc_stream.h | 4 ++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index af81c28357380..b904b03cf94e4 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1907,9 +1907,10 @@ enum dc_status dc_commit_streams(struct dc *dc, struct dc_stream_state *streams[], uint8_t stream_count) { - int i; + int i, j; struct dc_state *context; enum dc_status res = DC_OK; + struct dc_validation_set set[MAX_STREAMS] = {0}; if (!streams_changed(dc, streams, stream_count)) return res; @@ -1918,8 +1919,17 @@ enum dc_status dc_commit_streams(struct dc *dc, for (i = 0; i < stream_count; i++) { struct dc_stream_state *stream = streams[i]; + struct dc_stream_status *status = dc_stream_get_status(stream); dc_stream_log(dc, stream); + + set[i].stream = stream; + + if (status) { + set[i].plane_count = status->plane_count; + for (j = 0; j < status->plane_count; j++) + set[i].plane_states[j] = status->plane_states[j]; + } } context = dc_create_state(dc); diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 6adecb62e534e..836cdf5d41141 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1269,12 +1269,23 @@ void dc_post_update_surfaces_to_stream( #include "dc_stream.h" -/* - * Structure to store surface/stream associations for validation +/** + * struct dc_validation_set - Struct to store surface/stream associations for validation */ struct dc_validation_set { + /** + * @stream: Stream state properties + */ struct dc_stream_state *stream; + + /** + * @plane_state: Surface state + */ struct dc_plane_state *plane_states[MAX_SURFACES]; + + /** + * @plane_count: Total of active planes + */ uint8_t plane_count; }; diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index 9e6025c98db91..73dccd485895d 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -41,6 +41,10 @@ struct timing_sync_info { struct dc_stream_status { int primary_otg_inst; int stream_enc_inst; + + /** + * @plane_count: Total of planes attached to a single stream + */ int plane_count; int audio_inst; struct timing_sync_info timing_sync_info; -- GitLab From 987b96eb860036ab79051fb271f7fbdc01c9daf5 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:20 -0400 Subject: [PATCH 0377/1927] drm/amd/display: Handle virtual hardware detect If virtual hardware is detected, there is no reason to run the full dc_commit_streams process, and DC can return true immediately. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index b904b03cf94e4..61b574b9e7363 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1912,6 +1912,9 @@ enum dc_status dc_commit_streams(struct dc *dc, enum dc_status res = DC_OK; struct dc_validation_set set[MAX_STREAMS] = {0}; + if (dc->ctx->dce_environment == DCE_ENV_VIRTUAL_HW) + return res; + if (!streams_changed(dc, streams, stream_count)) return res; -- GitLab From a5e39ae27c3a305c6aafc0e423b0cb2c677facde Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:21 -0400 Subject: [PATCH 0378/1927] drm/amd/display: Add function for validate and update new stream DC stream can be seen as a representation of the DCN backend or the data struct that represents the center of the display pipeline. The front end (i.e., planes) is connected to the DC stream, and in its turn, streams are connected to the DC link. Due to this dynamic, DC must handle the following scenarios: 1. A stream is removed; 2. A new stream is created; 3. An unchanged stream had some updates on its planes. These combinations require that the new stream data struct become updated and has a valid global state. For handling multiple corner cases associated with stream operations, this commit introduces a function dedicated to manipulating stream changes and invokes the state validation function after that. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 16 +- .../gpu/drm/amd/display/dc/core/dc_resource.c | 219 +++++++++++++++++- drivers/gpu/drm/amd/display/dc/dc.h | 6 + 3 files changed, 227 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 61b574b9e7363..d568387c4bda4 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1941,21 +1941,17 @@ enum dc_status dc_commit_streams(struct dc *dc, dc_resource_state_copy_construct_current(dc, context); - /* - * Previous validation was perfomred with fast_validation = true and - * the full DML state required for hardware programming was skipped. - * - * Re-validate here to calculate these parameters / watermarks. - */ - res = dc_validate_global_state(dc, context, false); + res = dc_validate_with_context(dc, set, stream_count, context, false); if (res != DC_OK) { - DC_LOG_ERROR("DC commit global validation failure: %s (%d)", - dc_status_to_str(res), res); - return res; + BREAK_TO_DEBUGGER(); + goto fail; } res = dc_commit_state_no_check(dc, context); +fail: + dc_release_state(context); + context_alloc_fail: DC_LOG_DC("%s Finished.\n", __func__); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index fd8db482e56f9..e001b138b2ac6 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -2593,15 +2593,226 @@ bool dc_resource_is_dsc_encoding_supported(const struct dc *dc) return dc->res_pool->res_cap->num_dsc > 0; } +static bool planes_changed_for_existing_stream(struct dc_state *context, + struct dc_stream_state *stream, + const struct dc_validation_set set[], + int set_count) +{ + int i, j; + struct dc_stream_status *stream_status = NULL; + + for (i = 0; i < context->stream_count; i++) { + if (context->streams[i] == stream) { + stream_status = &context->stream_status[i]; + break; + } + } + + if (!stream_status) + ASSERT(0); + + for (i = 0; i < set_count; i++) + if (set[i].stream == stream) + break; + + if (i == set_count) + ASSERT(0); + + if (set[i].plane_count != stream_status->plane_count) + return true; + + for (j = 0; j < set[i].plane_count; j++) + if (set[i].plane_states[j] != stream_status->plane_states[j]) + return true; + + return false; +} + +/** + * dc_validate_with_context - Validate and update the potential new stream in the context object + * + * @dc: Used to get the current state status + * @set: An array of dc_validation_set with all the current streams reference + * @set_count: Total of streams + * @context: New context + * @fast_validate: Enable or disable fast validation + * + * This function updates the potential new stream in the context object. It + * creates multiple lists for the add, remove, and unchanged streams. In + * particular, if the unchanged streams have a plane that changed, it is + * necessary to remove all planes from the unchanged streams. In summary, this + * function is responsible for validating the new context. + * + * Return: + * In case of success, return DC_OK (1), otherwise, return a DC error. + */ +enum dc_status dc_validate_with_context(struct dc *dc, + const struct dc_validation_set set[], + int set_count, + struct dc_state *context, + bool fast_validate) +{ + struct dc_stream_state *unchanged_streams[MAX_PIPES] = { 0 }; + struct dc_stream_state *del_streams[MAX_PIPES] = { 0 }; + struct dc_stream_state *add_streams[MAX_PIPES] = { 0 }; + int old_stream_count = context->stream_count; + enum dc_status res = DC_ERROR_UNEXPECTED; + int unchanged_streams_count = 0; + int del_streams_count = 0; + int add_streams_count = 0; + bool found = false; + int i, j, k; + + DC_LOGGER_INIT(dc->ctx->logger); + + /* First build a list of streams to be remove from current context */ + for (i = 0; i < old_stream_count; i++) { + struct dc_stream_state *stream = context->streams[i]; + + for (j = 0; j < set_count; j++) { + if (stream == set[j].stream) { + found = true; + break; + } + } + + if (!found) + del_streams[del_streams_count++] = stream; + + found = false; + } + + /* Second, build a list of new streams */ + for (i = 0; i < set_count; i++) { + struct dc_stream_state *stream = set[i].stream; + + for (j = 0; j < old_stream_count; j++) { + if (stream == context->streams[j]) { + found = true; + break; + } + } + + if (!found) + add_streams[add_streams_count++] = stream; + + found = false; + } + + /* Build a list of unchanged streams which is necessary for handling + * planes change such as added, removed, and updated. + */ + for (i = 0; i < set_count; i++) { + /* Check if stream is part of the delete list */ + for (j = 0; j < del_streams_count; j++) { + if (set[i].stream == del_streams[j]) { + found = true; + break; + } + } + + if (!found) { + /* Check if stream is part of the add list */ + for (j = 0; j < add_streams_count; j++) { + if (set[i].stream == add_streams[j]) { + found = true; + break; + } + } + } + + if (!found) + unchanged_streams[unchanged_streams_count++] = set[i].stream; + + found = false; + } + + /* Remove all planes for unchanged streams if planes changed */ + for (i = 0; i < unchanged_streams_count; i++) { + if (planes_changed_for_existing_stream(context, + unchanged_streams[i], + set, + set_count)) { + if (!dc_rem_all_planes_for_stream(dc, + unchanged_streams[i], + context)) { + res = DC_FAIL_DETACH_SURFACES; + goto fail; + } + } + } + + /* Remove all planes for removed streams and then remove the streams */ + for (i = 0; i < del_streams_count; i++) { + /* Need to cpy the dwb data from the old stream in order to efc to work */ + if (del_streams[i]->num_wb_info > 0) { + for (j = 0; j < add_streams_count; j++) { + if (del_streams[i]->sink == add_streams[j]->sink) { + add_streams[j]->num_wb_info = del_streams[i]->num_wb_info; + for (k = 0; k < del_streams[i]->num_wb_info; k++) + add_streams[j]->writeback_info[k] = del_streams[i]->writeback_info[k]; + } + } + } + + if (!dc_rem_all_planes_for_stream(dc, del_streams[i], context)) { + res = DC_FAIL_DETACH_SURFACES; + goto fail; + } + + res = dc_remove_stream_from_ctx(dc, context, del_streams[i]); + if (res != DC_OK) + goto fail; + } + + /* Add new streams and then add all planes for the new stream */ + for (i = 0; i < add_streams_count; i++) { + calculate_phy_pix_clks(add_streams[i]); + res = dc_add_stream_to_ctx(dc, context, add_streams[i]); + if (res != DC_OK) + goto fail; + + if (!add_all_planes_for_stream(dc, add_streams[i], set, set_count, context)) { + res = DC_FAIL_ATTACH_SURFACES; + goto fail; + } + } + + /* Add all planes for unchanged streams if planes changed */ + for (i = 0; i < unchanged_streams_count; i++) { + if (planes_changed_for_existing_stream(context, + unchanged_streams[i], + set, + set_count)) { + if (!add_all_planes_for_stream(dc, unchanged_streams[i], set, set_count, context)) { + res = DC_FAIL_ATTACH_SURFACES; + goto fail; + } + } + } + + res = dc_validate_global_state(dc, context, fast_validate); + +fail: + if (res != DC_OK) + DC_LOG_WARNING("%s:resource validation failed, dc_status:%d\n", + __func__, + res); + + return res; +} /** - * dc_validate_global_state() - Determine if HW can support a given state - * Checks HW resource availability and bandwidth requirement. + * dc_validate_global_state() - Determine if hardware can support a given state + * * @dc: dc struct for this driver * @new_ctx: state to be validated * @fast_validate: set to true if only yes/no to support matters * - * Return: DC_OK if the result can be programmed. Otherwise, an error code. + * Checks hardware resource availability and bandwidth requirement. + * + * Return: + * DC_OK if the result can be programmed. Otherwise, an error code. */ enum dc_status dc_validate_global_state( struct dc *dc, @@ -3734,4 +3945,4 @@ bool dc_resource_acquire_secondary_pipe_for_mpc_odm( } return true; -} \ No newline at end of file +} diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 836cdf5d41141..bf938cda379a9 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -1297,6 +1297,12 @@ enum dc_status dc_validate_plane(struct dc *dc, const struct dc_plane_state *pla void get_clock_requirements_for_state(struct dc_state *state, struct AsicStateEx *info); +enum dc_status dc_validate_with_context(struct dc *dc, + const struct dc_validation_set set[], + int set_count, + struct dc_state *context, + bool fast_validate); + bool dc_set_generic_gpio_for_stereo(bool enable, struct gpio_service *gpio_service); -- GitLab From 170390e587a69b2a24abac39eb3ae6ec28a4d7f2 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:22 -0400 Subject: [PATCH 0379/1927] drm/amd/display: Handle seamless boot stream A seamless boot stream has hardware resources assigned to it, and adding a new stream means rebuilding the current assignment. It is desirable to avoid this situation since it may cause light-up issues on the VGA monitor on USB-C. This commit swaps the seamless boot stream to pipe 0 (if necessary) to ensure that the pipe context matches. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc_resource.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index e001b138b2ac6..ba2d4347f7224 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -2765,6 +2765,21 @@ enum dc_status dc_validate_with_context(struct dc *dc, goto fail; } + /* Swap seamless boot stream to pipe 0 (if needed) to ensure pipe_ctx + * matches. This may change in the future if seamless_boot_stream can be + * multiple. + */ + for (i = 0; i < add_streams_count; i++) { + mark_seamless_boot_stream(dc, add_streams[i]); + if (add_streams[i]->apply_seamless_boot_optimization && i != 0) { + struct dc_stream_state *temp = add_streams[0]; + + add_streams[0] = add_streams[i]; + add_streams[i] = temp; + break; + } + } + /* Add new streams and then add all planes for the new stream */ for (i = 0; i < add_streams_count; i++) { calculate_phy_pix_clks(add_streams[i]); -- GitLab From eef019eabc3cd0fddcffefbf67806a4d8cca29bb Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:23 -0400 Subject: [PATCH 0380/1927] drm/amd/display: Update OTG instance in the commit stream OTG instance is not updated in dc_commit_state_no_check for newly committed streams because mode_change is not set. Notice that OTG update is part of the software state, and after hardware programming, it must be updated; for this reason, this commit updates the OTG offset right after hardware programming. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index d568387c4bda4..b7afada35b24e 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1949,6 +1949,12 @@ enum dc_status dc_commit_streams(struct dc *dc, res = dc_commit_state_no_check(dc, context); + for (i = 0; i < stream_count; i++) { + for (j = 0; j < context->stream_count; j++) + if (streams[i]->stream_id == context->streams[j]->stream_id) + streams[i]->out.otg_offset = context->stream_status[j].primary_otg_inst; + } + fail: dc_release_state(context); -- GitLab From 7fffb03b4045c862f904a88b852dc509c4e46406 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:24 -0400 Subject: [PATCH 0381/1927] drm/amd/display: Avoid ABM when ODM combine is enabled for eDP ODM to combine on the eDP panel with ABM causes the color difference to the panel since the ABM module only sets one pipe. Hence, this commit blocks ABM in case of ODM combined on eDP. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index b7afada35b24e..375439c51cd30 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1950,9 +1950,19 @@ enum dc_status dc_commit_streams(struct dc *dc, res = dc_commit_state_no_check(dc, context); for (i = 0; i < stream_count; i++) { - for (j = 0; j < context->stream_count; j++) + for (j = 0; j < context->stream_count; j++) { if (streams[i]->stream_id == context->streams[j]->stream_id) streams[i]->out.otg_offset = context->stream_status[j].primary_otg_inst; + + if (dc_is_embedded_signal(streams[i]->signal)) { + struct dc_stream_status *status = dc_stream_get_status_from_state(context, streams[i]); + + if (dc->hwss.is_abm_supported) + status->is_abm_supported = dc->hwss.is_abm_supported(dc, context, streams[i]); + else + status->is_abm_supported = true; + } + } } fail: -- GitLab From dddde627807c22d6f15f4417eb395b13a1ca88f9 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 08:28:25 -0400 Subject: [PATCH 0382/1927] drm/amd/display: Use update plane and stream routine for DCN32x Sub-viewport (Subvp) feature is used for changing MCLK without causing any display artifact, requiring special treatment from the plane and stream perspective since DC needs to read data from the cache when using subvp. However, the function dc_commit_updates_for_stream does not provide all the support needed by this feature which will make this function legacy at some point. For this reason, this commit enables dc_update_planes_and_stream for ASICs that support this feature but preserves the old behavior for other ASICs. However, dc_update_planes_and_stream should replace dc_commit_updates_for_stream for all ASICs since it does most of the tasks executed by dc_commit_updates_for_stream with other extra operations, but we need to run tests before making this change. Reviewed-by: Harry Wentland Signed-off-by: Rodrigo Siqueira Co-developed-by: Aurabindo Pillai Signed-off-by: Aurabindo Pillai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 375439c51cd30..56629197a6a15 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -3904,6 +3904,18 @@ void dc_commit_updates_for_stream(struct dc *dc, struct dc_context *dc_ctx = dc->ctx; int i, j; + /* TODO: Since change commit sequence can have a huge impact, + * we decided to only enable it for DCN3x. However, as soon as + * we get more confident about this change we'll need to enable + * the new sequence for all ASICs. + */ + if (dc->ctx->dce_version >= DCN_VERSION_3_2) { + dc_update_planes_and_stream(dc, srf_updates, + surface_count, stream, + stream_update); + return; + } + stream_status = dc_stream_get_status(stream); context = dc->current_state; -- GitLab From fa16dec20440156fda208c3d289dfa15af3953dc Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Thu, 20 Oct 2022 15:25:25 +0800 Subject: [PATCH 0383/1927] drm/amd/pm: allow gfxoff on gc_11_0_3 allow gfxoff on gc_11_0_3 Signed-off-by: Kenneth Feng Reviewed-by: Yang Wang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 1 + drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index e3842dc100d66..5eefba2948a55 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -5051,6 +5051,7 @@ static int gfx_v11_0_set_powergating_state(void *handle, switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(11, 0, 0): case IP_VERSION(11, 0, 2): + case IP_VERSION(11, 0, 3): amdgpu_gfx_off_ctrl(adev, enable); break; case IP_VERSION(11, 0, 1): diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index feb4d68f3fd9a..43fb102a65f56 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -840,6 +840,7 @@ int smu_v13_0_gfx_off_control(struct smu_context *smu, bool enable) case IP_VERSION(13, 0, 5): case IP_VERSION(13, 0, 7): case IP_VERSION(13, 0, 8): + case IP_VERSION(13, 0, 10): if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) return 0; if (enable) -- GitLab From 345b16961afe0deae8633ce10dbff46c7e64e35f Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 6 Oct 2022 17:26:48 -0400 Subject: [PATCH 0384/1927] drm/amd/display: Remove wrong pipe control lock When using a device based on DCN32/321, we have an issue where a second 4k@60Hz display does not light up, and the system becomes unresponsive for a few minutes. In the debug process, it was possible to see a hang in the function dcn20_post_unlock_program_front_end in this part: for (j = 0; j < TIMEOUT_FOR_PIPE_ENABLE_MS*1000 && hubp->funcs->hubp_is_flip_pending(hubp); j++) mdelay(1); } The hubp_is_flip_pending always returns positive for waiting pending flips which is a symptom of pipe hang. Additionally, the dmesg log shows this message after a few minutes: BUG: soft lockup - CPU#4 stuck for 26s! ... [ +0.000003] dcn20_post_unlock_program_front_end+0x112/0x340 [amdgpu] [ +0.000171] dc_commit_state_no_check+0x63d/0xbf0 [amdgpu] [ +0.000155] ? dc_validate_global_state+0x358/0x3d0 [amdgpu] [ +0.000154] dc_commit_state+0xe2/0xf0 [amdgpu] This confirmed the hypothesis that we had a pipe hanging somewhere. Next, after checking the ftrace entries, we have the below weird sequence: [..] 2) | dcn10_lock_all_pipes [amdgpu]() { 2) 0.120 us | optc1_is_tg_enabled [amdgpu](); 2) | dcn20_pipe_control_lock [amdgpu]() { 2) | dc_dmub_srv_clear_inbox0_ack [amdgpu]() { 2) 0.121 us | amdgpu_dm_dmub_reg_write [amdgpu](); 2) 0.551 us | } 2) | dc_dmub_srv_send_inbox0_cmd [amdgpu]() { 2) 0.110 us | amdgpu_dm_dmub_reg_write [amdgpu](); 2) 0.511 us | } 2) | dc_dmub_srv_wait_for_inbox0_ack [amdgpu]() { 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); [..] We are not expected to read from dmub register so many times and for so long. From the trace log, it was possible to identify that the function dcn20_pipe_control_lock was triggering the dmub operation when it was unnecessary and causing the hang issue. This commit drops the unnecessary dmub code and, consequently, fixes the second display not lighting up the issue. Tested-by: Daniel Wheeler Acked-by: Qingqing Zhuo Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index d732b6f031a12..a7e0001a8f46d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1270,16 +1270,6 @@ void dcn20_pipe_control_lock( lock, &hw_locks, &inst_flags); - } else if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_MAIN) { - union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 }; - hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK; - hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER; - hw_lock_cmd.bits.lock_pipe = 1; - hw_lock_cmd.bits.otg_inst = pipe->stream_res.tg->inst; - hw_lock_cmd.bits.lock = lock; - if (!lock) - hw_lock_cmd.bits.should_release = 1; - dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd); } else if (pipe->plane_state != NULL && pipe->plane_state->triplebuffer_flips) { if (lock) pipe->stream_res.tg->funcs->triplebuffer_lock(pipe->stream_res.tg); @@ -1856,7 +1846,7 @@ void dcn20_post_unlock_program_front_end( for (j = 0; j < TIMEOUT_FOR_PIPE_ENABLE_MS*1000 && hubp->funcs->hubp_is_flip_pending(hubp); j++) - mdelay(1); + udelay(1); } } -- GitLab From 27fcf77830d0e8af53cd562909ff37aa3ec345dd Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 6 Oct 2022 17:26:49 -0400 Subject: [PATCH 0385/1927] drm/amd/display: Don't return false if no stream pipe_ctx[i] exists even if the pipe is not in use. If the pipe is not in use it will always have a null stream, so don't return false in this case. Tested-by: Daniel Wheeler Reviewed-by: Rodrigo Siqueira Acked-by: Qingqing Zhuo Signed-off-by: Alvin Lee Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c index d51d0c40ae5bc..b03a7814e96d8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c @@ -200,7 +200,7 @@ bool dcn32_all_pipes_have_stream_and_plane(struct dc *dc, struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; if (!pipe->stream) - return false; + continue; if (!pipe->plane_state) return false; -- GitLab From 6c16afdcec20b01879ad17440f42c9944786dc7d Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 19 Oct 2022 16:57:42 -0400 Subject: [PATCH 0386/1927] drm/amdgpu: fix sdma doorbell init ordering on APUs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 8795e182b02d ("PCI/portdrv: Don't disable AER reporting in get_port_device_capability()") uncovered a bug in amdgpu that required a reordering of the driver init sequence to avoid accessing a special register on the GPU before it was properly set up leading to an PCI AER error. This reordering uncovered a different hw programming ordering dependency in some APUs where the SDMA doorbells need to be programmed before the GFX doorbells. To fix this, move the SDMA doorbell programming back into the soc15 common code, but use the actual doorbell range values directly rather than the values stored in the ring structure since those will not be initialized at this point. This is a partial revert, but with the doorbell assignment fixed so the proper doorbell index is set before it's used. Fixes: e3163bc8ffdfdb ("drm/amdgpu: move nbio sdma_doorbell_range() into sdma code for vega") Acked-by: Christian König Signed-off-by: Alex Deucher Cc: skhan@linuxfoundation.org --- drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c | 5 ----- drivers/gpu/drm/amd/amdgpu/soc15.c | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c index 298fa11702e75..1122bd4eae98c 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c @@ -1417,11 +1417,6 @@ static int sdma_v4_0_start(struct amdgpu_device *adev) WREG32_SDMA(i, mmSDMA0_CNTL, temp); if (!amdgpu_sriov_vf(adev)) { - ring = &adev->sdma.instance[i].ring; - adev->nbio.funcs->sdma_doorbell_range(adev, i, - ring->use_doorbell, ring->doorbell_index, - adev->doorbell_index.sdma_doorbell_range); - /* unhalt engine */ temp = RREG32_SDMA(i, mmSDMA0_F32_CNTL); temp = REG_SET_FIELD(temp, SDMA0_F32_CNTL, HALT, 0); diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 183024d7c184e..e3b2b6b4f1a66 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -1211,6 +1211,20 @@ static int soc15_common_sw_fini(void *handle) return 0; } +static void soc15_sdma_doorbell_range_init(struct amdgpu_device *adev) +{ + int i; + + /* sdma doorbell range is programed by hypervisor */ + if (!amdgpu_sriov_vf(adev)) { + for (i = 0; i < adev->sdma.num_instances; i++) { + adev->nbio.funcs->sdma_doorbell_range(adev, i, + true, adev->doorbell_index.sdma_engine[i] << 1, + adev->doorbell_index.sdma_doorbell_range); + } + } +} + static int soc15_common_hw_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -1230,6 +1244,13 @@ static int soc15_common_hw_init(void *handle) /* enable the doorbell aperture */ soc15_enable_doorbell_aperture(adev, true); + /* HW doorbell routing policy: doorbell writing not + * in SDMA/IH/MM/ACV range will be routed to CP. So + * we need to init SDMA doorbell range prior + * to CP ip block init and ring test. IH already + * happens before CP. + */ + soc15_sdma_doorbell_range_init(adev); return 0; } -- GitLab From 9feece0d96124096e7268007bd2d3905a29d92ac Mon Sep 17 00:00:00 2001 From: ruanjinjie Date: Wed, 19 Oct 2022 15:45:02 +0800 Subject: [PATCH 0387/1927] drm/amd/display: Make some symbols static These symbols qp_table_422_10bpc_min, qp_table_444_8bpc_max, qp_table_420_12bpc_max, qp_table_444_10bpc_min, qp_table_420_8bpc_max, qp_table_444_8bpc_min, qp_table_444_12bpc_min, qp_table_420_12bpc_min, qp_table_422_12bpc_min, qp_table_422_12bpc_max, qp_table_444_12bpc_max, qp_table_420_8bpc_min, qp_table_422_8bpc_min, qp_table_422_10bpc_max, qp_table_420_10bpc_max, qp_table_420_10bpc_min, qp_table_444_10bpc_max, qp_table_422_8bpc_max are not used outside of the file, so mark them static. ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:28:18: warning: symbol 'qp_table_422_10bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:61:18: warning: symbol 'qp_table_444_8bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:102:18: warning: symbol 'qp_table_420_12bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:135:18: warning: symbol 'qp_table_444_10bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:188:18: warning: symbol 'qp_table_420_8bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:209:18: warning: symbol 'qp_table_444_8bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:250:18: warning: symbol 'qp_table_444_12bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:315:18: warning: symbol 'qp_table_420_12bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:348:18: warning: symbol 'qp_table_422_12bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:389:18: warning: symbol 'qp_table_422_12bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:430:18: warning: symbol 'qp_table_444_12bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:495:18: warning: symbol 'qp_table_420_8bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:516:18: warning: symbol 'qp_table_422_8bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:541:18: warning: symbol 'qp_table_422_10bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:574:16: warning: symbol 'qp_table_420_10bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:601:18: warning: symbol 'qp_table_420_10bpc_min' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:628:18: warning: symbol 'qp_table_444_10bpc_max' was not declared. Should it be static? ./drivers/gpu/drm/amd/amdgpu/../display/dc/dml/dsc/qp_tables.h:681:18: warning: symbol 'qp_table_422_8bpc_max' was not declared. Should it be static? Signed-off-by: ruanjinjie Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/dml/dsc/qp_tables.h | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h b/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h index e5fac9f4181d8..dcff0dd2b6a12 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/qp_tables.h @@ -25,7 +25,7 @@ */ -const qp_table qp_table_422_10bpc_min = { +static const qp_table qp_table_422_10bpc_min = { { 6, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} }, { 6.5, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 9, 12, 16} }, { 7, { 0, 4, 5, 6, 6, 6, 6, 7, 7, 7, 9, 9, 9, 11, 15} }, @@ -58,7 +58,7 @@ const qp_table qp_table_422_10bpc_min = { }; -const qp_table qp_table_444_8bpc_max = { +static const qp_table qp_table_444_8bpc_max = { { 6, { 4, 6, 8, 8, 9, 9, 9, 10, 11, 12, 12, 12, 12, 13, 15} }, { 6.5, { 4, 6, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 12, 13, 15} }, { 7, { 4, 5, 7, 7, 8, 8, 8, 9, 10, 11, 11, 12, 12, 13, 14} }, @@ -99,7 +99,7 @@ const qp_table qp_table_444_8bpc_max = { }; -const qp_table qp_table_420_12bpc_max = { +static const qp_table qp_table_420_12bpc_max = { { 4, {11, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 21, 22} }, { 4.5, {10, 11, 12, 13, 14, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} }, { 5, { 9, 11, 12, 13, 14, 15, 15, 16, 17, 17, 18, 18, 19, 20, 21} }, @@ -132,7 +132,7 @@ const qp_table qp_table_420_12bpc_max = { }; -const qp_table qp_table_444_10bpc_min = { +static const qp_table qp_table_444_10bpc_min = { { 6, { 0, 4, 7, 7, 9, 9, 9, 9, 9, 10, 10, 10, 10, 12, 18} }, { 6.5, { 0, 4, 6, 7, 8, 8, 9, 9, 9, 9, 10, 10, 10, 12, 18} }, { 7, { 0, 4, 6, 6, 8, 8, 8, 8, 8, 9, 9, 10, 10, 12, 17} }, @@ -185,7 +185,7 @@ const qp_table qp_table_444_10bpc_min = { }; -const qp_table qp_table_420_8bpc_max = { +static const qp_table qp_table_420_8bpc_max = { { 4, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 13, 14} }, { 4.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} }, { 5, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 12, 13} }, @@ -206,7 +206,7 @@ const qp_table qp_table_420_8bpc_max = { }; -const qp_table qp_table_444_8bpc_min = { +static const qp_table qp_table_444_8bpc_min = { { 6, { 0, 1, 3, 3, 5, 5, 5, 5, 5, 6, 6, 6, 6, 9, 14} }, { 6.5, { 0, 1, 2, 3, 4, 4, 5, 5, 5, 5, 6, 6, 6, 9, 14} }, { 7, { 0, 0, 2, 2, 4, 4, 4, 4, 4, 5, 5, 6, 6, 9, 13} }, @@ -247,7 +247,7 @@ const qp_table qp_table_444_8bpc_min = { }; -const qp_table qp_table_444_12bpc_min = { +static const qp_table qp_table_444_12bpc_min = { { 6, { 0, 5, 11, 11, 13, 13, 13, 13, 13, 14, 14, 14, 14, 17, 22} }, { 6.5, { 0, 5, 10, 11, 12, 12, 13, 13, 13, 13, 14, 14, 14, 17, 22} }, { 7, { 0, 5, 10, 10, 12, 12, 12, 12, 12, 13, 13, 14, 14, 17, 21} }, @@ -312,7 +312,7 @@ const qp_table qp_table_444_12bpc_min = { }; -const qp_table qp_table_420_12bpc_min = { +static const qp_table qp_table_420_12bpc_min = { { 4, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 21} }, { 4.5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} }, { 5, { 0, 4, 8, 9, 10, 11, 11, 11, 11, 11, 13, 13, 13, 15, 20} }, @@ -345,7 +345,7 @@ const qp_table qp_table_420_12bpc_min = { }; -const qp_table qp_table_422_12bpc_min = { +static const qp_table qp_table_422_12bpc_min = { { 6, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} }, { 6.5, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 16, 20} }, { 7, { 0, 4, 9, 10, 11, 11, 11, 11, 11, 11, 13, 13, 13, 15, 19} }, @@ -386,7 +386,7 @@ const qp_table qp_table_422_12bpc_min = { }; -const qp_table qp_table_422_12bpc_max = { +static const qp_table qp_table_422_12bpc_max = { { 6, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} }, { 6.5, {12, 12, 13, 14, 15, 15, 15, 16, 17, 18, 18, 19, 19, 20, 21} }, { 7, {11, 12, 13, 14, 15, 15, 15, 16, 17, 17, 18, 18, 19, 19, 20} }, @@ -427,7 +427,7 @@ const qp_table qp_table_422_12bpc_max = { }; -const qp_table qp_table_444_12bpc_max = { +static const qp_table qp_table_444_12bpc_max = { { 6, {12, 14, 16, 16, 17, 17, 17, 18, 19, 20, 20, 20, 20, 21, 23} }, { 6.5, {12, 14, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 20, 21, 23} }, { 7, {12, 13, 15, 15, 16, 16, 16, 17, 18, 19, 19, 20, 20, 21, 22} }, @@ -492,7 +492,7 @@ const qp_table qp_table_444_12bpc_max = { }; -const qp_table qp_table_420_8bpc_min = { +static const qp_table qp_table_420_8bpc_min = { { 4, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 9, 13} }, { 4.5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} }, { 5, { 0, 0, 1, 1, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} }, @@ -513,7 +513,7 @@ const qp_table qp_table_420_8bpc_min = { }; -const qp_table qp_table_422_8bpc_min = { +static const qp_table qp_table_422_8bpc_min = { { 6, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} }, { 6.5, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 8, 12} }, { 7, { 0, 0, 1, 2, 3, 3, 3, 3, 3, 3, 5, 5, 5, 7, 11} }, @@ -538,7 +538,7 @@ const qp_table qp_table_422_8bpc_min = { }; -const qp_table qp_table_422_10bpc_max = { +static const qp_table qp_table_422_10bpc_max = { { 6, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} }, { 6.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} }, { 7, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 15, 16} }, @@ -571,7 +571,7 @@ const qp_table qp_table_422_10bpc_max = { }; -const qp_table qp_table_420_10bpc_max = { +static const qp_table qp_table_420_10bpc_max = { { 4, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 17, 18} }, { 4.5, { 8, 8, 9, 10, 11, 11, 11, 12, 13, 14, 14, 15, 15, 16, 17} }, { 5, { 7, 8, 9, 10, 11, 11, 11, 12, 13, 13, 14, 14, 15, 16, 17} }, @@ -598,7 +598,7 @@ const qp_table qp_table_420_10bpc_max = { }; -const qp_table qp_table_420_10bpc_min = { +static const qp_table qp_table_420_10bpc_min = { { 4, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 13, 17} }, { 4.5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} }, { 5, { 0, 4, 4, 5, 7, 7, 7, 7, 7, 7, 9, 9, 9, 12, 16} }, @@ -625,7 +625,7 @@ const qp_table qp_table_420_10bpc_min = { }; -const qp_table qp_table_444_10bpc_max = { +static const qp_table qp_table_444_10bpc_max = { { 6, { 8, 10, 12, 12, 13, 13, 13, 14, 15, 16, 16, 16, 16, 17, 19} }, { 6.5, { 8, 10, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 16, 17, 19} }, { 7, { 8, 9, 11, 11, 12, 12, 12, 13, 14, 15, 15, 16, 16, 17, 18} }, @@ -678,7 +678,7 @@ const qp_table qp_table_444_10bpc_max = { }; -const qp_table qp_table_422_8bpc_max = { +static const qp_table qp_table_422_8bpc_max = { { 6, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} }, { 6.5, { 4, 4, 5, 6, 7, 7, 7, 8, 9, 10, 10, 11, 11, 12, 13} }, { 7, { 3, 4, 5, 6, 7, 7, 7, 8, 9, 9, 10, 10, 11, 11, 12} }, -- GitLab From 72c9abd5399d9f5c20a5b93737e67edba3189079 Mon Sep 17 00:00:00 2001 From: ruanjinjie Date: Wed, 19 Oct 2022 14:31:15 +0800 Subject: [PATCH 0388/1927] drm/amd/display: make dcn32_mmhubbub_funcs static The symbol is not used outside of the file, so mark it static. Fixes the following warning: drivers/gpu/drm/amd/amdgpu/../display/dc/dcn32/dcn32_mmhubbub.c:214:28: warning: symbol 'dcn32_mmhubbub_funcs' was not declared. Should it be static? Reviewed-by: Harry Wentland Signed-off-by: ruanjinjie Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c index 41b0baf8e1837..c3b089ba511a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c @@ -211,7 +211,7 @@ static void mmhubbub32_config_mcif_arb(struct mcif_wb *mcif_wb, REG_UPDATE(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB_CLIENT_ARBITRATION_SLICE, params->arbitration_slice); } -const struct mcif_wb_funcs dcn32_mmhubbub_funcs = { +static const struct mcif_wb_funcs dcn32_mmhubbub_funcs = { .warmup_mcif = mmhubbub32_warmup_mcif, .enable_mcif = mmhubbub2_enable_mcif, .disable_mcif = mmhubbub2_disable_mcif, -- GitLab From 178919f8ff2d4cd1ad32680eb767e6761e9d00ea Mon Sep 17 00:00:00 2001 From: ruanjinjie Date: Wed, 19 Oct 2022 14:23:47 +0800 Subject: [PATCH 0389/1927] drm/amd/display: make dcn32_mpc_funcs static The symbol is not used outside of the file, so mark it static. Fixes the following warning: drivers/gpu/drm/amd/amdgpu/../display/dc/dcn32/dcn32_mpc.c:985:24: warning: symbol 'dcn32_mpc_funcs' was not declared. Should it be static? Reviewed-by: Harry Wentland Signed-off-by: ruanjinjie Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c index 4edd0655965b8..206a5ddbaf6d1 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c @@ -982,7 +982,7 @@ static bool mpc32_program_3dlut( return true; } -const struct mpc_funcs dcn32_mpc_funcs = { +static const struct mpc_funcs dcn32_mpc_funcs = { .read_mpcc_state = mpc1_read_mpcc_state, .insert_plane = mpc1_insert_plane, .remove_mpcc = mpc1_remove_mpcc, -- GitLab From 3a3e841d050d9d919317ad2b65d80fc9c8c799dc Mon Sep 17 00:00:00 2001 From: Deming Wang Date: Thu, 6 Oct 2022 22:26:51 -0400 Subject: [PATCH 0390/1927] drm/amdkfd: use vma_lookup() instead of find_vma() Using vma_lookup() verifies the start address is contained in the found vma. This results in easier to read the code. Signed-off-by: Deming Wang Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 8 ++++---- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 2797029bd5001..d4e6de2a6bf67 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -529,8 +529,8 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc, for (addr = start; addr < end;) { unsigned long next; - vma = find_vma(mm, addr); - if (!vma || addr < vma->vm_start) + vma = vma_lookup(mm, addr); + if (!vma) break; next = min(vma->vm_end, end); @@ -798,8 +798,8 @@ int svm_migrate_vram_to_ram(struct svm_range *prange, struct mm_struct *mm, for (addr = start; addr < end;) { unsigned long next; - vma = find_vma(mm, addr); - if (!vma || addr < vma->vm_start) { + vma = vma_lookup(mm, addr); + if (!vma) { pr_debug("failed to find vma for prange %p\n", prange); r = -EFAULT; break; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 64fdf63093a00..0100812478b2e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -1586,8 +1586,8 @@ static int svm_range_validate_and_map(struct mm_struct *mm, unsigned long npages; bool readonly; - vma = find_vma(mm, addr); - if (!vma || addr < vma->vm_start) { + vma = vma_lookup(mm, addr); + if (!vma) { r = -EFAULT; goto unreserve_out; } @@ -2542,8 +2542,8 @@ svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr, struct interval_tree_node *node; unsigned long start_limit, end_limit; - vma = find_vma(p->mm, addr << PAGE_SHIFT); - if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) { + vma = vma_lookup(p->mm, addr << PAGE_SHIFT); + if (!vma) { pr_debug("VMA does not exist in address [0x%llx]\n", addr); return -EFAULT; } @@ -2871,8 +2871,8 @@ svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid, /* __do_munmap removed VMA, return success as we are handling stale * retry fault. */ - vma = find_vma(mm, addr << PAGE_SHIFT); - if (!vma || (addr << PAGE_SHIFT) < vma->vm_start) { + vma = vma_lookup(mm, addr << PAGE_SHIFT); + if (!vma) { pr_debug("address 0x%llx VMA is removed\n", addr); r = 0; goto out_unlock_range; @@ -3152,9 +3152,8 @@ svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size) start <<= PAGE_SHIFT; end = start + (size << PAGE_SHIFT); do { - vma = find_vma(p->mm, start); - if (!vma || start < vma->vm_start || - (vma->vm_flags & device_vma)) + vma = vma_lookup(p->mm, start); + if (!vma || (vma->vm_flags & device_vma)) return -EFAULT; start = min(end, vma->vm_end); } while (start < end); -- GitLab From b72362962a66693cd095389cbe41dd005bfcfb44 Mon Sep 17 00:00:00 2001 From: David Francis Date: Fri, 16 Sep 2022 11:15:01 -0400 Subject: [PATCH 0391/1927] drm/amd: Add IMU fw version to fw version queries IMU is a new firmware for GFX11. There are four means by which firmware version can be queried from the driver: device attributes, vf2pf, debugfs, and the AMDGPU_INFO_FW_VERSION option in the amdgpu info ioctl. Add IMU as an option for those four methods. V2: Added debugfs Reviewed-by: Likun Gao Reviewed-by: Alex Deucher Signed-off-by: David Francis Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 13 +++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c | 4 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h | 1 + include/uapi/drm/amdgpu_drm.h | 2 ++ 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index fe23e09eec985..bf1ff8f0e7120 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -344,6 +344,10 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->ver = adev->mes.ucode_fw_version[1]; fw_info->feature = 0; break; + case AMDGPU_INFO_FW_IMU: + fw_info->ver = adev->gfx.imu_fw_version; + fw_info->feature = 0; + break; default: return -EINVAL; } @@ -1520,6 +1524,15 @@ static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused) fw_info.feature, fw_info.ver); } + /* IMU */ + query_fw.fw_type = AMDGPU_INFO_FW_IMU; + query_fw.index = 0; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "IMU feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + /* PSP SOS */ query_fw.fw_type = AMDGPU_INFO_FW_SOS; ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index dd0bc649a57d1..5cb62e6249c23 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -698,6 +698,7 @@ FW_VERSION_ATTR(rlc_srlg_fw_version, 0444, gfx.rlc_srlg_fw_version); FW_VERSION_ATTR(rlc_srls_fw_version, 0444, gfx.rlc_srls_fw_version); FW_VERSION_ATTR(mec_fw_version, 0444, gfx.mec_fw_version); FW_VERSION_ATTR(mec2_fw_version, 0444, gfx.mec2_fw_version); +FW_VERSION_ATTR(imu_fw_version, 0444, gfx.imu_fw_version); FW_VERSION_ATTR(sos_fw_version, 0444, psp.sos.fw_version); FW_VERSION_ATTR(asd_fw_version, 0444, psp.asd_context.bin_desc.fw_version); FW_VERSION_ATTR(ta_ras_fw_version, 0444, psp.ras_context.context.bin_desc.fw_version); @@ -719,7 +720,8 @@ static struct attribute *fw_attrs[] = { &dev_attr_ta_ras_fw_version.attr, &dev_attr_ta_xgmi_fw_version.attr, &dev_attr_smc_fw_version.attr, &dev_attr_sdma_fw_version.attr, &dev_attr_sdma2_fw_version.attr, &dev_attr_vcn_fw_version.attr, - &dev_attr_dmcu_fw_version.attr, NULL + &dev_attr_dmcu_fw_version.attr, &dev_attr_imu_fw_version.attr, + NULL }; static const struct attribute_group fw_attr_group = { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 9c765b04aae3b..c73abe54d9747 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -547,6 +547,7 @@ static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev) POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version); + POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_IMU, adev->gfx.imu_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, adev->psp.asd_context.bin_desc.fw_version); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h index e78e4c27b62ad..6c97148ca0ed3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h @@ -70,6 +70,7 @@ enum amd_sriov_ucode_engine_id { AMD_SRIOV_UCODE_ID_RLC_SRLS, AMD_SRIOV_UCODE_ID_MEC, AMD_SRIOV_UCODE_ID_MEC2, + AMD_SRIOV_UCODE_ID_IMU, AMD_SRIOV_UCODE_ID_SOS, AMD_SRIOV_UCODE_ID_ASD, AMD_SRIOV_UCODE_ID_TA_RAS, diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 7ee65c0b4f70a..0d93ec132ebbc 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -763,6 +763,8 @@ struct drm_amdgpu_cs_chunk_data { #define AMDGPU_INFO_FW_MES_KIQ 0x19 /* Subquery id: Query MES firmware version */ #define AMDGPU_INFO_FW_MES 0x1a + /* Subquery id: Query IMU firmware version */ + #define AMDGPU_INFO_FW_IMU 0x1b /* number of bytes moved for TTM migration */ #define AMDGPU_INFO_NUM_BYTES_MOVED 0x0f -- GitLab From a9232b068e3698b8aa4a29fb7292bfdeb0c8f4ff Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Thu, 20 Oct 2022 14:44:26 +0800 Subject: [PATCH 0392/1927] drm/amdkfd: update gfx1037 Lx cache setting Update the gfx1037 L1/L2 cache setting. Signed-off-by: Prike Liang Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index cd5f8b219bf94..d6fa787ff4901 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -795,6 +795,54 @@ static struct kfd_gpu_cache_info yellow_carp_cache_info[] = { }, }; +static struct kfd_gpu_cache_info gfx1037_cache_info[] = { + { + /* TCP L1 Cache per CU */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 1, + }, + { + /* Scalar L1 Instruction Cache per SQC */ + .cache_size = 32, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_INST_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* Scalar L1 Data Cache per SQC */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* GL1 Data Cache per SA */ + .cache_size = 128, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* L2 Data Cache per GPU (Total Tex Cache) */ + .cache_size = 256, + .cache_level = 2, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, +}; + static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, struct crat_subtype_computeunit *cu) { @@ -1515,10 +1563,13 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, break; case IP_VERSION(10, 3, 3): case IP_VERSION(10, 3, 6): /* TODO: Double check these on production silicon */ - case IP_VERSION(10, 3, 7): /* TODO: Double check these on production silicon */ pcache_info = yellow_carp_cache_info; num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info); break; + case IP_VERSION(10, 3, 7): + pcache_info = gfx1037_cache_info; + num_of_cache_types = ARRAY_SIZE(gfx1037_cache_info); + break; case IP_VERSION(11, 0, 0): case IP_VERSION(11, 0, 1): case IP_VERSION(11, 0, 2): -- GitLab From d62eaddbc9ae27e1d98389b74778b29d4c1388d4 Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Tue, 11 Oct 2022 05:23:10 +0000 Subject: [PATCH 0393/1927] drm/amdkfd: correct the cache info for gfx1036 correct the cache information for gfx1036 Acked-by: Alex Deucher Reviewed-by: Yifan Zhang Signed-off-by: Yifan Zhang Signed-off-by: Jesse Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index d6fa787ff4901..8bfdfd062ff64 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -843,6 +843,54 @@ static struct kfd_gpu_cache_info gfx1037_cache_info[] = { }, }; +static struct kfd_gpu_cache_info gc_10_3_6_cache_info[] = { + { + /* TCP L1 Cache per CU */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 1, + }, + { + /* Scalar L1 Instruction Cache per SQC */ + .cache_size = 32, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_INST_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* Scalar L1 Data Cache per SQC */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* GL1 Data Cache per SA */ + .cache_size = 128, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* L2 Data Cache per GPU (Total Tex Cache) */ + .cache_size = 256, + .cache_level = 2, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, +}; + static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, struct crat_subtype_computeunit *cu) { @@ -1562,10 +1610,13 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, num_of_cache_types = ARRAY_SIZE(beige_goby_cache_info); break; case IP_VERSION(10, 3, 3): - case IP_VERSION(10, 3, 6): /* TODO: Double check these on production silicon */ pcache_info = yellow_carp_cache_info; num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info); break; + case IP_VERSION(10, 3, 6): + pcache_info = gc_10_3_6_cache_info; + num_of_cache_types = ARRAY_SIZE(gc_10_3_6_cache_info); + break; case IP_VERSION(10, 3, 7): pcache_info = gfx1037_cache_info; num_of_cache_types = ARRAY_SIZE(gfx1037_cache_info); -- GitLab From fd72e2cb2f9dd2734e8013b3e185a21f0d605d3e Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Fri, 21 Oct 2022 16:38:48 -0400 Subject: [PATCH 0394/1927] drm/amdkfd: introduce dummy cache info for property asic This dummy cache info will enable kfd base function support. Signed-off-by: Prike Liang Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 8bfdfd062ff64..5e0bad7e0b774 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -891,6 +891,54 @@ static struct kfd_gpu_cache_info gc_10_3_6_cache_info[] = { }, }; +static struct kfd_gpu_cache_info dummy_cache_info[] = { + { + /* TCP L1 Cache per CU */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 1, + }, + { + /* Scalar L1 Instruction Cache per SQC */ + .cache_size = 32, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_INST_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* Scalar L1 Data Cache per SQC */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* GL1 Data Cache per SA */ + .cache_size = 128, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 6, + }, + { + /* L2 Data Cache per GPU (Total Tex Cache) */ + .cache_size = 2048, + .cache_level = 2, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 6, + }, +}; + static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, struct crat_subtype_computeunit *cu) { @@ -1630,7 +1678,10 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, kfd_fill_gpu_cache_info_from_gfx_config(kdev, pcache_info); break; default: - return -EINVAL; + pcache_info = dummy_cache_info; + num_of_cache_types = ARRAY_SIZE(dummy_cache_info); + pr_warn("dummy cache info is used temporarily and real cache info need update later.\n"); + break; } } -- GitLab From 693073a04de491c82cd0e1d6bf979192375373c1 Mon Sep 17 00:00:00 2001 From: YuBiao Wang Date: Wed, 19 Oct 2022 11:36:32 +0800 Subject: [PATCH 0395/1927] drm/amdgpu: skip mes self test for gc 11.0.3 in recover Temporary disable mes self teset for gc 11.0.3 during gpu_recovery. Signed-off-by: YuBiao Wang Acked-by: Luben Tuikov Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index e0445e8cc3424..5b8362727226d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -5381,7 +5381,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res); } - if (adev->enable_mes) + if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3)) amdgpu_mes_self_test(tmp_adev); if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) { -- GitLab From f543d28687480fad06b708bc6e0b0b6ec953b078 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Fri, 21 Oct 2022 10:04:40 +0800 Subject: [PATCH 0396/1927] drm/amdgpu: disallow gfxoff until GC IP blocks complete s2idle resume In the S2idle suspend/resume phase the gfxoff is keeping functional so some IP blocks will be likely to reinitialize at gfxoff entry and that will result in failing to program GC registers.Therefore, let disallow gfxoff until AMDGPU IPs reinitialized completely. Signed-off-by: Prike Liang Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 5b8362727226d..ddaecb2610c94 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3210,6 +3210,15 @@ static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) return r; } adev->ip_blocks[i].status.hw = true; + + if (adev->in_s0ix && adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { + /* disable gfxoff for IP resume. The gfxoff will be re-enabled in + * amdgpu_device_resume() after IP resume. + */ + amdgpu_gfx_off_ctrl(adev, false); + DRM_DEBUG("will disable gfxoff for re-initializing other blocks\n"); + } + } return 0; @@ -4185,6 +4194,13 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) /* Make sure IB tests flushed */ flush_delayed_work(&adev->delayed_init_work); + if (adev->in_s0ix) { + /* re-enable gfxoff after IP resume. This re-enables gfxoff after + * it was disabled for IP resume in amdgpu_device_ip_resume_phase2(). + */ + amdgpu_gfx_off_ctrl(adev, true); + DRM_DEBUG("will enable gfxoff for the mission mode\n"); + } if (fbcon) drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false); -- GitLab From b66fca42c4e5739fa7ce17edd1c4d451b0d3c78d Mon Sep 17 00:00:00 2001 From: Fangzhi Zuo Date: Thu, 20 Oct 2022 11:46:30 -0400 Subject: [PATCH 0397/1927] drm/amd/display: Add UHBR135 and UHBR20 into debugfs Add support to manually force link rate to UHBR135 (0x546) and UHBR20 (0x7d0). Tested-by: Mark Broadworth Reviewed-by: Roman Li Acked-by: Rodrigo Siqueira Signed-off-by: Fangzhi Zuo Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index ee242d9d8b060..0c717d7b12e56 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -299,6 +299,8 @@ static ssize_t dp_link_settings_write(struct file *f, const char __user *buf, case LINK_RATE_HIGH2: case LINK_RATE_HIGH3: case LINK_RATE_UHBR10: + case LINK_RATE_UHBR13_5: + case LINK_RATE_UHBR20: break; default: valid_input = false; -- GitLab From bbfbf09d193ac831c40db50ef4b31d11548a9eef Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:46:31 -0400 Subject: [PATCH 0398/1927] drm/amd/display: Remove FPU guards from the DML folder As part of the programming expectation for using DML functions, DC requires that any DML function invoked outside DML uses: DC_FP_START(); ... dml function ... DC_FP_END(); Additionally, all the DML functions that can be invoked outside the DML folder call the function dc_assert_fp_enabled(), which is responsible for triggering a warning in the case that the DML function was not guarded by the DC_FP_START/END. For this reason, call DC_FP_START/END inside DML is wrong, and this commit removes all of those references. Tested-by: Mark Broadworth Reviewed-by: Nevenko Stupar Reviewed-by: Jun Lei Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/dml/dcn30/dcn30_fpu.c | 2 -- .../drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 17 +---------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c index e1e92daba6686..d4c0f9cdac8e2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c @@ -520,9 +520,7 @@ void dcn30_fpu_calculate_wm_and_dlg( pipe_idx++; } - DC_FP_START(); dcn20_calculate_dlg_params(dc, context, pipes, pipe_cnt, vlevel); - DC_FP_END(); if (!pstate_en) /* Restore full p-state latency */ diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index 819de0f110126..d8e68f13c2886 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -1191,9 +1191,7 @@ static void dcn32_full_validate_bw_helper(struct dc *dc, } } else { // Most populate phantom DLG params before programming hardware / timing for phantom pipe - DC_FP_START(); dcn32_helper_populate_phantom_dlg_params(dc, context, pipes, *pipe_cnt); - DC_FP_END(); /* Call validate_apply_pipe_split flags after calling DML getters for * phantom dlg params, or some of the VBA params indicating pipe split @@ -1494,11 +1492,8 @@ bool dcn32_internal_validate_bw(struct dc *dc, dml_log_pipe_params(&context->bw_ctx.dml, pipes, pipe_cnt); - if (!fast_validate) { - DC_FP_START(); + if (!fast_validate) dcn32_full_validate_bw_helper(dc, context, pipes, &vlevel, split, merge, &pipe_cnt); - DC_FP_END(); - } if (fast_validate || (dc->debug.dml_disallow_alternate_prefetch_modes && @@ -2145,9 +2140,7 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params, entry.fabricclk_mhz = 0; entry.dram_speed_mts = 0; - DC_FP_START(); insert_entry_into_table_sorted(table, num_entries, &entry); - DC_FP_END(); } // Insert the max DCFCLK @@ -2155,9 +2148,7 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params, entry.fabricclk_mhz = 0; entry.dram_speed_mts = 0; - DC_FP_START(); insert_entry_into_table_sorted(table, num_entries, &entry); - DC_FP_END(); // Insert the UCLK DPMS for (i = 0; i < num_uclk_dpms; i++) { @@ -2165,9 +2156,7 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params, entry.fabricclk_mhz = 0; entry.dram_speed_mts = bw_params->clk_table.entries[i].memclk_mhz * 16; - DC_FP_START(); insert_entry_into_table_sorted(table, num_entries, &entry); - DC_FP_END(); } // If FCLK is coarse grained, insert individual DPMs. @@ -2177,9 +2166,7 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params, entry.fabricclk_mhz = bw_params->clk_table.entries[i].fclk_mhz; entry.dram_speed_mts = 0; - DC_FP_START(); insert_entry_into_table_sorted(table, num_entries, &entry); - DC_FP_END(); } } // If FCLK fine grained, only insert max @@ -2188,9 +2175,7 @@ static int build_synthetic_soc_states(struct clk_bw_params *bw_params, entry.fabricclk_mhz = max_fclk_mhz; entry.dram_speed_mts = 0; - DC_FP_START(); insert_entry_into_table_sorted(table, num_entries, &entry); - DC_FP_END(); } // At this point, the table contains all "points of interest" based on -- GitLab From 14e2739c3695734bb4f5a0deca6fff4e10e5ef5d Mon Sep 17 00:00:00 2001 From: Wenjing Liu Date: Thu, 20 Oct 2022 11:46:32 -0400 Subject: [PATCH 0399/1927] drm/amd/display: refactor enable/disable audio stream hw sequence [why] 1. As recommended by hardware team, don't enable APG when stream is not enabled. 2. Move audio stream encoder programming into link_hwss. [how] 1. Merge dp_audio_enable into enable audio stream hw sequence. 2. Move stream encoder programming into link hwss level to unify stream encoder programming interface. Tested-by: Mark Broadworth Reviewed-by: Charlene Liu Acked-by: Rodrigo Siqueira Signed-off-by: Wenjing Liu Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc_link.c | 6 +-- .../gpu/drm/amd/display/dc/core/dc_link_dp.c | 14 +----- .../display/dc/dce110/dce110_hw_sequencer.c | 50 ++++--------------- .../drm/amd/display/dc/dcn20/dcn20_hwseq.c | 8 --- .../gpu/drm/amd/display/dc/dcn31/dcn31_apg.c | 2 - .../gpu/drm/amd/display/dc/inc/link_hwss.h | 2 + .../drm/amd/display/dc/link/link_hwss_dio.c | 35 +++++++++++++ .../drm/amd/display/dc/link/link_hwss_dio.h | 2 + .../drm/amd/display/dc/link/link_hwss_dpia.c | 2 + .../amd/display/dc/link/link_hwss_hpo_dp.c | 15 ++++++ 10 files changed, 68 insertions(+), 68 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index d7b1ace6328a0..24ed057414e1e 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -4229,6 +4229,7 @@ static void fpga_dp_hpo_enable_link_and_stream(struct dc_state *state, struct pi link_hwss->ext.set_throttled_vcp_size(pipe_ctx, avg_time_slots_per_mtp); dc->hwss.unblank_stream(pipe_ctx, &stream->link->cur_link_settings); + dc->hwss.enable_audio_stream(pipe_ctx); } void core_link_enable_stream( @@ -4308,10 +4309,7 @@ void core_link_enable_stream( /* Still enable stream features & audio on seamless boot for DP external displays */ if (pipe_ctx->stream->signal == SIGNAL_TYPE_DISPLAY_PORT) { enable_stream_features(pipe_ctx); - if (pipe_ctx->stream_res.audio != NULL) { - pipe_ctx->stream_res.stream_enc->funcs->dp_audio_enable(pipe_ctx->stream_res.stream_enc); - dc->hwss.enable_audio_stream(pipe_ctx); - } + dc->hwss.enable_audio_stream(pipe_ctx); } #if defined(CONFIG_DRM_AMD_DC_HDCP) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index 1254d38f1778a..bd4cbe02d353a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -7339,19 +7339,7 @@ void dp_retrain_link_dp_test(struct dc_link *link, link->dc->hwss.unblank_stream(&pipes[i], link_setting); - if (pipes[i].stream_res.audio) { - /* notify audio driver for - * audio modes of monitor */ - pipes[i].stream_res.audio->funcs->az_enable( - pipes[i].stream_res.audio); - - /* un-mute audio */ - /* TODO: audio should be per stream rather than - * per link */ - pipes[i].stream_res.stream_enc->funcs-> - audio_mute_control( - pipes[i].stream_res.stream_enc, false); - } + link->dc->hwss.enable_audio_stream(&pipes[i]); } } } diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index d260eaa1509ed..a34bc631be008 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -688,16 +688,6 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx) early_control = lane_count; tg->funcs->set_early_control(tg, early_control); - - /* enable audio only within mode set */ - if (pipe_ctx->stream_res.audio != NULL) { - if (dc_is_dp_signal(pipe_ctx->stream->signal)) - pipe_ctx->stream_res.stream_enc->funcs->dp_audio_enable(pipe_ctx->stream_res.stream_enc); - } - - - - } static enum bp_result link_transmitter_control( @@ -1081,12 +1071,14 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx) struct dc *dc; struct clk_mgr *clk_mgr; unsigned int i, num_audio = 1; + const struct link_hwss *link_hwss; if (!pipe_ctx->stream) return; dc = pipe_ctx->stream->ctx->dc; clk_mgr = dc->clk_mgr; + link_hwss = get_link_hwss(pipe_ctx->stream->link, &pipe_ctx->link_res); if (pipe_ctx->stream_res.audio && pipe_ctx->stream_res.audio->enabled == true) return; @@ -1103,56 +1095,35 @@ void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx) if (num_audio >= 1 && clk_mgr->funcs->enable_pme_wa) /*this is the first audio. apply the PME w/a in order to wake AZ from D3*/ clk_mgr->funcs->enable_pme_wa(clk_mgr); - /* un-mute audio */ - /* TODO: audio should be per stream rather than per link */ - if (is_dp_128b_132b_signal(pipe_ctx)) - pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->audio_mute_control( - pipe_ctx->stream_res.hpo_dp_stream_enc, false); - else - pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control( - pipe_ctx->stream_res.stream_enc, false); + + link_hwss->enable_audio_packet(pipe_ctx); + if (pipe_ctx->stream_res.audio) pipe_ctx->stream_res.audio->enabled = true; } - - if (dc_is_dp_signal(pipe_ctx->stream->signal)) - dp_source_sequence_trace(pipe_ctx->stream->link, DPCD_SOURCE_SEQ_AFTER_ENABLE_AUDIO_STREAM); } void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx) { struct dc *dc; struct clk_mgr *clk_mgr; + const struct link_hwss *link_hwss; if (!pipe_ctx || !pipe_ctx->stream) return; dc = pipe_ctx->stream->ctx->dc; clk_mgr = dc->clk_mgr; + link_hwss = get_link_hwss(pipe_ctx->stream->link, &pipe_ctx->link_res); if (pipe_ctx->stream_res.audio && pipe_ctx->stream_res.audio->enabled == false) return; - if (is_dp_128b_132b_signal(pipe_ctx)) - pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->audio_mute_control( - pipe_ctx->stream_res.hpo_dp_stream_enc, true); - else - pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control( - pipe_ctx->stream_res.stream_enc, true); + link_hwss->disable_audio_packet(pipe_ctx); + if (pipe_ctx->stream_res.audio) { pipe_ctx->stream_res.audio->enabled = false; - if (dc_is_dp_signal(pipe_ctx->stream->signal)) - if (is_dp_128b_132b_signal(pipe_ctx)) - pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_disable( - pipe_ctx->stream_res.hpo_dp_stream_enc); - else - pipe_ctx->stream_res.stream_enc->funcs->dp_audio_disable( - pipe_ctx->stream_res.stream_enc); - else - pipe_ctx->stream_res.stream_enc->funcs->hdmi_audio_disable( - pipe_ctx->stream_res.stream_enc); - if (clk_mgr->funcs->enable_pme_wa) /*this is the first audio. apply the PME w/a in order to wake AZ from D3*/ clk_mgr->funcs->enable_pme_wa(clk_mgr); @@ -1163,9 +1134,6 @@ void dce110_disable_audio_stream(struct pipe_ctx *pipe_ctx) * stream->stream_engine_id); */ } - - if (dc_is_dp_signal(pipe_ctx->stream->signal)) - dp_source_sequence_trace(pipe_ctx->stream->link, DPCD_SOURCE_SEQ_AFTER_DISABLE_AUDIO_STREAM); } void dce110_disable_stream(struct pipe_ctx *pipe_ctx) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index a7e0001a8f46d..1ac0ed5be605d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -2601,14 +2601,6 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx) if (dc->hwseq->funcs.set_pixels_per_cycle) dc->hwseq->funcs.set_pixels_per_cycle(pipe_ctx); - - /* enable audio only within mode set */ - if (pipe_ctx->stream_res.audio != NULL) { - if (is_dp_128b_132b_signal(pipe_ctx)) - pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_enable(pipe_ctx->stream_res.hpo_dp_stream_enc); - else if (dc_is_dp_signal(pipe_ctx->stream->signal)) - pipe_ctx->stream_res.stream_enc->funcs->dp_audio_enable(pipe_ctx->stream_res.stream_enc); - } } void dcn20_program_dmdata_engine(struct pipe_ctx *pipe_ctx) diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c index de5e18c2a3ac8..2cec7131a96f8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c @@ -134,8 +134,6 @@ static void apg31_se_audio_setup( /* Disable forced mem power off */ REG_UPDATE(APG_MEM_PWR, APG_MEM_PWR_FORCE, 0); - - apg31_enable(apg); } static void apg31_audio_mute_control( diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h index 89964c980b871..7a2c8bc280d50 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h +++ b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h @@ -79,6 +79,8 @@ struct link_hwss { void (*disable_link_output)(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal); + void (*enable_audio_packet)(struct pipe_ctx *pipe_ctx); + void (*disable_audio_packet)(struct pipe_ctx *pipe_ctx); }; #endif /* __DC_LINK_HWSS_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c index 4227adbc646a8..fcc3ce2b8fbf8 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c @@ -170,11 +170,46 @@ static void update_dio_stream_allocation_table(struct dc_link *link, link_enc->funcs->update_mst_stream_allocation_table(link_enc, table); } +void enable_dio_audio_packet(struct pipe_ctx *pipe_ctx) +{ + if (dc_is_dp_signal(pipe_ctx->stream->signal)) + pipe_ctx->stream_res.stream_enc->funcs->dp_audio_enable( + pipe_ctx->stream_res.stream_enc); + + pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control( + pipe_ctx->stream_res.stream_enc, false); + + if (dc_is_dp_signal(pipe_ctx->stream->signal)) + dp_source_sequence_trace(pipe_ctx->stream->link, + DPCD_SOURCE_SEQ_AFTER_ENABLE_AUDIO_STREAM); +} + +void disable_dio_audio_packet(struct pipe_ctx *pipe_ctx) +{ + pipe_ctx->stream_res.stream_enc->funcs->audio_mute_control( + pipe_ctx->stream_res.stream_enc, true); + + if (pipe_ctx->stream_res.audio) { + if (dc_is_dp_signal(pipe_ctx->stream->signal)) + pipe_ctx->stream_res.stream_enc->funcs->dp_audio_disable( + pipe_ctx->stream_res.stream_enc); + else + pipe_ctx->stream_res.stream_enc->funcs->hdmi_audio_disable( + pipe_ctx->stream_res.stream_enc); + } + + if (dc_is_dp_signal(pipe_ctx->stream->signal)) + dp_source_sequence_trace(pipe_ctx->stream->link, + DPCD_SOURCE_SEQ_AFTER_DISABLE_AUDIO_STREAM); +} + static const struct link_hwss dio_link_hwss = { .setup_stream_encoder = setup_dio_stream_encoder, .reset_stream_encoder = reset_dio_stream_encoder, .setup_stream_attribute = setup_dio_stream_attribute, .disable_link_output = disable_dio_link_output, + .enable_audio_packet = enable_dio_audio_packet, + .disable_audio_packet = disable_dio_audio_packet, .ext = { .set_throttled_vcp_size = set_dio_throttled_vcp_size, .enable_dp_link_output = enable_dio_dp_link_output, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h index 126d37f847a15..006830376e7de 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h @@ -50,5 +50,7 @@ void set_dio_dp_lane_settings(struct dc_link *link, const struct link_resource *link_res, const struct dc_link_settings *link_settings, const struct dc_lane_settings lane_settings[LANE_COUNT_DP_MAX]); +void enable_dio_audio_packet(struct pipe_ctx *pipe_ctx); +void disable_dio_audio_packet(struct pipe_ctx *pipe_ctx); #endif /* __LINK_HWSS_DIO_H__ */ diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c index 64f7ea6a9aa32..f81949221e0d0 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c @@ -57,6 +57,8 @@ static const struct link_hwss dpia_link_hwss = { .reset_stream_encoder = reset_dio_stream_encoder, .setup_stream_attribute = setup_dio_stream_attribute, .disable_link_output = disable_dio_link_output, + .enable_audio_packet = enable_dio_audio_packet, + .disable_audio_packet = disable_dio_audio_packet, .ext = { .set_throttled_vcp_size = set_dio_throttled_vcp_size, .enable_dp_link_output = enable_dio_dp_link_output, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c index 153a88381f2c7..668a66832579d 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c @@ -262,11 +262,26 @@ static void update_hpo_dp_stream_allocation_table(struct dc_link *link, table); } +static void enable_hpo_dp_audio_packet(struct pipe_ctx *pipe_ctx) +{ + pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_enable( + pipe_ctx->stream_res.hpo_dp_stream_enc); +} + +static void disable_hpo_dp_audio_packet(struct pipe_ctx *pipe_ctx) +{ + if (pipe_ctx->stream_res.audio) + pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_disable( + pipe_ctx->stream_res.hpo_dp_stream_enc); +} + static const struct link_hwss hpo_dp_link_hwss = { .setup_stream_encoder = setup_hpo_dp_stream_encoder, .reset_stream_encoder = reset_hpo_dp_stream_encoder, .setup_stream_attribute = setup_hpo_dp_stream_attribute, .disable_link_output = disable_hpo_dp_link_output, + .enable_audio_packet = enable_hpo_dp_audio_packet, + .disable_audio_packet = disable_hpo_dp_audio_packet, .ext = { .set_throttled_vcp_size = set_hpo_dp_throttled_vcp_size, .set_hblank_min_symbol_width = set_hpo_dp_hblank_min_symbol_width, -- GitLab From c859181cfeacf87336fc385c5665444f0e59eafe Mon Sep 17 00:00:00 2001 From: Wenjing Liu Date: Thu, 20 Oct 2022 11:46:33 -0400 Subject: [PATCH 0400/1927] drm/amd/display: move stream encoder audio setup to link_hwss Unify stream encoder audio setup interface. Tested-by: Mark Broadworth Reviewed-by: Charlene Liu Acked-by: Rodrigo Siqueira Signed-off-by: Wenjing Liu Signed-off-by: Alex Deucher --- .../display/dc/dce110/dce110_hw_sequencer.c | 22 +++++-------------- .../gpu/drm/amd/display/dc/inc/link_hwss.h | 3 +++ .../drm/amd/display/dc/link/link_hwss_dio.c | 17 ++++++++++++++ .../drm/amd/display/dc/link/link_hwss_dio.h | 2 ++ .../drm/amd/display/dc/link/link_hwss_dpia.c | 1 + .../amd/display/dc/link/link_hwss_hpo_dp.c | 10 +++++++++ 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index a34bc631be008..c5380ce706535 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -1455,6 +1455,9 @@ static enum dc_status apply_single_controller_ctx_to_hw( unsigned int event_triggers = 0; struct pipe_ctx *odm_pipe = pipe_ctx->next_odm_pipe; struct dce_hwseq *hws = dc->hwseq; + const struct link_hwss *link_hwss = get_link_hwss( + link, &pipe_ctx->link_res); + if (hws->funcs.disable_stream_gating) { hws->funcs.disable_stream_gating(dc, pipe_ctx); @@ -1465,23 +1468,8 @@ static enum dc_status apply_single_controller_ctx_to_hw( build_audio_output(context, pipe_ctx, &audio_output); - if (dc_is_dp_signal(pipe_ctx->stream->signal)) - if (is_dp_128b_132b_signal(pipe_ctx)) - pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_setup( - pipe_ctx->stream_res.hpo_dp_stream_enc, - pipe_ctx->stream_res.audio->inst, - &pipe_ctx->stream->audio_info); - else - pipe_ctx->stream_res.stream_enc->funcs->dp_audio_setup( - pipe_ctx->stream_res.stream_enc, - pipe_ctx->stream_res.audio->inst, - &pipe_ctx->stream->audio_info); - else - pipe_ctx->stream_res.stream_enc->funcs->hdmi_audio_setup( - pipe_ctx->stream_res.stream_enc, - pipe_ctx->stream_res.audio->inst, - &pipe_ctx->stream->audio_info, - &audio_output.crtc_info); + link_hwss->setup_audio_output(pipe_ctx, &audio_output, + pipe_ctx->stream_res.audio->inst); pipe_ctx->stream_res.audio->funcs->az_configure( pipe_ctx->stream_res.audio, diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h index 7a2c8bc280d50..0f69946cce9f6 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h +++ b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h @@ -38,6 +38,7 @@ struct link_resource; struct pipe_ctx; struct encoder_set_dp_phy_pattern_param; struct link_mst_stream_allocation_table; +struct audio_output; struct link_hwss_ext { /* function pointers below may require to check for NULL if caller @@ -79,6 +80,8 @@ struct link_hwss { void (*disable_link_output)(struct dc_link *link, const struct link_resource *link_res, enum signal_type signal); + void (*setup_audio_output)(struct pipe_ctx *pipe_ctx, + struct audio_output *audio_output, uint32_t audio_inst); void (*enable_audio_packet)(struct pipe_ctx *pipe_ctx); void (*disable_audio_packet)(struct pipe_ctx *pipe_ctx); }; diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c index fcc3ce2b8fbf8..33148b753c03b 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c @@ -170,6 +170,22 @@ static void update_dio_stream_allocation_table(struct dc_link *link, link_enc->funcs->update_mst_stream_allocation_table(link_enc, table); } +void setup_dio_audio_output(struct pipe_ctx *pipe_ctx, + struct audio_output *audio_output, uint32_t audio_inst) +{ + if (dc_is_dp_signal(pipe_ctx->stream->signal)) + pipe_ctx->stream_res.stream_enc->funcs->dp_audio_setup( + pipe_ctx->stream_res.stream_enc, + audio_inst, + &pipe_ctx->stream->audio_info); + else + pipe_ctx->stream_res.stream_enc->funcs->hdmi_audio_setup( + pipe_ctx->stream_res.stream_enc, + audio_inst, + &pipe_ctx->stream->audio_info, + &audio_output->crtc_info); +} + void enable_dio_audio_packet(struct pipe_ctx *pipe_ctx) { if (dc_is_dp_signal(pipe_ctx->stream->signal)) @@ -208,6 +224,7 @@ static const struct link_hwss dio_link_hwss = { .reset_stream_encoder = reset_dio_stream_encoder, .setup_stream_attribute = setup_dio_stream_attribute, .disable_link_output = disable_dio_link_output, + .setup_audio_output = setup_dio_audio_output, .enable_audio_packet = enable_dio_audio_packet, .disable_audio_packet = disable_dio_audio_packet, .ext = { diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h index 006830376e7de..9a108c3d78315 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h @@ -50,6 +50,8 @@ void set_dio_dp_lane_settings(struct dc_link *link, const struct link_resource *link_res, const struct dc_link_settings *link_settings, const struct dc_lane_settings lane_settings[LANE_COUNT_DP_MAX]); +void setup_dio_audio_output(struct pipe_ctx *pipe_ctx, + struct audio_output *audio_output, uint32_t audio_inst); void enable_dio_audio_packet(struct pipe_ctx *pipe_ctx); void disable_dio_audio_packet(struct pipe_ctx *pipe_ctx); diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c index f81949221e0d0..861f3cd5b3560 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c @@ -57,6 +57,7 @@ static const struct link_hwss dpia_link_hwss = { .reset_stream_encoder = reset_dio_stream_encoder, .setup_stream_attribute = setup_dio_stream_attribute, .disable_link_output = disable_dio_link_output, + .setup_audio_output = setup_dio_audio_output, .enable_audio_packet = enable_dio_audio_packet, .disable_audio_packet = disable_dio_audio_packet, .ext = { diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c index 668a66832579d..2f46e1ac4ce0e 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c @@ -262,6 +262,15 @@ static void update_hpo_dp_stream_allocation_table(struct dc_link *link, table); } +static void setup_hpo_dp_audio_output(struct pipe_ctx *pipe_ctx, + struct audio_output *audio_output, uint32_t audio_inst) +{ + pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_setup( + pipe_ctx->stream_res.hpo_dp_stream_enc, + audio_inst, + &pipe_ctx->stream->audio_info); +} + static void enable_hpo_dp_audio_packet(struct pipe_ctx *pipe_ctx) { pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->dp_audio_enable( @@ -280,6 +289,7 @@ static const struct link_hwss hpo_dp_link_hwss = { .reset_stream_encoder = reset_hpo_dp_stream_encoder, .setup_stream_attribute = setup_hpo_dp_stream_attribute, .disable_link_output = disable_hpo_dp_link_output, + .setup_audio_output = setup_hpo_dp_audio_output, .enable_audio_packet = enable_hpo_dp_audio_packet, .disable_audio_packet = disable_hpo_dp_audio_packet, .ext = { -- GitLab From 0094f042f2f88f6e0fea01f0753773a95cc975bd Mon Sep 17 00:00:00 2001 From: Wenjing Liu Date: Thu, 20 Oct 2022 11:46:34 -0400 Subject: [PATCH 0401/1927] drm/amd/display: remove audio mute control in hpo dp VPG doesn't have the ability to mute audio output by sending all 0s in audio SDP. The existing implemention is disabling audio SDP instead. This is same as what dp_audio_enable does. Since it is no longer referenced by any callers, we decided to remove this interface for simplicity. Tested-by: Mark Broadworth Reviewed-by: Charlene Liu Acked-by: Rodrigo Siqueira Signed-off-by: Wenjing Liu Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c | 11 ----------- drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.h | 4 ---- .../display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c | 9 --------- .../gpu/drm/amd/display/dc/inc/hw/stream_encoder.h | 3 --- 4 files changed, 27 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c index 2cec7131a96f8..24e9ff65434d3 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.c @@ -136,19 +136,8 @@ static void apg31_se_audio_setup( REG_UPDATE(APG_MEM_PWR, APG_MEM_PWR_FORCE, 0); } -static void apg31_audio_mute_control( - struct apg *apg, - bool mute) -{ - if (mute) - apg31_disable(apg); - else - apg31_enable(apg); -} - static struct apg_funcs dcn31_apg_funcs = { .se_audio_setup = apg31_se_audio_setup, - .audio_mute_control = apg31_audio_mute_control, .enable_apg = apg31_enable, .disable_apg = apg31_disable, }; diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.h index 24f568e120d84..1b81f6773c53f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.h +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_apg.h @@ -84,10 +84,6 @@ struct apg_funcs { unsigned int az_inst, struct audio_info *audio_info); - void (*audio_mute_control)( - struct apg *apg, - bool mute); - void (*enable_apg)( struct apg *apg); diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c index 814f401db3b34..16639bd03adf7 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c @@ -600,14 +600,6 @@ static void dcn31_hpo_dp_stream_enc_map_stream_to_link( } } -static void dcn31_hpo_dp_stream_enc_mute_control( - struct hpo_dp_stream_encoder *enc, - bool mute) -{ - ASSERT(enc->apg); - enc->apg->funcs->audio_mute_control(enc->apg, mute); -} - static void dcn31_hpo_dp_stream_enc_audio_setup( struct hpo_dp_stream_encoder *enc, unsigned int az_inst, @@ -726,7 +718,6 @@ static const struct hpo_dp_stream_encoder_funcs dcn30_str_enc_funcs = { .stop_dp_info_packets = dcn31_hpo_dp_stream_enc_stop_dp_info_packets, .dp_set_dsc_pps_info_packet = dcn31_hpo_dp_stream_enc_set_dsc_pps_info_packet, .map_stream_to_link = dcn31_hpo_dp_stream_enc_map_stream_to_link, - .audio_mute_control = dcn31_hpo_dp_stream_enc_mute_control, .dp_audio_setup = dcn31_hpo_dp_stream_enc_audio_setup, .dp_audio_enable = dcn31_hpo_dp_stream_enc_audio_enable, .dp_audio_disable = dcn31_hpo_dp_stream_enc_audio_disable, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h index 42afa1952890e..6b3e6de374bdb 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h @@ -317,9 +317,6 @@ struct hpo_dp_stream_encoder_funcs { uint32_t stream_enc_inst, uint32_t link_enc_inst); - void (*audio_mute_control)( - struct hpo_dp_stream_encoder *enc, bool mute); - void (*dp_audio_setup)( struct hpo_dp_stream_encoder *enc, unsigned int az_inst, -- GitLab From 174fc82410a8c75d3937320658fca5a240ca8176 Mon Sep 17 00:00:00 2001 From: Dmytro Laktyushkin Date: Thu, 20 Oct 2022 11:46:35 -0400 Subject: [PATCH 0402/1927] drm/amd/display: correctly populate dcn315 clock table Fix incorrect pstate read order as well as min and max state logic. Tested-by: Mark Broadworth Reviewed-by: Charlene Liu Acked-by: Rodrigo Siqueira Signed-off-by: Dmytro Laktyushkin Signed-off-by: Alex Deucher --- .../dc/clk_mgr/dcn315/dcn315_clk_mgr.c | 71 ++++++------------- 1 file changed, 23 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c index 893991a0eb971..07edd9777edfb 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c @@ -458,19 +458,6 @@ static void dcn315_get_dpm_table_from_smu(struct clk_mgr_internal *clk_mgr, dcn315_smu_transfer_dpm_table_smu_2_dram(clk_mgr); } -static uint32_t find_max_clk_value(const uint32_t clocks[], uint32_t num_clocks) -{ - uint32_t max = 0; - int i; - - for (i = 0; i < num_clocks; ++i) { - if (clocks[i] > max) - max = clocks[i]; - } - - return max; -} - static void dcn315_clk_mgr_helper_populate_bw_params( struct clk_mgr_internal *clk_mgr, struct integrated_info *bios_info, @@ -478,29 +465,21 @@ static void dcn315_clk_mgr_helper_populate_bw_params( { int i; struct clk_bw_params *bw_params = clk_mgr->base.bw_params; - uint32_t max_pstate = 0, max_fclk = 0, min_pstate = 0; + uint32_t max_pstate = clock_table->NumDfPstatesEnabled - 1; struct clk_limit_table_entry def_max = bw_params->clk_table.entries[bw_params->clk_table.num_entries - 1]; - /* Find highest fclk pstate */ - for (i = 0; i < clock_table->NumDfPstatesEnabled; i++) { - if (clock_table->DfPstateTable[i].FClk > max_fclk) { - max_fclk = clock_table->DfPstateTable[i].FClk; - max_pstate = i; - } - } - /* For 315 we want to base clock table on dcfclk, need at least one entry regardless of pmfw table */ for (i = 0; i < clock_table->NumDcfClkLevelsEnabled; i++) { int j; - uint32_t min_fclk = clock_table->DfPstateTable[0].FClk; - for (j = 1; j < clock_table->NumDfPstatesEnabled; j++) { - if (clock_table->DfPstateTable[j].Voltage <= clock_table->SocVoltage[i] - && clock_table->DfPstateTable[j].FClk < min_fclk) { - min_fclk = clock_table->DfPstateTable[j].FClk; - min_pstate = j; - } + /* DF table is sorted with clocks decreasing */ + for (j = clock_table->NumDfPstatesEnabled - 2; j >= 0; j--) { + if (clock_table->DfPstateTable[j].Voltage <= clock_table->SocVoltage[i]) + max_pstate = j; } + /* Max DCFCLK should match up with max pstate */ + if (i == clock_table->NumDcfClkLevelsEnabled - 1) + max_pstate = 0; /* First search defaults for the clocks we don't read using closest lower or equal default dcfclk */ for (j = bw_params->clk_table.num_entries - 1; j > 0; j--) @@ -511,9 +490,9 @@ static void dcn315_clk_mgr_helper_populate_bw_params( bw_params->clk_table.entries[i].dtbclk_mhz = bw_params->clk_table.entries[j].dtbclk_mhz; /* Now update clocks we do read */ - bw_params->clk_table.entries[i].fclk_mhz = min_fclk; - bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[min_pstate].MemClk; - bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[min_pstate].Voltage; + bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[max_pstate].FClk; + bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[max_pstate].MemClk; + bw_params->clk_table.entries[i].voltage = clock_table->SocVoltage[i]; bw_params->clk_table.entries[i].dcfclk_mhz = clock_table->DcfClocks[i]; bw_params->clk_table.entries[i].socclk_mhz = clock_table->SocClocks[i]; bw_params->clk_table.entries[i].dispclk_mhz = clock_table->DispClocks[i]; @@ -521,25 +500,16 @@ static void dcn315_clk_mgr_helper_populate_bw_params( bw_params->clk_table.entries[i].wck_ratio = 1; } - /* Make sure to include at least one entry and highest pstate */ - if (max_pstate != min_pstate || i == 0) { - bw_params->clk_table.entries[i].fclk_mhz = max_fclk; - bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[max_pstate].MemClk; - bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[max_pstate].Voltage; - bw_params->clk_table.entries[i].dcfclk_mhz = find_max_clk_value(clock_table->DcfClocks, NUM_DCFCLK_DPM_LEVELS); + /* Make sure to include at least one entry */ + if (i == 0) { + bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[0].FClk; + bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[0].MemClk; + bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[0].Voltage; + bw_params->clk_table.entries[i].dcfclk_mhz = clock_table->DcfClocks[0]; bw_params->clk_table.entries[i].wck_ratio = 1; i++; } - bw_params->clk_table.num_entries = i--; - - /* Make sure all highest clocks are included*/ - bw_params->clk_table.entries[i].socclk_mhz = find_max_clk_value(clock_table->SocClocks, NUM_SOCCLK_DPM_LEVELS); - bw_params->clk_table.entries[i].dispclk_mhz = find_max_clk_value(clock_table->DispClocks, NUM_DISPCLK_DPM_LEVELS); - bw_params->clk_table.entries[i].dppclk_mhz = find_max_clk_value(clock_table->DppClocks, NUM_DPPCLK_DPM_LEVELS); - ASSERT(clock_table->DcfClocks[i] == find_max_clk_value(clock_table->DcfClocks, NUM_DCFCLK_DPM_LEVELS)); - bw_params->clk_table.entries[i].phyclk_mhz = def_max.phyclk_mhz; - bw_params->clk_table.entries[i].phyclk_d18_mhz = def_max.phyclk_d18_mhz; - bw_params->clk_table.entries[i].dtbclk_mhz = def_max.dtbclk_mhz; + bw_params->clk_table.num_entries = i; /* Set any 0 clocks to max default setting. Not an issue for * power since we aren't doing switching in such case anyway @@ -565,6 +535,11 @@ static void dcn315_clk_mgr_helper_populate_bw_params( if (!bw_params->clk_table.entries[i].dtbclk_mhz) bw_params->clk_table.entries[i].dtbclk_mhz = def_max.dtbclk_mhz; } + + /* Make sure all highest default clocks are included*/ + ASSERT(bw_params->clk_table.entries[i-1].phyclk_mhz == def_max.phyclk_mhz); + ASSERT(bw_params->clk_table.entries[i-1].phyclk_d18_mhz == def_max.phyclk_d18_mhz); + ASSERT(bw_params->clk_table.entries[i-1].dtbclk_mhz == def_max.dtbclk_mhz); ASSERT(bw_params->clk_table.entries[i-1].dcfclk_mhz); bw_params->vram_type = bios_info->memory_type; bw_params->num_channels = bios_info->ma_channel_number; -- GitLab From bd829d5707730072fecc3267016a675a4789905b Mon Sep 17 00:00:00 2001 From: Ian Chen Date: Thu, 20 Oct 2022 11:46:36 -0400 Subject: [PATCH 0403/1927] drm/amd/display: Refactor eDP PSR codes We split out PSR config from "global" to "per-panel" config settings. Tested-by: Mark Broadworth Reviewed-by: Robin Chen Acked-by: Rodrigo Siqueira Signed-off-by: Ian Chen Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 - drivers/gpu/drm/amd/display/dc/dc_link.h | 14 +++++++++++--- .../gpu/drm/amd/display/dc/dcn21/dcn21_resource.c | 5 ++++- .../gpu/drm/amd/display/dc/dcn30/dcn30_resource.c | 15 +++++++++++++-- .../drm/amd/display/dc/dcn302/dcn302_resource.c | 14 +++++++++++++- .../drm/amd/display/dc/dcn303/dcn303_resource.c | 13 ++++++++++++- .../gpu/drm/amd/display/dc/dcn31/dcn31_resource.c | 4 ++++ .../drm/amd/display/dc/dcn314/dcn314_resource.c | 4 ++++ .../drm/amd/display/dc/dcn315/dcn315_resource.c | 4 ++++ .../drm/amd/display/dc/dcn316/dcn316_resource.c | 4 ++++ .../gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c | 2 +- 11 files changed, 70 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index bf938cda379a9..e4e41f2e50540 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -764,7 +764,6 @@ struct dc_debug_options { bool disable_mem_low_power; bool pstate_enabled; bool disable_dmcu; - bool disable_psr; bool force_abm_enable; bool disable_stereo_support; bool vsr_support; diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h index caf0c7af2d0b9..17f080f8af6cd 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_link.h +++ b/drivers/gpu/drm/amd/display/dc/dc_link.h @@ -117,7 +117,7 @@ struct psr_settings { * Add a struct dc_panel_config under dc_link */ struct dc_panel_config { - // extra panel power sequence parameters + /* extra panel power sequence parameters */ struct pps { unsigned int extra_t3_ms; unsigned int extra_t7_ms; @@ -127,13 +127,21 @@ struct dc_panel_config { unsigned int extra_t12_ms; unsigned int extra_post_OUI_ms; } pps; - // ABM + /* PSR */ + struct psr { + bool disable_psr; + bool disallow_psrsu; + bool rc_disable; + bool rc_allow_static_screen; + bool rc_allow_fullscreen_VPB; + } psr; + /* ABM */ struct varib { unsigned int varibright_feature_enable; unsigned int def_varibright_level; unsigned int abm_config_setting; } varib; - // edp DSC + /* edp DSC */ struct dsc { bool disable_dsc_edp; unsigned int force_dsc_edp_policy; diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c index 887081472c0d8..ce6c70e25703d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c @@ -671,12 +671,15 @@ static const struct dc_debug_options debug_defaults_diags = { .disable_pplib_wm_range = true, .disable_stutter = true, .disable_48mhz_pwrdwn = true, - .disable_psr = true, .enable_tri_buf = true, .use_max_lb = true }; static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, .ilr = { .optimize_edp_link_rate = true, }, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index 020f512e9690e..af4fe695535ed 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -723,7 +723,6 @@ static const struct dc_debug_options debug_defaults_drv = { .underflow_assert_delay_us = 0xFFFFFFFF, .dwb_fi_phase = -1, // -1 = disable, .dmub_command_table = true, - .disable_psr = false, .use_max_lb = true, .exit_idle_opt_for_cursor_updates = true }; @@ -742,11 +741,17 @@ static const struct dc_debug_options debug_defaults_diags = { .scl_reset_length10 = true, .dwb_fi_phase = -1, // -1 = disable .dmub_command_table = true, - .disable_psr = true, .enable_tri_buf = true, .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, +}; + static void dcn30_dpp_destroy(struct dpp **dpp) { kfree(TO_DCN20_DPP(*dpp)); @@ -2212,6 +2217,11 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params } } +static void dcn30_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + static const struct resource_funcs dcn30_res_pool_funcs = { .destroy = dcn30_destroy_resource_pool, .link_enc_create = dcn30_link_encoder_create, @@ -2231,6 +2241,7 @@ static const struct resource_funcs dcn30_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn30_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn30_get_panel_config_defaults, }; #define CTX ctx diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c index b925b6ddde5a3..d3945876aceda 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c @@ -112,10 +112,16 @@ static const struct dc_debug_options debug_defaults_diags = { .dwb_fi_phase = -1, // -1 = disable .dmub_command_table = true, .enable_tri_buf = true, - .disable_psr = true, .use_max_lb = true }; +static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, +}; + enum dcn302_clk_src_array_id { DCN302_CLK_SRC_PLL0, DCN302_CLK_SRC_PLL1, @@ -1132,6 +1138,11 @@ void dcn302_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param DC_FP_END(); } +static void dcn302_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} + static struct resource_funcs dcn302_res_pool_funcs = { .destroy = dcn302_destroy_resource_pool, .link_enc_create = dcn302_link_encoder_create, @@ -1151,6 +1162,7 @@ static struct resource_funcs dcn302_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn302_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn302_get_panel_config_defaults, }; static struct dc_cap_funcs cap_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c index 527d5c9028785..7e7f18bef0986 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c @@ -96,7 +96,13 @@ static const struct dc_debug_options debug_defaults_diags = { .dwb_fi_phase = -1, // -1 = disable .dmub_command_table = true, .enable_tri_buf = true, - .disable_psr = true, +}; + +static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, }; enum dcn303_clk_src_array_id { @@ -1055,6 +1061,10 @@ static void dcn303_destroy_resource_pool(struct resource_pool **pool) *pool = NULL; } +static void dcn303_get_panel_config_defaults(struct dc_panel_config *panel_config) +{ + *panel_config = panel_config_defaults; +} void dcn303_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params) { @@ -1082,6 +1092,7 @@ static struct resource_funcs dcn303_res_pool_funcs = { .release_post_bldn_3dlut = dcn30_release_post_bldn_3dlut, .update_bw_bounding_box = dcn303_update_bw_bounding_box, .patch_unknown_plane_state = dcn20_patch_unknown_plane_state, + .get_panel_config_defaults = dcn303_get_panel_config_defaults, }; static struct dc_cap_funcs cap_funcs = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c index fddc21a5a04c4..8729d6305996a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c @@ -911,6 +911,10 @@ static const struct dc_debug_options debug_defaults_diags = { }; static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, .ilr = { .optimize_edp_link_rate = true, }, diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c index d0ad72caead28..92eb56873eee6 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c @@ -937,6 +937,10 @@ static const struct dc_debug_options debug_defaults_diags = { }; static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, .ilr = { .optimize_edp_link_rate = true, }, diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c index 58746c437554f..31cbc5762eab3 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c @@ -907,6 +907,10 @@ static const struct dc_debug_options debug_defaults_diags = { }; static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, .ilr = { .optimize_edp_link_rate = true, }, diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c index 6b40a11ac83a9..af3eddc0cf32e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c @@ -906,6 +906,10 @@ static const struct dc_debug_options debug_defaults_diags = { }; static const struct dc_panel_config panel_config_defaults = { + .psr = { + .disable_psr = false, + .disallow_psrsu = false, + }, .ilr = { .optimize_edp_link_rate = true, }, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c index d680f1c5b69f8..ec2eae99b8929 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c @@ -989,7 +989,7 @@ static enum dcn_zstate_support_state decide_zstate_support(struct dc *dc, struc if (context->bw_ctx.dml.vba.StutterPeriod > 5000.0 || optimized_min_dst_y_next_start_us > 5000) return DCN_ZSTATE_SUPPORT_ALLOW; - else if (link->psr_settings.psr_version == DC_PSR_VERSION_1 && !dc->debug.disable_psr) + else if (link->psr_settings.psr_version == DC_PSR_VERSION_1 && !link->panel_config.psr.disable_psr) return DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY; else return DCN_ZSTATE_SUPPORT_DISALLOW; -- GitLab From 6be153dc8c88409e63309bcec300b03cd7bce9a7 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:46:37 -0400 Subject: [PATCH 0404/1927] drm/amd/display: Convert documentation to a kernel-doc The dc_dmub_srv file has a lot of documentation associated with SubVP that could be converted to a kernel-doc. This commit just changes the comment style to a kernel-doc. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c | 124 +++++++------------ 1 file changed, 48 insertions(+), 76 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index 0541e87e4f389..774e069366617 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -423,25 +423,20 @@ void dc_dmub_srv_get_visual_confirm_color_cmd(struct dc *dc, struct pipe_ctx *pi #ifdef CONFIG_DRM_AMD_DC_DCN /** - * *********************************************************************************************** - * populate_subvp_cmd_drr_info: Helper to populate DRR pipe info for the DMCUB subvp command + * populate_subvp_cmd_drr_info - Helper to populate DRR pipe info for the DMCUB subvp command * - * Populate the DMCUB SubVP command with DRR pipe info. All the information required for calculating - * the SubVP + DRR microschedule is populated here. + * @dc: [in] current dc state + * @subvp_pipe: [in] pipe_ctx for the SubVP pipe + * @vblank_pipe: [in] pipe_ctx for the DRR pipe + * @pipe_data: [in] Pipe data which stores the VBLANK/DRR info + * + * Populate the DMCUB SubVP command with DRR pipe info. All the information + * required for calculating the SubVP + DRR microschedule is populated here. * * High level algorithm: * 1. Get timing for SubVP pipe, phantom pipe, and DRR pipe * 2. Calculate the min and max vtotal which supports SubVP + DRR microschedule * 3. Populate the drr_info with the min and max supported vtotal values - * - * @param [in] dc: current dc state - * @param [in] subvp_pipe: pipe_ctx for the SubVP pipe - * @param [in] vblank_pipe: pipe_ctx for the DRR pipe - * @param [in] pipe_data: Pipe data which stores the VBLANK/DRR info - * - * @return: void - * - * *********************************************************************************************** */ static void populate_subvp_cmd_drr_info(struct dc *dc, struct pipe_ctx *subvp_pipe, @@ -493,22 +488,18 @@ static void populate_subvp_cmd_drr_info(struct dc *dc, } /** - * *********************************************************************************************** - * populate_subvp_cmd_vblank_pipe_info: Helper to populate VBLANK pipe info for the DMUB subvp command + * populate_subvp_cmd_vblank_pipe_info - Helper to populate VBLANK pipe info for the DMUB subvp command * - * Populate the DMCUB SubVP command with VBLANK pipe info. All the information required to calculate - * the microschedule for SubVP + VBLANK case is stored in the pipe_data (subvp_data and vblank_data). - * Also check if the VBLANK pipe is a DRR display -- if it is make a call to populate drr_info. + * @dc: [in] current dc state + * @context: [in] new dc state + * @cmd: [in] DMUB cmd to be populated with SubVP info + * @vblank_pipe: [in] pipe_ctx for the VBLANK pipe + * @cmd_pipe_index: [in] index for the pipe array in DMCUB SubVP cmd * - * @param [in] dc: current dc state - * @param [in] context: new dc state - * @param [in] cmd: DMUB cmd to be populated with SubVP info - * @param [in] vblank_pipe: pipe_ctx for the VBLANK pipe - * @param [in] cmd_pipe_index: index for the pipe array in DMCUB SubVP cmd - * - * @return: void - * - * *********************************************************************************************** + * Populate the DMCUB SubVP command with VBLANK pipe info. All the information + * required to calculate the microschedule for SubVP + VBLANK case is stored in + * the pipe_data (subvp_data and vblank_data). Also check if the VBLANK pipe + * is a DRR display -- if it is make a call to populate drr_info. */ static void populate_subvp_cmd_vblank_pipe_info(struct dc *dc, struct dc_state *context, @@ -551,22 +542,18 @@ static void populate_subvp_cmd_vblank_pipe_info(struct dc *dc, } /** - * *********************************************************************************************** - * update_subvp_prefetch_end_to_mall_start: Helper for SubVP + SubVP case - * - * For SubVP + SubVP, we use a single vertical interrupt to start the microschedule for both - * SubVP pipes. In order for this to work correctly, the MALL REGION of both SubVP pipes must - * start at the same time. This function lengthens the prefetch end to mall start delay of the - * SubVP pipe that has the shorter prefetch so that both MALL REGION's will start at the same time. - * - * @param [in] dc: current dc state - * @param [in] context: new dc state - * @param [in] cmd: DMUB cmd to be populated with SubVP info - * @param [in] subvp_pipes: Array of SubVP pipes (should always be length 2) + * update_subvp_prefetch_end_to_mall_start - Helper for SubVP + SubVP case * - * @return: void + * @dc: [in] current dc state + * @context: [in] new dc state + * @cmd: [in] DMUB cmd to be populated with SubVP info + * @subvp_pipes: [in] Array of SubVP pipes (should always be length 2) * - * *********************************************************************************************** + * For SubVP + SubVP, we use a single vertical interrupt to start the + * microschedule for both SubVP pipes. In order for this to work correctly, the + * MALL REGION of both SubVP pipes must start at the same time. This function + * lengthens the prefetch end to mall start delay of the SubVP pipe that has + * the shorter prefetch so that both MALL REGION's will start at the same time. */ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc, struct dc_state *context, @@ -608,22 +595,17 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc, } /** - * *************************************************************************************** - * setup_subvp_dmub_command: Helper to populate the SubVP pipe info for the DMUB subvp command + * setup_subvp_dmub_command - Helper to populate the SubVP pipe info for the DMUB subvp command * - * Populate the DMCUB SubVP command with SubVP pipe info. All the information required to - * calculate the microschedule for the SubVP pipe is stored in the pipe_data of the DMCUB - * SubVP command. + * @dc: [in] current dc state + * @context: [in] new dc state + * @cmd: [in] DMUB cmd to be populated with SubVP info + * @subvp_pipe: [in] pipe_ctx for the SubVP pipe + * @cmd_pipe_index: [in] index for the pipe array in DMCUB SubVP cmd * - * @param [in] dc: current dc state - * @param [in] context: new dc state - * @param [in] cmd: DMUB cmd to be populated with SubVP info - * @param [in] subvp_pipe: pipe_ctx for the SubVP pipe - * @param [in] cmd_pipe_index: index for the pipe array in DMCUB SubVP cmd - * - * @return: void - * - * *************************************************************************************** + * Populate the DMCUB SubVP command with SubVP pipe info. All the information + * required to calculate the microschedule for the SubVP pipe is stored in the + * pipe_data of the DMCUB SubVP command. */ static void populate_subvp_cmd_pipe_info(struct dc *dc, struct dc_state *context, @@ -703,19 +685,14 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc, } /** - * *************************************************************************************** - * dc_dmub_setup_subvp_dmub_command: Populate the DMCUB SubVP command - * - * This function loops through each pipe and populates the DMUB - * SubVP CMD info based on the pipe (e.g. SubVP, VBLANK). - * - * @param [in] dc: current dc state - * @param [in] context: new dc state - * @param [in] cmd: DMUB cmd to be populated with SubVP info + * dc_dmub_setup_subvp_dmub_command - Populate the DMCUB SubVP command * - * @return: void + * @dc: [in] current dc state + * @context: [in] new dc state + * @cmd: [in] DMUB cmd to be populated with SubVP info * - * *************************************************************************************** + * This function loops through each pipe and populates the DMUB SubVP CMD info + * based on the pipe (e.g. SubVP, VBLANK). */ void dc_dmub_setup_subvp_dmub_command(struct dc *dc, struct dc_state *context, @@ -962,19 +939,14 @@ static void dc_build_cursor_attribute_update_payload1( } /** - * *************************************************************************************** - * dc_send_update_cursor_info_to_dmu: Populate the DMCUB Cursor update info command + * dc_send_update_cursor_info_to_dmu - Populate the DMCUB Cursor update info command * - * This function would store the cursor related information and pass it into dmub + * @pCtx: [in] pipe context + * @pipe_idx: [in] pipe index * - * @param [in] pCtx: pipe context - * @param [in] pipe_idx: pipe index - * - * @return: void - * - * *************************************************************************************** + * This function would store the cursor related information and pass it into + * dmub */ - void dc_send_update_cursor_info_to_dmu( struct pipe_ctx *pCtx, uint8_t pipe_idx) { -- GitLab From f1943a51f0f9ce732e344fffb576b54f3e230a4d Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:46:38 -0400 Subject: [PATCH 0405/1927] drm/amd/display: Add events log to trace OPTC lock and unlock As an attempt to offer more DCN debug tools for cases where the OPTC can hang, this commit introduces a trace event responsible for showing OPTC status when it requests lock and unlock. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../amd/display/amdgpu_dm/amdgpu_dm_trace.h | 64 +++++++++++++++++++ drivers/gpu/drm/amd/display/dc/dc_trace.h | 2 + .../gpu/drm/amd/display/dc/dcn10/dcn10_optc.c | 5 ++ .../gpu/drm/amd/display/dc/dcn30/dcn30_optc.c | 9 +++ 4 files changed, 80 insertions(+) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h index d3bc9dc217717..0f580ea37576d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h @@ -37,6 +37,7 @@ #include #include #include +#include "dcn10/dcn10_optc.h" #include "dc/inc/core_types.h" @@ -662,6 +663,69 @@ TRACE_EVENT(dcn_fpu, ) ); +TRACE_EVENT(dcn_optc_lock_unlock_state, + TP_PROTO(const struct optc *optc_state, int instance, bool lock, const char *function, const int line), + TP_ARGS(optc_state, instance, lock, function, line), + + TP_STRUCT__entry( + __field(const char *, function) + __field(int, instance) + __field(bool, lock) + __field(int, line) + __field(int, opp_count) + __field(int, max_h_total) + __field(int, max_v_total) + __field(int, min_h_blank) + __field(int, min_h_sync_width) + __field(int, min_v_sync_width) + __field(int, min_v_blank) + __field(int, min_v_blank_interlace) + __field(int, vstartup_start) + __field(int, vupdate_offset) + __field(int, vupdate_width) + __field(int, vready_offset) + ), + TP_fast_assign( + __entry->function = function; + __entry->instance = instance; + __entry->lock = lock; + __entry->line = line; + __entry->opp_count = optc_state->opp_count; + __entry->max_h_total = optc_state->max_h_total; + __entry->max_v_total = optc_state->max_v_total; + __entry->min_h_blank = optc_state->min_h_blank; + __entry->min_h_sync_width = optc_state->min_h_sync_width; + __entry->min_v_sync_width = optc_state->min_v_sync_width; + __entry->min_v_blank = optc_state->min_v_blank; + __entry->min_v_blank_interlace = optc_state->min_v_blank_interlace; + __entry->vstartup_start = optc_state->vstartup_start; + __entry->vupdate_offset = optc_state->vupdate_offset; + __entry->vupdate_width = optc_state->vupdate_width; + __entry->vready_offset = optc_state->vupdate_offset; + ), + TP_printk("%s: %s()+%d: optc_instance=%d opp_count=%d max_h_total=%d max_v_total=%d " + "min_h_blank=%d min_h_sync_width=%d min_v_sync_width=%d min_v_blank=%d " + "min_v_blank_interlace=%d vstartup_start=%d vupdate_offset=%d vupdate_width=%d " + "vready_offset=%d", + __entry->lock ? "Lock" : "Unlock", + __entry->function, + __entry->line, + __entry->instance, + __entry->opp_count, + __entry->max_h_total, + __entry->max_v_total, + __entry->min_h_blank, + __entry->min_h_sync_width, + __entry->min_v_sync_width, + __entry->min_v_blank, + __entry->min_v_blank_interlace, + __entry->vstartup_start, + __entry->vupdate_offset, + __entry->vupdate_width, + __entry->vready_offset + ) +); + #endif /* _AMDGPU_DM_TRACE_H_ */ #undef TRACE_INCLUDE_PATH diff --git a/drivers/gpu/drm/amd/display/dc/dc_trace.h b/drivers/gpu/drm/amd/display/dc/dc_trace.h index c711797e5c9e0..bbec308a3a5e2 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_trace.h +++ b/drivers/gpu/drm/amd/display/dc/dc_trace.h @@ -40,3 +40,5 @@ #define TRACE_DCN_FPU(begin, function, line, ref_count) \ trace_dcn_fpu(begin, function, line, ref_count) +#define TRACE_OPTC_LOCK_UNLOCK_STATE(optc, inst, lock) \ + trace_dcn_optc_lock_unlock_state(optc, inst, lock, __func__, __LINE__) diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c index 33d7802187900..c9e53dc49c92b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c @@ -27,6 +27,7 @@ #include "reg_helper.h" #include "dcn10_optc.h" #include "dc.h" +#include "dc_trace.h" #define REG(reg)\ optc1->tg_regs->reg @@ -657,6 +658,8 @@ void optc1_lock(struct timing_generator *optc) REG_WAIT(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, 1, 1, 10); + + TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, true); } void optc1_unlock(struct timing_generator *optc) @@ -665,6 +668,8 @@ void optc1_unlock(struct timing_generator *optc) REG_SET(OTG_MASTER_UPDATE_LOCK, 0, OTG_MASTER_UPDATE_LOCK, 0); + + TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, false); } void optc1_get_position(struct timing_generator *optc, diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c index 892d3c4d01a1e..867d60151aebb 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c @@ -30,6 +30,7 @@ #include "dc_dmub_srv.h" #include "dml/dcn30/dcn30_fpu.h" +#include "dc_trace.h" #define REG(reg)\ optc1->tg_regs->reg @@ -58,6 +59,8 @@ void optc3_triplebuffer_lock(struct timing_generator *optc) REG_WAIT(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, 1, 1, 10); + + TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, true); } void optc3_lock_doublebuffer_enable(struct timing_generator *optc) @@ -93,6 +96,8 @@ void optc3_lock_doublebuffer_enable(struct timing_generator *optc) MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_START_OFFSET, 0, MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_END_OFFSET, 100, OTG_MASTER_UPDATE_LOCK_VUPDATE_KEEPOUT_EN, 1); + + TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, true); } void optc3_lock_doublebuffer_disable(struct timing_generator *optc) @@ -108,6 +113,8 @@ void optc3_lock_doublebuffer_disable(struct timing_generator *optc) REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 0); REG_UPDATE(OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_EN, 0); + + TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, true); } void optc3_lock(struct timing_generator *optc) @@ -122,6 +129,8 @@ void optc3_lock(struct timing_generator *optc) REG_WAIT(OTG_MASTER_UPDATE_LOCK, UPDATE_LOCK_STATUS, 1, 1, 10); + + TRACE_OPTC_LOCK_UNLOCK_STATE(optc1, optc->inst, true); } void optc3_set_out_mux(struct timing_generator *optc, enum otg_out_mux_dest dest) -- GitLab From b473bd5fc333ab02479444dd4a51be5f5e324d97 Mon Sep 17 00:00:00 2001 From: Lewis Huang Date: Thu, 20 Oct 2022 11:46:39 -0400 Subject: [PATCH 0406/1927] drm/amd/display: refine wake up aux in retrieve link caps [Why] Read set_power_state dpcd after HPD cause USB4 CTS 4.2.1.1 [How] Read LTTPR caps first. If aux channel not ready, wake up aux channel. If wake up aux channel return pass, retrieve lttpr caps again. If wake up aux channel return false, register a detection retry timer. Tested-by: Mark Broadworth Reviewed-by: Wenjing Liu Acked-by: Rodrigo Siqueira Signed-off-by: Lewis Huang Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/core/dc_link_dp.c | 148 +++++------------- .../gpu/drm/amd/display/dc/inc/dc_link_dp.h | 2 +- 2 files changed, 41 insertions(+), 109 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index bd4cbe02d353a..420119efcf5f5 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -5031,7 +5031,7 @@ static bool dpcd_read_sink_ext_caps(struct dc_link *link) return true; } -bool dp_retrieve_lttpr_cap(struct dc_link *link) +enum dc_status dp_retrieve_lttpr_cap(struct dc_link *link) { uint8_t lttpr_dpcd_data[8]; enum dc_status status = DC_ERROR_UNEXPECTED; @@ -5099,7 +5099,7 @@ bool dp_retrieve_lttpr_cap(struct dc_link *link) CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: "); DC_LOG_DC("is_lttpr_present = %d\n", is_lttpr_present); - return is_lttpr_present; + return status; } bool dp_is_lttpr_present(struct dc_link *link) @@ -5227,76 +5227,45 @@ static void retrieve_cable_id(struct dc_link *link) &link->dpcd_caps.cable_id, &usbc_cable_id); } -/* DPRX may take some time to respond to AUX messages after HPD asserted. - * If AUX read unsuccessful, try to wake unresponsive DPRX by toggling DPCD SET_POWER (0x600). - */ -static enum dc_status wa_try_to_wake_dprx(struct dc_link *link, uint64_t timeout_ms) +enum dc_status wake_up_aux_channel(struct dc_link *link) { enum dc_status status = DC_ERROR_UNEXPECTED; - uint8_t dpcd_data = 0; - uint64_t start_ts = 0; - uint64_t current_ts = 0; - uint64_t time_taken_ms = 0; - enum dc_connection_type type = dc_connection_none; - bool lttpr_present; - bool vbios_lttpr_interop = link->dc->caps.vbios_lttpr_aware; + uint32_t aux_channel_retry_cnt = 0; + uint8_t dpcd_power_state = '\0'; - lttpr_present = dp_is_lttpr_present(link) || - (!vbios_lttpr_interop || !link->dc->caps.extended_aux_timeout_support); - DC_LOG_DC("lttpr_present = %d.\n", lttpr_present ? 1 : 0); + while (status != DC_OK && aux_channel_retry_cnt < 10) { + status = core_link_read_dpcd(link, DP_SET_POWER, + &dpcd_power_state, sizeof(dpcd_power_state)); - /* Issue an AUX read to test DPRX responsiveness. If LTTPR is supported the first read is expected to - * be to determine LTTPR capabilities. Otherwise trying to read power state should be an innocuous AUX read. - */ - if (lttpr_present) - status = core_link_read_dpcd( - link, - DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV, - &dpcd_data, - sizeof(dpcd_data)); - else - status = core_link_read_dpcd( - link, - DP_SET_POWER, - &dpcd_data, - sizeof(dpcd_data)); + /* Delay 1 ms if AUX CH is in power down state. Based on spec + * section 2.3.1.2, if AUX CH may be powered down due to + * write to DPCD 600h = 2. Sink AUX CH is monitoring differential + * signal and may need up to 1 ms before being able to reply. + */ + if (status != DC_OK || dpcd_power_state == DP_SET_POWER_D3) { + udelay(1000); + aux_channel_retry_cnt++; + } + } if (status != DC_OK) { - DC_LOG_WARNING("%s: Read DPCD LTTPR_CAP failed - try to toggle DPCD SET_POWER for %lld ms.", - __func__, - timeout_ms); - start_ts = dm_get_timestamp(link->ctx); - - do { - if (!dc_link_detect_sink(link, &type) || type == dc_connection_none) - break; - - dpcd_data = DP_SET_POWER_D3; - status = core_link_write_dpcd( - link, - DP_SET_POWER, - &dpcd_data, - sizeof(dpcd_data)); - - dpcd_data = DP_SET_POWER_D0; - status = core_link_write_dpcd( - link, - DP_SET_POWER, - &dpcd_data, - sizeof(dpcd_data)); - - current_ts = dm_get_timestamp(link->ctx); - time_taken_ms = div_u64(dm_get_elapse_time_in_ns(link->ctx, current_ts, start_ts), 1000000); - } while (status != DC_OK && time_taken_ms < timeout_ms); + dpcd_power_state = DP_SET_POWER_D0; + status = core_link_write_dpcd( + link, + DP_SET_POWER, + &dpcd_power_state, + sizeof(dpcd_power_state)); - DC_LOG_WARNING("%s: DPCD SET_POWER %s after %lld ms%s", - __func__, - (status == DC_OK) ? "succeeded" : "failed", - time_taken_ms, - (type == dc_connection_none) ? ". Unplugged." : "."); + dpcd_power_state = DP_SET_POWER_D3; + status = core_link_write_dpcd( + link, + DP_SET_POWER, + &dpcd_power_state, + sizeof(dpcd_power_state)); + return DC_ERROR_UNEXPECTED; } - return status; + return DC_OK; } static bool retrieve_link_cap(struct dc_link *link) @@ -5308,7 +5277,6 @@ static bool retrieve_link_cap(struct dc_link *link) /*Only need to read 1 byte starting from DP_DPRX_FEATURE_ENUMERATION_LIST. */ uint8_t dpcd_dprx_data = '\0'; - uint8_t dpcd_power_state = '\0'; struct dp_device_vendor_id sink_id; union down_stream_port_count down_strm_port_count; @@ -5316,11 +5284,9 @@ static bool retrieve_link_cap(struct dc_link *link) union dp_downstream_port_present ds_port = { 0 }; enum dc_status status = DC_ERROR_UNEXPECTED; uint32_t read_dpcd_retry_cnt = 3; - uint32_t aux_channel_retry_cnt = 0; int i; struct dp_sink_hw_fw_revision dp_hw_fw_revision; const uint32_t post_oui_delay = 30; // 30ms - bool is_lttpr_present = false; memset(dpcd_data, '\0', sizeof(dpcd_data)); memset(&down_strm_port_count, @@ -5335,51 +5301,17 @@ static bool retrieve_link_cap(struct dc_link *link) dc_link_aux_try_to_configure_timeout(link->ddc, LINK_AUX_DEFAULT_LTTPR_TIMEOUT_PERIOD); - /* Try to ensure AUX channel active before proceeding. */ - if (link->dc->debug.aux_wake_wa.bits.enable_wa) { - uint64_t timeout_ms = link->dc->debug.aux_wake_wa.bits.timeout_ms; - - if (link->dc->debug.aux_wake_wa.bits.use_default_timeout) - timeout_ms = LINK_AUX_WAKE_TIMEOUT_MS; - status = wa_try_to_wake_dprx(link, timeout_ms); - } - - while (status != DC_OK && aux_channel_retry_cnt < 10) { - status = core_link_read_dpcd(link, DP_SET_POWER, - &dpcd_power_state, sizeof(dpcd_power_state)); - - /* Delay 1 ms if AUX CH is in power down state. Based on spec - * section 2.3.1.2, if AUX CH may be powered down due to - * write to DPCD 600h = 2. Sink AUX CH is monitoring differential - * signal and may need up to 1 ms before being able to reply. - */ - if (status != DC_OK || dpcd_power_state == DP_SET_POWER_D3) { - udelay(1000); - aux_channel_retry_cnt++; - } - } + status = dp_retrieve_lttpr_cap(link); - /* If aux channel is not active, return false and trigger another detect*/ if (status != DC_OK) { - dpcd_power_state = DP_SET_POWER_D0; - status = core_link_write_dpcd( - link, - DP_SET_POWER, - &dpcd_power_state, - sizeof(dpcd_power_state)); - - dpcd_power_state = DP_SET_POWER_D3; - status = core_link_write_dpcd( - link, - DP_SET_POWER, - &dpcd_power_state, - sizeof(dpcd_power_state)); - return false; + status = wake_up_aux_channel(link); + if (status == DC_OK) + dp_retrieve_lttpr_cap(link); + else + return false; } - is_lttpr_present = dp_retrieve_lttpr_cap(link); - - if (is_lttpr_present) + if (dp_is_lttpr_present(link)) configure_lttpr_mode_transparent(link); /* Read DP tunneling information. */ @@ -5406,7 +5338,7 @@ static bool retrieve_link_cap(struct dc_link *link) return false; } - if (!is_lttpr_present) + if (!dp_is_lttpr_present(link)) dc_link_aux_try_to_configure_timeout(link->ddc, LINK_AUX_DEFAULT_TIMEOUT_PERIOD); { diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h index b304d450b038a..e8d8c5cb13092 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h +++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h @@ -193,7 +193,7 @@ enum dc_status dpcd_configure_lttpr_mode( struct link_training_settings *lt_settings); enum dp_link_encoding dp_get_link_encoding_format(const struct dc_link_settings *link_settings); -bool dp_retrieve_lttpr_cap(struct dc_link *link); +enum dc_status dp_retrieve_lttpr_cap(struct dc_link *link); bool dp_is_lttpr_present(struct dc_link *link); enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link, struct dc_link_settings *link_setting); void dp_get_lttpr_mode_override(struct dc_link *link, enum lttpr_mode *override); -- GitLab From 12024b17617af9b953cf8801a6a3d3c769ebd06f Mon Sep 17 00:00:00 2001 From: wangjianli Date: Sat, 22 Oct 2022 14:01:55 +0800 Subject: [PATCH 0407/1927] amd/amdgpu: fix repeated words in comments Delete the redundant word 'the'. Signed-off-by: wangjianli Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index ddaecb2610c94..6f958603c8cc2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -1568,7 +1568,7 @@ static int amdgpu_device_check_arguments(struct amdgpu_device *adev) * @pdev: pci dev pointer * @state: vga_switcheroo state * - * Callback for the switcheroo driver. Suspends or resumes the + * Callback for the switcheroo driver. Suspends or resumes * the asics before or after it is powered up using ACPI methods. */ static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, -- GitLab From 31bc2485e88537009fad9246bf48319fc9172230 Mon Sep 17 00:00:00 2001 From: wangjianli Date: Sat, 22 Oct 2022 14:04:17 +0800 Subject: [PATCH 0408/1927] drm/radeon: fix repeated words in comments Delete the redundant word 'the'. Signed-off-by: wangjianli Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index a556b6be11374..262e2bcb30c3d 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -1207,7 +1207,7 @@ static void radeon_check_arguments(struct radeon_device *rdev) * @pdev: pci dev pointer * @state: vga_switcheroo state * - * Callback for the switcheroo driver. Suspends or resumes the + * Callback for the switcheroo driver. Suspends or resumes * the asics before or after it is powered up using ACPI methods. */ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) -- GitLab From 93e28cf81406519622383ae51d6fd183b744ac46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Ignacio=20Aramend=C3=ADa?= Date: Mon, 24 Oct 2022 08:33:59 -0300 Subject: [PATCH 0409/1927] drm/amd/display: Revert logic for plane modifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This file was split in commit 5d945cbcd4b16a29d6470a80dfb19738f9a4319f ("drm/amd/display: Create a file dedicated to planes") and the logic in dm_plane_format_mod_supported() function got changed by a switch logic. That change broke drm_plane modifiers setting on series 5000 APUs (tested on OXP mini AMD 5800U and HP Dev One 5850U PRO) leading to Gamescope not working as reported on GitHub[1] To reproduce the issue, enter a TTY and run: $ gamescope -- vkcube With said commit applied it will abort. This one restores the old logic, fixing the issue that affects Gamescope. [1](https://github.com/Plagman/gamescope/issues/624) Cc: # 6.0.x Signed-off-by: Joaquín Ignacio Aramendía Reviewed-by: Bas Nieuwenhuizen Signed-off-by: Alex Deucher --- .../amd/display/amdgpu_dm/amdgpu_dm_plane.c | 50 +++---------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index dfd3be49eac87..e6854f7270a66 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1369,7 +1369,7 @@ static bool dm_plane_format_mod_supported(struct drm_plane *plane, { struct amdgpu_device *adev = drm_to_adev(plane->dev); const struct drm_format_info *info = drm_format_info(format); - struct hw_asic_id asic_id = adev->dm.dc->ctx->asic_id; + int i; enum dm_micro_swizzle microtile = modifier_gfx9_swizzle_mode(modifier) & 3; @@ -1386,49 +1386,13 @@ static bool dm_plane_format_mod_supported(struct drm_plane *plane, return true; } - /* check if swizzle mode is supported by this version of DCN */ - switch (asic_id.chip_family) { - case FAMILY_SI: - case FAMILY_CI: - case FAMILY_KV: - case FAMILY_CZ: - case FAMILY_VI: - /* asics before AI does not have modifier support */ - return false; - case FAMILY_AI: - case FAMILY_RV: - case FAMILY_NV: - case FAMILY_VGH: - case FAMILY_YELLOW_CARP: - case AMDGPU_FAMILY_GC_10_3_6: - case AMDGPU_FAMILY_GC_10_3_7: - switch (AMD_FMT_MOD_GET(TILE, modifier)) { - case AMD_FMT_MOD_TILE_GFX9_64K_R_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D_X: - case AMD_FMT_MOD_TILE_GFX9_64K_S_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D: - return true; - default: - return false; - } - break; - case AMDGPU_FAMILY_GC_11_0_0: - case AMDGPU_FAMILY_GC_11_0_1: - switch (AMD_FMT_MOD_GET(TILE, modifier)) { - case AMD_FMT_MOD_TILE_GFX11_256K_R_X: - case AMD_FMT_MOD_TILE_GFX9_64K_R_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D_X: - case AMD_FMT_MOD_TILE_GFX9_64K_S_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D: - return true; - default: - return false; - } - break; - default: - ASSERT(0); /* Unknown asic */ - break; + /* Check that the modifier is on the list of the plane's supported modifiers. */ + for (i = 0; i < plane->modifier_count; i++) { + if (modifier == plane->modifiers[i]) + break; } + if (i == plane->modifier_count) + return false; /* * For D swizzle the canonical modifier depends on the bpp, so check -- GitLab From d2c4c1569a7d7d5c8f75963bf2d62d7aeac30e2a Mon Sep 17 00:00:00 2001 From: Lijo Lazar Date: Fri, 30 Sep 2022 10:43:08 +0530 Subject: [PATCH 0410/1927] drm/amdgpu: Remove ATC L2 access for MMHUB 2.1.x MMHUB 2.1.x versions don't have ATCL2. Remove accesses to ATCL2 registers. Since they are non-existing registers, read access will cause a 'Completer Abort' and gets reported when AER is enabled with the below patch. Tagging with the patch so that this is backported along with it. v2: squash in uninitialized warning fix (Nathan Chancellor) Fixes: 8795e182b02d ("PCI/portdrv: Don't disable AER reporting in get_port_device_capability()") Signed-off-by: Lijo Lazar Reviewed-by: Guchun Chen Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c | 28 +++++++------------------ 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c index 4d304f22889e4..998b5d17b271b 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v2_0.c @@ -32,8 +32,6 @@ #include "gc/gc_10_1_0_offset.h" #include "soc15_common.h" -#define mmMM_ATC_L2_MISC_CG_Sienna_Cichlid 0x064d -#define mmMM_ATC_L2_MISC_CG_Sienna_Cichlid_BASE_IDX 0 #define mmDAGB0_CNTL_MISC2_Sienna_Cichlid 0x0070 #define mmDAGB0_CNTL_MISC2_Sienna_Cichlid_BASE_IDX 0 @@ -574,7 +572,6 @@ static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *ad case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid); def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_Sienna_Cichlid); break; default: @@ -608,8 +605,6 @@ static void mmhub_v2_0_update_medium_grain_clock_gating(struct amdgpu_device *ad case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - if (def != data) - WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid, data); if (def1 != data1) WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_Sienna_Cichlid, data1); break; @@ -634,8 +629,8 @@ static void mmhub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *ade case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid); - break; + /* There is no ATCL2 in MMHUB for 2.1.x */ + return; default: def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG); break; @@ -646,18 +641,8 @@ static void mmhub_v2_0_update_medium_grain_light_sleep(struct amdgpu_device *ade else data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK; - if (def != data) { - switch (adev->ip_versions[MMHUB_HWIP][0]) { - case IP_VERSION(2, 1, 0): - case IP_VERSION(2, 1, 1): - case IP_VERSION(2, 1, 2): - WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid, data); - break; - default: - WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG, data); - break; - } - } + if (def != data) + WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG, data); } static int mmhub_v2_0_set_clockgating(struct amdgpu_device *adev, @@ -695,7 +680,10 @@ static void mmhub_v2_0_get_clockgating(struct amdgpu_device *adev, u64 *flags) case IP_VERSION(2, 1, 0): case IP_VERSION(2, 1, 1): case IP_VERSION(2, 1, 2): - data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_MISC_CG_Sienna_Cichlid); + /* There is no ATCL2 in MMHUB for 2.1.x. Keep the status + * based on DAGB + */ + data = MM_ATC_L2_MISC_CG__ENABLE_MASK; data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_Sienna_Cichlid); break; default: -- GitLab From 90bfee142af0f0e9d3bec80e7acd5f49b230acf7 Mon Sep 17 00:00:00 2001 From: Rafael Mendonca Date: Mon, 17 Oct 2022 22:27:50 -0300 Subject: [PATCH 0411/1927] drm/amdkfd: Fix memory leak in kfd_mem_dmamap_userptr() If the number of pages from the userptr BO differs from the SG BO then the allocated memory for the SG table doesn't get freed before returning -EINVAL, which may lead to a memory leak in some error paths. Fix this by checking the number of pages before allocating memory for the SG table. Fixes: 264fb4d332f5 ("drm/amdgpu: Add multi-GPU DMA mapping helpers") Signed-off-by: Rafael Mendonca Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 978d3970b5cc4..84f44f7e41118 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -510,13 +510,13 @@ kfd_mem_dmamap_userptr(struct kgd_mem *mem, struct ttm_tt *ttm = bo->tbo.ttm; int ret; + if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) + return -EINVAL; + ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); if (unlikely(!ttm->sg)) return -ENOMEM; - if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) - return -EINVAL; - /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, ttm->num_pages, 0, -- GitLab From 08841950db932dc3ba8bbd4c0f1f7f27ccfbae42 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Thu, 20 Oct 2022 15:25:25 +0800 Subject: [PATCH 0412/1927] drm/amd/pm: allow gfxoff on gc_11_0_3 allow gfxoff on gc_11_0_3 Signed-off-by: Kenneth Feng Reviewed-by: Yang Wang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 1 + drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 671ca5a0f208a..0fecc5bf45bc5 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -5051,6 +5051,7 @@ static int gfx_v11_0_set_powergating_state(void *handle, switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(11, 0, 0): case IP_VERSION(11, 0, 2): + case IP_VERSION(11, 0, 3): amdgpu_gfx_off_ctrl(adev, enable); break; case IP_VERSION(11, 0, 1): diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c index feb4d68f3fd9a..43fb102a65f56 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c @@ -840,6 +840,7 @@ int smu_v13_0_gfx_off_control(struct smu_context *smu, bool enable) case IP_VERSION(13, 0, 5): case IP_VERSION(13, 0, 7): case IP_VERSION(13, 0, 8): + case IP_VERSION(13, 0, 10): if (!(adev->pm.pp_feature & PP_GFXOFF_MASK)) return 0; if (enable) -- GitLab From ca08a1725d0d78efca8d2dbdbce5ea70355da0f2 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 6 Oct 2022 17:26:48 -0400 Subject: [PATCH 0413/1927] drm/amd/display: Remove wrong pipe control lock When using a device based on DCN32/321, we have an issue where a second 4k@60Hz display does not light up, and the system becomes unresponsive for a few minutes. In the debug process, it was possible to see a hang in the function dcn20_post_unlock_program_front_end in this part: for (j = 0; j < TIMEOUT_FOR_PIPE_ENABLE_MS*1000 && hubp->funcs->hubp_is_flip_pending(hubp); j++) mdelay(1); } The hubp_is_flip_pending always returns positive for waiting pending flips which is a symptom of pipe hang. Additionally, the dmesg log shows this message after a few minutes: BUG: soft lockup - CPU#4 stuck for 26s! ... [ +0.000003] dcn20_post_unlock_program_front_end+0x112/0x340 [amdgpu] [ +0.000171] dc_commit_state_no_check+0x63d/0xbf0 [amdgpu] [ +0.000155] ? dc_validate_global_state+0x358/0x3d0 [amdgpu] [ +0.000154] dc_commit_state+0xe2/0xf0 [amdgpu] This confirmed the hypothesis that we had a pipe hanging somewhere. Next, after checking the ftrace entries, we have the below weird sequence: [..] 2) | dcn10_lock_all_pipes [amdgpu]() { 2) 0.120 us | optc1_is_tg_enabled [amdgpu](); 2) | dcn20_pipe_control_lock [amdgpu]() { 2) | dc_dmub_srv_clear_inbox0_ack [amdgpu]() { 2) 0.121 us | amdgpu_dm_dmub_reg_write [amdgpu](); 2) 0.551 us | } 2) | dc_dmub_srv_send_inbox0_cmd [amdgpu]() { 2) 0.110 us | amdgpu_dm_dmub_reg_write [amdgpu](); 2) 0.511 us | } 2) | dc_dmub_srv_wait_for_inbox0_ack [amdgpu]() { 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); 2) 0.110 us | amdgpu_dm_dmub_reg_read [amdgpu](); [..] We are not expected to read from dmub register so many times and for so long. From the trace log, it was possible to identify that the function dcn20_pipe_control_lock was triggering the dmub operation when it was unnecessary and causing the hang issue. This commit drops the unnecessary dmub code and, consequently, fixes the second display not lighting up the issue. Tested-by: Daniel Wheeler Acked-by: Qingqing Zhuo Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index d732b6f031a12..a7e0001a8f46d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1270,16 +1270,6 @@ void dcn20_pipe_control_lock( lock, &hw_locks, &inst_flags); - } else if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_MAIN) { - union dmub_inbox0_cmd_lock_hw hw_lock_cmd = { 0 }; - hw_lock_cmd.bits.command_code = DMUB_INBOX0_CMD__HW_LOCK; - hw_lock_cmd.bits.hw_lock_client = HW_LOCK_CLIENT_DRIVER; - hw_lock_cmd.bits.lock_pipe = 1; - hw_lock_cmd.bits.otg_inst = pipe->stream_res.tg->inst; - hw_lock_cmd.bits.lock = lock; - if (!lock) - hw_lock_cmd.bits.should_release = 1; - dmub_hw_lock_mgr_inbox0_cmd(dc->ctx->dmub_srv, hw_lock_cmd); } else if (pipe->plane_state != NULL && pipe->plane_state->triplebuffer_flips) { if (lock) pipe->stream_res.tg->funcs->triplebuffer_lock(pipe->stream_res.tg); @@ -1856,7 +1846,7 @@ void dcn20_post_unlock_program_front_end( for (j = 0; j < TIMEOUT_FOR_PIPE_ENABLE_MS*1000 && hubp->funcs->hubp_is_flip_pending(hubp); j++) - mdelay(1); + udelay(1); } } -- GitLab From abe4d9f03fae76c9650b0d942faf6990b35c377b Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 6 Oct 2022 17:26:49 -0400 Subject: [PATCH 0414/1927] drm/amd/display: Don't return false if no stream pipe_ctx[i] exists even if the pipe is not in use. If the pipe is not in use it will always have a null stream, so don't return false in this case. Tested-by: Daniel Wheeler Reviewed-by: Rodrigo Siqueira Acked-by: Qingqing Zhuo Signed-off-by: Alvin Lee Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c index d51d0c40ae5bc..b03a7814e96d8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c @@ -200,7 +200,7 @@ bool dcn32_all_pipes_have_stream_and_plane(struct dc *dc, struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i]; if (!pipe->stream) - return false; + continue; if (!pipe->plane_state) return false; -- GitLab From 68bc147363bd9769a07d1cbf5cbe2bb4573f4e3c Mon Sep 17 00:00:00 2001 From: David Francis Date: Fri, 16 Sep 2022 11:15:01 -0400 Subject: [PATCH 0415/1927] drm/amd: Add IMU fw version to fw version queries IMU is a new firmware for GFX11. There are four means by which firmware version can be queried from the driver: device attributes, vf2pf, debugfs, and the AMDGPU_INFO_FW_VERSION option in the amdgpu info ioctl. Add IMU as an option for those four methods. V2: Added debugfs Reviewed-by: Likun Gao Reviewed-by: Alex Deucher Signed-off-by: David Francis Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 13 +++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c | 4 +++- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 1 + drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h | 1 + include/uapi/drm/amdgpu_drm.h | 2 ++ 5 files changed, 20 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index fe23e09eec985..bf1ff8f0e7120 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -344,6 +344,10 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->ver = adev->mes.ucode_fw_version[1]; fw_info->feature = 0; break; + case AMDGPU_INFO_FW_IMU: + fw_info->ver = adev->gfx.imu_fw_version; + fw_info->feature = 0; + break; default: return -EINVAL; } @@ -1520,6 +1524,15 @@ static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused) fw_info.feature, fw_info.ver); } + /* IMU */ + query_fw.fw_type = AMDGPU_INFO_FW_IMU; + query_fw.index = 0; + ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); + if (ret) + return ret; + seq_printf(m, "IMU feature version: %u, firmware version: 0x%08x\n", + fw_info.feature, fw_info.ver); + /* PSP SOS */ query_fw.fw_type = AMDGPU_INFO_FW_SOS; ret = amdgpu_firmware_info(&fw_info, &query_fw, adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c index dd0bc649a57d1..5cb62e6249c23 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c @@ -698,6 +698,7 @@ FW_VERSION_ATTR(rlc_srlg_fw_version, 0444, gfx.rlc_srlg_fw_version); FW_VERSION_ATTR(rlc_srls_fw_version, 0444, gfx.rlc_srls_fw_version); FW_VERSION_ATTR(mec_fw_version, 0444, gfx.mec_fw_version); FW_VERSION_ATTR(mec2_fw_version, 0444, gfx.mec2_fw_version); +FW_VERSION_ATTR(imu_fw_version, 0444, gfx.imu_fw_version); FW_VERSION_ATTR(sos_fw_version, 0444, psp.sos.fw_version); FW_VERSION_ATTR(asd_fw_version, 0444, psp.asd_context.bin_desc.fw_version); FW_VERSION_ATTR(ta_ras_fw_version, 0444, psp.ras_context.context.bin_desc.fw_version); @@ -719,7 +720,8 @@ static struct attribute *fw_attrs[] = { &dev_attr_ta_ras_fw_version.attr, &dev_attr_ta_xgmi_fw_version.attr, &dev_attr_smc_fw_version.attr, &dev_attr_sdma_fw_version.attr, &dev_attr_sdma2_fw_version.attr, &dev_attr_vcn_fw_version.attr, - &dev_attr_dmcu_fw_version.attr, NULL + &dev_attr_dmcu_fw_version.attr, &dev_attr_imu_fw_version.attr, + NULL }; static const struct attribute_group fw_attr_group = { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index 9c765b04aae3b..c73abe54d9747 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -547,6 +547,7 @@ static void amdgpu_virt_populate_vf2pf_ucode_info(struct amdgpu_device *adev) POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_RLC_SRLS, adev->gfx.rlc_srls_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC, adev->gfx.mec_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_MEC2, adev->gfx.mec2_fw_version); + POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_IMU, adev->gfx.imu_fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_SOS, adev->psp.sos.fw_version); POPULATE_UCODE_INFO(vf2pf_info, AMD_SRIOV_UCODE_ID_ASD, adev->psp.asd_context.bin_desc.fw_version); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h index e78e4c27b62ad..6c97148ca0ed3 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h @@ -70,6 +70,7 @@ enum amd_sriov_ucode_engine_id { AMD_SRIOV_UCODE_ID_RLC_SRLS, AMD_SRIOV_UCODE_ID_MEC, AMD_SRIOV_UCODE_ID_MEC2, + AMD_SRIOV_UCODE_ID_IMU, AMD_SRIOV_UCODE_ID_SOS, AMD_SRIOV_UCODE_ID_ASD, AMD_SRIOV_UCODE_ID_TA_RAS, diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 7ee65c0b4f70a..0d93ec132ebbc 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -763,6 +763,8 @@ struct drm_amdgpu_cs_chunk_data { #define AMDGPU_INFO_FW_MES_KIQ 0x19 /* Subquery id: Query MES firmware version */ #define AMDGPU_INFO_FW_MES 0x1a + /* Subquery id: Query IMU firmware version */ + #define AMDGPU_INFO_FW_IMU 0x1b /* number of bytes moved for TTM migration */ #define AMDGPU_INFO_NUM_BYTES_MOVED 0x0f -- GitLab From e105b6212f1f90c56c04439279d0ef0f8dd1c308 Mon Sep 17 00:00:00 2001 From: YuBiao Wang Date: Wed, 19 Oct 2022 11:36:32 +0800 Subject: [PATCH 0416/1927] drm/amdgpu: skip mes self test for gc 11.0.3 in recover Temporary disable mes self teset for gc 11.0.3 during gpu_recovery. Signed-off-by: YuBiao Wang Acked-by: Luben Tuikov Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index e0445e8cc3424..5b8362727226d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -5381,7 +5381,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res); } - if (adev->enable_mes) + if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3)) amdgpu_mes_self_test(tmp_adev); if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) { -- GitLab From 9656db1b933caf6ffaaef10322093fe018359090 Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Thu, 20 Oct 2022 14:44:26 +0800 Subject: [PATCH 0417/1927] drm/amdkfd: update gfx1037 Lx cache setting Update the gfx1037 L1/L2 cache setting. Signed-off-by: Prike Liang Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index cd5f8b219bf94..d6fa787ff4901 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -795,6 +795,54 @@ static struct kfd_gpu_cache_info yellow_carp_cache_info[] = { }, }; +static struct kfd_gpu_cache_info gfx1037_cache_info[] = { + { + /* TCP L1 Cache per CU */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 1, + }, + { + /* Scalar L1 Instruction Cache per SQC */ + .cache_size = 32, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_INST_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* Scalar L1 Data Cache per SQC */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* GL1 Data Cache per SA */ + .cache_size = 128, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* L2 Data Cache per GPU (Total Tex Cache) */ + .cache_size = 256, + .cache_level = 2, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, +}; + static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, struct crat_subtype_computeunit *cu) { @@ -1515,10 +1563,13 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, break; case IP_VERSION(10, 3, 3): case IP_VERSION(10, 3, 6): /* TODO: Double check these on production silicon */ - case IP_VERSION(10, 3, 7): /* TODO: Double check these on production silicon */ pcache_info = yellow_carp_cache_info; num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info); break; + case IP_VERSION(10, 3, 7): + pcache_info = gfx1037_cache_info; + num_of_cache_types = ARRAY_SIZE(gfx1037_cache_info); + break; case IP_VERSION(11, 0, 0): case IP_VERSION(11, 0, 1): case IP_VERSION(11, 0, 2): -- GitLab From 969758bbf5e9360b63bbb2328ac3fda46bbbc9f5 Mon Sep 17 00:00:00 2001 From: Jesse Zhang Date: Tue, 11 Oct 2022 05:23:10 +0000 Subject: [PATCH 0418/1927] drm/amdkfd: correct the cache info for gfx1036 correct the cache information for gfx1036 Acked-by: Alex Deucher Reviewed-by: Yifan Zhang Signed-off-by: Yifan Zhang Signed-off-by: Jesse Zhang Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 53 ++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index d6fa787ff4901..8bfdfd062ff64 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -843,6 +843,54 @@ static struct kfd_gpu_cache_info gfx1037_cache_info[] = { }, }; +static struct kfd_gpu_cache_info gc_10_3_6_cache_info[] = { + { + /* TCP L1 Cache per CU */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 1, + }, + { + /* Scalar L1 Instruction Cache per SQC */ + .cache_size = 32, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_INST_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* Scalar L1 Data Cache per SQC */ + .cache_size = 16, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* GL1 Data Cache per SA */ + .cache_size = 128, + .cache_level = 1, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, + { + /* L2 Data Cache per GPU (Total Tex Cache) */ + .cache_size = 256, + .cache_level = 2, + .flags = (CRAT_CACHE_FLAGS_ENABLED | + CRAT_CACHE_FLAGS_DATA_CACHE | + CRAT_CACHE_FLAGS_SIMD_CACHE), + .num_cu_shared = 2, + }, +}; + static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev, struct crat_subtype_computeunit *cu) { @@ -1562,10 +1610,13 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, num_of_cache_types = ARRAY_SIZE(beige_goby_cache_info); break; case IP_VERSION(10, 3, 3): - case IP_VERSION(10, 3, 6): /* TODO: Double check these on production silicon */ pcache_info = yellow_carp_cache_info; num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info); break; + case IP_VERSION(10, 3, 6): + pcache_info = gc_10_3_6_cache_info; + num_of_cache_types = ARRAY_SIZE(gc_10_3_6_cache_info); + break; case IP_VERSION(10, 3, 7): pcache_info = gfx1037_cache_info; num_of_cache_types = ARRAY_SIZE(gfx1037_cache_info); -- GitLab From 809734c110548dca410fb0cca52e6b1540319f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joaqu=C3=ADn=20Ignacio=20Aramend=C3=ADa?= Date: Mon, 24 Oct 2022 08:33:59 -0300 Subject: [PATCH 0419/1927] drm/amd/display: Revert logic for plane modifiers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This file was split in commit 5d945cbcd4b16a29d6470a80dfb19738f9a4319f ("drm/amd/display: Create a file dedicated to planes") and the logic in dm_plane_format_mod_supported() function got changed by a switch logic. That change broke drm_plane modifiers setting on series 5000 APUs (tested on OXP mini AMD 5800U and HP Dev One 5850U PRO) leading to Gamescope not working as reported on GitHub[1] To reproduce the issue, enter a TTY and run: $ gamescope -- vkcube With said commit applied it will abort. This one restores the old logic, fixing the issue that affects Gamescope. [1](https://github.com/Plagman/gamescope/issues/624) Cc: # 6.0.x Signed-off-by: Joaquín Ignacio Aramendía Reviewed-by: Bas Nieuwenhuizen Signed-off-by: Alex Deucher --- .../amd/display/amdgpu_dm/amdgpu_dm_plane.c | 50 +++---------------- 1 file changed, 7 insertions(+), 43 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index dfd3be49eac87..e6854f7270a66 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1369,7 +1369,7 @@ static bool dm_plane_format_mod_supported(struct drm_plane *plane, { struct amdgpu_device *adev = drm_to_adev(plane->dev); const struct drm_format_info *info = drm_format_info(format); - struct hw_asic_id asic_id = adev->dm.dc->ctx->asic_id; + int i; enum dm_micro_swizzle microtile = modifier_gfx9_swizzle_mode(modifier) & 3; @@ -1386,49 +1386,13 @@ static bool dm_plane_format_mod_supported(struct drm_plane *plane, return true; } - /* check if swizzle mode is supported by this version of DCN */ - switch (asic_id.chip_family) { - case FAMILY_SI: - case FAMILY_CI: - case FAMILY_KV: - case FAMILY_CZ: - case FAMILY_VI: - /* asics before AI does not have modifier support */ - return false; - case FAMILY_AI: - case FAMILY_RV: - case FAMILY_NV: - case FAMILY_VGH: - case FAMILY_YELLOW_CARP: - case AMDGPU_FAMILY_GC_10_3_6: - case AMDGPU_FAMILY_GC_10_3_7: - switch (AMD_FMT_MOD_GET(TILE, modifier)) { - case AMD_FMT_MOD_TILE_GFX9_64K_R_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D_X: - case AMD_FMT_MOD_TILE_GFX9_64K_S_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D: - return true; - default: - return false; - } - break; - case AMDGPU_FAMILY_GC_11_0_0: - case AMDGPU_FAMILY_GC_11_0_1: - switch (AMD_FMT_MOD_GET(TILE, modifier)) { - case AMD_FMT_MOD_TILE_GFX11_256K_R_X: - case AMD_FMT_MOD_TILE_GFX9_64K_R_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D_X: - case AMD_FMT_MOD_TILE_GFX9_64K_S_X: - case AMD_FMT_MOD_TILE_GFX9_64K_D: - return true; - default: - return false; - } - break; - default: - ASSERT(0); /* Unknown asic */ - break; + /* Check that the modifier is on the list of the plane's supported modifiers. */ + for (i = 0; i < plane->modifier_count; i++) { + if (modifier == plane->modifiers[i]) + break; } + if (i == plane->modifier_count) + return false; /* * For D swizzle the canonical modifier depends on the bpp, so check -- GitLab From a970174d7a1010cb29a5b0c9fa0626abdefcfcbe Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Mon, 24 Oct 2022 11:45:36 -0400 Subject: [PATCH 0420/1927] x86/mm: Do not verify W^X at boot up Adding on the kernel command line "ftrace=function" triggered: CPA detected W^X violation: 8000000000000063 -> 0000000000000063 range: 0xffffffffc0013000 - 0xffffffffc0013fff PFN 10031b WARNING: CPU: 0 PID: 0 at arch/x86/mm/pat/set_memory.c:609 verify_rwx+0x61/0x6d Call Trace: __change_page_attr_set_clr+0x146/0x8a6 change_page_attr_set_clr+0x135/0x268 change_page_attr_clear.constprop.0+0x16/0x1c set_memory_x+0x2c/0x32 arch_ftrace_update_trampoline+0x218/0x2db ftrace_update_trampoline+0x16/0xa1 __register_ftrace_function+0x93/0xb2 ftrace_startup+0x21/0xf0 register_ftrace_function_nolock+0x26/0x40 register_ftrace_function+0x4e/0x143 function_trace_init+0x7d/0xc3 tracer_init+0x23/0x2c tracing_set_tracer+0x1d5/0x206 register_tracer+0x1c0/0x1e4 init_function_trace+0x90/0x96 early_trace_init+0x25c/0x352 start_kernel+0x424/0x6e4 x86_64_start_reservations+0x24/0x2a x86_64_start_kernel+0x8c/0x95 secondary_startup_64_no_verify+0xe0/0xeb This is because at boot up, kernel text is writable, and there's no reason to do tricks to updated it. But the verifier does not distinguish updates at boot up and at run time, and causes a warning at time of boot. Add a check for system_state == SYSTEM_BOOTING and allow it if that is the case. [ These SYSTEM_BOOTING special cases are all pretty horrid, but the x86 text_poke() code does some odd things at bootup, forcing this for now - Linus ] Link: https://lore.kernel.org/r/20221024112730.180916b3@gandalf.local.home Fixes: 652c5bf380ad0 ("x86/mm: Refuse W^X violations") Signed-off-by: Steven Rostedt (Google) Signed-off-by: Linus Torvalds --- arch/x86/mm/pat/set_memory.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index 97342c42dda8e..2e5a045731dec 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -587,6 +587,10 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star { unsigned long end; + /* Kernel text is rw at boot up */ + if (system_state == SYSTEM_BOOTING) + return new; + /* * 32-bit has some unfixable W+X issues, like EFI code * and writeable data being in the same page. Disable -- GitLab From 9d9effca9d7d7cf6341182a7c5cabcbd6fa28063 Mon Sep 17 00:00:00 2001 From: Xin Long Date: Fri, 21 Oct 2022 10:22:47 -0400 Subject: [PATCH 0421/1927] ethtool: eeprom: fix null-deref on genl_info in dump The similar fix as commit 46cdedf2a0fa ("ethtool: pse-pd: fix null-deref on genl_info in dump") is also needed for ethtool eeprom. Fixes: c781ff12a2f3 ("ethtool: Allow network drivers to dump arbitrary EEPROM data") Signed-off-by: Xin Long Link: https://lore.kernel.org/r/5575919a2efc74cd9ad64021880afc3805c54166.1666362167.git.lucien.xin@gmail.com Signed-off-by: Jakub Kicinski --- net/ethtool/eeprom.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/ethtool/eeprom.c b/net/ethtool/eeprom.c index 1c94bb8ea03f2..49c0a2a77f02d 100644 --- a/net/ethtool/eeprom.c +++ b/net/ethtool/eeprom.c @@ -124,7 +124,7 @@ static int eeprom_prepare_data(const struct ethnl_req_info *req_base, if (ret) goto err_free; - ret = get_module_eeprom_by_page(dev, &page_data, info->extack); + ret = get_module_eeprom_by_page(dev, &page_data, info ? info->extack : NULL); if (ret < 0) goto err_ops; -- GitLab From 4fa86555d1cd338afc6e6308cc1ff890a014ec8c Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Fri, 21 Oct 2022 12:35:32 -0700 Subject: [PATCH 0422/1927] genetlink: piggy back on resv_op to default to a reject policy To keep backward compatibility we used to leave attribute parsing to the family if no policy is specified. This becomes tedious as we move to more strict validation. Families must define reject all policies if they don't want any attributes accepted. Piggy back on the resv_start_op field as the switchover point. AFAICT only ethtool has added new commands since the resv_start_op was defined, and it has per-op policies so this should be a no-op. Nonetheless the patch should still go into v6.1 for consistency. Link: https://lore.kernel.org/all/20221019125745.3f2e7659@kernel.org/ Link: https://lore.kernel.org/r/20221021193532.1511293-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- include/net/genetlink.h | 10 +++++++++- net/netlink/genetlink.c | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/include/net/genetlink.h b/include/net/genetlink.h index 3d08e67b3cfcc..9f97f73615b69 100644 --- a/include/net/genetlink.h +++ b/include/net/genetlink.h @@ -41,13 +41,21 @@ struct genl_info; * @mcgrps: multicast groups used by this family * @n_mcgrps: number of multicast groups * @resv_start_op: first operation for which reserved fields of the header - * can be validated, new families should leave this field at zero + * can be validated and policies are required (see below); + * new families should leave this field at zero * @mcgrp_offset: starting number of multicast group IDs in this family * (private) * @ops: the operations supported by this family * @n_ops: number of operations supported by this family * @small_ops: the small-struct operations supported by this family * @n_small_ops: number of small-struct operations supported by this family + * + * Attribute policies (the combination of @policy and @maxattr fields) + * can be attached at the family level or at the operation level. + * If both are present the per-operation policy takes precedence. + * For operations before @resv_start_op lack of policy means that the core + * will perform no attribute parsing or validation. For newer operations + * if policy is not provided core will reject all TLV attributes. */ struct genl_family { int id; /* private */ diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index 39b7c00e4cef0..b1fd059c99928 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -78,10 +78,29 @@ static unsigned long mc_group_start = 0x3 | BIT(GENL_ID_CTRL) | static unsigned long *mc_groups = &mc_group_start; static unsigned long mc_groups_longs = 1; +/* We need the last attribute with non-zero ID therefore a 2-entry array */ +static struct nla_policy genl_policy_reject_all[] = { + { .type = NLA_REJECT }, + { .type = NLA_REJECT }, +}; + static int genl_ctrl_event(int event, const struct genl_family *family, const struct genl_multicast_group *grp, int grp_id); +static void +genl_op_fill_in_reject_policy(const struct genl_family *family, + struct genl_ops *op) +{ + BUILD_BUG_ON(ARRAY_SIZE(genl_policy_reject_all) - 1 != 1); + + if (op->policy || op->cmd < family->resv_start_op) + return; + + op->policy = genl_policy_reject_all; + op->maxattr = 1; +} + static const struct genl_family *genl_family_find_byid(unsigned int id) { return idr_find(&genl_fam_idr, id); @@ -113,6 +132,8 @@ static void genl_op_from_full(const struct genl_family *family, op->maxattr = family->maxattr; if (!op->policy) op->policy = family->policy; + + genl_op_fill_in_reject_policy(family, op); } static int genl_get_cmd_full(u32 cmd, const struct genl_family *family, @@ -142,6 +163,8 @@ static void genl_op_from_small(const struct genl_family *family, op->maxattr = family->maxattr; op->policy = family->policy; + + genl_op_fill_in_reject_policy(family, op); } static int genl_get_cmd_small(u32 cmd, const struct genl_family *family, -- GitLab From 4a4b6848d1e932b977e6a00cda393adf7e839ff8 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Fri, 21 Oct 2022 11:07:11 +0200 Subject: [PATCH 0423/1927] net: lan966x: Stop replacing tx dcbs and dcbs_buf when changing MTU When a frame is sent using FDMA, the skb is mapped and then the mapped address is given to an tx dcb that is different than the last used tx dcb. Once the HW finish with this frame, it would generate an interrupt and then the dcb can be reused and memory can be freed. For each dcb there is an dcb buf that contains some meta-data(is used by PTP, is it free). There is 1 to 1 relationship between dcb and dcb_buf. The following issue was observed. That sometimes after changing the MTU to allocate new tx dcbs and dcbs_buf, two frames were not transmitted. The frames were not transmitted because when reloading the tx dcbs, it was always presuming to use the first dcb but that was not always happening. Because it could be that the last tx dcb used before changing MTU was first dcb and then when it tried to get the next dcb it would take dcb 1 instead of 0. Because it is supposed to take a different dcb than the last used one. This can be fixed simply by changing tx->last_in_use to -1 when the fdma is disabled to reload the new dcb and dcbs_buff. But there could be a different issue. For example, right after the frame is sent, the MTU is changed. Now all the dcbs and dcbs_buf will be cleared. And now get the interrupt from HW that it finished with the frame. So when we try to clear the skb, it is not possible because we lost all the dcbs_buf. The solution here is to stop replacing the tx dcbs and dcbs_buf when changing MTU because the TX doesn't care what is the MTU size, it is only the RX that needs this information. Fixes: 2ea1cbac267e ("net: lan966x: Update FDMA to change MTU.") Signed-off-by: Horatiu Vultur Link: https://lore.kernel.org/r/20221021090711.3749009-1-horatiu.vultur@microchip.com Signed-off-by: Jakub Kicinski --- .../ethernet/microchip/lan966x/lan966x_fdma.c | 24 +++---------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index 7e4061c854f0e..a42035cec611c 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -309,6 +309,7 @@ static void lan966x_fdma_tx_disable(struct lan966x_tx *tx) lan966x, FDMA_CH_DB_DISCARD); tx->activated = false; + tx->last_in_use = -1; } static void lan966x_fdma_tx_reload(struct lan966x_tx *tx) @@ -687,17 +688,14 @@ static int lan966x_qsys_sw_status(struct lan966x *lan966x) static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) { - void *rx_dcbs, *tx_dcbs, *tx_dcbs_buf; - dma_addr_t rx_dma, tx_dma; + dma_addr_t rx_dma; + void *rx_dcbs; u32 size; int err; /* Store these for later to free them */ rx_dma = lan966x->rx.dma; - tx_dma = lan966x->tx.dma; rx_dcbs = lan966x->rx.dcbs; - tx_dcbs = lan966x->tx.dcbs; - tx_dcbs_buf = lan966x->tx.dcbs_buf; napi_synchronize(&lan966x->napi); napi_disable(&lan966x->napi); @@ -715,17 +713,6 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) size = ALIGN(size, PAGE_SIZE); dma_free_coherent(lan966x->dev, size, rx_dcbs, rx_dma); - lan966x_fdma_tx_disable(&lan966x->tx); - err = lan966x_fdma_tx_alloc(&lan966x->tx); - if (err) - goto restore_tx; - - size = sizeof(struct lan966x_tx_dcb) * FDMA_DCB_MAX; - size = ALIGN(size, PAGE_SIZE); - dma_free_coherent(lan966x->dev, size, tx_dcbs, tx_dma); - - kfree(tx_dcbs_buf); - lan966x_fdma_wakeup_netdev(lan966x); napi_enable(&lan966x->napi); @@ -735,11 +722,6 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) lan966x->rx.dcbs = rx_dcbs; lan966x_fdma_rx_start(&lan966x->rx); -restore_tx: - lan966x->tx.dma = tx_dma; - lan966x->tx.dcbs = tx_dcbs; - lan966x->tx.dcbs_buf = tx_dcbs_buf; - return err; } -- GitLab From e72e4032637f4646554794ac28a3abecc6c2416d Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 21 Oct 2022 15:58:54 -0700 Subject: [PATCH 0424/1927] mptcp: set msk local address earlier The mptcp_pm_nl_get_local_id() code assumes that the msk local address is available at that point. For passive sockets, we initialize such address at accept() time. Depending on the running configuration and the user-space timing, a passive MPJ subflow can join the msk socket before accept() completes. In such case, the PM assigns a wrong local id to the MPJ subflow and later PM netlink operations will end-up touching the wrong/unexpected subflow. All the above causes sporadic self-tests failures, especially when the host is heavy loaded. Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/308 Fixes: 01cacb00b35c ("mptcp: add netlink-based PM") Fixes: d045b9eb95a9 ("mptcp: introduce implicit endpoints") Reviewed-by: Mat Martineau Signed-off-by: Paolo Abeni Signed-off-by: Mat Martineau Signed-off-by: Jakub Kicinski --- net/mptcp/protocol.c | 3 +-- net/mptcp/protocol.h | 1 + net/mptcp/subflow.c | 7 +++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index f599ad44ed24c..e33f9caf409d6 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -2952,7 +2952,7 @@ static void mptcp_close(struct sock *sk, long timeout) sock_put(sk); } -static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) +void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) { #if IS_ENABLED(CONFIG_MPTCP_IPV6) const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); @@ -3699,7 +3699,6 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, if (mptcp_is_fully_established(newsk)) mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL); - mptcp_copy_inaddrs(newsk, msk->first); mptcp_rcv_space_init(msk, msk->first); mptcp_propagate_sndbuf(newsk, msk->first); diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index c0b5b4628f650..be19592441df9 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -599,6 +599,7 @@ int mptcp_is_checksum_enabled(const struct net *net); int mptcp_allow_join_id0(const struct net *net); unsigned int mptcp_stale_loss_cnt(const struct net *net); int mptcp_get_pm_type(const struct net *net); +void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk); void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, struct mptcp_options_received *mp_opt); bool __mptcp_retransmit_pending_data(struct sock *sk); diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 07dd23d0fe04a..02a54d59697b5 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -723,6 +723,8 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, goto dispose_child; } + if (new_msk) + mptcp_copy_inaddrs(new_msk, child); subflow_drop_ctx(child); goto out; } @@ -750,6 +752,11 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, ctx->conn = new_msk; new_msk = NULL; + /* set msk addresses early to ensure mptcp_pm_get_local_id() + * uses the correct data + */ + mptcp_copy_inaddrs(ctx->conn, child); + /* with OoO packets we can reach here without ingress * mpc option */ -- GitLab From 54f1944ed6d2554475f39a4921dc5422fa692c4f Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 21 Oct 2022 15:58:55 -0700 Subject: [PATCH 0425/1927] mptcp: factor out mptcp_connect() The current MPTCP connect implementation duplicates a bit of inet code and does not use nor provide a struct proto->connect callback, which in turn will not fit the upcoming fastopen implementation. Refactor such implementation to use the common helper, moving the MPTCP-specific bits into mptcp_connect(). Additionally, avoid an indirect call to the subflow connect callback. Note that the fastopen call-path invokes mptcp_connect() while already holding the subflow socket lock. Explicitly keep track of such path via a new MPTCP-level flag and handle the locking accordingly. Additionally, track the connect flags in a new msk field to allow propagating them to the subflow inet_stream_connect call. Fixes: d98a82a6afc7 ("mptcp: handle defer connect in mptcp_sendmsg") Reviewed-by: Mat Martineau Signed-off-by: Paolo Abeni Signed-off-by: Mat Martineau Signed-off-by: Jakub Kicinski --- net/mptcp/protocol.c | 136 ++++++++++++++++++++++--------------------- net/mptcp/protocol.h | 4 +- 2 files changed, 73 insertions(+), 67 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index e33f9caf409d6..f2930699c6d3a 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -1698,7 +1698,10 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) lock_sock(ssk); + msk->connect_flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; + msk->is_sendmsg = 1; ret = tcp_sendmsg_fastopen(ssk, msg, &copied_syn, len, NULL); + msk->is_sendmsg = 0; copied += copied_syn; if (ret == -EINPROGRESS && copied_syn > 0) { /* reflect the new state on the MPTCP socket */ @@ -3507,10 +3510,73 @@ static int mptcp_ioctl(struct sock *sk, int cmd, unsigned long arg) return put_user(answ, (int __user *)arg); } +static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, + struct mptcp_subflow_context *subflow) +{ + subflow->request_mptcp = 0; + __mptcp_do_fallback(msk); +} + +static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) +{ + struct mptcp_subflow_context *subflow; + struct mptcp_sock *msk = mptcp_sk(sk); + struct socket *ssock; + int err = -EINVAL; + + ssock = __mptcp_nmpc_socket(msk); + if (!ssock) + return -EINVAL; + + mptcp_token_destroy(msk); + inet_sk_state_store(sk, TCP_SYN_SENT); + subflow = mptcp_subflow_ctx(ssock->sk); +#ifdef CONFIG_TCP_MD5SIG + /* no MPTCP if MD5SIG is enabled on this socket or we may run out of + * TCP option space. + */ + if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) + mptcp_subflow_early_fallback(msk, subflow); +#endif + if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) { + MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT); + mptcp_subflow_early_fallback(msk, subflow); + } + if (likely(!__mptcp_check_fallback(msk))) + MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE); + + /* if reaching here via the fastopen/sendmsg path, the caller already + * acquired the subflow socket lock, too. + */ + if (msk->is_sendmsg) + err = __inet_stream_connect(ssock, uaddr, addr_len, msk->connect_flags, 1); + else + err = inet_stream_connect(ssock, uaddr, addr_len, msk->connect_flags); + inet_sk(sk)->defer_connect = inet_sk(ssock->sk)->defer_connect; + + /* on successful connect, the msk state will be moved to established by + * subflow_finish_connect() + */ + if (unlikely(err && err != -EINPROGRESS)) { + inet_sk_state_store(sk, inet_sk_state_load(ssock->sk)); + return err; + } + + mptcp_copy_inaddrs(sk, ssock->sk); + + /* unblocking connect, mptcp-level inet_stream_connect will error out + * without changing the socket state, update it here. + */ + if (err == -EINPROGRESS) + sk->sk_socket->state = ssock->state; + return err; +} + static struct proto mptcp_prot = { .name = "MPTCP", .owner = THIS_MODULE, .init = mptcp_init_sock, + .connect = mptcp_connect, .disconnect = mptcp_disconnect, .close = mptcp_close, .accept = mptcp_accept, @@ -3562,78 +3628,16 @@ static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) return err; } -static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, - struct mptcp_subflow_context *subflow) -{ - subflow->request_mptcp = 0; - __mptcp_do_fallback(msk); -} - static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, int addr_len, int flags) { - struct mptcp_sock *msk = mptcp_sk(sock->sk); - struct mptcp_subflow_context *subflow; - struct socket *ssock; - int err = -EINVAL; + int ret; lock_sock(sock->sk); - if (uaddr) { - if (addr_len < sizeof(uaddr->sa_family)) - goto unlock; - - if (uaddr->sa_family == AF_UNSPEC) { - err = mptcp_disconnect(sock->sk, flags); - sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED; - goto unlock; - } - } - - if (sock->state != SS_UNCONNECTED && msk->subflow) { - /* pending connection or invalid state, let existing subflow - * cope with that - */ - ssock = msk->subflow; - goto do_connect; - } - - ssock = __mptcp_nmpc_socket(msk); - if (!ssock) - goto unlock; - - mptcp_token_destroy(msk); - inet_sk_state_store(sock->sk, TCP_SYN_SENT); - subflow = mptcp_subflow_ctx(ssock->sk); -#ifdef CONFIG_TCP_MD5SIG - /* no MPTCP if MD5SIG is enabled on this socket or we may run out of - * TCP option space. - */ - if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) - mptcp_subflow_early_fallback(msk, subflow); -#endif - if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) { - MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT); - mptcp_subflow_early_fallback(msk, subflow); - } - if (likely(!__mptcp_check_fallback(msk))) - MPTCP_INC_STATS(sock_net(sock->sk), MPTCP_MIB_MPCAPABLEACTIVE); - -do_connect: - err = ssock->ops->connect(ssock, uaddr, addr_len, flags); - inet_sk(sock->sk)->defer_connect = inet_sk(ssock->sk)->defer_connect; - sock->state = ssock->state; - - /* on successful connect, the msk state will be moved to established by - * subflow_finish_connect() - */ - if (!err || err == -EINPROGRESS) - mptcp_copy_inaddrs(sock->sk, ssock->sk); - else - inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); - -unlock: + mptcp_sk(sock->sk)->connect_flags = flags; + ret = __inet_stream_connect(sock, uaddr, addr_len, flags, 0); release_sock(sock->sk); - return err; + return ret; } static int mptcp_listen(struct socket *sock, int backlog) diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index be19592441df9..6a09ab99a12de 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -285,7 +285,9 @@ struct mptcp_sock { u8 mpc_endpoint_id; u8 recvmsg_inq:1, cork:1, - nodelay:1; + nodelay:1, + is_sendmsg:1; + int connect_flags; struct work_struct work; struct sk_buff *ooo_last_skb; struct rb_root out_of_order_queue; -- GitLab From fa9e57468aa10e91deca6d82ccd17c73ffdd1e40 Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 21 Oct 2022 15:58:56 -0700 Subject: [PATCH 0426/1927] mptcp: fix abba deadlock on fastopen Our CI reported lockdep splat in the fastopen code: ====================================================== WARNING: possible circular locking dependency detected 6.0.0.mptcp_f5e8bfe9878d+ #1558 Not tainted ------------------------------------------------------ packetdrill/1071 is trying to acquire lock: ffff8881bd198140 (sk_lock-AF_INET){+.+.}-{0:0}, at: inet_wait_for_connect+0x19c/0x310 but task is already holding lock: ffff8881b8346540 (k-sk_lock-AF_INET){+.+.}-{0:0}, at: mptcp_sendmsg+0xfdf/0x1740 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #1 (k-sk_lock-AF_INET){+.+.}-{0:0}: __lock_acquire+0xb6d/0x1860 lock_acquire+0x1d8/0x620 lock_sock_nested+0x37/0xd0 inet_stream_connect+0x3f/0xa0 mptcp_connect+0x411/0x800 __inet_stream_connect+0x3ab/0x800 mptcp_stream_connect+0xac/0x110 __sys_connect+0x101/0x130 __x64_sys_connect+0x6e/0xb0 do_syscall_64+0x59/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd -> #0 (sk_lock-AF_INET){+.+.}-{0:0}: check_prev_add+0x15e/0x2110 validate_chain+0xace/0xdf0 __lock_acquire+0xb6d/0x1860 lock_acquire+0x1d8/0x620 lock_sock_nested+0x37/0xd0 inet_wait_for_connect+0x19c/0x310 __inet_stream_connect+0x26c/0x800 tcp_sendmsg_fastopen+0x341/0x650 mptcp_sendmsg+0x109d/0x1740 sock_sendmsg+0xe1/0x120 __sys_sendto+0x1c7/0x2a0 __x64_sys_sendto+0xdc/0x1b0 do_syscall_64+0x59/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd other info that might help us debug this: Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(k-sk_lock-AF_INET); lock(sk_lock-AF_INET); lock(k-sk_lock-AF_INET); lock(sk_lock-AF_INET); *** DEADLOCK *** 1 lock held by packetdrill/1071: #0: ffff8881b8346540 (k-sk_lock-AF_INET){+.+.}-{0:0}, at: mptcp_sendmsg+0xfdf/0x1740 ====================================================== The problem is caused by the blocking inet_wait_for_connect() releasing and re-acquiring the msk socket lock while the subflow socket lock is still held and the MPTCP socket requires that the msk socket lock must be acquired before the subflow socket lock. Address the issue always invoking tcp_sendmsg_fastopen() in an unblocking manner, and later eventually complete the blocking __inet_stream_connect() as needed. Fixes: d98a82a6afc7 ("mptcp: handle defer connect in mptcp_sendmsg") Reviewed-by: Mat Martineau Reviewed-by: Matthieu Baerts Signed-off-by: Paolo Abeni Signed-off-by: Mat Martineau Signed-off-by: Jakub Kicinski --- net/mptcp/protocol.c | 49 ++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index f2930699c6d3a..b6dc6e260334f 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -1673,6 +1673,37 @@ static void mptcp_set_nospace(struct sock *sk) set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags); } +static int mptcp_sendmsg_fastopen(struct sock *sk, struct sock *ssk, struct msghdr *msg, + size_t len, int *copied_syn) +{ + unsigned int saved_flags = msg->msg_flags; + struct mptcp_sock *msk = mptcp_sk(sk); + int ret; + + lock_sock(ssk); + msg->msg_flags |= MSG_DONTWAIT; + msk->connect_flags = O_NONBLOCK; + msk->is_sendmsg = 1; + ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL); + msk->is_sendmsg = 0; + msg->msg_flags = saved_flags; + release_sock(ssk); + + /* do the blocking bits of inet_stream_connect outside the ssk socket lock */ + if (ret == -EINPROGRESS && !(msg->msg_flags & MSG_DONTWAIT)) { + ret = __inet_stream_connect(sk->sk_socket, msg->msg_name, + msg->msg_namelen, msg->msg_flags, 1); + + /* Keep the same behaviour of plain TCP: zero the copied bytes in + * case of any error, except timeout or signal + */ + if (ret && ret != -EINPROGRESS && ret != -ERESTARTSYS && ret != -EINTR) + *copied_syn = 0; + } + + return ret; +} + static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) { struct mptcp_sock *msk = mptcp_sk(sk); @@ -1693,26 +1724,14 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) ssock = __mptcp_nmpc_socket(msk); if (unlikely(ssock && inet_sk(ssock->sk)->defer_connect)) { - struct sock *ssk = ssock->sk; int copied_syn = 0; - lock_sock(ssk); - - msk->connect_flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0; - msk->is_sendmsg = 1; - ret = tcp_sendmsg_fastopen(ssk, msg, &copied_syn, len, NULL); - msk->is_sendmsg = 0; + ret = mptcp_sendmsg_fastopen(sk, ssock->sk, msg, len, &copied_syn); copied += copied_syn; - if (ret == -EINPROGRESS && copied_syn > 0) { - /* reflect the new state on the MPTCP socket */ - inet_sk_state_store(sk, inet_sk_state_load(ssk)); - release_sock(ssk); + if (ret == -EINPROGRESS && copied_syn > 0) goto out; - } else if (ret) { - release_sock(ssk); + else if (ret) goto do_error; - } - release_sock(ssk); } timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); -- GitLab From b5f9a01fae42684648c2ee3cd9985f80c67ab9f7 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Thu, 13 Oct 2022 12:03:33 +0800 Subject: [PATCH 0427/1927] RDMA/rxe: Fix mr leak in RESPST_ERR_RNR rxe_recheck_mr() will increase mr's ref_cnt, so we should call rxe_put(mr) to drop mr's ref_cnt in RESPST_ERR_RNR to avoid below warning: WARNING: CPU: 0 PID: 4156 at drivers/infiniband/sw/rxe/rxe_pool.c:259 __rxe_cleanup+0x1df/0x240 [rdma_rxe] ... Call Trace: rxe_dereg_mr+0x4c/0x60 [rdma_rxe] ib_dereg_mr_user+0xa8/0x200 [ib_core] ib_mr_pool_destroy+0x77/0xb0 [ib_core] nvme_rdma_destroy_queue_ib+0x89/0x240 [nvme_rdma] nvme_rdma_free_queue+0x40/0x50 [nvme_rdma] nvme_rdma_teardown_io_queues.part.0+0xc3/0x120 [nvme_rdma] nvme_rdma_error_recovery_work+0x4d/0xf0 [nvme_rdma] process_one_work+0x582/0xa40 ? pwq_dec_nr_in_flight+0x100/0x100 ? rwlock_bug.part.0+0x60/0x60 worker_thread+0x2a9/0x700 ? process_one_work+0xa40/0xa40 kthread+0x168/0x1a0 ? kthread_complete_and_exit+0x20/0x20 ret_from_fork+0x22/0x30 Link: https://lore.kernel.org/r/20221024052049.20577-1-lizhijian@fujitsu.com Fixes: 8a1a0be894da ("RDMA/rxe: Replace mr by rkey in responder resources") Signed-off-by: Li Zhijian Signed-off-by: Jason Gunthorpe Signed-off-by: Leon Romanovsky --- drivers/infiniband/sw/rxe/rxe_resp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index ed5a09e86417e..693081e813ec0 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -806,8 +806,10 @@ static enum resp_states read_reply(struct rxe_qp *qp, skb = prepare_ack_packet(qp, &ack_pkt, opcode, payload, res->cur_psn, AETH_ACK_UNLIMITED); - if (!skb) + if (!skb) { + rxe_put(mr); return RESPST_ERR_RNR; + } rxe_mr_copy(mr, res->read.va, payload_addr(&ack_pkt), payload, RXE_FROM_MR_OBJ); -- GitLab From 3e5b3418827cefb5e1cc658806f02965791b8f07 Mon Sep 17 00:00:00 2001 From: Dongliang Mu Date: Mon, 24 Oct 2022 19:48:07 +0800 Subject: [PATCH 0428/1927] can: mscan: mpc5xxx: mpc5xxx_can_probe(): add missing put_clock() in error path The commit 1149108e2fbf ("can: mscan: improve clock API use") only adds put_clock() in mpc5xxx_can_remove() function, forgetting to add put_clock() in the error handling code. Fix this bug by adding put_clock() in the error handling code. Fixes: 1149108e2fbf ("can: mscan: improve clock API use") Signed-off-by: Dongliang Mu Link: https://lore.kernel.org/all/20221024133828.35881-1-mkl@pengutronix.de Signed-off-by: Marc Kleine-Budde --- drivers/net/can/mscan/mpc5xxx_can.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/net/can/mscan/mpc5xxx_can.c b/drivers/net/can/mscan/mpc5xxx_can.c index c469b2f3e57d5..b0ed798ae70fe 100644 --- a/drivers/net/can/mscan/mpc5xxx_can.c +++ b/drivers/net/can/mscan/mpc5xxx_can.c @@ -322,14 +322,14 @@ static int mpc5xxx_can_probe(struct platform_device *ofdev) &mscan_clksrc); if (!priv->can.clock.freq) { dev_err(&ofdev->dev, "couldn't get MSCAN clock properties\n"); - goto exit_free_mscan; + goto exit_put_clock; } err = register_mscandev(dev, mscan_clksrc); if (err) { dev_err(&ofdev->dev, "registering %s failed (err=%d)\n", DRV_NAME, err); - goto exit_free_mscan; + goto exit_put_clock; } dev_info(&ofdev->dev, "MSCAN at 0x%p, irq %d, clock %d Hz\n", @@ -337,7 +337,9 @@ static int mpc5xxx_can_probe(struct platform_device *ofdev) return 0; -exit_free_mscan: +exit_put_clock: + if (data->put_clock) + data->put_clock(ofdev); free_candev(dev); exit_dispose_irq: irq_dispose_mapping(irq); -- GitLab From b1a09b63684cea56774786ca14c13b7041ffee63 Mon Sep 17 00:00:00 2001 From: Dongliang Mu Date: Mon, 24 Oct 2022 17:02:52 +0800 Subject: [PATCH 0429/1927] can: mcp251x: mcp251x_can_probe(): add missing unregister_candev() in error path In mcp251x_can_probe(), if mcp251x_gpio_setup() fails, it forgets to unregister the CAN device. Fix this by unregistering can device in mcp251x_can_probe(). Fixes: 2d52dabbef60 ("can: mcp251x: add GPIO support") Signed-off-by: Dongliang Mu Link: https://lore.kernel.org/all/20221024090256.717236-1-dzm91@hust.edu.cn [mkl: adjust label] Signed-off-by: Marc Kleine-Budde --- drivers/net/can/spi/mcp251x.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index c320de474f406..24883a65ca66a 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -1415,11 +1415,14 @@ static int mcp251x_can_probe(struct spi_device *spi) ret = mcp251x_gpio_setup(priv); if (ret) - goto error_probe; + goto out_unregister_candev; netdev_info(net, "MCP%x successfully initialized.\n", priv->model); return 0; +out_unregister_candev: + unregister_candev(net); + error_probe: destroy_workqueue(priv->wq); priv->wq = NULL; -- GitLab From 1a2dcbdde82e3a5f1db9b2f4c48aa1aeba534fb2 Mon Sep 17 00:00:00 2001 From: Sreekanth Reddy Date: Tue, 13 Sep 2022 17:35:38 +0530 Subject: [PATCH 0430/1927] scsi: mpt3sas: re-do lost mpt3sas DMA mask fix This is a re-do of commit e0e0747de0ea ("scsi: mpt3sas: Fix return value check of dma_get_required_mask()"), which I ended up undoing in a mis-merge in commit 62e6e5940c0c ("Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi"). The original commit message was scsi: mpt3sas: Fix return value check of dma_get_required_mask() Fix the incorrect return value check of dma_get_required_mask(). Due to this incorrect check, the driver was always setting the DMA mask to 63 bit. Link: https://lore.kernel.org/r/20220913120538.18759-2-sreekanth.reddy@broadcom.com Fixes: ba27c5cf286d ("scsi: mpt3sas: Don't change the DMA coherent mask after allocations") Signed-off-by: Sreekanth Reddy Signed-off-by: Martin K. Petersen and this fix was lost when I mis-merged the conflict with commit 9df650963bf6 ("scsi: mpt3sas: Don't change DMA mask while reallocating pools"). Reported-by: Juergen Gross Fixes: 62e6e5940c0c ("Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi") Link: https://lore.kernel.org/all/CAHk-=wjaK-TxrNaGtFDpL9qNHL1MVkWXO1TT6vObD5tXMSC4Zg@mail.gmail.com Signed-off-by: Linus Torvalds --- drivers/scsi/mpt3sas/mpt3sas_base.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 8b22df8c1792d..4e981ccaac416 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -2993,7 +2993,7 @@ _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev) u64 coherent_dma_mask, dma_mask; if (ioc->is_mcpu_endpoint || sizeof(dma_addr_t) == 4 || - dma_get_required_mask(&pdev->dev) <= 32) { + dma_get_required_mask(&pdev->dev) <= DMA_BIT_MASK(32)) { ioc->dma_mask = 32; coherent_dma_mask = dma_mask = DMA_BIT_MASK(32); /* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */ -- GitLab From 76a66ba101329316a5d7f4275070be22eb85fdf2 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Fri, 21 Oct 2022 08:43:45 +0800 Subject: [PATCH 0431/1927] btrfs: don't use btrfs_chunk::sub_stripes from disk [BUG] There are two reports (the earliest one from LKP, a more recent one from kernel bugzilla) that we can have some chunks with 0 as sub_stripes. This will cause divide-by-zero errors at btrfs_rmap_block, which is introduced by a recent kernel patch ac0677348f3c ("btrfs: merge calculations for simple striped profiles in btrfs_rmap_block"): if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) { stripe_nr = stripe_nr * map->num_stripes + i; stripe_nr = div_u64(stripe_nr, map->sub_stripes); <<< } [CAUSE] From the more recent report, it has been proven that we have some chunks with 0 as sub_stripes, mostly caused by older mkfs. It turns out that the mkfs.btrfs fix is only introduced in 6718ab4d33aa ("btrfs-progs: Initialize sub_stripes to 1 in btrfs_alloc_data_chunk") which is included in v5.4 btrfs-progs release. So there would be quite some old filesystems with such 0 sub_stripes. [FIX] Just don't trust the sub_stripes values from disk. We have a trusted btrfs_raid_array[] to fetch the correct sub_stripes numbers for each profile and that are fixed. By this, we can keep the compatibility with older filesystems while still avoid divide-by-zero bugs. Reported-by: kernel test robot Reported-by: Viktor Kuzmin Bugzilla: https://bugzilla.kernel.org/show_bug.cgi?id=216559 Fixes: ac0677348f3c ("btrfs: merge calculations for simple striped profiles in btrfs_rmap_block") CC: stable@vger.kernel.org # 6.0 Reviewed-by: Su Yue Reviewed-by: Johannes Thumshirn Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 94ba46d579205..a8d4bc6a19379 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -7142,6 +7142,7 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, u64 devid; u64 type; u8 uuid[BTRFS_UUID_SIZE]; + int index; int num_stripes; int ret; int i; @@ -7149,6 +7150,7 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, logical = key->offset; length = btrfs_chunk_length(leaf, chunk); type = btrfs_chunk_type(leaf, chunk); + index = btrfs_bg_flags_to_raid_index(type); num_stripes = btrfs_chunk_num_stripes(leaf, chunk); #if BITS_PER_LONG == 32 @@ -7202,7 +7204,15 @@ static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, map->io_align = btrfs_chunk_io_align(leaf, chunk); map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk); map->type = type; - map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk); + /* + * We can't use the sub_stripes value, as for profiles other than + * RAID10, they may have 0 as sub_stripes for filesystems created by + * older mkfs (sub_stripes = btrfs_raid_array[index].sub_stripes; map->verified_stripes = 0; em->orig_block_len = btrfs_calc_stripe_length(em); for (i = 0; i < num_stripes; i++) { -- GitLab From 88619e77b33d5718fae3c13d29f94b2646facfcd Mon Sep 17 00:00:00 2001 From: Benjamin Gaignard Date: Fri, 21 Oct 2022 19:24:22 +0200 Subject: [PATCH 0432/1927] net: stmmac: rk3588: Allow multiple gmac controller RK3588(s) can have multiple gmac controllers. Re-use rk3568 logic to distinguish them. Fixes: 2f2b60a0ec28 ("net: ethernet: stmmac: dwmac-rk: Add gmac support for rk3588") Signed-off-by: Benjamin Gaignard Signed-off-by: Sebastian Reichel Link: https://lore.kernel.org/r/20221021172422.88534-1-sebastian.reichel@collabora.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index f7269d79a3851..6656d76b6766b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -1243,6 +1243,12 @@ static const struct rk_gmac_ops rk3588_ops = { .set_rgmii_speed = rk3588_set_gmac_speed, .set_rmii_speed = rk3588_set_gmac_speed, .set_clock_selection = rk3588_set_clock_selection, + .regs_valid = true, + .regs = { + 0xfe1b0000, /* gmac0 */ + 0xfe1c0000, /* gmac1 */ + 0x0, /* sentinel */ + }, }; #define RV1108_GRF_GMAC_CON0 0X0900 -- GitLab From b3af84383e7abdc5e63435817bb73a268e7c3637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 27 Sep 2022 18:43:03 +0200 Subject: [PATCH 0433/1927] drm/scheduler: fix fence ref counting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We leaked dependency fences when processes were beeing killed. Additional to that grab a reference to the last scheduled fence. Signed-off-by: Christian König Reviewed-by: Andrey Grodzovsky Link: https://patchwork.freedesktop.org/patch/msgid/20220929180151.139751-1-christian.koenig@amd.com --- drivers/gpu/drm/scheduler/sched_entity.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 6137537aaea4d..4b913dbb7d7b6 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -207,6 +207,7 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, struct drm_sched_job *job = container_of(cb, struct drm_sched_job, finish_cb); + dma_fence_put(f); INIT_WORK(&job->work, drm_sched_entity_kill_jobs_work); schedule_work(&job->work); } @@ -234,8 +235,10 @@ static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) struct drm_sched_fence *s_fence = job->s_fence; /* Wait for all dependencies to avoid data corruptions */ - while ((f = drm_sched_job_dependency(job, entity))) + while ((f = drm_sched_job_dependency(job, entity))) { dma_fence_wait(f, false); + dma_fence_put(f); + } drm_sched_fence_scheduled(s_fence); dma_fence_set_error(&s_fence->finished, -ESRCH); @@ -250,6 +253,7 @@ static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) continue; } + dma_fence_get(entity->last_scheduled); r = dma_fence_add_callback(entity->last_scheduled, &job->finish_cb, drm_sched_entity_kill_jobs_cb); -- GitLab From 85850af4fc47132f3f2f0dd698b90f67906600b4 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 12 Oct 2022 22:50:17 -0500 Subject: [PATCH 0434/1927] PM: hibernate: Allow hybrid sleep to work with s2idle Hybrid sleep is currently hardcoded to only operate with S3 even on systems that might not support it. Instead of assuming this mode is what the user wants to use, for hybrid sleep follow the setting of `mem_sleep_current` which will respect mem_sleep_default kernel command line and policy decisions made by the presence of the FADT low power idle bit. Fixes: 81d45bdf8913 ("PM / hibernate: Untangle power_down()") Reported-and-tested-by: kolAflash Link: https://bugzilla.kernel.org/show_bug.cgi?id=216574 Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- kernel/power/hibernate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index f58a0aa92310c..793c55a2becba 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -645,7 +645,7 @@ static void power_down(void) int error; if (hibernation_mode == HIBERNATION_SUSPEND) { - error = suspend_devices_and_enter(PM_SUSPEND_MEM); + error = suspend_devices_and_enter(mem_sleep_current); if (error) { hibernation_mode = hibernation_ops ? HIBERNATION_PLATFORM : -- GitLab From 8dbab94d45fb1094cefac7956b7fb987a36e2b12 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 24 Oct 2022 21:21:00 +0200 Subject: [PATCH 0435/1927] cpufreq: intel_pstate: Read all MSRs on the target CPU Some of the MSR accesses in intel_pstate are carried out on the CPU that is running the code, but the values coming from them are used for the performance scaling of the other CPUs. This is problematic, for example, on hybrid platforms where MSR_TURBO_RATIO_LIMIT for P-cores and E-cores is different, so the values read from it on a P-core are generally not applicable to E-cores and the other way around. For this reason, make the driver access all MSRs on the target CPU on platforms using the "core" pstate_funcs callbacks which is the case for all of the hybrid platforms released to date. For this purpose, pass a CPU argument to the ->get_max(), ->get_max_physical(), ->get_min() and ->get_turbo() pstate_funcs callbacks and from there pass it to rdmsrl_on_cpu() or rdmsrl_safe_on_cpu() to access the MSR on the target CPU. Fixes: 46573fd6369f ("cpufreq: intel_pstate: hybrid: Rework HWP calibration") Acked-by: Srinivas Pandruvada Tested-by: Srinivas Pandruvada Cc: 5.15+ # 5.15+ Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 66 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index fc3ebeb0bbe59..457694a2d0e83 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -280,10 +280,10 @@ static struct cpudata **all_cpu_data; * structure is used to store those callbacks. */ struct pstate_funcs { - int (*get_max)(void); - int (*get_max_physical)(void); - int (*get_min)(void); - int (*get_turbo)(void); + int (*get_max)(int cpu); + int (*get_max_physical)(int cpu); + int (*get_min)(int cpu); + int (*get_turbo)(int cpu); int (*get_scaling)(void); int (*get_cpu_scaling)(int cpu); int (*get_aperf_mperf_shift)(void); @@ -531,12 +531,12 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) { int perf_ctl_max_phys = cpu->pstate.max_pstate_physical; int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; - int perf_ctl_turbo = pstate_funcs.get_turbo(); + int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu); int turbo_freq = perf_ctl_turbo * perf_ctl_scaling; int scaling = cpu->pstate.scaling; pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); - pr_debug("CPU%d: perf_ctl_max = %d\n", cpu->cpu, pstate_funcs.get_max()); + pr_debug("CPU%d: perf_ctl_max = %d\n", cpu->cpu, pstate_funcs.get_max(cpu->cpu)); pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu->cpu, perf_ctl_turbo); pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu->cpu, perf_ctl_scaling); pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate); @@ -1740,7 +1740,7 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata) intel_pstate_update_epp_defaults(cpudata); } -static int atom_get_min_pstate(void) +static int atom_get_min_pstate(int not_used) { u64 value; @@ -1748,7 +1748,7 @@ static int atom_get_min_pstate(void) return (value >> 8) & 0x7F; } -static int atom_get_max_pstate(void) +static int atom_get_max_pstate(int not_used) { u64 value; @@ -1756,7 +1756,7 @@ static int atom_get_max_pstate(void) return (value >> 16) & 0x7F; } -static int atom_get_turbo_pstate(void) +static int atom_get_turbo_pstate(int not_used) { u64 value; @@ -1834,23 +1834,23 @@ static void atom_get_vid(struct cpudata *cpudata) cpudata->vid.turbo = value & 0x7f; } -static int core_get_min_pstate(void) +static int core_get_min_pstate(int cpu) { u64 value; - rdmsrl(MSR_PLATFORM_INFO, value); + rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value); return (value >> 40) & 0xFF; } -static int core_get_max_pstate_physical(void) +static int core_get_max_pstate_physical(int cpu) { u64 value; - rdmsrl(MSR_PLATFORM_INFO, value); + rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &value); return (value >> 8) & 0xFF; } -static int core_get_tdp_ratio(u64 plat_info) +static int core_get_tdp_ratio(int cpu, u64 plat_info) { /* Check how many TDP levels present */ if (plat_info & 0x600000000) { @@ -1860,13 +1860,13 @@ static int core_get_tdp_ratio(u64 plat_info) int err; /* Get the TDP level (0, 1, 2) to get ratios */ - err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); + err = rdmsrl_safe_on_cpu(cpu, MSR_CONFIG_TDP_CONTROL, &tdp_ctrl); if (err) return err; /* TDP MSR are continuous starting at 0x648 */ tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03); - err = rdmsrl_safe(tdp_msr, &tdp_ratio); + err = rdmsrl_safe_on_cpu(cpu, tdp_msr, &tdp_ratio); if (err) return err; @@ -1883,7 +1883,7 @@ static int core_get_tdp_ratio(u64 plat_info) return -ENXIO; } -static int core_get_max_pstate(void) +static int core_get_max_pstate(int cpu) { u64 tar; u64 plat_info; @@ -1891,10 +1891,10 @@ static int core_get_max_pstate(void) int tdp_ratio; int err; - rdmsrl(MSR_PLATFORM_INFO, plat_info); + rdmsrl_on_cpu(cpu, MSR_PLATFORM_INFO, &plat_info); max_pstate = (plat_info >> 8) & 0xFF; - tdp_ratio = core_get_tdp_ratio(plat_info); + tdp_ratio = core_get_tdp_ratio(cpu, plat_info); if (tdp_ratio <= 0) return max_pstate; @@ -1903,7 +1903,7 @@ static int core_get_max_pstate(void) return tdp_ratio; } - err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar); + err = rdmsrl_safe_on_cpu(cpu, MSR_TURBO_ACTIVATION_RATIO, &tar); if (!err) { int tar_levels; @@ -1918,13 +1918,13 @@ static int core_get_max_pstate(void) return max_pstate; } -static int core_get_turbo_pstate(void) +static int core_get_turbo_pstate(int cpu) { u64 value; int nont, ret; - rdmsrl(MSR_TURBO_RATIO_LIMIT, value); - nont = core_get_max_pstate(); + rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); + nont = core_get_max_pstate(cpu); ret = (value) & 255; if (ret <= nont) ret = nont; @@ -1952,13 +1952,13 @@ static int knl_get_aperf_mperf_shift(void) return 10; } -static int knl_get_turbo_pstate(void) +static int knl_get_turbo_pstate(int cpu) { u64 value; int nont, ret; - rdmsrl(MSR_TURBO_RATIO_LIMIT, value); - nont = core_get_max_pstate(); + rdmsrl_on_cpu(cpu, MSR_TURBO_RATIO_LIMIT, &value); + nont = core_get_max_pstate(cpu); ret = (((value) >> 8) & 0xFF); if (ret <= nont) ret = nont; @@ -2025,10 +2025,10 @@ static void intel_pstate_max_within_limits(struct cpudata *cpu) static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) { - int perf_ctl_max_phys = pstate_funcs.get_max_physical(); + int perf_ctl_max_phys = pstate_funcs.get_max_physical(cpu->cpu); int perf_ctl_scaling = pstate_funcs.get_scaling(); - cpu->pstate.min_pstate = pstate_funcs.get_min(); + cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu); cpu->pstate.max_pstate_physical = perf_ctl_max_phys; cpu->pstate.perf_ctl_scaling = perf_ctl_scaling; @@ -2044,8 +2044,8 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) } } else { cpu->pstate.scaling = perf_ctl_scaling; - cpu->pstate.max_pstate = pstate_funcs.get_max(); - cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(); + cpu->pstate.max_pstate = pstate_funcs.get_max(cpu->cpu); + cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(cpu->cpu); } if (cpu->pstate.scaling == perf_ctl_scaling) { @@ -3221,9 +3221,9 @@ static unsigned int force_load __initdata; static int __init intel_pstate_msrs_not_valid(void) { - if (!pstate_funcs.get_max() || - !pstate_funcs.get_min() || - !pstate_funcs.get_turbo()) + if (!pstate_funcs.get_max(0) || + !pstate_funcs.get_min(0) || + !pstate_funcs.get_turbo(0)) return -ENODEV; return 0; -- GitLab From f5c8cf2a4992dd929fa0c2f25c09ee69b8dcbce1 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 24 Oct 2022 21:22:48 +0200 Subject: [PATCH 0436/1927] cpufreq: intel_pstate: hybrid: Use known scaling factor for P-cores Commit 46573fd6369f ("cpufreq: intel_pstate: hybrid: Rework HWP calibration") attempted to use the information from CPPC (the nominal performance in particular) to obtain the scaling factor allowing the frequency to be computed if the HWP performance level of the given CPU is known or vice versa. However, it turns out that on some platforms this doesn't work, because the CPPC information on them does not align with the contents of the MSR_HWP_CAPABILITIES registers. This basically means that the only way to make intel_pstate work on all of the hybrid platforms to date is to use the observation that on all of them the scaling factor between the HWP performance levels and frequency for P-cores is 78741 (approximately 100000/1.27). For E-cores it is 100000, which is the same as for all of the non-hybrid "core" platforms and does not require any changes. Accordingly, make intel_pstate use 78741 as the scaling factor between HWP performance levels and frequency for P-cores on all hybrid platforms and drop the dependency of the HWP calibration code on CPPC. Fixes: 46573fd6369f ("cpufreq: intel_pstate: hybrid: Rework HWP calibration") Reported-by: Srinivas Pandruvada Acked-by: Srinivas Pandruvada Tested-by: Srinivas Pandruvada Cc: 5.15+ # 5.15+ Signed-off-by: Rafael J. Wysocki --- drivers/cpufreq/intel_pstate.c | 69 ++++++++-------------------------- 1 file changed, 15 insertions(+), 54 deletions(-) diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 457694a2d0e83..6ff73c30769fa 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -398,16 +399,6 @@ static int intel_pstate_get_cppc_guaranteed(int cpu) return cppc_perf.nominal_perf; } - -static u32 intel_pstate_cppc_nominal(int cpu) -{ - u64 nominal_perf; - - if (cppc_get_nominal_perf(cpu, &nominal_perf)) - return 0; - - return nominal_perf; -} #else /* CONFIG_ACPI_CPPC_LIB */ static inline void intel_pstate_set_itmt_prio(int cpu) { @@ -532,34 +523,17 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) int perf_ctl_max_phys = cpu->pstate.max_pstate_physical; int perf_ctl_scaling = cpu->pstate.perf_ctl_scaling; int perf_ctl_turbo = pstate_funcs.get_turbo(cpu->cpu); - int turbo_freq = perf_ctl_turbo * perf_ctl_scaling; int scaling = cpu->pstate.scaling; pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); - pr_debug("CPU%d: perf_ctl_max = %d\n", cpu->cpu, pstate_funcs.get_max(cpu->cpu)); pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu->cpu, perf_ctl_turbo); pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu->cpu, perf_ctl_scaling); pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate); pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate); pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling); - /* - * If the product of the HWP performance scaling factor and the HWP_CAP - * highest performance is greater than the maximum turbo frequency - * corresponding to the pstate_funcs.get_turbo() return value, the - * scaling factor is too high, so recompute it to make the HWP_CAP - * highest performance correspond to the maximum turbo frequency. - */ - cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * scaling; - if (turbo_freq < cpu->pstate.turbo_freq) { - cpu->pstate.turbo_freq = turbo_freq; - scaling = DIV_ROUND_UP(turbo_freq, cpu->pstate.turbo_pstate); - cpu->pstate.scaling = scaling; - - pr_debug("CPU%d: refined HWP-to-frequency scaling factor: %d\n", - cpu->cpu, scaling); - } - + cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling, + perf_ctl_scaling); cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling, perf_ctl_scaling); @@ -1965,37 +1939,24 @@ static int knl_get_turbo_pstate(int cpu) return ret; } -#ifdef CONFIG_ACPI_CPPC_LIB -static u32 hybrid_ref_perf; - -static int hybrid_get_cpu_scaling(int cpu) +static void hybrid_get_type(void *data) { - return DIV_ROUND_UP(core_get_scaling() * hybrid_ref_perf, - intel_pstate_cppc_nominal(cpu)); + u8 *cpu_type = data; + + *cpu_type = get_this_hybrid_cpu_type(); } -static void intel_pstate_cppc_set_cpu_scaling(void) +static int hybrid_get_cpu_scaling(int cpu) { - u32 min_nominal_perf = U32_MAX; - int cpu; + u8 cpu_type = 0; - for_each_present_cpu(cpu) { - u32 nominal_perf = intel_pstate_cppc_nominal(cpu); + smp_call_function_single(cpu, hybrid_get_type, &cpu_type, 1); + /* P-cores have a smaller perf level-to-freqency scaling factor. */ + if (cpu_type == 0x40) + return 78741; - if (nominal_perf && nominal_perf < min_nominal_perf) - min_nominal_perf = nominal_perf; - } - - if (min_nominal_perf < U32_MAX) { - hybrid_ref_perf = min_nominal_perf; - pstate_funcs.get_cpu_scaling = hybrid_get_cpu_scaling; - } + return core_get_scaling(); } -#else -static inline void intel_pstate_cppc_set_cpu_scaling(void) -{ -} -#endif /* CONFIG_ACPI_CPPC_LIB */ static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate) { @@ -3450,7 +3411,7 @@ static int __init intel_pstate_init(void) default_driver = &intel_pstate; if (boot_cpu_has(X86_FEATURE_HYBRID_CPU)) - intel_pstate_cppc_set_cpu_scaling(); + pstate_funcs.get_cpu_scaling = hybrid_get_cpu_scaling; goto hwp_cpu_matched; } -- GitLab From ee03c0f200eb0d9f22dd8732d9fb7956d91019c2 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 24 Oct 2022 18:29:29 +0200 Subject: [PATCH 0437/1927] ALSA: au88x0: use explicitly signed char With char becoming unsigned by default, and with `char` alone being ambiguous and based on architecture, signed chars need to be marked explicitly as such. This fixes warnings like: sound/pci/au88x0/au88x0_core.c:2029 vortex_adb_checkinout() warn: signedness bug returning '(-22)' sound/pci/au88x0/au88x0_core.c:2046 vortex_adb_checkinout() warn: signedness bug returning '(-12)' sound/pci/au88x0/au88x0_core.c:2125 vortex_adb_allocroute() warn: 'vortex_adb_checkinout(vortex, (0), en, 0)' is unsigned sound/pci/au88x0/au88x0_core.c:2170 vortex_adb_allocroute() warn: 'vortex_adb_checkinout(vortex, stream->resources, en, 4)' is unsigned As well, since one function returns errnos, return an `int` rather than a `signed char`. Signed-off-by: Jason A. Donenfeld Cc: Link: https://lore.kernel.org/r/20221024162929.536004-1-Jason@zx2c4.com Signed-off-by: Takashi Iwai --- sound/pci/au88x0/au88x0.h | 6 +++--- sound/pci/au88x0/au88x0_core.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sound/pci/au88x0/au88x0.h b/sound/pci/au88x0/au88x0.h index 0aa7af049b1b9..6cbb2bc4a0483 100644 --- a/sound/pci/au88x0/au88x0.h +++ b/sound/pci/au88x0/au88x0.h @@ -141,7 +141,7 @@ struct snd_vortex { #ifndef CHIP_AU8810 stream_t dma_wt[NR_WT]; wt_voice_t wt_voice[NR_WT]; /* WT register cache. */ - char mixwt[(NR_WT / NR_WTPB) * 6]; /* WT mixin objects */ + s8 mixwt[(NR_WT / NR_WTPB) * 6]; /* WT mixin objects */ #endif /* Global resources */ @@ -235,8 +235,8 @@ static int vortex_alsafmt_aspfmt(snd_pcm_format_t alsafmt, vortex_t *v); static void vortex_connect_default(vortex_t * vortex, int en); static int vortex_adb_allocroute(vortex_t * vortex, int dma, int nr_ch, int dir, int type, int subdev); -static char vortex_adb_checkinout(vortex_t * vortex, int resmap[], int out, - int restype); +static int vortex_adb_checkinout(vortex_t * vortex, int resmap[], int out, + int restype); #ifndef CHIP_AU8810 static int vortex_wt_allocroute(vortex_t * vortex, int dma, int nr_ch); static void vortex_wt_connect(vortex_t * vortex, int en); diff --git a/sound/pci/au88x0/au88x0_core.c b/sound/pci/au88x0/au88x0_core.c index 2ed5100b8caea..f217c02dfdfa4 100644 --- a/sound/pci/au88x0/au88x0_core.c +++ b/sound/pci/au88x0/au88x0_core.c @@ -1998,7 +1998,7 @@ static const int resnum[VORTEX_RESOURCE_LAST] = out: Mean checkout if != 0. Else mean Checkin resource. restype: Indicates type of resource to be checked in or out. */ -static char +static int vortex_adb_checkinout(vortex_t * vortex, int resmap[], int out, int restype) { int i, qty = resnum[restype], resinuse = 0; -- GitLab From 50895a55bcfde8ac6f22a37c6bc8cff506b3c7c6 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 25 Oct 2022 02:03:13 +0200 Subject: [PATCH 0438/1927] ALSA: rme9652: use explicitly signed char With char becoming unsigned by default, and with `char` alone being ambiguous and based on architecture, signed chars need to be marked explicitly as such. This fixes warnings like: sound/pci/rme9652/hdsp.c:3953 hdsp_channel_buffer_location() warn: 'hdsp->channel_map[channel]' is unsigned sound/pci/rme9652/hdsp.c:4153 snd_hdsp_channel_info() warn: impossible condition '(hdsp->channel_map[channel] < 0) => (0-255 < 0)' sound/pci/rme9652/rme9652.c:1833 rme9652_channel_buffer_location() warn: 'rme9652->channel_map[channel]' is unsigned Signed-off-by: Jason A. Donenfeld Cc: Link: https://lore.kernel.org/r/20221025000313.546261-1-Jason@zx2c4.com Signed-off-by: Takashi Iwai --- sound/pci/rme9652/hdsp.c | 26 +++++++++++++------------- sound/pci/rme9652/rme9652.c | 22 +++++++++++----------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/sound/pci/rme9652/hdsp.c b/sound/pci/rme9652/hdsp.c index dcc43a81ae0e8..65add92c88aa6 100644 --- a/sound/pci/rme9652/hdsp.c +++ b/sound/pci/rme9652/hdsp.c @@ -433,7 +433,7 @@ struct hdsp_midi { struct snd_rawmidi *rmidi; struct snd_rawmidi_substream *input; struct snd_rawmidi_substream *output; - char istimer; /* timer in use */ + signed char istimer; /* timer in use */ struct timer_list timer; spinlock_t lock; int pending; @@ -480,7 +480,7 @@ struct hdsp { pid_t playback_pid; int running; int system_sample_rate; - const char *channel_map; + const signed char *channel_map; int dev; int irq; unsigned long port; @@ -502,7 +502,7 @@ struct hdsp { where the data for that channel can be read/written from/to. */ -static const char channel_map_df_ss[HDSP_MAX_CHANNELS] = { +static const signed char channel_map_df_ss[HDSP_MAX_CHANNELS] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 }; @@ -517,7 +517,7 @@ static const char channel_map_mf_ss[HDSP_MAX_CHANNELS] = { /* Multiface */ -1, -1, -1, -1, -1, -1, -1, -1 }; -static const char channel_map_ds[HDSP_MAX_CHANNELS] = { +static const signed char channel_map_ds[HDSP_MAX_CHANNELS] = { /* ADAT channels are remapped */ 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, /* channels 12 and 13 are S/PDIF */ @@ -526,7 +526,7 @@ static const char channel_map_ds[HDSP_MAX_CHANNELS] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; -static const char channel_map_H9632_ss[HDSP_MAX_CHANNELS] = { +static const signed char channel_map_H9632_ss[HDSP_MAX_CHANNELS] = { /* ADAT channels */ 0, 1, 2, 3, 4, 5, 6, 7, /* SPDIF */ @@ -540,7 +540,7 @@ static const char channel_map_H9632_ss[HDSP_MAX_CHANNELS] = { -1, -1 }; -static const char channel_map_H9632_ds[HDSP_MAX_CHANNELS] = { +static const signed char channel_map_H9632_ds[HDSP_MAX_CHANNELS] = { /* ADAT */ 1, 3, 5, 7, /* SPDIF */ @@ -554,7 +554,7 @@ static const char channel_map_H9632_ds[HDSP_MAX_CHANNELS] = { -1, -1, -1, -1, -1, -1 }; -static const char channel_map_H9632_qs[HDSP_MAX_CHANNELS] = { +static const signed char channel_map_H9632_qs[HDSP_MAX_CHANNELS] = { /* ADAT is disabled in this mode */ /* SPDIF */ 8, 9, @@ -3939,7 +3939,7 @@ static snd_pcm_uframes_t snd_hdsp_hw_pointer(struct snd_pcm_substream *substream return hdsp_hw_pointer(hdsp); } -static char *hdsp_channel_buffer_location(struct hdsp *hdsp, +static signed char *hdsp_channel_buffer_location(struct hdsp *hdsp, int stream, int channel) @@ -3964,7 +3964,7 @@ static int snd_hdsp_playback_copy(struct snd_pcm_substream *substream, void __user *src, unsigned long count) { struct hdsp *hdsp = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; if (snd_BUG_ON(pos + count > HDSP_CHANNEL_BUFFER_BYTES)) return -EINVAL; @@ -3982,7 +3982,7 @@ static int snd_hdsp_playback_copy_kernel(struct snd_pcm_substream *substream, void *src, unsigned long count) { struct hdsp *hdsp = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; channel_buf = hdsp_channel_buffer_location(hdsp, substream->pstr->stream, channel); if (snd_BUG_ON(!channel_buf)) @@ -3996,7 +3996,7 @@ static int snd_hdsp_capture_copy(struct snd_pcm_substream *substream, void __user *dst, unsigned long count) { struct hdsp *hdsp = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; if (snd_BUG_ON(pos + count > HDSP_CHANNEL_BUFFER_BYTES)) return -EINVAL; @@ -4014,7 +4014,7 @@ static int snd_hdsp_capture_copy_kernel(struct snd_pcm_substream *substream, void *dst, unsigned long count) { struct hdsp *hdsp = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; channel_buf = hdsp_channel_buffer_location(hdsp, substream->pstr->stream, channel); if (snd_BUG_ON(!channel_buf)) @@ -4028,7 +4028,7 @@ static int snd_hdsp_hw_silence(struct snd_pcm_substream *substream, unsigned long count) { struct hdsp *hdsp = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; channel_buf = hdsp_channel_buffer_location (hdsp, substream->pstr->stream, channel); if (snd_BUG_ON(!channel_buf)) diff --git a/sound/pci/rme9652/rme9652.c b/sound/pci/rme9652/rme9652.c index 1d614fe89a6ae..e7c320afefe86 100644 --- a/sound/pci/rme9652/rme9652.c +++ b/sound/pci/rme9652/rme9652.c @@ -230,7 +230,7 @@ struct snd_rme9652 { int last_spdif_sample_rate; /* so that we can catch externally ... */ int last_adat_sample_rate; /* ... induced rate changes */ - const char *channel_map; + const signed char *channel_map; struct snd_card *card; struct snd_pcm *pcm; @@ -247,12 +247,12 @@ struct snd_rme9652 { where the data for that channel can be read/written from/to. */ -static const char channel_map_9652_ss[26] = { +static const signed char channel_map_9652_ss[26] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 }; -static const char channel_map_9636_ss[26] = { +static const signed char channel_map_9636_ss[26] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* channels 16 and 17 are S/PDIF */ 24, 25, @@ -260,7 +260,7 @@ static const char channel_map_9636_ss[26] = { -1, -1, -1, -1, -1, -1, -1, -1 }; -static const char channel_map_9652_ds[26] = { +static const signed char channel_map_9652_ds[26] = { /* ADAT channels are remapped */ 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, /* channels 12 and 13 are S/PDIF */ @@ -269,7 +269,7 @@ static const char channel_map_9652_ds[26] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }; -static const char channel_map_9636_ds[26] = { +static const signed char channel_map_9636_ds[26] = { /* ADAT channels are remapped */ 1, 3, 5, 7, 9, 11, 13, 15, /* channels 8 and 9 are S/PDIF */ @@ -1819,7 +1819,7 @@ static snd_pcm_uframes_t snd_rme9652_hw_pointer(struct snd_pcm_substream *substr return rme9652_hw_pointer(rme9652); } -static char *rme9652_channel_buffer_location(struct snd_rme9652 *rme9652, +static signed char *rme9652_channel_buffer_location(struct snd_rme9652 *rme9652, int stream, int channel) @@ -1847,7 +1847,7 @@ static int snd_rme9652_playback_copy(struct snd_pcm_substream *substream, void __user *src, unsigned long count) { struct snd_rme9652 *rme9652 = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; if (snd_BUG_ON(pos + count > RME9652_CHANNEL_BUFFER_BYTES)) return -EINVAL; @@ -1867,7 +1867,7 @@ static int snd_rme9652_playback_copy_kernel(struct snd_pcm_substream *substream, void *src, unsigned long count) { struct snd_rme9652 *rme9652 = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; channel_buf = rme9652_channel_buffer_location(rme9652, substream->pstr->stream, @@ -1883,7 +1883,7 @@ static int snd_rme9652_capture_copy(struct snd_pcm_substream *substream, void __user *dst, unsigned long count) { struct snd_rme9652 *rme9652 = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; if (snd_BUG_ON(pos + count > RME9652_CHANNEL_BUFFER_BYTES)) return -EINVAL; @@ -1903,7 +1903,7 @@ static int snd_rme9652_capture_copy_kernel(struct snd_pcm_substream *substream, void *dst, unsigned long count) { struct snd_rme9652 *rme9652 = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; channel_buf = rme9652_channel_buffer_location(rme9652, substream->pstr->stream, @@ -1919,7 +1919,7 @@ static int snd_rme9652_hw_silence(struct snd_pcm_substream *substream, unsigned long count) { struct snd_rme9652 *rme9652 = snd_pcm_substream_chip(substream); - char *channel_buf; + signed char *channel_buf; channel_buf = rme9652_channel_buffer_location (rme9652, substream->pstr->stream, -- GitLab From 5fa9add66b00ad0c796185ff7438eaa3e67c1187 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Sat, 22 Oct 2022 19:46:36 +0200 Subject: [PATCH 0439/1927] nvme-tcp: replace sg_init_marker() with sg_init_table() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In nvme_tcp_ddgst_update(), sg_init_marker() is called with an uninitialized scatterlist. This is probably fine, but gcc complains: CC [M] drivers/nvme/host/tcp.o In file included from ./include/linux/dma-mapping.h:10, from ./include/linux/skbuff.h:31, from ./include/net/net_namespace.h:43, from ./include/linux/netdevice.h:38, from ./include/net/sock.h:46, from drivers/nvme/host/tcp.c:12: In function ‘sg_mark_end’, inlined from ‘sg_init_marker’ at ./include/linux/scatterlist.h:356:2, inlined from ‘nvme_tcp_ddgst_update’ at drivers/nvme/host/tcp.c:390:2: ./include/linux/scatterlist.h:234:11: error: ‘sg.page_link’ is used uninitialized [-Werror=uninitialized] 234 | sg->page_link |= SG_END; | ~~^~~~~~~~~~~ drivers/nvme/host/tcp.c: In function ‘nvme_tcp_ddgst_update’: drivers/nvme/host/tcp.c:388:28: note: ‘sg’ declared here 388 | struct scatterlist sg; | ^~ cc1: all warnings being treated as errors Use sg_init_table() instead, which basically memset the scatterlist to zero first before calling sg_init_marker(). Signed-off-by: Nam Cao Reviewed-by: Sagi Grimberg Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/tcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 1eed0fc26b3ae..dc2def86076d0 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -387,7 +387,7 @@ static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, { struct scatterlist sg; - sg_init_marker(&sg, 1); + sg_init_table(&sg, 1); sg_set_page(&sg, page, len, off); ahash_request_set_crypt(hash, &sg, NULL, len); crypto_ahash_update(hash); -- GitLab From 83e1226b0ee2d7e3fb6e002fbbfc6ab36aabdc35 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Sun, 23 Oct 2022 11:04:43 +0300 Subject: [PATCH 0440/1927] nvme-tcp: fix possible circular locking when deleting a controller under memory pressure When destroying a queue, when calling sock_release, the network stack might need to allocate an skb to send a FIN/RST. When that happens during memory pressure, there is a need to reclaim memory, which in turn may ask the nvme-tcp device to write out dirty pages, however this is not possible due to a ctrl teardown that is going on. Set PF_MEMALLOC to the task that releases the socket to grant access to PF_MEMALLOC reserves. In addition, do the same for the nvme-tcp thread as this may also originate from the swap itself and should be more resilient to memory pressure situations. This fixes the following lockdep complaint: -- ====================================================== WARNING: possible circular locking dependency detected 6.0.0-rc2+ #25 Tainted: G W ------------------------------------------------------ kswapd0/92 is trying to acquire lock: ffff888114003240 (sk_lock-AF_INET-NVME){+.+.}-{0:0}, at: tcp_sendpage+0x23/0xa0 but task is already holding lock: ffffffff97e95ca0 (fs_reclaim){+.+.}-{0:0}, at: balance_pgdat+0x987/0x10d0 which lock already depends on the new lock. the existing dependency chain (in reverse order) is: -> #1 (fs_reclaim){+.+.}-{0:0}: fs_reclaim_acquire+0x11e/0x160 kmem_cache_alloc_node+0x44/0x530 __alloc_skb+0x158/0x230 tcp_send_active_reset+0x7e/0x730 tcp_disconnect+0x1272/0x1ae0 __tcp_close+0x707/0xd90 tcp_close+0x26/0x80 inet_release+0xfa/0x220 sock_release+0x85/0x1a0 nvme_tcp_free_queue+0x1fd/0x470 [nvme_tcp] nvme_do_delete_ctrl+0x130/0x13d [nvme_core] nvme_sysfs_delete.cold+0x8/0xd [nvme_core] kernfs_fop_write_iter+0x356/0x530 vfs_write+0x4e8/0xce0 ksys_write+0xfd/0x1d0 do_syscall_64+0x58/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd -> #0 (sk_lock-AF_INET-NVME){+.+.}-{0:0}: __lock_acquire+0x2a0c/0x5690 lock_acquire+0x18e/0x4f0 lock_sock_nested+0x37/0xc0 tcp_sendpage+0x23/0xa0 inet_sendpage+0xad/0x120 kernel_sendpage+0x156/0x440 nvme_tcp_try_send+0x48a/0x2630 [nvme_tcp] nvme_tcp_queue_rq+0xefb/0x17e0 [nvme_tcp] __blk_mq_try_issue_directly+0x452/0x660 blk_mq_plug_issue_direct.constprop.0+0x207/0x700 blk_mq_flush_plug_list+0x6f5/0xc70 __blk_flush_plug+0x264/0x410 blk_finish_plug+0x4b/0xa0 shrink_lruvec+0x1263/0x1ea0 shrink_node+0x736/0x1a80 balance_pgdat+0x740/0x10d0 kswapd+0x5f2/0xaf0 kthread+0x256/0x2f0 ret_from_fork+0x1f/0x30 other info that might help us debug this: Possible unsafe locking scenario: CPU0 CPU1 ---- ---- lock(fs_reclaim); lock(sk_lock-AF_INET-NVME); lock(fs_reclaim); lock(sk_lock-AF_INET-NVME); *** DEADLOCK *** 3 locks held by kswapd0/92: #0: ffffffff97e95ca0 (fs_reclaim){+.+.}-{0:0}, at: balance_pgdat+0x987/0x10d0 #1: ffff88811f21b0b0 (q->srcu){....}-{0:0}, at: blk_mq_flush_plug_list+0x6b3/0xc70 #2: ffff888170b11470 (&queue->send_mutex){+.+.}-{3:3}, at: nvme_tcp_queue_rq+0xeb9/0x17e0 [nvme_tcp] Fixes: 3f2304f8c6d6 ("nvme-tcp: add NVMe over TCP host driver") Reported-by: Daniel Wagner Signed-off-by: Sagi Grimberg Tested-by: Daniel Wagner Signed-off-by: Christoph Hellwig --- drivers/nvme/host/tcp.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index dc2def86076d0..9b47dcb2a7d97 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -1141,6 +1141,7 @@ static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) { struct nvme_tcp_request *req; + unsigned int noreclaim_flag; int ret = 1; if (!queue->request) { @@ -1150,12 +1151,13 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) } req = queue->request; + noreclaim_flag = memalloc_noreclaim_save(); if (req->state == NVME_TCP_SEND_CMD_PDU) { ret = nvme_tcp_try_send_cmd_pdu(req); if (ret <= 0) goto done; if (!nvme_tcp_has_inline_data(req)) - return ret; + goto out; } if (req->state == NVME_TCP_SEND_H2C_PDU) { @@ -1181,6 +1183,8 @@ static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) nvme_tcp_fail_request(queue->request); nvme_tcp_done_send_req(queue); } +out: + memalloc_noreclaim_restore(noreclaim_flag); return ret; } @@ -1296,6 +1300,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) struct page *page; struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); struct nvme_tcp_queue *queue = &ctrl->queues[qid]; + unsigned int noreclaim_flag; if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) return; @@ -1308,7 +1313,11 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); queue->pf_cache.va = NULL; } + + noreclaim_flag = memalloc_noreclaim_save(); sock_release(queue->sock); + memalloc_noreclaim_restore(noreclaim_flag); + kfree(queue->pdu); mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); -- GitLab From fe8714b04fb137aa62e9a69424c48b5301b721b9 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 24 Oct 2022 11:57:45 -0700 Subject: [PATCH 0441/1927] nvme-multipath: set queue dma alignment to 3 NVMe spec requires all transports support dword aligned addresses, which is already set in the namespace request_queue. Set the same limit in the multipath device's request_queue as well. Signed-off-by: Keith Busch Reviewed-by: Sagi Grimberg Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/host/multipath.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 0ea7e441e080f..93e2138a8b427 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -516,6 +516,7 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) /* set to a default value of 512 until the disk is validated */ blk_queue_logical_block_size(head->disk->queue, 512); blk_set_stacking_limits(&head->disk->queue->limits); + blk_queue_dma_alignment(head->disk->queue, 3); /* we need to propagate up the VMC settings */ if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) -- GitLab From 6853a71726b6f5930b4450889faf02e8f1cfe35c Mon Sep 17 00:00:00 2001 From: Quentin Perret Date: Tue, 25 Oct 2022 14:51:56 +0000 Subject: [PATCH 0442/1927] KVM: arm64: Use correct accessor to parse stage-1 PTEs hyp_get_page_state() is used with pKVM to retrieve metadata about a page by parsing a hypervisor stage-1 PTE. However, it incorrectly uses a helper which parses *stage-2* mappings. Ouch. Luckily, pkvm_getstate() only looks at the software bits, which happen to be in the same place for stage-1 and stage-2 PTEs, and this all ends up working correctly by accident. But clearly, we should do better. Fix hyp_get_page_state() to use the correct helper. Fixes: e82edcc75c4e ("KVM: arm64: Implement do_share() helper for sharing memory") Signed-off-by: Quentin Perret Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20221025145156.855308-1-qperret@google.com --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 1e78acf9662eb..07f9dc9848ef1 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -516,7 +516,7 @@ static enum pkvm_page_state hyp_get_page_state(kvm_pte_t pte) if (!kvm_pte_valid(pte)) return PKVM_NOPAGE; - return pkvm_getstate(kvm_pgtable_stage2_pte_prot(pte)); + return pkvm_getstate(kvm_pgtable_hyp_pte_prot(pte)); } static int __hyp_check_page_state_range(u64 addr, u64 size, -- GitLab From f8bcaf714abfc94818dff8c0db84d750433984f4 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Wed, 12 Oct 2022 15:32:28 +0100 Subject: [PATCH 0443/1927] media: vivid: s_fbuf: add more sanity checks VIDIOC_S_FBUF is by definition a scary ioctl, which is why only root can use it. But at least check if the framebuffer parameters match that of one of the framebuffer created by vivid, and reject anything else. Signed-off-by: Hans Verkuil Fixes: ef834f7836ec ([media] vivid: add the video capture and output parts) Signed-off-by: Mauro Carvalho Chehab --- drivers/media/test-drivers/vivid/vivid-core.c | 22 +++++++++++++++++++ drivers/media/test-drivers/vivid/vivid-core.h | 2 ++ .../media/test-drivers/vivid/vivid-vid-cap.c | 9 +++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c index 04b75666bad4d..61d48fbc3d15d 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.c +++ b/drivers/media/test-drivers/vivid/vivid-core.c @@ -339,6 +339,28 @@ static int vidioc_g_fbuf(struct file *file, void *fh, struct v4l2_framebuffer *a return vivid_vid_out_g_fbuf(file, fh, a); } +/* + * Only support the framebuffer of one of the vivid instances. + * Anything else is rejected. + */ +bool vivid_validate_fb(const struct v4l2_framebuffer *a) +{ + struct vivid_dev *dev; + int i; + + for (i = 0; i < n_devs; i++) { + dev = vivid_devs[i]; + if (!dev || !dev->video_pbase) + continue; + if ((unsigned long)a->base == dev->video_pbase && + a->fmt.width <= dev->display_width && + a->fmt.height <= dev->display_height && + a->fmt.bytesperline <= dev->display_byte_stride) + return true; + } + return false; +} + static int vidioc_s_fbuf(struct file *file, void *fh, const struct v4l2_framebuffer *a) { struct video_device *vdev = video_devdata(file); diff --git a/drivers/media/test-drivers/vivid/vivid-core.h b/drivers/media/test-drivers/vivid/vivid-core.h index bfcfb35159013..473f3598db5af 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.h +++ b/drivers/media/test-drivers/vivid/vivid-core.h @@ -613,4 +613,6 @@ static inline bool vivid_is_hdmi_out(const struct vivid_dev *dev) return dev->output_type[dev->output] == HDMI; } +bool vivid_validate_fb(const struct v4l2_framebuffer *a); + #endif diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index 86b158eeb2d81..e3e78b5bd2271 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -1276,7 +1276,14 @@ int vivid_vid_cap_s_fbuf(struct file *file, void *fh, return -EINVAL; if (a->fmt.bytesperline < (a->fmt.width * fmt->bit_depth[0]) / 8) return -EINVAL; - if (a->fmt.height * a->fmt.bytesperline < a->fmt.sizeimage) + if (a->fmt.bytesperline > a->fmt.sizeimage / a->fmt.height) + return -EINVAL; + + /* + * Only support the framebuffer of one of the vivid instances. + * Anything else is rejected. + */ + if (!vivid_validate_fb(a)) return -EINVAL; dev->fb_vbase_cap = phys_to_virt((unsigned long)a->base); -- GitLab From 1f65ea411cc7b6ff128d82a3493d7b5648054e6f Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Thu, 13 Oct 2022 15:18:46 +0100 Subject: [PATCH 0444/1927] media: vivid: dev->bitmap_cap wasn't freed in all cases Whenever the compose width/height values change, the dev->bitmap_cap vmalloc'ed array must be freed and dev->bitmap_cap set to NULL. This was done in some places, but not all. This is only an issue if overlay support is enabled and the bitmap clipping is used. Signed-off-by: Hans Verkuil Fixes: ef834f7836ec ([media] vivid: add the video capture and output parts) Signed-off-by: Mauro Carvalho Chehab --- .../media/test-drivers/vivid/vivid-vid-cap.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index e3e78b5bd2271..d52d24b61d347 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -453,6 +453,12 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) tpg_reset_source(&dev->tpg, dev->src_rect.width, dev->src_rect.height, dev->field_cap); dev->crop_cap = dev->src_rect; dev->crop_bounds_cap = dev->src_rect; + if (dev->bitmap_cap && + (dev->compose_cap.width != dev->crop_cap.width || + dev->compose_cap.height != dev->crop_cap.height)) { + vfree(dev->bitmap_cap); + dev->bitmap_cap = NULL; + } dev->compose_cap = dev->crop_cap; if (V4L2_FIELD_HAS_T_OR_B(dev->field_cap)) dev->compose_cap.height /= 2; @@ -913,6 +919,8 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection struct vivid_dev *dev = video_drvdata(file); struct v4l2_rect *crop = &dev->crop_cap; struct v4l2_rect *compose = &dev->compose_cap; + unsigned orig_compose_w = compose->width; + unsigned orig_compose_h = compose->height; unsigned factor = V4L2_FIELD_HAS_T_OR_B(dev->field_cap) ? 2 : 1; int ret; @@ -1029,17 +1037,17 @@ int vivid_vid_cap_s_selection(struct file *file, void *fh, struct v4l2_selection s->r.height /= factor; } v4l2_rect_map_inside(&s->r, &dev->fmt_cap_rect); - if (dev->bitmap_cap && (compose->width != s->r.width || - compose->height != s->r.height)) { - vfree(dev->bitmap_cap); - dev->bitmap_cap = NULL; - } *compose = s->r; break; default: return -EINVAL; } + if (dev->bitmap_cap && (compose->width != orig_compose_w || + compose->height != orig_compose_h)) { + vfree(dev->bitmap_cap); + dev->bitmap_cap = NULL; + } tpg_s_crop_compose(&dev->tpg, crop, compose); return 0; } -- GitLab From 4b6d66a45ed34a15721cb9e11492fa1a24bc83df Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Thu, 13 Oct 2022 09:00:34 +0100 Subject: [PATCH 0445/1927] media: v4l2-dv-timings: add sanity checks for blanking values Add sanity checks to v4l2_valid_dv_timings() to ensure that the provided blanking values are reasonable. Signed-off-by: Hans Verkuil Fixes: b18787ed1ce3 ([media] v4l2-dv-timings: add new helper module) Signed-off-by: Mauro Carvalho Chehab --- drivers/media/v4l2-core/v4l2-dv-timings.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/media/v4l2-core/v4l2-dv-timings.c b/drivers/media/v4l2-core/v4l2-dv-timings.c index af48705c704f8..003c32fed3f75 100644 --- a/drivers/media/v4l2-core/v4l2-dv-timings.c +++ b/drivers/media/v4l2-core/v4l2-dv-timings.c @@ -161,6 +161,20 @@ bool v4l2_valid_dv_timings(const struct v4l2_dv_timings *t, (bt->interlaced && !(caps & V4L2_DV_BT_CAP_INTERLACED)) || (!bt->interlaced && !(caps & V4L2_DV_BT_CAP_PROGRESSIVE))) return false; + + /* sanity checks for the blanking timings */ + if (!bt->interlaced && + (bt->il_vbackporch || bt->il_vsync || bt->il_vfrontporch)) + return false; + if (bt->hfrontporch > 2 * bt->width || + bt->hsync > 1024 || bt->hbackporch > 1024) + return false; + if (bt->vfrontporch > 4096 || + bt->vsync > 128 || bt->vbackporch > 4096) + return false; + if (bt->interlaced && (bt->il_vfrontporch > 4096 || + bt->il_vsync > 128 || bt->il_vbackporch > 4096)) + return false; return fnc == NULL || fnc(t, fnc_handle); } EXPORT_SYMBOL_GPL(v4l2_valid_dv_timings); -- GitLab From 8da7f0976b9071b528c545008de9d10cc81883b1 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Wed, 12 Oct 2022 16:46:17 +0100 Subject: [PATCH 0446/1927] media: videodev2.h: V4L2_DV_BT_BLANKING_HEIGHT should check 'interlaced' If it is a progressive (non-interlaced) format, then ignore the interlaced timing values. Signed-off-by: Hans Verkuil Fixes: 7f68127fa11f ([media] videodev2.h: defines to calculate blanking and frame sizes) Signed-off-by: Mauro Carvalho Chehab --- include/uapi/linux/videodev2.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index 86cae23cc4469..29da1f4b4578e 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -1601,7 +1601,8 @@ struct v4l2_bt_timings { ((bt)->width + V4L2_DV_BT_BLANKING_WIDTH(bt)) #define V4L2_DV_BT_BLANKING_HEIGHT(bt) \ ((bt)->vfrontporch + (bt)->vsync + (bt)->vbackporch + \ - (bt)->il_vfrontporch + (bt)->il_vsync + (bt)->il_vbackporch) + ((bt)->interlaced ? \ + ((bt)->il_vfrontporch + (bt)->il_vsync + (bt)->il_vbackporch) : 0)) #define V4L2_DV_BT_FRAME_HEIGHT(bt) \ ((bt)->height + V4L2_DV_BT_BLANKING_HEIGHT(bt)) -- GitLab From eb1d969203eb8212741751f88dcf5cb56bb11830 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Fri, 21 Oct 2022 13:21:25 +0100 Subject: [PATCH 0447/1927] media: vivid: fix control handler mutex deadlock vivid_update_format_cap() can be called from an s_ctrl callback. In that case (keep_controls == true) no control framework functions can be called that take the control handler mutex. The new call to v4l2_ctrl_modify_dimensions() did exactly that. Signed-off-by: Hans Verkuil Fixes: 6bc7643d1b9c (media: vivid: add pixel_array test control) Signed-off-by: Mauro Carvalho Chehab --- drivers/media/test-drivers/vivid/vivid-vid-cap.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index d52d24b61d347..11620eaf941e3 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -466,6 +466,14 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) tpg_s_video_aspect(&dev->tpg, vivid_get_video_aspect(dev)); tpg_s_pixel_aspect(&dev->tpg, vivid_get_pixel_aspect(dev)); tpg_update_mv_step(&dev->tpg); + + /* + * We can be called from within s_ctrl, in that case we can't + * modify controls. Luckily we don't need to in that case. + */ + if (keep_controls) + return; + dims[0] = roundup(dev->src_rect.width, PIXEL_ARRAY_DIV); dims[1] = roundup(dev->src_rect.height, PIXEL_ARRAY_DIV); v4l2_ctrl_modify_dimensions(dev->pixel_array, dims); -- GitLab From 957148e22870e4021082dddd12f8619a49d9bb50 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Wed, 12 Oct 2022 15:31:51 +0100 Subject: [PATCH 0448/1927] media: vivid: drop GFP_DMA32 >From what I can see, this is not needed. And since using it issues a 'deprecated' warning, just drop it. Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab --- drivers/media/test-drivers/vivid/vivid-osd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/media/test-drivers/vivid/vivid-osd.c b/drivers/media/test-drivers/vivid/vivid-osd.c index fbaec8acc161e..ec25edc679b39 100644 --- a/drivers/media/test-drivers/vivid/vivid-osd.c +++ b/drivers/media/test-drivers/vivid/vivid-osd.c @@ -357,7 +357,7 @@ int vivid_fb_init(struct vivid_dev *dev) int ret; dev->video_buffer_size = MAX_OSD_HEIGHT * MAX_OSD_WIDTH * 2; - dev->video_vbase = kzalloc(dev->video_buffer_size, GFP_KERNEL | GFP_DMA32); + dev->video_vbase = kzalloc(dev->video_buffer_size, GFP_KERNEL); if (dev->video_vbase == NULL) return -ENOMEM; dev->video_pbase = virt_to_phys(dev->video_vbase); -- GitLab From 69d78a80da4ef12faf2a6f9cfa2097ab4ac43983 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Mon, 17 Oct 2022 15:09:06 +0100 Subject: [PATCH 0449/1927] media: vivid: set num_in/outputs to 0 if not supported If node_types does not have video/vbi/meta inputs or outputs, then set num_inputs/num_outputs to 0 instead of 1. Signed-off-by: Hans Verkuil Fixes: 0c90f649d2f5 (media: vivid: add vivid_create_queue() helper) Signed-off-by: Mauro Carvalho Chehab --- drivers/media/test-drivers/vivid/vivid-core.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c index 61d48fbc3d15d..f28440e6c9f89 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.c +++ b/drivers/media/test-drivers/vivid/vivid-core.c @@ -942,8 +942,12 @@ static int vivid_detect_feature_set(struct vivid_dev *dev, int inst, /* how many inputs do we have and of what type? */ dev->num_inputs = num_inputs[inst]; - if (dev->num_inputs < 1) - dev->num_inputs = 1; + if (node_type & 0x20007) { + if (dev->num_inputs < 1) + dev->num_inputs = 1; + } else { + dev->num_inputs = 0; + } if (dev->num_inputs >= MAX_INPUTS) dev->num_inputs = MAX_INPUTS; for (i = 0; i < dev->num_inputs; i++) { @@ -960,8 +964,12 @@ static int vivid_detect_feature_set(struct vivid_dev *dev, int inst, /* how many outputs do we have and of what type? */ dev->num_outputs = num_outputs[inst]; - if (dev->num_outputs < 1) - dev->num_outputs = 1; + if (node_type & 0x40300) { + if (dev->num_outputs < 1) + dev->num_outputs = 1; + } else { + dev->num_outputs = 0; + } if (dev->num_outputs >= MAX_OUTPUTS) dev->num_outputs = MAX_OUTPUTS; for (i = 0; i < dev->num_outputs; i++) { -- GitLab From de547896aac606a00435a219757a940ece142bf0 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Mon, 17 Oct 2022 15:15:05 +0100 Subject: [PATCH 0450/1927] media: vivid.rst: loop_video is set on the capture devnode The example on how to use and test Capture Overlay specified the wrong video device node. Back in 2015 the loop_video control moved from the output device to the capture device, but this example code is still referring to the output video device. Signed-off-by: Hans Verkuil Signed-off-by: Mauro Carvalho Chehab --- Documentation/admin-guide/media/vivid.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/admin-guide/media/vivid.rst b/Documentation/admin-guide/media/vivid.rst index 4f680dc9661cb..abd90ed310907 100644 --- a/Documentation/admin-guide/media/vivid.rst +++ b/Documentation/admin-guide/media/vivid.rst @@ -1318,7 +1318,7 @@ instance. This setup would require the following commands: $ v4l2-ctl -d2 -i2 $ v4l2-ctl -d2 -c horizontal_movement=4 $ v4l2-ctl -d1 --overlay=1 - $ v4l2-ctl -d1 -c loop_video=1 + $ v4l2-ctl -d0 -c loop_video=1 $ v4l2-ctl -d2 --stream-mmap --overlay=1 And from another console: -- GitLab From 9bfb09774ea40937b6c6d6e07858e0f7ad1991ec Mon Sep 17 00:00:00 2001 From: Todd Brandt Date: Thu, 20 Oct 2022 02:23:10 -0700 Subject: [PATCH 0451/1927] pm-graph v5.10 sleepgraph: - add -wifitrace argument for tracing all the way to wifi reconnect - include more data in ftrace to mark the end of kernel resume - add async_synchronize_full to the list of funcs to chart - add thermal zone info to the log data - include a check for s0ix support (s2idle is the default mem_sleep) - if s2idle does not support s0ix, remove the SYS%LPI turbostat var - fix -dev crash when kprobe caller is just an address (not a symbol) - fix the cpuexec data in -proc to display in resume sleepgraph.8: - add -wifitrace documentation README: - change links from 01.org to developer.intel.com Signed-off-by: Todd Brandt Signed-off-by: Rafael J. Wysocki --- tools/power/pm-graph/README | 12 +- tools/power/pm-graph/sleepgraph.8 | 3 + tools/power/pm-graph/sleepgraph.py | 225 ++++++++++++++--------------- 3 files changed, 118 insertions(+), 122 deletions(-) diff --git a/tools/power/pm-graph/README b/tools/power/pm-graph/README index e6020c0d59ec3..3213dbe63b74f 100644 --- a/tools/power/pm-graph/README +++ b/tools/power/pm-graph/README @@ -6,22 +6,22 @@ |_| |___/ |_| pm-graph: suspend/resume/boot timing analysis tools - Version: 5.9 + Version: 5.10 Author: Todd Brandt - Home Page: https://01.org/pm-graph + Home Page: https://www.intel.com/content/www/us/en/developer/topic-technology/open/pm-graph/overview.html Report bugs/issues at bugzilla.kernel.org Tools/pm-graph - https://bugzilla.kernel.org/buglist.cgi?component=pm-graph&product=Tools Full documentation available online & in man pages - Getting Started: - https://01.org/pm-graph/documentation/getting-started + https://www.intel.com/content/www/us/en/developer/articles/technical/usage.html - - Config File Format: - https://01.org/pm-graph/documentation/3-config-file-format + - Feature Summary: + https://www.intel.com/content/www/us/en/developer/topic-technology/open/pm-graph/features.html - upstream version in git: - https://github.com/intel/pm-graph/ + git clone https://github.com/intel/pm-graph/ Table of Contents - Overview diff --git a/tools/power/pm-graph/sleepgraph.8 b/tools/power/pm-graph/sleepgraph.8 index 5126271de98a6..643271b6fc6f0 100644 --- a/tools/power/pm-graph/sleepgraph.8 +++ b/tools/power/pm-graph/sleepgraph.8 @@ -78,6 +78,9 @@ This helps maintain the consistency of test data for better comparison. If a wifi connection is available, check that it reconnects after resume. Include the reconnect time in the total resume time calculation and treat wifi timeouts as resume failures. +.TP +\fB-wifitrace\fR +Trace through the wifi reconnect time and include it in the timeline. .SS "advanced" .TP diff --git a/tools/power/pm-graph/sleepgraph.py b/tools/power/pm-graph/sleepgraph.py index 33981adcdd687..cfe343306e08c 100755 --- a/tools/power/pm-graph/sleepgraph.py +++ b/tools/power/pm-graph/sleepgraph.py @@ -86,7 +86,7 @@ def ascii(text): # store system values and test parameters class SystemValues: title = 'SleepGraph' - version = '5.9' + version = '5.10' ansi = False rs = 0 display = '' @@ -100,6 +100,7 @@ class SystemValues: ftracelog = False acpidebug = True tstat = True + wifitrace = False mindevlen = 0.0001 mincglen = 0.0 cgphase = '' @@ -124,6 +125,7 @@ class SystemValues: epath = '/sys/kernel/debug/tracing/events/power/' pmdpath = '/sys/power/pm_debug_messages' s0ixpath = '/sys/module/intel_pmc_core/parameters/warn_on_s0ix_failures' + s0ixres = '/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us' acpipath='/sys/module/acpi/parameters/debug_level' traceevents = [ 'suspend_resume', @@ -180,6 +182,7 @@ class SystemValues: tmstart = 'SUSPEND START %Y%m%d-%H:%M:%S.%f' tmend = 'RESUME COMPLETE %Y%m%d-%H:%M:%S.%f' tracefuncs = { + 'async_synchronize_full': {}, 'sys_sync': {}, 'ksys_sync': {}, '__pm_notifier_call_chain': {}, @@ -304,6 +307,7 @@ class SystemValues: [2, 'suspendstats', 'sh', '-c', 'grep -v invalid /sys/power/suspend_stats/*'], [2, 'cpuidle', 'sh', '-c', 'grep -v invalid /sys/devices/system/cpu/cpu*/cpuidle/state*/s2idle/*'], [2, 'battery', 'sh', '-c', 'grep -v invalid /sys/class/power_supply/*/*'], + [2, 'thermal', 'sh', '-c', 'grep . /sys/class/thermal/thermal_zone*/temp'], ] cgblacklist = [] kprobes = dict() @@ -777,7 +781,7 @@ class SystemValues: return if not quiet: sysvals.printSystemInfo(False) - pprint('INITIALIZING FTRACE...') + pprint('INITIALIZING FTRACE') # turn trace off self.fsetVal('0', 'tracing_on') self.cleanupFtrace() @@ -841,7 +845,7 @@ class SystemValues: for name in self.dev_tracefuncs: self.defaultKprobe(name, self.dev_tracefuncs[name]) if not quiet: - pprint('INITIALIZING KPROBES...') + pprint('INITIALIZING KPROBES') self.addKprobes(self.verbose) if(self.usetraceevents): # turn trace events on @@ -1133,6 +1137,15 @@ class SystemValues: self.cfgdef[file] = fp.read().strip() fp.write(value) fp.close() + def s0ixSupport(self): + if not os.path.exists(self.s0ixres) or not os.path.exists(self.mempowerfile): + return False + fp = open(sysvals.mempowerfile, 'r') + data = fp.read().strip() + fp.close() + if '[s2idle]' in data: + return True + return False def haveTurbostat(self): if not self.tstat: return False @@ -1146,7 +1159,7 @@ class SystemValues: self.vprint(out) return True return False - def turbostat(self): + def turbostat(self, s0ixready): cmd = self.getExec('turbostat') rawout = keyline = valline = '' fullcmd = '%s -q -S echo freeze > %s' % (cmd, self.powerfile) @@ -1173,6 +1186,8 @@ class SystemValues: for key in keyline: idx = keyline.index(key) val = valline[idx] + if key == 'SYS%LPI' and not s0ixready and re.match('^[0\.]*$', val): + continue out.append('%s=%s' % (key, val)) return '|'.join(out) def netfixon(self, net='both'): @@ -1183,14 +1198,6 @@ class SystemValues: out = ascii(fp.read()).strip() fp.close() return out - def wifiRepair(self): - out = self.netfixon('wifi') - if not out or 'error' in out.lower(): - return '' - m = re.match('WIFI \S* ONLINE (?P\S*)', out) - if not m: - return 'dead' - return m.group('action') def wifiDetails(self, dev): try: info = open('/sys/class/net/%s/device/uevent' % dev, 'r').read().strip() @@ -1220,11 +1227,6 @@ class SystemValues: return '%s reconnected %.2f' % \ (self.wifiDetails(dev), max(0, time.time() - start)) time.sleep(0.01) - if self.netfix: - res = self.wifiRepair() - if res: - timeout = max(0, time.time() - start) - return '%s %s %d' % (self.wifiDetails(dev), res, timeout) return '%s timeout %d' % (self.wifiDetails(dev), timeout) def errorSummary(self, errinfo, msg): found = False @@ -1346,6 +1348,20 @@ class SystemValues: for i in self.rslist: self.setVal(self.rstgt, i) pprint('runtime suspend settings restored on %d devices' % len(self.rslist)) + def start(self, pm): + if self.useftrace: + self.dlog('start ftrace tracing') + self.fsetVal('1', 'tracing_on') + if self.useprocmon: + self.dlog('start the process monitor') + pm.start() + def stop(self, pm): + if self.useftrace: + if self.useprocmon: + self.dlog('stop the process monitor') + pm.stop() + self.dlog('stop ftrace tracing') + self.fsetVal('0', 'tracing_on') sysvals = SystemValues() switchvalues = ['enable', 'disable', 'on', 'off', 'true', 'false', '1', '0'] @@ -1643,19 +1659,20 @@ class Data: ubiquitous = False if kprobename in dtf and 'ub' in dtf[kprobename]: ubiquitous = True - title = cdata+' '+rdata - mstr = '\(.*\) *(?P.*) *\((?P.*)\+.* arg1=(?P.*)' - m = re.match(mstr, title) - if m: - c = m.group('caller') - a = m.group('args').strip() - r = m.group('ret') + mc = re.match('\(.*\) *(?P.*)', cdata) + mr = re.match('\((?P\S*).* arg1=(?P.*)', rdata) + if mc and mr: + c = mr.group('caller').split('+')[0] + a = mc.group('args').strip() + r = mr.group('ret') if len(r) > 6: r = '' else: r = 'ret=%s ' % r if ubiquitous and c in dtf and 'ub' in dtf[c]: return False + else: + return False color = sysvals.kprobeColor(kprobename) e = DevFunction(displayname, a, c, r, start, end, ubiquitous, proc, pid, color) tgtdev['src'].append(e) @@ -1772,6 +1789,14 @@ class Data: e.time = self.trimTimeVal(e.time, t0, dT, left) e.end = self.trimTimeVal(e.end, t0, dT, left) e.length = e.end - e.time + if('cpuexec' in d): + cpuexec = dict() + for e in d['cpuexec']: + c0, cN = e + c0 = self.trimTimeVal(c0, t0, dT, left) + cN = self.trimTimeVal(cN, t0, dT, left) + cpuexec[(c0, cN)] = d['cpuexec'][e] + d['cpuexec'] = cpuexec for dir in ['suspend', 'resume']: list = [] for e in self.errorinfo[dir]: @@ -2086,75 +2111,43 @@ class Data: return d def addProcessUsageEvent(self, name, times): # get the start and end times for this process - maxC = 0 - tlast = 0 - start = -1 - end = -1 + cpuexec = dict() + tlast = start = end = -1 for t in sorted(times): - if tlast == 0: + if tlast < 0: tlast = t continue - if name in self.pstl[t]: - if start == -1 or tlast < start: + if name in self.pstl[t] and self.pstl[t][name] > 0: + if start < 0: start = tlast - if end == -1 or t > end: - end = t + end, key = t, (tlast, t) + maxj = (t - tlast) * 1024.0 + cpuexec[key] = min(1.0, float(self.pstl[t][name]) / maxj) tlast = t - if start == -1 or end == -1: - return 0 + if start < 0 or end < 0: + return # add a new action for this process and get the object out = self.newActionGlobal(name, start, end, -3) - if not out: - return 0 - phase, devname = out - dev = self.dmesg[phase]['list'][devname] - # get the cpu exec data - tlast = 0 - clast = 0 - cpuexec = dict() - for t in sorted(times): - if tlast == 0 or t <= start or t > end: - tlast = t - continue - list = self.pstl[t] - c = 0 - if name in list: - c = list[name] - if c > maxC: - maxC = c - if c != clast: - key = (tlast, t) - cpuexec[key] = c - tlast = t - clast = c - dev['cpuexec'] = cpuexec - return maxC + if out: + phase, devname = out + dev = self.dmesg[phase]['list'][devname] + dev['cpuexec'] = cpuexec def createProcessUsageEvents(self): - # get an array of process names - proclist = [] - for t in sorted(self.pstl): - pslist = self.pstl[t] - for ps in sorted(pslist): - if ps not in proclist: - proclist.append(ps) - # get a list of data points for suspend and resume - tsus = [] - tres = [] + # get an array of process names and times + proclist = {'sus': dict(), 'res': dict()} + tdata = {'sus': [], 'res': []} for t in sorted(self.pstl): - if t < self.tSuspended: - tsus.append(t) - else: - tres.append(t) + dir = 'sus' if t < self.tSuspended else 'res' + for ps in sorted(self.pstl[t]): + if ps not in proclist[dir]: + proclist[dir][ps] = 0 + tdata[dir].append(t) # process the events for suspend and resume - if len(proclist) > 0: + if len(proclist['sus']) > 0 or len(proclist['res']) > 0: sysvals.vprint('Process Execution:') - for ps in proclist: - c = self.addProcessUsageEvent(ps, tsus) - if c > 0: - sysvals.vprint('%25s (sus): %d' % (ps, c)) - c = self.addProcessUsageEvent(ps, tres) - if c > 0: - sysvals.vprint('%25s (res): %d' % (ps, c)) + for dir in ['sus', 'res']: + for ps in sorted(proclist[dir]): + self.addProcessUsageEvent(ps, tdata[dir]) def handleEndMarker(self, time, msg=''): dm = self.dmesg self.setEnd(time, msg) @@ -3218,7 +3211,7 @@ class ProcessMonitor: # markers, and/or kprobes required for primary parsing. def doesTraceLogHaveTraceEvents(): kpcheck = ['_cal: (', '_ret: ('] - techeck = ['suspend_resume', 'device_pm_callback'] + techeck = ['suspend_resume', 'device_pm_callback', 'tracing_mark_write'] tmcheck = ['SUSPEND START', 'RESUME COMPLETE'] sysvals.usekprobes = False fp = sysvals.openlog(sysvals.ftracefile, 'r') @@ -3241,7 +3234,7 @@ def doesTraceLogHaveTraceEvents(): check.remove(i) tmcheck = check fp.close() - sysvals.usetraceevents = True if len(techeck) < 2 else False + sysvals.usetraceevents = True if len(techeck) < 3 else False sysvals.usetracemarkers = True if len(tmcheck) == 0 else False # Function: appendIncompleteTraceLog @@ -3456,6 +3449,8 @@ def parseTraceLog(live=False): continue # process cpu exec line if t.type == 'tracing_mark_write': + if t.name == 'CMD COMPLETE' and data.tKernRes == 0: + data.tKernRes = t.time m = re.match(tp.procexecfmt, t.name) if(m): parts, msg = 1, m.group('ps') @@ -3674,6 +3669,9 @@ def parseTraceLog(live=False): e = next((x for x in reversed(tp.ktemp[key]) if x['end'] < 0), 0) if not e: continue + if (t.time - e['begin']) * 1000 < sysvals.mindevlen: + tp.ktemp[key].pop() + continue e['end'] = t.time e['rdata'] = kprobedata # end of kernel resume @@ -4213,6 +4211,8 @@ def callgraphHTML(sv, hf, num, cg, title, color, devid): fmt = '(%.3f ms @ '+sv.timeformat+')' flen = fmt % (line.length*1000, line.time) if line.isLeaf(): + if line.length * 1000 < sv.mincglen: + continue hf.write(html_func_leaf.format(line.name, flen)) elif line.freturn: hf.write(html_func_end) @@ -4827,14 +4827,11 @@ def createHTML(testruns, testfail): if('cpuexec' in dev): for t in sorted(dev['cpuexec']): start, end = t - j = float(dev['cpuexec'][t]) / 5 - if j > 1.0: - j = 1.0 height = '%.3f' % (rowheight/3) top = '%.3f' % (rowtop + devtl.scaleH + 2*rowheight/3) left = '%f' % (((start-m0)*100)/mTotal) width = '%f' % ((end-start)*100/mTotal) - color = 'rgba(255, 0, 0, %f)' % j + color = 'rgba(255, 0, 0, %f)' % dev['cpuexec'][t] devtl.html += \ html_cpuexec.format(left, top, height, width, color) if('src' not in dev): @@ -5453,17 +5450,9 @@ def executeSuspend(quiet=False): call('sync', shell=True) sv.dlog('read dmesg') sv.initdmesg() - # start ftrace - if sv.useftrace: - if not quiet: - pprint('START TRACING') - sv.dlog('start ftrace tracing') - sv.fsetVal('1', 'tracing_on') - if sv.useprocmon: - sv.dlog('start the process monitor') - pm.start() - sv.dlog('run the cmdinfo list before') + sv.dlog('cmdinfo before') sv.cmdinfo(True) + sv.start(pm) # execute however many s/r runs requested for count in range(1,sv.execcount+1): # x2delay in between test runs @@ -5500,6 +5489,7 @@ def executeSuspend(quiet=False): if res != 0: tdata['error'] = 'cmd returned %d' % res else: + s0ixready = sv.s0ixSupport() mode = sv.suspendmode if sv.memmode and os.path.exists(sv.mempowerfile): mode = 'mem' @@ -5509,9 +5499,10 @@ def executeSuspend(quiet=False): sv.testVal(sv.diskpowerfile, 'radio', sv.diskmode) if sv.acpidebug: sv.testVal(sv.acpipath, 'acpi', '0xe') - if mode == 'freeze' and sv.haveTurbostat(): + if ((mode == 'freeze') or (sv.memmode == 's2idle')) \ + and sv.haveTurbostat(): # execution will pause here - turbo = sv.turbostat() + turbo = sv.turbostat(s0ixready) if turbo: tdata['turbo'] = turbo else: @@ -5522,7 +5513,8 @@ def executeSuspend(quiet=False): pf.close() except Exception as e: tdata['error'] = str(e) - sv.dlog('system returned from resume') + sv.fsetVal('CMD COMPLETE', 'trace_marker') + sv.dlog('system returned') # reset everything sv.testVal('restoreall') if(sv.rtcwake): @@ -5535,33 +5527,29 @@ def executeSuspend(quiet=False): sv.fsetVal('WAIT END', 'trace_marker') # return from suspend pprint('RESUME COMPLETE') - sv.fsetVal(datetime.now().strftime(sv.tmend), 'trace_marker') + if(count < sv.execcount): + sv.fsetVal(datetime.now().strftime(sv.tmend), 'trace_marker') + elif(not sv.wifitrace): + sv.fsetVal(datetime.now().strftime(sv.tmend), 'trace_marker') + sv.stop(pm) if sv.wifi and wifi: tdata['wifi'] = sv.pollWifi(wifi) sv.dlog('wifi check, %s' % tdata['wifi']) - if sv.netfix: - netfixout = sv.netfixon('wired') - elif sv.netfix: - netfixout = sv.netfixon() - if sv.netfix and netfixout: - tdata['netfix'] = netfixout + if(count == sv.execcount and sv.wifitrace): + sv.fsetVal(datetime.now().strftime(sv.tmend), 'trace_marker') + sv.stop(pm) + if sv.netfix: + tdata['netfix'] = sv.netfixon() sv.dlog('netfix, %s' % tdata['netfix']) if(sv.suspendmode == 'mem' or sv.suspendmode == 'command'): sv.dlog('read the ACPI FPDT') tdata['fw'] = getFPDT(False) testdata.append(tdata) - sv.dlog('run the cmdinfo list after') + sv.dlog('cmdinfo after') cmdafter = sv.cmdinfo(False) - # stop ftrace - if sv.useftrace: - if sv.useprocmon: - sv.dlog('stop the process monitor') - pm.stop() - sv.fsetVal('0', 'tracing_on') # grab a copy of the dmesg output if not quiet: pprint('CAPTURING DMESG') - sysvals.dlog('EXECUTION TRACE END') sv.getdmesg(testdata) # grab a copy of the ftrace output if sv.useftrace: @@ -6350,6 +6338,8 @@ def data_from_html(file, outpath, issues, fulldetail=False): if not m: continue name, time, phase = m.group('n'), m.group('t'), m.group('p') + if name == 'async_synchronize_full': + continue if ' async' in name or ' sync' in name: name = ' '.join(name.split(' ')[:-1]) if phase.startswith('suspend'): @@ -6701,6 +6691,7 @@ def printHelp(): ' -skiphtml Run the test and capture the trace logs, but skip the timeline (default: disabled)\n'\ ' -result fn Export a results table to a text file for parsing.\n'\ ' -wifi If a wifi connection is available, check that it reconnects after resume.\n'\ + ' -wifitrace Trace kernel execution through wifi reconnect.\n'\ ' -netfix Use netfix to reset the network in the event it fails to resume.\n'\ ' [testprep]\n'\ ' -sync Sync the filesystems before starting the test\n'\ @@ -6828,6 +6819,8 @@ if __name__ == '__main__': sysvals.sync = True elif(arg == '-wifi'): sysvals.wifi = True + elif(arg == '-wifitrace'): + sysvals.wifitrace = True elif(arg == '-netfix'): sysvals.netfix = True elif(arg == '-gzip'): -- GitLab From 19905240aef0181d1e6944070eb85fce75f75bcd Mon Sep 17 00:00:00 2001 From: Michael Grzeschik Date: Tue, 18 Oct 2022 00:11:41 +0200 Subject: [PATCH 0452/1927] usb: gadget: uvc: limit isoc_sg to super speed gadgets The overhead of preparing sg data is high for transfers with limited payload. When transferring isoc over high-speed usb the maximum payload is rather small which is a good argument no to use sg. This patch is changing the uvc_video_encode_isoc_sg encode function only to be used for super speed gadgets. Signed-off-by: Michael Grzeschik Cc: stable Link: https://lore.kernel.org/r/20221017221141.3134818-1-m.grzeschik@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/function/uvc_queue.c | 9 +++------ drivers/usb/gadget/function/uvc_video.c | 9 +++++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index 0aa3d7e1f3cc3..782eaae931407 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -84,12 +84,9 @@ static int uvc_buffer_prepare(struct vb2_buffer *vb) return -ENODEV; buf->state = UVC_BUF_STATE_QUEUED; - if (queue->use_sg) { - buf->sgt = vb2_dma_sg_plane_desc(vb, 0); - buf->sg = buf->sgt->sgl; - } else { - buf->mem = vb2_plane_vaddr(vb, 0); - } + buf->sgt = vb2_dma_sg_plane_desc(vb, 0); + buf->sg = buf->sgt->sgl; + buf->mem = vb2_plane_vaddr(vb, 0); buf->length = vb2_plane_size(vb, 0); if (vb->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) buf->bytesused = 0; diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index dd1c6b2ca7c6f..b6ea600b01118 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -459,6 +459,9 @@ static void uvcg_video_pump(struct work_struct *work) */ int uvcg_video_enable(struct uvc_video *video, int enable) { + struct uvc_device *uvc = video->uvc; + struct usb_composite_dev *cdev = uvc->func.config->cdev; + struct usb_gadget *gadget = cdev->gadget; unsigned int i; int ret; @@ -490,9 +493,11 @@ int uvcg_video_enable(struct uvc_video *video, int enable) if (video->max_payload_size) { video->encode = uvc_video_encode_bulk; video->payload_size = 0; - } else - video->encode = video->queue.use_sg ? + } else { + video->encode = (video->queue.use_sg && + !(gadget->speed <= USB_SPEED_HIGH)) ? uvc_video_encode_isoc_sg : uvc_video_encode_isoc; + } video->req_int_count = 0; -- GitLab From 3f53c329b31d53b2a2e7992819242fc0d4f883e0 Mon Sep 17 00:00:00 2001 From: Patrice Chotard Date: Fri, 30 Sep 2022 16:20:18 +0200 Subject: [PATCH 0453/1927] usb: dwc3: st: Rely on child's compatible instead of name To ensure that child node is found, don't rely on child's node name which can take different value, but on child's compatible name. Fixes: f5c5936d6b4d ("usb: dwc3: st: Fix node's child name") Cc: stable Cc: Jerome Audu Reported-by: Felipe Balbi Signed-off-by: Patrice Chotard Link: https://lore.kernel.org/r/20220930142018.890535-1-patrice.chotard@foss.st.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/dwc3-st.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/dwc3-st.c b/drivers/usb/dwc3/dwc3-st.c index 6c14a79279f9a..fea5290de83fb 100644 --- a/drivers/usb/dwc3/dwc3-st.c +++ b/drivers/usb/dwc3/dwc3-st.c @@ -251,7 +251,7 @@ static int st_dwc3_probe(struct platform_device *pdev) /* Manage SoftReset */ reset_control_deassert(dwc3_data->rstc_rst); - child = of_get_child_by_name(node, "usb"); + child = of_get_compatible_child(node, "snps,dwc3"); if (!child) { dev_err(&pdev->dev, "failed to find dwc3 core node\n"); ret = -ENODEV; -- GitLab From 6746eae4bbaddcc16b40efb33dab79210828b3ce Mon Sep 17 00:00:00 2001 From: James Clark Date: Tue, 25 Oct 2022 14:10:32 +0100 Subject: [PATCH 0454/1927] coresight: cti: Fix hang in cti_disable_hw() cti_enable_hw() and cti_disable_hw() are called from an atomic context so shouldn't use runtime PM because it can result in a sleep when communicating with firmware. Since commit 3c6656337852 ("Revert "firmware: arm_scmi: Add clock management to the SCMI power domain""), this causes a hang on Juno when running the Perf Coresight tests or running this command: perf record -e cs_etm//u -- ls This was also missed until the revert commit because pm_runtime_put() was called with the wrong device until commit 692c9a499b28 ("coresight: cti: Correct the parameter for pm_runtime_put") With lock and scheduler debugging enabled the following is output: coresight cti_sys0: cti_enable_hw -- dev:cti_sys0 parent: 20020000.cti BUG: sleeping function called from invalid context at drivers/base/power/runtime.c:1151 in_atomic(): 1, irqs_disabled(): 128, non_block: 0, pid: 330, name: perf-exec preempt_count: 2, expected: 0 RCU nest depth: 0, expected: 0 INFO: lockdep is turned off. irq event stamp: 0 hardirqs last enabled at (0): [<0000000000000000>] 0x0 hardirqs last disabled at (0): [] copy_process+0xa0c/0x1948 softirqs last enabled at (0): [] copy_process+0xa0c/0x1948 softirqs last disabled at (0): [<0000000000000000>] 0x0 CPU: 3 PID: 330 Comm: perf-exec Not tainted 6.0.0-00053-g042116d99298 #7 Hardware name: ARM LTD ARM Juno Development Platform/ARM Juno Development Platform, BIOS EDK II Sep 13 2022 Call trace: dump_backtrace+0x134/0x140 show_stack+0x20/0x58 dump_stack_lvl+0x8c/0xb8 dump_stack+0x18/0x34 __might_resched+0x180/0x228 __might_sleep+0x50/0x88 __pm_runtime_resume+0xac/0xb0 cti_enable+0x44/0x120 coresight_control_assoc_ectdev+0xc0/0x150 coresight_enable_path+0xb4/0x288 etm_event_start+0x138/0x170 etm_event_add+0x48/0x70 event_sched_in.isra.122+0xb4/0x280 merge_sched_in+0x1fc/0x3d0 visit_groups_merge.constprop.137+0x16c/0x4b0 ctx_sched_in+0x114/0x1f0 perf_event_sched_in+0x60/0x90 ctx_resched+0x68/0xb0 perf_event_exec+0x138/0x508 begin_new_exec+0x52c/0xd40 load_elf_binary+0x6b8/0x17d0 bprm_execve+0x360/0x7f8 do_execveat_common.isra.47+0x218/0x238 __arm64_sys_execve+0x48/0x60 invoke_syscall+0x4c/0x110 el0_svc_common.constprop.4+0xfc/0x120 do_el0_svc+0x34/0xc0 el0_svc+0x40/0x98 el0t_64_sync_handler+0x98/0xc0 el0t_64_sync+0x170/0x174 Fix the issue by removing the runtime PM calls completely. They are not needed here because it must have already been done when building the path for a trace. Fixes: 835d722ba10a ("coresight: cti: Initial CoreSight CTI Driver") Cc: stable Reported-by: Aishwarya TCV Reported-by: Cristian Marussi Suggested-by: Suzuki K Poulose Signed-off-by: James Clark Reviewed-by: Mike Leach Tested-by: Mike Leach [ Fix build warnings ] Signed-off-by: Suzuki K Poulose Link: https://lore.kernel.org/r/20221025131032.1149459-1-suzuki.poulose@arm.com Signed-off-by: Greg Kroah-Hartman --- drivers/hwtracing/coresight/coresight-cti-core.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c index 1be92342b5b98..c6e8c6542f24b 100644 --- a/drivers/hwtracing/coresight/coresight-cti-core.c +++ b/drivers/hwtracing/coresight/coresight-cti-core.c @@ -90,11 +90,9 @@ void cti_write_all_hw_regs(struct cti_drvdata *drvdata) static int cti_enable_hw(struct cti_drvdata *drvdata) { struct cti_config *config = &drvdata->config; - struct device *dev = &drvdata->csdev->dev; unsigned long flags; int rc = 0; - pm_runtime_get_sync(dev->parent); spin_lock_irqsave(&drvdata->spinlock, flags); /* no need to do anything if enabled or unpowered*/ @@ -119,7 +117,6 @@ static int cti_enable_hw(struct cti_drvdata *drvdata) /* cannot enable due to error */ cti_err_not_enabled: spin_unlock_irqrestore(&drvdata->spinlock, flags); - pm_runtime_put(dev->parent); return rc; } @@ -153,7 +150,6 @@ static void cti_cpuhp_enable_hw(struct cti_drvdata *drvdata) static int cti_disable_hw(struct cti_drvdata *drvdata) { struct cti_config *config = &drvdata->config; - struct device *dev = &drvdata->csdev->dev; struct coresight_device *csdev = drvdata->csdev; spin_lock(&drvdata->spinlock); @@ -175,7 +171,6 @@ static int cti_disable_hw(struct cti_drvdata *drvdata) coresight_disclaim_device_unlocked(csdev); CS_LOCK(drvdata->base); spin_unlock(&drvdata->spinlock); - pm_runtime_put(dev->parent); return 0; /* not disabled this call */ -- GitLab From 677047383296ea25fdfc001be3cdcdf5cc874be2 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 25 Oct 2022 04:52:23 +0200 Subject: [PATCH 0455/1927] misc: sgi-gru: use explicitly signed char With char becoming unsigned by default, and with `char` alone being ambiguous and based on architecture, signed chars need to be marked explicitly as such. This fixes warnings like: drivers/misc/sgi-gru/grumain.c:711 gru_check_chiplet_assignment() warn: 'gts->ts_user_chiplet_id' is unsigned Cc: Dimitri Sivanich Cc: Arnd Bergmann Cc: Greg Kroah-Hartman Signed-off-by: Jason A. Donenfeld Link: https://lore.kernel.org/r/20221025025223.573543-1-Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/sgi-gru/grumain.c | 6 +++--- drivers/misc/sgi-gru/grutables.h | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 9afda47efbf2e..6706ef3c59776 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -152,7 +152,7 @@ static int gru_assign_asid(struct gru_state *gru) * Optionally, build an array of chars that contain the bit numbers allocated. */ static unsigned long reserve_resources(unsigned long *p, int n, int mmax, - char *idx) + signed char *idx) { unsigned long bits = 0; int i; @@ -170,14 +170,14 @@ static unsigned long reserve_resources(unsigned long *p, int n, int mmax, } unsigned long gru_reserve_cb_resources(struct gru_state *gru, int cbr_au_count, - char *cbmap) + signed char *cbmap) { return reserve_resources(&gru->gs_cbr_map, cbr_au_count, GRU_CBR_AU, cbmap); } unsigned long gru_reserve_ds_resources(struct gru_state *gru, int dsr_au_count, - char *dsmap) + signed char *dsmap) { return reserve_resources(&gru->gs_dsr_map, dsr_au_count, GRU_DSR_AU, dsmap); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 5efc869fe59a0..8c52776db2341 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -351,7 +351,7 @@ struct gru_thread_state { pid_t ts_tgid_owner; /* task that is using the context - for migration */ short ts_user_blade_id;/* user selected blade */ - char ts_user_chiplet_id;/* user selected chiplet */ + signed char ts_user_chiplet_id;/* user selected chiplet */ unsigned short ts_sizeavail; /* Pagesizes in use */ int ts_tsid; /* thread that owns the structure */ @@ -364,11 +364,11 @@ struct gru_thread_state { required for contest */ unsigned char ts_cbr_au_count;/* Number of CBR resources required for contest */ - char ts_cch_req_slice;/* CCH packet slice */ - char ts_blade; /* If >= 0, migrate context if + signed char ts_cch_req_slice;/* CCH packet slice */ + signed char ts_blade; /* If >= 0, migrate context if ref from different blade */ - char ts_force_cch_reload; - char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each + signed char ts_force_cch_reload; + signed char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each allocated CB */ int ts_data_valid; /* Indicates if ts_gdata has valid data */ @@ -643,9 +643,9 @@ extern struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, int cbr_au_count, int dsr_au_count, unsigned char tlb_preload_count, int options, int tsid); extern unsigned long gru_reserve_cb_resources(struct gru_state *gru, - int cbr_au_count, char *cbmap); + int cbr_au_count, signed char *cbmap); extern unsigned long gru_reserve_ds_resources(struct gru_state *gru, - int dsr_au_count, char *dsmap); + int dsr_au_count, signed char *dsmap); extern vm_fault_t gru_fault(struct vm_fault *vmf); extern struct gru_mm_struct *gru_register_mmu_notifier(void); extern void gru_drop_mmu_notifier(struct gru_mm_struct *gms); -- GitLab From 937ec9f7d5f2625d60077bb7824fee35dc447c6e Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 25 Oct 2022 14:21:50 +0200 Subject: [PATCH 0456/1927] staging: rtl8192e: remove bogus ssid character sign test This error triggers on some architectures with unsigned `char` types: drivers/staging/rtl8192e/rtllib_softmac_wx.c:459 rtllib_wx_set_essid() warn: impossible condition '(extra[i] < 0) => (0-255 < 0)' But actually, the entire test is bogus, as ssids don't have any sign validity rules like that. So just remove this check look all together. Cc: Greg Kroah-Hartman Cc: linux-staging@lists.linux.dev Signed-off-by: Jason A. Donenfeld Reviewed-by: Dan Carpenter Link: https://lore.kernel.org/r/20221025122150.583617-1-Jason@zx2c4.com Signed-off-by: Greg Kroah-Hartman --- drivers/staging/rtl8192e/rtllib_softmac_wx.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/staging/rtl8192e/rtllib_softmac_wx.c b/drivers/staging/rtl8192e/rtllib_softmac_wx.c index f9589c5b62bac..1e5ad3b476ef7 100644 --- a/drivers/staging/rtl8192e/rtllib_softmac_wx.c +++ b/drivers/staging/rtl8192e/rtllib_softmac_wx.c @@ -439,7 +439,7 @@ int rtllib_wx_set_essid(struct rtllib_device *ieee, union iwreq_data *wrqu, char *extra) { - int ret = 0, len, i; + int ret = 0, len; short proto_started; unsigned long flags; @@ -455,13 +455,6 @@ int rtllib_wx_set_essid(struct rtllib_device *ieee, goto out; } - for (i = 0; i < len; i++) { - if (extra[i] < 0) { - ret = -1; - goto out; - } - } - if (proto_started) rtllib_stop_protocol(ieee, true); -- GitLab From 4f547472380136718b56064ea5689a61e135f904 Mon Sep 17 00:00:00 2001 From: Jens Glathe Date: Mon, 24 Oct 2022 17:27:17 +0300 Subject: [PATCH 0457/1927] usb: xhci: add XHCI_SPURIOUS_SUCCESS to ASM1042 despite being a V0.96 controller This appears to fix the error: "xhci_hcd
; ERROR Transfer event TRB DMA ptr not part of current TD ep_index 2 comp_code 13" that appear spuriously (or pretty often) when using a r8152 USB3 ethernet adapter with integrated hub. ASM1042 reports as a 0.96 controller, but appears to behave more like 1.0 Inspired by this email thread: https://markmail.org/thread/7vzqbe7t6du6qsw3 Cc: stable@vger.kernel.org Signed-off-by: Jens Glathe Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20221024142720.4122053-2-mathias.nyman@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/xhci-pci.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 40228a3d77a0b..6dd3102749b70 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -306,8 +306,14 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) } if (pdev->vendor == PCI_VENDOR_ID_ASMEDIA && - pdev->device == PCI_DEVICE_ID_ASMEDIA_1042_XHCI) + pdev->device == PCI_DEVICE_ID_ASMEDIA_1042_XHCI) { + /* + * try to tame the ASMedia 1042 controller which reports 0.96 + * but appears to behave more like 1.0 + */ + xhci->quirks |= XHCI_SPURIOUS_SUCCESS; xhci->quirks |= XHCI_BROKEN_STREAMS; + } if (pdev->vendor == PCI_VENDOR_ID_ASMEDIA && pdev->device == PCI_DEVICE_ID_ASMEDIA_1042A_XHCI) { xhci->quirks |= XHCI_TRUST_TX_LENGTH; -- GitLab From 34cd2db408d591bc15771cbcc90939ade0a99a21 Mon Sep 17 00:00:00 2001 From: Mathias Nyman Date: Mon, 24 Oct 2022 17:27:18 +0300 Subject: [PATCH 0458/1927] xhci: Add quirk to reset host back to default state at shutdown Systems based on Alder Lake P see significant boot time delay if boot firmware tries to control usb ports in unexpected link states. This is seen with self-powered usb devices that survive in U3 link suspended state over S5. A more generic solution to power off ports at shutdown was attempted in commit 83810f84ecf1 ("xhci: turn off port power in shutdown") but it caused regression. Add host specific XHCI_RESET_TO_DEFAULT quirk which will reset host and ports back to default state in shutdown. Cc: stable@vger.kernel.org Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20221024142720.4122053-3-mathias.nyman@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/xhci-pci.c | 4 ++++ drivers/usb/host/xhci.c | 10 ++++++++-- drivers/usb/host/xhci.h | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 6dd3102749b70..fbbd547ba12a7 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -257,6 +257,10 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == PCI_DEVICE_ID_INTEL_DNV_XHCI)) xhci->quirks |= XHCI_MISSING_CAS; + if (pdev->vendor == PCI_VENDOR_ID_INTEL && + pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI) + xhci->quirks |= XHCI_RESET_TO_DEFAULT; + if (pdev->vendor == PCI_VENDOR_ID_INTEL && (pdev->device == PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_XHCI || diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 5176765c40131..79d7931c048a8 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -810,9 +810,15 @@ void xhci_shutdown(struct usb_hcd *hcd) spin_lock_irq(&xhci->lock); xhci_halt(xhci); - /* Workaround for spurious wakeups at shutdown with HSW */ - if (xhci->quirks & XHCI_SPURIOUS_WAKEUP) + + /* + * Workaround for spurious wakeps at shutdown with HSW, and for boot + * firmware delay in ADL-P PCH if port are left in U3 at shutdown + */ + if (xhci->quirks & XHCI_SPURIOUS_WAKEUP || + xhci->quirks & XHCI_RESET_TO_DEFAULT) xhci_reset(xhci, XHCI_RESET_SHORT_USEC); + spin_unlock_irq(&xhci->lock); xhci_cleanup_msix(xhci); diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index c0964fe8ac12f..cc084d9505cdf 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -1897,6 +1897,7 @@ struct xhci_hcd { #define XHCI_BROKEN_D3COLD BIT_ULL(41) #define XHCI_EP_CTX_BROKEN_DCS BIT_ULL(42) #define XHCI_SUSPEND_RESUME_CLKS BIT_ULL(43) +#define XHCI_RESET_TO_DEFAULT BIT_ULL(44) unsigned int num_active_eps; unsigned int limit_active_eps; -- GitLab From a611bf473d1f77b70f7188b5577542cb39b4701b Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 24 Oct 2022 17:27:19 +0300 Subject: [PATCH 0459/1927] xhci-pci: Set runtime PM as default policy on all xHC 1.2 or later devices For optimal power consumption of USB4 routers the XHCI PCIe endpoint used for tunneling must be in D3. Historically this is accomplished by a long list of PCIe IDs that correspond to these endpoints because the xhci_hcd driver will not default to allowing runtime PM for all devices. As both AMD and Intel have released new products with new XHCI controllers this list continues to grow. In reviewing the XHCI specification v1.2 on page 607 there is already a requirement that the PCI power management states D3hot and D3cold must be supported. In the quirk list, use this to indicate that runtime PM should be allowed on XHCI controllers. The following controllers are known to be xHC 1.2 and dropped explicitly: * AMD Yellow Carp * Intel Alder Lake * Intel Meteor Lake * Intel Raptor Lake [keep PCI ID for Alder Lake PCH for recently added quirk -Mathias] Cc: stable@vger.kernel.org Suggested-by: Mathias Nyman Link: https://www.intel.com/content/dam/www/public/us/en/documents/technical-specifications/extensible-host-controler-interface-usb-xhci.pdf Signed-off-by: Mario Limonciello Reviewed-by: Mika Westerberg Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20221024142720.4122053-4-mathias.nyman@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/xhci-pci.c | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index fbbd547ba12a7..7bccbe50bab15 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -58,25 +58,13 @@ #define PCI_DEVICE_ID_INTEL_CML_XHCI 0xa3af #define PCI_DEVICE_ID_INTEL_TIGER_LAKE_XHCI 0x9a13 #define PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI 0x1138 -#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI 0x461e -#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_N_XHCI 0x464e -#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI 0x51ed -#define PCI_DEVICE_ID_INTEL_RAPTOR_LAKE_XHCI 0xa71e -#define PCI_DEVICE_ID_INTEL_METEOR_LAKE_XHCI 0x7ec0 +#define PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI 0x51ed #define PCI_DEVICE_ID_AMD_RENOIR_XHCI 0x1639 #define PCI_DEVICE_ID_AMD_PROMONTORYA_4 0x43b9 #define PCI_DEVICE_ID_AMD_PROMONTORYA_3 0x43ba #define PCI_DEVICE_ID_AMD_PROMONTORYA_2 0x43bb #define PCI_DEVICE_ID_AMD_PROMONTORYA_1 0x43bc -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_1 0x161a -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_2 0x161b -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_3 0x161d -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_4 0x161e -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_5 0x15d6 -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_6 0x15d7 -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_7 0x161c -#define PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_8 0x161f #define PCI_DEVICE_ID_ASMEDIA_1042_XHCI 0x1042 #define PCI_DEVICE_ID_ASMEDIA_1042A_XHCI 0x1142 @@ -272,12 +260,7 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == PCI_DEVICE_ID_INTEL_TITAN_RIDGE_DD_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_ICE_LAKE_XHCI || pdev->device == PCI_DEVICE_ID_INTEL_TIGER_LAKE_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_N_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_ALDER_LAKE_PCH_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_RAPTOR_LAKE_XHCI || - pdev->device == PCI_DEVICE_ID_INTEL_METEOR_LAKE_XHCI)) + pdev->device == PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_XHCI)) xhci->quirks |= XHCI_DEFAULT_PM_RUNTIME_ALLOW; if (pdev->vendor == PCI_VENDOR_ID_ETRON && @@ -346,15 +329,8 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) pdev->device == PCI_DEVICE_ID_AMD_PROMONTORYA_4)) xhci->quirks |= XHCI_NO_SOFT_RETRY; - if (pdev->vendor == PCI_VENDOR_ID_AMD && - (pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_1 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_2 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_3 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_4 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_5 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_6 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_7 || - pdev->device == PCI_DEVICE_ID_AMD_YELLOW_CARP_XHCI_8)) + /* xHC spec requires PCI devices to support D3hot and D3cold */ + if (xhci->hci_version >= 0x120) xhci->quirks |= XHCI_DEFAULT_PM_RUNTIME_ALLOW; if (xhci->quirks & XHCI_RESET_ON_RESUME) -- GitLab From 5aed5b7c2430ce318a8e62f752f181e66f0d1053 Mon Sep 17 00:00:00 2001 From: Mathias Nyman Date: Mon, 24 Oct 2022 17:27:20 +0300 Subject: [PATCH 0460/1927] xhci: Remove device endpoints from bandwidth list when freeing the device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Endpoints are normally deleted from the bandwidth list when they are dropped, before the virt device is freed. If xHC host is dying or being removed then the endpoints aren't dropped cleanly due to functions returning early to avoid interacting with a non-accessible host controller. So check and delete endpoints that are still on the bandwidth list when freeing the virt device. Solves a list_del corruption kernel crash when unbinding xhci-pci, caused by xhci_mem_cleanup() when it later tried to delete already freed endpoints from the bandwidth list. This only affects hosts that use software bandwidth checking, which currenty is only the xHC in intel Panther Point PCH (Ivy Bridge) Cc: stable@vger.kernel.org Reported-by: Marek Marczykowski-Górecki Tested-by: Marek Marczykowski-Górecki Signed-off-by: Mathias Nyman Link: https://lore.kernel.org/r/20221024142720.4122053-5-mathias.nyman@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/xhci-mem.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index 9e56aa28efcd4..81ca2bc1f0bef 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -889,15 +889,19 @@ void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) if (dev->eps[i].stream_info) xhci_free_stream_info(xhci, dev->eps[i].stream_info); - /* Endpoints on the TT/root port lists should have been removed - * when usb_disable_device() was called for the device. - * We can't drop them anyway, because the udev might have gone - * away by this point, and we can't tell what speed it was. + /* + * Endpoints are normally deleted from the bandwidth list when + * endpoints are dropped, before device is freed. + * If host is dying or being removed then endpoints aren't + * dropped cleanly, so delete the endpoint from list here. + * Only applicable for hosts with software bandwidth checking. */ - if (!list_empty(&dev->eps[i].bw_endpoint_list)) - xhci_warn(xhci, "Slot %u endpoint %u " - "not removed from BW list!\n", - slot_id, i); + + if (!list_empty(&dev->eps[i].bw_endpoint_list)) { + list_del_init(&dev->eps[i].bw_endpoint_list); + xhci_dbg(xhci, "Slot %u endpoint %u not removed from BW list!\n", + slot_id, i); + } } /* If this is a hub, free the TT(s) from the TT list */ xhci_free_tt_info(xhci, dev, slot_id); -- GitLab From 92e10465acaffcf65e803f40e884ffa86fd3ff2f Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Thu, 6 Oct 2022 15:38:07 +0300 Subject: [PATCH 0461/1927] device property: Fix documentation for *_match_string() APIs The returned value on success is an index of the matching string, starting from 0. Reflect this in the documentation. Fixes: 3f5c8d318785 ("device property: Add fwnode_property_match_string()") Signed-off-by: Andy Shevchenko Reviewed-by: Sakari Ailus Signed-off-by: Rafael J. Wysocki --- drivers/base/property.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/base/property.c b/drivers/base/property.c index 4d6278a848686..2a5a37fcd9987 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -229,7 +229,7 @@ EXPORT_SYMBOL_GPL(device_property_read_string); * Find a given string in a string array and if it is found return the * index back. * - * Return: %0 if the property was found (success), + * Return: index, starting from %0, if the property was found (success), * %-EINVAL if given arguments are not valid, * %-ENODATA if the property does not have a value, * %-EPROTO if the property is not an array of strings, @@ -450,7 +450,7 @@ EXPORT_SYMBOL_GPL(fwnode_property_read_string); * Find a given string in a string array and if it is found return the * index back. * - * Return: %0 if the property was found (success), + * Return: index, starting from %0, if the property was found (success), * %-EINVAL if given arguments are not valid, * %-ENODATA if the property does not have a value, * %-EPROTO if the property is not an array of strings, -- GitLab From 88864611940ae7dd5a9d40667287c4c9fa455140 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 8 Sep 2021 16:09:08 -0300 Subject: [PATCH 0462/1927] tools headers UAPI: Sync powerpc syscall tables with the kernel sources To pick the changes in these csets: e237506238352f3b ("powerpc/32: fix syscall wrappers with 64-bit arguments of unaligned register-pairs") That doesn't cause any changes in the perf tools. As a reminder, this table is used in tools perf to allow features such as: [root@five ~]# perf trace -e set_mempolicy_home_node ^C[root@five ~]# [root@five ~]# perf trace -v -e set_mempolicy_home_node Using CPUID AuthenticAMD-25-21-0 event qualifier tracepoint filter: (common_pid != 253729 && common_pid != 3585) && (id == 450) mmap size 528384B ^C[root@five ~] [root@five ~]# perf trace -v -e set* --max-events 5 Using CPUID AuthenticAMD-25-21-0 event qualifier tracepoint filter: (common_pid != 253734 && common_pid != 3585) && (id == 38 || id == 54 || id == 105 || id == 106 || id == 109 || id == 112 || id == 113 || id == 114 || id == 116 || id == 117 || id == 119 || id == 122 || id == 123 || id == 141 || id == 160 || id == 164 || id == 170 || id == 171 || id == 188 || id == 205 || id == 218 || id == 238 || id == 273 || id == 308 || id == 450) mmap size 528384B 0.000 ( 0.008 ms): bash/253735 setpgid(pid: 253735 (bash), pgid: 253735 (bash)) = 0 6849.011 ( 0.008 ms): bash/16046 setpgid(pid: 253736 (bash), pgid: 253736 (bash)) = 0 6849.080 ( 0.005 ms): bash/253736 setpgid(pid: 253736 (bash), pgid: 253736 (bash)) = 0 7437.718 ( 0.009 ms): gnome-shell/253737 set_robust_list(head: 0x7f34b527e920, len: 24) = 0 13445.986 ( 0.010 ms): bash/16046 setpgid(pid: 253738 (bash), pgid: 253738 (bash)) = 0 [root@five ~]# That is the filter expression attached to the raw_syscalls:sys_{enter,exit} tracepoints. $ find tools/perf/arch/ -name "syscall*tbl" | xargs grep -w set_mempolicy_home_node tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl:450 common set_mempolicy_home_node sys_set_mempolicy_home_node tools/perf/arch/powerpc/entry/syscalls/syscall.tbl:450 nospu set_mempolicy_home_node sys_set_mempolicy_home_node tools/perf/arch/s390/entry/syscalls/syscall.tbl:450 common set_mempolicy_home_node sys_set_mempolicy_home_node sys_set_mempolicy_home_node tools/perf/arch/x86/entry/syscalls/syscall_64.tbl:450 common set_mempolicy_home_node sys_set_mempolicy_home_node $ $ grep -w set_mempolicy_home_node /tmp/build/perf/arch/x86/include/generated/asm/syscalls_64.c [450] = "set_mempolicy_home_node", $ This addresses these perf build warnings: Warning: Kernel ABI header at 'tools/include/uapi/asm-generic/unistd.h' differs from latest version at 'include/uapi/asm-generic/unistd.h' diff -u tools/include/uapi/asm-generic/unistd.h include/uapi/asm-generic/unistd.h Warning: Kernel ABI header at 'tools/perf/arch/x86/entry/syscalls/syscall_64.tbl' differs from latest version at 'arch/x86/entry/syscalls/syscall_64.tbl' diff -u tools/perf/arch/x86/entry/syscalls/syscall_64.tbl arch/x86/entry/syscalls/syscall_64.tbl Warning: Kernel ABI header at 'tools/perf/arch/powerpc/entry/syscalls/syscall.tbl' differs from latest version at 'arch/powerpc/kernel/syscalls/syscall.tbl' diff -u tools/perf/arch/powerpc/entry/syscalls/syscall.tbl arch/powerpc/kernel/syscalls/syscall.tbl Warning: Kernel ABI header at 'tools/perf/arch/s390/entry/syscalls/syscall.tbl' differs from latest version at 'arch/s390/kernel/syscalls/syscall.tbl' diff -u tools/perf/arch/s390/entry/syscalls/syscall.tbl arch/s390/kernel/syscalls/syscall.tbl Warning: Kernel ABI header at 'tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl' differs from latest version at 'arch/mips/kernel/syscalls/syscall_n64.tbl' diff -u tools/perf/arch/mips/entry/syscalls/syscall_n64.tbl arch/mips/kernel/syscalls/syscall_n64.tbl Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Cc: Michael Ellerman Cc: Nicholas Piggin Link: https://lore.kernel.org/lkml/Y01HN2DGkWz8tC%2FJ@kernel.org/ Signed-off-by: Arnaldo Carvalho de Melo --- .../perf/arch/powerpc/entry/syscalls/syscall.tbl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl index 2bca64f96164a..e9e0df4f9a61a 100644 --- a/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl +++ b/tools/perf/arch/powerpc/entry/syscalls/syscall.tbl @@ -228,8 +228,10 @@ 176 64 rt_sigtimedwait sys_rt_sigtimedwait 177 nospu rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo 178 nospu rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend -179 common pread64 sys_pread64 compat_sys_ppc_pread64 -180 common pwrite64 sys_pwrite64 compat_sys_ppc_pwrite64 +179 32 pread64 sys_ppc_pread64 compat_sys_ppc_pread64 +179 64 pread64 sys_pread64 +180 32 pwrite64 sys_ppc_pwrite64 compat_sys_ppc_pwrite64 +180 64 pwrite64 sys_pwrite64 181 common chown sys_chown 182 common getcwd sys_getcwd 183 common capget sys_capget @@ -242,10 +244,11 @@ 188 common putpmsg sys_ni_syscall 189 nospu vfork sys_vfork 190 common ugetrlimit sys_getrlimit compat_sys_getrlimit -191 common readahead sys_readahead compat_sys_ppc_readahead +191 32 readahead sys_ppc_readahead compat_sys_ppc_readahead +191 64 readahead sys_readahead 192 32 mmap2 sys_mmap2 compat_sys_mmap2 -193 32 truncate64 sys_truncate64 compat_sys_ppc_truncate64 -194 32 ftruncate64 sys_ftruncate64 compat_sys_ppc_ftruncate64 +193 32 truncate64 sys_ppc_truncate64 compat_sys_ppc_truncate64 +194 32 ftruncate64 sys_ppc_ftruncate64 compat_sys_ppc_ftruncate64 195 32 stat64 sys_stat64 196 32 lstat64 sys_lstat64 197 32 fstat64 sys_fstat64 @@ -288,7 +291,8 @@ 230 common io_submit sys_io_submit compat_sys_io_submit 231 common io_cancel sys_io_cancel 232 nospu set_tid_address sys_set_tid_address -233 common fadvise64 sys_fadvise64 compat_sys_ppc32_fadvise64 +233 32 fadvise64 sys_ppc32_fadvise64 compat_sys_ppc32_fadvise64 +233 64 fadvise64 sys_fadvise64 234 nospu exit_group sys_exit_group 235 nospu lookup_dcookie sys_lookup_dcookie compat_sys_lookup_dcookie 236 common epoll_create sys_epoll_create -- GitLab From 231e61bc2e87486270686f7bc3c43c4fb3b0e0b9 Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Mon, 17 Oct 2022 12:35:49 +0300 Subject: [PATCH 0463/1927] perf docs: Fix man page build wrt perf-arm-coresight.txt perf build assumes documentation files starting with "perf-" are man pages but perf-arm-coresight.txt is not a man page: asciidoc: ERROR: perf-arm-coresight.txt: line 2: malformed manpage title asciidoc: ERROR: perf-arm-coresight.txt: line 3: name section expected asciidoc: FAILED: perf-arm-coresight.txt: line 3: section title expected make[3]: *** [Makefile:266: perf-arm-coresight.xml] Error 1 make[3]: *** Waiting for unfinished jobs.... make[2]: *** [Makefile.perf:895: man] Error 2 Fix by renaming it. Fixes: dc2e0fb00bb2b24f ("perf test coresight: Add relevant documentation about ARM64 CoreSight testing") Reported-by: Christian Borntraeger Reported-by: Sven Schnelle Reviewed-by: Leo Yan Signed-off-by: Adrian Hunter Cc: Alexander Shishkin Cc: Carsten Haitzler Cc: coresight@lists.linaro.org Cc: Ingo Molnar Cc: James Clark Cc: Jiri Olsa Cc: Mark Rutland Cc: Mathieu Poirier Cc: Mike Leach Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Suzuki Poulouse Link: https://lore.kernel.org/r/a176a3e1-6ddc-bb63-e41c-15cda8c2d5d2@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- .../Documentation/{perf-arm-coresight.txt => arm-coresight.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/perf/Documentation/{perf-arm-coresight.txt => arm-coresight.txt} (100%) diff --git a/tools/perf/Documentation/perf-arm-coresight.txt b/tools/perf/Documentation/arm-coresight.txt similarity index 100% rename from tools/perf/Documentation/perf-arm-coresight.txt rename to tools/perf/Documentation/arm-coresight.txt -- GitLab From b92dd11725a7c57f55e148c7d3ce58a86f480575 Mon Sep 17 00:00:00 2001 From: Kajol Jain Date: Fri, 14 Oct 2022 19:32:20 +0530 Subject: [PATCH 0464/1927] perf vendor events power10: Fix hv-24x7 metric events Testcase stat_all_metrics.sh fails in powerpc: 90: perf all metrics test : FAILED! The testcase "stat_all_metrics.sh" verifies perf stat result for all the metric events present in perf list. It runs perf metric events with various commands and expects non-empty metric result. Incase of powerpc:hv-24x7 events, some of the event count can be 0 based on system configuration. And if that event used as denominator in divide equation, it can cause divide by 0 error. The current nest_metric.json file creating divide by 0 issue for some of the metric events, which results in failure of the "stat_all_metrics.sh" test case. Most of the metrics events have cycles or an event which expect to have a larger value as denominator, so adding 1 to the denominator of the metric expression as a fix. Result in powerpc box after this patch changes: 90: perf all metrics test : Ok Fixes: a3cbcadfdfc330c2 ("perf vendor events power10: Adds 24x7 nest metric events for power10 platform") Signed-off-by: Kajol Jain Reviewed-by: Madhavan Srinivasan Cc: Athira Jajeev Cc: Disha Goel Cc: Jiri Olsa Cc: Kajol Jain Cc: linuxppc-dev@lists.ozlabs.org Cc: Michael Ellerman Cc: Nageswara R Sastry Link: https://lore.kernel.org/r/20221014140220.122251-1-kjain@linux.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../arch/powerpc/power10/nest_metrics.json | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tools/perf/pmu-events/arch/powerpc/power10/nest_metrics.json b/tools/perf/pmu-events/arch/powerpc/power10/nest_metrics.json index 8ba3e81c9808b..fe050d44374ba 100644 --- a/tools/perf/pmu-events/arch/powerpc/power10/nest_metrics.json +++ b/tools/perf/pmu-events/arch/powerpc/power10/nest_metrics.json @@ -1,13 +1,13 @@ [ { "MetricName": "VEC_GROUP_PUMP_RETRY_RATIO_P01", - "MetricExpr": "(hv_24x7@PM_PB_RTY_VG_PUMP01\\,chip\\=?@ / hv_24x7@PM_PB_VG_PUMP01\\,chip\\=?@) * 100", + "MetricExpr": "(hv_24x7@PM_PB_RTY_VG_PUMP01\\,chip\\=?@ / (1 + hv_24x7@PM_PB_VG_PUMP01\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "VEC_GROUP_PUMP_RETRY_RATIO_P23", - "MetricExpr": "(hv_24x7@PM_PB_RTY_VG_PUMP23\\,chip\\=?@ / hv_24x7@PM_PB_VG_PUMP23\\,chip\\=?@) * 100", + "MetricExpr": "(hv_24x7@PM_PB_RTY_VG_PUMP23\\,chip\\=?@ / (1 + hv_24x7@PM_PB_VG_PUMP23\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, @@ -61,13 +61,13 @@ }, { "MetricName": "REMOTE_NODE_PUMPS_RETRIES_RATIO_P01", - "MetricExpr": "(hv_24x7@PM_PB_RTY_RNS_PUMP01\\,chip\\=?@ / hv_24x7@PM_PB_RNS_PUMP01\\,chip\\=?@) * 100", + "MetricExpr": "(hv_24x7@PM_PB_RTY_RNS_PUMP01\\,chip\\=?@ / (1 + hv_24x7@PM_PB_RNS_PUMP01\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "REMOTE_NODE_PUMPS_RETRIES_RATIO_P23", - "MetricExpr": "(hv_24x7@PM_PB_RTY_RNS_PUMP23\\,chip\\=?@ / hv_24x7@PM_PB_RNS_PUMP23\\,chip\\=?@) * 100", + "MetricExpr": "(hv_24x7@PM_PB_RTY_RNS_PUMP23\\,chip\\=?@ / (1 + hv_24x7@PM_PB_RNS_PUMP23\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, @@ -151,193 +151,193 @@ }, { "MetricName": "XLINK0_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK0_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK0_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK1_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK1_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK1_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK2_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK2_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK2_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK3_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK3_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK3_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK4_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK4_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK4_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK5_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK5_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK5_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK6_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK6_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK6_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK7_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK7_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_XLINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK7_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK0_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK0_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK0_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK1_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK1_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK1_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK2_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK2_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK2_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK3_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK3_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK3_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK4_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK4_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK4_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK5_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK5_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK5_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK6_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK6_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK6_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "XLINK7_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_XLINK7_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_XLINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_XLINK7_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_XLINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_XLINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK0_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK0_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK0_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK1_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK1_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK1_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK2_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK2_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK2_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK3_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK3_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK3_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK4_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK4_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK4_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK5_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK5_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK5_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK6_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK6_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK6_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK7_OUT_TOTAL_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK7_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (hv_24x7@PM_ALINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK7_OUT_ODD_TOTAL_UTIL\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_TOTAL_UTIL\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK0_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK0_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK0_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK0_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK0_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK1_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK1_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK1_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK1_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK1_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK2_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK2_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK2_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK2_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK2_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK3_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK3_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK3_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK3_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK3_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK4_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK4_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK4_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK4_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK4_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK5_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK5_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK5_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK5_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK5_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK6_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK6_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK6_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK6_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK6_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, { "MetricName": "ALINK7_OUT_DATA_UTILIZATION", - "MetricExpr": "((hv_24x7@PM_ALINK7_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_DATA\\,chip\\=?@) / (hv_24x7@PM_ALINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", + "MetricExpr": "((hv_24x7@PM_ALINK7_OUT_ODD_DATA\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_DATA\\,chip\\=?@) / (1 + hv_24x7@PM_ALINK7_OUT_ODD_AVLBL_CYCLES\\,chip\\=?@ + hv_24x7@PM_ALINK7_OUT_EVEN_AVLBL_CYCLES\\,chip\\=?@)) * 100", "ScaleUnit": "1.063%", "AggregationMode": "PerChip" }, -- GitLab From 409fb6bdd6eaf64c2da174b48b30ae032a9d7554 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Tue, 18 Oct 2022 21:47:51 -0300 Subject: [PATCH 0465/1927] perf bpf: Fix build with libbpf 0.7.0 by adding prototype for bpf_load_program() The bpf_load_program() prototype appeared in tools/lib/bpf/bpf.h as deprecated, but nowadays its completely removed, so add it back for building with the system libbpf when using 'make LIBBPF_DYNAMIC=1'. This is a stop gap hack till we do like tools/bpf does with bpftool, i.e. bootstrap the libbpf build and install it in the perf build directory when not using 'make LIBBPF_DYNAMIC=1'. That has to be done to all libraries in tools/lib/, so tha we can remove -Itools/lib/ from the tools/perf CFLAGS. Noticed when building with LIBBPF_DYNAMIC=1 and libbpf 0.7.0 on RHEL9. Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/bpf-event.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index eee64ddb766df..cc7c1f90cf629 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -36,6 +36,11 @@ struct btf *btf__load_from_kernel_by_id(__u32 id) #endif #ifndef HAVE_LIBBPF_BPF_PROG_LOAD +LIBBPF_API int bpf_load_program(enum bpf_prog_type type, + const struct bpf_insn *insns, size_t insns_cnt, + const char *license, __u32 kern_version, + char *log_buf, size_t log_buf_sz); + int bpf_prog_load(enum bpf_prog_type prog_type, const char *prog_name __maybe_unused, const char *license, -- GitLab From f1bdebbb67bd21d7c5dfc42f313f2f54002440b8 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 19 Oct 2022 08:44:17 -0400 Subject: [PATCH 0466/1927] perf bpf: Fix build with libbpf 0.7.0 by checking if bpf_program__set_insns() is available During the transition to libbpf 1.0 some functions that perf used were deprecated and finally removed from libbpf, so bpf_program__set_insns() was introduced for perf to continue to use its bpf loader. But when build with LIBBPF_DYNAMIC=1 we now need to check if that function is available so that perf can build with older libbpf versions, even if the end result is emitting a warning to the user that the use of the perf BPF loader requires a newer libbpf, since bpf_program__set_insns() touches libbpf objects internal state. This affects only 'perf trace' when using bpf C code or pre-compiled bytecode as an event. Noticed on RHEL9, that has libbpf 0.7.0, where bpf_program__set_insns() isn't available. Signed-off-by: Arnaldo Carvalho de Melo --- tools/build/Makefile.feature | 1 + tools/build/feature/Makefile | 4 ++++ .../test-libbpf-bpf_program__set_insns.c | 8 ++++++++ tools/perf/Makefile.config | 5 +++++ tools/perf/util/bpf-loader.c | 18 ++++++++++++++++++ 5 files changed, 36 insertions(+) create mode 100644 tools/build/feature/test-libbpf-bpf_program__set_insns.c diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index 57619f240b560..38f8851bd7cbd 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -103,6 +103,7 @@ FEATURE_TESTS_EXTRA := \ libbpf-bpf_prog_load \ libbpf-bpf_object__next_program \ libbpf-bpf_object__next_map \ + libbpf-bpf_program__set_insns \ libbpf-bpf_create_map \ libpfm4 \ libdebuginfod \ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 04b07ff882348..690fe97be1904 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -63,6 +63,7 @@ FILES= \ test-libbpf-bpf_map_create.bin \ test-libbpf-bpf_object__next_program.bin \ test-libbpf-bpf_object__next_map.bin \ + test-libbpf-bpf_program__set_insns.bin \ test-libbpf-btf__raw_data.bin \ test-get_cpuid.bin \ test-sdt.bin \ @@ -316,6 +317,9 @@ $(OUTPUT)test-libbpf-bpf_object__next_program.bin: $(OUTPUT)test-libbpf-bpf_object__next_map.bin: $(BUILD) -lbpf +$(OUTPUT)test-libbpf-bpf_program__set_insns.bin: + $(BUILD) -lbpf + $(OUTPUT)test-libbpf-btf__raw_data.bin: $(BUILD) -lbpf diff --git a/tools/build/feature/test-libbpf-bpf_program__set_insns.c b/tools/build/feature/test-libbpf-bpf_program__set_insns.c new file mode 100644 index 0000000000000..f3b7f18c8f49f --- /dev/null +++ b/tools/build/feature/test-libbpf-bpf_program__set_insns.c @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +int main(void) +{ + bpf_program__set_insns(NULL /* prog */, NULL /* new_insns */, 0 /* new_insn_cnt */); + return 0; +} diff --git a/tools/perf/Makefile.config b/tools/perf/Makefile.config index 6fd4b1384b975..898226ea8cadc 100644 --- a/tools/perf/Makefile.config +++ b/tools/perf/Makefile.config @@ -588,6 +588,10 @@ ifndef NO_LIBELF ifeq ($(feature-libbpf-bpf_object__next_map), 1) CFLAGS += -DHAVE_LIBBPF_BPF_OBJECT__NEXT_MAP endif + $(call feature_check,libbpf-bpf_program__set_insns) + ifeq ($(feature-libbpf-bpf_program__set_insns), 1) + CFLAGS += -DHAVE_LIBBPF_BPF_PROGRAM__SET_INSNS + endif $(call feature_check,libbpf-btf__raw_data) ifeq ($(feature-libbpf-btf__raw_data), 1) CFLAGS += -DHAVE_LIBBPF_BTF__RAW_DATA @@ -604,6 +608,7 @@ ifndef NO_LIBELF CFLAGS += -DHAVE_LIBBPF_BPF_PROG_LOAD CFLAGS += -DHAVE_LIBBPF_BPF_OBJECT__NEXT_PROGRAM CFLAGS += -DHAVE_LIBBPF_BPF_OBJECT__NEXT_MAP + CFLAGS += -DHAVE_LIBBPF_BPF_PROGRAM__SET_INSNS CFLAGS += -DHAVE_LIBBPF_BTF__RAW_DATA CFLAGS += -DHAVE_LIBBPF_BPF_MAP_CREATE endif diff --git a/tools/perf/util/bpf-loader.c b/tools/perf/util/bpf-loader.c index d657594894cf6..f4adeccdbbcb4 100644 --- a/tools/perf/util/bpf-loader.c +++ b/tools/perf/util/bpf-loader.c @@ -36,6 +36,24 @@ #include +#ifndef HAVE_LIBBPF_BPF_PROGRAM__SET_INSNS +int bpf_program__set_insns(struct bpf_program *prog __maybe_unused, + struct bpf_insn *new_insns __maybe_unused, size_t new_insn_cnt __maybe_unused) +{ + pr_err("%s: not support, update libbpf\n", __func__); + return -ENOTSUP; +} + +int libbpf_register_prog_handler(const char *sec __maybe_unused, + enum bpf_prog_type prog_type __maybe_unused, + enum bpf_attach_type exp_attach_type __maybe_unused, + const struct libbpf_prog_handler_opts *opts __maybe_unused) +{ + pr_err("%s: not support, update libbpf\n", __func__); + return -ENOTSUP; +} +#endif + /* temporarily disable libbpf deprecation warnings */ #pragma GCC diagnostic ignored "-Wdeprecated-declarations" -- GitLab From 304f0a2f6a6d9336fb5e474d7f62b8677d5ee167 Mon Sep 17 00:00:00 2001 From: Ian Rogers Date: Sun, 23 Oct 2022 18:10:24 -0700 Subject: [PATCH 0467/1927] perf record: Fix event fd races The write call may set errno which is problematic if occurring in a function also setting errno. Save and restore errno around the write call. done_fd may be used after close, clear it as part of the close and check its validity in the signal handler. Suggested-by: Reviewed-by: Leo Yan Signed-off-by: Ian Rogers Cc: Alexander Shishkin Cc: Anand K Mistry Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Stephane Eranian Link: https://lore.kernel.org/r/20221024011024.462518-1-irogers@google.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/builtin-record.c | 41 ++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index 52d254b1530c9..e128b855dddec 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -649,7 +649,7 @@ static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size) static volatile int signr = -1; static volatile int child_finished; #ifdef HAVE_EVENTFD_SUPPORT -static int done_fd = -1; +static volatile int done_fd = -1; #endif static void sig_handler(int sig) @@ -661,19 +661,24 @@ static void sig_handler(int sig) done = 1; #ifdef HAVE_EVENTFD_SUPPORT -{ - u64 tmp = 1; - /* - * It is possible for this signal handler to run after done is checked - * in the main loop, but before the perf counter fds are polled. If this - * happens, the poll() will continue to wait even though done is set, - * and will only break out if either another signal is received, or the - * counters are ready for read. To ensure the poll() doesn't sleep when - * done is set, use an eventfd (done_fd) to wake up the poll(). - */ - if (write(done_fd, &tmp, sizeof(tmp)) < 0) - pr_err("failed to signal wakeup fd, error: %m\n"); -} + if (done_fd >= 0) { + u64 tmp = 1; + int orig_errno = errno; + + /* + * It is possible for this signal handler to run after done is + * checked in the main loop, but before the perf counter fds are + * polled. If this happens, the poll() will continue to wait + * even though done is set, and will only break out if either + * another signal is received, or the counters are ready for + * read. To ensure the poll() doesn't sleep when done is set, + * use an eventfd (done_fd) to wake up the poll(). + */ + if (write(done_fd, &tmp, sizeof(tmp)) < 0) + pr_err("failed to signal wakeup fd, error: %m\n"); + + errno = orig_errno; + } #endif // HAVE_EVENTFD_SUPPORT } @@ -2834,8 +2839,12 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) out_delete_session: #ifdef HAVE_EVENTFD_SUPPORT - if (done_fd >= 0) - close(done_fd); + if (done_fd >= 0) { + fd = done_fd; + done_fd = -1; + + close(fd); + } #endif zstd_fini(&session->zstd_data); perf_session__delete(session); -- GitLab From 5a6c184a72a375072cffe788d93ad6052c48f16b Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Fri, 21 Oct 2022 10:25:57 +0200 Subject: [PATCH 0468/1927] perf list: Fix PMU name pai_crypto in perf list on s390 Commit e0b23af82d6f454c ("perf list: Add PMU pai_crypto event description for IBM z16") introduced the "Processor Activity Instrumentation" for cryptographic counters for z16. The PMU device driver exports the counters via sysfs files listed in directory /sys/devices/pai_crypto. To specify an event from that PMU, use 'perf stat -e pai_crypto/XXX/'. However the JSON file mentioned in above commit exports the counter decriptions in file pmu-events/arch/s390/cf_z16/pai.json. Rename this file to pmu-events/arch/s390/cf_z16/pai_crypto.json to make the naming consistent. Now 'perf list' shows the counter names under pai_crypto section: pai_crypto: CRYPTO_ALL [CRYPTO ALL. Unit: pai_crypto] ... Output before was pai: CRYPTO_ALL [CRYPTO ALL. Unit: pai_crypto] ... Fixes: e0b23af82d6f454c ("perf list: Add PMU pai_crypto event description for IBM z16") Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Cc: Heiko Carstens Cc: Sven Schnelle Cc: Thomas Richter Cc: Vasily Gorbik Link: https://lore.kernel.org/r/20221021082557.2695382-1-tmricht@linux.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- .../pmu-events/arch/s390/cf_z16/{pai.json => pai_crypto.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/perf/pmu-events/arch/s390/cf_z16/{pai.json => pai_crypto.json} (100%) diff --git a/tools/perf/pmu-events/arch/s390/cf_z16/pai.json b/tools/perf/pmu-events/arch/s390/cf_z16/pai_crypto.json similarity index 100% rename from tools/perf/pmu-events/arch/s390/cf_z16/pai.json rename to tools/perf/pmu-events/arch/s390/cf_z16/pai_crypto.json -- GitLab From 246122a856faddd87df3063c0dd38a62b40ceeab Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Fri, 21 Oct 2022 11:10:55 -0700 Subject: [PATCH 0469/1927] perf test: Do not fail Intel-PT misc test w/o libpython The virtual LBR test uses a python script to check the max size of branch stack in the Intel-PT generated LBR. But it didn't check whether python scripting is available (as it's optional). Let's skip the test if the python support is not available. Fixes: f77811a0f62577d2 ("perf test: test_intel_pt.sh: Add 9 tests") Reviewed-by: Adrian Hunter Signed-off-by: Namhyung Kim Acked-by: Ian Rogers Cc: Ammy Yi Cc: Ingo Molnar Cc: Jiri Olsa Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20221021181055.60183-1-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/shell/test_intel_pt.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/perf/tests/shell/test_intel_pt.sh b/tools/perf/tests/shell/test_intel_pt.sh index 4c0aabbe33bdf..f5ed7b1af419c 100755 --- a/tools/perf/tests/shell/test_intel_pt.sh +++ b/tools/perf/tests/shell/test_intel_pt.sh @@ -526,6 +526,12 @@ test_kernel_trace() test_virtual_lbr() { echo "--- Test virtual LBR ---" + # Check if python script is supported + libpython=$(perf version --build-options | grep python | grep -cv OFF) + if [ "${libpython}" != "1" ] ; then + echo "SKIP: python scripting is not supported" + return 2 + fi # Python script to determine the maximum size of branch stacks cat << "_end_of_file_" > "${maxbrstack}" -- GitLab From ffc1df3dc97ee2aad6d2a94e4615c2a96cf291ad Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Sat, 9 Apr 2022 11:48:15 -0300 Subject: [PATCH 0470/1927] tools headers arm64: Sync arm64's cputype.h with the kernel sources To get the changes in: 0e5d5ae837c8ce04 ("arm64: Add AMPERE1 to the Spectre-BHB affected list") That addresses this perf build warning: Warning: Kernel ABI header at 'tools/arch/arm64/include/asm/cputype.h' differs from latest version at 'arch/arm64/include/asm/cputype.h' diff -u tools/arch/arm64/include/asm/cputype.h arch/arm64/include/asm/cputype.h Cc: Catalin Marinas Cc: D Scott Phillips https://lore.kernel.org/lkml/Y1fy5GD7ZYvkeufv@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/arch/arm64/include/asm/cputype.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/arch/arm64/include/asm/cputype.h b/tools/arch/arm64/include/asm/cputype.h index 8aa0d276a6362..abc418650fec0 100644 --- a/tools/arch/arm64/include/asm/cputype.h +++ b/tools/arch/arm64/include/asm/cputype.h @@ -60,6 +60,7 @@ #define ARM_CPU_IMP_FUJITSU 0x46 #define ARM_CPU_IMP_HISI 0x48 #define ARM_CPU_IMP_APPLE 0x61 +#define ARM_CPU_IMP_AMPERE 0xC0 #define ARM_CPU_PART_AEM_V8 0xD0F #define ARM_CPU_PART_FOUNDATION 0xD00 @@ -123,6 +124,8 @@ #define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 #define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 +#define AMPERE_CPU_PART_AMPERE1 0xAC3 + #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) #define MIDR_CORTEX_A72 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A72) @@ -172,6 +175,7 @@ #define MIDR_APPLE_M1_FIRESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO) #define MIDR_APPLE_M1_ICESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX) #define MIDR_APPLE_M1_FIRESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX) +#define MIDR_AMPERE1 MIDR_CPU_MODEL(ARM_CPU_IMP_AMPERE, AMPERE_CPU_PART_AMPERE1) /* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */ #define MIDR_FUJITSU_ERRATUM_010001 MIDR_FUJITSU_A64FX -- GitLab From 4402e360d0f833c8c67b2fda0d3f612f4fd8b2cc Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Mon, 2 Dec 2019 11:40:57 -0300 Subject: [PATCH 0471/1927] tools headers: Update the copy of x86's memcpy_64.S used in 'perf bench' We also need to add SYM_TYPED_FUNC_START() to util/include/linux/linkage.h and update tools/perf/check_headers.sh to ignore the include cfi_types.h line when checking if the kernel original files drifted from the copies we carry. This is to get the changes from: ccace936eec7b805 ("x86: Add types to indirectly called assembly functions") Addressing these tools/perf build warnings: Warning: Kernel ABI header at 'tools/arch/x86/lib/memcpy_64.S' differs from latest version at 'arch/x86/lib/memcpy_64.S' diff -u tools/arch/x86/lib/memcpy_64.S arch/x86/lib/memcpy_64.S Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Cc: Kees Cook Cc: Sami Tolvanen Link: https://lore.kernel.org/lkml/Y1f3VRIec9EBgX6F@kernel.org/ Signed-off-by: Arnaldo Carvalho de Melo --- tools/arch/x86/lib/memcpy_64.S | 2 +- tools/perf/check-headers.sh | 2 +- tools/perf/util/include/linux/linkage.h | 13 +++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tools/arch/x86/lib/memcpy_64.S b/tools/arch/x86/lib/memcpy_64.S index d0d7b9bc6cad3..5418e2f99834e 100644 --- a/tools/arch/x86/lib/memcpy_64.S +++ b/tools/arch/x86/lib/memcpy_64.S @@ -27,7 +27,7 @@ * Output: * rax original destination */ -SYM_FUNC_START(__memcpy) +SYM_TYPED_FUNC_START(__memcpy) ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ "jmp memcpy_erms", X86_FEATURE_ERMS diff --git a/tools/perf/check-headers.sh b/tools/perf/check-headers.sh index 6ee44b18c6b57..eacca9a874e2f 100755 --- a/tools/perf/check-headers.sh +++ b/tools/perf/check-headers.sh @@ -143,7 +143,7 @@ for i in $SYNC_CHECK_FILES; do done # diff with extra ignore lines -check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memcpy_\(erms\|orig\))"' +check arch/x86/lib/memcpy_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memcpy_\(erms\|orig\))" -I"^#include "' check arch/x86/lib/memset_64.S '-I "^EXPORT_SYMBOL" -I "^#include " -I"^SYM_FUNC_START\(_LOCAL\)*(memset_\(erms\|orig\))"' check arch/x86/include/asm/amd-ibs.h '-I "^#include [<\"]\(asm/\)*msr-index.h"' check arch/arm64/include/asm/cputype.h '-I "^#include [<\"]\(asm/\)*sysreg.h"' diff --git a/tools/perf/util/include/linux/linkage.h b/tools/perf/util/include/linux/linkage.h index aa0c5179836d1..75e2248416f55 100644 --- a/tools/perf/util/include/linux/linkage.h +++ b/tools/perf/util/include/linux/linkage.h @@ -115,4 +115,17 @@ SYM_ALIAS(alias, name, SYM_T_FUNC, SYM_L_WEAK) #endif +// In the kernel sources (include/linux/cfi_types.h), this has a different +// definition when CONFIG_CFI_CLANG is used, for tools/ just use the !clang +// definition: +#ifndef SYM_TYPED_START +#define SYM_TYPED_START(name, linkage, align...) \ + SYM_START(name, linkage, align) +#endif + +#ifndef SYM_TYPED_FUNC_START +#define SYM_TYPED_FUNC_START(name) \ + SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN) +#endif + #endif /* PERF_LINUX_LINKAGE_H_ */ -- GitLab From 036b8f5b8970e387eb3224eda45348de39135177 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Thu, 19 Mar 2020 11:42:56 -0300 Subject: [PATCH 0472/1927] tools headers uapi: Update linux/in.h copy To get the changes in: 65b32f801bfbc54d ("uapi: move IPPROTO_L2TP to in.h") 5854a09b49574da5 ("net/ipv4: Use __DECLARE_FLEX_ARRAY() helper") That ends up automatically adding the new IPPROTO_L2TP to the socket args beautifiers: $ tools/perf/trace/beauty/socket.sh > before $ cp include/uapi/linux/in.h tools/include/uapi/linux/in.h $ tools/perf/trace/beauty/socket.sh > after $ diff -u before after --- before 2022-10-25 12:17:02.577892416 -0300 +++ after 2022-10-25 12:17:10.806113033 -0300 @@ -20,6 +20,7 @@ [98] = "ENCAP", [103] = "PIM", [108] = "COMP", + [115] = "L2TP", [132] = "SCTP", [136] = "UDPLITE", [137] = "MPLS", $ Now 'perf trace' will decode that 115 into "L2TP" and it will also be possible to use it in tracepoint filter expressions. Addresses this tools/perf build warning: Warning: Kernel ABI header at 'tools/include/uapi/linux/in.h' differs from latest version at 'include/uapi/linux/in.h' diff -u tools/include/uapi/linux/in.h include/uapi/linux/in.h Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Cc: Paolo Abeni Cc: Wojciech Drewek Cc: David S. Miller Cc: Gustavo A. R. Silva Link: https://lore.kernel.org/lkml/Y1f%2FGe6vjQrGjYiK@kernel.org/ Signed-off-by: Arnaldo Carvalho de Melo --- tools/include/uapi/linux/in.h | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/tools/include/uapi/linux/in.h b/tools/include/uapi/linux/in.h index 14168225cecdc..f243ce665f74f 100644 --- a/tools/include/uapi/linux/in.h +++ b/tools/include/uapi/linux/in.h @@ -68,6 +68,8 @@ enum { #define IPPROTO_PIM IPPROTO_PIM IPPROTO_COMP = 108, /* Compression Header Protocol */ #define IPPROTO_COMP IPPROTO_COMP + IPPROTO_L2TP = 115, /* Layer 2 Tunnelling Protocol */ +#define IPPROTO_L2TP IPPROTO_L2TP IPPROTO_SCTP = 132, /* Stream Control Transport Protocol */ #define IPPROTO_SCTP IPPROTO_SCTP IPPROTO_UDPLITE = 136, /* UDP-Lite (RFC 3828) */ @@ -188,21 +190,13 @@ struct ip_mreq_source { }; struct ip_msfilter { + __be32 imsf_multiaddr; + __be32 imsf_interface; + __u32 imsf_fmode; + __u32 imsf_numsrc; union { - struct { - __be32 imsf_multiaddr_aux; - __be32 imsf_interface_aux; - __u32 imsf_fmode_aux; - __u32 imsf_numsrc_aux; - __be32 imsf_slist[1]; - }; - struct { - __be32 imsf_multiaddr; - __be32 imsf_interface; - __u32 imsf_fmode; - __u32 imsf_numsrc; - __be32 imsf_slist_flex[]; - }; + __be32 imsf_slist[1]; + __DECLARE_FLEX_ARRAY(__be32, imsf_slist_flex); }; }; -- GitLab From 82c50d8937f32f911a41739f7a51934a9ea8b92a Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 12 Feb 2020 11:04:23 -0300 Subject: [PATCH 0473/1927] tools include UAPI: Sync sound/asound.h copy with the kernel sources Picking the changes from: 69ab6f5b00b1804e ("ALSA: Remove some left-over license text in include/uapi/sound/") Which entails no changes in the tooling side as it doesn't introduce new SNDRV_PCM_IOCTL_ ioctls. To silence this perf tools build warning: Warning: Kernel ABI header at 'tools/include/uapi/sound/asound.h' differs from latest version at 'include/uapi/sound/asound.h' diff -u tools/include/uapi/sound/asound.h include/uapi/sound/asound.h Signed-off-by: Arnaldo Carvalho de Melo --- tools/include/uapi/sound/asound.h | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tools/include/uapi/sound/asound.h b/tools/include/uapi/sound/asound.h index 3974a2a911cc4..de6810e94abed 100644 --- a/tools/include/uapi/sound/asound.h +++ b/tools/include/uapi/sound/asound.h @@ -3,22 +3,6 @@ * Advanced Linux Sound Architecture - ALSA - Driver * Copyright (c) 1994-2003 by Jaroslav Kysela , * Abramo Bagnara - * - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - * */ #ifndef _UAPI__SOUND_ASOUND_H -- GitLab From 49c75d30b0078d304bc0ae41026e629b23f6711e Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Mon, 8 Jun 2020 11:32:48 -0300 Subject: [PATCH 0474/1927] tools headers uapi: Sync linux/stat.h with the kernel sources To pick the changes from: 825cf206ed510c4a ("statx: add direct I/O alignment information") That add a constant that was manually added to tools/perf/trace/beauty/statx.c, at some point this should move to the shell based automated way. This silences this perf build warning: Warning: Kernel ABI header at 'tools/include/uapi/linux/stat.h' differs from latest version at 'include/uapi/linux/stat.h' diff -u tools/include/uapi/linux/stat.h include/uapi/linux/stat.h Cc: Eric Biggers Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Link: https://lore.kernel.org/lkml/Y1gGQL5LonnuzeYd@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/include/uapi/linux/stat.h | 4 +++- tools/perf/trace/beauty/statx.c | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/include/uapi/linux/stat.h b/tools/include/uapi/linux/stat.h index 1500a0f58041a..7cab2c65d3d7f 100644 --- a/tools/include/uapi/linux/stat.h +++ b/tools/include/uapi/linux/stat.h @@ -124,7 +124,8 @@ struct statx { __u32 stx_dev_minor; /* 0x90 */ __u64 stx_mnt_id; - __u64 __spare2; + __u32 stx_dio_mem_align; /* Memory buffer alignment for direct I/O */ + __u32 stx_dio_offset_align; /* File offset alignment for direct I/O */ /* 0xa0 */ __u64 __spare3[12]; /* Spare space for future expansion */ /* 0x100 */ @@ -152,6 +153,7 @@ struct statx { #define STATX_BASIC_STATS 0x000007ffU /* The stuff in the normal stat struct */ #define STATX_BTIME 0x00000800U /* Want/got stx_btime */ #define STATX_MNT_ID 0x00001000U /* Got stx_mnt_id */ +#define STATX_DIOALIGN 0x00002000U /* Want/got direct I/O alignment info */ #define STATX__RESERVED 0x80000000U /* Reserved for future struct statx expansion */ diff --git a/tools/perf/trace/beauty/statx.c b/tools/perf/trace/beauty/statx.c index 110f0c609d840..5f5320f7c6e27 100644 --- a/tools/perf/trace/beauty/statx.c +++ b/tools/perf/trace/beauty/statx.c @@ -66,6 +66,7 @@ size_t syscall_arg__scnprintf_statx_mask(char *bf, size_t size, struct syscall_a P_FLAG(BLOCKS); P_FLAG(BTIME); P_FLAG(MNT_ID); + P_FLAG(DIOALIGN); #undef P_FLAG -- GitLab From 74455fd7e459566198c8f1b2b33ca43a0c3ee8cb Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Thu, 1 Jul 2021 13:39:15 -0300 Subject: [PATCH 0475/1927] tools headers cpufeatures: Sync with the kernel sources To pick the changes from: 257449c6a50298bd ("x86/cpufeatures: Add LbrExtV2 feature bit") This only causes these perf files to be rebuilt: CC /tmp/build/perf/bench/mem-memcpy-x86-64-asm.o CC /tmp/build/perf/bench/mem-memset-x86-64-asm.o And addresses this perf build warning: Warning: Kernel ABI header at 'tools/arch/x86/include/asm/cpufeatures.h' differs from latest version at 'arch/x86/include/asm/cpufeatures.h' diff -u tools/arch/x86/include/asm/cpufeatures.h arch/x86/include/asm/cpufeatures.h Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Sandipan Das Link: https://lore.kernel.org/lkml/Y1g6vGPqPhOrXoaN@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/arch/x86/include/asm/cpufeatures.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index ef4775c6db01c..b71f4f2ecdd57 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -96,7 +96,7 @@ #define X86_FEATURE_SYSCALL32 ( 3*32+14) /* "" syscall in IA32 userspace */ #define X86_FEATURE_SYSENTER32 ( 3*32+15) /* "" sysenter in IA32 userspace */ #define X86_FEATURE_REP_GOOD ( 3*32+16) /* REP microcode works well */ -/* FREE! ( 3*32+17) */ +#define X86_FEATURE_AMD_LBR_V2 ( 3*32+17) /* AMD Last Branch Record Extension Version 2 */ #define X86_FEATURE_LFENCE_RDTSC ( 3*32+18) /* "" LFENCE synchronizes RDTSC */ #define X86_FEATURE_ACC_POWER ( 3*32+19) /* AMD Accumulated Power Mechanism */ #define X86_FEATURE_NOPL ( 3*32+20) /* The NOPL (0F 1F) instructions */ -- GitLab From 31970608a6d3796c3adbfbfd379fa3092de65c5d Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Mon, 26 Sep 2022 12:45:32 -0700 Subject: [PATCH 0476/1927] overflow: Fix kern-doc markup for functions Fix the kern-doc markings for several of the overflow helpers and move their location into the core kernel API documentation, where it belongs (it's not driver-specific). Cc: Jonathan Corbet Cc: linux-doc@vger.kernel.org Cc: linux-hardening@vger.kernel.org Reviewed-by: Akira Yokosawa Signed-off-by: Kees Cook --- Documentation/core-api/kernel-api.rst | 6 +++++ Documentation/driver-api/basics.rst | 3 --- include/linux/overflow.h | 38 +++++++++++---------------- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/Documentation/core-api/kernel-api.rst b/Documentation/core-api/kernel-api.rst index 0793c400d4b05..06f4ab1226979 100644 --- a/Documentation/core-api/kernel-api.rst +++ b/Documentation/core-api/kernel-api.rst @@ -118,6 +118,12 @@ Text Searching CRC and Math Functions in Linux =============================== +Arithmetic Overflow Checking +---------------------------- + +.. kernel-doc:: include/linux/overflow.h + :internal: + CRC Functions ------------- diff --git a/Documentation/driver-api/basics.rst b/Documentation/driver-api/basics.rst index 3e2dae9548984..4b4d8e28d3be4 100644 --- a/Documentation/driver-api/basics.rst +++ b/Documentation/driver-api/basics.rst @@ -107,9 +107,6 @@ Kernel utility functions .. kernel-doc:: kernel/panic.c :export: -.. kernel-doc:: include/linux/overflow.h - :internal: - Device Resource Management -------------------------- diff --git a/include/linux/overflow.h b/include/linux/overflow.h index 19dfdd74835ec..1d3be1a2204c3 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -51,8 +51,8 @@ static inline bool __must_check __must_check_overflow(bool overflow) return unlikely(overflow); } -/** check_add_overflow() - Calculate addition with overflow checking - * +/** + * check_add_overflow() - Calculate addition with overflow checking * @a: first addend * @b: second addend * @d: pointer to store sum @@ -66,8 +66,8 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define check_add_overflow(a, b, d) \ __must_check_overflow(__builtin_add_overflow(a, b, d)) -/** check_sub_overflow() - Calculate subtraction with overflow checking - * +/** + * check_sub_overflow() - Calculate subtraction with overflow checking * @a: minuend; value to subtract from * @b: subtrahend; value to subtract from @a * @d: pointer to store difference @@ -81,8 +81,8 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define check_sub_overflow(a, b, d) \ __must_check_overflow(__builtin_sub_overflow(a, b, d)) -/** check_mul_overflow() - Calculate multiplication with overflow checking - * +/** + * check_mul_overflow() - Calculate multiplication with overflow checking * @a: first factor * @b: second factor * @d: pointer to store product @@ -96,23 +96,24 @@ static inline bool __must_check __must_check_overflow(bool overflow) #define check_mul_overflow(a, b, d) \ __must_check_overflow(__builtin_mul_overflow(a, b, d)) -/** check_shl_overflow() - Calculate a left-shifted value and check overflow - * +/** + * check_shl_overflow() - Calculate a left-shifted value and check overflow * @a: Value to be shifted * @s: How many bits left to shift * @d: Pointer to where to store the result * * Computes *@d = (@a << @s) * - * Returns true if '*d' cannot hold the result or when 'a << s' doesn't + * Returns true if '*@d' cannot hold the result or when '@a << @s' doesn't * make sense. Example conditions: - * - 'a << s' causes bits to be lost when stored in *d. - * - 's' is garbage (e.g. negative) or so large that the result of - * 'a << s' is guaranteed to be 0. - * - 'a' is negative. - * - 'a << s' sets the sign bit, if any, in '*d'. * - * '*d' will hold the results of the attempted shift, but is not + * - '@a << @s' causes bits to be lost when stored in *@d. + * - '@s' is garbage (e.g. negative) or so large that the result of + * '@a << @s' is guaranteed to be 0. + * - '@a' is negative. + * - '@a << @s' sets the sign bit, if any, in '*@d'. + * + * '*@d' will hold the results of the attempted shift, but is not * considered "safe for use" if true is returned. */ #define check_shl_overflow(a, s, d) __must_check_overflow(({ \ @@ -129,7 +130,6 @@ static inline bool __must_check __must_check_overflow(bool overflow) /** * size_mul() - Calculate size_t multiplication with saturation at SIZE_MAX - * * @factor1: first factor * @factor2: second factor * @@ -149,7 +149,6 @@ static inline size_t __must_check size_mul(size_t factor1, size_t factor2) /** * size_add() - Calculate size_t addition with saturation at SIZE_MAX - * * @addend1: first addend * @addend2: second addend * @@ -169,7 +168,6 @@ static inline size_t __must_check size_add(size_t addend1, size_t addend2) /** * size_sub() - Calculate size_t subtraction with saturation at SIZE_MAX - * * @minuend: value to subtract from * @subtrahend: value to subtract from @minuend * @@ -192,7 +190,6 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) /** * array_size() - Calculate size of 2-dimensional array. - * * @a: dimension one * @b: dimension two * @@ -205,7 +202,6 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) /** * array3_size() - Calculate size of 3-dimensional array. - * * @a: dimension one * @b: dimension two * @c: dimension three @@ -220,7 +216,6 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) /** * flex_array_size() - Calculate size of a flexible array member * within an enclosing structure. - * * @p: Pointer to the structure. * @member: Name of the flexible array member. * @count: Number of elements in the array. @@ -237,7 +232,6 @@ static inline size_t __must_check size_sub(size_t minuend, size_t subtrahend) /** * struct_size() - Calculate size of structure with trailing flexible array. - * * @p: Pointer to the structure. * @member: Name of the array member. * @count: Number of elements in the array. -- GitLab From 0e5b9f25b27a7a92880f88f5dba3edf726ec5f61 Mon Sep 17 00:00:00 2001 From: Nick Desaulniers Date: Thu, 6 Oct 2022 10:17:51 -0700 Subject: [PATCH 0477/1927] overflow: disable failing tests for older clang versions Building the overflow kunit tests with clang-11 fails with: $ ./tools/testing/kunit/kunit.py run --arch=arm --make_options LLVM=1 \ overflow ... ld.lld: error: undefined symbol: __mulodi4 ... Clang 11 and earlier generate unwanted libcalls for signed output, unsigned input. Disable these tests for now, but should these become used in the kernel we might consider that as justification for dropping clang-11 support. Keep the clang-11 build alive a little bit longer. Avoid -Wunused-function warnings via __maybe_unused. To test W=1: $ make LLVM=1 -j128 defconfig $ ./scripts/config -e KUNIT -e KUNIT_ALL $ make LLVM=1 -j128 olddefconfig lib/overflow_kunit.o W=1 Link: https://github.com/ClangBuiltLinux/linux/issues/1711 Link: https://github.com/llvm/llvm-project/commit/3203143f1356a4e4e3ada231156fc6da6e1a9f9d Reported-by: Nathan Chancellor Signed-off-by: Nick Desaulniers Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20221006171751.3444575-1-ndesaulniers@google.com --- lib/overflow_kunit.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c index 5369634701fa3..ca5f1aa3e91d9 100644 --- a/lib/overflow_kunit.c +++ b/lib/overflow_kunit.c @@ -254,6 +254,7 @@ static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of); \ } \ \ +__maybe_unused \ static void n ## _overflow_test(struct kunit *test) { \ unsigned i; \ \ @@ -720,8 +721,14 @@ static struct kunit_case overflow_test_cases[] = { KUNIT_CASE(u64_u64__u64_overflow_test), KUNIT_CASE(s64_s64__s64_overflow_test), #endif - KUNIT_CASE(u32_u32__u8_overflow_test), +/* + * Clang 11 and earlier generate unwanted libcalls for signed output, unsigned + * input. + */ +#if !(defined(CONFIG_CC_IS_CLANG) && __clang_major__ <= 11) KUNIT_CASE(u32_u32__int_overflow_test), +#endif + KUNIT_CASE(u32_u32__u8_overflow_test), KUNIT_CASE(u8_u8__int_overflow_test), KUNIT_CASE(int_int__u8_overflow_test), KUNIT_CASE(shift_sane_test), -- GitLab From 72c3ebea375c39413d02113758319b74ecd790bd Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 6 Oct 2022 16:00:17 -0700 Subject: [PATCH 0478/1927] overflow: Refactor test skips for Clang-specific issues Convert test exclusion into test skipping. This brings the logic for why a test is being skipped into the test itself, instead of having to spread ifdefs around the code. This will make cleanup easier as minimum tests get raised. Drop __maybe_unused so missed tests will be noticed again and clean up whitespace. For example, clang-11 on i386: [15:52:32] ================== overflow (18 subtests) ================== [15:52:32] [PASSED] u8_u8__u8_overflow_test [15:52:32] [PASSED] s8_s8__s8_overflow_test [15:52:32] [PASSED] u16_u16__u16_overflow_test [15:52:32] [PASSED] s16_s16__s16_overflow_test [15:52:32] [PASSED] u32_u32__u32_overflow_test [15:52:32] [PASSED] s32_s32__s32_overflow_test [15:52:32] [SKIPPED] u64_u64__u64_overflow_test [15:52:32] [SKIPPED] s64_s64__s64_overflow_test [15:52:32] [SKIPPED] u32_u32__int_overflow_test [15:52:32] [PASSED] u32_u32__u8_overflow_test [15:52:32] [PASSED] u8_u8__int_overflow_test [15:52:32] [PASSED] int_int__u8_overflow_test [15:52:32] [PASSED] shift_sane_test [15:52:32] [PASSED] shift_overflow_test [15:52:32] [PASSED] shift_truncate_test [15:52:32] [PASSED] shift_nonsense_test [15:52:32] [PASSED] overflow_allocation_test [15:52:32] [PASSED] overflow_size_helpers_test [15:52:32] ==================== [PASSED] overflow ===================== [15:52:32] ============================================================ [15:52:32] Testing complete. Ran 18 tests: passed: 15, skipped: 3 Cc: Nick Desaulniers Cc: Nathan Chancellor Cc: Tom Rix Cc: Daniel Latypov Cc: "Gustavo A. R. Silva" Cc: Gwan-gyeong Mun Cc: llvm@lists.linux.dev Signed-off-by: Kees Cook Reviewed-by: Nick Desaulniers Tested-by: Nick Desaulniers Link: https://lore.kernel.org/r/20221006230017.1833458-1-keescook@chromium.org --- lib/overflow_kunit.c | 52 +++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c index ca5f1aa3e91d9..b8556a2e7bb1d 100644 --- a/lib/overflow_kunit.c +++ b/lib/overflow_kunit.c @@ -16,6 +16,34 @@ #include #include +#define SKIP(cond, reason) do { \ + if (cond) { \ + kunit_skip(test, reason); \ + return; \ + } \ +} while (0) + +/* + * Clang 11 and earlier generate unwanted libcalls for signed output + * on unsigned input. + */ +#if defined(CONFIG_CC_IS_CLANG) && __clang_major__ <= 11 +# define SKIP_SIGN_MISMATCH(t) SKIP(t, "Clang 11 unwanted libcalls") +#else +# define SKIP_SIGN_MISMATCH(t) do { } while (0) +#endif + +/* + * Clang 13 and earlier generate unwanted libcalls for 64-bit tests on + * 32-bit hosts. + */ +#if defined(CONFIG_CC_IS_CLANG) && __clang_major__ <= 13 && \ + BITS_PER_LONG != 64 +# define SKIP_64_ON_32(t) SKIP(t, "Clang 13 unwanted libcalls") +#else +# define SKIP_64_ON_32(t) do { } while (0) +#endif + #define DEFINE_TEST_ARRAY_TYPED(t1, t2, t) \ static const struct test_ ## t1 ## _ ## t2 ## __ ## t { \ t1 a; \ @@ -94,7 +122,6 @@ DEFINE_TEST_ARRAY(u32) = { {-4U, 5U, 1U, -9U, -20U, true, false, true}, }; -#if BITS_PER_LONG == 64 DEFINE_TEST_ARRAY(u64) = { {0, 0, 0, 0, 0, false, false, false}, {1, 1, 2, 0, 1, false, false, false}, @@ -118,7 +145,6 @@ DEFINE_TEST_ARRAY(u64) = { false, true, false}, {-15ULL, 10ULL, -5ULL, -25ULL, -150ULL, false, false, true}, }; -#endif DEFINE_TEST_ARRAY(s8) = { {0, 0, 0, 0, 0, false, false, false}, @@ -194,7 +220,6 @@ DEFINE_TEST_ARRAY(s32) = { {S32_MAX, S32_MAX, -2, 0, 1, true, false, true}, }; -#if BITS_PER_LONG == 64 DEFINE_TEST_ARRAY(s64) = { {0, 0, 0, 0, 0, false, false, false}, @@ -223,7 +248,6 @@ DEFINE_TEST_ARRAY(s64) = { {-128, -1, -129, -127, 128, false, false, false}, {0, -S64_MAX, -S64_MAX, S64_MAX, 0, false, false, false}, }; -#endif #define check_one_op(t, fmt, op, sym, a, b, r, of) do { \ int _a_orig = a, _a_bump = a + 1; \ @@ -246,7 +270,7 @@ DEFINE_TEST_ARRAY(s64) = { #define DEFINE_TEST_FUNC_TYPED(n, t, fmt) \ static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ -{ \ +{ \ check_one_op(t, fmt, add, "+", p->a, p->b, p->sum, p->s_of); \ check_one_op(t, fmt, add, "+", p->b, p->a, p->sum, p->s_of); \ check_one_op(t, fmt, sub, "-", p->a, p->b, p->diff, p->d_of); \ @@ -254,10 +278,15 @@ static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \ check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of); \ } \ \ -__maybe_unused \ static void n ## _overflow_test(struct kunit *test) { \ unsigned i; \ \ + SKIP_64_ON_32(__same_type(t, u64)); \ + SKIP_64_ON_32(__same_type(t, s64)); \ + SKIP_SIGN_MISMATCH(__same_type(n ## _tests[0].a, u32) && \ + __same_type(n ## _tests[0].b, u32) && \ + __same_type(n ## _tests[0].sum, int)); \ + \ for (i = 0; i < ARRAY_SIZE(n ## _tests); ++i) \ do_test_ ## n(test, &n ## _tests[i]); \ kunit_info(test, "%zu %s arithmetic tests finished\n", \ @@ -273,10 +302,8 @@ DEFINE_TEST_FUNC(u16, "%d"); DEFINE_TEST_FUNC(s16, "%d"); DEFINE_TEST_FUNC(u32, "%u"); DEFINE_TEST_FUNC(s32, "%d"); -#if BITS_PER_LONG == 64 DEFINE_TEST_FUNC(u64, "%llu"); DEFINE_TEST_FUNC(s64, "%lld"); -#endif DEFINE_TEST_ARRAY_TYPED(u32, u32, u8) = { {0, 0, 0, 0, 0, false, false, false}, @@ -716,18 +743,9 @@ static struct kunit_case overflow_test_cases[] = { KUNIT_CASE(s16_s16__s16_overflow_test), KUNIT_CASE(u32_u32__u32_overflow_test), KUNIT_CASE(s32_s32__s32_overflow_test), -/* Clang 13 and earlier generate unwanted libcalls on 32-bit. */ -#if BITS_PER_LONG == 64 KUNIT_CASE(u64_u64__u64_overflow_test), KUNIT_CASE(s64_s64__s64_overflow_test), -#endif -/* - * Clang 11 and earlier generate unwanted libcalls for signed output, unsigned - * input. - */ -#if !(defined(CONFIG_CC_IS_CLANG) && __clang_major__ <= 11) KUNIT_CASE(u32_u32__int_overflow_test), -#endif KUNIT_CASE(u32_u32__u8_overflow_test), KUNIT_CASE(u8_u8__int_overflow_test), KUNIT_CASE(int_int__u8_overflow_test), -- GitLab From 5bf2fedca8f59379025b0d52f917b9ddb9bfe17e Mon Sep 17 00:00:00 2001 From: Bernd Edlinger Date: Mon, 7 Jun 2021 15:54:27 +0200 Subject: [PATCH 0479/1927] exec: Copy oldsighand->action under spin-lock unshare_sighand should only access oldsighand->action while holding oldsighand->siglock, to make sure that newsighand->action is in a consistent state. Signed-off-by: Bernd Edlinger Cc: stable@vger.kernel.org Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/AM8PR10MB470871DEBD1DED081F9CC391E4389@AM8PR10MB4708.EURPRD10.PROD.OUTLOOK.COM --- fs/exec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fs/exec.c b/fs/exec.c index 349a5da91efe8..32dc8cf5fceb9 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1197,11 +1197,11 @@ static int unshare_sighand(struct task_struct *me) return -ENOMEM; refcount_set(&newsighand->count, 1); - memcpy(newsighand->action, oldsighand->action, - sizeof(newsighand->action)); write_lock_irq(&tasklist_lock); spin_lock(&oldsighand->siglock); + memcpy(newsighand->action, oldsighand->action, + sizeof(newsighand->action)); rcu_assign_pointer(me->sighand, newsighand); spin_unlock(&oldsighand->siglock); write_unlock_irq(&tasklist_lock); -- GitLab From 594d2a14f2168c09b13b114c3d457aa939403e52 Mon Sep 17 00:00:00 2001 From: Li Zetao Date: Mon, 24 Oct 2022 23:44:21 +0800 Subject: [PATCH 0480/1927] fs/binfmt_elf: Fix memory leak in load_elf_binary() There is a memory leak reported by kmemleak: unreferenced object 0xffff88817104ef80 (size 224): comm "xfs_admin", pid 47165, jiffies 4298708825 (age 1333.476s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 60 a8 b3 00 81 88 ff ff a8 10 5a 00 81 88 ff ff `.........Z..... backtrace: [] __alloc_file+0x21/0x250 [] alloc_empty_file+0x41/0xf0 [] path_openat+0xea/0x3d30 [] do_filp_open+0x1b9/0x290 [] do_open_execat+0xce/0x5b0 [] open_exec+0x27/0x50 [] load_elf_binary+0x510/0x3ed0 [] bprm_execve+0x599/0x1240 [] do_execveat_common.isra.0+0x4c7/0x680 [] __x64_sys_execve+0x88/0xb0 [] do_syscall_64+0x35/0x80 If "interp_elf_ex" fails to allocate memory in load_elf_binary(), the program will take the "out_free_ph" error handing path, resulting in "interpreter" file resource is not released. Fix it by adding an error handing path "out_free_file", which will release the file resource when "interp_elf_ex" failed to allocate memory. Fixes: 0693ffebcfe5 ("fs/binfmt_elf.c: allocate less for static executable") Signed-off-by: Li Zetao Reviewed-by: Alexey Dobriyan Signed-off-by: Kees Cook Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221024154421.982230-1-lizetao1@huawei.com --- fs/binfmt_elf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 63c7ebb0da898..6a11025e58502 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -911,7 +911,7 @@ static int load_elf_binary(struct linux_binprm *bprm) interp_elf_ex = kmalloc(sizeof(*interp_elf_ex), GFP_KERNEL); if (!interp_elf_ex) { retval = -ENOMEM; - goto out_free_ph; + goto out_free_file; } /* Get the exec headers */ @@ -1354,6 +1354,7 @@ static int load_elf_binary(struct linux_binprm *bprm) out_free_dentry: kfree(interp_elf_ex); kfree(interp_elf_phdata); +out_free_file: allow_write_access(interpreter); if (interpreter) fput(interpreter); -- GitLab From d89d7ff01235f218dad37de84457717f699dee79 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Sun, 23 Oct 2022 19:01:24 -0700 Subject: [PATCH 0481/1927] ipv6: ensure sane device mtu in tunnels Another syzbot report [1] with no reproducer hints at a bug in ip6_gre tunnel (dev:ip6gretap0) Since ipv6 mcast code makes sure to read dev->mtu once and applies a sanity check on it (see commit b9b312a7a451 "ipv6: mcast: better catch silly mtu values"), a remaining possibility is that a layer is able to set dev->mtu to an underflowed value (high order bit set). This could happen indeed in ip6gre_tnl_link_config_route(), ip6_tnl_link_config() and ipip6_tunnel_bind_dev() Make sure to sanitize mtu value in a local variable before it is written once on dev->mtu, as lockless readers could catch wrong temporary value. [1] skbuff: skb_over_panic: text:ffff80000b7a2f38 len:40 put:40 head:ffff000149dcf200 data:ffff000149dcf2b0 tail:0xd8 end:0xc0 dev:ip6gretap0 ------------[ cut here ]------------ kernel BUG at net/core/skbuff.c:120 Internal error: Oops - BUG: 00000000f2000800 [#1] PREEMPT SMP Modules linked in: CPU: 1 PID: 10241 Comm: kworker/1:1 Not tainted 6.0.0-rc7-syzkaller-18095-gbbed346d5a96 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/30/2022 Workqueue: mld mld_ifc_work pstate: 60400005 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : skb_panic+0x4c/0x50 net/core/skbuff.c:116 lr : skb_panic+0x4c/0x50 net/core/skbuff.c:116 sp : ffff800020dd3b60 x29: ffff800020dd3b70 x28: 0000000000000000 x27: ffff00010df2a800 x26: 00000000000000c0 x25: 00000000000000b0 x24: ffff000149dcf200 x23: 00000000000000c0 x22: 00000000000000d8 x21: ffff80000b7a2f38 x20: ffff00014c2f7800 x19: 0000000000000028 x18: 00000000000001a9 x17: 0000000000000000 x16: ffff80000db49158 x15: ffff000113bf1a80 x14: 0000000000000000 x13: 00000000ffffffff x12: ffff000113bf1a80 x11: ff808000081c0d5c x10: 0000000000000000 x9 : 73f125dc5c63ba00 x8 : 73f125dc5c63ba00 x7 : ffff800008161d1c x6 : 0000000000000000 x5 : 0000000000000080 x4 : 0000000000000001 x3 : 0000000000000000 x2 : ffff0001fefddcd0 x1 : 0000000100000000 x0 : 0000000000000089 Call trace: skb_panic+0x4c/0x50 net/core/skbuff.c:116 skb_over_panic net/core/skbuff.c:125 [inline] skb_put+0xd4/0xdc net/core/skbuff.c:2049 ip6_mc_hdr net/ipv6/mcast.c:1714 [inline] mld_newpack+0x14c/0x270 net/ipv6/mcast.c:1765 add_grhead net/ipv6/mcast.c:1851 [inline] add_grec+0xa20/0xae0 net/ipv6/mcast.c:1989 mld_send_cr+0x438/0x5a8 net/ipv6/mcast.c:2115 mld_ifc_work+0x38/0x290 net/ipv6/mcast.c:2653 process_one_work+0x2d8/0x504 kernel/workqueue.c:2289 worker_thread+0x340/0x610 kernel/workqueue.c:2436 kthread+0x12c/0x158 kernel/kthread.c:376 ret_from_fork+0x10/0x20 arch/arm64/kernel/entry.S:860 Code: 91011400 aa0803e1 a90027ea 94373093 (d4210000) Fixes: c12b395a4664 ("gre: Support GRE over IPv6") Reported-by: syzbot Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20221024020124.3756833-1-eric.dumazet@gmail.com Signed-off-by: Jakub Kicinski --- net/ipv6/ip6_gre.c | 12 +++++++----- net/ipv6/ip6_tunnel.c | 11 ++++++----- net/ipv6/sit.c | 8 +++++--- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 48b4ff0294f6c..c035a96fba3a4 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -1175,14 +1175,16 @@ static void ip6gre_tnl_link_config_route(struct ip6_tnl *t, int set_mtu, dev->needed_headroom = dst_len; if (set_mtu) { - dev->mtu = rt->dst.dev->mtu - t_hlen; + int mtu = rt->dst.dev->mtu - t_hlen; + if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT)) - dev->mtu -= 8; + mtu -= 8; if (dev->type == ARPHRD_ETHER) - dev->mtu -= ETH_HLEN; + mtu -= ETH_HLEN; - if (dev->mtu < IPV6_MIN_MTU) - dev->mtu = IPV6_MIN_MTU; + if (mtu < IPV6_MIN_MTU) + mtu = IPV6_MIN_MTU; + WRITE_ONCE(dev->mtu, mtu); } } ip6_rt_put(rt); diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index cc5d5e75b658f..2fb4c6ad72432 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -1450,8 +1450,8 @@ static void ip6_tnl_link_config(struct ip6_tnl *t) struct net_device *tdev = NULL; struct __ip6_tnl_parm *p = &t->parms; struct flowi6 *fl6 = &t->fl.u.ip6; - unsigned int mtu; int t_hlen; + int mtu; __dev_addr_set(dev, &p->laddr, sizeof(struct in6_addr)); memcpy(dev->broadcast, &p->raddr, sizeof(struct in6_addr)); @@ -1498,12 +1498,13 @@ static void ip6_tnl_link_config(struct ip6_tnl *t) dev->hard_header_len = tdev->hard_header_len + t_hlen; mtu = min_t(unsigned int, tdev->mtu, IP6_MAX_MTU); - dev->mtu = mtu - t_hlen; + mtu = mtu - t_hlen; if (!(t->parms.flags & IP6_TNL_F_IGN_ENCAP_LIMIT)) - dev->mtu -= 8; + mtu -= 8; - if (dev->mtu < IPV6_MIN_MTU) - dev->mtu = IPV6_MIN_MTU; + if (mtu < IPV6_MIN_MTU) + mtu = IPV6_MIN_MTU; + WRITE_ONCE(dev->mtu, mtu); } } } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index d27683e3fc971..5703d3cbea9ba 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -1124,10 +1124,12 @@ static void ipip6_tunnel_bind_dev(struct net_device *dev) if (tdev && !netif_is_l3_master(tdev)) { int t_hlen = tunnel->hlen + sizeof(struct iphdr); + int mtu; - dev->mtu = tdev->mtu - t_hlen; - if (dev->mtu < IPV6_MIN_MTU) - dev->mtu = IPV6_MIN_MTU; + mtu = tdev->mtu - t_hlen; + if (mtu < IPV6_MIN_MTU) + mtu = IPV6_MIN_MTU; + WRITE_ONCE(dev->mtu, mtu); } } -- GitLab From 54b5af5a438076082d482cab105b1bd484ab5074 Mon Sep 17 00:00:00 2001 From: Slawomir Laba Date: Mon, 24 Oct 2022 03:05:24 -0700 Subject: [PATCH 0482/1927] i40e: Fix ethtool rx-flow-hash setting for X722 When enabling flow type for RSS hash via ethtool: ethtool -N $pf rx-flow-hash tcp4|tcp6|udp4|udp6 s|d the driver would fail to setup this setting on X722 device since it was using the mask on the register dedicated for X710 devices. Apply a different mask on the register when setting the RSS hash for the X722 device. When displaying the flow types enabled via ethtool: ethtool -n $pf rx-flow-hash tcp4|tcp6|udp4|udp6 the driver would print wrong values for X722 device. Fix this issue by testing masks for X722 device in i40e_get_rss_hash_opts function. Fixes: eb0dd6e4a3b3 ("i40e: Allow RSS Hash set with less than four parameters") Signed-off-by: Slawomir Laba Signed-off-by: Michal Jaron Signed-off-by: Mateusz Palczewski Tested-by: Gurucharan (A Contingent worker at Intel) Signed-off-by: Jacob Keller Link: https://lore.kernel.org/r/20221024100526.1874914-1-jacob.e.keller@intel.com Signed-off-by: Jakub Kicinski --- .../net/ethernet/intel/i40e/i40e_ethtool.c | 31 ++++++++++++++----- drivers/net/ethernet/intel/i40e/i40e_type.h | 4 +++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 87f36d1ce8008..314ef40aa2606 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -3185,10 +3185,17 @@ static int i40e_get_rss_hash_opts(struct i40e_pf *pf, struct ethtool_rxnfc *cmd) if (cmd->flow_type == TCP_V4_FLOW || cmd->flow_type == UDP_V4_FLOW) { - if (i_set & I40E_L3_SRC_MASK) - cmd->data |= RXH_IP_SRC; - if (i_set & I40E_L3_DST_MASK) - cmd->data |= RXH_IP_DST; + if (hw->mac.type == I40E_MAC_X722) { + if (i_set & I40E_X722_L3_SRC_MASK) + cmd->data |= RXH_IP_SRC; + if (i_set & I40E_X722_L3_DST_MASK) + cmd->data |= RXH_IP_DST; + } else { + if (i_set & I40E_L3_SRC_MASK) + cmd->data |= RXH_IP_SRC; + if (i_set & I40E_L3_DST_MASK) + cmd->data |= RXH_IP_DST; + } } else if (cmd->flow_type == TCP_V6_FLOW || cmd->flow_type == UDP_V6_FLOW) { if (i_set & I40E_L3_V6_SRC_MASK) @@ -3546,12 +3553,15 @@ static int i40e_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, /** * i40e_get_rss_hash_bits - Read RSS Hash bits from register + * @hw: hw structure * @nfc: pointer to user request * @i_setc: bits currently set * * Returns value of bits to be set per user request **/ -static u64 i40e_get_rss_hash_bits(struct ethtool_rxnfc *nfc, u64 i_setc) +static u64 i40e_get_rss_hash_bits(struct i40e_hw *hw, + struct ethtool_rxnfc *nfc, + u64 i_setc) { u64 i_set = i_setc; u64 src_l3 = 0, dst_l3 = 0; @@ -3570,8 +3580,13 @@ static u64 i40e_get_rss_hash_bits(struct ethtool_rxnfc *nfc, u64 i_setc) dst_l3 = I40E_L3_V6_DST_MASK; } else if (nfc->flow_type == TCP_V4_FLOW || nfc->flow_type == UDP_V4_FLOW) { - src_l3 = I40E_L3_SRC_MASK; - dst_l3 = I40E_L3_DST_MASK; + if (hw->mac.type == I40E_MAC_X722) { + src_l3 = I40E_X722_L3_SRC_MASK; + dst_l3 = I40E_X722_L3_DST_MASK; + } else { + src_l3 = I40E_L3_SRC_MASK; + dst_l3 = I40E_L3_DST_MASK; + } } else { /* Any other flow type are not supported here */ return i_set; @@ -3686,7 +3701,7 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc) flow_pctype)) | ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_pctype)) << 32); - i_set = i40e_get_rss_hash_bits(nfc, i_setc); + i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc); i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_pctype), (u32)i_set); i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_pctype), diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h index 7b3f30beb757a..388c3d36d96a5 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_type.h +++ b/drivers/net/ethernet/intel/i40e/i40e_type.h @@ -1404,6 +1404,10 @@ struct i40e_lldp_variables { #define I40E_PFQF_CTL_0_HASHLUTSIZE_512 0x00010000 /* INPUT SET MASK for RSS, flow director, and flexible payload */ +#define I40E_X722_L3_SRC_SHIFT 49 +#define I40E_X722_L3_SRC_MASK (0x3ULL << I40E_X722_L3_SRC_SHIFT) +#define I40E_X722_L3_DST_SHIFT 41 +#define I40E_X722_L3_DST_MASK (0x3ULL << I40E_X722_L3_DST_SHIFT) #define I40E_L3_SRC_SHIFT 47 #define I40E_L3_SRC_MASK (0x3ULL << I40E_L3_SRC_SHIFT) #define I40E_L3_V6_SRC_SHIFT 43 -- GitLab From 52424f974bc53c26ba3f00300a00e9de9afcd972 Mon Sep 17 00:00:00 2001 From: Sylwester Dziedziuch Date: Mon, 24 Oct 2022 03:05:25 -0700 Subject: [PATCH 0483/1927] i40e: Fix VF hang when reset is triggered on another VF When a reset was triggered on one VF with i40e_reset_vf global PF state __I40E_VF_DISABLE was set on a PF until the reset finished. If immediately after triggering reset on one VF there is a request to reset on another it will cause a hang on VF side because VF will be notified of incoming reset but the reset will never happen because of this global state, we will get such error message: [ +4.890195] iavf 0000:86:02.1: Never saw reset and VF will hang waiting for the reset to be triggered. Fix this by introducing new VF state I40E_VF_STATE_RESETTING that will be set on a VF if it is currently resetting instead of the global __I40E_VF_DISABLE PF state. Fixes: 3ba9bcb4b68f ("i40e: add locking around VF reset") Signed-off-by: Sylwester Dziedziuch Signed-off-by: Mateusz Palczewski Tested-by: Konrad Jankowski Signed-off-by: Jacob Keller Link: https://lore.kernel.org/r/20221024100526.1874914-2-jacob.e.keller@intel.com Signed-off-by: Jakub Kicinski --- .../ethernet/intel/i40e/i40e_virtchnl_pf.c | 43 ++++++++++++++----- .../ethernet/intel/i40e/i40e_virtchnl_pf.h | 1 + 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index 7e9f6a69eb10c..72ddcefc45b1e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -1536,10 +1536,12 @@ bool i40e_reset_vf(struct i40e_vf *vf, bool flr) if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) return true; - /* If the VFs have been disabled, this means something else is - * resetting the VF, so we shouldn't continue. - */ - if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) + /* Bail out if VFs are disabled. */ + if (test_bit(__I40E_VF_DISABLE, pf->state)) + return true; + + /* If VF is being reset already we don't need to continue. */ + if (test_and_set_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) return true; i40e_trigger_vf_reset(vf, flr); @@ -1576,7 +1578,7 @@ bool i40e_reset_vf(struct i40e_vf *vf, bool flr) i40e_cleanup_reset_vf(vf); i40e_flush(hw); - clear_bit(__I40E_VF_DISABLE, pf->state); + clear_bit(I40E_VF_STATE_RESETTING, &vf->vf_states); return true; } @@ -1609,8 +1611,12 @@ bool i40e_reset_all_vfs(struct i40e_pf *pf, bool flr) return false; /* Begin reset on all VFs at once */ - for (v = 0; v < pf->num_alloc_vfs; v++) - i40e_trigger_vf_reset(&pf->vf[v], flr); + for (v = 0; v < pf->num_alloc_vfs; v++) { + vf = &pf->vf[v]; + /* If VF is being reset no need to trigger reset again */ + if (!test_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) + i40e_trigger_vf_reset(&pf->vf[v], flr); + } /* HW requires some time to make sure it can flush the FIFO for a VF * when it resets it. Poll the VPGEN_VFRSTAT register for each VF in @@ -1626,9 +1632,11 @@ bool i40e_reset_all_vfs(struct i40e_pf *pf, bool flr) */ while (v < pf->num_alloc_vfs) { vf = &pf->vf[v]; - reg = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_id)); - if (!(reg & I40E_VPGEN_VFRSTAT_VFRD_MASK)) - break; + if (!test_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) { + reg = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_id)); + if (!(reg & I40E_VPGEN_VFRSTAT_VFRD_MASK)) + break; + } /* If the current VF has finished resetting, move on * to the next VF in sequence. @@ -1656,6 +1664,10 @@ bool i40e_reset_all_vfs(struct i40e_pf *pf, bool flr) if (pf->vf[v].lan_vsi_idx == 0) continue; + /* If VF is reset in another thread just continue */ + if (test_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) + continue; + i40e_vsi_stop_rings_no_wait(pf->vsi[pf->vf[v].lan_vsi_idx]); } @@ -1667,6 +1679,10 @@ bool i40e_reset_all_vfs(struct i40e_pf *pf, bool flr) if (pf->vf[v].lan_vsi_idx == 0) continue; + /* If VF is reset in another thread just continue */ + if (test_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) + continue; + i40e_vsi_wait_queues_disabled(pf->vsi[pf->vf[v].lan_vsi_idx]); } @@ -1676,8 +1692,13 @@ bool i40e_reset_all_vfs(struct i40e_pf *pf, bool flr) mdelay(50); /* Finish the reset on each VF */ - for (v = 0; v < pf->num_alloc_vfs; v++) + for (v = 0; v < pf->num_alloc_vfs; v++) { + /* If VF is reset in another thread just continue */ + if (test_bit(I40E_VF_STATE_RESETTING, &vf->vf_states)) + continue; + i40e_cleanup_reset_vf(&pf->vf[v]); + } i40e_flush(hw); clear_bit(__I40E_VF_DISABLE, pf->state); diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h index a554d0a0b09bd..358bbdb587951 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h @@ -39,6 +39,7 @@ enum i40e_vf_states { I40E_VF_STATE_MC_PROMISC, I40E_VF_STATE_UC_PROMISC, I40E_VF_STATE_PRE_ENABLE, + I40E_VF_STATE_RESETTING }; /* VF capabilities */ -- GitLab From 3b32c9932853e11d71f9db012d69e92e4669ba23 Mon Sep 17 00:00:00 2001 From: Slawomir Laba Date: Mon, 24 Oct 2022 03:05:26 -0700 Subject: [PATCH 0484/1927] i40e: Fix flow-type by setting GL_HASH_INSET registers Fix setting bits for specific flow_type for GLQF_HASH_INSET register. In previous version all of the bits were set only in hena register, while in inset only one bit was set. In order for this working correctly on all types of cards these bits needs to be set correctly for both hena and inset registers. Fixes: eb0dd6e4a3b3 ("i40e: Allow RSS Hash set with less than four parameters") Signed-off-by: Slawomir Laba Signed-off-by: Michal Jaron Signed-off-by: Mateusz Palczewski Signed-off-by: Jacob Keller Link: https://lore.kernel.org/r/20221024100526.1874914-3-jacob.e.keller@intel.com Signed-off-by: Jakub Kicinski --- .../net/ethernet/intel/i40e/i40e_ethtool.c | 71 ++++++++++--------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 314ef40aa2606..4a6a6e48c615b 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -3604,6 +3604,7 @@ static u64 i40e_get_rss_hash_bits(struct i40e_hw *hw, return i_set; } +#define FLOW_PCTYPES_SIZE 64 /** * i40e_set_rss_hash_opt - Enable/Disable flow types for RSS hash * @pf: pointer to the physical function struct @@ -3616,9 +3617,11 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc) struct i40e_hw *hw = &pf->hw; u64 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); - u8 flow_pctype = 0; + DECLARE_BITMAP(flow_pctypes, FLOW_PCTYPES_SIZE); u64 i_set, i_setc; + bitmap_zero(flow_pctypes, FLOW_PCTYPES_SIZE); + if (pf->flags & I40E_FLAG_MFP_ENABLED) { dev_err(&pf->pdev->dev, "Change of RSS hash input set is not supported when MFP mode is enabled\n"); @@ -3634,36 +3637,35 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc) switch (nfc->flow_type) { case TCP_V4_FLOW: - flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP; + set_bit(I40E_FILTER_PCTYPE_NONF_IPV4_TCP, flow_pctypes); if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) - hena |= - BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK); + set_bit(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK, + flow_pctypes); break; case TCP_V6_FLOW: - flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_TCP; - if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) - hena |= - BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK); + set_bit(I40E_FILTER_PCTYPE_NONF_IPV6_TCP, flow_pctypes); if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) - hena |= - BIT_ULL(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK); + set_bit(I40E_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK, + flow_pctypes); break; case UDP_V4_FLOW: - flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; - if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) - hena |= - BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP) | - BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP); - + set_bit(I40E_FILTER_PCTYPE_NONF_IPV4_UDP, flow_pctypes); + if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) { + set_bit(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP, + flow_pctypes); + set_bit(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP, + flow_pctypes); + } hena |= BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV4); break; case UDP_V6_FLOW: - flow_pctype = I40E_FILTER_PCTYPE_NONF_IPV6_UDP; - if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) - hena |= - BIT_ULL(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP) | - BIT_ULL(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP); - + set_bit(I40E_FILTER_PCTYPE_NONF_IPV6_UDP, flow_pctypes); + if (pf->hw_features & I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE) { + set_bit(I40E_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP, + flow_pctypes); + set_bit(I40E_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP, + flow_pctypes); + } hena |= BIT_ULL(I40E_FILTER_PCTYPE_FRAG_IPV6); break; case AH_ESP_V4_FLOW: @@ -3696,17 +3698,20 @@ static int i40e_set_rss_hash_opt(struct i40e_pf *pf, struct ethtool_rxnfc *nfc) return -EINVAL; } - if (flow_pctype) { - i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, - flow_pctype)) | - ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, - flow_pctype)) << 32); - i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc); - i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_pctype), - (u32)i_set); - i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_pctype), - (u32)(i_set >> 32)); - hena |= BIT_ULL(flow_pctype); + if (bitmap_weight(flow_pctypes, FLOW_PCTYPES_SIZE)) { + u8 flow_id; + + for_each_set_bit(flow_id, flow_pctypes, FLOW_PCTYPES_SIZE) { + i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id)) | + ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id)) << 32); + i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc); + + i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id), + (u32)i_set); + i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id), + (u32)(i_set >> 32)); + hena |= BIT_ULL(flow_id); + } } i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); -- GitLab From 5da6d65590a0698199df44d095e54b0ed1708178 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 24 Oct 2022 21:13:38 +0800 Subject: [PATCH 0485/1927] net: ksz884x: fix missing pci_disable_device() on error in pcidev_init() pci_disable_device() need be called while module exiting, switch to use pcim_enable(), pci_disable_device() will be called in pcim_release() while unbinding device. Fixes: 8ca86fd83eae ("net: Micrel KSZ8841/2 PCI Ethernet driver") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221024131338.2848959-1-yangyingliang@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/micrel/ksz884x.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c index 468520079c65e..e6acd1e7b263a 100644 --- a/drivers/net/ethernet/micrel/ksz884x.c +++ b/drivers/net/ethernet/micrel/ksz884x.c @@ -6851,7 +6851,7 @@ static int pcidev_init(struct pci_dev *pdev, const struct pci_device_id *id) char banner[sizeof(version)]; struct ksz_switch *sw = NULL; - result = pci_enable_device(pdev); + result = pcim_enable_device(pdev); if (result) return result; -- GitLab From f23a566bbfc0896c97b1949216eb87fcdcb154bb Mon Sep 17 00:00:00 2001 From: Caleb Connolly Date: Mon, 24 Oct 2022 22:03:31 +0100 Subject: [PATCH 0486/1927] net: ipa: fix v3.5.1 resource limit max values Some resource limits on IPA v3.5.1 have their max values set to 255, this causes a few splats in ipa_reg_encode and prevents the IPA from booting properly. The limits are all 6 bits wide so adjust the max values to 63. Fixes: 1c418c4a929c ("net: ipa: define resource group/type IPA register fields") Signed-off-by: Caleb Connolly Reviewed-by: Alex Elder Link: https://lore.kernel.org/r/20221024210336.4014983-1-caleb.connolly@linaro.org Signed-off-by: Jakub Kicinski --- drivers/net/ipa/data/ipa_data-v3.5.1.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ipa/data/ipa_data-v3.5.1.c b/drivers/net/ipa/data/ipa_data-v3.5.1.c index 383ef18900654..42f2c88a92d4b 100644 --- a/drivers/net/ipa/data/ipa_data-v3.5.1.c +++ b/drivers/net/ipa/data/ipa_data-v3.5.1.c @@ -179,10 +179,10 @@ static const struct ipa_gsi_endpoint_data ipa_gsi_endpoint_data[] = { static const struct ipa_resource ipa_resource_src[] = { [IPA_RESOURCE_TYPE_SRC_PKT_CONTEXTS] = { .limits[IPA_RSRC_GROUP_SRC_LWA_DL] = { - .min = 1, .max = 255, + .min = 1, .max = 63, }, .limits[IPA_RSRC_GROUP_SRC_UL_DL] = { - .min = 1, .max = 255, + .min = 1, .max = 63, }, .limits[IPA_RSRC_GROUP_SRC_UC_RX_Q] = { .min = 1, .max = 63, -- GitLab From 05a31b94af3226ee47dcb6802229a7a576105d47 Mon Sep 17 00:00:00 2001 From: Caleb Connolly Date: Mon, 24 Oct 2022 22:03:32 +0100 Subject: [PATCH 0487/1927] net: ipa: fix v3.1 resource limit masks The resource group limits for IPA v3.1 mistakenly used 6 bit wide mask values, when the hardware actually uses 8. Out of range values were silently ignored before, so the IPA worked as expected. However the new generalised register definitions introduce stricter checking here, they now cause some splats and result in the value 0 being written instead. Fix the limit bitmask widths so that the correct values can be written. Fixes: 1c418c4a929c ("net: ipa: define resource group/type IPA register fields") Signed-off-by: Caleb Connolly Reviewed-by: Alex Elder Tested-by: Jami Kettunen Link: https://lore.kernel.org/r/20221024210336.4014983-2-caleb.connolly@linaro.org Signed-off-by: Jakub Kicinski --- drivers/net/ipa/reg/ipa_reg-v3.1.c | 96 ++++++++++-------------------- 1 file changed, 32 insertions(+), 64 deletions(-) diff --git a/drivers/net/ipa/reg/ipa_reg-v3.1.c b/drivers/net/ipa/reg/ipa_reg-v3.1.c index 116b27717e3d7..0d002c3c38a26 100644 --- a/drivers/net/ipa/reg/ipa_reg-v3.1.c +++ b/drivers/net/ipa/reg/ipa_reg-v3.1.c @@ -127,112 +127,80 @@ static const u32 ipa_reg_counter_cfg_fmask[] = { IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0); static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type, 0x00000400, 0x0020); static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type, 0x00000404, 0x0020); static const u32 ipa_reg_src_rsrc_grp_45_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_45_RSRC_TYPE, src_rsrc_grp_45_rsrc_type, 0x00000408, 0x0020); static const u32 ipa_reg_src_rsrc_grp_67_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_67_RSRC_TYPE, src_rsrc_grp_67_rsrc_type, 0x0000040c, 0x0020); static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type, 0x00000500, 0x0020); static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type, 0x00000504, 0x0020); static const u32 ipa_reg_dst_rsrc_grp_45_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_45_RSRC_TYPE, dst_rsrc_grp_45_rsrc_type, 0x00000508, 0x0020); static const u32 ipa_reg_dst_rsrc_grp_67_rsrc_type_fmask[] = { - [X_MIN_LIM] = GENMASK(5, 0), - /* Bits 6-7 reserved */ - [X_MAX_LIM] = GENMASK(13, 8), - /* Bits 14-15 reserved */ - [Y_MIN_LIM] = GENMASK(21, 16), - /* Bits 22-23 reserved */ - [Y_MAX_LIM] = GENMASK(29, 24), - /* Bits 30-31 reserved */ + [X_MIN_LIM] = GENMASK(7, 0), + [X_MAX_LIM] = GENMASK(15, 8), + [Y_MIN_LIM] = GENMASK(23, 16), + [Y_MAX_LIM] = GENMASK(31, 24), }; IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_67_RSRC_TYPE, dst_rsrc_grp_67_rsrc_type, -- GitLab From 95a0396a0642d3c28b6cefdc76697e0b8f594825 Mon Sep 17 00:00:00 2001 From: Caleb Connolly Date: Tue, 25 Oct 2022 00:48:50 +0100 Subject: [PATCH 0488/1927] net: ipa: don't configure IDLE_INDICATION on v3.1 IPA v3.1 doesn't support the IDLE_INDICATION_CFG register, this was causing a harmless splat in ipa_idle_indication_cfg(), add a version check to prevent trying to fetch this register on v3.1 Fixes: 6a244b75cfab ("net: ipa: introduce ipa_reg()") Signed-off-by: Caleb Connolly Reviewed-by: Alex Elder Tested-by: Jami Kettunen Link: https://lore.kernel.org/r/20221024234850.4049778-1-caleb.connolly@linaro.org Signed-off-by: Jakub Kicinski --- drivers/net/ipa/ipa_main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 3461ad3029ab8..49537fccf6ad0 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -434,6 +434,9 @@ static void ipa_idle_indication_cfg(struct ipa *ipa, const struct ipa_reg *reg; u32 val; + if (ipa->version < IPA_VERSION_3_5_1) + return; + reg = ipa_reg(ipa, IDLE_INDICATION_CFG); val = ipa_reg_encode(reg, ENTER_IDLE_DEBOUNCE_THRESH, enter_idle_debounce_thresh); -- GitLab From 89fd4a1df829187d4d35f6a520cc531de622e6f0 Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sat, 8 Oct 2022 22:54:37 +0800 Subject: [PATCH 0489/1927] riscv: jump_label: mark arguments as const to satisfy asm constraints Samuel reported that the static branch usage in cpu_relax() breaks building with CONFIG_CC_OPTIMIZE_FOR_SIZE: In file included from : ./arch/riscv/include/asm/jump_label.h: In function 'cpu_relax': ././include/linux/compiler_types.h:285:33: warning: 'asm' operand 0 probably does not match constraints 285 | #define asm_volatile_goto(x...) asm goto(x) | ^~~ ./arch/riscv/include/asm/jump_label.h:41:9: note: in expansion of macro 'asm_volatile_goto' 41 | asm_volatile_goto( | ^~~~~~~~~~~~~~~~~ ././include/linux/compiler_types.h:285:33: error: impossible constraint in 'asm' 285 | #define asm_volatile_goto(x...) asm goto(x) | ^~~ ./arch/riscv/include/asm/jump_label.h:41:9: note: in expansion of macro 'asm_volatile_goto' 41 | asm_volatile_goto( | ^~~~~~~~~~~~~~~~~ make[1]: *** [scripts/Makefile.build:249: arch/riscv/kernel/vdso/vgettimeofday.o] Error 1 make: *** [arch/riscv/Makefile:128: vdso_prepare] Error 2 Maybe "-Os" prevents GCC from detecting that the key/branch arguments can be treated as constants and used as immediate operands. Inspired by x86's commit 864b435514b2("x86/jump_label: Mark arguments as const to satisfy asm constraints"), and as pointed out by Steven: "The "i" constraint needs to be a constant.", let's do similar modifications to riscv. Tested by CC_OPTIMIZE_FOR_SIZE + gcc and CC_OPTIMIZE_FOR_SIZE + clang. Link: https://lore.kernel.org/linux-riscv/20220922060958.44203-1-samuel@sholland.org/ Link: https://lore.kernel.org/all/20210212094059.5f8d05e8@gandalf.local.home/ Fixes: 8eb060e10185 ("arch/riscv: add Zihintpause support") Reported-by: Samuel Holland Signed-off-by: Jisheng Zhang Reviewed-by: Andrew Jones Tested-by: Heiko Stuebner Link: https://lore.kernel.org/r/20221008145437.491-1-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/include/asm/jump_label.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 38af2ec7b9bf9..6d58bbb5da467 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -14,8 +14,8 @@ #define JUMP_LABEL_NOP_SIZE 4 -static __always_inline bool arch_static_branch(struct static_key *key, - bool branch) +static __always_inline bool arch_static_branch(struct static_key * const key, + const bool branch) { asm_volatile_goto( " .option push \n\t" @@ -35,8 +35,8 @@ static __always_inline bool arch_static_branch(struct static_key *key, return true; } -static __always_inline bool arch_static_branch_jump(struct static_key *key, - bool branch) +static __always_inline bool arch_static_branch_jump(struct static_key * const key, + const bool branch) { asm_volatile_goto( " .option push \n\t" -- GitLab From 389ec68c83ee142f2edde954751fb67dafb5be32 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 13 Oct 2022 14:46:37 -0700 Subject: [PATCH 0490/1927] MAINTAINERS: git://github.com -> https://github.com for sifive Github deprecated the git:// links about a year ago, so let's move to the https:// URLs instead. Reported-by: Conor Dooley Link: https://github.blog/2021-09-01-improving-git-protocol-security-github/ Signed-off-by: Palmer Dabbelt --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..5b1827daffb98 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -18777,7 +18777,7 @@ M: Palmer Dabbelt M: Paul Walmsley L: linux-riscv@lists.infradead.org S: Supported -T: git git://github.com/sifive/riscv-linux.git +T: git https://github.com/sifive/riscv-linux.git N: sifive K: [^@]sifive -- GitLab From 8d280b1df87e0b3d1355aeac7e62b62214b93f1c Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Mon, 24 Oct 2022 11:02:59 -0700 Subject: [PATCH 0491/1927] mmc: sdhci_am654: 'select', not 'depends' REGMAP_MMIO REGMAP_MMIO is not user-configurable, so we can only satisfy this dependency by enabling some other Kconfig symbol that properly 'select's it. Use select like everybody else. Noticed when trying to enable this driver for compile testing. Fixes: 59592cc1f593 ("mmc: sdhci_am654: Add dependency on MMC_SDHCI_AM654") Signed-off-by: Brian Norris Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221024180300.2292208-1-briannorris@chromium.org Signed-off-by: Ulf Hansson --- drivers/mmc/host/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig index f324daadaf701..fb1062a6394c1 100644 --- a/drivers/mmc/host/Kconfig +++ b/drivers/mmc/host/Kconfig @@ -1075,9 +1075,10 @@ config MMC_SDHCI_OMAP config MMC_SDHCI_AM654 tristate "Support for the SDHCI Controller in TI's AM654 SOCs" - depends on MMC_SDHCI_PLTFM && OF && REGMAP_MMIO + depends on MMC_SDHCI_PLTFM && OF select MMC_SDHCI_IO_ACCESSORS select MMC_CQHCI + select REGMAP_MMIO help This selects the Secure Digital Host Controller Interface (SDHCI) support present in TI's AM654 SOCs. The controller supports -- GitLab From a9003f74f5a2f487e101f3aa1dd5c3d3a78c6999 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 7 Oct 2022 15:10:01 +0200 Subject: [PATCH 0492/1927] clk: renesas: r8a779g0: Fix HSCIF parent clocks As serial communication requires a clean clock signal, the High Speed Serial Communication Interfaces with FIFO (HSCIF) is clocked by a clock that is not affected by Spread Spectrum or Fractional Multiplication. Hence change the parent clocks for the HSCIF modules from the S0D3_PER clock to the SASYNCPERD1 clock (which has the same clock rate), cfr. R-Car V4H Hardware User's Manual rev. 0.54. Fixes: 0ab55cf1834177a2 ("clk: renesas: cpg-mssr: Add support for R-Car V4H") Signed-off-by: Geert Uytterhoeven Acked-by: Stephen Boyd Link: https://lore.kernel.org/r/b7928abc8b9f53d5b06ec8624342f449de3d24ec.1665147497.git.geert+renesas@glider.be --- drivers/clk/renesas/r8a779g0-cpg-mssr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/clk/renesas/r8a779g0-cpg-mssr.c b/drivers/clk/renesas/r8a779g0-cpg-mssr.c index 3e8c93facfa14..d5b325e3c5398 100644 --- a/drivers/clk/renesas/r8a779g0-cpg-mssr.c +++ b/drivers/clk/renesas/r8a779g0-cpg-mssr.c @@ -158,10 +158,10 @@ static const struct mssr_mod_clk r8a779g0_mod_clks[] __initconst = { DEF_MOD("avb0", 211, R8A779G0_CLK_S0D4_HSC), DEF_MOD("avb1", 212, R8A779G0_CLK_S0D4_HSC), DEF_MOD("avb2", 213, R8A779G0_CLK_S0D4_HSC), - DEF_MOD("hscif0", 514, R8A779G0_CLK_S0D3_PER), - DEF_MOD("hscif1", 515, R8A779G0_CLK_S0D3_PER), - DEF_MOD("hscif2", 516, R8A779G0_CLK_S0D3_PER), - DEF_MOD("hscif3", 517, R8A779G0_CLK_S0D3_PER), + DEF_MOD("hscif0", 514, R8A779G0_CLK_SASYNCPERD1), + DEF_MOD("hscif1", 515, R8A779G0_CLK_SASYNCPERD1), + DEF_MOD("hscif2", 516, R8A779G0_CLK_SASYNCPERD1), + DEF_MOD("hscif3", 517, R8A779G0_CLK_SASYNCPERD1), DEF_MOD("i2c0", 518, R8A779G0_CLK_S0D6_PER), DEF_MOD("i2c1", 519, R8A779G0_CLK_S0D6_PER), DEF_MOD("i2c2", 520, R8A779G0_CLK_S0D6_PER), -- GitLab From e0c57a5c70c13317238cb19a7ded0eab4a5f7de5 Mon Sep 17 00:00:00 2001 From: Sudeep Holla Date: Tue, 25 Oct 2022 13:34:32 +0100 Subject: [PATCH 0493/1927] PM: domains: Fix handling of unavailable/disabled idle states Platforms can provide the information about the availability of each idle states via status flag. Platforms may have to disable one or more idle states for various reasons like broken firmware or other unmet dependencies. Fix handling of such unavailable/disabled idle states by ignoring them while parsing the states. Fixes: a3381e3a65cb ("PM / domains: Fix up domain-idle-states OF parsing") Signed-off-by: Sudeep Holla Reviewed-by: Ulf Hansson Signed-off-by: Rafael J. Wysocki --- drivers/base/power/domain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index ead135c7044c6..6471b559230e9 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -2952,6 +2952,10 @@ static int genpd_iterate_idle_states(struct device_node *dn, np = it.node; if (!of_match_node(idle_state_match, np)) continue; + + if (!of_device_is_available(np)) + continue; + if (states) { ret = genpd_parse_state(&states[i], np); if (ret) { -- GitLab From 8338b74a750c534c223e8943cc0ed0e198ece261 Mon Sep 17 00:00:00 2001 From: Manank Patel Date: Tue, 18 Oct 2022 00:13:39 +0530 Subject: [PATCH 0494/1927] ACPI: PCC: Fix unintentional integer overflow Fix an unintentional u32 overflow by changing PCC_CMD_WAIT_RETRIES_NUM to 500ULL. Fixes: 91cefefb6991 ("ACPI: PCC: replace wait_for_completion()") Signed-off-by: Manank Patel Acked-by: Sudeep Holla Acked-by: Huisong Li Signed-off-by: Rafael J. Wysocki --- drivers/acpi/acpi_pcc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c index ee4ce5ba1fb24..3e252be047b85 100644 --- a/drivers/acpi/acpi_pcc.c +++ b/drivers/acpi/acpi_pcc.c @@ -27,7 +27,7 @@ * Arbitrary retries in case the remote processor is slow to respond * to PCC commands */ -#define PCC_CMD_WAIT_RETRIES_NUM 500 +#define PCC_CMD_WAIT_RETRIES_NUM 500ULL struct pcc_data { struct pcc_mbox_chan *pcc_chan; -- GitLab From b5f9223a105d9b56954ad1ca3eace4eaf26c99ed Mon Sep 17 00:00:00 2001 From: Tamim Khan Date: Fri, 14 Oct 2022 01:19:31 -0400 Subject: [PATCH 0495/1927] ACPI: resource: Skip IRQ override on Asus Vivobook S5602ZA Like the Asus Vivobook K3402ZA/K3502ZA/S5402ZA Asus Vivobook S5602ZA has an ACPI DSDT table the describes IRQ 1 as ActiveLow while the kernel overrides it to Edge_High. This prevents the keyboard on this laptop from working. To fix this add this laptop to the skip_override_table so that the kernel does not override IRQ 1. Link: https://bugzilla.kernel.org/show_bug.cgi?id=216579 Tested-by: Dzmitry Signed-off-by: Tamim Khan Signed-off-by: Rafael J. Wysocki --- drivers/acpi/resource.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c index 78c2804164c6f..f27914aedbd5a 100644 --- a/drivers/acpi/resource.c +++ b/drivers/acpi/resource.c @@ -425,6 +425,13 @@ static const struct dmi_system_id asus_laptop[] = { DMI_MATCH(DMI_BOARD_NAME, "S5402ZA"), }, }, + { + .ident = "Asus Vivobook S5602ZA", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_BOARD_NAME, "S5602ZA"), + }, + }, { } }; -- GitLab From fa153b7cddce795662d38f78a87612c166c0f692 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 25 Oct 2022 14:12:23 +0200 Subject: [PATCH 0496/1927] ACPI: scan: Add LATT2021 to acpi_ignore_dep_ids[] Some x86/ACPI laptops with MIPI cameras have a LATT2021 ACPI device in the _DEP dependency list of the ACPI devices for the camera-sensors (which have flags.honor_deps set). The _DDN for the LATT2021 device is "Lattice FW Update Client Driver", suggesting that this is used for firmware updates of something. There is no Linux driver for this and if Linux gets support for updates it will likely be in userspace through fwupd. For now add the LATT2021 HID to acpi_ignore_dep_ids[] so that acpi_dev_ready_for_enumeration() will return true once the other _DEP dependencies are met. Signed-off-by: Hans de Goede Signed-off-by: Rafael J. Wysocki --- drivers/acpi/scan.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 024cc373a197f..b47e93a24a9a4 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -789,6 +789,7 @@ static bool acpi_info_matches_ids(struct acpi_device_info *info, static const char * const acpi_ignore_dep_ids[] = { "PNP0D80", /* Windows-compatible System Power Management Controller */ "INT33BD", /* Intel Baytrail Mailbox Device */ + "LATT2021", /* Lattice FW Update Client Driver */ NULL }; -- GitLab From 88c8e05ed5c0f05a637e654bbe4e49a1ebe7013c Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Wed, 26 Oct 2022 14:09:06 +0200 Subject: [PATCH 0497/1927] Revert "usb: gadget: uvc: limit isoc_sg to super speed gadgets" This reverts commit 19905240aef0181d1e6944070eb85fce75f75bcd. It was a new feature, and it doesn't even work properly yet, so revert it from this branch as it is not needed for 6.1-final. Cc: Michael Grzeschik Cc: stable Fixes: 19905240aef0 ("usb: gadget: uvc: limit isoc_sg to super speed gadgets") Signed-off-by: Greg Kroah-Hartman --- drivers/usb/gadget/function/uvc_queue.c | 9 ++++++--- drivers/usb/gadget/function/uvc_video.c | 9 ++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/drivers/usb/gadget/function/uvc_queue.c b/drivers/usb/gadget/function/uvc_queue.c index 782eaae931407..0aa3d7e1f3cc3 100644 --- a/drivers/usb/gadget/function/uvc_queue.c +++ b/drivers/usb/gadget/function/uvc_queue.c @@ -84,9 +84,12 @@ static int uvc_buffer_prepare(struct vb2_buffer *vb) return -ENODEV; buf->state = UVC_BUF_STATE_QUEUED; - buf->sgt = vb2_dma_sg_plane_desc(vb, 0); - buf->sg = buf->sgt->sgl; - buf->mem = vb2_plane_vaddr(vb, 0); + if (queue->use_sg) { + buf->sgt = vb2_dma_sg_plane_desc(vb, 0); + buf->sg = buf->sgt->sgl; + } else { + buf->mem = vb2_plane_vaddr(vb, 0); + } buf->length = vb2_plane_size(vb, 0); if (vb->type == V4L2_BUF_TYPE_VIDEO_CAPTURE) buf->bytesused = 0; diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index b6ea600b01118..dd1c6b2ca7c6f 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -459,9 +459,6 @@ static void uvcg_video_pump(struct work_struct *work) */ int uvcg_video_enable(struct uvc_video *video, int enable) { - struct uvc_device *uvc = video->uvc; - struct usb_composite_dev *cdev = uvc->func.config->cdev; - struct usb_gadget *gadget = cdev->gadget; unsigned int i; int ret; @@ -493,11 +490,9 @@ int uvcg_video_enable(struct uvc_video *video, int enable) if (video->max_payload_size) { video->encode = uvc_video_encode_bulk; video->payload_size = 0; - } else { - video->encode = (video->queue.use_sg && - !(gadget->speed <= USB_SPEED_HIGH)) ? + } else + video->encode = video->queue.use_sg ? uvc_video_encode_isoc_sg : uvc_video_encode_isoc; - } video->req_int_count = 0; -- GitLab From c5f0a17288740573f4de72965c5294a60244c5fc Mon Sep 17 00:00:00 2001 From: Rolf Eike Beer Date: Fri, 21 Oct 2022 15:47:03 +0200 Subject: [PATCH 0498/1927] rhashtable: make test actually random The "random rhlist add/delete operations" actually wasn't very random, as all cases tested the same bit. Since the later parts of this loop depend on the first case execute this unconditionally, and then test on different bits for the remaining tests. While at it only request as much random bits as are actually used. Signed-off-by: Rolf Eike Beer Acked-by: Herbert Xu Signed-off-by: David S. Miller --- lib/test_rhashtable.c | 58 ++++++++++++++++--------------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/lib/test_rhashtable.c b/lib/test_rhashtable.c index b358a74ed7ed8..f2ba5787055a9 100644 --- a/lib/test_rhashtable.c +++ b/lib/test_rhashtable.c @@ -369,18 +369,10 @@ static int __init test_rhltable(unsigned int entries) pr_info("test %d random rhlist add/delete operations\n", entries); for (j = 0; j < entries; j++) { u32 i = prandom_u32_max(entries); - u32 prand = get_random_u32(); + u32 prand = prandom_u32_max(4); cond_resched(); - if (prand == 0) - prand = get_random_u32(); - - if (prand & 1) { - prand >>= 1; - continue; - } - err = rhltable_remove(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); if (test_bit(i, obj_in_table)) { clear_bit(i, obj_in_table); @@ -393,35 +385,29 @@ static int __init test_rhltable(unsigned int entries) } if (prand & 1) { - prand >>= 1; - continue; - } - - err = rhltable_insert(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); - if (err == 0) { - if (WARN(test_and_set_bit(i, obj_in_table), "succeeded to insert same object %d", i)) - continue; - } else { - if (WARN(!test_bit(i, obj_in_table), "failed to insert object %d", i)) - continue; - } - - if (prand & 1) { - prand >>= 1; - continue; + err = rhltable_insert(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); + if (err == 0) { + if (WARN(test_and_set_bit(i, obj_in_table), "succeeded to insert same object %d", i)) + continue; + } else { + if (WARN(!test_bit(i, obj_in_table), "failed to insert object %d", i)) + continue; + } } - i = prandom_u32_max(entries); - if (test_bit(i, obj_in_table)) { - err = rhltable_remove(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); - WARN(err, "cannot remove element at slot %d", i); - if (err == 0) - clear_bit(i, obj_in_table); - } else { - err = rhltable_insert(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); - WARN(err, "failed to insert object %d", i); - if (err == 0) - set_bit(i, obj_in_table); + if (prand & 2) { + i = prandom_u32_max(entries); + if (test_bit(i, obj_in_table)) { + err = rhltable_remove(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); + WARN(err, "cannot remove element at slot %d", i); + if (err == 0) + clear_bit(i, obj_in_table); + } else { + err = rhltable_insert(&rhlt, &rhl_test_objects[i].list_node, test_rht_params); + WARN(err, "failed to insert object %d", i); + if (err == 0) + set_bit(i, obj_in_table); + } } } -- GitLab From d6d9875e22cb760b433cade5cf19d8f105ad4621 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 12 Oct 2022 14:14:57 +0200 Subject: [PATCH 0499/1927] MAINTAINERS: remove outdated linux390 link Reported-by: Alexandra Winter Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- MAINTAINERS | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index e04d944005ba8..ea5fcf9047ead 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11258,7 +11258,6 @@ M: Claudio Imbrenda R: David Hildenbrand L: kvm@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git F: Documentation/virt/kvm/s390* F: arch/s390/include/asm/gmap.h @@ -17996,7 +17995,6 @@ R: Christian Borntraeger R: Sven Schnelle L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git F: Documentation/driver-api/s390-drivers.rst F: Documentation/s390/ @@ -18008,7 +18006,6 @@ M: Vineeth Vijayan M: Peter Oberparleiter L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: drivers/s390/cio/ S390 DASD DRIVER @@ -18016,7 +18013,6 @@ M: Stefan Haberland M: Jan Hoeppner L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: block/partitions/ibm.c F: drivers/s390/block/dasd* F: include/linux/dasd_mod.h @@ -18026,7 +18022,6 @@ M: Matthew Rosato M: Gerald Schaefer L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: drivers/iommu/s390-iommu.c S390 IUCV NETWORK LAYER @@ -18035,7 +18030,6 @@ M: Wenjia Zhang L: linux-s390@vger.kernel.org L: netdev@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: drivers/s390/net/*iucv* F: include/net/iucv/ F: net/iucv/ @@ -18046,7 +18040,6 @@ M: Wenjia Zhang L: linux-s390@vger.kernel.org L: netdev@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: drivers/s390/net/ S390 PCI SUBSYSTEM @@ -18054,7 +18047,6 @@ M: Niklas Schnelle M: Gerald Schaefer L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: arch/s390/pci/ F: drivers/pci/hotplug/s390_pci_hpc.c F: Documentation/s390/pci.rst @@ -18065,7 +18057,6 @@ M: Halil Pasic M: Jason Herne L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: Documentation/s390/vfio-ap* F: drivers/s390/crypto/vfio_ap* @@ -18094,7 +18085,6 @@ S390 ZCRYPT DRIVER M: Harald Freudenberger L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: drivers/s390/crypto/ S390 ZFCP DRIVER @@ -18102,7 +18092,6 @@ M: Steffen Maier M: Benjamin Block L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: drivers/s390/scsi/zfcp_* S3C ADC BATTERY DRIVER @@ -18674,7 +18663,6 @@ M: Wenjia Zhang M: Jan Karcher L: linux-s390@vger.kernel.org S: Supported -W: http://www.ibm.com/developerworks/linux/linux390/ F: net/smc/ SHARP GP2AP002A00F/GP2AP002S00F SENSOR DRIVER -- GitLab From 4e1b5a86a5edfbefc9396d41b0fc1a2ebd0101b6 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 18 Oct 2022 13:39:43 +0200 Subject: [PATCH 0500/1927] s390/uaccess: add missing EX_TABLE entries to __clear_user() For some exception types the instruction address points behind the instruction that caused the exception. Take that into account and add the missing exception table entries. Cc: Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/lib/uaccess.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c index 58033dfcb6d45..720036fb19242 100644 --- a/arch/s390/lib/uaccess.c +++ b/arch/s390/lib/uaccess.c @@ -157,7 +157,7 @@ unsigned long __clear_user(void __user *to, unsigned long size) asm volatile( " lr 0,%[spec]\n" "0: mvcos 0(%1),0(%4),%0\n" - " jz 4f\n" + "6: jz 4f\n" "1: algr %0,%2\n" " slgr %1,%2\n" " j 0b\n" @@ -167,11 +167,11 @@ unsigned long __clear_user(void __user *to, unsigned long size) " clgr %0,%3\n" /* copy crosses next page boundary? */ " jnh 5f\n" "3: mvcos 0(%1),0(%4),%3\n" - " slgr %0,%3\n" + "7: slgr %0,%3\n" " j 5f\n" "4: slgr %0,%0\n" "5:\n" - EX_TABLE(0b,2b) EX_TABLE(3b,5b) + EX_TABLE(0b,2b) EX_TABLE(6b,2b) EX_TABLE(3b,5b) EX_TABLE(7b,5b) : "+a" (size), "+a" (to), "+a" (tmp1), "=a" (tmp2) : "a" (empty_zero_page), [spec] "d" (spec.val) : "cc", "memory", "0"); -- GitLab From a262d3ad6a433e4080cecd0a8841104a5906355e Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 18 Oct 2022 13:44:11 +0200 Subject: [PATCH 0501/1927] s390/futex: add missing EX_TABLE entry to __futex_atomic_op() For some exception types the instruction address points behind the instruction that caused the exception. Take that into account and add the missing exception table entry. Cc: Reviewed-by: Vasily Gorbik Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/include/asm/futex.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/include/asm/futex.h b/arch/s390/include/asm/futex.h index e08c882dccaae..eaeaeb3ff0be3 100644 --- a/arch/s390/include/asm/futex.h +++ b/arch/s390/include/asm/futex.h @@ -17,7 +17,8 @@ "3: jl 1b\n" \ " lhi %0,0\n" \ "4: sacf 768\n" \ - EX_TABLE(0b,4b) EX_TABLE(2b,4b) EX_TABLE(3b,4b) \ + EX_TABLE(0b,4b) EX_TABLE(1b,4b) \ + EX_TABLE(2b,4b) EX_TABLE(3b,4b) \ : "=d" (ret), "=&d" (oldval), "=&d" (newval), \ "=m" (*uaddr) \ : "0" (-EFAULT), "d" (oparg), "a" (uaddr), \ -- GitLab From 6ec803025cf3173a57222e4411097166bd06fa98 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Tue, 18 Oct 2022 13:48:34 +0200 Subject: [PATCH 0502/1927] s390/pci: add missing EX_TABLE entries to __pcistg_mio_inuser()/__pcilg_mio_inuser() For some exception types the instruction address points behind the instruction that caused the exception. Take that into account and add the missing exception table entry. Cc: Fixes: f058599e22d5 ("s390/pci: Fix s390_mmio_read/write with MIO") Reviewed-by: Niklas Schnelle Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/pci/pci_mmio.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 080c88620723a..5880893329310 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -64,7 +64,7 @@ static inline int __pcistg_mio_inuser( asm volatile ( " sacf 256\n" "0: llgc %[tmp],0(%[src])\n" - " sllg %[val],%[val],8\n" + "4: sllg %[val],%[val],8\n" " aghi %[src],1\n" " ogr %[val],%[tmp]\n" " brctg %[cnt],0b\n" @@ -72,7 +72,7 @@ static inline int __pcistg_mio_inuser( "2: ipm %[cc]\n" " srl %[cc],28\n" "3: sacf 768\n" - EX_TABLE(0b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) + EX_TABLE(0b, 3b) EX_TABLE(4b, 3b) EX_TABLE(1b, 3b) EX_TABLE(2b, 3b) : [src] "+a" (src), [cnt] "+d" (cnt), [val] "+d" (val), [tmp] "=d" (tmp), @@ -215,10 +215,10 @@ static inline int __pcilg_mio_inuser( "2: ahi %[shift],-8\n" " srlg %[tmp],%[val],0(%[shift])\n" "3: stc %[tmp],0(%[dst])\n" - " aghi %[dst],1\n" + "5: aghi %[dst],1\n" " brctg %[cnt],2b\n" "4: sacf 768\n" - EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) + EX_TABLE(0b, 4b) EX_TABLE(1b, 4b) EX_TABLE(3b, 4b) EX_TABLE(5b, 4b) : [ioaddr_len] "+&d" (ioaddr_len.pair), [cc] "+d" (cc), [val] "=d" (val), -- GitLab From aa127a069ef312aca02b730d5137e1778d0c3ba7 Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Fri, 16 Sep 2022 15:01:36 +0200 Subject: [PATCH 0503/1927] s390/boot: add secure boot trailer This patch enhances the kernel image adding a trailer as required for secure boot by future firmware versions. Cc: # 5.2+ Signed-off-by: Peter Oberparleiter Reviewed-by: Sven Schnelle Signed-off-by: Vasily Gorbik --- arch/s390/boot/vmlinux.lds.S | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index af5c6860e0a11..fa9d33b01b858 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -102,8 +102,17 @@ SECTIONS _compressed_start = .; *(.vmlinux.bin.compressed) _compressed_end = .; - FILL(0xff); - . = ALIGN(4096); + } + +#define SB_TRAILER_SIZE 32 + /* Trailer needed for Secure Boot */ + . += SB_TRAILER_SIZE; /* make sure .sb.trailer does not overwrite the previous section */ + . = ALIGN(4096) - SB_TRAILER_SIZE; + .sb.trailer : { + QUAD(0) + QUAD(0) + QUAD(0) + QUAD(0x000000207a49504c) } _end = .; -- GitLab From 8b1e6a3fb3feecdce8521154bfe30f9d1ebb70e6 Mon Sep 17 00:00:00 2001 From: Thomas Richter Date: Thu, 20 Oct 2022 16:51:34 +0200 Subject: [PATCH 0504/1927] s390/pai: fix raw data collection for PMU pai_ext Commit 838d9bb62d13 ("perf: Use sample_flags for raw_data") changed the way the raw data of an event is collected. Adjust the PMU pai_ext to the new scheme. Fixes: 838d9bb62d13 ("perf: Use sample_flags for raw_data") Signed-off-by: Thomas Richter Acked-by: Sumanth Korikkar Signed-off-by: Vasily Gorbik --- arch/s390/kernel/perf_pai_ext.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/s390/kernel/perf_pai_ext.c b/arch/s390/kernel/perf_pai_ext.c index d5c7c1e30c179..74b53c531e0cd 100644 --- a/arch/s390/kernel/perf_pai_ext.c +++ b/arch/s390/kernel/perf_pai_ext.c @@ -459,6 +459,7 @@ static int paiext_push_sample(void) raw.frag.data = cpump->save; raw.size = raw.frag.size; data.raw = &raw; + data.sample_flags |= PERF_SAMPLE_RAW; } overflow = perf_event_overflow(event, &data, ®s); -- GitLab From 1b6074112742f65ece71b0f299ca5a6a887d2db6 Mon Sep 17 00:00:00 2001 From: Peter Oberparleiter Date: Fri, 14 Oct 2022 12:24:58 +0200 Subject: [PATCH 0505/1927] s390/cio: fix out-of-bounds access on cio_ignore free The channel-subsystem-driver scans for newly available devices whenever device-IDs are removed from the cio_ignore list using a command such as: echo free >/proc/cio_ignore Since an I/O device scan might interfer with running I/Os, commit 172da89ed0ea ("s390/cio: avoid excessive path-verification requests") introduced an optimization to exclude online devices from the scan. The newly added check for online devices incorrectly assumes that an I/O-subchannel's drvdata points to a struct io_subchannel_private. For devices that are bound to a non-default I/O subchannel driver, such as the vfio_ccw driver, this results in an out-of-bounds read access during each scan. Fix this by changing the scan logic to rely on a driver-independent online indication. For this we can use struct subchannel->config.ena, which is the driver's requested subchannel-enabled state. Since I/Os can only be started on enabled subchannels, this matches the intent of the original optimization of not scanning devices where I/O might be running. Fixes: 172da89ed0ea ("s390/cio: avoid excessive path-verification requests") Fixes: 0c3812c347bf ("s390/cio: derive cdev information only for IO-subchannels") Cc: # v5.15 Reported-by: Alexander Egorenkov Reviewed-by: Vineeth Vijayan Signed-off-by: Peter Oberparleiter Signed-off-by: Vasily Gorbik --- drivers/s390/cio/css.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index 913b6ddd040b8..c7db953985002 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -753,13 +753,9 @@ static int __unset_online(struct device *dev, void *data) { struct idset *set = data; struct subchannel *sch = to_subchannel(dev); - struct ccw_device *cdev; - if (sch->st == SUBCHANNEL_TYPE_IO) { - cdev = sch_get_cdev(sch); - if (cdev && cdev->online) - idset_sch_del(set, sch->schid); - } + if (sch->st == SUBCHANNEL_TYPE_IO && sch->config.ena) + idset_sch_del(set, sch->schid); return 0; } -- GitLab From e38de4804421b064a9c73c5a9b7f3df96b863e4b Mon Sep 17 00:00:00 2001 From: "Jason J. Herne" Date: Fri, 21 Oct 2022 10:50:02 -0400 Subject: [PATCH 0506/1927] s390/vfio-ap: Fix memory allocation for mdev_types array The vfio-ap crypto driver fails to allocate memory for an array of pointers used to pass supported mdev types to mdev_register_parent(). Since we only support a single mdev type, the fix is to allocate a single entry in the ap_matrix_dev->mdev_types array. Link: https://lore.kernel.org/r/20221021145905.15100-1-jjherne@linux.ibm.com Fixes: da44c340c4fe ("vfio/mdev: simplify mdev_type handling") Cc: stable@vger.kernel.org Cc: Tony Krowiak Reported-by: Christian Borntraeger Reviewed-by: Matthew Rosato Signed-off-by: Jason J. Herne Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/vfio_ap_private.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h index 2eddd5f34ed34..976a65f32e7d1 100644 --- a/drivers/s390/crypto/vfio_ap_private.h +++ b/drivers/s390/crypto/vfio_ap_private.h @@ -52,7 +52,7 @@ struct ap_matrix_dev { struct mutex guests_lock; /* serializes access to each KVM guest */ struct mdev_parent parent; struct mdev_type mdev_type; - struct mdev_type *mdev_types[]; + struct mdev_type *mdev_types[1]; }; extern struct ap_matrix_dev *matrix_dev; -- GitLab From 1683d3282f240336a2b4b6b541d435facfe8bbb6 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Tue, 25 Oct 2022 16:01:49 +0100 Subject: [PATCH 0507/1927] ASoC: dapm: Don't use prefix for regulator name When a component has a prefix, and uses a SND_SOC_DAPM_REGULATOR_SUPPLY, the name of the regulator should not use the prefix, otherwise it won't be properly matched in the DT/ACPI. Fixes: 3caac759681e ("ASoC: soc-dapm.c: fixup snd_soc_dapm_new_control_unlocked() error handling") Signed-off-by: Paul Cercueil Link: https://lore.kernel.org/r/20221025150149.113129-1-paul@crapouillou.net Signed-off-by: Mark Brown --- sound/soc/soc-dapm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index d515e7a78ea86..879cf1be67a9f 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -3645,7 +3645,7 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm, switch (w->id) { case snd_soc_dapm_regulator_supply: - w->regulator = devm_regulator_get(dapm->dev, w->name); + w->regulator = devm_regulator_get(dapm->dev, widget->name); if (IS_ERR(w->regulator)) { ret = PTR_ERR(w->regulator); goto request_failed; -- GitLab From d40b6529c6269cd5afddb1116a383cab9f126694 Mon Sep 17 00:00:00 2001 From: Brent Mendelsohn Date: Mon, 24 Oct 2022 18:42:27 +0100 Subject: [PATCH 0508/1927] ASoC: amd: yc: Add Alienware m17 R5 AMD into DMI table This model requires an additional detection quirk to enable the internal microphone - BIOS doesn't seem to support AcpDmicConnected (nothing in acpidump output). Link: https://bugzilla.kernel.org/show_bug.cgi?id=216590 Signed-off-by: Brent Mendelsohn Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20221024174227.4160-1-mendiebm@gmail.com Signed-off-by: Mark Brown --- sound/soc/amd/yc/acp6x-mach.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index 6c0f1de10429a..d9715bea965e1 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -206,6 +206,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "UM5302TA"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware m17 R5 AMD"), + } + }, {} }; -- GitLab From 8bb0ac0e6f64ebdf15d963c26b028de391c9bcf9 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 25 Oct 2022 16:09:42 +0200 Subject: [PATCH 0509/1927] ASoC: Intel: bytcht_es8316: Add quirk for the Nanote UMPC-01 The Nanote UMPC-01 mini laptop has stereo speakers, while the default bytcht_es8316 settings assume a mono speaker setup. Add a quirk for this. Signed-off-by: Hans de Goede Acked-by: Pierre-Louis Bossart Link: https://lore.kernel.org/r/20221025140942.509066-1-hdegoede@redhat.com Signed-off-by: Mark Brown --- sound/soc/intel/boards/bytcht_es8316.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sound/soc/intel/boards/bytcht_es8316.c b/sound/soc/intel/boards/bytcht_es8316.c index 6432b83f616f3..a935c5fd9edbc 100644 --- a/sound/soc/intel/boards/bytcht_es8316.c +++ b/sound/soc/intel/boards/bytcht_es8316.c @@ -443,6 +443,13 @@ static const struct dmi_system_id byt_cht_es8316_quirk_table[] = { | BYT_CHT_ES8316_INTMIC_IN2_MAP | BYT_CHT_ES8316_JD_INVERTED), }, + { /* Nanote UMPC-01 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "RWC CO.,LTD"), + DMI_MATCH(DMI_PRODUCT_NAME, "UMPC-01"), + }, + .driver_data = (void *)BYT_CHT_ES8316_INTMIC_IN1_MAP, + }, { /* Teclast X98 Plus II */ .matches = { DMI_MATCH(DMI_SYS_VENDOR, "TECLAST"), -- GitLab From 69d1abc0214e944dff1d30e201f8fc332a1adf1a Mon Sep 17 00:00:00 2001 From: Tzung-Bi Shih Date: Tue, 25 Oct 2022 10:49:29 +0800 Subject: [PATCH 0510/1927] MAINTAINERS: update Tzung-Bi's email address Use kernel.org account instead. Signed-off-by: Tzung-Bi Shih Link: https://lore.kernel.org/r/20221025024929.2652134-1-tzungbi@kernel.org Signed-off-by: Mark Brown --- .mailmap | 1 + .../devicetree/bindings/sound/google,cros-ec-codec.yaml | 2 +- Documentation/devicetree/bindings/sound/realtek,rt1015p.yaml | 2 +- MAINTAINERS | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.mailmap b/.mailmap index 380378e2db368..84342d7814077 100644 --- a/.mailmap +++ b/.mailmap @@ -414,6 +414,7 @@ TripleX Chung TripleX Chung Tsuneo Yoshioka Tycho Andersen +Tzung-Bi Shih Uwe Kleine-König Uwe Kleine-König Uwe Kleine-König diff --git a/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml b/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml index c3e9f3485449e..dea293f403d90 100644 --- a/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml +++ b/Documentation/devicetree/bindings/sound/google,cros-ec-codec.yaml @@ -8,7 +8,7 @@ title: Audio codec controlled by ChromeOS EC maintainers: - Cheng-Yi Chiang - - Tzung-Bi Shih + - Tzung-Bi Shih description: | Google's ChromeOS EC codec is a digital mic codec provided by the diff --git a/Documentation/devicetree/bindings/sound/realtek,rt1015p.yaml b/Documentation/devicetree/bindings/sound/realtek,rt1015p.yaml index 1d73204451b15..ea7d4900ee4a5 100644 --- a/Documentation/devicetree/bindings/sound/realtek,rt1015p.yaml +++ b/Documentation/devicetree/bindings/sound/realtek,rt1015p.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Realtek rt1015p codec devicetree bindings maintainers: - - Tzung-Bi Shih + - Tzung-Bi Shih description: | Rt1015p is a rt1015 variant which does not support I2C and diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..f9749afc0b9d7 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4906,7 +4906,7 @@ F: drivers/platform/chrome/ CHROMEOS EC CODEC DRIVER M: Cheng-Yi Chiang -M: Tzung-Bi Shih +M: Tzung-Bi Shih R: Guenter Roeck L: chrome-platform@lists.linux.dev S: Maintained -- GitLab From 04694e50020b62b10bd0d46ff9e9708a6e1c7eb3 Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Wed, 26 Oct 2022 09:58:28 +0200 Subject: [PATCH 0511/1927] spi: meson-spicc: move wait completion in driver to take bursts delay in account Some delay occurs between each bursts, thus the default delay is wrong and a timeout will occur with big enough transfers. The solution is to handle the timeout management in the driver and add some delay for each bursts in the timeout calculation. Reported-by: Da Xue Signed-off-by: Neil Armstrong Link: https://lore.kernel.org/r/20221026-spicc-burst-delay-v1-0-1be5ffb7051a@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-meson-spicc.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c index bad201510a992..52bffab183298 100644 --- a/drivers/spi/spi-meson-spicc.c +++ b/drivers/spi/spi-meson-spicc.c @@ -160,6 +160,7 @@ struct meson_spicc_device { struct clk *clk; struct spi_message *message; struct spi_transfer *xfer; + struct completion done; const struct meson_spicc_data *data; u8 *tx_buf; u8 *rx_buf; @@ -282,7 +283,7 @@ static irqreturn_t meson_spicc_irq(int irq, void *data) /* Disable all IRQs */ writel(0, spicc->base + SPICC_INTREG); - spi_finalize_current_transfer(spicc->master); + complete(&spicc->done); return IRQ_HANDLED; } @@ -386,6 +387,7 @@ static int meson_spicc_transfer_one(struct spi_master *master, struct spi_transfer *xfer) { struct meson_spicc_device *spicc = spi_master_get_devdata(master); + unsigned long timeout; /* Store current transfer */ spicc->xfer = xfer; @@ -410,13 +412,29 @@ static int meson_spicc_transfer_one(struct spi_master *master, /* Setup burst */ meson_spicc_setup_burst(spicc); + /* Setup wait for completion */ + reinit_completion(&spicc->done); + + /* For each byte we wait for 8 cycles of the SPI clock */ + timeout = 8LL * MSEC_PER_SEC * xfer->len; + do_div(timeout, xfer->speed_hz); + + /* Add 10us delay between each fifo bursts */ + timeout += ((xfer->len >> 4) * 10) / MSEC_PER_SEC; + + /* Increase it twice and add 200 ms tolerance */ + timeout += timeout + 200; + /* Start burst */ writel_bits_relaxed(SPICC_XCH, SPICC_XCH, spicc->base + SPICC_CONREG); /* Enable interrupts */ writel_relaxed(SPICC_TC_EN, spicc->base + SPICC_INTREG); - return 1; + if (!wait_for_completion_timeout(&spicc->done, msecs_to_jiffies(timeout))) + return -ETIMEDOUT; + + return 0; } static int meson_spicc_prepare_message(struct spi_master *master, @@ -743,6 +761,8 @@ static int meson_spicc_probe(struct platform_device *pdev) spicc->pdev = pdev; platform_set_drvdata(pdev, spicc); + init_completion(&spicc->done); + spicc->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(spicc->base)) { dev_err(&pdev->dev, "io resource mapping failed\n"); -- GitLab From 65722736c3baf29e02e964a09e85c9ef71c48e8d Mon Sep 17 00:00:00 2001 From: Nicholas Piggin Date: Sat, 22 Oct 2022 15:22:07 +1000 Subject: [PATCH 0512/1927] powerpc/64s/interrupt: Fix clear of PACA_IRQS_HARD_DIS when returning to soft-masked context Commit a4cb3651a1743 ("powerpc/64s/interrupt: Fix lost interrupts when returning to soft-masked context") fixed the problem of pending irqs being cleared when clearing the HARD_DIS bit, but then it didn't clear the bit at all. This change clears HARD_DIS without affecting other bits in the mask. When an interrupt hits in a soft-masked section that has MSR[EE]=1, it can hard disable and set PACA_IRQS_HARD_DIS, which must be cleared when returning to the EE=1 caller (unless it was set due to a MUST_HARD_MASK interrupt becoming pending). Failure to clear this leaves the returned-to context running with MSR[EE]=1 and PACA_IRQS_HARD_DIS, which confuses irq assertions and could be dangerous for code that might test the flag. This was observed in a hash MMU kernel where a kernel hash fault hits in a local_irqs_disabled region that has EE=1. The hash fault also runs with EE=1, then as it returns, a decrementer hits in the restart section and the irq restart code hard-masks which sets the PACA_IRQ_HARD_DIS flag, which is not clear when the original context is returned to. Reported-by: Sachin Sant Fixes: a4cb3651a1743 ("powerpc/64s/interrupt: Fix lost interrupts when returning to soft-masked context") Signed-off-by: Nicholas Piggin Tested-by: Sachin Sant Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221022052207.471328-1-npiggin@gmail.com --- arch/powerpc/kernel/interrupt_64.S | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S index 978a173eb3396..a019ed6fc8393 100644 --- a/arch/powerpc/kernel/interrupt_64.S +++ b/arch/powerpc/kernel/interrupt_64.S @@ -532,15 +532,24 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel) * Returning to soft-disabled context. * Check if a MUST_HARD_MASK interrupt has become pending, in which * case we need to disable MSR[EE] in the return context. + * + * The MSR[EE] check catches among other things the short incoherency + * in hard_irq_disable() between clearing MSR[EE] and setting + * PACA_IRQ_HARD_DIS. */ ld r12,_MSR(r1) andi. r10,r12,MSR_EE beq .Lfast_kernel_interrupt_return_\srr\() // EE already disabled lbz r11,PACAIRQHAPPENED(r13) andi. r10,r11,PACA_IRQ_MUST_HARD_MASK - beq .Lfast_kernel_interrupt_return_\srr\() // No HARD_MASK pending + bne 1f // HARD_MASK is pending + // No HARD_MASK pending, clear possible HARD_DIS set by interrupt + andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l + stb r11,PACAIRQHAPPENED(r13) + b .Lfast_kernel_interrupt_return_\srr\() + - /* Must clear MSR_EE from _MSR */ +1: /* Must clear MSR_EE from _MSR */ #ifdef CONFIG_PPC_BOOK3S li r10,0 /* Clear valid before changing _MSR */ -- GitLab From 831c05a7621b96944b0b4dbede57ed7cf0578f1c Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Fri, 21 May 2021 16:00:31 -0300 Subject: [PATCH 0513/1927] tools headers UAPI: Sync linux/perf_event.h with the kernel sources To pick the changes in: cfef80bad4cf79cd ("perf/uapi: Define PERF_MEM_SNOOPX_PEER in kernel header file") ee3e88dfec23153d ("perf/mem: Introduce PERF_MEM_LVLNUM_{EXTN_MEM|IO}") b4e12b2d70fd9ecc ("perf: Kill __PERF_SAMPLE_CALLCHAIN_EARLY") There is a kernel patch pending that renames PERF_MEM_LVLNUM_EXTN_MEM to PERF_MEM_LVLNUM_CXL, tooling this time is ahead of the kernel :-) This thus partially addresses this perf build warning: Warning: Kernel ABI header at 'tools/include/uapi/linux/perf_event.h' differs from latest version at 'include/uapi/linux/perf_event.h' diff -u tools/include/uapi/linux/perf_event.h include/uapi/linux/perf_event.h Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Ravi Bangoria Link: https://lore.kernel.org/lkml/Y1k53KMdzypmU0WS@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/include/uapi/linux/perf_event.h | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h index ea6defacc1a7d..ccb7f5dad59be 100644 --- a/tools/include/uapi/linux/perf_event.h +++ b/tools/include/uapi/linux/perf_event.h @@ -164,8 +164,6 @@ enum perf_event_sample_format { PERF_SAMPLE_WEIGHT_STRUCT = 1U << 24, PERF_SAMPLE_MAX = 1U << 25, /* non-ABI */ - - __PERF_SAMPLE_CALLCHAIN_EARLY = 1ULL << 63, /* non-ABI; internal use */ }; #define PERF_SAMPLE_WEIGHT_TYPE (PERF_SAMPLE_WEIGHT | PERF_SAMPLE_WEIGHT_STRUCT) @@ -263,6 +261,17 @@ enum { PERF_BR_MAX, }; +/* + * Common branch speculation outcome classification + */ +enum { + PERF_BR_SPEC_NA = 0, /* Not available */ + PERF_BR_SPEC_WRONG_PATH = 1, /* Speculative but on wrong path */ + PERF_BR_NON_SPEC_CORRECT_PATH = 2, /* Non-speculative but on correct path */ + PERF_BR_SPEC_CORRECT_PATH = 3, /* Speculative and on correct path */ + PERF_BR_SPEC_MAX, +}; + enum { PERF_BR_NEW_FAULT_ALGN = 0, /* Alignment fault */ PERF_BR_NEW_FAULT_DATA = 1, /* Data fault */ @@ -282,11 +291,11 @@ enum { PERF_BR_PRIV_HV = 3, }; -#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1 -#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2 -#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3 -#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4 -#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5 +#define PERF_BR_ARM64_FIQ PERF_BR_NEW_ARCH_1 +#define PERF_BR_ARM64_DEBUG_HALT PERF_BR_NEW_ARCH_2 +#define PERF_BR_ARM64_DEBUG_EXIT PERF_BR_NEW_ARCH_3 +#define PERF_BR_ARM64_DEBUG_INST PERF_BR_NEW_ARCH_4 +#define PERF_BR_ARM64_DEBUG_DATA PERF_BR_NEW_ARCH_5 #define PERF_SAMPLE_BRANCH_PLM_ALL \ (PERF_SAMPLE_BRANCH_USER|\ @@ -1397,6 +1406,7 @@ union perf_mem_data_src { * abort: aborting a hardware transaction * cycles: cycles from last branch (or 0 if not supported) * type: branch type + * spec: branch speculation info (or 0 if not supported) */ struct perf_branch_entry { __u64 from; @@ -1407,9 +1417,10 @@ struct perf_branch_entry { abort:1, /* transaction abort */ cycles:16, /* cycle count to last branch */ type:4, /* branch type */ + spec:2, /* branch speculation info */ new_type:4, /* additional branch type */ priv:3, /* privilege level */ - reserved:33; + reserved:31; }; union perf_sample_weight { -- GitLab From cba04f3136b658583adb191556f99d087589c1cc Mon Sep 17 00:00:00 2001 From: Adrian Hunter Date: Wed, 26 Oct 2022 10:27:36 +0300 Subject: [PATCH 0514/1927] perf auxtrace: Fix address filter symbol name match for modules For modules, names from kallsyms__parse() contain the module name which meant that module symbols did not match exactly by name. Fix by matching the name string up to the separating tab character. Fixes: 1b36c03e356936d6 ("perf record: Add support for using symbols in address filters") Signed-off-by: Adrian Hunter Cc: Adrian Hunter Cc: Ian Rogers Cc: Jiri Olsa Cc: Namhyung Kim Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221026072736.2982-1-adrian.hunter@intel.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/auxtrace.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 60d8beb662aa3..46ada5ec3f9a2 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -2325,11 +2325,19 @@ struct sym_args { bool near; }; +static bool kern_sym_name_match(const char *kname, const char *name) +{ + size_t n = strlen(name); + + return !strcmp(kname, name) || + (!strncmp(kname, name, n) && kname[n] == '\t'); +} + static bool kern_sym_match(struct sym_args *args, const char *name, char type) { /* A function with the same name, and global or the n'th found or any */ return kallsyms__is_function(type) && - !strcmp(name, args->name) && + kern_sym_name_match(name, args->name) && ((args->global && isupper(type)) || (args->selected && ++(args->cnt) == args->idx) || (!args->global && !args->selected)); -- GitLab From e9229d5b6254a75291536f582652c599957344d2 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Fri, 21 Oct 2022 18:50:33 +0800 Subject: [PATCH 0515/1927] perf vendor events arm64: Fix incorrect Hisi hip08 L3 metrics Commit 0cc177cfc95d565e ("perf vendor events arm64: Add Hisi hip08 L3 metrics") add L3 metrics of hip08, but some metrics (IF_BP_MISP_BR_RET, IF_BP_MISP_BR_RET, IF_BP_MISP_BR_BL) have incorrect event number due to the mistakes in document, which caused incorrect result. Fix the incorrect metrics. Before: 65,811,214,308 armv8_pmuv3_0/event=0x1014/ # 18.87 push_branch # -40.19 other_branch 3,564,316,780 BR_MIS_PRED # 0.51 indirect_branch # 21.81 pop_branch After: 6,537,146,245 BR_MIS_PRED # 0.48 indirect_branch # 0.47 pop_branch # 0.00 push_branch # 0.05 other_branch Fixes: 0cc177cfc95d565e ("perf vendor events arm64: Add Hisi hip08 L3 metrics") Reviewed-by: John Garry Signed-off-by: Shang XiaoJing Acked-by: James Clark Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: John Garry Cc: Kajol Jain Cc: Leo Yan Cc: linux-arm-kernel@lists.infradead.org Cc: Mark Rutland Cc: Mike Leach Cc: Namhyung Kim Cc: Peter Zijlstra Cc: Will Deacon Link: https://lore.kernel.org/r/20221021105035.10000-2-shangxiaojing@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- .../perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json b/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json index 6970203cb2470..6443a061e22a1 100644 --- a/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json +++ b/tools/perf/pmu-events/arch/arm64/hisilicon/hip08/metrics.json @@ -112,21 +112,21 @@ "MetricName": "indirect_branch" }, { - "MetricExpr": "(armv8_pmuv3_0@event\\=0x1014@ + armv8_pmuv3_0@event\\=0x1018@) / BR_MIS_PRED", + "MetricExpr": "(armv8_pmuv3_0@event\\=0x1013@ + armv8_pmuv3_0@event\\=0x1016@) / BR_MIS_PRED", "PublicDescription": "Push branch L3 topdown metric", "BriefDescription": "Push branch L3 topdown metric", "MetricGroup": "TopDownL3", "MetricName": "push_branch" }, { - "MetricExpr": "armv8_pmuv3_0@event\\=0x100c@ / BR_MIS_PRED", + "MetricExpr": "armv8_pmuv3_0@event\\=0x100d@ / BR_MIS_PRED", "PublicDescription": "Pop branch L3 topdown metric", "BriefDescription": "Pop branch L3 topdown metric", "MetricGroup": "TopDownL3", "MetricName": "pop_branch" }, { - "MetricExpr": "(BR_MIS_PRED - armv8_pmuv3_0@event\\=0x1010@ - armv8_pmuv3_0@event\\=0x1014@ - armv8_pmuv3_0@event\\=0x1018@ - armv8_pmuv3_0@event\\=0x100c@) / BR_MIS_PRED", + "MetricExpr": "(BR_MIS_PRED - armv8_pmuv3_0@event\\=0x1010@ - armv8_pmuv3_0@event\\=0x1013@ - armv8_pmuv3_0@event\\=0x1016@ - armv8_pmuv3_0@event\\=0x100d@) / BR_MIS_PRED", "PublicDescription": "Other branch L3 topdown metric", "BriefDescription": "Other branch L3 topdown metric", "MetricGroup": "TopDownL3", -- GitLab From f78961f8380b940e0cfc7e549336c21a2ad44f4d Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Tue, 25 Oct 2022 15:10:14 -0700 Subject: [PATCH 0516/1927] usb: dwc3: gadget: Stop processing more requests on IMI When servicing a transfer completion event, the dwc3 driver will reclaim TRBs of started requests up to the request associated with the interrupt event. Currently we don't check for interrupt due to missed isoc, and the driver may attempt to reclaim TRBs beyond the associated event. This causes invalid memory access when the hardware still owns the TRB. If there's a missed isoc TRB with IMI (interrupt on missed isoc), make sure to stop servicing further. Note that only the last TRB of chained TRBs has its status updated with missed isoc. Fixes: 72246da40f37 ("usb: Introduce DesignWare USB3 DRD Driver") Cc: stable@vger.kernel.org Reported-by: Jeff Vanhoof Reported-by: Dan Vacura Signed-off-by: Thinh Nguyen Reviewed-by: Jeff Vanhoof Tested-by: Jeff Vanhoof Link: https://lore.kernel.org/r/b29acbeab531b666095dfdafd8cb5c7654fbb3e1.1666735451.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/gadget.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index dd8ecbe61bec1..230b3c6600540 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -3248,6 +3248,10 @@ static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, if (event->status & DEPEVT_STATUS_SHORT && !chain) return 1; + if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && + DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) + return 1; + if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || (trb->ctrl & DWC3_TRB_CTRL_LST)) return 1; -- GitLab From 308c316d16cbad99bb834767382baa693ac42169 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Tue, 25 Oct 2022 15:10:20 -0700 Subject: [PATCH 0517/1927] usb: dwc3: gadget: Don't set IMI for no_interrupt The gadget driver may have a certain expectation of how the request completion flow should be from to its configuration. Make sure the controller driver respect that. That is, don't set IMI (Interrupt on Missed Isoc) when usb_request->no_interrupt is set. Also, the driver should only set IMI to the last TRB of a chain. Fixes: 72246da40f37 ("usb: Introduce DesignWare USB3 DRD Driver") Cc: stable@vger.kernel.org Signed-off-by: Thinh Nguyen Reviewed-by: Jeff Vanhoof Tested-by: Jeff Vanhoof Link: https://lore.kernel.org/r/ced336c84434571340c07994e3667a0ee284fefe.1666735451.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/gadget.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 230b3c6600540..5fe2d136dff51 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1292,8 +1292,8 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; } - /* always enable Interrupt on Missed ISOC */ - trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; + if (!no_interrupt && !chain) + trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; break; case USB_ENDPOINT_XFER_BULK: -- GitLab From e07ee6fe21f47cfd72ae566395c67a80e7c66163 Mon Sep 17 00:00:00 2001 From: Allison Henderson Date: Tue, 25 Oct 2022 12:16:27 -0700 Subject: [PATCH 0518/1927] xfs: increase rename inode reservation xfs_rename can update up to 5 inodes: src_dp, target_dp, src_ip, target_ip and wip. So we need to increase the inode reservation to match. Signed-off-by: Allison Henderson Reviewed-by: Darrick J. Wong Signed-off-by: Darrick J. Wong --- fs/xfs/libxfs/xfs_trans_resv.c | 4 ++-- fs/xfs/xfs_inode.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_trans_resv.c b/fs/xfs/libxfs/xfs_trans_resv.c index 2c4ad6e4bb149..5b2f27cbdb808 100644 --- a/fs/xfs/libxfs/xfs_trans_resv.c +++ b/fs/xfs/libxfs/xfs_trans_resv.c @@ -422,7 +422,7 @@ xfs_calc_itruncate_reservation_minlogsize( /* * In renaming a files we can modify: - * the four inodes involved: 4 * inode size + * the five inodes involved: 5 * inode size * the two directory btrees: 2 * (max depth + v2) * dir block size * the two directory bmap btrees: 2 * max depth * block size * And the bmap_finish transaction can free dir and bmap blocks (two sets @@ -437,7 +437,7 @@ xfs_calc_rename_reservation( struct xfs_mount *mp) { return XFS_DQUOT_LOGRES(mp) + - max((xfs_calc_inode_res(mp, 4) + + max((xfs_calc_inode_res(mp, 5) + xfs_calc_buf_res(2 * XFS_DIROP_LOG_COUNT(mp), XFS_FSB_TO_B(mp, 1))), (xfs_calc_buf_res(7, mp->m_sb.sb_sectsize) + diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index c000b74dd2035..aa303be11576f 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2818,7 +2818,7 @@ xfs_rename( * Lock all the participating inodes. Depending upon whether * the target_name exists in the target directory, and * whether the target directory is the same as the source - * directory, we can lock from 2 to 4 inodes. + * directory, we can lock from 2 to 5 inodes. */ xfs_lock_inodes(inodes, num_inodes, XFS_ILOCK_EXCL); -- GitLab From d61e1d1d5225a9baeb995bcbdb904f66f70ed87e Mon Sep 17 00:00:00 2001 From: Prike Liang Date: Fri, 21 Oct 2022 10:04:40 +0800 Subject: [PATCH 0519/1927] drm/amdgpu: disallow gfxoff until GC IP blocks complete s2idle resume In the S2idle suspend/resume phase the gfxoff is keeping functional so some IP blocks will be likely to reinitialize at gfxoff entry and that will result in failing to program GC registers.Therefore, let disallow gfxoff until AMDGPU IPs reinitialized completely. Signed-off-by: Prike Liang Acked-by: Alex Deucher Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 5.15.x --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 5b8362727226d..ddaecb2610c94 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -3210,6 +3210,15 @@ static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) return r; } adev->ip_blocks[i].status.hw = true; + + if (adev->in_s0ix && adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { + /* disable gfxoff for IP resume. The gfxoff will be re-enabled in + * amdgpu_device_resume() after IP resume. + */ + amdgpu_gfx_off_ctrl(adev, false); + DRM_DEBUG("will disable gfxoff for re-initializing other blocks\n"); + } + } return 0; @@ -4185,6 +4194,13 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) /* Make sure IB tests flushed */ flush_delayed_work(&adev->delayed_init_work); + if (adev->in_s0ix) { + /* re-enable gfxoff after IP resume. This re-enables gfxoff after + * it was disabled for IP resume in amdgpu_device_ip_resume_phase2(). + */ + amdgpu_gfx_off_ctrl(adev, true); + DRM_DEBUG("will enable gfxoff for the mission mode\n"); + } if (fbcon) drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false); -- GitLab From 0a8b43b12dd78daa77a7dc007b92770d262a2714 Mon Sep 17 00:00:00 2001 From: Juergen Borleis Date: Mon, 24 Oct 2022 10:05:52 +0200 Subject: [PATCH 0520/1927] net: fec: limit register access on i.MX6UL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using 'ethtool -d […]' on an i.MX6UL leads to a kernel crash: Unhandled fault: external abort on non-linefetch (0x1008) at […] due to this SoC has less registers in its FEC implementation compared to other i.MX6 variants. Thus, a run-time decision is required to avoid access to non-existing registers. Fixes: a51d3ab50702 ("net: fec: use a more proper compatible string for i.MX6UL type device") Signed-off-by: Juergen Borleis Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20221024080552.21004-1-jbe@pengutronix.de Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/freescale/fec_main.c | 46 ++++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 98d5cd313fddf..28ef4d3c18789 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -2432,6 +2432,31 @@ static u32 fec_enet_register_offset[] = { IEEE_R_DROP, IEEE_R_FRAME_OK, IEEE_R_CRC, IEEE_R_ALIGN, IEEE_R_MACERR, IEEE_R_FDXFC, IEEE_R_OCTETS_OK }; +/* for i.MX6ul */ +static u32 fec_enet_register_offset_6ul[] = { + FEC_IEVENT, FEC_IMASK, FEC_R_DES_ACTIVE_0, FEC_X_DES_ACTIVE_0, + FEC_ECNTRL, FEC_MII_DATA, FEC_MII_SPEED, FEC_MIB_CTRLSTAT, FEC_R_CNTRL, + FEC_X_CNTRL, FEC_ADDR_LOW, FEC_ADDR_HIGH, FEC_OPD, FEC_TXIC0, FEC_RXIC0, + FEC_HASH_TABLE_HIGH, FEC_HASH_TABLE_LOW, FEC_GRP_HASH_TABLE_HIGH, + FEC_GRP_HASH_TABLE_LOW, FEC_X_WMRK, FEC_R_DES_START_0, + FEC_X_DES_START_0, FEC_R_BUFF_SIZE_0, FEC_R_FIFO_RSFL, FEC_R_FIFO_RSEM, + FEC_R_FIFO_RAEM, FEC_R_FIFO_RAFL, FEC_RACC, + RMON_T_DROP, RMON_T_PACKETS, RMON_T_BC_PKT, RMON_T_MC_PKT, + RMON_T_CRC_ALIGN, RMON_T_UNDERSIZE, RMON_T_OVERSIZE, RMON_T_FRAG, + RMON_T_JAB, RMON_T_COL, RMON_T_P64, RMON_T_P65TO127, RMON_T_P128TO255, + RMON_T_P256TO511, RMON_T_P512TO1023, RMON_T_P1024TO2047, + RMON_T_P_GTE2048, RMON_T_OCTETS, + IEEE_T_DROP, IEEE_T_FRAME_OK, IEEE_T_1COL, IEEE_T_MCOL, IEEE_T_DEF, + IEEE_T_LCOL, IEEE_T_EXCOL, IEEE_T_MACERR, IEEE_T_CSERR, IEEE_T_SQE, + IEEE_T_FDXFC, IEEE_T_OCTETS_OK, + RMON_R_PACKETS, RMON_R_BC_PKT, RMON_R_MC_PKT, RMON_R_CRC_ALIGN, + RMON_R_UNDERSIZE, RMON_R_OVERSIZE, RMON_R_FRAG, RMON_R_JAB, + RMON_R_RESVD_O, RMON_R_P64, RMON_R_P65TO127, RMON_R_P128TO255, + RMON_R_P256TO511, RMON_R_P512TO1023, RMON_R_P1024TO2047, + RMON_R_P_GTE2048, RMON_R_OCTETS, + IEEE_R_DROP, IEEE_R_FRAME_OK, IEEE_R_CRC, IEEE_R_ALIGN, IEEE_R_MACERR, + IEEE_R_FDXFC, IEEE_R_OCTETS_OK +}; #else static __u32 fec_enet_register_version = 1; static u32 fec_enet_register_offset[] = { @@ -2456,7 +2481,24 @@ static void fec_enet_get_regs(struct net_device *ndev, u32 *buf = (u32 *)regbuf; u32 i, off; int ret; +#if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \ + defined(CONFIG_ARM64) || defined(CONFIG_COMPILE_TEST) + u32 *reg_list; + u32 reg_cnt; + if (!of_machine_is_compatible("fsl,imx6ul")) { + reg_list = fec_enet_register_offset; + reg_cnt = ARRAY_SIZE(fec_enet_register_offset); + } else { + reg_list = fec_enet_register_offset_6ul; + reg_cnt = ARRAY_SIZE(fec_enet_register_offset_6ul); + } +#else + /* coldfire */ + static u32 *reg_list = fec_enet_register_offset; + static const u32 reg_cnt = ARRAY_SIZE(fec_enet_register_offset); +#endif ret = pm_runtime_resume_and_get(dev); if (ret < 0) return; @@ -2465,8 +2507,8 @@ static void fec_enet_get_regs(struct net_device *ndev, memset(buf, 0, regs->len); - for (i = 0; i < ARRAY_SIZE(fec_enet_register_offset); i++) { - off = fec_enet_register_offset[i]; + for (i = 0; i < reg_cnt; i++) { + off = reg_list[i]; if ((off == FEC_R_BOUND || off == FEC_R_FSTART) && !(fep->quirks & FEC_QUIRK_HAS_FRREG)) -- GitLab From e2badb4bd33abe13ddc35975bd7f7f8693955a4b Mon Sep 17 00:00:00 2001 From: Kunihiko Hayashi Date: Mon, 24 Oct 2022 16:22:27 +0900 Subject: [PATCH 0521/1927] net: ethernet: ave: Fix MAC to be in charge of PHY PM The phylib callback is called after MAC driver's own resume callback is called. For AVE driver, after resuming immediately, PHY state machine is in PHY_NOLINK because there is a time lag from link-down to link-up due to autoneg. The result is WARN_ON() dump in mdio_bus_phy_resume(). Since ave_resume() itself calls phy_resume(), AVE driver should manage PHY PM. To indicate that MAC driver manages PHY PM, set phydev->mac_managed_pm to true to avoid the unnecessary phylib call and add missing phy_init_hw() to ave_resume(). Suggested-by: Heiner Kallweit Fixes: fba863b81604 ("net: phy: make PHY PM ops a no-op if MAC driver manages PHY PM") Signed-off-by: Kunihiko Hayashi Link: https://lore.kernel.org/r/20221024072227.24769-1-hayashi.kunihiko@socionext.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/socionext/sni_ave.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c index 1fa09b49ba7fa..d2c6a5dfdc0e1 100644 --- a/drivers/net/ethernet/socionext/sni_ave.c +++ b/drivers/net/ethernet/socionext/sni_ave.c @@ -1229,6 +1229,8 @@ static int ave_init(struct net_device *ndev) phy_support_asym_pause(phydev); + phydev->mac_managed_pm = true; + phy_attached_info(phydev); return 0; @@ -1756,6 +1758,10 @@ static int ave_resume(struct device *dev) ave_global_reset(ndev); + ret = phy_init_hw(ndev->phydev); + if (ret) + return ret; + ave_ethtool_get_wol(ndev, &wol); wol.wolopts = priv->wolopts; __ave_ethtool_set_wol(ndev, &wol); -- GitLab From 4a4c8482e370d697738a78dcd7bf2780832cb712 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 27 Oct 2022 09:34:38 +0800 Subject: [PATCH 0522/1927] ALSA: aoa: i2sbus: fix possible memory leak in i2sbus_add_dev() dev_set_name() in soundbus_add_one() allocates memory for name, it need be freed when of_device_register() fails, call soundbus_dev_put() to give up the reference that hold in device_initialize(), so that it can be freed in kobject_cleanup() when the refcount hit to 0. And other resources are also freed in i2sbus_release_dev(), so it can return 0 directly. Fixes: f3d9478b2ce4 ("[ALSA] snd-aoa: add snd-aoa") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221027013438.991920-1-yangyingliang@huawei.com Signed-off-by: Takashi Iwai --- sound/aoa/soundbus/i2sbus/core.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sound/aoa/soundbus/i2sbus/core.c b/sound/aoa/soundbus/i2sbus/core.c index faf6b03131ee4..f6841daf9e3be 100644 --- a/sound/aoa/soundbus/i2sbus/core.c +++ b/sound/aoa/soundbus/i2sbus/core.c @@ -302,6 +302,10 @@ static int i2sbus_add_dev(struct macio_dev *macio, if (soundbus_add_one(&dev->sound)) { printk(KERN_DEBUG "i2sbus: device registration error!\n"); + if (dev->sound.ofdev.dev.kobj.state_initialized) { + soundbus_dev_put(&dev->sound); + return 0; + } goto err; } -- GitLab From f0a868788fcbf63cdab51f5adcf73b271ede8164 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Wed, 26 Oct 2022 23:12:36 -0400 Subject: [PATCH 0523/1927] ALSA: Use del_timer_sync() before freeing timer The current code for freeing the emux timer is extremely dangerous: CPU0 CPU1 ---- ---- snd_emux_timer_callback() snd_emux_free() spin_lock(&emu->voice_lock) del_timer(&emu->tlist); <-- returns immediately spin_unlock(&emu->voice_lock); [..] kfree(emu); spin_lock(&emu->voice_lock); [BOOM!] Instead just use del_timer_sync() which will wait for the timer to finish before continuing. No need to check if the timer is active or not when doing so. This doesn't fix the race of a possible re-arming of the timer, but at least it won't use the data that has just been freed. [ Fixed unused variable warning by tiwai ] Cc: stable@vger.kernel.org Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Steven Rostedt (Google) Reviewed-by: Guenter Roeck Link: https://lore.kernel.org/r/20221026231236.6834b551@gandalf.local.home Signed-off-by: Takashi Iwai --- sound/synth/emux/emux.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sound/synth/emux/emux.c b/sound/synth/emux/emux.c index 5ed8e36d2e043..a870759d179ed 100644 --- a/sound/synth/emux/emux.c +++ b/sound/synth/emux/emux.c @@ -126,15 +126,10 @@ EXPORT_SYMBOL(snd_emux_register); */ int snd_emux_free(struct snd_emux *emu) { - unsigned long flags; - if (! emu) return -EINVAL; - spin_lock_irqsave(&emu->voice_lock, flags); - if (emu->timer_active) - del_timer(&emu->tlist); - spin_unlock_irqrestore(&emu->voice_lock, flags); + del_timer_sync(&emu->tlist); snd_emux_proc_free(emu); snd_emux_delete_virmidi(emu); -- GitLab From f1fae475f10a26b7e34da4ff2e2f19b7feb3548e Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Thu, 27 Oct 2022 08:52:33 +0200 Subject: [PATCH 0524/1927] ALSA: aoa: Fix I2S device accounting i2sbus_add_dev() is supposed to return the number of probed devices, i.e. either 1 or 0. However, i2sbus_add_dev() has one error handling that returns -ENODEV; this will screw up the accumulation number counted in the caller, i2sbus_probe(). Fix the return value to 0 and add the comment for better understanding for readers. Fixes: f3d9478b2ce4 ("[ALSA] snd-aoa: add snd-aoa") Link: https://lore.kernel.org/r/20221027065233.13292-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/aoa/soundbus/i2sbus/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sound/aoa/soundbus/i2sbus/core.c b/sound/aoa/soundbus/i2sbus/core.c index f6841daf9e3be..51ed2f34b276d 100644 --- a/sound/aoa/soundbus/i2sbus/core.c +++ b/sound/aoa/soundbus/i2sbus/core.c @@ -147,6 +147,7 @@ static int i2sbus_get_and_fixup_rsrc(struct device_node *np, int index, return rc; } +/* Returns 1 if added, 0 for otherwise; don't return a negative value! */ /* FIXME: look at device node refcounting */ static int i2sbus_add_dev(struct macio_dev *macio, struct i2sbus_control *control, @@ -213,7 +214,7 @@ static int i2sbus_add_dev(struct macio_dev *macio, * either as the second one in that case is just a modem. */ if (!ok) { kfree(dev); - return -ENODEV; + return 0; } mutex_init(&dev->lock); -- GitLab From 2871edb32f4622c3a25ce4b3977bad9050b91974 Mon Sep 17 00:00:00 2001 From: Anssi Hannula Date: Mon, 10 Oct 2022 20:52:27 +0200 Subject: [PATCH 0525/1927] can: kvaser_usb: Fix possible completions during init_completion kvaser_usb uses completions to signal when a response event is received for outgoing commands. However, it uses init_completion() to reinitialize the start_comp and stop_comp completions before sending the start/stop commands. In case the device sends the corresponding response just before the actual command is sent, complete() may be called concurrently with init_completion() which is not safe. This might be triggerable even with a properly functioning device by stopping the interface (CMD_STOP_CHIP) just after it goes bus-off (which also causes the driver to send CMD_STOP_CHIP when restart-ms is off), but that was not tested. Fix the issue by using reinit_completion() instead. Fixes: 080f40a6fa28 ("can: kvaser_usb: Add support for Kvaser CAN/USB devices") Tested-by: Jimmy Assarsson Signed-off-by: Anssi Hannula Signed-off-by: Jimmy Assarsson Link: https://lore.kernel.org/all/20221010185237.319219-2-extja@kvaser.com Cc: stable@vger.kernel.org Signed-off-by: Marc Kleine-Budde --- drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c | 4 ++-- drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index 7b52fda73d827..66f672ea631b8 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -1875,7 +1875,7 @@ static int kvaser_usb_hydra_start_chip(struct kvaser_usb_net_priv *priv) { int err; - init_completion(&priv->start_comp); + reinit_completion(&priv->start_comp); err = kvaser_usb_hydra_send_simple_cmd(priv->dev, CMD_START_CHIP_REQ, priv->channel); @@ -1893,7 +1893,7 @@ static int kvaser_usb_hydra_stop_chip(struct kvaser_usb_net_priv *priv) { int err; - init_completion(&priv->stop_comp); + reinit_completion(&priv->stop_comp); /* Make sure we do not report invalid BUS_OFF from CMD_CHIP_STATE_EVENT * see comment in kvaser_usb_hydra_update_state() diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index 50f2ac8319ff8..19958037720f4 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -1320,7 +1320,7 @@ static int kvaser_usb_leaf_start_chip(struct kvaser_usb_net_priv *priv) { int err; - init_completion(&priv->start_comp); + reinit_completion(&priv->start_comp); err = kvaser_usb_leaf_send_simple_cmd(priv->dev, CMD_START_CHIP, priv->channel); @@ -1338,7 +1338,7 @@ static int kvaser_usb_leaf_stop_chip(struct kvaser_usb_net_priv *priv) { int err; - init_completion(&priv->stop_comp); + reinit_completion(&priv->stop_comp); err = kvaser_usb_leaf_send_simple_cmd(priv->dev, CMD_STOP_CHIP, priv->channel); -- GitLab From a0c9f1f2e53b8eb2ae43987a30e547ba56b4fa18 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Thu, 27 Oct 2022 09:12:05 +0200 Subject: [PATCH 0526/1927] parisc: Export iosapic_serial_irq() symbol for serial port driver The parisc serial port driver needs this symbol when it's compiled as module. Signed-off-by: Helge Deller Reported-by: kernel test robot Cc: --- drivers/parisc/iosapic.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/parisc/iosapic.c b/drivers/parisc/iosapic.c index bdef7a8d6ab8e..bcc1dae007803 100644 --- a/drivers/parisc/iosapic.c +++ b/drivers/parisc/iosapic.c @@ -866,6 +866,7 @@ int iosapic_serial_irq(struct parisc_device *dev) return vi->txn_irq; } +EXPORT_SYMBOL(iosapic_serial_irq); #endif -- GitLab From e0ba1a39b8dfe4f005bebdd85daa89e7382e26b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Miros=C5=82aw?= Date: Thu, 27 Oct 2022 02:06:16 +0200 Subject: [PATCH 0527/1927] fbdev/core: Avoid uninitialized read in aperture_remove_conflicting_pci_device() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return on error directly from the BAR-iterating loop instead of break+return. This is actually a cosmetic fix, since it would be highly unusual to have this called for a PCI device without any memory BARs. Fixes: 9d69ef183815 ("fbdev/core: Remove remove_conflicting_pci_framebuffers()") Signed-off-by: Michał Mirosław Signed-off-by: Thomas Zimmermann Link: https://patchwork.freedesktop.org/patch/msgid/e75323732bedc46d613d72ecb40f97e3bc75eea8.1666829073.git.mirq-linux@rere.qmqm.pl --- drivers/video/aperture.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/video/aperture.c b/drivers/video/aperture.c index 9e6bcc03a1a4a..41e77de1ea82c 100644 --- a/drivers/video/aperture.c +++ b/drivers/video/aperture.c @@ -340,12 +340,9 @@ int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *na size = pci_resource_len(pdev, bar); ret = aperture_remove_conflicting_devices(base, size, primary, name); if (ret) - break; + return ret; } - if (ret) - return ret; - /* * WARNING: Apparently we must kick fbdev drivers before vgacon, * otherwise the vga fbdev driver falls over. -- GitLab From 702de2c21eed04c67cefaaedc248ef16e5f6b293 Mon Sep 17 00:00:00 2001 From: Biju Das Date: Tue, 25 Oct 2022 16:56:55 +0100 Subject: [PATCH 0528/1927] can: rcar_canfd: rcar_canfd_handle_global_receive(): fix IRQ storm on global FIFO receive We are seeing an IRQ storm on the global receive IRQ line under heavy CAN bus load conditions with both CAN channels enabled. Conditions: The global receive IRQ line is shared between can0 and can1, either of the channels can trigger interrupt while the other channel's IRQ line is disabled (RFIE). When global a receive IRQ interrupt occurs, we mask the interrupt in the IRQ handler. Clearing and unmasking of the interrupt is happening in rx_poll(). There is a race condition where rx_poll() unmasks the interrupt, but the next IRQ handler does not mask the IRQ due to NAPIF_STATE_MISSED flag (e.g.: can0 RX FIFO interrupt is disabled and can1 is triggering RX interrupt, the delay in rx_poll() processing results in setting NAPIF_STATE_MISSED flag) leading to an IRQ storm. This patch fixes the issue by checking IRQ active and enabled before handling the IRQ on a particular channel. Fixes: dd3bd23eb438 ("can: rcar_canfd: Add Renesas R-Car CAN FD driver") Suggested-by: Marc Kleine-Budde Signed-off-by: Biju Das Link: https://lore.kernel.org/all/20221025155657.1426948-2-biju.das.jz@bp.renesas.com Cc: stable@vger.kernel.org [mkl: adjust commit message] Signed-off-by: Marc Kleine-Budde --- drivers/net/can/rcar/rcar_canfd.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 567620d215f83..ea828c1bd3a1a 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1157,11 +1157,13 @@ static void rcar_canfd_handle_global_receive(struct rcar_canfd_global *gpriv, u3 { struct rcar_canfd_channel *priv = gpriv->ch[ch]; u32 ridx = ch + RCANFD_RFFIFO_IDX; - u32 sts; + u32 sts, cc; /* Handle Rx interrupts */ sts = rcar_canfd_read(priv->base, RCANFD_RFSTS(gpriv, ridx)); - if (likely(sts & RCANFD_RFSTS_RFIF)) { + cc = rcar_canfd_read(priv->base, RCANFD_RFCC(gpriv, ridx)); + if (likely(sts & RCANFD_RFSTS_RFIF && + cc & RCANFD_RFCC_RFIE)) { if (napi_schedule_prep(&priv->napi)) { /* Disable Rx FIFO interrupts */ rcar_canfd_clear_bit(priv->base, -- GitLab From d887087c896881715c1a82f1d4f71fbfe5344ffd Mon Sep 17 00:00:00 2001 From: Biju Das Date: Tue, 25 Oct 2022 16:56:56 +0100 Subject: [PATCH 0529/1927] can: rcar_canfd: fix channel specific IRQ handling for RZ/G2L RZ/G2L has separate channel specific IRQs for transmit and error interrupts. But the IRQ handler processes both channels, even if there no interrupt occurred on one of the channels. This patch fixes the issue by passing a channel specific context parameter instead of global one for the IRQ register and the IRQ handler, it just handles the channel which is triggered the interrupt. Fixes: 76e9353a80e9 ("can: rcar_canfd: Add support for RZ/G2L family") Signed-off-by: Biju Das Link: https://lore.kernel.org/all/20221025155657.1426948-3-biju.das.jz@bp.renesas.com Cc: stable@vger.kernel.org [mkl: adjust commit message] Signed-off-by: Marc Kleine-Budde --- drivers/net/can/rcar/rcar_canfd.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index ea828c1bd3a1a..198da643ee6dd 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1246,11 +1246,9 @@ static void rcar_canfd_handle_channel_tx(struct rcar_canfd_global *gpriv, u32 ch static irqreturn_t rcar_canfd_channel_tx_interrupt(int irq, void *dev_id) { - struct rcar_canfd_global *gpriv = dev_id; - u32 ch; + struct rcar_canfd_channel *priv = dev_id; - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) - rcar_canfd_handle_channel_tx(gpriv, ch); + rcar_canfd_handle_channel_tx(priv->gpriv, priv->channel); return IRQ_HANDLED; } @@ -1278,11 +1276,9 @@ static void rcar_canfd_handle_channel_err(struct rcar_canfd_global *gpriv, u32 c static irqreturn_t rcar_canfd_channel_err_interrupt(int irq, void *dev_id) { - struct rcar_canfd_global *gpriv = dev_id; - u32 ch; + struct rcar_canfd_channel *priv = dev_id; - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->max_channels) - rcar_canfd_handle_channel_err(gpriv, ch); + rcar_canfd_handle_channel_err(priv->gpriv, priv->channel); return IRQ_HANDLED; } @@ -1723,6 +1719,7 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, priv->ndev = ndev; priv->base = gpriv->base; priv->channel = ch; + priv->gpriv = gpriv; priv->can.clock.freq = fcan_freq; dev_info(&pdev->dev, "can_clk rate is %u\n", priv->can.clock.freq); @@ -1751,7 +1748,7 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, } err = devm_request_irq(&pdev->dev, err_irq, rcar_canfd_channel_err_interrupt, 0, - irq_name, gpriv); + irq_name, priv); if (err) { dev_err(&pdev->dev, "devm_request_irq CH Err(%d) failed, error %d\n", err_irq, err); @@ -1765,7 +1762,7 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, } err = devm_request_irq(&pdev->dev, tx_irq, rcar_canfd_channel_tx_interrupt, 0, - irq_name, gpriv); + irq_name, priv); if (err) { dev_err(&pdev->dev, "devm_request_irq Tx (%d) failed, error %d\n", tx_irq, err); @@ -1791,7 +1788,6 @@ static int rcar_canfd_channel_probe(struct rcar_canfd_global *gpriv, u32 ch, priv->can.do_set_mode = rcar_canfd_do_set_mode; priv->can.do_get_berr_counter = rcar_canfd_get_berr_counter; - priv->gpriv = gpriv; SET_NETDEV_DEV(ndev, &pdev->dev); netif_napi_add_weight(ndev, &priv->napi, rcar_canfd_rx_poll, -- GitLab From 4b66ff46f2e18b1d32e18c881799ef911606f3be Mon Sep 17 00:00:00 2001 From: James Clark Date: Wed, 12 Oct 2022 15:38:57 +0100 Subject: [PATCH 0530/1927] perf: Fix missing raw data on tracepoint events Since commit 838d9bb62d13 ("perf: Use sample_flags for raw_data") raw data is not being output on tracepoints due to the PERF_SAMPLE_RAW field not being set. Fix this by setting it for tracepoint events. This fixes the following test failure: perf test "sched_switch" -vvv 35: Track with sched_switch --- start --- test child forked, pid 1828 ... Using CPUID 0x00000000410fd400 sched_switch: cpu: 2 prev_tid -14687 next_tid 0 sched_switch: cpu: 2 prev_tid -14687 next_tid 0 Missing sched_switch events 4613 events recorded test child finished with -1 ---- end ---- Track with sched_switch: FAILED! Fixes: 838d9bb62d13 ("perf: Use sample_flags for raw_data") Signed-off-by: James Clark Signed-off-by: Peter Zijlstra (Intel) Acked-by: Namhyung Kim Tested-by: SeongJae Park Tested-by: Athira Rajeev Link: https://lore.kernel.org/r/20221012143857.48198-1-james.clark@arm.com --- kernel/events/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/events/core.c b/kernel/events/core.c index 01933db7629c1..4ec3717003d55 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9846,6 +9846,7 @@ void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, perf_sample_data_init(&data, 0, 0); data.raw = &raw; + data.sample_flags |= PERF_SAMPLE_RAW; perf_trace_buf_update(record, event_type); -- GitLab From 1ab28f17eeeecf7d832e686fdd903d74569854ed Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Sun, 23 Oct 2022 20:51:19 +0800 Subject: [PATCH 0531/1927] perf/x86/rapl: Add support for Intel AlderLake-N AlderLake-N RAPL support is the same as previous Sky Lake. Add AlderLake-N model for RAPL. Signed-off-by: Zhang Rui Signed-off-by: Peter Zijlstra (Intel) Tested-by: Wang Wendy Link: https://lkml.kernel.org/r/20221023125120.2727-1-rui.zhang@intel.com --- arch/x86/events/rapl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index 77e3a47af5ad5..165c506cd37cd 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -806,6 +806,7 @@ static const struct x86_cpu_id rapl_model_match[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &model_skl), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &model_skl), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &model_skl), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &model_spr), {}, }; -- GitLab From eff98a7421b3ee73d62268115ffa5bfc0ba94544 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Sun, 23 Oct 2022 20:51:20 +0800 Subject: [PATCH 0532/1927] perf/x86/rapl: Add support for Intel Raptor Lake Raptor Lake RAPL support is the same as previous Sky Lake. Add Raptor Lake model for RAPL. Signed-off-by: Zhang Rui Signed-off-by: Peter Zijlstra (Intel) Tested-by: Wang Wendy Link: https://lkml.kernel.org/r/20221023125120.2727-2-rui.zhang@intel.com --- arch/x86/events/rapl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index 165c506cd37cd..fea544e5842a1 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -808,6 +808,9 @@ static const struct x86_cpu_id rapl_model_match[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &model_skl), X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N, &model_skl), X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &model_spr), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P, &model_skl), + X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S, &model_skl), {}, }; MODULE_DEVICE_TABLE(x86cpu, rapl_model_match); -- GitLab From cb6c18b5a41622c7a439508f7421f8766a91cb87 Mon Sep 17 00:00:00 2001 From: Ravi Bangoria Date: Sat, 1 Oct 2022 11:37:05 +0530 Subject: [PATCH 0533/1927] perf/mem: Rename PERF_MEM_LVLNUM_EXTN_MEM to PERF_MEM_LVLNUM_CXL PERF_MEM_LVLNUM_EXTN_MEM was introduced to cover CXL devices but it's bit ambiguous name and also not generic enough to cover cxl.cache and cxl.io devices. Rename it to PERF_MEM_LVLNUM_CXL to be more specific. Signed-off-by: Ravi Bangoria Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/f6268268-b4e9-9ed6-0453-65792644d953@amd.com --- arch/x86/events/amd/ibs.c | 2 +- include/uapi/linux/perf_event.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 3271735f00702..4cb710efbdd9a 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -801,7 +801,7 @@ static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2, /* Extension Memory */ if (ibs_caps & IBS_CAPS_ZEN4 && ibs_data_src == IBS_DATA_SRC_EXT_EXT_MEM) { - data_src->mem_lvl_num = PERF_MEM_LVLNUM_EXTN_MEM; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_CXL; if (op_data2->rmt_node) { data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; /* IBS doesn't provide Remote socket detail */ diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h index 85be78e0e7f65..ccb7f5dad59be 100644 --- a/include/uapi/linux/perf_event.h +++ b/include/uapi/linux/perf_event.h @@ -1337,7 +1337,7 @@ union perf_mem_data_src { #define PERF_MEM_LVLNUM_L3 0x03 /* L3 */ #define PERF_MEM_LVLNUM_L4 0x04 /* L4 */ /* 5-0x8 available */ -#define PERF_MEM_LVLNUM_EXTN_MEM 0x09 /* Extension memory */ +#define PERF_MEM_LVLNUM_CXL 0x09 /* CXL */ #define PERF_MEM_LVLNUM_IO 0x0a /* I/O */ #define PERF_MEM_LVLNUM_ANY_CACHE 0x0b /* Any cache */ #define PERF_MEM_LVLNUM_LFB 0x0c /* LFB */ -- GitLab From 86c4f0d547f6460d0426ebb3ba0614f1134b8cda Mon Sep 17 00:00:00 2001 From: Jim Mattson Date: Thu, 29 Sep 2022 15:52:03 -0700 Subject: [PATCH 0534/1927] KVM: x86: Mask off reserved bits in CPUID.8000001FH KVM_GET_SUPPORTED_CPUID should only enumerate features that KVM actually supports. CPUID.8000001FH:EBX[31:16] are reserved bits and should be masked off. Fixes: 8765d75329a3 ("KVM: X86: Extend CPUID range to include new leaf") Signed-off-by: Jim Mattson Message-Id: <20220929225203.2234702-6-jmattson@google.com> Cc: stable@vger.kernel.org [Clear NumVMPL too. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index a0292ba650df3..0810e93cbedc0 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1199,7 +1199,8 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function) entry->eax = entry->ebx = entry->ecx = entry->edx = 0; } else { cpuid_entry_override(entry, CPUID_8000_001F_EAX); - + /* Clear NumVMPL since KVM does not support VMPL. */ + entry->ebx &= ~GENMASK(31, 12); /* * Enumerate '0' for "PA bits reduction", the adjusted * MAXPHYADDR is enumerated directly (see 0x80000008). -- GitLab From 5aa02366773376a1fd3a5c6a815e5f6e026ab391 Mon Sep 17 00:00:00 2001 From: Hou Wenlong Date: Fri, 14 Oct 2022 15:55:11 +0800 Subject: [PATCH 0535/1927] KVM: x86: Reduce refcount if single_open() fails in kvm_mmu_rmaps_stat_open() Refcount is increased before calling single_open() in kvm_mmu_rmaps_stat_open(), If single_open() fails, refcount should be restored, otherwise the vm couldn't be destroyed. Fixes: 3bcd0662d66fd ("KVM: X86: Introduce mmu_rmaps_stat per-vm debugfs file") Signed-off-by: Hou Wenlong Message-Id: [Preserved return value of single_open. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/debugfs.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index cfed36aba2f70..c1390357126ab 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -158,11 +158,16 @@ static int kvm_mmu_rmaps_stat_show(struct seq_file *m, void *v) static int kvm_mmu_rmaps_stat_open(struct inode *inode, struct file *file) { struct kvm *kvm = inode->i_private; + int r; if (!kvm_get_kvm_safe(kvm)) return -ENOENT; - return single_open(file, kvm_mmu_rmaps_stat_show, kvm); + r = single_open(file, kvm_mmu_rmaps_stat_show, kvm); + if (r < 0) + kvm_put_kvm(kvm); + + return r; } static int kvm_mmu_rmaps_stat_release(struct inode *inode, struct file *file) -- GitLab From 180418e2eb33be5c8d0b703c843e0ebc045aef80 Mon Sep 17 00:00:00 2001 From: Hou Wenlong Date: Mon, 17 Oct 2022 11:06:10 +0800 Subject: [PATCH 0536/1927] KVM: debugfs: Return retval of simple_attr_open() if it fails Although simple_attr_open() fails only with -ENOMEM with current code base, it would be nicer to return retval of simple_attr_open() directly in kvm_debugfs_open(). No functional change intended. Signed-off-by: Hou Wenlong Message-Id: <69d64d93accd1f33691b8a383ae555baee80f943.1665975828.git.houwenlong.hwl@antgroup.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- virt/kvm/kvm_main.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 1376a47fedeed..f1df24c2bc84f 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5409,6 +5409,7 @@ static int kvm_debugfs_open(struct inode *inode, struct file *file, int (*get)(void *, u64 *), int (*set)(void *, u64), const char *fmt) { + int ret; struct kvm_stat_data *stat_data = (struct kvm_stat_data *) inode->i_private; @@ -5420,15 +5421,13 @@ static int kvm_debugfs_open(struct inode *inode, struct file *file, if (!kvm_get_kvm_safe(stat_data->kvm)) return -ENOENT; - if (simple_attr_open(inode, file, get, - kvm_stats_debugfs_mode(stat_data->desc) & 0222 - ? set : NULL, - fmt)) { + ret = simple_attr_open(inode, file, get, + kvm_stats_debugfs_mode(stat_data->desc) & 0222 + ? set : NULL, fmt); + if (ret) kvm_put_kvm(stat_data->kvm); - return -ENOMEM; - } - return 0; + return ret; } static int kvm_debugfs_release(struct inode *inode, struct file *file) -- GitLab From 44fc40a015af7511408f7b447e2c0c2da056fd95 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 13 Oct 2022 23:46:38 +0200 Subject: [PATCH 0537/1927] MAINTAINERS: git://github -> https://github.com for kvm-riscv Github deprecated the git:// links about a year ago, so let's move to the https:// URLs instead. Reported-by: Conor Dooley Link: https://github.blog/2021-09-01-improving-git-protocol-security-github/ Signed-off-by: Palmer Dabbelt Signed-off-by: Paolo Bonzini --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..7d62b16409306 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11241,7 +11241,7 @@ L: kvm@vger.kernel.org L: kvm-riscv@lists.infradead.org L: linux-riscv@lists.infradead.org S: Maintained -T: git git://github.com/kvm-riscv/linux.git +T: git https://github.com/kvm-riscv/linux.git F: arch/riscv/include/asm/kvm* F: arch/riscv/include/uapi/asm/kvm* F: arch/riscv/kvm/ -- GitLab From dea0d5a2fde62237ff14c41cb05dd151cebf84c0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 30 Sep 2022 23:00:08 +0000 Subject: [PATCH 0538/1927] KVM: x86: Exempt pending triple fault from event injection sanity check Exempt pending triple faults, a.k.a. KVM_REQ_TRIPLE_FAULT, when asserting that KVM didn't attempt to queue a new exception during event injection. KVM needs to emulate the injection itself when emulating Real Mode due to lack of unrestricted guest support (VMX) and will queue a triple fault if that emulation fails. Ideally the assertion would more precisely filter out the emulated Real Mode triple fault case, but rmode.vm86_active is buried in vcpu_vmx and can't be queried without a new kvm_x86_ops. And unlike "regular" exceptions, triple fault cannot put the vCPU into an infinite loop; the triple fault will force either an exit to userspace or a nested VM-Exit, and triple fault after nested VM-Exit will force an exit to userspace. I.e. there is no functional issue, so just suppress the warning for triple faults. Opportunistically convert the warning to a one-time thing, when it fires, it fires _a lot_, and is usually user triggerable, i.e. can be used to spam the kernel log. Fixes: 7055fb113116 ("KVM: x86: Treat pending TRIPLE_FAULT requests as pending exceptions") Reported-by: kernel test robot Link: https://lore.kernel.org/r/202209301338.aca913c3-yujie.liu@intel.com Signed-off-by: Sean Christopherson Message-Id: <20220930230008.1636044-1-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 9cf1ba865562e..104b72df33d6a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10044,7 +10044,20 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, kvm_x86_ops.nested_ops->has_events(vcpu)) *req_immediate_exit = true; - WARN_ON(kvm_is_exception_pending(vcpu)); + /* + * KVM must never queue a new exception while injecting an event; KVM + * is done emulating and should only propagate the to-be-injected event + * to the VMCS/VMCB. Queueing a new exception can put the vCPU into an + * infinite loop as KVM will bail from VM-Enter to inject the pending + * exception and start the cycle all over. + * + * Exempt triple faults as they have special handling and won't put the + * vCPU into an infinite loop. Triple fault can be queued when running + * VMX without unrestricted guest, as that requires KVM to emulate Real + * Mode events (see kvm_inject_realmode_interrupt()). + */ + WARN_ON_ONCE(vcpu->arch.exception.pending || + vcpu->arch.exception_vmexit.pending); return 0; out: -- GitLab From 1c1a41497ab879ac9608f3047f230af833eeef3d Mon Sep 17 00:00:00 2001 From: Emanuele Giuseppe Esposito Date: Tue, 25 Oct 2022 08:37:49 -0400 Subject: [PATCH 0539/1927] KVM: VMX: fully disable SGX if SECONDARY_EXEC_ENCLS_EXITING unavailable Clear enable_sgx if ENCLS-exiting is not supported, i.e. if SGX cannot be virtualized. When KVM is loaded, adjust_vmx_controls checks that the bit is available before enabling the feature; however, other parts of the code check enable_sgx and not clearing the variable caused two different bugs, mostly affecting nested virtualization scenarios. First, because enable_sgx remained true, SECONDARY_EXEC_ENCLS_EXITING would be marked available in the capability MSR that are accessed by a nested hypervisor. KVM would then propagate the control from vmcs12 to vmcs02 even if it isn't supported by the processor, thus causing an unexpected VM-Fail (exit code 0x7) in L1. Second, vmx_set_cpu_caps() would not clear the SGX bits when hardware support is unavailable. This is a much less problematic bug as it only happens if SGX is soft-disabled (available in the processor but hidden in CPUID) or if SGX is supported for bare metal but not in the VMCS (will never happen when running on bare metal, but can theoertically happen when running in a VM). Last but not least, this ensures that module params in sysfs reflect KVM's actual configuration. RHBZ: https://bugzilla.redhat.com/show_bug.cgi?id=2127128 Fixes: 72add915fbd5 ("KVM: VMX: Enable SGX virtualization for SGX1, SGX2 and LC") Cc: stable@vger.kernel.org Suggested-by: Sean Christopherson Suggested-by: Bandan Das Signed-off-by: Emanuele Giuseppe Esposito Message-Id: <20221025123749.2201649-1-eesposit@redhat.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 9dba04b6b019a..65f092e4a81b3 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8263,6 +8263,11 @@ static __init int hardware_setup(void) if (!cpu_has_virtual_nmis()) enable_vnmi = 0; +#ifdef CONFIG_X86_SGX_KVM + if (!cpu_has_vmx_encls_vmexit()) + enable_sgx = false; +#endif + /* * set_apic_access_page_addr() is used to reload apic access * page upon invalidation. No need to do anything if not -- GitLab From fd954cc1919e35cb92f78671cab6e42d661945a3 Mon Sep 17 00:00:00 2001 From: Aaron Conole Date: Tue, 25 Oct 2022 06:50:17 -0400 Subject: [PATCH 0540/1927] openvswitch: switch from WARN to pr_warn As noted by Paolo Abeni, pr_warn doesn't generate any splat and can still preserve the warning to the user that feature downgrade occurred. We likely cannot introduce other kinds of checks / enforcement here because syzbot can generate different genl versions to the datapath. Reported-by: syzbot+31cde0bef4bbf8ba2d86@syzkaller.appspotmail.com Fixes: 44da5ae5fbea ("openvswitch: Drop user features if old user space attempted to create datapath") Cc: Thomas Graf Signed-off-by: Aaron Conole Acked-by: Ilya Maximets Signed-off-by: Paolo Abeni --- net/openvswitch/datapath.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index c8a9075ddd0a8..155263e735124 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -1616,7 +1616,8 @@ static void ovs_dp_reset_user_features(struct sk_buff *skb, if (IS_ERR(dp)) return; - WARN(dp->user_features, "Dropping previously announced user features\n"); + pr_warn("%s: Dropping previously announced user features\n", + ovs_dp_name(dp)); dp->user_features = 0; } -- GitLab From 25f16c873fb1aa8ba870319c9614f7ff7502d35b Mon Sep 17 00:00:00 2001 From: Aaron Conole Date: Tue, 25 Oct 2022 06:50:18 -0400 Subject: [PATCH 0541/1927] selftests: add openvswitch selftest suite Previous commit resolves a WARN splat that can be difficult to reproduce, but with the ovs-dpctl.py utility, it can be trivial. Introduce a test case which creates a DP, and then downgrades the feature set. This will include a utility 'ovs-dpctl.py' that can be extended to do additional tests and diagnostics. Signed-off-by: Aaron Conole Signed-off-by: Paolo Abeni --- MAINTAINERS | 1 + tools/testing/selftests/Makefile | 1 + .../selftests/net/openvswitch/Makefile | 13 + .../selftests/net/openvswitch/openvswitch.sh | 218 +++++++++++ .../selftests/net/openvswitch/ovs-dpctl.py | 351 ++++++++++++++++++ 5 files changed, 584 insertions(+) create mode 100644 tools/testing/selftests/net/openvswitch/Makefile create mode 100755 tools/testing/selftests/net/openvswitch/openvswitch.sh create mode 100644 tools/testing/selftests/net/openvswitch/ovs-dpctl.py diff --git a/MAINTAINERS b/MAINTAINERS index 3bb30c0d1cb44..10c1344b44730 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -15436,6 +15436,7 @@ S: Maintained W: http://openvswitch.org F: include/uapi/linux/openvswitch.h F: net/openvswitch/ +F: tools/testing/selftests/net/openvswitch/ OPERATING PERFORMANCE POINTS (OPP) M: Viresh Kumar diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index 0464b2c6c1e4b..f07aef7c592c2 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -49,6 +49,7 @@ TARGETS += net TARGETS += net/af_unix TARGETS += net/forwarding TARGETS += net/mptcp +TARGETS += net/openvswitch TARGETS += netfilter TARGETS += nsfs TARGETS += pidfd diff --git a/tools/testing/selftests/net/openvswitch/Makefile b/tools/testing/selftests/net/openvswitch/Makefile new file mode 100644 index 0000000000000..2f1508abc826b --- /dev/null +++ b/tools/testing/selftests/net/openvswitch/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 + +top_srcdir = ../../../../.. + +CFLAGS = -Wall -Wl,--no-as-needed -O2 -g -I$(top_srcdir)/usr/include $(KHDR_INCLUDES) + +TEST_PROGS := openvswitch.sh + +TEST_FILES := ovs-dpctl.py + +EXTRA_CLEAN := test_netlink_checks + +include ../../lib.mk diff --git a/tools/testing/selftests/net/openvswitch/openvswitch.sh b/tools/testing/selftests/net/openvswitch/openvswitch.sh new file mode 100755 index 0000000000000..7ce46700a3ae3 --- /dev/null +++ b/tools/testing/selftests/net/openvswitch/openvswitch.sh @@ -0,0 +1,218 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# OVS kernel module self tests + +# Kselftest framework requirement - SKIP code is 4. +ksft_skip=4 + +PAUSE_ON_FAIL=no +VERBOSE=0 +TRACING=0 + +tests=" + netlink_checks ovsnl: validate netlink attrs and settings" + +info() { + [ $VERBOSE = 0 ] || echo $* +} + +ovs_base=`pwd` +sbxs= +sbx_add () { + info "adding sandbox '$1'" + + sbxs="$sbxs $1" + + NO_BIN=0 + + # Create sandbox. + local d="$ovs_base"/$1 + if [ -e $d ]; then + info "removing $d" + rm -rf "$d" + fi + mkdir "$d" || return 1 + ovs_setenv $1 +} + +ovs_exit_sig() { + [ -e ${ovs_dir}/cleanup ] && . "$ovs_dir/cleanup" +} + +on_exit() { + echo "$1" > ${ovs_dir}/cleanup.tmp + cat ${ovs_dir}/cleanup >> ${ovs_dir}/cleanup.tmp + mv ${ovs_dir}/cleanup.tmp ${ovs_dir}/cleanup +} + +ovs_setenv() { + sandbox=$1 + + ovs_dir=$ovs_base${1:+/$1}; export ovs_dir + + test -e ${ovs_dir}/cleanup || : > ${ovs_dir}/cleanup +} + +ovs_sbx() { + if test "X$2" != X; then + (ovs_setenv $1; shift; "$@" >> ${ovs_dir}/debug.log) + else + ovs_setenv $1 + fi +} + +ovs_add_dp () { + info "Adding DP/Bridge IF: sbx:$1 dp:$2 {$3, $4, $5}" + sbxname="$1" + shift + ovs_sbx "$sbxname" python3 $ovs_base/ovs-dpctl.py add-dp $* + on_exit "ovs_sbx $sbxname python3 $ovs_base/ovs-dpctl.py del-dp $1;" +} + +usage() { + echo + echo "$0 [OPTIONS] [TEST]..." + echo "If no TEST argument is given, all tests will be run." + echo + echo "Options" + echo " -t: capture traffic via tcpdump" + echo " -v: verbose" + echo " -p: pause on failure" + echo + echo "Available tests${tests}" + exit 1 +} + +# netlink_validation +# - Create a dp +# - check no warning with "old version" simulation +test_netlink_checks () { + sbx_add "test_netlink_checks" || return 1 + + info "setting up new DP" + ovs_add_dp "test_netlink_checks" nv0 || return 1 + # now try again + PRE_TEST=$(dmesg | grep -E "RIP: [0-9a-fA-Fx]+:ovs_dp_cmd_new\+") + ovs_add_dp "test_netlink_checks" nv0 -V 0 || return 1 + POST_TEST=$(dmesg | grep -E "RIP: [0-9a-fA-Fx]+:ovs_dp_cmd_new\+") + if [ "$PRE_TEST" != "$POST_TEST" ]; then + info "failed - gen warning" + return 1 + fi + + return 0 +} + +run_test() { + ( + tname="$1" + tdesc="$2" + + if ! lsmod | grep openvswitch >/dev/null 2>&1; then + stdbuf -o0 printf "TEST: %-60s [NOMOD]\n" "${tdesc}" + return $ksft_skip + fi + + if python3 ovs-dpctl.py -h 2>&1 | \ + grep "Need to install the python" >/dev/null 2>&1; then + stdbuf -o0 printf "TEST: %-60s [PYLIB]\n" "${tdesc}" + return $ksft_skip + fi + printf "TEST: %-60s [START]\n" "${tname}" + + unset IFS + + eval test_${tname} + ret=$? + + if [ $ret -eq 0 ]; then + printf "TEST: %-60s [ OK ]\n" "${tdesc}" + ovs_exit_sig + rm -rf "$ovs_dir" + elif [ $ret -eq 1 ]; then + printf "TEST: %-60s [FAIL]\n" "${tdesc}" + if [ "${PAUSE_ON_FAIL}" = "yes" ]; then + echo + echo "Pausing. Logs in $ovs_dir/. Hit enter to continue" + read a + fi + ovs_exit_sig + [ "${PAUSE_ON_FAIL}" = "yes" ] || rm -rf "$ovs_dir" + exit 1 + elif [ $ret -eq $ksft_skip ]; then + printf "TEST: %-60s [SKIP]\n" "${tdesc}" + elif [ $ret -eq 2 ]; then + rm -rf test_${tname} + run_test "$1" "$2" + fi + + return $ret + ) + ret=$? + case $ret in + 0) + [ $all_skipped = true ] && [ $exitcode=$ksft_skip ] && exitcode=0 + all_skipped=false + ;; + $ksft_skip) + [ $all_skipped = true ] && exitcode=$ksft_skip + ;; + *) + all_skipped=false + exitcode=1 + ;; + esac + + return $ret +} + + +exitcode=0 +desc=0 +all_skipped=true + +while getopts :pvt o +do + case $o in + p) PAUSE_ON_FAIL=yes;; + v) VERBOSE=1;; + t) if which tcpdump > /dev/null 2>&1; then + TRACING=1 + else + echo "=== tcpdump not available, tracing disabled" + fi + ;; + *) usage;; + esac +done +shift $(($OPTIND-1)) + +IFS=" +" + +for arg do + # Check first that all requested tests are available before running any + command -v > /dev/null "test_${arg}" || { echo "=== Test ${arg} not found"; usage; } +done + +name="" +desc="" +for t in ${tests}; do + [ "${name}" = "" ] && name="${t}" && continue + [ "${desc}" = "" ] && desc="${t}" + + run_this=1 + for arg do + [ "${arg}" != "${arg#--*}" ] && continue + [ "${arg}" = "${name}" ] && run_this=1 && break + run_this=0 + done + if [ $run_this -eq 1 ]; then + run_test "${name}" "${desc}" + fi + name="" + desc="" +done + +exit ${exitcode} diff --git a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py new file mode 100644 index 0000000000000..3243c90d449e6 --- /dev/null +++ b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +# Controls the openvswitch module. Part of the kselftest suite, but +# can be used for some diagnostic purpose as well. + +import argparse +import errno +import sys + +try: + from pyroute2 import NDB + + from pyroute2.netlink import NLM_F_ACK + from pyroute2.netlink import NLM_F_REQUEST + from pyroute2.netlink import genlmsg + from pyroute2.netlink import nla + from pyroute2.netlink.exceptions import NetlinkError + from pyroute2.netlink.generic import GenericNetlinkSocket +except ModuleNotFoundError: + print("Need to install the python pyroute2 package.") + sys.exit(0) + + +OVS_DATAPATH_FAMILY = "ovs_datapath" +OVS_VPORT_FAMILY = "ovs_vport" +OVS_FLOW_FAMILY = "ovs_flow" +OVS_PACKET_FAMILY = "ovs_packet" +OVS_METER_FAMILY = "ovs_meter" +OVS_CT_LIMIT_FAMILY = "ovs_ct_limit" + +OVS_DATAPATH_VERSION = 2 +OVS_DP_CMD_NEW = 1 +OVS_DP_CMD_DEL = 2 +OVS_DP_CMD_GET = 3 +OVS_DP_CMD_SET = 4 + +OVS_VPORT_CMD_NEW = 1 +OVS_VPORT_CMD_DEL = 2 +OVS_VPORT_CMD_GET = 3 +OVS_VPORT_CMD_SET = 4 + + +class ovs_dp_msg(genlmsg): + # include the OVS version + # We need a custom header rather than just being able to rely on + # genlmsg because fields ends up not expressing everything correctly + # if we use the canonical example of setting fields = (('customfield',),) + fields = genlmsg.fields + (("dpifindex", "I"),) + + +class OvsDatapath(GenericNetlinkSocket): + + OVS_DP_F_VPORT_PIDS = 1 << 1 + OVS_DP_F_DISPATCH_UPCALL_PER_CPU = 1 << 3 + + class dp_cmd_msg(ovs_dp_msg): + """ + Message class that will be used to communicate with the kernel module. + """ + + nla_map = ( + ("OVS_DP_ATTR_UNSPEC", "none"), + ("OVS_DP_ATTR_NAME", "asciiz"), + ("OVS_DP_ATTR_UPCALL_PID", "uint32"), + ("OVS_DP_ATTR_STATS", "dpstats"), + ("OVS_DP_ATTR_MEGAFLOW_STATS", "megaflowstats"), + ("OVS_DP_ATTR_USER_FEATURES", "uint32"), + ("OVS_DP_ATTR_PAD", "none"), + ("OVS_DP_ATTR_MASKS_CACHE_SIZE", "uint32"), + ("OVS_DP_ATTR_PER_CPU_PIDS", "array(uint32)"), + ) + + class dpstats(nla): + fields = ( + ("hit", "=Q"), + ("missed", "=Q"), + ("lost", "=Q"), + ("flows", "=Q"), + ) + + class megaflowstats(nla): + fields = ( + ("mask_hit", "=Q"), + ("masks", "=I"), + ("padding", "=I"), + ("cache_hits", "=Q"), + ("pad1", "=Q"), + ) + + def __init__(self): + GenericNetlinkSocket.__init__(self) + self.bind(OVS_DATAPATH_FAMILY, OvsDatapath.dp_cmd_msg) + + def info(self, dpname, ifindex=0): + msg = OvsDatapath.dp_cmd_msg() + msg["cmd"] = OVS_DP_CMD_GET + msg["version"] = OVS_DATAPATH_VERSION + msg["reserved"] = 0 + msg["dpifindex"] = ifindex + msg["attrs"].append(["OVS_DP_ATTR_NAME", dpname]) + + try: + reply = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST + ) + reply = reply[0] + except NetlinkError as ne: + if ne.code == errno.ENODEV: + reply = None + else: + raise ne + + return reply + + def create(self, dpname, shouldUpcall=False, versionStr=None): + msg = OvsDatapath.dp_cmd_msg() + msg["cmd"] = OVS_DP_CMD_NEW + if versionStr is None: + msg["version"] = OVS_DATAPATH_VERSION + else: + msg["version"] = int(versionStr.split(":")[0], 0) + msg["reserved"] = 0 + msg["dpifindex"] = 0 + msg["attrs"].append(["OVS_DP_ATTR_NAME", dpname]) + + dpfeatures = 0 + if versionStr is not None and versionStr.find(":") != -1: + dpfeatures = int(versionStr.split(":")[1], 0) + else: + dpfeatures = OvsDatapath.OVS_DP_F_VPORT_PIDS + + msg["attrs"].append(["OVS_DP_ATTR_USER_FEATURES", dpfeatures]) + if not shouldUpcall: + msg["attrs"].append(["OVS_DP_ATTR_UPCALL_PID", 0]) + + try: + reply = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) + reply = reply[0] + except NetlinkError as ne: + if ne.code == errno.EEXIST: + reply = None + else: + raise ne + + return reply + + def destroy(self, dpname): + msg = OvsDatapath.dp_cmd_msg() + msg["cmd"] = OVS_DP_CMD_DEL + msg["version"] = OVS_DATAPATH_VERSION + msg["reserved"] = 0 + msg["dpifindex"] = 0 + msg["attrs"].append(["OVS_DP_ATTR_NAME", dpname]) + + try: + reply = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST | NLM_F_ACK + ) + reply = reply[0] + except NetlinkError as ne: + if ne.code == errno.ENODEV: + reply = None + else: + raise ne + + return reply + + +class OvsVport(GenericNetlinkSocket): + class ovs_vport_msg(ovs_dp_msg): + nla_map = ( + ("OVS_VPORT_ATTR_UNSPEC", "none"), + ("OVS_VPORT_ATTR_PORT_NO", "uint32"), + ("OVS_VPORT_ATTR_TYPE", "uint32"), + ("OVS_VPORT_ATTR_NAME", "asciiz"), + ("OVS_VPORT_ATTR_OPTIONS", "none"), + ("OVS_VPORT_ATTR_UPCALL_PID", "array(uint32)"), + ("OVS_VPORT_ATTR_STATS", "vportstats"), + ("OVS_VPORT_ATTR_PAD", "none"), + ("OVS_VPORT_ATTR_IFINDEX", "uint32"), + ("OVS_VPORT_ATTR_NETNSID", "uint32"), + ) + + class vportstats(nla): + fields = ( + ("rx_packets", "=Q"), + ("tx_packets", "=Q"), + ("rx_bytes", "=Q"), + ("tx_bytes", "=Q"), + ("rx_errors", "=Q"), + ("tx_errors", "=Q"), + ("rx_dropped", "=Q"), + ("tx_dropped", "=Q"), + ) + + def type_to_str(vport_type): + if vport_type == 1: + return "netdev" + elif vport_type == 2: + return "internal" + elif vport_type == 3: + return "gre" + elif vport_type == 4: + return "vxlan" + elif vport_type == 5: + return "geneve" + return "unknown:%d" % vport_type + + def __init__(self): + GenericNetlinkSocket.__init__(self) + self.bind(OVS_VPORT_FAMILY, OvsVport.ovs_vport_msg) + + def info(self, vport_name, dpifindex=0, portno=None): + msg = OvsVport.ovs_vport_msg() + + msg["cmd"] = OVS_VPORT_CMD_GET + msg["version"] = OVS_DATAPATH_VERSION + msg["reserved"] = 0 + msg["dpifindex"] = dpifindex + + if portno is None: + msg["attrs"].append(["OVS_VPORT_ATTR_NAME", vport_name]) + else: + msg["attrs"].append(["OVS_VPORT_ATTR_PORT_NO", portno]) + + try: + reply = self.nlm_request( + msg, msg_type=self.prid, msg_flags=NLM_F_REQUEST + ) + reply = reply[0] + except NetlinkError as ne: + if ne.code == errno.ENODEV: + reply = None + else: + raise ne + return reply + + +def print_ovsdp_full(dp_lookup_rep, ifindex, ndb=NDB()): + dp_name = dp_lookup_rep.get_attr("OVS_DP_ATTR_NAME") + base_stats = dp_lookup_rep.get_attr("OVS_DP_ATTR_STATS") + megaflow_stats = dp_lookup_rep.get_attr("OVS_DP_ATTR_MEGAFLOW_STATS") + user_features = dp_lookup_rep.get_attr("OVS_DP_ATTR_USER_FEATURES") + masks_cache_size = dp_lookup_rep.get_attr("OVS_DP_ATTR_MASKS_CACHE_SIZE") + + print("%s:" % dp_name) + print( + " lookups: hit:%d missed:%d lost:%d" + % (base_stats["hit"], base_stats["missed"], base_stats["lost"]) + ) + print(" flows:%d" % base_stats["flows"]) + pkts = base_stats["hit"] + base_stats["missed"] + avg = (megaflow_stats["mask_hit"] / pkts) if pkts != 0 else 0.0 + print( + " masks: hit:%d total:%d hit/pkt:%f" + % (megaflow_stats["mask_hit"], megaflow_stats["masks"], avg) + ) + print(" caches:") + print(" masks-cache: size:%d" % masks_cache_size) + + if user_features is not None: + print(" features: 0x%X" % user_features) + + # port print out + vpl = OvsVport() + for iface in ndb.interfaces: + rep = vpl.info(iface.ifname, ifindex) + if rep is not None: + print( + " port %d: %s (%s)" + % ( + rep.get_attr("OVS_VPORT_ATTR_PORT_NO"), + rep.get_attr("OVS_VPORT_ATTR_NAME"), + OvsVport.type_to_str(rep.get_attr("OVS_VPORT_ATTR_TYPE")), + ) + ) + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument( + "-v", + "--verbose", + action="count", + help="Increment 'verbose' output counter.", + ) + subparsers = parser.add_subparsers() + + showdpcmd = subparsers.add_parser("show") + showdpcmd.add_argument( + "showdp", metavar="N", type=str, nargs="?", help="Datapath Name" + ) + + adddpcmd = subparsers.add_parser("add-dp") + adddpcmd.add_argument("adddp", help="Datapath Name") + adddpcmd.add_argument( + "-u", + "--upcall", + action="store_true", + help="Leave open a reader for upcalls", + ) + adddpcmd.add_argument( + "-V", + "--versioning", + required=False, + help="Specify a custom version / feature string", + ) + + deldpcmd = subparsers.add_parser("del-dp") + deldpcmd.add_argument("deldp", help="Datapath Name") + + args = parser.parse_args() + + ovsdp = OvsDatapath() + ndb = NDB() + + if hasattr(args, "showdp"): + found = False + for iface in ndb.interfaces: + rep = None + if args.showdp is None: + rep = ovsdp.info(iface.ifname, 0) + elif args.showdp == iface.ifname: + rep = ovsdp.info(iface.ifname, 0) + + if rep is not None: + found = True + print_ovsdp_full(rep, iface.index, ndb) + + if not found: + msg = "No DP found" + if args.showdp is not None: + msg += ":'%s'" % args.showdp + print(msg) + elif hasattr(args, "adddp"): + rep = ovsdp.create(args.adddp, args.upcall, args.versioning) + if rep is None: + print("DP '%s' already exists" % args.adddp) + else: + print("DP '%s' added" % args.adddp) + elif hasattr(args, "deldp"): + ovsdp.destroy(args.deldp) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) -- GitLab From 52491a38b2c2411f3f0229dc6ad610349c704a41 Mon Sep 17 00:00:00 2001 From: Michal Luczaj Date: Thu, 13 Oct 2022 21:12:19 +0000 Subject: [PATCH 0542/1927] KVM: Initialize gfn_to_pfn_cache locks in dedicated helper Move the gfn_to_pfn_cache lock initialization to another helper and call the new helper during VM/vCPU creation. There are race conditions possible due to kvm_gfn_to_pfn_cache_init()'s ability to re-initialize the cache's locks. For example: a race between ioctl(KVM_XEN_HVM_EVTCHN_SEND) and kvm_gfn_to_pfn_cache_init() leads to a corrupted shinfo gpc lock. (thread 1) | (thread 2) | kvm_xen_set_evtchn_fast | read_lock_irqsave(&gpc->lock, ...) | | kvm_gfn_to_pfn_cache_init | rwlock_init(&gpc->lock) read_unlock_irqrestore(&gpc->lock, ...) | Rename "cache_init" and "cache_destroy" to activate+deactivate to avoid implying that the cache really is destroyed/freed. Note, there more races in the newly named kvm_gpc_activate() that will be addressed separately. Fixes: 982ed0de4753 ("KVM: Reinstate gfn_to_pfn_cache with invalidation support") Cc: stable@vger.kernel.org Suggested-by: Sean Christopherson Signed-off-by: Michal Luczaj [sean: call out that this is a bug fix] Signed-off-by: Sean Christopherson Message-Id: <20221013211234.1318131-2-seanjc@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 12 +++++---- arch/x86/kvm/xen.c | 57 +++++++++++++++++++++------------------- include/linux/kvm_host.h | 24 ++++++++++++----- virt/kvm/pfncache.c | 21 ++++++++------- 4 files changed, 66 insertions(+), 48 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 104b72df33d6a..521b433f978c7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2315,11 +2315,11 @@ static void kvm_write_system_time(struct kvm_vcpu *vcpu, gpa_t system_time, /* we verify if the enable bit is set... */ if (system_time & 1) { - kvm_gfn_to_pfn_cache_init(vcpu->kvm, &vcpu->arch.pv_time, vcpu, - KVM_HOST_USES_PFN, system_time & ~1ULL, - sizeof(struct pvclock_vcpu_time_info)); + kvm_gpc_activate(vcpu->kvm, &vcpu->arch.pv_time, vcpu, + KVM_HOST_USES_PFN, system_time & ~1ULL, + sizeof(struct pvclock_vcpu_time_info)); } else { - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.pv_time); + kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.pv_time); } return; @@ -3388,7 +3388,7 @@ static int kvm_pv_enable_async_pf_int(struct kvm_vcpu *vcpu, u64 data) static void kvmclock_reset(struct kvm_vcpu *vcpu) { - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.pv_time); + kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.pv_time); vcpu->arch.time = 0; } @@ -11829,6 +11829,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.regs_avail = ~0; vcpu->arch.regs_dirty = ~0; + kvm_gpc_init(&vcpu->arch.pv_time); + if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; else diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 93c628d3e3a92..b2be60c6efa4b 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -42,13 +42,13 @@ static int kvm_xen_shared_info_init(struct kvm *kvm, gfn_t gfn) int idx = srcu_read_lock(&kvm->srcu); if (gfn == GPA_INVALID) { - kvm_gfn_to_pfn_cache_destroy(kvm, gpc); + kvm_gpc_deactivate(kvm, gpc); goto out; } do { - ret = kvm_gfn_to_pfn_cache_init(kvm, gpc, NULL, KVM_HOST_USES_PFN, - gpa, PAGE_SIZE); + ret = kvm_gpc_activate(kvm, gpc, NULL, KVM_HOST_USES_PFN, gpa, + PAGE_SIZE); if (ret) goto out; @@ -554,15 +554,15 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) offsetof(struct compat_vcpu_info, time)); if (data->u.gpa == GPA_INVALID) { - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache); + kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache); r = 0; break; } - r = kvm_gfn_to_pfn_cache_init(vcpu->kvm, - &vcpu->arch.xen.vcpu_info_cache, - NULL, KVM_HOST_USES_PFN, data->u.gpa, - sizeof(struct vcpu_info)); + r = kvm_gpc_activate(vcpu->kvm, + &vcpu->arch.xen.vcpu_info_cache, NULL, + KVM_HOST_USES_PFN, data->u.gpa, + sizeof(struct vcpu_info)); if (!r) kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); @@ -570,16 +570,16 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) case KVM_XEN_VCPU_ATTR_TYPE_VCPU_TIME_INFO: if (data->u.gpa == GPA_INVALID) { - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, - &vcpu->arch.xen.vcpu_time_info_cache); + kvm_gpc_deactivate(vcpu->kvm, + &vcpu->arch.xen.vcpu_time_info_cache); r = 0; break; } - r = kvm_gfn_to_pfn_cache_init(vcpu->kvm, - &vcpu->arch.xen.vcpu_time_info_cache, - NULL, KVM_HOST_USES_PFN, data->u.gpa, - sizeof(struct pvclock_vcpu_time_info)); + r = kvm_gpc_activate(vcpu->kvm, + &vcpu->arch.xen.vcpu_time_info_cache, + NULL, KVM_HOST_USES_PFN, data->u.gpa, + sizeof(struct pvclock_vcpu_time_info)); if (!r) kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); break; @@ -590,16 +590,15 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; } if (data->u.gpa == GPA_INVALID) { - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, - &vcpu->arch.xen.runstate_cache); + kvm_gpc_deactivate(vcpu->kvm, + &vcpu->arch.xen.runstate_cache); r = 0; break; } - r = kvm_gfn_to_pfn_cache_init(vcpu->kvm, - &vcpu->arch.xen.runstate_cache, - NULL, KVM_HOST_USES_PFN, data->u.gpa, - sizeof(struct vcpu_runstate_info)); + r = kvm_gpc_activate(vcpu->kvm, &vcpu->arch.xen.runstate_cache, + NULL, KVM_HOST_USES_PFN, data->u.gpa, + sizeof(struct vcpu_runstate_info)); break; case KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT: @@ -1816,7 +1815,12 @@ void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) { vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx; vcpu->arch.xen.poll_evtchn = 0; + timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0); + + kvm_gpc_init(&vcpu->arch.xen.runstate_cache); + kvm_gpc_init(&vcpu->arch.xen.vcpu_info_cache); + kvm_gpc_init(&vcpu->arch.xen.vcpu_time_info_cache); } void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) @@ -1824,18 +1828,17 @@ void kvm_xen_destroy_vcpu(struct kvm_vcpu *vcpu) if (kvm_xen_timer_enabled(vcpu)) kvm_xen_stop_timer(vcpu); - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, - &vcpu->arch.xen.runstate_cache); - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, - &vcpu->arch.xen.vcpu_info_cache); - kvm_gfn_to_pfn_cache_destroy(vcpu->kvm, - &vcpu->arch.xen.vcpu_time_info_cache); + kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.xen.runstate_cache); + kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.xen.vcpu_info_cache); + kvm_gpc_deactivate(vcpu->kvm, &vcpu->arch.xen.vcpu_time_info_cache); + del_timer_sync(&vcpu->arch.xen.poll_timer); } void kvm_xen_init_vm(struct kvm *kvm) { idr_init(&kvm->arch.xen.evtchn_ports); + kvm_gpc_init(&kvm->arch.xen.shinfo_cache); } void kvm_xen_destroy_vm(struct kvm *kvm) @@ -1843,7 +1846,7 @@ void kvm_xen_destroy_vm(struct kvm *kvm) struct evtchnfd *evtchnfd; int i; - kvm_gfn_to_pfn_cache_destroy(kvm, &kvm->arch.xen.shinfo_cache); + kvm_gpc_deactivate(kvm, &kvm->arch.xen.shinfo_cache); idr_for_each_entry(&kvm->arch.xen.evtchn_ports, evtchnfd, i) { if (!evtchnfd->deliver.port.port) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 00c3448ba7f8b..18592bdf4c1bf 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1240,8 +1240,18 @@ int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data, void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); /** - * kvm_gfn_to_pfn_cache_init - prepare a cached kernel mapping and HPA for a - * given guest physical address. + * kvm_gpc_init - initialize gfn_to_pfn_cache. + * + * @gpc: struct gfn_to_pfn_cache object. + * + * This sets up a gfn_to_pfn_cache by initializing locks. Note, the cache must + * be zero-allocated (or zeroed by the caller before init). + */ +void kvm_gpc_init(struct gfn_to_pfn_cache *gpc); + +/** + * kvm_gpc_activate - prepare a cached kernel mapping and HPA for a given guest + * physical address. * * @kvm: pointer to kvm instance. * @gpc: struct gfn_to_pfn_cache object. @@ -1265,9 +1275,9 @@ void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); * kvm_gfn_to_pfn_cache_check() to ensure that the cache is valid before * accessing the target page. */ -int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, - struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, - gpa_t gpa, unsigned long len); +int kvm_gpc_activate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, + struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, + gpa_t gpa, unsigned long len); /** * kvm_gfn_to_pfn_cache_check - check validity of a gfn_to_pfn_cache. @@ -1324,7 +1334,7 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); /** - * kvm_gfn_to_pfn_cache_destroy - destroy and unlink a gfn_to_pfn_cache. + * kvm_gpc_deactivate - deactivate and unlink a gfn_to_pfn_cache. * * @kvm: pointer to kvm instance. * @gpc: struct gfn_to_pfn_cache object. @@ -1332,7 +1342,7 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); * This removes a cache from the @kvm's list to be processed on MMU notifier * invocation. */ -void kvm_gfn_to_pfn_cache_destroy(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); +void kvm_gpc_deactivate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc); void kvm_sigset_activate(struct kvm_vcpu *vcpu); void kvm_sigset_deactivate(struct kvm_vcpu *vcpu); diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 68ff41d395452..08f97cf972643 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -346,17 +346,20 @@ void kvm_gfn_to_pfn_cache_unmap(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) } EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_unmap); +void kvm_gpc_init(struct gfn_to_pfn_cache *gpc) +{ + rwlock_init(&gpc->lock); + mutex_init(&gpc->refresh_lock); +} +EXPORT_SYMBOL_GPL(kvm_gpc_init); -int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, - struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, - gpa_t gpa, unsigned long len) +int kvm_gpc_activate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, + struct kvm_vcpu *vcpu, enum pfn_cache_usage usage, + gpa_t gpa, unsigned long len) { WARN_ON_ONCE(!usage || (usage & KVM_GUEST_AND_HOST_USE_PFN) != usage); if (!gpc->active) { - rwlock_init(&gpc->lock); - mutex_init(&gpc->refresh_lock); - gpc->khva = NULL; gpc->pfn = KVM_PFN_ERR_FAULT; gpc->uhva = KVM_HVA_ERR_BAD; @@ -371,9 +374,9 @@ int kvm_gfn_to_pfn_cache_init(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, } return kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpa, len); } -EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_init); +EXPORT_SYMBOL_GPL(kvm_gpc_activate); -void kvm_gfn_to_pfn_cache_destroy(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) +void kvm_gpc_deactivate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) { if (gpc->active) { spin_lock(&kvm->gpc_lock); @@ -384,4 +387,4 @@ void kvm_gfn_to_pfn_cache_destroy(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) gpc->active = false; } } -EXPORT_SYMBOL_GPL(kvm_gfn_to_pfn_cache_destroy); +EXPORT_SYMBOL_GPL(kvm_gpc_deactivate); -- GitLab From ecbcf030b45666ad11bc98565e71dfbcb7be4393 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 13 Oct 2022 21:12:20 +0000 Subject: [PATCH 0543/1927] KVM: Reject attempts to consume or refresh inactive gfn_to_pfn_cache Reject kvm_gpc_check() and kvm_gpc_refresh() if the cache is inactive. Not checking the active flag during refresh is particularly egregious, as KVM can end up with a valid, inactive cache, which can lead to a variety of use-after-free bugs, e.g. consuming a NULL kernel pointer or missing an mmu_notifier invalidation due to the cache not being on the list of gfns to invalidate. Note, "active" needs to be set if and only if the cache is on the list of caches, i.e. is reachable via mmu_notifier events. If a relevant mmu_notifier event occurs while the cache is "active" but not on the list, KVM will not acquire the cache's lock and so will not serailize the mmu_notifier event with active users and/or kvm_gpc_refresh(). A race between KVM_XEN_ATTR_TYPE_SHARED_INFO and KVM_XEN_HVM_EVTCHN_SEND can be exploited to trigger the bug. 1. Deactivate shinfo cache: kvm_xen_hvm_set_attr case KVM_XEN_ATTR_TYPE_SHARED_INFO kvm_gpc_deactivate kvm_gpc_unmap gpc->valid = false gpc->khva = NULL gpc->active = false Result: active = false, valid = false 2. Cause cache refresh: kvm_arch_vm_ioctl case KVM_XEN_HVM_EVTCHN_SEND kvm_xen_hvm_evtchn_send kvm_xen_set_evtchn kvm_xen_set_evtchn_fast kvm_gpc_check return -EWOULDBLOCK because !gpc->valid kvm_xen_set_evtchn_fast return -EWOULDBLOCK kvm_gpc_refresh hva_to_pfn_retry gpc->valid = true gpc->khva = not NULL Result: active = false, valid = true 3. Race ioctl KVM_XEN_HVM_EVTCHN_SEND against ioctl KVM_XEN_ATTR_TYPE_SHARED_INFO: kvm_arch_vm_ioctl case KVM_XEN_HVM_EVTCHN_SEND kvm_xen_hvm_evtchn_send kvm_xen_set_evtchn kvm_xen_set_evtchn_fast read_lock gpc->lock kvm_xen_hvm_set_attr case KVM_XEN_ATTR_TYPE_SHARED_INFO mutex_lock kvm->lock kvm_xen_shared_info_init kvm_gpc_activate gpc->khva = NULL kvm_gpc_check [ Check passes because gpc->valid is still true, even though gpc->khva is already NULL. ] shinfo = gpc->khva pending_bits = shinfo->evtchn_pending CRASH: test_and_set_bit(..., pending_bits) Fixes: 982ed0de4753 ("KVM: Reinstate gfn_to_pfn_cache with invalidation support") Cc: stable@vger.kernel.org Reported-by: : Michal Luczaj Signed-off-by: Sean Christopherson Message-Id: <20221013211234.1318131-3-seanjc@google.com> Signed-off-by: Paolo Bonzini --- virt/kvm/pfncache.c | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c index 08f97cf972643..346e47f155724 100644 --- a/virt/kvm/pfncache.c +++ b/virt/kvm/pfncache.c @@ -81,6 +81,9 @@ bool kvm_gfn_to_pfn_cache_check(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, { struct kvm_memslots *slots = kvm_memslots(kvm); + if (!gpc->active) + return false; + if ((gpa & ~PAGE_MASK) + len > PAGE_SIZE) return false; @@ -240,10 +243,11 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, { struct kvm_memslots *slots = kvm_memslots(kvm); unsigned long page_offset = gpa & ~PAGE_MASK; - kvm_pfn_t old_pfn, new_pfn; + bool unmap_old = false; unsigned long old_uhva; + kvm_pfn_t old_pfn; void *old_khva; - int ret = 0; + int ret; /* * If must fit within a single page. The 'len' argument is @@ -261,6 +265,11 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, write_lock_irq(&gpc->lock); + if (!gpc->active) { + ret = -EINVAL; + goto out_unlock; + } + old_pfn = gpc->pfn; old_khva = gpc->khva - offset_in_page(gpc->khva); old_uhva = gpc->uhva; @@ -291,6 +300,7 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, /* If the HVA→PFN mapping was already valid, don't unmap it. */ old_pfn = KVM_PFN_ERR_FAULT; old_khva = NULL; + ret = 0; } out: @@ -305,14 +315,15 @@ int kvm_gfn_to_pfn_cache_refresh(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, gpc->khva = NULL; } - /* Snapshot the new pfn before dropping the lock! */ - new_pfn = gpc->pfn; + /* Detect a pfn change before dropping the lock! */ + unmap_old = (old_pfn != gpc->pfn); +out_unlock: write_unlock_irq(&gpc->lock); mutex_unlock(&gpc->refresh_lock); - if (old_pfn != new_pfn) + if (unmap_old) gpc_unmap_khva(kvm, old_pfn, old_khva); return ret; @@ -366,11 +377,19 @@ int kvm_gpc_activate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc, gpc->vcpu = vcpu; gpc->usage = usage; gpc->valid = false; - gpc->active = true; spin_lock(&kvm->gpc_lock); list_add(&gpc->list, &kvm->gpc_list); spin_unlock(&kvm->gpc_lock); + + /* + * Activate the cache after adding it to the list, a concurrent + * refresh must not establish a mapping until the cache is + * reachable by mmu_notifier events. + */ + write_lock_irq(&gpc->lock); + gpc->active = true; + write_unlock_irq(&gpc->lock); } return kvm_gfn_to_pfn_cache_refresh(kvm, gpc, gpa, len); } @@ -379,12 +398,20 @@ EXPORT_SYMBOL_GPL(kvm_gpc_activate); void kvm_gpc_deactivate(struct kvm *kvm, struct gfn_to_pfn_cache *gpc) { if (gpc->active) { + /* + * Deactivate the cache before removing it from the list, KVM + * must stall mmu_notifier events until all users go away, i.e. + * until gpc->lock is dropped and refresh is guaranteed to fail. + */ + write_lock_irq(&gpc->lock); + gpc->active = false; + write_unlock_irq(&gpc->lock); + spin_lock(&kvm->gpc_lock); list_del(&gpc->list); spin_unlock(&kvm->gpc_lock); kvm_gfn_to_pfn_cache_unmap(kvm, gpc); - gpc->active = false; } } EXPORT_SYMBOL_GPL(kvm_gpc_deactivate); -- GitLab From 0e7ce23a917a9cc83ca3c779fbba836bca3bcf1e Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 25 Oct 2022 21:00:11 +0800 Subject: [PATCH 0544/1927] net: ehea: fix possible memory leak in ehea_register_port() If of_device_register() returns error, the of node and the name allocated in dev_set_name() is leaked, call put_device() to give up the reference that was set in device_initialize(), so that of node is put in logical_port_release() and the name is freed in kobject_cleanup(). Fixes: 1acf2318dd13 ("ehea: dynamic add / remove port") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221025130011.1071357-1-yangyingliang@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/ibm/ehea/ehea_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c index 294bdbbeacc33..b4aff59b3eb4f 100644 --- a/drivers/net/ethernet/ibm/ehea/ehea_main.c +++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c @@ -2900,6 +2900,7 @@ static struct device *ehea_register_port(struct ehea_port *port, ret = of_device_register(&port->ofdev); if (ret) { pr_err("failed to register device. ret=%d\n", ret); + put_device(&port->ofdev.dev); goto out; } -- GitLab From 03b9a6e18d325ede28c6dc218cedda53969eb41b Mon Sep 17 00:00:00 2001 From: Zhao Liu Date: Thu, 20 Oct 2022 16:38:20 +0800 Subject: [PATCH 0545/1927] x86/hyperv: Remove BUG_ON() for kmap_local_page() The commit 154fb14df7a3c ("x86/hyperv: Replace kmap() with kmap_local_page()") keeps the BUG_ON() to check if kmap_local_page() fails. But in fact, kmap_local_page() always returns a valid kernel address and won't return NULL here. It will BUG on its own if it fails. [1] So directly use memcpy_to_page() which creates local mapping to copy. [1]: https://lore.kernel.org/lkml/YztFEyUA48et0yTt@iweiny-mobl/ Suggested-by: Fabio M. De Francesco Suggested-by: Ira Weiny Reviewed-by: Ira Weiny Signed-off-by: Zhao Liu Link: https://lore.kernel.org/r/20221020083820.2341088-1-zhao1.liu@linux.intel.com Signed-off-by: Wei Liu --- arch/x86/hyperv/hv_init.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index 29774126e9314..d46f70522ddf2 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -444,7 +444,7 @@ void __init hyperv_init(void) if (hv_root_partition) { struct page *pg; - void *src, *dst; + void *src; /* * For the root partition, the hypervisor will set up its @@ -459,13 +459,11 @@ void __init hyperv_init(void) wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); pg = vmalloc_to_page(hv_hypercall_pg); - dst = kmap_local_page(pg); src = memremap(hypercall_msr.guest_physical_address << PAGE_SHIFT, PAGE_SIZE, MEMREMAP_WB); - BUG_ON(!(src && dst)); - memcpy(dst, src, HV_HYP_PAGE_SIZE); + BUG_ON(!src); + memcpy_to_page(pg, 0, src, HV_HYP_PAGE_SIZE); memunmap(src); - kunmap_local(dst); } else { hypercall_msr.guest_physical_address = vmalloc_to_pfn(hv_hypercall_pg); wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); -- GitLab From 99632e3dbeb25a0ff86c4af4bba2dcf638624423 Mon Sep 17 00:00:00 2001 From: Jilin Yuan Date: Wed, 19 Oct 2022 20:56:04 +0800 Subject: [PATCH 0546/1927] Drivers: hv: fix repeated words in comments Delete the redundant word 'of'. Signed-off-by: Jilin Yuan Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20221019125604.52999-1-yuanjilin@cdjrlc.com Signed-off-by: Wei Liu --- drivers/hv/hv_balloon.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c index fdf6decacf067..6c127f061f06d 100644 --- a/drivers/hv/hv_balloon.c +++ b/drivers/hv/hv_balloon.c @@ -905,7 +905,7 @@ static unsigned long handle_pg_range(unsigned long pg_start, * We have some residual hot add range * that needs to be hot added; hot add * it now. Hot add a multiple of - * of HA_CHUNK that fully covers the pages + * HA_CHUNK that fully covers the pages * we have. */ size = (has->end_pfn - has->ha_end_pfn); -- GitLab From c3c06c61890da80494bb196f75d89b791adda87f Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 27 Oct 2022 17:12:37 +0800 Subject: [PATCH 0547/1927] can: j1939: transport: j1939_session_skb_drop_old(): spin_unlock_irqrestore() before kfree_skb() It is not allowed to call kfree_skb() from hardware interrupt context or with interrupts being disabled. The skb is unlinked from the queue, so it can be freed after spin_unlock_irqrestore(). Fixes: 9d71dd0c7009 ("can: add support of SAE J1939 protocol") Signed-off-by: Yang Yingliang Acked-by: Oleksij Rempel Link: https://lore.kernel.org/all/20221027091237.2290111-1-yangyingliang@huawei.com Cc: stable@vger.kernel.org [mkl: adjust subject] Signed-off-by: Marc Kleine-Budde --- net/can/j1939/transport.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c index d7d86c944d76d..55f29c9f9e08e 100644 --- a/net/can/j1939/transport.c +++ b/net/can/j1939/transport.c @@ -342,10 +342,12 @@ static void j1939_session_skb_drop_old(struct j1939_session *session) __skb_unlink(do_skb, &session->skb_queue); /* drop ref taken in j1939_session_skb_queue() */ skb_unref(do_skb); + spin_unlock_irqrestore(&session->skb_queue.lock, flags); kfree_skb(do_skb); + } else { + spin_unlock_irqrestore(&session->skb_queue.lock, flags); } - spin_unlock_irqrestore(&session->skb_queue.lock, flags); } void j1939_session_skb_queue(struct j1939_session *session, -- GitLab From 7f21735ffb2648a29e0fc79c4bdcb1b9ed8602cd Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 27 Oct 2022 17:19:18 +0800 Subject: [PATCH 0548/1927] rbd: fix possible memory leak in rbd_sysfs_init() If device_register() returns error in rbd_sysfs_init(), name of kobject which is allocated in dev_set_name() called in device_add() is leaked. As comment of device_add() says, it should call put_device() to drop the reference count that was set in device_initialize() when it fails, so the name can be freed in kobject_cleanup(). Fault injection test can trigger this problem: unreferenced object 0xffff88810173aa78 (size 8): comm "modprobe", pid 247, jiffies 4294714278 (age 31.789s) hex dump (first 8 bytes): 72 62 64 00 81 88 ff ff rbd..... backtrace: [<00000000f58fae56>] __kmalloc_node_track_caller+0x44/0x1b0 [<00000000bdd44fe7>] kstrdup+0x3a/0x70 [<00000000f7844d0b>] kstrdup_const+0x63/0x80 [<000000001b0a0eeb>] kvasprintf_const+0x10b/0x190 [<00000000a47bd894>] kobject_set_name_vargs+0x56/0x150 [<00000000d5edbf18>] dev_set_name+0xab/0xe0 [<00000000f5153e80>] device_add+0x106/0x1f20 Fixes: dfc5606dc513 ("rbd: replace the rbd sysfs interface") Signed-off-by: Yang Yingliang Reviewed-by: Alex Elder Link: https://lore.kernel.org/r/20221027091918.2294132-1-yangyingliang@huawei.com Signed-off-by: Jens Axboe --- drivers/block/rbd.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index f9e39301c4afa..04453f4a319cb 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -7222,8 +7222,10 @@ static int __init rbd_sysfs_init(void) int ret; ret = device_register(&rbd_root_dev); - if (ret < 0) + if (ret < 0) { + put_device(&rbd_root_dev); return ret; + } ret = bus_register(&rbd_bus_type); if (ret < 0) -- GitLab From 2d87d455ead2cbdee7e60463cddc5bff3f98c912 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Thu, 27 Oct 2022 16:57:09 +0800 Subject: [PATCH 0549/1927] blk-mq: don't add non-pt request with ->end_io to batch dm-rq implements ->end_io callback for request issued to underlying queue, and it isn't passthrough request. Commit ab3e1d3bbab9 ("block: allow end_io based requests in the completion batch handling") doesn't clear rq->bio and rq->__data_len for request with ->end_io in blk_mq_end_request_batch(), and this way is actually dangerous, but so far it is only for nvme passthrough request. dm-rq needs to clean up remained bios in case of partial completion, and req->bio is required, then use-after-free is triggered, so the underlying clone request can't be completed in blk_mq_end_request_batch. Fix panic by not adding such request into batch list, and the issue can be triggered simply by exposing nvme pci to dm-mpath simply. Fixes: ab3e1d3bbab9 ("block: allow end_io based requests in the completion batch handling") Cc: dm-devel@redhat.com Cc: Mike Snitzer Reported-by: Changhui Zhong Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20221027085709.513175-1-ming.lei@redhat.com Signed-off-by: Jens Axboe --- include/linux/blk-mq.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index ba18e9bdb799b..d6119c5d1069b 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -853,7 +853,8 @@ static inline bool blk_mq_add_to_batch(struct request *req, struct io_comp_batch *iob, int ioerror, void (*complete)(struct io_comp_batch *)) { - if (!iob || (req->rq_flags & RQF_ELV) || ioerror) + if (!iob || (req->rq_flags & RQF_ELV) || ioerror || + (req->end_io && !blk_rq_is_passthrough(req))) return false; if (!iob->complete) -- GitLab From 9f172134dde7e4f5bf4b9139f23a1e741ec1c36e Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Tue, 25 Oct 2022 16:42:01 -0700 Subject: [PATCH 0550/1927] net: bcmsysport: Indicate MAC is in charge of PHY PM Avoid the PHY library call unnecessarily into the suspend/resume functions by setting phydev->mac_managed_pm to true. The SYSTEMPORT driver essentially does exactly what mdio_bus_phy_resume() does by calling phy_resume(). Fixes: fba863b81604 ("net: phy: make PHY PM ops a no-op if MAC driver manages PHY PM") Signed-off-by: Florian Fainelli Link: https://lore.kernel.org/r/20221025234201.2549360-1-f.fainelli@gmail.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/broadcom/bcmsysport.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index 867f14c30e09f..425d6ccd5413a 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -1991,6 +1991,9 @@ static int bcm_sysport_open(struct net_device *dev) goto out_clk_disable; } + /* Indicate that the MAC is responsible for PHY PM */ + phydev->mac_managed_pm = true; + /* Reset house keeping link status */ priv->old_duplex = -1; priv->old_link = -1; -- GitLab From df3414b0a245f43476061fddd78cee7d6cff797f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Monin?= Date: Thu, 13 Oct 2022 16:26:48 +0200 Subject: [PATCH 0551/1927] USB: serial: option: add Sierra Wireless EM9191 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for the AT and diag ports, similar to other qualcomm SDX55 modems. In QDL mode, the modem uses a different device ID and support is provided by qcserial in commit 11c52d250b34 ("USB: serial: qcserial: add EM9191 QDL support"). T: Bus=08 Lev=01 Prnt=01 Port=01 Cnt=01 Dev#= 3 Spd=5000 MxCh= 0 D: Ver= 3.20 Cls=00(>ifc ) Sub=00 Prot=00 MxPS= 9 #Cfgs= 1 P: Vendor=1199 ProdID=90d3 Rev=00.06 S: Manufacturer=Sierra Wireless, Incorporated S: Product=Sierra Wireless EM9191 S: SerialNumber=xxxxxxxxxxxxxxxx C: #Ifs= 4 Cfg#= 1 Atr=a0 MxPwr=896mA I: If#=0x0 Alt= 0 #EPs= 1 Cls=02(commc) Sub=0e Prot=00 Driver=cdc_mbim I: If#=0x1 Alt= 1 #EPs= 2 Cls=0a(data ) Sub=00 Prot=02 Driver=cdc_mbim I: If#=0x3 Alt= 0 #EPs= 3 Cls=ff(vend.) Sub=00 Prot=00 Driver=(none) I: If#=0x4 Alt= 0 #EPs= 2 Cls=ff(vend.) Sub=ff Prot=30 Driver=(none) Signed-off-by: Benoît Monin Cc: stable@vger.kernel.org Signed-off-by: Johan Hovold --- drivers/usb/serial/option.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 697683e3fbffa..37257a52287d5 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -581,6 +581,9 @@ static void option_instat_callback(struct urb *urb); #define OPPO_VENDOR_ID 0x22d9 #define OPPO_PRODUCT_R11 0x276c +/* Sierra Wireless products */ +#define SIERRA_VENDOR_ID 0x1199 +#define SIERRA_PRODUCT_EM9191 0x90d3 /* Device flags */ @@ -2176,6 +2179,8 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) }, /* GosunCn GM500 MBIM */ { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1406, 0xff) }, /* GosunCn GM500 ECM/NCM */ { USB_DEVICE_AND_INTERFACE_INFO(OPPO_VENDOR_ID, OPPO_PRODUCT_R11, 0xff, 0xff, 0x30) }, + { USB_DEVICE_AND_INTERFACE_INFO(SIERRA_VENDOR_ID, SIERRA_PRODUCT_EM9191, 0xff, 0xff, 0x30) }, + { USB_DEVICE_AND_INTERFACE_INFO(SIERRA_VENDOR_ID, SIERRA_PRODUCT_EM9191, 0xff, 0, 0) }, { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(usb, option_ids); -- GitLab From 5c1df62ca65936139a4b008561110d9fc285c58a Mon Sep 17 00:00:00 2001 From: Stefan Hansson Date: Mon, 17 Oct 2022 17:01:14 +0200 Subject: [PATCH 0552/1927] kbuild: use POSIX-compatible grep option --file is a GNU extension to grep which is not available in all implementations (such as BusyBox). Use the -f option instead which is eqvuialent according to the GNU grep manpage[1] and is present in POSIX[2]. [1] https://www.gnu.org/software/grep/manual/grep.html [2] https://pubs.opengroup.org/onlinepubs/9699919799/utilities/grep.html Signed-off-by: Stefan Hansson Signed-off-by: Masahiro Yamada --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d148a55bfd0f5..e90bb2b38607a 100644 --- a/Makefile +++ b/Makefile @@ -1218,7 +1218,7 @@ quiet_cmd_ar_vmlinux.a = AR $@ cmd_ar_vmlinux.a = \ rm -f $@; \ $(AR) cDPrST $@ $(KBUILD_VMLINUX_OBJS); \ - $(AR) mPiT $$($(AR) t $@ | head -n1) $@ $$($(AR) t $@ | grep -F --file=$(srctree)/scripts/head-object-list.txt) + $(AR) mPiT $$($(AR) t $@ | head -n1) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) targets += vmlinux.a vmlinux.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt autoksyms_recursive FORCE -- GitLab From 114ff6fe6cfbe81659f9e517d0b25f53db5dfc5d Mon Sep 17 00:00:00 2001 From: Dan Li Date: Thu, 20 Oct 2022 03:38:23 -0700 Subject: [PATCH 0553/1927] Documentation: kbuild: Add description of git for reproducible builds The status of git will affect the final compilation result, add it to the documentation of reproducible builds. Signed-off-by: Dan Li Signed-off-by: Masahiro Yamada --- Documentation/kbuild/reproducible-builds.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/kbuild/reproducible-builds.rst b/Documentation/kbuild/reproducible-builds.rst index 071f0151a7a4e..f2dcc39044e66 100644 --- a/Documentation/kbuild/reproducible-builds.rst +++ b/Documentation/kbuild/reproducible-builds.rst @@ -119,6 +119,16 @@ To avoid this, you can make the vDSO different for different kernel versions by including an arbitrary string of "salt" in it. This is specified by the Kconfig symbol ``CONFIG_BUILD_SALT``. +Git +--- + +Uncommitted changes or different commit ids in git can also lead +to different compilation results. For example, after executing +``git reset HEAD^``, even if the code is the same, the +``include/config/kernel.release`` generated during compilation +will be different, which will eventually lead to binary differences. +See ``scripts/setlocalversion`` for details. + .. _KBUILD_BUILD_TIMESTAMP: kbuild.html#kbuild-build-timestamp .. _KBUILD_BUILD_USER and KBUILD_BUILD_HOST: kbuild.html#kbuild-build-user-kbuild-build-host .. _KCFLAGS: kbuild.html#kcflags -- GitLab From 3b1e0dd2dc8a280b1e89c0df6d38cd28768575a5 Mon Sep 17 00:00:00 2001 From: Will McVicker Date: Tue, 25 Oct 2022 13:17:44 -0700 Subject: [PATCH 0554/1927] kbuild: fix typo in modpost Commit f73edc8951b2 ("kbuild: unify two modpost invocations") introduced a typo (moudle.symvers-if-present) which results in the kernel's Module.symvers to not be included as a prerequisite for $(KBUILD_EXTMOD)/Module.symvers. Fix the typo to restore the intended functionality. Fixes: f73edc8951b2 ("kbuild: unify two modpost invocations") Signed-off-by: Will McVicker Signed-off-by: Masahiro Yamada --- scripts/Makefile.modpost | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Makefile.modpost b/scripts/Makefile.modpost index 8489a3402eb8c..e41dee64d429c 100644 --- a/scripts/Makefile.modpost +++ b/scripts/Makefile.modpost @@ -122,7 +122,7 @@ quiet_cmd_modpost = MODPOST $@ sed 's/ko$$/o/' $(or $(modorder-if-needed), /dev/null) | $(MODPOST) $(modpost-args) -T - $(vmlinux.o-if-present) targets += $(output-symdump) -$(output-symdump): $(modorder-if-needed) $(vmlinux.o-if-present) $(moudle.symvers-if-present) $(MODPOST) FORCE +$(output-symdump): $(modorder-if-needed) $(vmlinux.o-if-present) $(module.symvers-if-present) $(MODPOST) FORCE $(call if_changed,modpost) __modpost: $(output-symdump) -- GitLab From ce48ebdd56513fa5ad9dab683a96399e00dbf464 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Tue, 25 Oct 2022 17:15:24 -0700 Subject: [PATCH 0555/1927] genetlink: limit the use of validation workarounds to old ops During review of previous change another thing came up - we should limit the use of validation workarounds to old commands. Don't list the workarounds one by one, as we're rejecting all existing ones. We can deal with the masking in the unlikely event that new flag is added. Link: https://lore.kernel.org/all/6ba9f727e555fd376623a298d5d305ad408c3d47.camel@sipsolutions.net/ Link: https://lore.kernel.org/r/20221026001524.1892202-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- net/netlink/genetlink.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index b1fd059c99928..3e16527beb914 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -380,6 +380,8 @@ static int genl_validate_ops(const struct genl_family *family) genl_get_cmd_by_index(i, family, &op); if (op.dumpit == NULL && op.doit == NULL) return -EINVAL; + if (WARN_ON(op.cmd >= family->resv_start_op && op.validate)) + return -EINVAL; for (j = i + 1; j < genl_get_cmd_cnt(family); j++) { struct genl_ops op2; -- GitLab From 8de11cdc96bf58b324c59a28512eb9513fd02553 Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Thu, 27 Oct 2022 07:44:28 -0700 Subject: [PATCH 0556/1927] io_uring: use io_run_local_work_locked helper prefer to use io_run_local_work_locked helper for consistency Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20221027144429.3971400-2-dylany@meta.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 6cc16e39b27f0..8a0ce7379e897 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1446,8 +1446,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min) io_task_work_pending(ctx)) { u32 tail = ctx->cached_cq_tail; - if (!llist_empty(&ctx->work_llist)) - __io_run_local_work(ctx, true); + (void) io_run_local_work_locked(ctx); if (task_work_pending(current) || wq_list_empty(&ctx->iopoll_list)) { -- GitLab From b3026767e15b488860d4bbf1649d69612bab2c25 Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Thu, 27 Oct 2022 07:44:29 -0700 Subject: [PATCH 0557/1927] io_uring: unlock if __io_run_local_work locked inside It is possible for tw to lock the ring, and this was not propogated out to io_run_local_work. This can cause an unlock to be missed. Instead pass a pointer to locked into __io_run_local_work. Fixes: 8ac5d85a89b4 ("io_uring: add local task_work run helper that is entered locked") Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20221027144429.3971400-3-dylany@meta.com [axboe: WARN_ON() -> WARN_ON_ONCE() and add a minor comment] Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 8 ++++---- io_uring/io_uring.h | 13 +++++++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 8a0ce7379e897..ac8c488e3077d 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -1173,7 +1173,7 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx) } } -int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) +int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked) { struct llist_node *node; struct llist_node fake; @@ -1192,7 +1192,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) struct io_kiocb *req = container_of(node, struct io_kiocb, io_task_work.node); prefetch(container_of(next, struct io_kiocb, io_task_work.node)); - req->io_task_work.func(req, &locked); + req->io_task_work.func(req, locked); ret++; node = next; } @@ -1208,7 +1208,7 @@ int __io_run_local_work(struct io_ring_ctx *ctx, bool locked) goto again; } - if (locked) + if (*locked) io_submit_flush_completions(ctx); trace_io_uring_local_work_run(ctx, ret, loops); return ret; @@ -1225,7 +1225,7 @@ int io_run_local_work(struct io_ring_ctx *ctx) __set_current_state(TASK_RUNNING); locked = mutex_trylock(&ctx->uring_lock); - ret = __io_run_local_work(ctx, locked); + ret = __io_run_local_work(ctx, &locked); if (locked) mutex_unlock(&ctx->uring_lock); diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index ef77d2aa3172c..e99a79f2df9b1 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -27,7 +27,7 @@ enum { struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow); bool io_req_cqe_overflow(struct io_kiocb *req); int io_run_task_work_sig(struct io_ring_ctx *ctx); -int __io_run_local_work(struct io_ring_ctx *ctx, bool locked); +int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked); int io_run_local_work(struct io_ring_ctx *ctx); void io_req_complete_failed(struct io_kiocb *req, s32 res); void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); @@ -277,9 +277,18 @@ static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx) static inline int io_run_local_work_locked(struct io_ring_ctx *ctx) { + bool locked; + int ret; + if (llist_empty(&ctx->work_llist)) return 0; - return __io_run_local_work(ctx, true); + + locked = true; + ret = __io_run_local_work(ctx, &locked); + /* shouldn't happen! */ + if (WARN_ON_ONCE(!locked)) + mutex_lock(&ctx->uring_lock); + return ret; } static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked) -- GitLab From 745b913a59947919454658dd44cddf5ee8d8f899 Mon Sep 17 00:00:00 2001 From: Nicolas Dichtel Date: Thu, 20 Oct 2022 12:09:50 +0200 Subject: [PATCH 0558/1927] Revert "ip: fix triggering of 'icmp redirect'" This reverts commit eb55dc09b5dd040232d5de32812cc83001a23da6. The patch that introduces this bug is reverted right after this one. Signed-off-by: Nicolas Dichtel Reviewed-by: Julian Anastasov Signed-off-by: Jakub Kicinski --- net/ipv4/fib_frontend.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/ipv4/fib_frontend.c b/net/ipv4/fib_frontend.c index 943edf4ad4db0..f361d3d56be27 100644 --- a/net/ipv4/fib_frontend.c +++ b/net/ipv4/fib_frontend.c @@ -389,7 +389,7 @@ static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst, dev_match = dev_match || (res.type == RTN_LOCAL && dev == net->loopback_dev); if (dev_match) { - ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_LINK; + ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_HOST; return ret; } if (no_addr) @@ -401,7 +401,7 @@ static int __fib_validate_source(struct sk_buff *skb, __be32 src, __be32 dst, ret = 0; if (fib_lookup(net, &fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE) == 0) { if (res.type == RTN_UNICAST) - ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_LINK; + ret = FIB_RES_NHC(res)->nhc_scope >= RT_SCOPE_HOST; } return ret; -- GitLab From e021c329ee198f1cd0cb3855f221137dda49256a Mon Sep 17 00:00:00 2001 From: Nicolas Dichtel Date: Thu, 20 Oct 2022 12:09:51 +0200 Subject: [PATCH 0559/1927] Revert "ip: fix dflt addr selection for connected nexthop" This reverts commit 747c14307214b55dbd8250e1ab44cad8305756f1. As explained by Julian, nhc_scope is related to nhc_gw, not to the route. Revert the original patch. The initial problem is fixed differently in the next commit. Link: https://lore.kernel.org/netdev/6c8a44ba-c2d5-cdf-c5c7-5baf97cba38@ssi.bg/ Signed-off-by: Nicolas Dichtel Reviewed-by: Julian Anastasov Signed-off-by: Jakub Kicinski --- net/ipv4/fib_semantics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index e9a7f70a54df4..f721c308248ba 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -1231,7 +1231,7 @@ static int fib_check_nh_nongw(struct net *net, struct fib_nh *nh, nh->fib_nh_dev = in_dev->dev; netdev_hold(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); - nh->fib_nh_scope = RT_SCOPE_LINK; + nh->fib_nh_scope = RT_SCOPE_HOST; if (!netif_carrier_ok(nh->fib_nh_dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; err = 0; -- GitLab From bac0f937c343d651874f83b265ca8f5070ed4f06 Mon Sep 17 00:00:00 2001 From: Nicolas Dichtel Date: Thu, 20 Oct 2022 12:09:52 +0200 Subject: [PATCH 0560/1927] nh: fix scope used to find saddr when adding non gw nh As explained by Julian, fib_nh_scope is related to fib_nh_gw4, but fib_info_update_nhc_saddr() needs the scope of the route, which is the scope "before" fib_nh_scope, ie fib_nh_scope - 1. This patch fixes the problem described in commit 747c14307214 ("ip: fix dflt addr selection for connected nexthop"). Fixes: 597cfe4fc339 ("nexthop: Add support for IPv4 nexthops") Link: https://lore.kernel.org/netdev/6c8a44ba-c2d5-cdf-c5c7-5baf97cba38@ssi.bg/ Signed-off-by: Nicolas Dichtel Reviewed-by: Julian Anastasov Signed-off-by: Jakub Kicinski --- net/ipv4/nexthop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 853a75a8fbafc..d8ef05347fd98 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -2534,7 +2534,7 @@ static int nh_create_ipv4(struct net *net, struct nexthop *nh, if (!err) { nh->nh_flags = fib_nh->fib_nh_flags; fib_info_update_nhc_saddr(net, &fib_nh->nh_common, - fib_nh->fib_nh_scope); + !fib_nh->fib_nh_scope ? 0 : fib_nh->fib_nh_scope - 1); } else { fib_nh_release(net, fib_nh); } -- GitLab From ef3556ee16c68735ec69bd08df41d1cd83b14ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mi=C5=82ecki?= Date: Thu, 27 Oct 2022 13:24:30 +0200 Subject: [PATCH 0561/1927] net: broadcom: bcm4908_enet: update TX stats after actual transmission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Queueing packets doesn't guarantee their transmission. Update TX stats after hardware confirms consuming submitted data. This also fixes a possible race and NULL dereference. bcm4908_enet_start_xmit() could try to access skb after freeing it in the bcm4908_enet_poll_tx(). Reported-by: Florian Fainelli Fixes: 4feffeadbcb2e ("net: broadcom: bcm4908enet: add BCM4908 controller driver") Signed-off-by: Rafał Miłecki Reviewed-by: Florian Fainelli Link: https://lore.kernel.org/r/20221027112430.8696-1-zajec5@gmail.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bcm4908_enet.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bcm4908_enet.c b/drivers/net/ethernet/broadcom/bcm4908_enet.c index 93ccf549e2ed3..a737b1913cf95 100644 --- a/drivers/net/ethernet/broadcom/bcm4908_enet.c +++ b/drivers/net/ethernet/broadcom/bcm4908_enet.c @@ -561,8 +561,6 @@ static netdev_tx_t bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_devic if (++ring->write_idx == ring->length - 1) ring->write_idx = 0; - enet->netdev->stats.tx_bytes += skb->len; - enet->netdev->stats.tx_packets++; return NETDEV_TX_OK; } @@ -635,6 +633,7 @@ static int bcm4908_enet_poll_tx(struct napi_struct *napi, int weight) struct bcm4908_enet_dma_ring_bd *buf_desc; struct bcm4908_enet_dma_ring_slot *slot; struct device *dev = enet->dev; + unsigned int bytes = 0; int handled = 0; while (handled < weight && tx_ring->read_idx != tx_ring->write_idx) { @@ -645,12 +644,17 @@ static int bcm4908_enet_poll_tx(struct napi_struct *napi, int weight) dma_unmap_single(dev, slot->dma_addr, slot->len, DMA_TO_DEVICE); dev_kfree_skb(slot->skb); - if (++tx_ring->read_idx == tx_ring->length) - tx_ring->read_idx = 0; handled++; + bytes += slot->len; + + if (++tx_ring->read_idx == tx_ring->length) + tx_ring->read_idx = 0; } + enet->netdev->stats.tx_packets += handled; + enet->netdev->stats.tx_bytes += bytes; + if (handled < weight) { napi_complete_done(napi, handled); bcm4908_enet_dma_ring_intrs_on(enet, tx_ring); -- GitLab From cf2010aa1c739bab067cbc90b690d28eaa0b47da Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 26 Oct 2022 09:54:05 +0800 Subject: [PATCH 0562/1927] netdevsim: fix memory leak in nsim_bus_dev_new() If device_register() failed in nsim_bus_dev_new(), the value of reference in nsim_bus_dev->dev is 1. obj->name in nsim_bus_dev->dev will not be released. unreferenced object 0xffff88810352c480 (size 16): comm "echo", pid 5691, jiffies 4294945921 (age 133.270s) hex dump (first 16 bytes): 6e 65 74 64 65 76 73 69 6d 31 00 00 00 00 00 00 netdevsim1...... backtrace: [<000000005e2e5e26>] __kmalloc_node_track_caller+0x3a/0xb0 [<0000000094ca4fc8>] kvasprintf+0xc3/0x160 [<00000000aad09bcc>] kvasprintf_const+0x55/0x180 [<000000009bac868d>] kobject_set_name_vargs+0x56/0x150 [<000000007c1a5d70>] dev_set_name+0xbb/0xf0 [<00000000ad0d126b>] device_add+0x1f8/0x1cb0 [<00000000c222ae24>] new_device_store+0x3b6/0x5e0 [<0000000043593421>] bus_attr_store+0x72/0xa0 [<00000000cbb1833a>] sysfs_kf_write+0x106/0x160 [<00000000d0dedb8a>] kernfs_fop_write_iter+0x3a8/0x5a0 [<00000000770b66e2>] vfs_write+0x8f0/0xc80 [<0000000078bb39be>] ksys_write+0x106/0x210 [<00000000005e55a4>] do_syscall_64+0x35/0x80 [<00000000eaa40bbc>] entry_SYSCALL_64_after_hwframe+0x46/0xb0 Fixes: 40e4fe4ce115 ("netdevsim: move device registration and related code to bus.c") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221026015405.128795-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/netdevsim/bus.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/net/netdevsim/bus.c b/drivers/net/netdevsim/bus.c index b5f4df1a07a3d..0052968e881e7 100644 --- a/drivers/net/netdevsim/bus.c +++ b/drivers/net/netdevsim/bus.c @@ -117,6 +117,10 @@ static const struct attribute_group *nsim_bus_dev_attr_groups[] = { static void nsim_bus_dev_release(struct device *dev) { + struct nsim_bus_dev *nsim_bus_dev; + + nsim_bus_dev = container_of(dev, struct nsim_bus_dev, dev); + kfree(nsim_bus_dev); } static struct device_type nsim_bus_dev_type = { @@ -291,6 +295,8 @@ nsim_bus_dev_new(unsigned int id, unsigned int port_count, unsigned int num_queu err_nsim_bus_dev_id_free: ida_free(&nsim_bus_dev_ids, nsim_bus_dev->dev.id); + put_device(&nsim_bus_dev->dev); + nsim_bus_dev = NULL; err_nsim_bus_dev_free: kfree(nsim_bus_dev); return ERR_PTR(err); @@ -300,9 +306,8 @@ static void nsim_bus_dev_del(struct nsim_bus_dev *nsim_bus_dev) { /* Disallow using nsim_bus_dev */ smp_store_release(&nsim_bus_dev->init, false); - device_unregister(&nsim_bus_dev->dev); ida_free(&nsim_bus_dev_ids, nsim_bus_dev->dev.id); - kfree(nsim_bus_dev); + device_unregister(&nsim_bus_dev->dev); } static struct device_driver nsim_driver = { -- GitLab From 6b1da9f7126f05e857da6db24c6a04aa7974d644 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 26 Oct 2022 09:46:41 +0800 Subject: [PATCH 0563/1927] netdevsim: fix memory leak in nsim_drv_probe() when nsim_dev_resources_register() failed If some items in nsim_dev_resources_register() fail, memory leak will occur. The following is the memory leak information. unreferenced object 0xffff888074c02600 (size 128): comm "echo", pid 8159, jiffies 4294945184 (age 493.530s) hex dump (first 32 bytes): 40 47 ea 89 ff ff ff ff 01 00 00 00 00 00 00 00 @G.............. ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ................ backtrace: [<0000000011a31c98>] kmalloc_trace+0x22/0x60 [<0000000027384c69>] devl_resource_register+0x144/0x4e0 [<00000000a16db248>] nsim_drv_probe+0x37a/0x1260 [<000000007d1f448c>] really_probe+0x20b/0xb10 [<00000000c416848a>] __driver_probe_device+0x1b3/0x4a0 [<00000000077e0351>] driver_probe_device+0x49/0x140 [<0000000054f2465a>] __device_attach_driver+0x18c/0x2a0 [<000000008538f359>] bus_for_each_drv+0x151/0x1d0 [<0000000038e09747>] __device_attach+0x1c9/0x4e0 [<00000000dd86e533>] bus_probe_device+0x1d5/0x280 [<00000000839bea35>] device_add+0xae0/0x1cb0 [<000000009c2abf46>] new_device_store+0x3b6/0x5f0 [<00000000fb823d7f>] bus_attr_store+0x72/0xa0 [<000000007acc4295>] sysfs_kf_write+0x106/0x160 [<000000005f50cb4d>] kernfs_fop_write_iter+0x3a8/0x5a0 [<0000000075eb41bf>] vfs_write+0x8f0/0xc80 Fixes: 37923ed6b8ce ("netdevsim: Add simple FIB resource controller via devlink") Signed-off-by: Zhengchao Shao Signed-off-by: Jakub Kicinski --- drivers/net/netdevsim/dev.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index 794fc0cc73b88..81c3e14af0636 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -442,7 +442,7 @@ static int nsim_dev_resources_register(struct devlink *devlink) ¶ms); if (err) { pr_err("Failed to register IPv4 top resource\n"); - goto out; + goto err_out; } err = devl_resource_register(devlink, "fib", (u64)-1, @@ -450,7 +450,7 @@ static int nsim_dev_resources_register(struct devlink *devlink) NSIM_RESOURCE_IPV4, ¶ms); if (err) { pr_err("Failed to register IPv4 FIB resource\n"); - return err; + goto err_out; } err = devl_resource_register(devlink, "fib-rules", (u64)-1, @@ -458,7 +458,7 @@ static int nsim_dev_resources_register(struct devlink *devlink) NSIM_RESOURCE_IPV4, ¶ms); if (err) { pr_err("Failed to register IPv4 FIB rules resource\n"); - return err; + goto err_out; } /* Resources for IPv6 */ @@ -468,7 +468,7 @@ static int nsim_dev_resources_register(struct devlink *devlink) ¶ms); if (err) { pr_err("Failed to register IPv6 top resource\n"); - goto out; + goto err_out; } err = devl_resource_register(devlink, "fib", (u64)-1, @@ -476,7 +476,7 @@ static int nsim_dev_resources_register(struct devlink *devlink) NSIM_RESOURCE_IPV6, ¶ms); if (err) { pr_err("Failed to register IPv6 FIB resource\n"); - return err; + goto err_out; } err = devl_resource_register(devlink, "fib-rules", (u64)-1, @@ -484,7 +484,7 @@ static int nsim_dev_resources_register(struct devlink *devlink) NSIM_RESOURCE_IPV6, ¶ms); if (err) { pr_err("Failed to register IPv6 FIB rules resource\n"); - return err; + goto err_out; } /* Resources for nexthops */ @@ -492,8 +492,14 @@ static int nsim_dev_resources_register(struct devlink *devlink) NSIM_RESOURCE_NEXTHOPS, DEVLINK_RESOURCE_ID_PARENT_TOP, ¶ms); + if (err) { + pr_err("Failed to register NEXTHOPS resource\n"); + goto err_out; + } + return 0; -out: +err_out: + devl_resources_unregister(devlink); return err; } -- GitLab From a6aa8d0ce2cfba57ac0f23293fcb3be0b9f53fba Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 26 Oct 2022 09:46:42 +0800 Subject: [PATCH 0564/1927] netdevsim: remove dir in nsim_dev_debugfs_init() when creating ports dir failed Remove dir in nsim_dev_debugfs_init() when creating ports dir failed. Otherwise, the netdevsim device will not be created next time. Kernel reports an error: debugfs: Directory 'netdevsim1' with parent 'netdevsim' already present! Fixes: ab1d0cc004d7 ("netdevsim: change debugfs tree topology") Signed-off-by: Zhengchao Shao Signed-off-by: Jakub Kicinski --- drivers/net/netdevsim/dev.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index 81c3e14af0636..a7880c7ce94ca 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -309,8 +309,10 @@ static int nsim_dev_debugfs_init(struct nsim_dev *nsim_dev) if (IS_ERR(nsim_dev->ddir)) return PTR_ERR(nsim_dev->ddir); nsim_dev->ports_ddir = debugfs_create_dir("ports", nsim_dev->ddir); - if (IS_ERR(nsim_dev->ports_ddir)) - return PTR_ERR(nsim_dev->ports_ddir); + if (IS_ERR(nsim_dev->ports_ddir)) { + err = PTR_ERR(nsim_dev->ports_ddir); + goto err_ddir; + } debugfs_create_bool("fw_update_status", 0600, nsim_dev->ddir, &nsim_dev->fw_update_status); debugfs_create_u32("fw_update_overwrite_mask", 0600, nsim_dev->ddir, @@ -346,7 +348,7 @@ static int nsim_dev_debugfs_init(struct nsim_dev *nsim_dev) nsim_dev->nodes_ddir = debugfs_create_dir("rate_nodes", nsim_dev->ddir); if (IS_ERR(nsim_dev->nodes_ddir)) { err = PTR_ERR(nsim_dev->nodes_ddir); - goto err_out; + goto err_ports_ddir; } debugfs_create_bool("fail_trap_drop_counter_get", 0600, nsim_dev->ddir, @@ -354,8 +356,9 @@ static int nsim_dev_debugfs_init(struct nsim_dev *nsim_dev) nsim_udp_tunnels_debugfs_create(nsim_dev); return 0; -err_out: +err_ports_ddir: debugfs_remove_recursive(nsim_dev->ports_ddir); +err_ddir: debugfs_remove_recursive(nsim_dev->ddir); return err; } -- GitLab From 888be6b279b7257b5f6e4c9527675bff0a335596 Mon Sep 17 00:00:00 2001 From: Hyong Youb Kim Date: Wed, 26 Oct 2022 14:51:39 +0100 Subject: [PATCH 0565/1927] net/mlx5e: Do not increment ESN when updating IPsec ESN state An offloaded SA stops receiving after about 2^32 + replay_window packets. For example, when SA reaches , all subsequent packets get dropped with SA-icv-failure (integrity_failed). To reproduce the bug: - ConnectX-6 Dx with crypto enabled (FW 22.30.1004) - ipsec.conf: nic-offload = yes replay-window = 32 esn = yes salifetime=24h - Run netperf for a long time to send more than 2^32 packets netperf -H -t TCP_STREAM -l 20000 When 2^32 + replay_window packets are received, the replay window moves from the 2nd half of subspace (overlap=1) to the 1st half (overlap=0). The driver then updates the 'esn' value in NIC (i.e. seq_hi) as follows. seq_hi = xfrm_replay_seqhi(seq_bottom) new esn in NIC = seq_hi + 1 The +1 increment is wrong, as seq_hi already contains the correct seq_hi. For example, when seq_hi=1, the driver actually tells NIC to use seq_hi=2 (esn). This incorrect esn value causes all subsequent packets to fail integrity checks (SA-icv-failure). So, do not increment. Fixes: cb01008390bb ("net/mlx5: IPSec, Add support for ESN") Signed-off-by: Hyong Youb Kim Acked-by: Leon Romanovsky Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-2-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 2a8fd70206220..a715601865d31 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -101,7 +101,6 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) struct xfrm_replay_state_esn *replay_esn; u32 seq_bottom = 0; u8 overlap; - u32 *esn; if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) { sa_entry->esn_state.trigger = 0; @@ -116,11 +115,9 @@ static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry) sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x, htonl(seq_bottom)); - esn = &sa_entry->esn_state.esn; sa_entry->esn_state.trigger = 1; if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) { - ++(*esn); sa_entry->esn_state.overlap = 0; return true; } else if (unlikely(!overlap && -- GitLab From 212b4d7251c169f87fa734e79bdec8dd413be5cf Mon Sep 17 00:00:00 2001 From: Moshe Shemesh Date: Wed, 26 Oct 2022 14:51:40 +0100 Subject: [PATCH 0566/1927] net/mlx5: Wait for firmware to enable CRS before pci_restore_state After firmware reset driver should verify firmware already enabled CRS and became responsive to pci config cycles before restoring pci state. Fix that by waiting till device_id is readable through PCI again. Fixes: eabe8e5e88f5 ("net/mlx5: Handle sync reset now event") Signed-off-by: Moshe Shemesh Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-3-saeed@kernel.org Signed-off-by: Jakub Kicinski --- .../net/ethernet/mellanox/mlx5/core/fw_reset.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index e8896f3683626..07c583996c297 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -358,6 +358,23 @@ static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev) err = -ETIMEDOUT; } + do { + err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, ®16); + if (err) + return err; + if (reg16 == dev_id) + break; + msleep(20); + } while (!time_after(jiffies, timeout)); + + if (reg16 == dev_id) { + mlx5_core_info(dev, "Firmware responds to PCI config cycles again\n"); + } else { + mlx5_core_err(dev, "Firmware is not responsive (0x%04x) after %llu ms\n", + reg16, mlx5_tout_ms(dev, PCI_TOGGLE)); + err = -ETIMEDOUT; + } + restore: list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { pci_cfg_access_unlock(sdev); -- GitLab From 4ea9891d66410da5030dababb4b825d8e41cd7bb Mon Sep 17 00:00:00 2001 From: Rongwei Liu Date: Wed, 26 Oct 2022 14:51:41 +0100 Subject: [PATCH 0567/1927] net/mlx5: DR, Fix matcher disconnect error flow When 2nd flow rules arrives, it will merge together with the 1st one if matcher criteria is the same. If merge fails, driver will rollback the merge contents, and reject the 2nd rule. At rollback stage, matcher can't be disconnected unconditionally, otherise the 1st rule can't be hit anymore. Add logic to check if the matcher should be disconnected or not. Fixes: cc2295cd54e4 ("net/mlx5: DR, Improve steering for empty or RX/TX-only matchers") Signed-off-by: Rongwei Liu Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-4-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c index ddfaf78911881..91ff19f676951 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_rule.c @@ -1200,7 +1200,8 @@ dr_rule_create_rule_nic(struct mlx5dr_rule *rule, } remove_from_nic_tbl: - mlx5dr_matcher_remove_from_tbl_nic(dmn, nic_matcher); + if (!nic_matcher->rules) + mlx5dr_matcher_remove_from_tbl_nic(dmn, nic_matcher); free_hw_ste: mlx5dr_domain_nic_unlock(nic_dmn); -- GitLab From 19b43a432e3e47db656a8269a74b50aef826950c Mon Sep 17 00:00:00 2001 From: Aya Levin Date: Wed, 26 Oct 2022 14:51:42 +0100 Subject: [PATCH 0568/1927] net/mlx5e: Extend SKB room check to include PTP-SQ When tx_port_ts is set, the driver diverts all UPD traffic over PTP port to a dedicated PTP-SQ. The SKBs are cached until the wire-CQE arrives. When the packet size is greater then MTU, the firmware might drop it and the packet won't be transmitted to the wire, hence the wire-CQE won't reach the driver. In this case the SKBs are accumulated in the SKB fifo. Add room check to consider the PTP-SQ SKB fifo, when the SKB fifo is full, driver stops the queue resulting in a TX timeout. Devlink TX-reporter can recover from it. Fixes: 1880bc4e4a96 ("net/mlx5e: Add TX port timestamp support") Signed-off-by: Aya Levin Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-5-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h | 9 +++++++++ drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h | 6 ++++++ drivers/net/ethernet/mellanox/mlx5/core/en_tx.c | 6 ++++++ 3 files changed, 21 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index 5bce554e131a9..cc7efde88ac3c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -6,6 +6,7 @@ #include "en.h" #include "en_stats.h" +#include "en/txrx.h" #include #define MLX5E_PTP_CHANNEL_IX 0 @@ -68,6 +69,14 @@ static inline bool mlx5e_use_ptpsq(struct sk_buff *skb) fk.ports.dst == htons(PTP_EV_PORT)); } +static inline bool mlx5e_ptpsq_fifo_has_room(struct mlx5e_txqsq *sq) +{ + if (!sq->ptpsq) + return true; + + return mlx5e_skb_fifo_has_room(&sq->ptpsq->skb_fifo); +} + int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params, u8 lag_port, struct mlx5e_ptp **cp); void mlx5e_ptp_close(struct mlx5e_ptp *c); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index 4456ad5cedf1e..cb164b62f5436 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -57,6 +57,12 @@ netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev); bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget); void mlx5e_free_txqsq_descs(struct mlx5e_txqsq *sq); +static inline bool +mlx5e_skb_fifo_has_room(struct mlx5e_skb_fifo *fifo) +{ + return (*fifo->pc - *fifo->cc) < fifo->mask; +} + static inline bool mlx5e_wqc_has_room_for(struct mlx5_wq_cyc *wq, u16 cc, u16 pc, u16 n) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index bf2232a2a836b..6adca01fbdc9d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -392,6 +392,11 @@ mlx5e_txwqe_complete(struct mlx5e_txqsq *sq, struct sk_buff *skb, if (unlikely(sq->ptpsq)) { mlx5e_skb_cb_hwtstamp_init(skb); mlx5e_skb_fifo_push(&sq->ptpsq->skb_fifo, skb); + if (!netif_tx_queue_stopped(sq->txq) && + !mlx5e_skb_fifo_has_room(&sq->ptpsq->skb_fifo)) { + netif_tx_stop_queue(sq->txq); + sq->stats->stopped++; + } skb_get(skb); } @@ -868,6 +873,7 @@ bool mlx5e_poll_tx_cq(struct mlx5e_cq *cq, int napi_budget) if (netif_tx_queue_stopped(sq->txq) && mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, sq->stop_room) && + mlx5e_ptpsq_fifo_has_room(sq) && !test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) { netif_tx_wake_queue(sq->txq); stats->wake++; -- GitLab From 8dc47c0527c1586e3ebe0efd323f1d8abb181c77 Mon Sep 17 00:00:00 2001 From: Paul Blakey Date: Wed, 26 Oct 2022 14:51:43 +0100 Subject: [PATCH 0569/1927] net/mlx5e: Update restore chain id for slow path packets Currently encap slow path rules just forward to software without setting the chain id miss register, so driver doesn't restore the chain, and packets hitting this rule will restart from tc chain 0 instead of continuing to the chain the encap rule was on. Fix this by setting the chain id miss register to the chain id mapping. Fixes: 8f1e0b97cc70 ("net/mlx5: E-Switch, Mark miss packets with new chain id mapping") Signed-off-by: Paul Blakey Reviewed-by: Oz Shlomo Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-6-saeed@kernel.org Signed-off-by: Jakub Kicinski --- .../ethernet/mellanox/mlx5/core/en/tc_priv.h | 2 + .../net/ethernet/mellanox/mlx5/core/en_tc.c | 62 ++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index 10c9a8a79d005..2e42d7c5451e9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -96,6 +96,7 @@ struct mlx5e_tc_flow { struct encap_flow_item encaps[MLX5_MAX_FLOW_FWD_VPORTS]; struct mlx5e_tc_flow *peer_flow; struct mlx5e_mod_hdr_handle *mh; /* attached mod header instance */ + struct mlx5e_mod_hdr_handle *slow_mh; /* attached mod header instance for slow path */ struct mlx5e_hairpin_entry *hpe; /* attached hairpin instance */ struct list_head hairpin; /* flows sharing the same hairpin */ struct list_head peer; /* flows with peer flow */ @@ -111,6 +112,7 @@ struct mlx5e_tc_flow { struct completion del_hw_done; struct mlx5_flow_attr *attr; struct list_head attrs; + u32 chain_mapping; }; struct mlx5_flow_handle * diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 70a7a61f97087..2cceace36c776 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -1405,8 +1405,13 @@ mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch *esw, struct mlx5e_tc_flow *flow, struct mlx5_flow_spec *spec) { + struct mlx5e_tc_mod_hdr_acts mod_acts = {}; + struct mlx5e_mod_hdr_handle *mh = NULL; struct mlx5_flow_attr *slow_attr; struct mlx5_flow_handle *rule; + bool fwd_and_modify_cap; + u32 chain_mapping = 0; + int err; slow_attr = mlx5_alloc_flow_attr(MLX5_FLOW_NAMESPACE_FDB); if (!slow_attr) @@ -1417,13 +1422,56 @@ mlx5e_tc_offload_to_slow_path(struct mlx5_eswitch *esw, slow_attr->esw_attr->split_count = 0; slow_attr->flags |= MLX5_ATTR_FLAG_SLOW_PATH; + fwd_and_modify_cap = MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table); + if (!fwd_and_modify_cap) + goto skip_restore; + + err = mlx5_chains_get_chain_mapping(esw_chains(esw), flow->attr->chain, &chain_mapping); + if (err) + goto err_get_chain; + + err = mlx5e_tc_match_to_reg_set(esw->dev, &mod_acts, MLX5_FLOW_NAMESPACE_FDB, + CHAIN_TO_REG, chain_mapping); + if (err) + goto err_reg_set; + + mh = mlx5e_mod_hdr_attach(esw->dev, get_mod_hdr_table(flow->priv, flow), + MLX5_FLOW_NAMESPACE_FDB, &mod_acts); + if (IS_ERR(mh)) { + err = PTR_ERR(mh); + goto err_attach; + } + + slow_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + slow_attr->modify_hdr = mlx5e_mod_hdr_get(mh); + +skip_restore: rule = mlx5e_tc_offload_fdb_rules(esw, flow, spec, slow_attr); - if (!IS_ERR(rule)) - flow_flag_set(flow, SLOW); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + goto err_offload; + } + flow->slow_mh = mh; + flow->chain_mapping = chain_mapping; + flow_flag_set(flow, SLOW); + + mlx5e_mod_hdr_dealloc(&mod_acts); kfree(slow_attr); return rule; + +err_offload: + if (fwd_and_modify_cap) + mlx5e_mod_hdr_detach(esw->dev, get_mod_hdr_table(flow->priv, flow), mh); +err_attach: +err_reg_set: + if (fwd_and_modify_cap) + mlx5_chains_put_chain_mapping(esw_chains(esw), chain_mapping); +err_get_chain: + mlx5e_mod_hdr_dealloc(&mod_acts); + kfree(slow_attr); + return ERR_PTR(err); } void mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw, @@ -1441,7 +1489,17 @@ void mlx5e_tc_unoffload_from_slow_path(struct mlx5_eswitch *esw, slow_attr->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; slow_attr->esw_attr->split_count = 0; slow_attr->flags |= MLX5_ATTR_FLAG_SLOW_PATH; + if (flow->slow_mh) { + slow_attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + slow_attr->modify_hdr = mlx5e_mod_hdr_get(flow->slow_mh); + } mlx5e_tc_unoffload_fdb_rules(esw, flow, slow_attr); + if (flow->slow_mh) { + mlx5e_mod_hdr_detach(esw->dev, get_mod_hdr_table(flow->priv, flow), flow->slow_mh); + mlx5_chains_put_chain_mapping(esw_chains(esw), flow->chain_mapping); + flow->chain_mapping = 0; + flow->slow_mh = NULL; + } flow_flag_clear(flow, SLOW); kfree(slow_attr); } -- GitLab From 0f3caaa2c6fbf9f892bd235c9dce9eb551f8d815 Mon Sep 17 00:00:00 2001 From: Saeed Mahameed Date: Wed, 26 Oct 2022 14:51:44 +0100 Subject: [PATCH 0570/1927] net/mlx5: ASO, Create the ASO SQ with the correct timestamp format mlx5 SQs must select the timestamp format explicitly according to the active clock mode, select the current active timestamp mode so ASO SQ create will succeed. This fixes the following error prints when trying to create ipsec ASO SQ while the timestamp format is real time mode. mlx5_cmd_out_err:778:(pid 34874): CREATE_SQ(0x904) op_mod(0x0) failed, status bad parameter(0x3), syndrome (0xd61c0b), err(-22) mlx5_aso_create_sq:285:(pid 34874): Failed to open aso wq sq, err=-22 mlx5e_ipsec_init:436:(pid 34874): IPSec initialization failed, -22 Fixes: cdd04f4d4d71 ("net/mlx5: Add support to create SQ and CQ for ASO") Signed-off-by: Saeed Mahameed Reported-by: Leon Romanovsky Reviewed-by: Leon Romanovsky Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-7-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c index baa8092f335e3..c971ff04dd046 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -3,6 +3,7 @@ #include #include +#include "clock.h" #include "aso.h" #include "wq.h" @@ -179,6 +180,7 @@ static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn, { void *in, *sqc, *wq; int inlen, err; + u8 ts_format; inlen = MLX5_ST_SZ_BYTES(create_sq_in) + sizeof(u64) * sq->wq_ctrl.buf.npages; @@ -195,6 +197,11 @@ static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn, MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); MLX5_SET(sqc, sqc, flush_in_error_en, 1); + ts_format = mlx5_is_real_time_sq(mdev) ? + MLX5_TIMESTAMP_FORMAT_REAL_TIME : + MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; + MLX5_SET(sqc, sqc, ts_format, ts_format); + MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - -- GitLab From bacd22df95147ed673bec4692ab2d4d585935241 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Wed, 26 Oct 2022 14:51:45 +0100 Subject: [PATCH 0571/1927] net/mlx5: Fix possible use-after-free in async command interface mlx5_cmd_cleanup_async_ctx should return only after all its callback handlers were completed. Before this patch, the below race between mlx5_cmd_cleanup_async_ctx and mlx5_cmd_exec_cb_handler was possible and lead to a use-after-free: 1. mlx5_cmd_cleanup_async_ctx is called while num_inflight is 2 (i.e. elevated by 1, a single inflight callback). 2. mlx5_cmd_cleanup_async_ctx decreases num_inflight to 1. 3. mlx5_cmd_exec_cb_handler is called, decreases num_inflight to 0 and is about to call wake_up(). 4. mlx5_cmd_cleanup_async_ctx calls wait_event, which returns immediately as the condition (num_inflight == 0) holds. 5. mlx5_cmd_cleanup_async_ctx returns. 6. The caller of mlx5_cmd_cleanup_async_ctx frees the mlx5_async_ctx object. 7. mlx5_cmd_exec_cb_handler goes on and calls wake_up() on the freed object. Fix it by syncing using a completion object. Mark it completed when num_inflight reaches 0. Trace: BUG: KASAN: use-after-free in do_raw_spin_lock+0x23d/0x270 Read of size 4 at addr ffff888139cd12f4 by task swapper/5/0 CPU: 5 PID: 0 Comm: swapper/5 Not tainted 6.0.0-rc3_for_upstream_debug_2022_08_30_13_10 #1 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014 Call Trace: dump_stack_lvl+0x57/0x7d print_report.cold+0x2d5/0x684 ? do_raw_spin_lock+0x23d/0x270 kasan_report+0xb1/0x1a0 ? do_raw_spin_lock+0x23d/0x270 do_raw_spin_lock+0x23d/0x270 ? rwlock_bug.part.0+0x90/0x90 ? __delete_object+0xb8/0x100 ? lock_downgrade+0x6e0/0x6e0 _raw_spin_lock_irqsave+0x43/0x60 ? __wake_up_common_lock+0xb9/0x140 __wake_up_common_lock+0xb9/0x140 ? __wake_up_common+0x650/0x650 ? destroy_tis_callback+0x53/0x70 [mlx5_core] ? kasan_set_track+0x21/0x30 ? destroy_tis_callback+0x53/0x70 [mlx5_core] ? kfree+0x1ba/0x520 ? do_raw_spin_unlock+0x54/0x220 mlx5_cmd_exec_cb_handler+0x136/0x1a0 [mlx5_core] ? mlx5_cmd_cleanup_async_ctx+0x220/0x220 [mlx5_core] ? mlx5_cmd_cleanup_async_ctx+0x220/0x220 [mlx5_core] mlx5_cmd_comp_handler+0x65a/0x12b0 [mlx5_core] ? dump_command+0xcc0/0xcc0 [mlx5_core] ? lockdep_hardirqs_on_prepare+0x400/0x400 ? cmd_comp_notifier+0x7e/0xb0 [mlx5_core] cmd_comp_notifier+0x7e/0xb0 [mlx5_core] atomic_notifier_call_chain+0xd7/0x1d0 mlx5_eq_async_int+0x3ce/0xa20 [mlx5_core] atomic_notifier_call_chain+0xd7/0x1d0 ? irq_release+0x140/0x140 [mlx5_core] irq_int_handler+0x19/0x30 [mlx5_core] __handle_irq_event_percpu+0x1f2/0x620 handle_irq_event+0xb2/0x1d0 handle_edge_irq+0x21e/0xb00 __common_interrupt+0x79/0x1a0 common_interrupt+0x78/0xa0 asm_common_interrupt+0x22/0x40 RIP: 0010:default_idle+0x42/0x60 Code: c1 83 e0 07 48 c1 e9 03 83 c0 03 0f b6 14 11 38 d0 7c 04 84 d2 75 14 8b 05 eb 47 22 02 85 c0 7e 07 0f 00 2d e0 9f 48 00 fb f4 48 c7 c7 80 08 7f 85 e8 d1 d3 3e fe eb de 66 66 2e 0f 1f 84 00 RSP: 0018:ffff888100dbfdf0 EFLAGS: 00000242 RAX: 0000000000000001 RBX: ffffffff84ecbd48 RCX: 1ffffffff0afe110 RDX: 0000000000000004 RSI: 0000000000000000 RDI: ffffffff835cc9bc RBP: 0000000000000005 R08: 0000000000000001 R09: ffff88881dec4ac3 R10: ffffed1103bd8958 R11: 0000017d0ca571c9 R12: 0000000000000005 R13: ffffffff84f024e0 R14: 0000000000000000 R15: dffffc0000000000 ? default_idle_call+0xcc/0x450 default_idle_call+0xec/0x450 do_idle+0x394/0x450 ? arch_cpu_idle_exit+0x40/0x40 ? do_idle+0x17/0x450 cpu_startup_entry+0x19/0x20 start_secondary+0x221/0x2b0 ? set_cpu_sibling_map+0x2070/0x2070 secondary_startup_64_no_verify+0xcd/0xdb Allocated by task 49502: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0x81/0xa0 kvmalloc_node+0x48/0xe0 mlx5e_bulk_async_init+0x35/0x110 [mlx5_core] mlx5e_tls_priv_tx_list_cleanup+0x84/0x3e0 [mlx5_core] mlx5e_ktls_cleanup_tx+0x38f/0x760 [mlx5_core] mlx5e_cleanup_nic_tx+0xa7/0x100 [mlx5_core] mlx5e_detach_netdev+0x1ca/0x2b0 [mlx5_core] mlx5e_suspend+0xdb/0x140 [mlx5_core] mlx5e_remove+0x89/0x190 [mlx5_core] auxiliary_bus_remove+0x52/0x70 device_release_driver_internal+0x40f/0x650 driver_detach+0xc1/0x180 bus_remove_driver+0x125/0x2f0 auxiliary_driver_unregister+0x16/0x50 mlx5e_cleanup+0x26/0x30 [mlx5_core] cleanup+0xc/0x4e [mlx5_core] __x64_sys_delete_module+0x2b5/0x450 do_syscall_64+0x3d/0x90 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Freed by task 49502: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x30 ____kasan_slab_free+0x11d/0x1b0 kfree+0x1ba/0x520 mlx5e_tls_priv_tx_list_cleanup+0x2e7/0x3e0 [mlx5_core] mlx5e_ktls_cleanup_tx+0x38f/0x760 [mlx5_core] mlx5e_cleanup_nic_tx+0xa7/0x100 [mlx5_core] mlx5e_detach_netdev+0x1ca/0x2b0 [mlx5_core] mlx5e_suspend+0xdb/0x140 [mlx5_core] mlx5e_remove+0x89/0x190 [mlx5_core] auxiliary_bus_remove+0x52/0x70 device_release_driver_internal+0x40f/0x650 driver_detach+0xc1/0x180 bus_remove_driver+0x125/0x2f0 auxiliary_driver_unregister+0x16/0x50 mlx5e_cleanup+0x26/0x30 [mlx5_core] cleanup+0xc/0x4e [mlx5_core] __x64_sys_delete_module+0x2b5/0x450 do_syscall_64+0x3d/0x90 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Fixes: e355477ed9e4 ("net/mlx5: Make mlx5_cmd_exec_cb() a safe API") Signed-off-by: Tariq Toukan Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-8-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 10 +++++----- include/linux/mlx5/driver.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 0377392848d92..46ba4c2faad21 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -2004,7 +2004,7 @@ void mlx5_cmd_init_async_ctx(struct mlx5_core_dev *dev, ctx->dev = dev; /* Starts at 1 to avoid doing wake_up if we are not cleaning up */ atomic_set(&ctx->num_inflight, 1); - init_waitqueue_head(&ctx->wait); + init_completion(&ctx->inflight_done); } EXPORT_SYMBOL(mlx5_cmd_init_async_ctx); @@ -2018,8 +2018,8 @@ EXPORT_SYMBOL(mlx5_cmd_init_async_ctx); */ void mlx5_cmd_cleanup_async_ctx(struct mlx5_async_ctx *ctx) { - atomic_dec(&ctx->num_inflight); - wait_event(ctx->wait, atomic_read(&ctx->num_inflight) == 0); + if (!atomic_dec_and_test(&ctx->num_inflight)) + wait_for_completion(&ctx->inflight_done); } EXPORT_SYMBOL(mlx5_cmd_cleanup_async_ctx); @@ -2032,7 +2032,7 @@ static void mlx5_cmd_exec_cb_handler(int status, void *_work) status = cmd_status_err(ctx->dev, status, work->opcode, work->out); work->user_callback(status, work); if (atomic_dec_and_test(&ctx->num_inflight)) - wake_up(&ctx->wait); + complete(&ctx->inflight_done); } int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, @@ -2050,7 +2050,7 @@ int mlx5_cmd_exec_cb(struct mlx5_async_ctx *ctx, void *in, int in_size, ret = cmd_exec(ctx->dev, in, in_size, out, out_size, mlx5_cmd_exec_cb_handler, work, false); if (ret && atomic_dec_and_test(&ctx->num_inflight)) - wake_up(&ctx->wait); + complete(&ctx->inflight_done); return ret; } diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index a12929bc31b22..af2ceb4160bcd 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -970,7 +970,7 @@ void mlx5_cmd_allowed_opcode(struct mlx5_core_dev *dev, u16 opcode); struct mlx5_async_ctx { struct mlx5_core_dev *dev; atomic_t num_inflight; - struct wait_queue_head wait; + struct completion inflight_done; }; struct mlx5_async_work; -- GitLab From f382a2413dae8c855226a72600812a4b37432c48 Mon Sep 17 00:00:00 2001 From: Ariel Levkovich Date: Wed, 26 Oct 2022 14:51:46 +0100 Subject: [PATCH 0572/1927] net/mlx5e: TC, Reject forwarding from internal port to internal port Reject TC rules that forward from internal port to internal port as it is not supported. This include rules that are explicitly have internal port as the filter device as well as rules that apply on tunnel interfaces as the route device for the tunnel interface can be an internal port. Fixes: 27484f7170ed ("net/mlx5e: Offload tc rules that redirect to ovs internal port") Signed-off-by: Ariel Levkovich Reviewed-by: Maor Dickman Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-9-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 2cceace36c776..73f91e54e9d07 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -4066,6 +4066,7 @@ parse_tc_fdb_actions(struct mlx5e_priv *priv, struct mlx5e_tc_flow_parse_attr *parse_attr; struct mlx5_flow_attr *attr = flow->attr; struct mlx5_esw_flow_attr *esw_attr; + struct net_device *filter_dev; int err; err = flow_action_supported(flow_action, extack); @@ -4074,6 +4075,7 @@ parse_tc_fdb_actions(struct mlx5e_priv *priv, esw_attr = attr->esw_attr; parse_attr = attr->parse_attr; + filter_dev = parse_attr->filter_dev; parse_state = &parse_attr->parse_state; mlx5e_tc_act_init_parse_state(parse_state, flow, flow_action, extack); parse_state->ct_priv = get_ct_priv(priv); @@ -4083,13 +4085,21 @@ parse_tc_fdb_actions(struct mlx5e_priv *priv, return err; /* Forward to/from internal port can only have 1 dest */ - if ((netif_is_ovs_master(parse_attr->filter_dev) || esw_attr->dest_int_port) && + if ((netif_is_ovs_master(filter_dev) || esw_attr->dest_int_port) && esw_attr->out_count > 1) { NL_SET_ERR_MSG_MOD(extack, "Rules with internal port can have only one destination"); return -EOPNOTSUPP; } + /* Forward from tunnel/internal port to internal port is not supported */ + if ((mlx5e_get_tc_tun(filter_dev) || netif_is_ovs_master(filter_dev)) && + esw_attr->dest_int_port) { + NL_SET_ERR_MSG_MOD(extack, + "Forwarding from tunnel/internal port to internal port is not supported"); + return -EOPNOTSUPP; + } + err = actions_prepare_mod_hdr_actions(priv, flow, attr, extack); if (err) return err; -- GitLab From 94d651739e17b0ee9b556e60f206fe538d06dc05 Mon Sep 17 00:00:00 2001 From: Roi Dayan Date: Wed, 26 Oct 2022 14:51:47 +0100 Subject: [PATCH 0573/1927] net/mlx5e: TC, Fix cloned flow attr instance dests are not zeroed On multi table split the driver creates a new attr instance with data being copied from prev attr instance zeroing action flags. Also need to reset dests properties to avoid incorrect dests per attr. Fixes: 8300f225268b ("net/mlx5e: Create new flow attr for multi table actions") Signed-off-by: Roi Dayan Reviewed-by: Maor Dickman Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-10-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 73f91e54e9d07..dd6fea9e9a5b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -3633,6 +3633,10 @@ mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr, attr2->action = 0; attr2->flags = 0; attr2->parse_attr = parse_attr; + attr2->esw_attr->out_count = 0; + attr2->esw_attr->split_count = 0; + attr2->dest_chain = 0; + attr2->dest_ft = NULL; return attr2; } -- GitLab From 416ef713631937cf5452476a7f1041a3ae7b06c6 Mon Sep 17 00:00:00 2001 From: Roy Novich Date: Wed, 26 Oct 2022 14:51:48 +0100 Subject: [PATCH 0574/1927] net/mlx5: Update fw fatal reporter state on PCI handlers successful recover Update devlink health fw fatal reporter state to "healthy" is needed by strictly calling devlink_health_reporter_state_update() after recovery was done by PCI error handler. This is needed when fw_fatal reporter was triggered due to PCI error. Poll health is called and set reporter state to error. Health recovery failed (since EEH didn't re-enable the PCI). PCI handlers keep on recover flow and succeed later without devlink acknowledgment. Fix this by adding devlink state update at the end of the PCI handler recovery process. Fixes: 6181e5cb752e ("devlink: add support for reporter recovery completion") Signed-off-by: Roy Novich Reviewed-by: Moshe Shemesh Reviewed-by: Aya Levin Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-11-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/main.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 0b459d841c3a3..283c4cc28944d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1872,6 +1872,10 @@ static void mlx5_pci_resume(struct pci_dev *pdev) err = mlx5_load_one(dev, false); + if (!err) + devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter, + DEVLINK_HEALTH_REPORTER_STATE_HEALTHY); + mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err, !err ? "recovered" : "Failed"); } -- GitLab From aefb62a9988749703435e941704624949a80a2a9 Mon Sep 17 00:00:00 2001 From: Suresh Devarakonda Date: Wed, 26 Oct 2022 14:51:49 +0100 Subject: [PATCH 0575/1927] net/mlx5: Fix crash during sync firmware reset When setting Bluefield to DPU NIC mode using mlxconfig tool + sync firmware reset flow, we run into scenario where the host was not eswitch manager at the time of mlx5 driver load but becomes eswitch manager after the sync firmware reset flow. This results in null pointer access of mpfs structure during mac filter add. This change prevents null pointer access but mpfs table entries will not be added. Fixes: 5ec697446f46 ("net/mlx5: Add support for devlink reload action fw activate") Signed-off-by: Suresh Devarakonda Reviewed-by: Moshe Shemesh Reviewed-by: Bodong Wang Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-12-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c index 839a01da110f3..8ff16318e32dc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c @@ -122,7 +122,7 @@ void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev) { struct mlx5_mpfs *mpfs = dev->priv.mpfs; - if (!MLX5_ESWITCH_MANAGER(dev)) + if (!mpfs) return; WARN_ON(!hlist_empty(mpfs->hash)); @@ -137,7 +137,7 @@ int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac) int err = 0; u32 index; - if (!MLX5_ESWITCH_MANAGER(dev)) + if (!mpfs) return 0; mutex_lock(&mpfs->lock); @@ -185,7 +185,7 @@ int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac) int err = 0; u32 index; - if (!MLX5_ESWITCH_MANAGER(dev)) + if (!mpfs) return 0; mutex_lock(&mpfs->lock); -- GitLab From d3ecf037569c64490a5cae5a1ac4605f4bedc607 Mon Sep 17 00:00:00 2001 From: Raed Salem Date: Wed, 26 Oct 2022 14:51:50 +0100 Subject: [PATCH 0576/1927] net/mlx5e: Fix macsec coverity issue at rx sa update The cited commit at update rx sa operation passes object attributes to MACsec object create function without initializing/setting all attributes fields leaving some of them with garbage values, therefore violating the implicit assumption at create object function, which assumes that all input object attributes fields are set. Fix by initializing the object attributes struct to zero, thus leaving unset fields with the legal zero value. Fixes: aae3454e4d4c ("net/mlx5e: Add MACsec offload Rx command support") Signed-off-by: Raed Salem Reviewed-by: Lior Nahmanson Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-13-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 4331235b21ee8..250c878ba2c90 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -432,7 +432,7 @@ static int mlx5e_macsec_update_rx_sa(struct mlx5e_macsec *macsec, bool active) { struct mlx5_core_dev *mdev = macsec->mdev; - struct mlx5_macsec_obj_attrs attrs; + struct mlx5_macsec_obj_attrs attrs = {}; int err = 0; if (rx_sa->active != active) -- GitLab From 74573e38e933a6dbb11691bea535c54d683cd06e Mon Sep 17 00:00:00 2001 From: Raed Salem Date: Wed, 26 Oct 2022 14:51:51 +0100 Subject: [PATCH 0577/1927] net/mlx5e: Fix macsec rx security association (SA) update/delete The cited commit adds the support for update/delete MACsec Rx SA, naturally, these operations need to check if the SA in question exists to update/delete the SA and return error code otherwise, however they do just the opposite i.e. return with error if the SA exists Fix by change the check to return error in case the SA in question does not exist, adjust error message and code accordingly. Fixes: aae3454e4d4c ("net/mlx5e: Add MACsec offload Rx command support") Signed-off-by: Raed Salem Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-14-saeed@kernel.org Signed-off-by: Jakub Kicinski --- .../ethernet/mellanox/mlx5/core/en_accel/macsec.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 250c878ba2c90..6ae9fcdbda07a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -999,11 +999,11 @@ static int mlx5e_macsec_upd_rxsa(struct macsec_context *ctx) } rx_sa = rx_sc->rx_sa[assoc_num]; - if (rx_sa) { + if (!rx_sa) { netdev_err(ctx->netdev, - "MACsec offload rx_sc sci %lld rx_sa %d already exist\n", + "MACsec offload rx_sc sci %lld rx_sa %d doesn't exist\n", sci, assoc_num); - err = -EEXIST; + err = -EINVAL; goto out; } @@ -1055,11 +1055,11 @@ static int mlx5e_macsec_del_rxsa(struct macsec_context *ctx) } rx_sa = rx_sc->rx_sa[assoc_num]; - if (rx_sa) { + if (!rx_sa) { netdev_err(ctx->netdev, - "MACsec offload rx_sc sci %lld rx_sa %d already exist\n", + "MACsec offload rx_sc sci %lld rx_sa %d doesn't exist\n", sci, assoc_num); - err = -EEXIST; + err = -EINVAL; goto out; } -- GitLab From d550956458a83cf87cb8fe24862f3340065c62c1 Mon Sep 17 00:00:00 2001 From: Raed Salem Date: Wed, 26 Oct 2022 14:51:52 +0100 Subject: [PATCH 0578/1927] net/mlx5e: Fix wrong bitwise comparison usage in macsec_fs_rx_add_rule function The cited commit produces a sparse check error of type "sparse: error: restricted __be64 degrades to integer". The offending line wrongly did a bitwise operation between two different storage types one of 64 bit when the other smaller side is 16 bit which caused the above sparse error, furthermore bitwise operation usage here is wrong in the first place as the constant MACSEC_PORT_ES is not a bitwise field. Fix by using the right mask to get the lower 16 bit if the sci number, and use comparison operator '==' instead of bitwise '&' operator. Fixes: 3b20949cb21b ("net/mlx5e: Add MACsec RX steering rules") Signed-off-by: Raed Salem Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-15-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c index 13dc628b988a1..1ac0cf04e8113 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c @@ -1180,7 +1180,7 @@ macsec_fs_rx_add_rule(struct mlx5e_macsec_fs *macsec_fs, rx_rule->rule[0] = rule; /* Rx crypto table without SCI rule */ - if (cpu_to_be64((__force u64)attrs->sci) & ntohs(MACSEC_PORT_ES)) { + if ((cpu_to_be64((__force u64)attrs->sci) & 0xFFFF) == ntohs(MACSEC_PORT_ES)) { memset(spec, 0, sizeof(struct mlx5_flow_spec)); memset(&dest, 0, sizeof(struct mlx5_flow_destination)); memset(&flow_act, 0, sizeof(flow_act)); -- GitLab From 12ba40ba3dc3a28ad579b7de2202ab6419da304a Mon Sep 17 00:00:00 2001 From: Raed Salem Date: Wed, 26 Oct 2022 14:51:53 +0100 Subject: [PATCH 0579/1927] net/mlx5e: Fix macsec sci endianness at rx sa update The cited commit at rx sa update operation passes the sci object attribute, in the wrong endianness and not as expected by the HW effectively create malformed hw sa context in case of update rx sa consequently, HW produces unexpected MACsec packets which uses this sa. Fix by passing sci to create macsec object with the correct endianness, while at it add __force u64 to prevent sparse check error of type "sparse: error: incorrect type in assignment". Fixes: aae3454e4d4c ("net/mlx5e: Add MACsec offload Rx command support") Signed-off-by: Raed Salem Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed Link: https://lore.kernel.org/r/20221026135153.154807-16-saeed@kernel.org Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c index 6ae9fcdbda07a..2ef36cb9555ab 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c @@ -444,7 +444,7 @@ static int mlx5e_macsec_update_rx_sa(struct mlx5e_macsec *macsec, return 0; } - attrs.sci = rx_sa->sci; + attrs.sci = cpu_to_be64((__force u64)rx_sa->sci); attrs.enc_key_id = rx_sa->enc_key_id; err = mlx5e_macsec_create_object(mdev, &attrs, false, &rx_sa->macsec_obj_id); if (err) -- GitLab From 228ebc41dfab5b5d34cd76835ddb0ca8ee12f513 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 27 Oct 2022 04:03:46 +0000 Subject: [PATCH 0580/1927] net: do not sense pfmemalloc status in skb_append_pagefrags() skb_append_pagefrags() is used by af_unix and udp sendpage() implementation so far. In commit 326140063946 ("tcp: TX zerocopy should not sense pfmemalloc status") we explained why we should not sense pfmemalloc status for pages owned by user space. We should also use skb_fill_page_desc_noacc() in skb_append_pagefrags() to avoid following KCSAN report: BUG: KCSAN: data-race in lru_add_fn / skb_append_pagefrags write to 0xffffea00058fc1c8 of 8 bytes by task 17319 on cpu 0: __list_add include/linux/list.h:73 [inline] list_add include/linux/list.h:88 [inline] lruvec_add_folio include/linux/mm_inline.h:323 [inline] lru_add_fn+0x327/0x410 mm/swap.c:228 folio_batch_move_lru+0x1e1/0x2a0 mm/swap.c:246 lru_add_drain_cpu+0x73/0x250 mm/swap.c:669 lru_add_drain+0x21/0x60 mm/swap.c:773 free_pages_and_swap_cache+0x16/0x70 mm/swap_state.c:311 tlb_batch_pages_flush mm/mmu_gather.c:59 [inline] tlb_flush_mmu_free mm/mmu_gather.c:256 [inline] tlb_flush_mmu+0x5b2/0x640 mm/mmu_gather.c:263 tlb_finish_mmu+0x86/0x100 mm/mmu_gather.c:363 exit_mmap+0x190/0x4d0 mm/mmap.c:3098 __mmput+0x27/0x1b0 kernel/fork.c:1185 mmput+0x3d/0x50 kernel/fork.c:1207 copy_process+0x19fc/0x2100 kernel/fork.c:2518 kernel_clone+0x166/0x550 kernel/fork.c:2671 __do_sys_clone kernel/fork.c:2812 [inline] __se_sys_clone kernel/fork.c:2796 [inline] __x64_sys_clone+0xc3/0xf0 kernel/fork.c:2796 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x2b/0x70 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd read to 0xffffea00058fc1c8 of 8 bytes by task 17325 on cpu 1: page_is_pfmemalloc include/linux/mm.h:1817 [inline] __skb_fill_page_desc include/linux/skbuff.h:2432 [inline] skb_fill_page_desc include/linux/skbuff.h:2453 [inline] skb_append_pagefrags+0x210/0x600 net/core/skbuff.c:3974 unix_stream_sendpage+0x45e/0x990 net/unix/af_unix.c:2338 kernel_sendpage+0x184/0x300 net/socket.c:3561 sock_sendpage+0x5a/0x70 net/socket.c:1054 pipe_to_sendpage+0x128/0x160 fs/splice.c:361 splice_from_pipe_feed fs/splice.c:415 [inline] __splice_from_pipe+0x222/0x4d0 fs/splice.c:559 splice_from_pipe fs/splice.c:594 [inline] generic_splice_sendpage+0x89/0xc0 fs/splice.c:743 do_splice_from fs/splice.c:764 [inline] direct_splice_actor+0x80/0xa0 fs/splice.c:931 splice_direct_to_actor+0x305/0x620 fs/splice.c:886 do_splice_direct+0xfb/0x180 fs/splice.c:974 do_sendfile+0x3bf/0x910 fs/read_write.c:1255 __do_sys_sendfile64 fs/read_write.c:1323 [inline] __se_sys_sendfile64 fs/read_write.c:1309 [inline] __x64_sys_sendfile64+0x10c/0x150 fs/read_write.c:1309 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x2b/0x70 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd value changed: 0x0000000000000000 -> 0xffffea00058fc188 Reported by Kernel Concurrency Sanitizer on: CPU: 1 PID: 17325 Comm: syz-executor.0 Not tainted 6.1.0-rc1-syzkaller-00158-g440b7895c990-dirty #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/11/2022 Fixes: 326140063946 ("tcp: TX zerocopy should not sense pfmemalloc status") Reported-by: syzbot Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20221027040346.1104204-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/core/skbuff.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 1d9719e72f9d9..d1a3fa6f3f126 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3971,7 +3971,7 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page, } else if (i < MAX_SKB_FRAGS) { skb_zcopy_downgrade_managed(skb); get_page(page); - skb_fill_page_desc(skb, i, page, offset, size); + skb_fill_page_desc_noacc(skb, i, page, offset, size); } else { return -EMSGSIZE; } -- GitLab From ee15e1f38dc201fa7d63c13aa258b728dce27f4d Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Thu, 27 Oct 2022 04:06:37 +0000 Subject: [PATCH 0581/1927] kcm: do not sense pfmemalloc status in kcm_sendpage() Similar to changes done in TCP in blamed commit. We should not sense pfmemalloc status in sendpage() methods. Fixes: 326140063946 ("tcp: TX zerocopy should not sense pfmemalloc status") Signed-off-by: Eric Dumazet Link: https://lore.kernel.org/r/20221027040637.1107703-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- net/kcm/kcmsock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index 63e32f181f435..a5004228111de 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -839,7 +839,7 @@ static ssize_t kcm_sendpage(struct socket *sock, struct page *page, } get_page(page); - skb_fill_page_desc(skb, i, page, offset, size); + skb_fill_page_desc_noacc(skb, i, page, offset, size); skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG; coalesced: -- GitLab From 3c6bf6bddc84888c0ce163b09dee0ddd23b5172a Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 24 Oct 2022 22:00:28 +0800 Subject: [PATCH 0582/1927] fbdev: cyber2000fb: fix missing pci_disable_device() Add missing pci_disable_device() in error path of probe() and remove() path. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Yang Yingliang Signed-off-by: Helge Deller --- drivers/video/fbdev/cyber2000fb.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/video/fbdev/cyber2000fb.c b/drivers/video/fbdev/cyber2000fb.c index 585af90a68a5f..31ff1da82c052 100644 --- a/drivers/video/fbdev/cyber2000fb.c +++ b/drivers/video/fbdev/cyber2000fb.c @@ -1796,6 +1796,7 @@ static int cyberpro_pci_probe(struct pci_dev *dev, failed_regions: cyberpro_free_fb_info(cfb); failed_release: + pci_disable_device(dev); return err; } @@ -1812,6 +1813,7 @@ static void cyberpro_pci_remove(struct pci_dev *dev) int_cfb_info = NULL; pci_release_regions(dev); + pci_disable_device(dev); } } -- GitLab From 84ce1ca3fe9e1249bf21176ff162200f1c4e5ed1 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Thu, 27 Oct 2022 21:29:25 +0300 Subject: [PATCH 0583/1927] net: enetc: survive memory pressure without crashing Under memory pressure, enetc_refill_rx_ring() may fail, and when called during the enetc_open() -> enetc_setup_rxbdr() procedure, this is not checked for. An extreme case of memory pressure will result in exactly zero buffers being allocated for the RX ring, and in such a case it is expected that hardware drops all RX packets due to lack of buffers. This does not happen, because the reset-default value of the consumer and produces index is 0, and this makes the ENETC think that all buffers have been initialized and that it owns them (when in reality none were). The hardware guide explains this best: | Configure the receive ring producer index register RBaPIR with a value | of 0. The producer index is initially configured by software but owned | by hardware after the ring has been enabled. Hardware increments the | index when a frame is received which may consume one or more BDs. | Hardware is not allowed to increment the producer index to match the | consumer index since it is used to indicate an empty condition. The ring | can hold at most RBLENR[LENGTH]-1 received BDs. | | Configure the receive ring consumer index register RBaCIR. The | consumer index is owned by software and updated during operation of the | of the BD ring by software, to indicate that any receive data occupied | in the BD has been processed and it has been prepared for new data. | - If consumer index and producer index are initialized to the same | value, it indicates that all BDs in the ring have been prepared and | hardware owns all of the entries. | - If consumer index is initialized to producer index plus N, it would | indicate N BDs have been prepared. Note that hardware cannot start if | only a single buffer is prepared due to the restrictions described in | (2). | - Software may write consumer index to match producer index anytime | while the ring is operational to indicate all received BDs prior have | been processed and new BDs prepared for hardware. Normally, the value of rx_ring->rcir (consumer index) is brought in sync with the rx_ring->next_to_use software index, but this only happens if page allocation ever succeeded. When PI==CI==0, the hardware appears to receive frames and write them to DMA address 0x0 (?!), then set the READY bit in the BD. The enetc_clean_rx_ring() function (and its XDP derivative) is naturally not prepared to handle such a condition. It will attempt to process those frames using the rx_swbd structure associated with index i of the RX ring, but that structure is not fully initialized (enetc_new_page() does all of that). So what happens next is undefined behavior. To operate using no buffer, we must initialize the CI to PI + 1, which will block the hardware from advancing the CI any further, and drop everything. The issue was seen while adding support for zero-copy AF_XDP sockets, where buffer memory comes from user space, which can even decide to supply no buffers at all (example: "xdpsock --txonly"). However, the bug is present also with the network stack code, even though it would take a very determined person to trigger a page allocation failure at the perfect time (a series of ifup/ifdown under memory pressure should eventually reproduce it given enough retries). Fixes: d4fd0404c1c9 ("enetc: Introduce basic PF and VF ENETC ethernet drivers") Signed-off-by: Vladimir Oltean Reviewed-by: Claudiu Manoil Link: https://lore.kernel.org/r/20221027182925.3256653-1-vladimir.oltean@nxp.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/freescale/enetc/enetc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 54bc92fc6bf07..f8c06c3f94646 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -2090,7 +2090,12 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) else enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE); + /* Also prepare the consumer index in case page allocation never + * succeeds. In that case, hardware will never advance producer index + * to match consumer index, and will drop all frames. + */ enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0); + enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1); /* enable Rx ints by setting pkt thr to 1 */ enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); -- GitLab From e37203511c1f2622398eafb7c94a00915bf8e1a1 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 20 Oct 2022 11:46:40 -0400 Subject: [PATCH 0584/1927] drm/amd/display: Update DSC capabilitie for DCN314 dcn314 has 4 DSC - conflicted hardware document updated and confirmed. Tested-by: Mark Broadworth Reviewed-by: Charlene Liu Acked-by: Rodrigo Siqueira Signed-off-by: Leo Chen Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c index 92eb56873eee6..85e2d87a8e44b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c @@ -847,7 +847,7 @@ static const struct resource_caps res_cap_dcn314 = { .num_ddc = 5, .num_vmid = 16, .num_mpc_3dlut = 2, - .num_dsc = 3, + .num_dsc = 4, }; static const struct dc_plane_cap plane_cap = { -- GitLab From d1bc26cb5cd51fd1e7984423b665bf8abfd69256 Mon Sep 17 00:00:00 2001 From: Fangzhi Zuo Date: Thu, 20 Oct 2022 11:46:41 -0400 Subject: [PATCH 0585/1927] drm/amd/display: Ignore Cable ID Feature Ignore cable ID for DP2 receivers that does not support the feature. Tested-by: Mark Broadworth Reviewed-by: Roman Li Acked-by: Rodrigo Siqueira Signed-off-by: Fangzhi Zuo Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index c053cb79cd063..589bee9acf162 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1549,6 +1549,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm; + /* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */ + adev->dm.dc->debug.ignore_cable_id = true; + r = dm_dmub_hw_init(adev); if (r) { DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); -- GitLab From c0459bddd1955df35c9331757671e64e3ea0a77a Mon Sep 17 00:00:00 2001 From: Alan Liu Date: Thu, 20 Oct 2022 11:46:42 -0400 Subject: [PATCH 0586/1927] drm/amd/display: Implement secure display on DCN21 [Why] Porting secure display feature from DCN10 to DCN21. Support single display for now and will extend to multiple displays. [How] - use workqueue to offload works for dmub or dmcu firmware - after receiving ROI update from userspace, set skip_frame_cnt to 1 - refactor amdgpu_dm_crtc_handle_crc_window_irq() - disable PSR before activating secure_display on a crtc - check if secure_display is activated before enabling psr - only work for single display for now. Tested-by: Mark Broadworth Reviewed-by: Wayne Lin Acked-by: Rodrigo Siqueira Signed-off-by: Alan Liu Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 7 +- .../drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c | 151 ++++++++---------- .../drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h | 17 +- .../amd/display/amdgpu_dm/amdgpu_dm_crtc.c | 3 + .../amd/display/amdgpu_dm/amdgpu_dm_debugfs.c | 55 ++++--- .../display/amdgpu_dm/amdgpu_dm_irq_params.h | 2 +- drivers/gpu/drm/amd/display/dc/core/dc.c | 115 +++++++------ drivers/gpu/drm/amd/display/dc/dc_stream.h | 8 +- drivers/gpu/drm/amd/display/dc/dc_types.h | 14 ++ drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h | 14 -- .../gpu/drm/amd/display/dmub/inc/dmub_cmd.h | 26 +++ 11 files changed, 223 insertions(+), 189 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 589bee9acf162..72aede3fcf9b6 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -7840,6 +7840,9 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, */ if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 && acrtc_attach->dm_irq_params.allow_psr_entry && +#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY + !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) && +#endif !acrtc_state->stream->link->psr_settings.psr_allow_active) amdgpu_dm_psr_enable(acrtc_state->stream); } else { @@ -8301,8 +8304,8 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state) #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) if (amdgpu_dm_crc_window_is_activated(crtc)) { spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - acrtc->dm_irq_params.crc_window.update_win = true; - acrtc->dm_irq_params.crc_window.skip_frame_cnt = 2; + acrtc->dm_irq_params.window_param.update_win = true; + acrtc->dm_irq_params.window_param.skip_frame_cnt = 2; spin_lock_irq(&crc_rd_wrk->crc_rd_work_lock); crc_rd_wrk->crtc = crtc; spin_unlock_irq(&crc_rd_wrk->crc_rd_work_lock); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c index 8a441a22c46ec..a857f64547536 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c @@ -89,13 +89,13 @@ static void amdgpu_dm_set_crc_window_default(struct drm_crtc *crtc) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.crc_window.x_start = 0; - acrtc->dm_irq_params.crc_window.y_start = 0; - acrtc->dm_irq_params.crc_window.x_end = 0; - acrtc->dm_irq_params.crc_window.y_end = 0; - acrtc->dm_irq_params.crc_window.activated = false; - acrtc->dm_irq_params.crc_window.update_win = false; - acrtc->dm_irq_params.crc_window.skip_frame_cnt = 0; + acrtc->dm_irq_params.window_param.roi.x_start = 0; + acrtc->dm_irq_params.window_param.roi.y_start = 0; + acrtc->dm_irq_params.window_param.roi.x_end = 0; + acrtc->dm_irq_params.window_param.roi.y_end = 0; + acrtc->dm_irq_params.window_param.activated = false; + acrtc->dm_irq_params.window_param.update_win = false; + acrtc->dm_irq_params.window_param.skip_frame_cnt = 0; spin_unlock_irq(&drm_dev->event_lock); } @@ -135,6 +135,22 @@ static void amdgpu_dm_crtc_notify_ta_to_read(struct work_struct *work) } } +static void +amdgpu_dm_forward_crc_window(struct work_struct *work) +{ + struct crc_fw_work *crc_fw_wrk; + struct amdgpu_display_manager *dm; + + crc_fw_wrk = container_of(work, struct crc_fw_work, forward_roi_work); + dm = crc_fw_wrk->dm; + + mutex_lock(&dm->dc_lock); + dc_stream_forward_crc_window(dm->dc, &crc_fw_wrk->roi, crc_fw_wrk->stream, crc_fw_wrk->is_stop_cmd); + mutex_unlock(&dm->dc_lock); + + kfree(crc_fw_wrk); +} + bool amdgpu_dm_crc_window_is_activated(struct drm_crtc *crtc) { struct drm_device *drm_dev = crtc->dev; @@ -142,7 +158,7 @@ bool amdgpu_dm_crc_window_is_activated(struct drm_crtc *crtc) bool ret = false; spin_lock_irq(&drm_dev->event_lock); - ret = acrtc->dm_irq_params.crc_window.activated; + ret = acrtc->dm_irq_params.window_param.activated; spin_unlock_irq(&drm_dev->event_lock); return ret; @@ -187,9 +203,11 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc, if (adev->dm.crc_rd_wrk) { flush_work(&adev->dm.crc_rd_wrk->notify_ta_work); spin_lock_irq(&adev->dm.crc_rd_wrk->crc_rd_work_lock); + if (adev->dm.crc_rd_wrk->crtc == crtc) { - dc_stream_stop_dmcu_crc_win_update(stream_state->ctx->dc, - dm_crtc_state->stream); + /* stop ROI update on this crtc */ + dc_stream_forward_crc_window(stream_state->ctx->dc, + NULL, stream_state, true); adev->dm.crc_rd_wrk->crtc = NULL; } spin_unlock_irq(&adev->dm.crc_rd_wrk->crc_rd_work_lock); @@ -439,14 +457,9 @@ void amdgpu_dm_crtc_handle_crc_window_irq(struct drm_crtc *crtc) enum amdgpu_dm_pipe_crc_source cur_crc_src; struct amdgpu_crtc *acrtc = NULL; struct amdgpu_device *adev = NULL; - struct crc_rd_work *crc_rd_wrk = NULL; - struct crc_params *crc_window = NULL, tmp_window; + struct crc_rd_work *crc_rd_wrk; + struct crc_fw_work *crc_fw_wrk; unsigned long flags1, flags2; - struct crtc_position position; - uint32_t v_blank; - uint32_t v_back_porch; - uint32_t crc_window_latch_up_line; - struct dc_crtc_timing *timing_out; if (crtc == NULL) return; @@ -458,74 +471,52 @@ void amdgpu_dm_crtc_handle_crc_window_irq(struct drm_crtc *crtc) spin_lock_irqsave(&drm_dev->event_lock, flags1); stream_state = acrtc->dm_irq_params.stream; cur_crc_src = acrtc->dm_irq_params.crc_src; - timing_out = &stream_state->timing; /* Early return if CRC capture is not enabled. */ if (!amdgpu_dm_is_valid_crc_source(cur_crc_src)) goto cleanup; - if (dm_is_crc_source_crtc(cur_crc_src)) { - if (acrtc->dm_irq_params.crc_window.activated) { - if (acrtc->dm_irq_params.crc_window.update_win) { - if (acrtc->dm_irq_params.crc_window.skip_frame_cnt) { - acrtc->dm_irq_params.crc_window.skip_frame_cnt -= 1; - goto cleanup; - } - crc_window = &tmp_window; - - tmp_window.windowa_x_start = - acrtc->dm_irq_params.crc_window.x_start; - tmp_window.windowa_y_start = - acrtc->dm_irq_params.crc_window.y_start; - tmp_window.windowa_x_end = - acrtc->dm_irq_params.crc_window.x_end; - tmp_window.windowa_y_end = - acrtc->dm_irq_params.crc_window.y_end; - tmp_window.windowb_x_start = - acrtc->dm_irq_params.crc_window.x_start; - tmp_window.windowb_y_start = - acrtc->dm_irq_params.crc_window.y_start; - tmp_window.windowb_x_end = - acrtc->dm_irq_params.crc_window.x_end; - tmp_window.windowb_y_end = - acrtc->dm_irq_params.crc_window.y_end; - - dc_stream_forward_dmcu_crc_window(stream_state->ctx->dc, - stream_state, crc_window); - - acrtc->dm_irq_params.crc_window.update_win = false; - - dc_stream_get_crtc_position(stream_state->ctx->dc, &stream_state, 1, - &position.vertical_count, - &position.nominal_vcount); - - v_blank = timing_out->v_total - timing_out->v_border_top - - timing_out->v_addressable - timing_out->v_border_bottom; - - v_back_porch = v_blank - timing_out->v_front_porch - - timing_out->v_sync_width; - - crc_window_latch_up_line = v_back_porch + timing_out->v_sync_width; - - /* take 3 lines margin*/ - if ((position.vertical_count + 3) >= crc_window_latch_up_line) - acrtc->dm_irq_params.crc_window.skip_frame_cnt = 1; - else - acrtc->dm_irq_params.crc_window.skip_frame_cnt = 0; - } else { - if (acrtc->dm_irq_params.crc_window.skip_frame_cnt == 0) { - if (adev->dm.crc_rd_wrk) { - crc_rd_wrk = adev->dm.crc_rd_wrk; - spin_lock_irqsave(&crc_rd_wrk->crc_rd_work_lock, flags2); - crc_rd_wrk->phy_inst = - stream_state->link->link_enc_hw_inst; - spin_unlock_irqrestore(&crc_rd_wrk->crc_rd_work_lock, flags2); - schedule_work(&crc_rd_wrk->notify_ta_work); - } - } else { - acrtc->dm_irq_params.crc_window.skip_frame_cnt -= 1; - } - } + if (!dm_is_crc_source_crtc(cur_crc_src)) + goto cleanup; + + if (!acrtc->dm_irq_params.window_param.activated) + goto cleanup; + + if (acrtc->dm_irq_params.window_param.update_win) { + if (acrtc->dm_irq_params.window_param.skip_frame_cnt) { + acrtc->dm_irq_params.window_param.skip_frame_cnt -= 1; + goto cleanup; + } + + /* prepare work for dmub to update ROI */ + crc_fw_wrk = kzalloc(sizeof(*crc_fw_wrk), GFP_ATOMIC); + if (!crc_fw_wrk) + goto cleanup; + + INIT_WORK(&crc_fw_wrk->forward_roi_work, amdgpu_dm_forward_crc_window); + crc_fw_wrk->dm = &adev->dm; + crc_fw_wrk->stream = stream_state; + crc_fw_wrk->roi.x_start = acrtc->dm_irq_params.window_param.roi.x_start; + crc_fw_wrk->roi.y_start = acrtc->dm_irq_params.window_param.roi.y_start; + crc_fw_wrk->roi.x_end = acrtc->dm_irq_params.window_param.roi.x_end; + crc_fw_wrk->roi.y_end = acrtc->dm_irq_params.window_param.roi.y_end; + schedule_work(&crc_fw_wrk->forward_roi_work); + + acrtc->dm_irq_params.window_param.update_win = false; + acrtc->dm_irq_params.window_param.skip_frame_cnt = 1; + + } else { + if (acrtc->dm_irq_params.window_param.skip_frame_cnt) { + acrtc->dm_irq_params.window_param.skip_frame_cnt -= 1; + goto cleanup; + } + + if (adev->dm.crc_rd_wrk) { + crc_rd_wrk = adev->dm.crc_rd_wrk; + spin_lock_irqsave(&crc_rd_wrk->crc_rd_work_lock, flags2); + crc_rd_wrk->phy_inst = stream_state->link->link_enc_hw_inst; + spin_unlock_irqrestore(&crc_rd_wrk->crc_rd_work_lock, flags2); + schedule_work(&crc_rd_wrk->notify_ta_work); } } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h index f07850db60a6a..16bfc2dc35ab3 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h @@ -40,11 +40,8 @@ enum amdgpu_dm_pipe_crc_source { }; #ifdef CONFIG_DRM_AMD_SECURE_DISPLAY -struct crc_window_parm { - uint16_t x_start; - uint16_t y_start; - uint16_t x_end; - uint16_t y_end; +struct crc_window_param { + struct crc_region roi; /* CRC windwo is activated or not*/ bool activated; /* Update crc window during vertical blank or not */ @@ -53,6 +50,7 @@ struct crc_window_parm { int skip_frame_cnt; }; +/* read_work for driver to call PSP to read */ struct crc_rd_work { struct work_struct notify_ta_work; /* To protect crc_rd_work carried fields*/ @@ -60,6 +58,15 @@ struct crc_rd_work { struct drm_crtc *crtc; uint8_t phy_inst; }; + +/* forward_work for driver to forward ROI to dmu */ +struct crc_fw_work { + struct work_struct forward_roi_work; + struct amdgpu_display_manager *dm; + struct dc_stream_state *stream; + struct crc_region roi; + bool is_stop_cmd; +}; #endif static inline bool amdgpu_dm_is_valid_crc_source(enum amdgpu_dm_pipe_crc_source source) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 594fe8a4d02b0..3675b39e297a4 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -127,6 +127,9 @@ static void vblank_control_worker(struct work_struct *work) amdgpu_dm_psr_disable(vblank_work->stream); } else if (vblank_work->stream->link->psr_settings.psr_feature_enabled && !vblank_work->stream->link->psr_settings.psr_allow_active && +#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY + !amdgpu_dm_crc_window_is_activated(&vblank_work->acrtc->base) && +#endif vblank_work->acrtc->dm_irq_params.allow_psr_entry) { amdgpu_dm_psr_enable(vblank_work->stream); } diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 0c717d7b12e56..8f5b98df26dbf 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -38,6 +38,10 @@ #include "link_hwss.h" #include "dc/dc_dmub_srv.h" +#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY +#include "amdgpu_dm_psr.h" +#endif + struct dmub_debugfs_trace_header { uint32_t entry_count; uint32_t reserved[3]; @@ -3081,8 +3085,8 @@ static int crc_win_x_start_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.crc_window.x_start = (uint16_t) val; - acrtc->dm_irq_params.crc_window.update_win = false; + acrtc->dm_irq_params.window_param.roi.x_start = (uint16_t) val; + acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3098,7 +3102,7 @@ static int crc_win_x_start_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.crc_window.x_start; + *val = acrtc->dm_irq_params.window_param.roi.x_start; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3118,8 +3122,8 @@ static int crc_win_y_start_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.crc_window.y_start = (uint16_t) val; - acrtc->dm_irq_params.crc_window.update_win = false; + acrtc->dm_irq_params.window_param.roi.y_start = (uint16_t) val; + acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3135,7 +3139,7 @@ static int crc_win_y_start_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.crc_window.y_start; + *val = acrtc->dm_irq_params.window_param.roi.y_start; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3154,8 +3158,8 @@ static int crc_win_x_end_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.crc_window.x_end = (uint16_t) val; - acrtc->dm_irq_params.crc_window.update_win = false; + acrtc->dm_irq_params.window_param.roi.x_end = (uint16_t) val; + acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3171,7 +3175,7 @@ static int crc_win_x_end_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.crc_window.x_end; + *val = acrtc->dm_irq_params.window_param.roi.x_end; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3190,8 +3194,8 @@ static int crc_win_y_end_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.crc_window.y_end = (uint16_t) val; - acrtc->dm_irq_params.crc_window.update_win = false; + acrtc->dm_irq_params.window_param.roi.y_end = (uint16_t) val; + acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3207,7 +3211,7 @@ static int crc_win_y_end_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.crc_window.y_end; + *val = acrtc->dm_irq_params.window_param.roi.y_end; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3230,31 +3234,38 @@ static int crc_win_update_set(void *data, u64 val) return 0; if (val) { + new_acrtc = to_amdgpu_crtc(new_crtc); + mutex_lock(&adev->dm.dc_lock); + /* PSR may write to OTG CRC window control register, + * so close it before starting secure_display. + */ + amdgpu_dm_psr_disable(new_acrtc->dm_irq_params.stream); + spin_lock_irq(&adev_to_drm(adev)->event_lock); spin_lock_irq(&crc_rd_wrk->crc_rd_work_lock); if (crc_rd_wrk->crtc) { old_crtc = crc_rd_wrk->crtc; old_acrtc = to_amdgpu_crtc(old_crtc); } - new_acrtc = to_amdgpu_crtc(new_crtc); if (old_crtc && old_crtc != new_crtc) { - old_acrtc->dm_irq_params.crc_window.activated = false; - old_acrtc->dm_irq_params.crc_window.update_win = false; - old_acrtc->dm_irq_params.crc_window.skip_frame_cnt = 0; + old_acrtc->dm_irq_params.window_param.activated = false; + old_acrtc->dm_irq_params.window_param.update_win = false; + old_acrtc->dm_irq_params.window_param.skip_frame_cnt = 0; - new_acrtc->dm_irq_params.crc_window.activated = true; - new_acrtc->dm_irq_params.crc_window.update_win = true; - new_acrtc->dm_irq_params.crc_window.skip_frame_cnt = 0; + new_acrtc->dm_irq_params.window_param.activated = true; + new_acrtc->dm_irq_params.window_param.update_win = true; + new_acrtc->dm_irq_params.window_param.skip_frame_cnt = 0; crc_rd_wrk->crtc = new_crtc; } else { - new_acrtc->dm_irq_params.crc_window.activated = true; - new_acrtc->dm_irq_params.crc_window.update_win = true; - new_acrtc->dm_irq_params.crc_window.skip_frame_cnt = 0; + new_acrtc->dm_irq_params.window_param.activated = true; + new_acrtc->dm_irq_params.window_param.update_win = true; + new_acrtc->dm_irq_params.window_param.skip_frame_cnt = 0; crc_rd_wrk->crtc = new_crtc; } spin_unlock_irq(&crc_rd_wrk->crc_rd_work_lock); spin_unlock_irq(&adev_to_drm(adev)->event_lock); + mutex_unlock(&adev->dm.dc_lock); } return 0; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h index 79b5f9999fec5..5c9303241aeb9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq_params.h @@ -39,7 +39,7 @@ struct dm_irq_params { #ifdef CONFIG_DEBUG_FS enum amdgpu_dm_pipe_crc_source crc_src; #ifdef CONFIG_DRM_AMD_SECURE_DISPLAY - struct crc_window_parm crc_window; + struct crc_window_param window_param; #endif #endif }; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 56629197a6a15..58352de24061b 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -491,86 +491,79 @@ bool dc_stream_get_crtc_position(struct dc *dc, } #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) -bool dc_stream_forward_dmcu_crc_window(struct dc *dc, struct dc_stream_state *stream, - struct crc_params *crc_window) +static inline void +dc_stream_forward_dmub_crc_window(struct dc_dmub_srv *dmub_srv, + struct crc_region *roi, struct otg_phy_mux *mux_mapping, bool is_stop) { - int i; - struct dmcu *dmcu = dc->res_pool->dmcu; - struct pipe_ctx *pipe; - struct crc_region tmp_win, *crc_win; - struct otg_phy_mux mapping_tmp, *mux_mapping; - - /*crc window can't be null*/ - if (!crc_window) - return false; - - if ((dmcu != NULL && dmcu->funcs->is_dmcu_initialized(dmcu))) { - crc_win = &tmp_win; - mux_mapping = &mapping_tmp; - /*set crc window*/ - tmp_win.x_start = crc_window->windowa_x_start; - tmp_win.y_start = crc_window->windowa_y_start; - tmp_win.x_end = crc_window->windowa_x_end; - tmp_win.y_end = crc_window->windowa_y_end; - - for (i = 0; i < MAX_PIPES; i++) { - pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - if (pipe->stream == stream && !pipe->top_pipe && !pipe->prev_odm_pipe) - break; - } - - /* Stream not found */ - if (i == MAX_PIPES) - return false; - + union dmub_rb_cmd cmd = {0}; - /*set mux routing info*/ - mapping_tmp.phy_output_num = stream->link->link_enc_hw_inst; - mapping_tmp.otg_output_num = pipe->stream_res.tg->inst; + cmd.secure_display.roi_info.phy_id = mux_mapping->phy_output_num; + cmd.secure_display.roi_info.otg_id = mux_mapping->otg_output_num; - dmcu->funcs->forward_crc_window(dmcu, crc_win, mux_mapping); + if (is_stop) { + cmd.secure_display.header.type = DMUB_CMD__SECURE_DISPLAY; + cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_CRC_STOP_UPDATE; } else { - DC_LOG_DC("dmcu is not initialized"); - return false; + cmd.secure_display.header.type = DMUB_CMD__SECURE_DISPLAY; + cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_CRC_WIN_NOTIFY; + cmd.secure_display.roi_info.x_start = roi->x_start; + cmd.secure_display.roi_info.y_start = roi->y_start; + cmd.secure_display.roi_info.x_end = roi->x_end; + cmd.secure_display.roi_info.y_end = roi->y_end; } - return true; + dc_dmub_srv_cmd_queue(dmub_srv, &cmd); + dc_dmub_srv_cmd_execute(dmub_srv); } -bool dc_stream_stop_dmcu_crc_win_update(struct dc *dc, struct dc_stream_state *stream) +static inline void +dc_stream_forward_dmcu_crc_window(struct dmcu *dmcu, + struct crc_region *roi, struct otg_phy_mux *mux_mapping, bool is_stop) { - int i; - struct dmcu *dmcu = dc->res_pool->dmcu; - struct pipe_ctx *pipe; - struct otg_phy_mux mapping_tmp, *mux_mapping; + if (is_stop) + dmcu->funcs->stop_crc_win_update(dmcu, mux_mapping); + else + dmcu->funcs->forward_crc_window(dmcu, roi, mux_mapping); +} - if ((dmcu != NULL && dmcu->funcs->is_dmcu_initialized(dmcu))) { - mux_mapping = &mapping_tmp; +bool +dc_stream_forward_crc_window(struct dc *dc, + struct crc_region *roi, struct dc_stream_state *stream, bool is_stop) +{ + struct dmcu *dmcu; + struct dc_dmub_srv *dmub_srv; + struct otg_phy_mux mux_mapping; + struct pipe_ctx *pipe; + int i; - for (i = 0; i < MAX_PIPES; i++) { - pipe = &dc->current_state->res_ctx.pipe_ctx[i]; - if (pipe->stream == stream && !pipe->top_pipe && !pipe->prev_odm_pipe) - break; - } + for (i = 0; i < MAX_PIPES; i++) { + pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + if (pipe->stream == stream && !pipe->top_pipe && !pipe->prev_odm_pipe) + break; + } - /* Stream not found */ - if (i == MAX_PIPES) - return false; + /* Stream not found */ + if (i == MAX_PIPES) + return false; + mux_mapping.phy_output_num = stream->link->link_enc_hw_inst; + mux_mapping.otg_output_num = pipe->stream_res.tg->inst; - /*set mux routing info*/ - mapping_tmp.phy_output_num = stream->link->link_enc_hw_inst; - mapping_tmp.otg_output_num = pipe->stream_res.tg->inst; + dmcu = dc->res_pool->dmcu; + dmub_srv = dc->ctx->dmub_srv; - dmcu->funcs->stop_crc_win_update(dmcu, mux_mapping); - } else { - DC_LOG_DC("dmcu is not initialized"); + /* forward to dmub */ + if (dmub_srv) + dc_stream_forward_dmub_crc_window(dmub_srv, roi, &mux_mapping, is_stop); + /* forward to dmcu */ + else if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) + dc_stream_forward_dmcu_crc_window(dmcu, roi, &mux_mapping, is_stop); + else return false; - } return true; } -#endif +#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */ /** * dc_stream_configure_crc() - Configure CRC capture for the given stream. diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index 73dccd485895d..26f8f18e675c8 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -521,10 +521,10 @@ bool dc_stream_get_crtc_position(struct dc *dc, unsigned int *nom_v_pos); #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) -bool dc_stream_forward_dmcu_crc_window(struct dc *dc, struct dc_stream_state *stream, - struct crc_params *crc_window); -bool dc_stream_stop_dmcu_crc_win_update(struct dc *dc, - struct dc_stream_state *stream); +bool dc_stream_forward_crc_window(struct dc *dc, + struct crc_region *roi, + struct dc_stream_state *stream, + bool is_stop); #endif bool dc_stream_configure_crc(struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index ad9041472ccae..27dd4304ce69f 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -993,4 +993,18 @@ struct display_endpoint_id { enum display_endpoint_type ep_type; }; +#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) +struct crc_region { + uint16_t x_start; + uint16_t y_start; + uint16_t x_end; + uint16_t y_end; +}; + +struct otg_phy_mux { + uint8_t phy_output_num; + uint8_t otg_output_num; +}; +#endif + #endif /* DC_TYPES_H_ */ diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h index 8df2765cce78b..9a2432608ddf4 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h @@ -56,20 +56,6 @@ struct dmcu { bool auto_load_dmcu; }; -#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) -struct crc_region { - uint16_t x_start; - uint16_t y_start; - uint16_t x_end; - uint16_t y_end; -}; - -struct otg_phy_mux { - uint8_t phy_output_num; - uint8_t otg_output_num; -}; -#endif - struct dmcu_funcs { bool (*dmcu_init)(struct dmcu *dmcu); bool (*load_iram)(struct dmcu *dmcu, diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index 7a8f61517424c..dc7c4d74cb5d9 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -729,6 +729,10 @@ enum dmub_cmd_type { /** * Command type used for all VBIOS interface commands. */ + /** + * Command type used for all SECURE_DISPLAY commands. + */ + DMUB_CMD__SECURE_DISPLAY = 85, /** * Command type used to set DPIA HPD interrupt state @@ -3143,6 +3147,24 @@ struct dmub_rb_cmd_get_usbc_cable_id { } data; }; +enum dmub_cmd_secure_display_type { + DMUB_CMD__SECURE_DISPLAY_TEST_CMD = 0, + DMUB_CMD__SECURE_DISPLAY_CRC_STOP_UPDATE, + DMUB_CMD__SECURE_DISPLAY_CRC_WIN_NOTIFY +}; + +struct dmub_rb_cmd_secure_display { + struct dmub_cmd_header header; + struct dmub_cmd_roi_info { + uint16_t x_start; + uint16_t x_end; + uint16_t y_start; + uint16_t y_end; + uint8_t otg_id; + uint8_t phy_id; + } roi_info; +}; + /** * union dmub_rb_cmd - DMUB inbox command. */ @@ -3347,6 +3369,10 @@ union dmub_rb_cmd { * Definition of a DMUB_CMD__QUERY_HPD_STATE command. */ struct dmub_rb_cmd_query_hpd_state query_hpd; + /** + * Definition of a DMUB_CMD__SECURE_DISPLAY command. + */ + struct dmub_rb_cmd_secure_display secure_display; /** * Definition of a DMUB_CMD__DPIA_HPD_INT_ENABLE command. */ -- GitLab From 62fa035b5605a993fa3b4526af8b6ed145e3ce1b Mon Sep 17 00:00:00 2001 From: Alan Liu Date: Thu, 20 Oct 2022 11:46:43 -0400 Subject: [PATCH 0587/1927] drm/amd/display: Drop struct crc_region and reuse struct rect [Why] reuse struct rect rather than adding a new struct. [How] - Userspace keeps inputting x_start, y_start, x_end, y_end - We translate data to x, y, width, height in code flow to store - translate back to x_start, y_start, x_end, y_end before programming HW Tested-by: Mark Broadworth Reviewed-by: Wayne Lin Acked-by: Rodrigo Siqueira Signed-off-by: Alan Liu Signed-off-by: Alex Deucher --- .../drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c | 20 ++++++++++--------- .../drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h | 7 +++++-- .../amd/display/amdgpu_dm/amdgpu_dm_debugfs.c | 16 +++++++-------- drivers/gpu/drm/amd/display/dc/core/dc.c | 20 +++++++++---------- drivers/gpu/drm/amd/display/dc/dc_stream.h | 2 +- drivers/gpu/drm/amd/display/dc/dc_types.h | 7 ------- drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c | 14 +++++++++---- drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h | 2 +- 8 files changed, 46 insertions(+), 42 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c index a857f64547536..fb077e9af99fc 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c @@ -89,10 +89,10 @@ static void amdgpu_dm_set_crc_window_default(struct drm_crtc *crtc) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.window_param.roi.x_start = 0; - acrtc->dm_irq_params.window_param.roi.y_start = 0; - acrtc->dm_irq_params.window_param.roi.x_end = 0; - acrtc->dm_irq_params.window_param.roi.y_end = 0; + acrtc->dm_irq_params.window_param.x_start = 0; + acrtc->dm_irq_params.window_param.y_start = 0; + acrtc->dm_irq_params.window_param.x_end = 0; + acrtc->dm_irq_params.window_param.y_end = 0; acrtc->dm_irq_params.window_param.activated = false; acrtc->dm_irq_params.window_param.update_win = false; acrtc->dm_irq_params.window_param.skip_frame_cnt = 0; @@ -145,7 +145,7 @@ amdgpu_dm_forward_crc_window(struct work_struct *work) dm = crc_fw_wrk->dm; mutex_lock(&dm->dc_lock); - dc_stream_forward_crc_window(dm->dc, &crc_fw_wrk->roi, crc_fw_wrk->stream, crc_fw_wrk->is_stop_cmd); + dc_stream_forward_crc_window(dm->dc, &crc_fw_wrk->rect, crc_fw_wrk->stream, crc_fw_wrk->is_stop_cmd); mutex_unlock(&dm->dc_lock); kfree(crc_fw_wrk); @@ -496,10 +496,12 @@ void amdgpu_dm_crtc_handle_crc_window_irq(struct drm_crtc *crtc) INIT_WORK(&crc_fw_wrk->forward_roi_work, amdgpu_dm_forward_crc_window); crc_fw_wrk->dm = &adev->dm; crc_fw_wrk->stream = stream_state; - crc_fw_wrk->roi.x_start = acrtc->dm_irq_params.window_param.roi.x_start; - crc_fw_wrk->roi.y_start = acrtc->dm_irq_params.window_param.roi.y_start; - crc_fw_wrk->roi.x_end = acrtc->dm_irq_params.window_param.roi.x_end; - crc_fw_wrk->roi.y_end = acrtc->dm_irq_params.window_param.roi.y_end; + crc_fw_wrk->rect.x = acrtc->dm_irq_params.window_param.x_start; + crc_fw_wrk->rect.y = acrtc->dm_irq_params.window_param.y_start; + crc_fw_wrk->rect.width = acrtc->dm_irq_params.window_param.x_end - + acrtc->dm_irq_params.window_param.x_start; + crc_fw_wrk->rect.height = acrtc->dm_irq_params.window_param.y_end - + acrtc->dm_irq_params.window_param.y_start; schedule_work(&crc_fw_wrk->forward_roi_work); acrtc->dm_irq_params.window_param.update_win = false; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h index 16bfc2dc35ab3..71bce608d751d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.h @@ -41,7 +41,10 @@ enum amdgpu_dm_pipe_crc_source { #ifdef CONFIG_DRM_AMD_SECURE_DISPLAY struct crc_window_param { - struct crc_region roi; + uint16_t x_start; + uint16_t y_start; + uint16_t x_end; + uint16_t y_end; /* CRC windwo is activated or not*/ bool activated; /* Update crc window during vertical blank or not */ @@ -64,7 +67,7 @@ struct crc_fw_work { struct work_struct forward_roi_work; struct amdgpu_display_manager *dm; struct dc_stream_state *stream; - struct crc_region roi; + struct rect rect; bool is_stop_cmd; }; #endif diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 8f5b98df26dbf..2c43cdd2e7073 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -3085,7 +3085,7 @@ static int crc_win_x_start_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.window_param.roi.x_start = (uint16_t) val; + acrtc->dm_irq_params.window_param.x_start = (uint16_t) val; acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); @@ -3102,7 +3102,7 @@ static int crc_win_x_start_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.window_param.roi.x_start; + *val = acrtc->dm_irq_params.window_param.x_start; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3122,7 +3122,7 @@ static int crc_win_y_start_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.window_param.roi.y_start = (uint16_t) val; + acrtc->dm_irq_params.window_param.y_start = (uint16_t) val; acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); @@ -3139,7 +3139,7 @@ static int crc_win_y_start_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.window_param.roi.y_start; + *val = acrtc->dm_irq_params.window_param.y_start; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3158,7 +3158,7 @@ static int crc_win_x_end_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.window_param.roi.x_end = (uint16_t) val; + acrtc->dm_irq_params.window_param.x_end = (uint16_t) val; acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); @@ -3175,7 +3175,7 @@ static int crc_win_x_end_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.window_param.roi.x_end; + *val = acrtc->dm_irq_params.window_param.x_end; spin_unlock_irq(&drm_dev->event_lock); return 0; @@ -3194,7 +3194,7 @@ static int crc_win_y_end_set(void *data, u64 val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - acrtc->dm_irq_params.window_param.roi.y_end = (uint16_t) val; + acrtc->dm_irq_params.window_param.y_end = (uint16_t) val; acrtc->dm_irq_params.window_param.update_win = false; spin_unlock_irq(&drm_dev->event_lock); @@ -3211,7 +3211,7 @@ static int crc_win_y_end_get(void *data, u64 *val) struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc); spin_lock_irq(&drm_dev->event_lock); - *val = acrtc->dm_irq_params.window_param.roi.y_end; + *val = acrtc->dm_irq_params.window_param.y_end; spin_unlock_irq(&drm_dev->event_lock); return 0; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 58352de24061b..20c16ca935f73 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -493,7 +493,7 @@ bool dc_stream_get_crtc_position(struct dc *dc, #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) static inline void dc_stream_forward_dmub_crc_window(struct dc_dmub_srv *dmub_srv, - struct crc_region *roi, struct otg_phy_mux *mux_mapping, bool is_stop) + struct rect *rect, struct otg_phy_mux *mux_mapping, bool is_stop) { union dmub_rb_cmd cmd = {0}; @@ -506,10 +506,10 @@ dc_stream_forward_dmub_crc_window(struct dc_dmub_srv *dmub_srv, } else { cmd.secure_display.header.type = DMUB_CMD__SECURE_DISPLAY; cmd.secure_display.header.sub_type = DMUB_CMD__SECURE_DISPLAY_CRC_WIN_NOTIFY; - cmd.secure_display.roi_info.x_start = roi->x_start; - cmd.secure_display.roi_info.y_start = roi->y_start; - cmd.secure_display.roi_info.x_end = roi->x_end; - cmd.secure_display.roi_info.y_end = roi->y_end; + cmd.secure_display.roi_info.x_start = rect->x; + cmd.secure_display.roi_info.y_start = rect->y; + cmd.secure_display.roi_info.x_end = rect->x + rect->width; + cmd.secure_display.roi_info.y_end = rect->y + rect->height; } dc_dmub_srv_cmd_queue(dmub_srv, &cmd); @@ -518,17 +518,17 @@ dc_stream_forward_dmub_crc_window(struct dc_dmub_srv *dmub_srv, static inline void dc_stream_forward_dmcu_crc_window(struct dmcu *dmcu, - struct crc_region *roi, struct otg_phy_mux *mux_mapping, bool is_stop) + struct rect *rect, struct otg_phy_mux *mux_mapping, bool is_stop) { if (is_stop) dmcu->funcs->stop_crc_win_update(dmcu, mux_mapping); else - dmcu->funcs->forward_crc_window(dmcu, roi, mux_mapping); + dmcu->funcs->forward_crc_window(dmcu, rect, mux_mapping); } bool dc_stream_forward_crc_window(struct dc *dc, - struct crc_region *roi, struct dc_stream_state *stream, bool is_stop) + struct rect *rect, struct dc_stream_state *stream, bool is_stop) { struct dmcu *dmcu; struct dc_dmub_srv *dmub_srv; @@ -554,10 +554,10 @@ dc_stream_forward_crc_window(struct dc *dc, /* forward to dmub */ if (dmub_srv) - dc_stream_forward_dmub_crc_window(dmub_srv, roi, &mux_mapping, is_stop); + dc_stream_forward_dmub_crc_window(dmub_srv, rect, &mux_mapping, is_stop); /* forward to dmcu */ else if (dmcu && dmcu->funcs->is_dmcu_initialized(dmcu)) - dc_stream_forward_dmcu_crc_window(dmcu, roi, &mux_mapping, is_stop); + dc_stream_forward_dmcu_crc_window(dmcu, rect, &mux_mapping, is_stop); else return false; diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index 26f8f18e675c8..4d30ae0ca721a 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -522,7 +522,7 @@ bool dc_stream_get_crtc_position(struct dc *dc, #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) bool dc_stream_forward_crc_window(struct dc *dc, - struct crc_region *roi, + struct rect *rect, struct dc_stream_state *stream, bool is_stop); #endif diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index 27dd4304ce69f..dc78e2404b487 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -994,13 +994,6 @@ struct display_endpoint_id { }; #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) -struct crc_region { - uint16_t x_start; - uint16_t y_start; - uint16_t x_end; - uint16_t y_end; -}; - struct otg_phy_mux { uint8_t phy_output_num; uint8_t otg_output_num; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c index fbb19e253f505..d3cc5ec46956d 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_dmcu.c @@ -927,19 +927,20 @@ static bool dcn10_recv_edid_cea_ack(struct dmcu *dmcu, int *offset) #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) static void dcn10_forward_crc_window(struct dmcu *dmcu, - struct crc_region *crc_win, + struct rect *rect, struct otg_phy_mux *mux_mapping) { struct dce_dmcu *dmcu_dce = TO_DCE_DMCU(dmcu); unsigned int dmcu_max_retry_on_wait_reg_ready = 801; unsigned int dmcu_wait_reg_ready_interval = 100; unsigned int crc_start = 0, crc_end = 0, otg_phy_mux = 0; + int x_start, y_start, x_end, y_end; /* If microcontroller is not running, do nothing */ if (dmcu->dmcu_state != DMCU_RUNNING) return; - if (!crc_win) + if (!rect) return; /* waitDMCUReadyForCmd */ @@ -947,9 +948,14 @@ static void dcn10_forward_crc_window(struct dmcu *dmcu, dmcu_wait_reg_ready_interval, dmcu_max_retry_on_wait_reg_ready); + x_start = rect->x; + y_start = rect->y; + x_end = x_start + rect->width; + y_end = y_start + rect->height; + /* build up nitification data */ - crc_start = (((unsigned int) crc_win->x_start) << 16) | crc_win->y_start; - crc_end = (((unsigned int) crc_win->x_end) << 16) | crc_win->y_end; + crc_start = (((unsigned int) x_start) << 16) | y_start; + crc_end = (((unsigned int) x_end) << 16) | y_end; otg_phy_mux = (((unsigned int) mux_mapping->otg_output_num) << 16) | mux_mapping->phy_output_num; diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h index 9a2432608ddf4..de3113ecbc775 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dmcu.h @@ -86,7 +86,7 @@ struct dmcu_funcs { bool (*recv_edid_cea_ack)(struct dmcu *dmcu, int *offset); #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) void (*forward_crc_window)(struct dmcu *dmcu, - struct crc_region *crc_win, + struct rect *rect, struct otg_phy_mux *mux_mapping); void (*stop_crc_win_update)(struct dmcu *dmcu, struct otg_phy_mux *mux_mapping); -- GitLab From f7f697403f2067ddbeff3344a4c2d20ff44ace79 Mon Sep 17 00:00:00 2001 From: Jun Lei Date: Thu, 20 Oct 2022 11:46:44 -0400 Subject: [PATCH 0588/1927] drm/amd/display: Limit dcn32 to 1950Mhz display clock [why] Hardware team recommends we limit dispclock to 1950Mhz for all DCN3.2.x [how] Limit to 1950 when initializing clocks. Tested-by: Mark Broadworth Reviewed-by: Alvin Lee Acked-by: Rodrigo Siqueira Signed-off-by: Jun Lei Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c index 1c612ccf1944a..fd0313468fdbc 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c @@ -157,6 +157,7 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base) struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); unsigned int num_levels; struct clk_limit_num_entries *num_entries_per_clk = &clk_mgr_base->bw_params->clk_table.num_entries_per_clk; + unsigned int i; memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks)); clk_mgr_base->clks.p_state_change_support = true; @@ -205,18 +206,17 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base) clk_mgr->dpm_present = true; if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) { - unsigned int i; - for (i = 0; i < num_levels; i++) if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz < khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz)) clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz = khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz); } + for (i = 0; i < num_levels; i++) + if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz > 1950) + clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz = 1950; if (clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz) { - unsigned int i; - for (i = 0; i < num_levels; i++) if (clk_mgr_base->bw_params->clk_table.entries[i].dppclk_mhz < khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz)) -- GitLab From 1fb695d9e67d86c28f888b22d246c4333afa8a71 Mon Sep 17 00:00:00 2001 From: Anthony Koo Date: Thu, 20 Oct 2022 11:46:45 -0400 Subject: [PATCH 0589/1927] drm/amd/display: Document part of the DMUB cmd Add a simple documentation in the dmub_cmd. Tested-by: Mark Broadworth Reviewed-by: Aric Cyr Acked-by: Rodrigo Siqueira Signed-off-by: Anthony Koo Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index dc7c4d74cb5d9..bc9344af19957 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -729,6 +729,7 @@ enum dmub_cmd_type { /** * Command type used for all VBIOS interface commands. */ + /** * Command type used for all SECURE_DISPLAY commands. */ @@ -3147,14 +3148,23 @@ struct dmub_rb_cmd_get_usbc_cable_id { } data; }; +/** + * Command type of a DMUB_CMD__SECURE_DISPLAY command + */ enum dmub_cmd_secure_display_type { - DMUB_CMD__SECURE_DISPLAY_TEST_CMD = 0, + DMUB_CMD__SECURE_DISPLAY_TEST_CMD = 0, /* test command to only check if inbox message works */ DMUB_CMD__SECURE_DISPLAY_CRC_STOP_UPDATE, DMUB_CMD__SECURE_DISPLAY_CRC_WIN_NOTIFY }; +/** + * Definition of a DMUB_CMD__SECURE_DISPLAY command + */ struct dmub_rb_cmd_secure_display { struct dmub_cmd_header header; + /** + * Data passed from driver to dmub firmware. + */ struct dmub_cmd_roi_info { uint16_t x_start; uint16_t x_end; @@ -3373,6 +3383,7 @@ union dmub_rb_cmd { * Definition of a DMUB_CMD__SECURE_DISPLAY command. */ struct dmub_rb_cmd_secure_display secure_display; + /** * Definition of a DMUB_CMD__DPIA_HPD_INT_ENABLE command. */ -- GitLab From 5384d0e2b0e6afc717a523420a4b30e96dc9a99b Mon Sep 17 00:00:00 2001 From: Aric Cyr Date: Thu, 20 Oct 2022 11:46:46 -0400 Subject: [PATCH 0590/1927] drm/amd/display: 3.2.208 DC version 3.2.208 brings along the following: * Add more kernel doc * Enable secure display on DCN21 * Limit dcn32 to 1950Mhz display clock * PSR code refactor * Rework audio stream sequence * Generic bug fixes Tested-by: Mark Broadworth Acked-by: Rodrigo Siqueira Signed-off-by: Aric Cyr Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index e4e41f2e50540..3450aca538b58 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -47,7 +47,7 @@ struct aux_payload; struct set_config_cmd_payload; struct dmub_notification; -#define DC_VER "3.2.207" +#define DC_VER "3.2.208" #define MAX_SURFACES 3 #define MAX_PLANES 6 -- GitLab From 4fd8575dc59c2e725c38f55f8167be9df1040ef9 Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Thu, 20 Oct 2022 11:46:47 -0400 Subject: [PATCH 0591/1927] drm/amd/display: Update latencies on DCN321 Update DF related latencies based on new measurements. Tested-by: Mark Broadworth Reviewed-by: Jun Lei Acked-by: Rodrigo Siqueira Signed-off-by: Dillon Varone Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index dd90f241e9065..7352f75144101 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -119,15 +119,15 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_21_soc = { }, }, .num_states = 1, - .sr_exit_time_us = 12.36, - .sr_enter_plus_exit_time_us = 16.72, + .sr_exit_time_us = 19.95, + .sr_enter_plus_exit_time_us = 24.36, .sr_exit_z8_time_us = 285.0, .sr_enter_plus_exit_z8_time_us = 320, .writeback_latency_us = 12.0, .round_trip_ping_latency_dcfclk_cycles = 263, - .urgent_latency_pixel_data_only_us = 4.0, - .urgent_latency_pixel_mixed_with_vm_data_us = 4.0, - .urgent_latency_vm_data_only_us = 4.0, + .urgent_latency_pixel_data_only_us = 9.35, + .urgent_latency_pixel_mixed_with_vm_data_us = 9.35, + .urgent_latency_vm_data_only_us = 9.35, .fclk_change_latency_us = 20, .usr_retraining_latency_us = 2, .smn_latency_us = 2, -- GitLab From 33151fb787692edf8275120435c8cc41bfe0cf63 Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Thu, 20 Oct 2022 11:46:48 -0400 Subject: [PATCH 0592/1927] drm/amd/display: Set memclk levels to be at least 1 for dcn32 [Why] Cannot report 0 memclk levels even when SMU does not provide any. [How] When memclk levels reported by SMU is 0, set levels to 1. Tested-by: Mark Broadworth Reviewed-by: Martin Leung Acked-by: Rodrigo Siqueira Signed-off-by: Dillon Varone Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c index fd0313468fdbc..6f77d8e538ab1 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c @@ -669,6 +669,9 @@ static void dcn32_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base) &clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz, &num_entries_per_clk->num_memclk_levels); + /* memclk must have at least one level */ + num_entries_per_clk->num_memclk_levels = num_entries_per_clk->num_memclk_levels ? num_entries_per_clk->num_memclk_levels : 1; + dcn32_init_single_clock(clk_mgr, PPCLK_FCLK, &clk_mgr_base->bw_params->clk_table.entries[0].fclk_mhz, &num_entries_per_clk->num_fclk_levels); -- GitLab From 3442f4e0e55555d14b099c17382453fdfd2508d5 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 20 Oct 2022 11:46:49 -0400 Subject: [PATCH 0593/1927] drm/amd/display: Remove optimization for VRR updates Optimization caused unexpected regression, so remove for now. Tested-by: Mark Broadworth Reviewed-by: Aric Cyr Acked-by: Rodrigo Siqueira Signed-off-by: Alvin Lee Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 20c16ca935f73..f6776ccf7a0eb 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -401,9 +401,6 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc, { int i; - if (memcmp(adjust, &stream->adjust, sizeof(struct dc_crtc_timing_adjust)) == 0) - return true; - stream->adjust.v_total_max = adjust->v_total_max; stream->adjust.v_total_mid = adjust->v_total_mid; stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num; -- GitLab From 35fe1915eebde4546cea5fd4e86e76f47d7b5dae Mon Sep 17 00:00:00 2001 From: Bhawanpreet Lakha Date: Thu, 20 Oct 2022 11:46:50 -0400 Subject: [PATCH 0594/1927] drm/amd/display: Fix HDCP 1.X 1A-04 failing [Why] On some linux based OS, the hotplug->HDCP start takes longer than 4seconds (by ~100ms) This is due to the HDCP delay (3 sec) so we only have 1 second to hotplug->stream enablement, which is not enough for certain OS configs. [How] Change the Delay to 0 seconds. From testing it seems like 0 Seconds can pass 1.x and 2.x compliances Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Bhawanpreet Lakha Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c index 6202e31c7e3a6..a7fd98f57f94c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_hdcp.c @@ -495,7 +495,7 @@ static void update_config(void *handle, struct cp_psp_stream_config *config) link->dp.mst_enabled = config->mst_enabled; link->dp.usb4_enabled = config->usb4_enabled; display->adjust.disable = MOD_HDCP_DISPLAY_DISABLE_AUTHENTICATION; - link->adjust.auth_delay = 3; + link->adjust.auth_delay = 0; link->adjust.hdcp1.disable = 0; conn_state = aconnector->base.state; -- GitLab From 6894534b56059f68a96a3fcf357e15671de0d24e Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 20 Oct 2022 11:46:51 -0400 Subject: [PATCH 0595/1927] drm/amd/display: Enable timing sync on DCN32 Missed enabling timing sync on DCN32 because DCN32 has a different DML param. Tested-by: Mark Broadworth Reviewed-by: Martin Leung Reviewed-by: Jun Lei Acked-by: Rodrigo Siqueira Signed-off-by: Alvin Lee Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c index ec2eae99b8929..602e885ed52c4 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c @@ -1228,6 +1228,7 @@ int dcn20_populate_dml_pipes_from_context( pipes[pipe_cnt].pipe.src.dcc = false; pipes[pipe_cnt].pipe.src.dcc_rate = 1; pipes[pipe_cnt].pipe.dest.synchronized_vblank_all_planes = synchronized_vblank; + pipes[pipe_cnt].pipe.dest.synchronize_timings = synchronized_vblank; pipes[pipe_cnt].pipe.dest.hblank_start = timing->h_total - timing->h_front_porch; pipes[pipe_cnt].pipe.dest.hblank_end = pipes[pipe_cnt].pipe.dest.hblank_start - timing->h_addressable -- GitLab From c3575dd201895aa5cbb50712e3879a650cde9eb4 Mon Sep 17 00:00:00 2001 From: Max Tseng Date: Thu, 20 Oct 2022 11:46:52 -0400 Subject: [PATCH 0596/1927] drm/amd/display: cursor update command incomplete Missing send cursor_rect width & Height into DMUB. PSR-SU would use these information. But missing these assignment in last refactor commit Tested-by: Mark Broadworth Reviewed-by: Anthony Koo Acked-by: Rodrigo Siqueira Signed-off-by: Max Tseng Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c index 4996d2810edb8..938dba5249d48 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c @@ -623,6 +623,10 @@ void hubp2_cursor_set_attributes( hubp->att.size.bits.width = attr->width; hubp->att.size.bits.height = attr->height; hubp->att.cur_ctl.bits.mode = attr->color_format; + + hubp->cur_rect.w = attr->width; + hubp->cur_rect.h = attr->height; + hubp->att.cur_ctl.bits.pitch = hw_pitch; hubp->att.cur_ctl.bits.line_per_chunk = lpc; hubp->att.cur_ctl.bits.cur_2x_magnify = attr->attribute_flags.bits.ENABLE_MAGNIFICATION; -- GitLab From 6bf20b9ba7863ecfa14f5b4171f719256b787b34 Mon Sep 17 00:00:00 2001 From: Aric Cyr Date: Thu, 20 Oct 2022 11:46:53 -0400 Subject: [PATCH 0597/1927] drm/amd/display: Fix SDR visual confirm Apply SDR visual confirm to RGB10 and FP16 formats as well when needed. Tested-by: Mark Broadworth Reviewed-by: Jun Lei Acked-by: Rodrigo Siqueira Signed-off-by: Aric Cyr Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c index 7c2e3b8dc26ad..471078fc3900d 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c @@ -366,6 +366,7 @@ void get_hdr_visual_confirm_color( struct tg_color *color) { uint32_t color_value = MAX_TG_COLOR_VALUE; + bool is_sdr = false; /* Determine the overscan color based on the top-most (desktop) plane's context */ struct pipe_ctx *top_pipe_ctx = pipe_ctx; @@ -382,7 +383,8 @@ void get_hdr_visual_confirm_color( /* FreeSync 2 ARGB2101010 - set border color to pink */ color->color_r_cr = color_value; color->color_b_cb = color_value; - } + } else + is_sdr = true; break; case PIXEL_FORMAT_FP16: if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_PQ) { @@ -391,14 +393,19 @@ void get_hdr_visual_confirm_color( } else if (top_pipe_ctx->stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) { /* FreeSync 2 HDR - set border color to green */ color->color_g_y = color_value; - } + } else + is_sdr = true; break; default: + is_sdr = true; + break; + } + + if (is_sdr) { /* SDR - set border color to Gray */ color->color_r_cr = color_value/2; color->color_b_cb = color_value/2; color->color_g_y = color_value/2; - break; } } -- GitLab From 203ccaf586446b578909de1b763278033fb74b51 Mon Sep 17 00:00:00 2001 From: Haiyi Zhou Date: Thu, 20 Oct 2022 11:46:54 -0400 Subject: [PATCH 0598/1927] drm/amd/display: wait for vblank during pipe programming [WHY] Skipping vblank during global sync update request can result in underflow on certain displays. [HOW] Roll back to the previous behavior where DC waits for vblank during pipe programming. Fixes: 5d3e14421410 ("drm/amd/display: do not wait for vblank during pipe programming") Tested-by: Mark Broadworth Reviewed-by: Martin Leung Acked-by: Rodrigo Siqueira Signed-off-by: Haiyi Zhou Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index 1ac0ed5be605d..feff834d5410e 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1640,10 +1640,8 @@ static void dcn20_program_pipe( pipe_ctx->pipe_dlg_param.vupdate_width); if (pipe_ctx->stream->mall_stream_config.type != SUBVP_PHANTOM) { - pipe_ctx->stream_res.tg->funcs->wait_for_state( - pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); - pipe_ctx->stream_res.tg->funcs->wait_for_state( - pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); } pipe_ctx->stream_res.tg->funcs->set_vtg_params( -- GitLab From 1e8fd864afdc7a52df375e888a03b8472fc24f5d Mon Sep 17 00:00:00 2001 From: zhikzhai Date: Thu, 20 Oct 2022 11:46:55 -0400 Subject: [PATCH 0599/1927] drm/amd/display: skip commit minimal transition state [WHY] Now dynamic ODM will now be disabled when MPO is required safe transitions to avoid underflow, but we are triggering the way of minimal transition too often. Commit state of dc with no check will do pipeline setup which may re-initialize the component with no need such as audio. [HOW] Just do the minimal transition when all of pipes are in use, otherwise return true to skip. Tested-by: Mark Broadworth Reviewed-by: Dillon Varone Acked-by: Rodrigo Siqueira Signed-off-by: zhikzhai Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index f6776ccf7a0eb..945b5210fba45 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -3738,10 +3738,32 @@ static bool commit_minimal_transition_state(struct dc *dc, bool temp_subvp_policy; enum dc_status ret = DC_ERROR_UNEXPECTED; unsigned int i, j; + unsigned int pipe_in_use = 0; if (!transition_context) return false; + /* check current pipes in use*/ + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe = &transition_base_context->res_ctx.pipe_ctx[i]; + + if (pipe->plane_state) + pipe_in_use++; + } + + /* When the OS add a new surface if we have been used all of pipes with odm combine + * and mpc split feature, it need use commit_minimal_transition_state to transition safely. + * After OS exit MPO, it will back to use odm and mpc split with all of pipes, we need + * call it again. Otherwise return true to skip. + * + * Reduce the scenarios to use dc_commit_state_no_check in the stage of flip. Especially + * enter/exit MPO when DCN still have enough resources. + */ + if (pipe_in_use != dc->res_pool->pipe_count) { + dc_release_state(transition_context); + return true; + } + if (!dc->config.is_vmin_only_asic) { tmp_mpc_policy = dc->debug.pipe_split_policy; dc->debug.pipe_split_policy = MPC_SPLIT_AVOID; -- GitLab From b1a98cf89a695d36c414653634ea7ba91b6e701f Mon Sep 17 00:00:00 2001 From: Ma Hanghong Date: Thu, 20 Oct 2022 11:46:56 -0400 Subject: [PATCH 0600/1927] drm/amd/display: Wrong colorimetry workaround [Why] For FreeSync HDR, native color space flag in AMD VSIF(BT.709) should be used when intepreting content and color space flag in VSC or AVI infoFrame should be ignored. However, it turned out some userspace application still use color flag in VSC or AVI infoFrame which is incorrect. [How] Transfer function is used when building the VSC and AVI infoFrame. Set colorimetry to BT.709 when all the following match: 1. Pixel format is YCbCr; 2. In FreeSync 2 HDR, color is COLOR_SPACE_2020_YCBCR; 3. Transfer function is TRANSFER_FUNC_GAMMA_22; Tested-by: Mark Broadworth Reviewed-by: Krunoslav Kovac Acked-by: Rodrigo Siqueira Signed-off-by: Ma Hanghong Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 5 ++++- drivers/gpu/drm/amd/display/dc/core/dc_resource.c | 6 ++++++ drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h | 3 ++- .../gpu/drm/amd/display/modules/info_packet/info_packet.c | 6 +++++- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 72aede3fcf9b6..d14e57b19066c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -5697,6 +5697,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, bool scale = dm_state ? (dm_state->scaling != RMX_OFF) : false; int mode_refresh; int preferred_refresh = 0; + enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN; #if defined(CONFIG_DRM_AMD_DC_DCN) struct dsc_dec_dpcd_caps dsc_caps; #endif @@ -5820,7 +5821,9 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, if (stream->link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED) stream->use_vsc_sdp_for_colorimetry = true; } - mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space); + if (stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) + tf = TRANSFER_FUNC_GAMMA_22; + mod_build_vsc_infopacket(stream, &stream->vsc_infopacket, stream->output_color_space, tf); aconnector->psr_skip_count = AMDGPU_DM_PSR_ENTRY_DELAY; } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index ba2d4347f7224..d99f1c4b95845 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -3015,6 +3015,12 @@ static void set_avi_info_frame( hdmi_info.bits.C0_C1 = COLORIMETRY_EXTENDED; } + if (pixel_encoding && color_space == COLOR_SPACE_2020_YCBCR && + stream->out_transfer_func->tf == TRANSFER_FUNCTION_GAMMA22) { + hdmi_info.bits.EC0_EC2 = 0; + hdmi_info.bits.C0_C1 = COLORIMETRY_ITU709; + } + /* TODO: un-hardcode aspect ratio */ aspect = stream->timing.aspect_ratio; diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h index 1d8b746b02f24..edf5845f6a1f7 100644 --- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h +++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h @@ -35,7 +35,8 @@ struct mod_vrr_params; void mod_build_vsc_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet, - enum dc_color_space cs); + enum dc_color_space cs, + enum color_transfer_func tf); void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet); diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c index 27ceba9d6d658..69691058ab898 100644 --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c @@ -132,7 +132,8 @@ enum ColorimetryYCCDP { void mod_build_vsc_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet, - enum dc_color_space cs) + enum dc_color_space cs, + enum color_transfer_func tf) { unsigned int vsc_packet_revision = vsc_packet_undefined; unsigned int i; @@ -382,6 +383,9 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, colorimetryFormat = ColorimetryYCC_DP_AdobeYCC; else if (cs == COLOR_SPACE_2020_YCBCR) colorimetryFormat = ColorimetryYCC_DP_ITU2020YCbCr; + + if (cs == COLOR_SPACE_2020_YCBCR && tf == TRANSFER_FUNC_GAMMA_22) + colorimetryFormat = ColorimetryYCC_DP_ITU709; break; default: -- GitLab From 1682bd1a6b5fb094e914d9b73b711821fd84dcbd Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:46:57 -0400 Subject: [PATCH 0601/1927] drm/amd/display: Expand kernel doc for DC This commit adds extra documentation for elements related to FAMs. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 19 +++++++++++--- drivers/gpu/drm/amd/display/dc/dc_stream.h | 11 ++++++++ .../gpu/drm/amd/display/dc/dml/dc_features.h | 7 ++++++ .../amd/display/dc/dml/display_mode_enums.h | 25 +++++++++++++++++++ .../drm/amd/display/dc/dml/display_mode_vba.h | 9 +++++++ .../gpu/drm/amd/display/dc/inc/core_types.h | 7 ++++++ .../gpu/drm/amd/display/dc/inc/hw/hw_shared.h | 7 ++++++ 7 files changed, 82 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 3450aca538b58..44f8db1605200 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -494,9 +494,12 @@ enum dcn_zstate_support_state { DCN_ZSTATE_SUPPORT_ALLOW_Z10_ONLY, DCN_ZSTATE_SUPPORT_DISALLOW, }; -/* - * For any clocks that may differ per pipe - * only the max is stored in this structure + +/** + * dc_clocks - DC pipe clocks + * + * For any clocks that may differ per pipe only the max is stored in this + * structure */ struct dc_clocks { int dispclk_khz; @@ -523,6 +526,16 @@ struct dc_clocks { bool prev_p_state_change_support; bool fclk_prev_p_state_change_support; int num_ways; + + /** + * @fw_based_mclk_switching + * + * DC has a mechanism that leverage the variable refresh rate to switch + * memory clock in cases that we have a large latency to achieve the + * memory clock change and a short vblank window. DC has some + * requirements to enable this feature, and this field describes if the + * system support or not such a feature. + */ bool fw_based_mclk_switching; bool fw_based_mclk_switching_shut_down; int prev_num_ways; diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h index 4d30ae0ca721a..f4dfd3a49b68e 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_stream.h +++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h @@ -201,7 +201,18 @@ struct dc_stream_state { bool use_vsc_sdp_for_colorimetry; bool ignore_msa_timing_param; + /** + * @allow_freesync: + * + * It say if Freesync is enabled or not. + */ bool allow_freesync; + + /** + * @vrr_active_variable: + * + * It describes if VRR is in use. + */ bool vrr_active_variable; bool freesync_on_desktop; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dc_features.h b/drivers/gpu/drm/amd/display/dc/dml/dc_features.h index 74e86732e3010..2cbdd75429ffd 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dc_features.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dc_features.h @@ -29,6 +29,13 @@ #define DC__PRESENT 1 #define DC__PRESENT__1 1 #define DC__NUM_DPP 4 + +/** + * @DC__VOLTAGE_STATES: + * + * Define the maximum amount of states supported by the ASIC. Every ASIC has a + * specific number of states; this macro defines the maximum number of states. + */ #define DC__VOLTAGE_STATES 20 #define DC__NUM_DPP__4 1 #define DC__NUM_DPP__0_PRESENT 1 diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h index f394b3f3922a8..0bffae95f3a29 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_enums.h @@ -105,14 +105,39 @@ enum source_macro_tile_size { enum cursor_bpp { dm_cur_2bit = 0, dm_cur_32bit = 1, dm_cur_64bit = 2 }; + +/** + * @enum clock_change_support - It represents possible reasons to change the DRAM clock. + * + * DC may change the DRAM clock during its execution, and this enum tracks all + * the available methods. Note that every ASIC has their specific way to deal + * with these clock switch. + */ enum clock_change_support { + /** + * @dm_dram_clock_change_uninitialized: If you see this, we might have + * a code initialization issue + */ dm_dram_clock_change_uninitialized = 0, + + /** + * @dm_dram_clock_change_vactive: Support DRAM switch in VActive + */ dm_dram_clock_change_vactive, + + /** + * @dm_dram_clock_change_vblank: Support DRAM switch in VBlank + */ dm_dram_clock_change_vblank, + dm_dram_clock_change_vactive_w_mall_full_frame, dm_dram_clock_change_vactive_w_mall_sub_vp, dm_dram_clock_change_vblank_w_mall_full_frame, dm_dram_clock_change_vblank_w_mall_sub_vp, + + /** + * @dm_dram_clock_change_unsupported: Do not support DRAM switch + */ dm_dram_clock_change_unsupported }; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h index 630f3395e90a0..d46aa4817e709 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h @@ -419,6 +419,15 @@ struct vba_vars_st { double MinPixelChunkSizeBytes; unsigned int DCCMetaBufferSizeBytes; // Pipe/Plane Parameters + + /** @VoltageLevel: + * Every ASIC has a fixed number of DPM states, and some devices might + * have some particular voltage configuration that does not map + * directly to the DPM states. This field tells how many states the + * target device supports; even though this field combines the DPM and + * special SOC voltages, it mostly matches the total number of DPM + * states. + */ int VoltageLevel; double FabricClock; double DRAMSpeed; diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index 9498105c98ab3..295845d49cb92 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -115,6 +115,13 @@ struct resource_funcs { int vlevel); void (*update_soc_for_wm_a)( struct dc *dc, struct dc_state *context); + + /** + * @populate_dml_pipes - Populate pipe data struct + * + * Returns: + * Total of pipes available in the specific ASIC. + */ int (*populate_dml_pipes)( struct dc *dc, struct dc_state *context, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h index cd2be729846b4..a819f0f97c5f3 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h @@ -35,6 +35,13 @@ ******************************************************************************/ #define MAX_AUDIOS 7 + +/** + * @MAX_PIPES: + * + * Every ASIC support a fixed number of pipes; MAX_PIPES defines a large number + * to be used inside loops and for determining array sizes. + */ #define MAX_PIPES 6 #define MAX_DIG_LINK_ENCODERS 7 #define MAX_DWB_PIPES 1 -- GitLab From 61708a47e84bc458fae9fbc8a9070fb62a9bcf6d Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:46:58 -0400 Subject: [PATCH 0602/1927] drm/amd/display: Add DEC/CRB basic doc Add a kernel-doc to the DE-Tile function hook. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c | 4 ++++ drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c index 9fbb72369c10e..a88a714605213 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c @@ -41,6 +41,10 @@ #define FN(reg_name, field_name) \ hubbub2->shifts->field_name, hubbub2->masks->field_name +/** + * @DCN32_CRB_SEGMENT_SIZE_KB: Maximum Configurable Return Buffer size for + * DCN32 + */ #define DCN32_CRB_SEGMENT_SIZE_KB 64 static void dcn32_init_crb(struct hubbub *hubbub) diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h index e7571c6f5ead4..58f758fcbce13 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/dchubbub.h @@ -167,6 +167,21 @@ struct hubbub_funcs { void (*force_pstate_change_control)(struct hubbub *hubbub, bool force, bool allow); void (*init_watermarks)(struct hubbub *hubbub); + + /** + * @program_det_size: + * + * DE-Tile buffers (DET) is a memory that is used to convert the tiled + * data into linear, which the rest of the display can use to generate + * the graphics output. One of the main features of this component is + * that each pipe has a configurable DET buffer which means that when a + * pipe is not enabled, the device can assign the memory to other + * enabled pipes to try to be more efficient. + * + * DET logic is handled by dchubbub. Some ASICs provide a feature named + * Configurable Return Buffer (CRB) segments which can be allocated to + * compressed or detiled buffers. + */ void (*program_det_size)(struct hubbub *hubbub, int hubp_inst, unsigned det_buffer_size_in_kbyte); void (*program_compbuf_size)(struct hubbub *hubbub, unsigned compbuf_size_kb, bool safe_to_increase); void (*init_crb)(struct hubbub *hubbub); -- GitLab From a6126e14cfb200ecb915c66bccbfc077c704c1fa Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:46:59 -0400 Subject: [PATCH 0603/1927] drm/amd/display: Add basic ODM description Add kernel-doc to some of the ODM-related functions. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/dcn20/dcn20_resource.c | 16 ++++++++++++++++ .../drm/amd/display/dc/inc/hw/stream_encoder.h | 3 +++ .../drm/amd/display/dc/inc/hw/timing_generator.h | 5 +++++ 3 files changed, 24 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c index 8224b9bf01d1b..d0199ec045cb0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c @@ -1454,6 +1454,22 @@ enum dc_status dcn20_remove_stream_from_ctx(struct dc *dc, struct dc_state *new_ return result; } +/** + * dcn20_split_stream_for_odm - Check if stream can be splited for ODM + * + * @dc: DC object with resource pool info required for pipe split + * @res_ctx: Persistent state of resources + * @prev_odm_pipe: Reference to the previous ODM pipe + * @next_odm_pipe: Reference to the next ODM pipe + * + * This function takes a logically active pipe and a logically free pipe and + * halves all the scaling parameters that need to be halved while populating + * the free pipe with the required resources and configuring the next/previous + * ODM pipe pointers. + * + * Return: + * Return true if split stream for ODM is possible, otherwise, return false. + */ bool dcn20_split_stream_for_odm( const struct dc *dc, struct resource_context *res_ctx, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h index 6b3e6de374bdb..42db4b7b79fdc 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h @@ -243,6 +243,9 @@ struct stream_encoder_funcs { uint32_t hubp_requestor_id, enum dynamic_metadata_mode dmdata_mode); + /** + * @dp_set_odm_combine: Sets up DP stream encoder for ODM. + */ void (*dp_set_odm_combine)( struct stream_encoder *enc, bool odm_combine); diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h index 25a1df45b2641..65f18f9dad346 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h @@ -301,6 +301,11 @@ struct timing_generator_funcs { void (*get_dsc_status)(struct timing_generator *optc, uint32_t *dsc_mode); void (*set_odm_bypass)(struct timing_generator *optc, const struct dc_crtc_timing *dc_crtc_timing); + + /** + * @set_odm_combine: Set up the ODM block to read from the correct + * OPP(s) and turn on/off ODM memory. + */ void (*set_odm_combine)(struct timing_generator *optc, int *opp_id, int opp_cnt, struct dc_crtc_timing *timing); void (*set_h_timing_div_manual_mode)(struct timing_generator *optc, bool manual_mode); -- GitLab From fa0fc4fb09b4306ce77093987ee7b84ead697182 Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:47:00 -0400 Subject: [PATCH 0604/1927] drm/amd/display: Add kernel doc to some of the dc fields Add kernel-doc to some important elements from DC struct that might help to understand DC sequence. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/core/dc_resource.c | 5 ++- drivers/gpu/drm/amd/display/dc/dc_link.h | 8 ++++ .../gpu/drm/amd/display/dc/inc/core_types.h | 38 ++++++++++++++----- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index d99f1c4b95845..7caf3139fabc1 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -2562,9 +2562,12 @@ enum dc_status resource_map_pool_resources( /** * dc_resource_state_copy_construct_current() - Creates a new dc_state from existing state - * Is a shallow copy. Increments refcounts on existing streams and planes. + * * @dc: copy out of dc->current_state * @dst_ctx: copy into this + * + * This function makes a shallow copy of the current DC state and increments + * refcounts on existing streams and planes. */ void dc_resource_state_copy_construct_current( const struct dc *dc, diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h index 17f080f8af6cd..edb4532eaa39e 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_link.h +++ b/drivers/gpu/drm/amd/display/dc/dc_link.h @@ -166,6 +166,14 @@ struct dc_link { enum dc_irq_source irq_source_hpd_rx;/* aka DP Short Pulse */ bool is_hpd_filter_disabled; bool dp_ss_off; + + /** + * @link_state_valid: + * + * If there is no link and local sink, this variable should be set to + * false. Otherwise, it should be set to true; usually, the function + * core_link_enable_stream sets this field to true. + */ bool link_state_valid; bool aux_access_disabled; bool sync_lt_in_progress; diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index 295845d49cb92..fe34cfde22a61 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -514,29 +514,47 @@ struct bw_context { union bw_output bw; struct display_mode_lib dml; }; + /** - * struct dc_state - The full description of a state requested by a user - * - * @streams: Stream properties - * @stream_status: The planes on a given stream - * @res_ctx: Persistent state of resources - * @bw_ctx: The output from bandwidth and watermark calculations and the DML - * @pp_display_cfg: PowerPlay clocks and settings - * @dcn_bw_vars: non-stack memory to support bandwidth calculations - * + * struct dc_state - The full description of a state requested by users */ struct dc_state { + /** + * @streams: Stream state properties + */ struct dc_stream_state *streams[MAX_PIPES]; + + /** + * @stream_status: Planes status on a given stream + */ struct dc_stream_status stream_status[MAX_PIPES]; + + /** + * @stream_count: Total of streams in use + */ uint8_t stream_count; uint8_t stream_mask; + /** + * @res_ctx: Persistent state of resources + */ struct resource_context res_ctx; + /** + * @bw_ctx: The output from bandwidth and watermark calculations and the DML + */ struct bw_context bw_ctx; - /* Note: these are big structures, do *not* put on stack! */ + /** + * @pp_display_cfg: PowerPlay clocks and settings + * Note: this is a big struct, do *not* put on stack! + */ struct dm_pp_display_configuration pp_display_cfg; + + /** + * @dcn_bw_vars: non-stack memory to support bandwidth calculations + * Note: this is a big struct, do *not* put on stack! + */ struct dcn_bw_internal_vars dcn_bw_vars; struct clk_mgr *clk_mgr; -- GitLab From 6a0114e0e31cdf391b96ab347618baa9201c12ad Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Thu, 20 Oct 2022 11:47:01 -0400 Subject: [PATCH 0605/1927] drm/amd/display: Add kernel doc for commit sequence Add basic kernel-doc that describes some of the struct and functions that are part of the DC commit sequence.. Tested-by: Mark Broadworth Reviewed-by: Aurabindo Pillai Acked-by: Rodrigo Siqueira Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 63 ++++++++++++++++--- .../gpu/drm/amd/display/dc/core/dc_resource.c | 11 ++++ .../gpu/drm/amd/display/dc/inc/core_types.h | 16 ++++- 3 files changed, 79 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 945b5210fba45..2858917a66be9 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1714,8 +1714,13 @@ void dc_z10_save_init(struct dc *dc) dc->hwss.z10_save_init(dc); } -/* - * Applies given context to HW and copy it into current context. +/** + * dc_commit_state_no_check - Apply context to the hardware + * + * @dc: DC object with the current status to be updated + * @context: New state that will become the current status at the end of this function + * + * Applies given context to the hardware and copy it into current context. * It's up to the user to release the src context afterwards. */ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *context) @@ -3651,10 +3656,24 @@ static void commit_planes_for_stream(struct dc *dc, } } -/* Determines if the incoming context requires a applying transition state with unnecessary - * pipe splitting and ODM disabled, due to hardware limitations. In a case where - * the OPP associated with an MPCC might change due to plane additions, this function +/** + * could_mpcc_tree_change_for_active_pipes - Check if an OPP associated with MPCC might change + * + * @dc: Used to get the current state status + * @stream: Target stream, which we want to remove the attached planes + * @surface_count: Number of surface update + * @is_plane_addition: [in] Fill out with true if it is a plane addition case + * + * DCN32x and newer support a feature named Dynamic ODM which can conflict with + * the MPO if used simultaneously in some specific configurations (e.g., + * 4k@144). This function checks if the incoming context requires applying a + * transition state with unnecessary pipe splitting and ODM disabled to + * circumvent our hardware limitations to prevent this edge case. If the OPP + * associated with an MPCC might change due to plane additions, this function * returns true. + * + * Return: + * Return true if OPP and MPCC might change, otherwise, return false. */ static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc, struct dc_stream_state *stream, @@ -3729,6 +3748,24 @@ static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc, return force_minimal_pipe_splitting; } +/** + * commit_minimal_transition_state - Create a transition pipe split state + * + * @dc: Used to get the current state status + * @transition_base_context: New transition state + * + * In some specific configurations, such as pipe split on multi-display with + * MPO and/or Dynamic ODM, removing a plane may cause unsupported pipe + * programming when moving to new planes. To mitigate those types of problems, + * this function adds a transition state that minimizes pipe usage before + * programming the new configuration. When adding a new plane, the current + * state requires the least pipes, so it is applied without splitting. When + * removing a plane, the new state requires the least pipes, so it is applied + * without splitting. + * + * Return: + * Return false if something is wrong in the transition state. + */ static bool commit_minimal_transition_state(struct dc *dc, struct dc_state *transition_base_context) { @@ -3742,6 +3779,10 @@ static bool commit_minimal_transition_state(struct dc *dc, if (!transition_context) return false; + /* Setup: + * Store the current ODM and MPC config in some temp variables to be + * restored after we commit the transition state. + */ /* check current pipes in use*/ for (i = 0; i < dc->res_pool->pipe_count; i++) { @@ -3777,7 +3818,7 @@ static bool commit_minimal_transition_state(struct dc *dc, dc_resource_state_copy_construct(transition_base_context, transition_context); - //commit minimal state + /* commit minimal state */ if (dc->res_pool->funcs->validate_bandwidth(dc, transition_context, false)) { for (i = 0; i < transition_context->stream_count; i++) { struct dc_stream_status *stream_status = &transition_context->stream_status[i]; @@ -3795,10 +3836,12 @@ static bool commit_minimal_transition_state(struct dc *dc, ret = dc_commit_state_no_check(dc, transition_context); } - /*always release as dc_commit_state_no_check retains in good case*/ + /* always release as dc_commit_state_no_check retains in good case */ dc_release_state(transition_context); - /*restore previous pipe split and odm policy*/ + /* TearDown: + * Restore original configuration for ODM and MPO. + */ if (!dc->config.is_vmin_only_asic) dc->debug.pipe_split_policy = tmp_mpc_policy; @@ -3806,12 +3849,12 @@ static bool commit_minimal_transition_state(struct dc *dc, dc->debug.force_disable_subvp = temp_subvp_policy; if (ret != DC_OK) { - /*this should never happen*/ + /* this should never happen */ BREAK_TO_DEBUGGER(); return false; } - /*force full surface update*/ + /* force full surface update */ for (i = 0; i < dc->current_state->stream_count; i++) { for (j = 0; j < dc->current_state->stream_status[i].plane_count; j++) { dc->current_state->stream_status[i].plane_states[j]->update_flags.raw = 0xFFFFFFFF; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 7caf3139fabc1..da164685547d9 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -1768,6 +1768,17 @@ bool dc_remove_plane_from_context( return true; } +/** + * dc_rem_all_planes_for_stream - Remove planes attached to the target stream. + * + * @dc: Current dc state. + * @stream: Target stream, which we want to remove the attached plans. + * @context: New context. + * + * Return: + * Return true if DC was able to remove all planes from the target + * stream, otherwise, return false. + */ bool dc_rem_all_planes_for_stream( const struct dc *dc, struct dc_stream_state *stream, diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h index fe34cfde22a61..85495ef026f53 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h +++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h @@ -420,7 +420,10 @@ struct pipe_ctx { struct pll_settings pll_settings; - /* link config records software decision for what link config should be + /** + * @link_config: + * + * link config records software decision for what link config should be * enabled given current link capability and stream during hw resource * mapping. This is to decouple the dependency on link capability during * dc commit or update. @@ -542,6 +545,10 @@ struct dc_state { /** * @bw_ctx: The output from bandwidth and watermark calculations and the DML + * + * Each context must have its own instance of VBA, and in order to + * initialize and obtain IP and SOC, the base DML instance from DC is + * initially copied into every context. */ struct bw_context bw_ctx; @@ -559,6 +566,13 @@ struct dc_state { struct clk_mgr *clk_mgr; + /** + * @refcount: refcount reference + * + * Notice that dc_state is used around the code to capture the current + * context, so we need to pass it everywhere. That's why we want to use + * kref in this struct. + */ struct kref refcount; struct { -- GitLab From 887e8cec55f20c39e14e04eb5f8a013ba75312ce Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Mon, 24 Oct 2022 12:47:47 +0800 Subject: [PATCH 0606/1927] drm/amdgpu: set fb_modifiers_not_supported in vkms This patch to fix the gdm3 start failure with virual display: /usr/libexec/gdm-x-session[1711]: (II) AMDGPU(0): Setting screen physical size to 270 x 203 /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): Failed to make import prime FD as pixmap: 22 /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): failed to set mode: Invalid argument /usr/libexec/gdm-x-session[1711]: (WW) AMDGPU(0): Failed to set mode on CRTC 0 /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): Failed to enable any CRTC gnome-shell[1840]: Running GNOME Shell (using mutter 42.2) as a X11 window and compositing manager /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): failed to set mode: Invalid argument vkms doesn't have modifiers support, set fb_modifiers_not_supported to bring the gdm back. Signed-off-by: Yifan Zhang Acked-by: Guchun Chen Reviewed-by: Tim Huang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c index f4b5301ea2a02..500a1dc4fe029 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c @@ -500,6 +500,8 @@ static int amdgpu_vkms_sw_init(void *handle) adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_modifiers_not_supported = true; + r = amdgpu_display_modeset_create_props(adev); if (r) return r; -- GitLab From 9552b19edcf640103e4b6ad2ab0b46ae8d92b6be Mon Sep 17 00:00:00 2001 From: Aric Cyr Date: Thu, 20 Oct 2022 11:47:02 -0400 Subject: [PATCH 0607/1927] drm/amd/display: 3.2.209 DC version 3.2.209 brings along the following: * Improve color manipulation * Add corrections to DML and some flag configuration * Updates for DCN32x * Expand kernel-doc Tested-by: Mark Broadworth Acked-by: Rodrigo Siqueira Signed-off-by: Aric Cyr Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 44f8db1605200..c43bafa289e6c 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -47,7 +47,7 @@ struct aux_payload; struct set_config_cmd_payload; struct dmub_notification; -#define DC_VER "3.2.208" +#define DC_VER "3.2.209" #define MAX_SURFACES 3 #define MAX_PLANES 6 -- GitLab From e366f36958f60c431a7430c8c421c9db0ec6738d Mon Sep 17 00:00:00 2001 From: Rodrigo Siqueira Date: Tue, 18 Oct 2022 16:27:59 -0400 Subject: [PATCH 0608/1927] drm/amd/display: Rework comments on dc file The file dc.c has multiple comments that do not follow the kernel-doc or are made in a distracting way. This commit alleviates part of this issue by reorganizing some comments inside the dc file. Signed-off-by: Rodrigo Siqueira Reviewed-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 161 +++++++++-------------- drivers/gpu/drm/amd/display/dc/dc.h | 32 ++--- 2 files changed, 70 insertions(+), 123 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 2858917a66be9..d446e60989484 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -135,9 +135,7 @@ static const char DC_BUILD_ID[] = "production-build"; * one or two (in the pipe-split case). */ -/******************************************************************************* - * Private functions - ******************************************************************************/ +/* Private functions */ static inline void elevate_update_type(enum surface_update_type *original, enum surface_update_type new) { @@ -421,18 +419,14 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc, } /** - ***************************************************************************** - * Function: dc_stream_get_last_vrr_vtotal + * dc_stream_get_last_used_drr_vtotal - dc_stream_get_last_vrr_vtotal * - * @brief - * Looks up the pipe context of dc_stream_state and gets the - * last VTOTAL used by DRR (Dynamic Refresh Rate) + * @dc: [in] dc reference + * @stream: [in] Initial dc stream state + * @adjust: [in] Updated parameters for vertical_total_min and * - * @param [in] dc: dc reference - * @param [in] stream: Initial dc stream state - * @param [in] adjust: Updated parameters for vertical_total_min and - * vertical_total_max - ***************************************************************************** + * Looks up the pipe context of dc_stream_state and gets the last VTOTAL used + * by DRR (Dynamic Refresh Rate) */ bool dc_stream_get_last_used_drr_vtotal(struct dc *dc, struct dc_stream_state *stream, @@ -1209,9 +1203,7 @@ static void wait_for_no_pipes_pending(struct dc *dc, struct dc_state *context) PERF_TRACE(); } -/******************************************************************************* - * Public functions - ******************************************************************************/ +/* Public functions */ struct dc *dc_create(const struct dc_init_data *init_params) { @@ -4552,21 +4544,17 @@ void dc_mclk_switch_using_fw_based_vblank_stretch_shut_down(struct dc *dc) dc->current_state->bw_ctx.bw.dcn.clk.fw_based_mclk_switching_shut_down = true; } -/* - ***************************************************************************** - * Function: dc_is_dmub_outbox_supported - +/** + * dc_is_dmub_outbox_supported - Check if DMUB firmware support outbox notification * - * @brief - * Checks whether DMUB FW supports outbox notifications, if supported - * DM should register outbox interrupt prior to actually enabling interrupts - * via dc_enable_dmub_outbox + * @dc: [in] dc structure * - * @param - * [in] dc: dc structure + * Checks whether DMUB FW supports outbox notifications, if supported DM + * should register outbox interrupt prior to actually enabling interrupts + * via dc_enable_dmub_outbox * - * @return - * True if DMUB FW supports outbox notifications, False otherwise - ***************************************************************************** + * Return: + * True if DMUB FW supports outbox notifications, False otherwise */ bool dc_is_dmub_outbox_supported(struct dc *dc) { @@ -4584,21 +4572,17 @@ bool dc_is_dmub_outbox_supported(struct dc *dc) return dc->debug.enable_dmub_aux_for_legacy_ddc; } -/* - ***************************************************************************** - * Function: dc_enable_dmub_notifications +/** + * dc_enable_dmub_notifications - Check if dmub fw supports outbox * - * @brief - * Calls dc_is_dmub_outbox_supported to check if dmub fw supports outbox - * notifications. All DMs shall switch to dc_is_dmub_outbox_supported. - * This API shall be removed after switching. + * @dc: [in] dc structure * - * @param - * [in] dc: dc structure + * Calls dc_is_dmub_outbox_supported to check if dmub fw supports outbox + * notifications. All DMs shall switch to dc_is_dmub_outbox_supported. This + * API shall be removed after switching. * - * @return - * True if DMUB FW supports outbox notifications, False otherwise - ***************************************************************************** + * Return: + * True if DMUB FW supports outbox notifications, False otherwise */ bool dc_enable_dmub_notifications(struct dc *dc) { @@ -4606,18 +4590,11 @@ bool dc_enable_dmub_notifications(struct dc *dc) } /** - ***************************************************************************** - * Function: dc_enable_dmub_outbox + * dc_enable_dmub_outbox - Enables DMUB unsolicited notification * - * @brief - * Enables DMUB unsolicited notifications to x86 via outbox + * dc: [in] dc structure * - * @param - * [in] dc: dc structure - * - * @return - * None - ***************************************************************************** + * Enables DMUB unsolicited notifications to x86 via outbox. */ void dc_enable_dmub_outbox(struct dc *dc) { @@ -4718,21 +4695,17 @@ uint8_t get_link_index_from_dpia_port_index(const struct dc *dc, } /** - ***************************************************************************** - * Function: dc_process_dmub_set_config_async + * dc_process_dmub_set_config_async - Submits set_config command * - * @brief - * Submits set_config command to dmub via inbox message + * @dc: [in] dc structure + * @link_index: [in] link_index: link index + * @payload: [in] aux payload + * @notify: [out] set_config immediate reply * - * @param - * [in] dc: dc structure - * [in] link_index: link index - * [in] payload: aux payload - * [out] notify: set_config immediate reply + * Submits set_config command to dmub via inbox message. * - * @return - * True if successful, False if failure - ***************************************************************************** + * Return: + * True if successful, False if failure */ bool dc_process_dmub_set_config_async(struct dc *dc, uint32_t link_index, @@ -4768,21 +4741,17 @@ bool dc_process_dmub_set_config_async(struct dc *dc, } /** - ***************************************************************************** - * Function: dc_process_dmub_set_mst_slots + * dc_process_dmub_set_mst_slots - Submits MST solt allocation * - * @brief - * Submits mst slot allocation command to dmub via inbox message + * @dc: [in] dc structure + * @link_index: [in] link index + * @mst_alloc_slots: [in] mst slots to be allotted + * @mst_slots_in_use: [out] mst slots in use returned in failure case * - * @param - * [in] dc: dc structure - * [in] link_index: link index - * [in] mst_alloc_slots: mst slots to be allotted - * [out] mst_slots_in_use: mst slots in use returned in failure case + * Submits mst slot allocation command to dmub via inbox message * - * @return - * DC_OK if successful, DC_ERROR if failure - ***************************************************************************** + * Return: + * DC_OK if successful, DC_ERROR if failure */ enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc, uint32_t link_index, @@ -4822,19 +4791,12 @@ enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc, } /** - ***************************************************************************** - * Function: dc_process_dmub_dpia_hpd_int_enable - * - * @brief - * Submits dpia hpd int enable command to dmub via inbox message + * dc_process_dmub_dpia_hpd_int_enable - Submits DPIA DPD interruption * - * @param - * [in] dc: dc structure - * [in] hpd_int_enable: 1 for hpd int enable, 0 to disable + * @dc [in]: dc structure + * @hpd_int_enable [in]: 1 for hpd int enable, 0 to disable * - * @return - * None - ***************************************************************************** + * Submits dpia hpd int enable command to dmub via inbox message */ void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc, uint32_t hpd_int_enable) @@ -4863,16 +4825,13 @@ void dc_disable_accelerated_mode(struct dc *dc) /** - ***************************************************************************** - * dc_notify_vsync_int_state() - notifies vsync enable/disable state + * dc_notify_vsync_int_state - notifies vsync enable/disable state * @dc: dc structure - * @stream: stream where vsync int state changed - * @enable: whether vsync is enabled or disabled + * @stream: stream where vsync int state changed + * @enable: whether vsync is enabled or disabled * - * Called when vsync is enabled/disabled - * Will notify DMUB to start/stop ABM interrupts after steady state is reached - * - ***************************************************************************** + * Called when vsync is enabled/disabled Will notify DMUB to start/stop ABM + * interrupts after steady state is reached. */ void dc_notify_vsync_int_state(struct dc *dc, struct dc_stream_state *stream, bool enable) { @@ -4914,14 +4873,18 @@ void dc_notify_vsync_int_state(struct dc *dc, struct dc_stream_state *stream, bo if (pipe->stream_res.abm && pipe->stream_res.abm->funcs->set_abm_pause) pipe->stream_res.abm->funcs->set_abm_pause(pipe->stream_res.abm, !enable, i, pipe->stream_res.tg->inst); } -/* - * dc_extended_blank_supported: Decide whether extended blank is supported + +/** + * dc_extended_blank_supported 0 Decide whether extended blank is supported * - * Extended blank is a freesync optimization feature to be enabled in the future. - * During the extra vblank period gained from freesync, we have the ability to enter z9/z10. + * @dc: [in] Current DC state * - * @param [in] dc: Current DC state - * @return: Indicate whether extended blank is supported (true or false) + * Extended blank is a freesync optimization feature to be enabled in the + * future. During the extra vblank period gained from freesync, we have the + * ability to enter z9/z10. + * + * Return: + * Indicate whether extended blank is supported (true or false) */ bool dc_extended_blank_supported(struct dc *dc) { diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index c43bafa289e6c..2a64064400326 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -56,9 +56,7 @@ struct dmub_notification; #define MIN_VIEWPORT_SIZE 12 #define MAX_NUM_EDP 2 -/******************************************************************************* - * Display Core Interfaces - ******************************************************************************/ +/* Display Core Interfaces */ struct dc_versions { const char *dc_ver; struct dmcu_version dmcu_version; @@ -1000,9 +998,7 @@ void dc_init_callbacks(struct dc *dc, void dc_deinit_callbacks(struct dc *dc); void dc_destroy(struct dc **dc); -/******************************************************************************* - * Surface Interfaces - ******************************************************************************/ +/* Surface Interfaces */ enum { TRANSFER_FUNC_POINTS = 1025 @@ -1363,9 +1359,7 @@ struct dc_state *dc_copy_state(struct dc_state *src_ctx); void dc_retain_state(struct dc_state *context); void dc_release_state(struct dc_state *context); -/******************************************************************************* - * Link Interfaces - ******************************************************************************/ +/* Link Interfaces */ struct dpcd_caps { union dpcd_rev dpcd_rev; @@ -1467,9 +1461,7 @@ struct hdcp_caps { uint32_t dc_get_opp_for_plane(struct dc *dc, struct dc_plane_state *plane); -/******************************************************************************* - * Sink Interfaces - A sink corresponds to a display output device - ******************************************************************************/ +/* Sink Interfaces - A sink corresponds to a display output device */ struct dc_container_id { // 128bit GUID in binary form @@ -1552,9 +1544,7 @@ struct dc_cursor { }; -/******************************************************************************* - * Interrupt interfaces - ******************************************************************************/ +/* Interrupt interfaces */ enum dc_irq_source dc_interrupt_to_irq_source( struct dc *dc, uint32_t src_id, @@ -1566,9 +1556,7 @@ enum dc_irq_source dc_get_hpd_irq_source_at_index( void dc_notify_vsync_int_state(struct dc *dc, struct dc_stream_state *stream, bool enable); -/******************************************************************************* - * Power Interfaces - ******************************************************************************/ +/* Power Interfaces */ void dc_set_power_state( struct dc *dc, @@ -1641,14 +1629,10 @@ enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc, void dc_process_dmub_dpia_hpd_int_enable(const struct dc *dc, uint32_t hpd_int_enable); -/******************************************************************************* - * DSC Interfaces - ******************************************************************************/ +/* DSC Interfaces */ #include "dc_dsc.h" -/******************************************************************************* - * Disable acc mode Interfaces - ******************************************************************************/ +/* Disable acc mode Interfaces */ void dc_disable_accelerated_mode(struct dc *dc); #endif /* DC_INTERFACE_H_ */ -- GitLab From 5468c36d628524effbb89a9503eb1a2318804759 Mon Sep 17 00:00:00 2001 From: Fangzhi Zuo Date: Thu, 20 Oct 2022 16:06:26 -0400 Subject: [PATCH 0609/1927] drm/amd/display: Filter Invalid 420 Modes for HDMI TMDS [Why] Invalidate unsupported 420 modes on HDMI TMDS. HDMI TMDS does not support ODM. Any modes that are horizontally wider than 4096, cannot be supported via TMDS. So they must be filtered out and should not pass validation. [How] Create fake plane for the new stream, and validate global state by going through dml validation routine. Tested-by: Mark Broadworth Acked-by: Rodrigo Siqueira Signed-off-by: Fangzhi Zuo Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index d14e57b19066c..9f2bf3488a786 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -6156,6 +6156,70 @@ static void handle_edid_mgmt(struct amdgpu_dm_connector *aconnector) create_eml_sink(aconnector); } +static enum dc_status dm_validate_stream_and_context(struct dc *dc, + struct dc_stream_state *stream) +{ + enum dc_status dc_result = DC_ERROR_UNEXPECTED; + struct dc_plane_state *dc_plane_state = NULL; + struct dc_state *dc_state = NULL; + + if (!stream) + goto cleanup; + + dc_plane_state = dc_create_plane_state(dc); + if (!dc_plane_state) + goto cleanup; + + dc_state = dc_create_state(dc); + if (!dc_state) + goto cleanup; + + /* populate stream to plane */ + dc_plane_state->src_rect.height = stream->src.height; + dc_plane_state->src_rect.width = stream->src.width; + dc_plane_state->dst_rect.height = stream->src.height; + dc_plane_state->dst_rect.width = stream->src.width; + dc_plane_state->clip_rect.height = stream->src.height; + dc_plane_state->clip_rect.width = stream->src.width; + dc_plane_state->plane_size.surface_pitch = ((stream->src.width + 255) / 256) * 256; + dc_plane_state->plane_size.surface_size.height = stream->src.height; + dc_plane_state->plane_size.surface_size.width = stream->src.width; + dc_plane_state->plane_size.chroma_size.height = stream->src.height; + dc_plane_state->plane_size.chroma_size.width = stream->src.width; + dc_plane_state->tiling_info.gfx9.swizzle = DC_SW_UNKNOWN; + dc_plane_state->format = SURFACE_PIXEL_FORMAT_GRPH_ARGB8888; + dc_plane_state->tiling_info.gfx9.swizzle = DC_SW_UNKNOWN; + dc_plane_state->rotation = ROTATION_ANGLE_0; + dc_plane_state->is_tiling_rotated = false; + dc_plane_state->tiling_info.gfx8.array_mode = DC_ARRAY_LINEAR_GENERAL; + + dc_result = dc_validate_stream(dc, stream); + if (dc_result == DC_OK) + dc_result = dc_validate_plane(dc, dc_plane_state); + + if (dc_result == DC_OK) + dc_result = dc_add_stream_to_ctx(dc, dc_state, stream); + + if (dc_result == DC_OK && !dc_add_plane_to_context( + dc, + stream, + dc_plane_state, + dc_state)) + dc_result = DC_FAIL_ATTACH_SURFACES; + + if (dc_result == DC_OK) + dc_result = dc_validate_global_state(dc, dc_state, true); + +cleanup: + if (dc_state) + dc_release_state(dc_state); + + if (dc_plane_state) + dc_plane_state_release(dc_plane_state); + + return dc_result; +} + struct dc_stream_state * create_validate_stream_for_sink(struct amdgpu_dm_connector *aconnector, const struct drm_display_mode *drm_mode, @@ -6182,6 +6246,9 @@ create_validate_stream_for_sink(struct amdgpu_dm_connector *aconnector, if (dc_result == DC_OK && stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) dc_result = dm_dp_mst_is_port_support_mode(aconnector, stream); + if (dc_result == DC_OK) + dc_result = dm_validate_stream_and_context(adev->dm.dc, stream); + if (dc_result != DC_OK) { DRM_DEBUG_KMS("Mode %dx%d (clk %d) failed DC validation with error %d (%s)\n", drm_mode->hdisplay, -- GitLab From b6bcdc9f6b8321e4471ff45413b6410e16762a8d Mon Sep 17 00:00:00 2001 From: Ryan Roberts Date: Thu, 27 Oct 2022 13:09:45 +0100 Subject: [PATCH 0610/1927] KVM: arm64: Fix bad dereference on MTE-enabled systems enter_exception64() performs an MTE check, which involves dereferencing vcpu->kvm. While vcpu has already been fixed up to be a HYP VA pointer, kvm is still a pointer in the kernel VA space. This only affects nVHE configurations with MTE enabled, as in other cases, the pointer is either valid (VHE) or not dereferenced (!MTE). Fix this by first converting kvm to a HYP VA pointer. Fixes: ea7fc1bb1cd1 ("KVM: arm64: Introduce MTE VM feature") Signed-off-by: Ryan Roberts Reviewed-by: Steven Price [maz: commit message tidy-up] Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221027120945.29679-1-ryan.roberts@arm.com --- arch/arm64/kvm/hyp/exception.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index b7557b25ed568..791d3de767713 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -13,6 +13,7 @@ #include #include #include +#include #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__) #error Hypervisor code only! @@ -115,7 +116,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, new |= (old & PSR_C_BIT); new |= (old & PSR_V_BIT); - if (kvm_has_mte(vcpu->kvm)) + if (kvm_has_mte(kern_hyp_va(vcpu->kvm))) new |= PSR_TCO_BIT; new |= (old & PSR_DIT_BIT); -- GitLab From 3f1a02ebeeebe3b8937391646cead0752144f1cb Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Tue, 25 Oct 2022 22:12:00 +1300 Subject: [PATCH 0611/1927] drm/amdkfd: remove unused kfd_pm4_headers_diq header file kfd_pm4_headers_diq.h header is a leftover from the old H/W debugger module support added on commit fbeb661bfa895dc ("drm/amdkfd: Add skeleton H/W debugger module support"). That implementation was removed after a while and the last file that included that header was removed on commit 5bdd3eb253544b1 ("drm/amdkfd: Remove unused old debugger implementation"). This patch removes the unused header file kfd_pm4_headers_diq.h Signed-off-by: Paulo Miguel Almeida Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h | 291 ------------------ 1 file changed, 291 deletions(-) delete mode 100644 drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h b/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h deleted file mode 100644 index f9cd28690151d..0000000000000 --- a/drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_diq.h +++ /dev/null @@ -1,291 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR MIT */ -/* - * Copyright 2014-2022 Advanced Micro Devices, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef KFD_PM4_HEADERS_DIQ_H_ -#define KFD_PM4_HEADERS_DIQ_H_ - -/*--------------------_INDIRECT_BUFFER-------------------- */ - -#ifndef _PM4__INDIRECT_BUFFER_DEFINED -#define _PM4__INDIRECT_BUFFER_DEFINED -enum _INDIRECT_BUFFER_cache_policy_enum { - cache_policy___indirect_buffer__lru = 0, - cache_policy___indirect_buffer__stream = 1, - cache_policy___indirect_buffer__bypass = 2 -}; - -enum { - IT_INDIRECT_BUFFER_PASID = 0x5C -}; - -struct pm4__indirect_buffer_pasid { - union { - union PM4_MES_TYPE_3_HEADER header; /* header */ - unsigned int ordinal1; - }; - - union { - struct { - unsigned int reserved1:2; - unsigned int ib_base_lo:30; - } bitfields2; - unsigned int ordinal2; - }; - - union { - struct { - unsigned int ib_base_hi:16; - unsigned int reserved2:16; - } bitfields3; - unsigned int ordinal3; - }; - - union { - unsigned int control; - unsigned int ordinal4; - }; - - union { - struct { - unsigned int pasid:10; - unsigned int reserved4:22; - } bitfields5; - unsigned int ordinal5; - }; - -}; - -#endif - -/*--------------------_RELEASE_MEM-------------------- */ - -#ifndef _PM4__RELEASE_MEM_DEFINED -#define _PM4__RELEASE_MEM_DEFINED -enum _RELEASE_MEM_event_index_enum { - event_index___release_mem__end_of_pipe = 5, - event_index___release_mem__shader_done = 6 -}; - -enum _RELEASE_MEM_cache_policy_enum { - cache_policy___release_mem__lru = 0, - cache_policy___release_mem__stream = 1, - cache_policy___release_mem__bypass = 2 -}; - -enum _RELEASE_MEM_dst_sel_enum { - dst_sel___release_mem__memory_controller = 0, - dst_sel___release_mem__tc_l2 = 1, - dst_sel___release_mem__queue_write_pointer_register = 2, - dst_sel___release_mem__queue_write_pointer_poll_mask_bit = 3 -}; - -enum _RELEASE_MEM_int_sel_enum { - int_sel___release_mem__none = 0, - int_sel___release_mem__send_interrupt_only = 1, - int_sel___release_mem__send_interrupt_after_write_confirm = 2, - int_sel___release_mem__send_data_after_write_confirm = 3 -}; - -enum _RELEASE_MEM_data_sel_enum { - data_sel___release_mem__none = 0, - data_sel___release_mem__send_32_bit_low = 1, - data_sel___release_mem__send_64_bit_data = 2, - data_sel___release_mem__send_gpu_clock_counter = 3, - data_sel___release_mem__send_cp_perfcounter_hi_lo = 4, - data_sel___release_mem__store_gds_data_to_memory = 5 -}; - -struct pm4__release_mem { - union { - union PM4_MES_TYPE_3_HEADER header; /*header */ - unsigned int ordinal1; - }; - - union { - struct { - unsigned int event_type:6; - unsigned int reserved1:2; - enum _RELEASE_MEM_event_index_enum event_index:4; - unsigned int tcl1_vol_action_ena:1; - unsigned int tc_vol_action_ena:1; - unsigned int reserved2:1; - unsigned int tc_wb_action_ena:1; - unsigned int tcl1_action_ena:1; - unsigned int tc_action_ena:1; - unsigned int reserved3:6; - unsigned int atc:1; - enum _RELEASE_MEM_cache_policy_enum cache_policy:2; - unsigned int reserved4:5; - } bitfields2; - unsigned int ordinal2; - }; - - union { - struct { - unsigned int reserved5:16; - enum _RELEASE_MEM_dst_sel_enum dst_sel:2; - unsigned int reserved6:6; - enum _RELEASE_MEM_int_sel_enum int_sel:3; - unsigned int reserved7:2; - enum _RELEASE_MEM_data_sel_enum data_sel:3; - } bitfields3; - unsigned int ordinal3; - }; - - union { - struct { - unsigned int reserved8:2; - unsigned int address_lo_32b:30; - } bitfields4; - struct { - unsigned int reserved9:3; - unsigned int address_lo_64b:29; - } bitfields5; - unsigned int ordinal4; - }; - - unsigned int address_hi; - - unsigned int data_lo; - - unsigned int data_hi; - -}; -#endif - - -/*--------------------_SET_CONFIG_REG-------------------- */ - -#ifndef _PM4__SET_CONFIG_REG_DEFINED -#define _PM4__SET_CONFIG_REG_DEFINED - -struct pm4__set_config_reg { - union { - union PM4_MES_TYPE_3_HEADER header; /*header */ - unsigned int ordinal1; - }; - - union { - struct { - unsigned int reg_offset:16; - unsigned int reserved1:7; - unsigned int vmid_shift:5; - unsigned int insert_vmid:1; - unsigned int reserved2:3; - } bitfields2; - unsigned int ordinal2; - }; - - unsigned int reg_data[1]; /*1..N of these fields */ - -}; -#endif - -/*--------------------_WAIT_REG_MEM-------------------- */ - -#ifndef _PM4__WAIT_REG_MEM_DEFINED -#define _PM4__WAIT_REG_MEM_DEFINED -enum _WAIT_REG_MEM_function_enum { - function___wait_reg_mem__always_pass = 0, - function___wait_reg_mem__less_than_ref_value = 1, - function___wait_reg_mem__less_than_equal_to_the_ref_value = 2, - function___wait_reg_mem__equal_to_the_reference_value = 3, - function___wait_reg_mem__not_equal_reference_value = 4, - function___wait_reg_mem__greater_than_or_equal_reference_value = 5, - function___wait_reg_mem__greater_than_reference_value = 6, - function___wait_reg_mem__reserved = 7 -}; - -enum _WAIT_REG_MEM_mem_space_enum { - mem_space___wait_reg_mem__register_space = 0, - mem_space___wait_reg_mem__memory_space = 1 -}; - -enum _WAIT_REG_MEM_operation_enum { - operation___wait_reg_mem__wait_reg_mem = 0, - operation___wait_reg_mem__wr_wait_wr_reg = 1 -}; - -struct pm4__wait_reg_mem { - union { - union PM4_MES_TYPE_3_HEADER header; /*header */ - unsigned int ordinal1; - }; - - union { - struct { - enum _WAIT_REG_MEM_function_enum function:3; - unsigned int reserved1:1; - enum _WAIT_REG_MEM_mem_space_enum mem_space:2; - enum _WAIT_REG_MEM_operation_enum operation:2; - unsigned int reserved2:24; - } bitfields2; - unsigned int ordinal2; - }; - - union { - struct { - unsigned int reserved3:2; - unsigned int memory_poll_addr_lo:30; - } bitfields3; - struct { - unsigned int register_poll_addr:16; - unsigned int reserved4:16; - } bitfields4; - struct { - unsigned int register_write_addr:16; - unsigned int reserved5:16; - } bitfields5; - unsigned int ordinal3; - }; - - union { - struct { - unsigned int poll_address_hi:16; - unsigned int reserved6:16; - } bitfields6; - struct { - unsigned int register_write_addr:16; - unsigned int reserved7:16; - } bitfields7; - unsigned int ordinal4; - }; - - unsigned int reference; - - unsigned int mask; - - union { - struct { - unsigned int poll_interval:16; - unsigned int reserved8:16; - } bitfields8; - unsigned int ordinal7; - }; - -}; -#endif - - -#endif /* KFD_PM4_HEADERS_DIQ_H_ */ -- GitLab From 2513574853f388008b81a900d56c2f75e7f35be2 Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Tue, 25 Oct 2022 21:20:19 +1300 Subject: [PATCH 0612/1927] drm/amdkfd: remove unused struct cdit_header struct cdit_header was never used across any of the amd drivers nor this is exposed to UAPI so it can be removed. This patch removes struct cdit_header and refactor code accordingly Signed-off-by: Paulo Miguel Almeida Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.h | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h index 482ba84a728d1..22893ff7b9a1a 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h @@ -29,11 +29,10 @@ #pragma pack(1) /* - * 4CC signature values for the CRAT and CDIT ACPI tables + * 4CC signature value for the CRAT ACPI table */ #define CRAT_SIGNATURE "CRAT" -#define CDIT_SIGNATURE "CDIT" /* * Component Resource Association Table (CRAT) @@ -292,27 +291,6 @@ struct crat_subtype_generic { uint32_t flags; }; -/* - * Component Locality Distance Information Table (CDIT) - */ -#define CDIT_OEMID_LENGTH 6 -#define CDIT_OEMTABLEID_LENGTH 8 - -struct cdit_header { - uint32_t signature; - uint32_t length; - uint8_t revision; - uint8_t checksum; - uint8_t oem_id[CDIT_OEMID_LENGTH]; - uint8_t oem_table_id[CDIT_OEMTABLEID_LENGTH]; - uint32_t oem_revision; - uint32_t creator_id; - uint32_t creator_revision; - uint32_t total_entries; - uint16_t num_domains; - uint8_t entry[1]; -}; - #pragma pack() struct kfd_dev; -- GitLab From cbe4d43ea5e903ae3e8555cc39047b0ed027738a Mon Sep 17 00:00:00 2001 From: Tao Zhou Date: Mon, 17 Oct 2022 18:22:13 +0800 Subject: [PATCH 0613/1927] drm/amdgpu: add RAS page retirement functions for MCA Define page retirement functions for MCA platform. v2: remove page retirement handling from MCA poison handler, let MCA notifier do page retirement. v3: remove specific poison handler for MCA to simplify code. Signed-off-by: Tao Zhou Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c | 53 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h | 2 + 2 files changed, 55 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c index aad3c8b4c8102..3c83129f40900 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c @@ -22,6 +22,59 @@ */ #include "amdgpu.h" +#include "umc_v6_7.h" + +static int amdgpu_umc_convert_error_address(struct amdgpu_device *adev, + struct ras_err_data *err_data, uint64_t err_addr, + uint32_t ch_inst, uint32_t umc_inst) +{ + switch (adev->ip_versions[UMC_HWIP][0]) { + case IP_VERSION(6, 7, 0): + umc_v6_7_convert_error_address(adev, + err_data, err_addr, ch_inst, umc_inst); + break; + default: + dev_warn(adev->dev, + "UMC address to Physical address translation is not supported\n"); + return AMDGPU_RAS_FAIL; + } + + return AMDGPU_RAS_SUCCESS; +} + +int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev, + uint64_t err_addr, uint32_t ch_inst, uint32_t umc_inst) +{ + struct ras_err_data err_data = {0, 0, 0, NULL}; + int ret = AMDGPU_RAS_FAIL; + + err_data.err_addr = + kcalloc(adev->umc.max_ras_err_cnt_per_query, + sizeof(struct eeprom_table_record), GFP_KERNEL); + if (!err_data.err_addr) { + dev_warn(adev->dev, + "Failed to alloc memory for umc error record in MCA notifier!\n"); + return AMDGPU_RAS_FAIL; + } + + /* + * Translate UMC channel address to Physical address + */ + ret = amdgpu_umc_convert_error_address(adev, &err_data, err_addr, + ch_inst, umc_inst); + if (ret) + goto out; + + if (amdgpu_bad_page_threshold != 0) { + amdgpu_ras_add_bad_pages(adev, err_data.err_addr, + err_data.err_addr_cnt); + amdgpu_ras_save_bad_pages(adev); + } + +out: + kfree(err_data.err_addr); + return ret; +} static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev, void *ras_error_status, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h index 3629d8f292ef9..659a10de29c95 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h @@ -98,4 +98,6 @@ void amdgpu_umc_fill_error_record(struct ras_err_data *err_data, int amdgpu_umc_process_ras_data_cb(struct amdgpu_device *adev, void *ras_error_status, struct amdgpu_iv_entry *entry); +int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev, + uint64_t err_addr, uint32_t ch_inst, uint32_t umc_inst); #endif -- GitLab From 24b822928b5139b85ee9a818a65e343b7e3bb4fe Mon Sep 17 00:00:00 2001 From: Tao Zhou Date: Mon, 17 Oct 2022 18:26:17 +0800 Subject: [PATCH 0614/1927] drm/amdgpu: use page retirement API in MCA notifier Make the code more readable. Signed-off-by: Tao Zhou Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 36 +++---------------------- 1 file changed, 3 insertions(+), 33 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 21a47f2bb87bc..28463b47ce333 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -36,7 +36,6 @@ #include "ivsrcid/nbio/irqsrcs_nbif_7_4.h" #include "atom.h" #include "amdgpu_reset.h" -#include "umc_v6_7.h" #ifdef CONFIG_X86_MCE_AMD #include @@ -2849,7 +2848,6 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb, struct amdgpu_device *adev = NULL; uint32_t gpu_id = 0; uint32_t umc_inst = 0, ch_inst = 0; - struct ras_err_data err_data = {0, 0, 0, NULL}; /* * If the error was generated in UMC_V2, which belongs to GPU UMCs, @@ -2888,38 +2886,10 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb, dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d", umc_inst, ch_inst); - err_data.err_addr = - kcalloc(adev->umc.max_ras_err_cnt_per_query, - sizeof(struct eeprom_table_record), GFP_KERNEL); - if (!err_data.err_addr) { - dev_warn(adev->dev, - "Failed to alloc memory for umc error record in mca notifier!\n"); - return NOTIFY_DONE; - } - - /* - * Translate UMC channel address to Physical address - */ - switch (adev->ip_versions[UMC_HWIP][0]) { - case IP_VERSION(6, 7, 0): - umc_v6_7_convert_error_address(adev, - &err_data, m->addr, ch_inst, umc_inst); - break; - default: - dev_warn(adev->dev, - "UMC address to Physical address translation is not supported\n"); - kfree(err_data.err_addr); + if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst)) + return NOTIFY_OK; + else return NOTIFY_DONE; - } - - if (amdgpu_bad_page_threshold != 0) { - amdgpu_ras_add_bad_pages(adev, err_data.err_addr, - err_data.err_addr_cnt); - amdgpu_ras_save_bad_pages(adev); - } - - kfree(err_data.err_addr); - return NOTIFY_OK; } static struct notifier_block amdgpu_bad_page_nb = { -- GitLab From ae45a18b80d9d0d29f0ecfc52fb4e7831671b299 Mon Sep 17 00:00:00 2001 From: Tao Zhou Date: Mon, 17 Oct 2022 18:31:20 +0800 Subject: [PATCH 0615/1927] drm/amdgpu: add RAS poison handling for MCA For MCA poison, if unmap queue fails, only gpu reset should be triggered without page retirement handling, MCA notifier will do it. v2: handle MCA poison consumption in umc_poison_handler directly. Signed-off-by: Tao Zhou Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c | 31 ++++++++++++++++--------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c index 3c83129f40900..758942150c097 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c @@ -169,19 +169,28 @@ int amdgpu_umc_poison_handler(struct amdgpu_device *adev, void *ras_error_status, bool reset) { - int ret; - struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; - struct ras_common_if head = { - .block = AMDGPU_RAS_BLOCK__UMC, - }; - struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head); + int ret = AMDGPU_RAS_SUCCESS; - ret = - amdgpu_umc_do_page_retirement(adev, ras_error_status, NULL, reset); + if (!adev->gmc.xgmi.connected_to_cpu) { + struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; + struct ras_common_if head = { + .block = AMDGPU_RAS_BLOCK__UMC, + }; + struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head); - if (ret == AMDGPU_RAS_SUCCESS && obj) { - obj->err_data.ue_count += err_data->ue_count; - obj->err_data.ce_count += err_data->ce_count; + ret = + amdgpu_umc_do_page_retirement(adev, ras_error_status, NULL, reset); + + if (ret == AMDGPU_RAS_SUCCESS && obj) { + obj->err_data.ue_count += err_data->ue_count; + obj->err_data.ce_count += err_data->ce_count; + } + } else if (reset) { + /* MCA poison handler is only responsible for GPU reset, + * let MCA notifier do page retirement. + */ + kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); + amdgpu_ras_reset_gpu(adev); } return ret; -- GitLab From 1ed0e176902483e67cd02530d387a7551b0e99a4 Mon Sep 17 00:00:00 2001 From: Tao Zhou Date: Tue, 18 Oct 2022 10:31:09 +0800 Subject: [PATCH 0616/1927] drm/amdgpu: remove ras_error_status parameter for UMC poison handler Make the code simpler. Signed-off-by: Tao Zhou Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 4 +--- drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c | 3 +-- drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c | 13 +++++-------- drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h | 4 +--- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 0561812aa0a43..37db39ba8718c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -753,9 +753,7 @@ bool amdgpu_amdkfd_have_atomics_support(struct amdgpu_device *adev) void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev, bool reset) { - struct ras_err_data err_data = {0, 0, 0, NULL}; - - amdgpu_umc_poison_handler(adev, &err_data, reset); + amdgpu_umc_poison_handler(adev, reset); } bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index 28463b47ce333..693bce07eb464 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -1561,7 +1561,6 @@ static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager * { bool poison_stat = false; struct amdgpu_device *adev = obj->adev; - struct ras_err_data err_data = {0, 0, 0, NULL}; struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, obj->head.block, 0); @@ -1584,7 +1583,7 @@ static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager * } if (!adev->gmc.xgmi.connected_to_cpu) - amdgpu_umc_poison_handler(adev, &err_data, false); + amdgpu_umc_poison_handler(adev, false); if (block_obj->hw_ops->handle_poison_consumption) poison_stat = block_obj->hw_ops->handle_poison_consumption(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c index 758942150c097..f76c19fc03926 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c @@ -165,25 +165,22 @@ static int amdgpu_umc_do_page_retirement(struct amdgpu_device *adev, return AMDGPU_RAS_SUCCESS; } -int amdgpu_umc_poison_handler(struct amdgpu_device *adev, - void *ras_error_status, - bool reset) +int amdgpu_umc_poison_handler(struct amdgpu_device *adev, bool reset) { int ret = AMDGPU_RAS_SUCCESS; if (!adev->gmc.xgmi.connected_to_cpu) { - struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; + struct ras_err_data err_data = {0, 0, 0, NULL}; struct ras_common_if head = { .block = AMDGPU_RAS_BLOCK__UMC, }; struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head); - ret = - amdgpu_umc_do_page_retirement(adev, ras_error_status, NULL, reset); + ret = amdgpu_umc_do_page_retirement(adev, &err_data, NULL, reset); if (ret == AMDGPU_RAS_SUCCESS && obj) { - obj->err_data.ue_count += err_data->ue_count; - obj->err_data.ce_count += err_data->ce_count; + obj->err_data.ue_count += err_data.ue_count; + obj->err_data.ce_count += err_data.ce_count; } } else if (reset) { /* MCA poison handler is only responsible for GPU reset, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h index 659a10de29c95..a6951160f13af 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h @@ -83,9 +83,7 @@ struct amdgpu_umc { }; int amdgpu_umc_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block); -int amdgpu_umc_poison_handler(struct amdgpu_device *adev, - void *ras_error_status, - bool reset); +int amdgpu_umc_poison_handler(struct amdgpu_device *adev, bool reset); int amdgpu_umc_process_ecc_irq(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry); -- GitLab From 7117007edad2299f2f093cdcfa813beb646d913e Mon Sep 17 00:00:00 2001 From: Alan Liu Date: Mon, 24 Oct 2022 14:15:30 +0800 Subject: [PATCH 0617/1927] drm/amdgpu: Move the mutex_lock to protect the return status of securedisplay command buffer [Why] Before we call psp_securedisplay_invoke(), we call psp_prep_securedisplay_cmd_buf() to prepare and initialize the command buffer. However, we didn't use the mutex_lock to protect the status of command buffer. So when multiple threads are using the command buffer, after thread A return from psp_securedisplay_invoke() and the command buffer status is set to SUCCESS, another thread B may call psp_prep_securedisplay_cmd_buf() and initialize the status to FAILURE again, and cause Thread A to get a failure return status. [How] Move the mutex_lock out of psp_securedisplay_invoke() to its caller to cover psp_prep_securedisplay_cmd_buf() and the code checking the return status of command buffer. Signed-off-by: Alan Liu Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 9 +++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c | 4 ++++ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index effa7df3ddbfa..7bbf869f4f0da 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -1938,10 +1938,15 @@ static int psp_securedisplay_initialize(struct psp_context *psp) } else return ret; + mutex_lock(&psp->securedisplay_context.mutex); + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, TA_SECUREDISPLAY_COMMAND__QUERY_TA); ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA); + + mutex_unlock(&psp->securedisplay_context.mutex); + if (ret) { psp_securedisplay_terminate(psp); /* free securedisplay shared memory */ @@ -1990,12 +1995,8 @@ int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id) ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC) return -EINVAL; - mutex_lock(&psp->securedisplay_context.mutex); - ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context); - mutex_unlock(&psp->securedisplay_context.mutex); - return ret; } /* SECUREDISPLAY end */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c index cc7597a15fe97..2c1d82fc4c345 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_securedisplay.c @@ -121,6 +121,7 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u switch (op) { case 1: + mutex_lock(&psp->securedisplay_context.mutex); psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, TA_SECUREDISPLAY_COMMAND__QUERY_TA); ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA); @@ -131,8 +132,10 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u else psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); } + mutex_unlock(&psp->securedisplay_context.mutex); break; case 2: + mutex_lock(&psp->securedisplay_context.mutex); psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC); securedisplay_cmd->securedisplay_in_message.send_roi_crc.phy_id = phy_id; @@ -146,6 +149,7 @@ static ssize_t amdgpu_securedisplay_debugfs_write(struct file *f, const char __u psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); } } + mutex_unlock(&psp->securedisplay_context.mutex); break; default: dev_err(adev->dev, "Invalid input: %s\n", str); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c index fb077e9af99fc..66df2394d7e4c 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c @@ -123,6 +123,8 @@ static void amdgpu_dm_crtc_notify_ta_to_read(struct work_struct *work) phy_id = crc_rd_wrk->phy_inst; spin_unlock_irq(&crc_rd_wrk->crc_rd_work_lock); + mutex_lock(&psp->securedisplay_context.mutex); + psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd, TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC); securedisplay_cmd->securedisplay_in_message.send_roi_crc.phy_id = @@ -133,6 +135,8 @@ static void amdgpu_dm_crtc_notify_ta_to_read(struct work_struct *work) psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status); } } + + mutex_unlock(&psp->securedisplay_context.mutex); } static void -- GitLab From 1d522b51e3ab3d65628a5e92c3176df10eab67d4 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 25 Oct 2022 14:42:13 -0400 Subject: [PATCH 0618/1927] drm/amdgpu: correct MES debugfs versions Use mes.sched_version, mes.kiq_version for debugfs as mes.ucode_fw_version does not contain correct versioning information. Signed-off-by: Graham Sider Reviewed-by: Jack Xiao Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index bf1ff8f0e7120..4e42dcb1950f7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -337,12 +337,14 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->feature = adev->psp.cap_feature_version; break; case AMDGPU_INFO_FW_MES_KIQ: - fw_info->ver = adev->mes.ucode_fw_version[0]; - fw_info->feature = 0; + fw_info->ver = adev->mes.kiq_version & AMDGPU_MES_VERSION_MASK; + fw_info->feature = (adev->mes.kiq_version & AMDGPU_MES_FEAT_VERSION_MASK) + >> AMDGPU_MES_FEAT_VERSION_SHIFT; break; case AMDGPU_INFO_FW_MES: - fw_info->ver = adev->mes.ucode_fw_version[1]; - fw_info->feature = 0; + fw_info->ver = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; + fw_info->feature = (adev->mes.sched_version & AMDGPU_MES_FEAT_VERSION_MASK) + >> AMDGPU_MES_FEAT_VERSION_SHIFT; break; case AMDGPU_INFO_FW_IMU: fw_info->ver = adev->gfx.imu_fw_version; -- GitLab From 6040517e4a29d3828160c571681eec9ffe10043f Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 25 Oct 2022 14:47:05 -0400 Subject: [PATCH 0619/1927] drm/amdgpu: remove deprecated MES version vars MES scheduler and kiq versions are stored in mes.sched_version and mes.kiq_version, respectively, which are read from a register after their queues are initialized. Remove mes.ucode_fw_version and mes.data_fw_version which tried to read this versioning info from the firmware headers (which don't contain this information). Signed-off-by: Graham Sider Reviewed-by: Jack Xiao Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h | 2 -- drivers/gpu/drm/amd/amdgpu/mes_v10_1.c | 4 ---- drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 4 ---- 3 files changed, 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h index ad980f4b66e19..97c05d08a551a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h @@ -91,14 +91,12 @@ struct amdgpu_mes { struct amdgpu_bo *ucode_fw_obj[AMDGPU_MAX_MES_PIPES]; uint64_t ucode_fw_gpu_addr[AMDGPU_MAX_MES_PIPES]; uint32_t *ucode_fw_ptr[AMDGPU_MAX_MES_PIPES]; - uint32_t ucode_fw_version[AMDGPU_MAX_MES_PIPES]; uint64_t uc_start_addr[AMDGPU_MAX_MES_PIPES]; /* mes ucode data */ struct amdgpu_bo *data_fw_obj[AMDGPU_MAX_MES_PIPES]; uint64_t data_fw_gpu_addr[AMDGPU_MAX_MES_PIPES]; uint32_t *data_fw_ptr[AMDGPU_MAX_MES_PIPES]; - uint32_t data_fw_version[AMDGPU_MAX_MES_PIPES]; uint64_t data_start_addr[AMDGPU_MAX_MES_PIPES]; /* eop gpu obj */ diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c index 067d10073a562..1abdf8b7ab50c 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c @@ -415,10 +415,6 @@ static int mes_v10_1_init_microcode(struct amdgpu_device *adev, mes_hdr = (const struct mes_firmware_header_v1_0 *) adev->mes.fw[pipe]->data; - adev->mes.ucode_fw_version[pipe] = - le32_to_cpu(mes_hdr->mes_ucode_version); - adev->mes.ucode_fw_version[pipe] = - le32_to_cpu(mes_hdr->mes_ucode_data_version); adev->mes.uc_start_addr[pipe] = le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index f141fadd2d86f..8ca3417fcfe53 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -485,10 +485,6 @@ static int mes_v11_0_init_microcode(struct amdgpu_device *adev, mes_hdr = (const struct mes_firmware_header_v1_0 *) adev->mes.fw[pipe]->data; - adev->mes.ucode_fw_version[pipe] = - le32_to_cpu(mes_hdr->mes_ucode_version); - adev->mes.ucode_fw_version[pipe] = - le32_to_cpu(mes_hdr->mes_ucode_data_version); adev->mes.uc_start_addr[pipe] = le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); -- GitLab From 64a9f19a5addffe585e872945c0544fed24f83a8 Mon Sep 17 00:00:00 2001 From: Ao Zhong Date: Wed, 26 Oct 2022 13:02:19 +0200 Subject: [PATCH 0620/1927] drm/amd/display: move remaining FPU code to dml folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the process of enabling DCN support for arm64, I found that the dcn10_resource_construct_fp function in dcn10/dcn10_resource.c still needs to use FPU. This will cause compilation to fail on ARM64 platforms because -mgeneral-regs-only is enabled by default to disable the hardware FPU. So move dcn10_resource_construct_fp from dcn10 folder to dml/dcn10 folder to enable hardware FPU for that function. Reviewed-by: Rodrigo Siqueira Acked-by: Christian König Signed-off-by: Ao Zhong Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/dcn10/dcn10_resource.c | 44 +------------------ .../drm/amd/display/dc/dml/dcn10/dcn10_fpu.c | 36 +++++++++++++++ .../drm/amd/display/dc/dml/dcn10/dcn10_fpu.h | 2 + 3 files changed, 40 insertions(+), 42 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c index 56d30baf12df2..6bfac8088ab0a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c @@ -1295,47 +1295,6 @@ static uint32_t read_pipe_fuses(struct dc_context *ctx) return value; } -/* - * Some architectures don't support soft-float (e.g. aarch64), on those - * this function has to be called with hardfloat enabled, make sure not - * to inline it so whatever fp stuff is done stays inside - */ -static noinline void dcn10_resource_construct_fp( - struct dc *dc) -{ - if (dc->ctx->dce_version == DCN_VERSION_1_01) { - struct dcn_soc_bounding_box *dcn_soc = dc->dcn_soc; - struct dcn_ip_params *dcn_ip = dc->dcn_ip; - struct display_mode_lib *dml = &dc->dml; - - dml->ip.max_num_dpp = 3; - /* TODO how to handle 23.84? */ - dcn_soc->dram_clock_change_latency = 23; - dcn_ip->max_num_dpp = 3; - } - if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { - dc->dcn_soc->urgent_latency = 3; - dc->debug.disable_dmcu = true; - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 41.60f; - } - - - dc->dcn_soc->number_of_channels = dc->ctx->asic_id.vram_width / ddr4_dram_width; - ASSERT(dc->dcn_soc->number_of_channels < 3); - if (dc->dcn_soc->number_of_channels == 0)/*old sbios bug*/ - dc->dcn_soc->number_of_channels = 2; - - if (dc->dcn_soc->number_of_channels == 1) { - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 19.2f; - dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = 17.066f; - dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = 14.933f; - dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 12.8f; - if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { - dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 20.80f; - } - } -} - static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks) { int i; @@ -1510,8 +1469,9 @@ static bool dcn10_resource_construct( memcpy(dc->dcn_ip, &dcn10_ip_defaults, sizeof(dcn10_ip_defaults)); memcpy(dc->dcn_soc, &dcn10_soc_defaults, sizeof(dcn10_soc_defaults)); - /* Other architectures we build for build this with soft-float */ + DC_FP_START(); dcn10_resource_construct_fp(dc); + DC_FP_END(); if (!dc->config.is_vmin_only_asic) if (ASICREV_IS_RAVEN2(dc->ctx->asic_id.hw_internal_rev)) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c index 99644d8962221..c5e84190c17a4 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.c @@ -27,6 +27,8 @@ #include "dcn10/dcn10_resource.h" #include "dcn10_fpu.h" +#include "resource.h" +#include "amdgpu_dm/dc_fpu.h" /** * DOC: DCN10 FPU manipulation Overview @@ -121,3 +123,37 @@ struct _vcs_dpi_soc_bounding_box_st dcn1_0_soc = { .writeback_dram_clock_change_latency_us = 23.0, .return_bus_width_bytes = 64, }; + +void dcn10_resource_construct_fp(struct dc *dc) +{ + dc_assert_fp_enabled(); + if (dc->ctx->dce_version == DCN_VERSION_1_01) { + struct dcn_soc_bounding_box *dcn_soc = dc->dcn_soc; + struct dcn_ip_params *dcn_ip = dc->dcn_ip; + struct display_mode_lib *dml = &dc->dml; + + dml->ip.max_num_dpp = 3; + /* TODO how to handle 23.84? */ + dcn_soc->dram_clock_change_latency = 23; + dcn_ip->max_num_dpp = 3; + } + if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) { + dc->dcn_soc->urgent_latency = 3; + dc->debug.disable_dmcu = true; + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 41.60f; + } + + dc->dcn_soc->number_of_channels = dc->ctx->asic_id.vram_width / ddr4_dram_width; + ASSERT(dc->dcn_soc->number_of_channels < 3); + if (dc->dcn_soc->number_of_channels == 0)/*old sbios bug*/ + dc->dcn_soc->number_of_channels = 2; + + if (dc->dcn_soc->number_of_channels == 1) { + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 19.2f; + dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 = 17.066f; + dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 = 14.933f; + dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 = 12.8f; + if (ASICREV_IS_RV1_F0(dc->ctx->asic_id.hw_internal_rev)) + dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 = 20.80f; + } +} diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.h index e74ed4b4ce5b8..63219ecd84789 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn10/dcn10_fpu.h @@ -27,4 +27,6 @@ #ifndef __DCN10_FPU_H__ #define __DCN10_FPU_H__ +void dcn10_resource_construct_fp(struct dc *dc); + #endif /* __DCN20_FPU_H__ */ -- GitLab From 58ddbecb14c792b7fe0d92ae5e25c9179d62ff25 Mon Sep 17 00:00:00 2001 From: Ao Zhong Date: Tue, 25 Oct 2022 23:17:49 +0200 Subject: [PATCH 0621/1927] drm/amd/display: move remaining FPU code to dml folder pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_luma = 0; pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_chroma = 0; these two operations in dcn32/dcn32_resource.c still need to use FPU, This will cause compilation to fail on ARM64 platforms because -mgeneral-regs-only is enabled by default to disable the hardware FPU. Therefore, imitate the dcn31_zero_pipe_dcc_fraction function in dml/dcn31/dcn31_fpu.c, declare the dcn32_zero_pipe_dcc_fraction function in dcn32_fpu.c, and move above two operations into this function. Reviewed-by: Rodrigo Siqueira Signed-off-by: Ao Zhong Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c | 5 +++-- drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 8 ++++++++ drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h | 3 +++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index a88dd7b3d1c10..287b7fa9bf419 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -1918,8 +1918,9 @@ int dcn32_populate_dml_pipes_from_context( timing = &pipe->stream->timing; pipes[pipe_cnt].pipe.src.gpuvm = true; - pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_luma = 0; - pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_chroma = 0; + DC_FP_START(); + dcn32_zero_pipe_dcc_fraction(pipes, pipe_cnt); + DC_FP_END(); pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch; pipes[pipe_cnt].pipe.src.gpuvm_min_page_size_kbytes = 256; // according to spreadsheet pipes[pipe_cnt].pipe.src.unbounded_req_mode = false; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index d8e68f13c2886..b6cd10f3181c3 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -2506,3 +2506,11 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa } } +void dcn32_zero_pipe_dcc_fraction(display_e2e_pipe_params_st *pipes, + int pipe_cnt) +{ + dc_assert_fp_enabled(); + + pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_luma = 0; + pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_chroma = 0; +} diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h index 3a3dc2ce4c739..ab010e7e840b8 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h @@ -73,4 +73,7 @@ int dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc, void dcn32_patch_dpm_table(struct clk_bw_params *bw_params); +void dcn32_zero_pipe_dcc_fraction(display_e2e_pipe_params_st *pipes, + int pipe_cnt); + #endif -- GitLab From d232afb1f3417ae8194ccf19ad3a8360e70e104e Mon Sep 17 00:00:00 2001 From: gehao Date: Wed, 26 Oct 2022 17:54:05 +0800 Subject: [PATCH 0622/1927] drm/amd/display: prevent memory leak In dce6(0,1,4)_create_resource_pool and dce80_create_resource_pool the allocated memory should be released if construct pool fails. Reviewed-by: Rodrigo Siqueira Signed-off-by: gehao Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c | 3 +++ drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c | 2 ++ 2 files changed, 5 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c index fc6aa098bda06..8db9f75144662 100644 --- a/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce60/dce60_resource.c @@ -1128,6 +1128,7 @@ struct resource_pool *dce60_create_resource_pool( if (dce60_construct(num_virtual_links, dc, pool)) return &pool->base; + kfree(pool); BREAK_TO_DEBUGGER(); return NULL; } @@ -1325,6 +1326,7 @@ struct resource_pool *dce61_create_resource_pool( if (dce61_construct(num_virtual_links, dc, pool)) return &pool->base; + kfree(pool); BREAK_TO_DEBUGGER(); return NULL; } @@ -1518,6 +1520,7 @@ struct resource_pool *dce64_create_resource_pool( if (dce64_construct(num_virtual_links, dc, pool)) return &pool->base; + kfree(pool); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c index b28025960050c..5825e6f412bd7 100644 --- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_resource.c @@ -1137,6 +1137,7 @@ struct resource_pool *dce80_create_resource_pool( if (dce80_construct(num_virtual_links, dc, pool)) return &pool->base; + kfree(pool); BREAK_TO_DEBUGGER(); return NULL; } @@ -1336,6 +1337,7 @@ struct resource_pool *dce81_create_resource_pool( if (dce81_construct(num_virtual_links, dc, pool)) return &pool->base; + kfree(pool); BREAK_TO_DEBUGGER(); return NULL; } -- GitLab From 9682069a03c1ff031a67d80e7a58cf2b11d7f758 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 26 Oct 2022 10:00:54 +0800 Subject: [PATCH 0623/1927] drm/amdkfd: Fix NULL pointer dereference in svm_migrate_to_ram() ./drivers/gpu/drm/amd/amdkfd/kfd_migrate.c:985:58-62: ERROR: p is NULL but dereferenced. Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=2549 Reported-by: Abaci Robot Signed-off-by: Yang Li Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index d4e6de2a6bf67..3723e90e3a904 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -973,12 +973,10 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf) out_unlock_svms: mutex_unlock(&p->svms.lock); out_unref_process: + pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr); kfd_unref_process(p); out_mmput: mmput(mm); - - pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr); - return r ? VM_FAULT_SIGBUS : 0; } -- GitLab From bf7d777289d106963fd2080d298e6b88b7263b66 Mon Sep 17 00:00:00 2001 From: Candice Li Date: Tue, 25 Oct 2022 18:07:44 +0800 Subject: [PATCH 0624/1927] drm/amdgpu: Optimize RAS TA initialization and TA unload funcs 1. Save TA unload psp response status 2. Add RAS TA loading status check for initializaiton 3. Drop RAS context teardown to allow RAS TA to be reloaded Reviewed-by: Hawking Zhang Signed-off-by: Candice Li Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 7bbf869f4f0da..b8d07e863ed81 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -988,6 +988,8 @@ int psp_ta_unload(struct psp_context *psp, struct ta_context *context) ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr); + context->resp_status = cmd->resp.status; + release_psp_cmd_buf(psp); return ret; @@ -1560,6 +1562,11 @@ static int psp_ras_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(adev)) return 0; + if (psp->ras_context.context.initialized) { + dev_warn(adev->dev, "RAS WARN: TA has already been loaded\n"); + return 0; + } + if (!adev->psp.ras_context.context.bin_desc.size_bytes || !adev->psp.ras_context.context.bin_desc.start_addr) { dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n"); @@ -1610,7 +1617,7 @@ static int psp_ras_initialize(struct psp_context *psp) psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE; psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA; - if (!psp->ras_context.context.initialized) { + if (!psp->ras_context.context.mem_context.shared_buf) { ret = psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context); if (ret) return ret; @@ -1631,7 +1638,6 @@ static int psp_ras_initialize(struct psp_context *psp) else { if (ras_cmd->ras_status) dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status); - amdgpu_ras_fini(psp->adev); } return ret; -- GitLab From 896b7addf2dc24908ad87f8ebeff83b2a83b8e04 Mon Sep 17 00:00:00 2001 From: Candice Li Date: Tue, 25 Oct 2022 18:32:15 +0800 Subject: [PATCH 0625/1927] drm/amdgpu: Optimize TA load/unload/invoke debugfs interfaces 1. Add a function pointer structure ta_funcs to psp context 2. Make the interfaces generic to all TAs 3. Leverage exisitng TA context and remove unused functions 4. Fix return code bugs v2: Add comments for ta funcs macros and correct typo Reviewed-by: Hawking Zhang Signed-off-by: Candice Li Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 38 +--- drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h | 12 +- drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c | 217 +++++++++++++++------ drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.h | 5 + 4 files changed, 168 insertions(+), 104 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index b8d07e863ed81..2fcb5bfbef89b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -1071,42 +1071,6 @@ int psp_ta_init_shared_buf(struct psp_context *psp, &mem_ctx->shared_buf); } -static void psp_prep_ta_invoke_indirect_cmd_buf(struct psp_gfx_cmd_resp *cmd, - uint32_t ta_cmd_id, - struct ta_context *context) -{ - cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD; - cmd->cmd.cmd_invoke_cmd.session_id = context->session_id; - cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id; - - cmd->cmd.cmd_invoke_cmd.buf.num_desc = 1; - cmd->cmd.cmd_invoke_cmd.buf.total_size = context->mem_context.shared_mem_size; - cmd->cmd.cmd_invoke_cmd.buf.buf_desc[0].buf_size = context->mem_context.shared_mem_size; - cmd->cmd.cmd_invoke_cmd.buf.buf_desc[0].buf_phy_addr_lo = - lower_32_bits(context->mem_context.shared_mc_addr); - cmd->cmd.cmd_invoke_cmd.buf.buf_desc[0].buf_phy_addr_hi = - upper_32_bits(context->mem_context.shared_mc_addr); -} - -int psp_ta_invoke_indirect(struct psp_context *psp, - uint32_t ta_cmd_id, - struct ta_context *context) -{ - int ret; - struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); - - psp_prep_ta_invoke_indirect_cmd_buf(cmd, ta_cmd_id, context); - - ret = psp_cmd_submit_buf(psp, NULL, cmd, - psp->fence_buf_mc_addr); - - context->resp_status = cmd->resp.status; - - release_psp_cmd_buf(psp); - - return ret; -} - static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd, uint32_t ta_cmd_id, uint32_t session_id) @@ -1549,7 +1513,7 @@ int psp_ras_terminate(struct psp_context *psp) return ret; } -static int psp_ras_initialize(struct psp_context *psp) +int psp_ras_initialize(struct psp_context *psp) { int ret; uint32_t boot_cfg = 0xFF; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 58ce3ebb446cf..cbd4194a28833 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -136,6 +136,12 @@ struct psp_funcs int (*vbflash_stat)(struct psp_context *psp); }; +struct ta_funcs { + int (*fn_ta_initialize)(struct psp_context *psp); + int (*fn_ta_invoke)(struct psp_context *psp, uint32_t ta_cmd_id); + int (*fn_ta_terminate)(struct psp_context *psp); +}; + #define AMDGPU_XGMI_MAX_CONNECTED_NODES 64 struct psp_xgmi_node_info { uint64_t node_id; @@ -309,6 +315,7 @@ struct psp_context struct psp_gfx_cmd_resp *cmd; const struct psp_funcs *funcs; + const struct ta_funcs *ta_funcs; /* firmware buffer */ struct amdgpu_bo *fw_pri_bo; @@ -463,9 +470,6 @@ int psp_ta_load(struct psp_context *psp, struct ta_context *context); int psp_ta_invoke(struct psp_context *psp, uint32_t ta_cmd_id, struct ta_context *context); -int psp_ta_invoke_indirect(struct psp_context *psp, - uint32_t ta_cmd_id, - struct ta_context *context); int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta); int psp_xgmi_terminate(struct psp_context *psp); @@ -479,7 +483,7 @@ int psp_xgmi_get_topology_info(struct psp_context *psp, int psp_xgmi_set_topology_info(struct psp_context *psp, int number_devices, struct psp_xgmi_topology_info *topology); - +int psp_ras_initialize(struct psp_context *psp); int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id); int psp_ras_enable_features(struct psp_context *psp, union ta_ras_cmd_input *info, bool enable); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c index 0988e00612e51..468a67b302d4c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.c @@ -41,30 +41,46 @@ static uint32_t get_bin_version(const uint8_t *bin) return hdr->ucode_version; } -static void prep_ta_mem_context(struct psp_context *psp, - struct ta_context *context, +static int prep_ta_mem_context(struct ta_mem_context *mem_context, uint8_t *shared_buf, uint32_t shared_buf_len) { - context->mem_context.shared_mem_size = PAGE_ALIGN(shared_buf_len); - psp_ta_init_shared_buf(psp, &context->mem_context); + if (mem_context->shared_mem_size < shared_buf_len) + return -EINVAL; + memset(mem_context->shared_buf, 0, mem_context->shared_mem_size); + memcpy((void *)mem_context->shared_buf, shared_buf, shared_buf_len); - memcpy((void *)context->mem_context.shared_buf, shared_buf, shared_buf_len); + return 0; } static bool is_ta_type_valid(enum ta_type_id ta_type) { - bool ret = false; + switch (ta_type) { + case TA_TYPE_RAS: + return true; + default: + return false; + } +} + +static const struct ta_funcs ras_ta_funcs = { + .fn_ta_initialize = psp_ras_initialize, + .fn_ta_invoke = psp_ras_invoke, + .fn_ta_terminate = psp_ras_terminate +}; +static void set_ta_context_funcs(struct psp_context *psp, + enum ta_type_id ta_type, + struct ta_context **pcontext) +{ switch (ta_type) { case TA_TYPE_RAS: - ret = true; + *pcontext = &psp->ras_context.context; + psp->ta_funcs = &ras_ta_funcs; break; default: break; } - - return ret; } static const struct file_operations ta_load_debugfs_fops = { @@ -85,8 +101,7 @@ static const struct file_operations ta_invoke_debugfs_fops = { .owner = THIS_MODULE }; - -/** +/* * DOC: AMDGPU TA debugfs interfaces * * Three debugfs interfaces can be opened by a program to @@ -111,15 +126,18 @@ static const struct file_operations ta_invoke_debugfs_fops = { * * - For TA invoke debugfs interface: * Transmit buffer: + * - TA type (4bytes) * - TA ID (4bytes) * - TA CMD ID (4bytes) - * - TA shard buf length (4bytes) + * - TA shard buf length + * (4bytes, value not beyond TA shared memory size) * - TA shared buf * Receive buffer: * - TA shared buf * * - For TA unload debugfs interface: * Transmit buffer: + * - TA type (4bytes) * - TA ID (4bytes) */ @@ -131,59 +149,92 @@ static ssize_t ta_if_load_debugfs_write(struct file *fp, const char *buf, size_t uint32_t copy_pos = 0; int ret = 0; - struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(fp)->i_private; - struct psp_context *psp = &adev->psp; - struct ta_context context = {0}; + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(fp)->i_private; + struct psp_context *psp = &adev->psp; + struct ta_context *context = NULL; if (!buf) return -EINVAL; ret = copy_from_user((void *)&ta_type, &buf[copy_pos], sizeof(uint32_t)); if (ret || (!is_ta_type_valid(ta_type))) - return -EINVAL; + return -EFAULT; copy_pos += sizeof(uint32_t); ret = copy_from_user((void *)&ta_bin_len, &buf[copy_pos], sizeof(uint32_t)); if (ret) - return -EINVAL; + return -EFAULT; copy_pos += sizeof(uint32_t); ta_bin = kzalloc(ta_bin_len, GFP_KERNEL); if (!ta_bin) - ret = -ENOMEM; + return -ENOMEM; if (copy_from_user((void *)ta_bin, &buf[copy_pos], ta_bin_len)) { ret = -EFAULT; goto err_free_bin; } - ret = psp_ras_terminate(psp); - if (ret) { - dev_err(adev->dev, "Failed to unload embedded RAS TA\n"); + /* Set TA context and functions */ + set_ta_context_funcs(psp, ta_type, &context); + + if (!psp->ta_funcs || !psp->ta_funcs->fn_ta_terminate) { + dev_err(adev->dev, "Unsupported function to terminate TA\n"); + ret = -EOPNOTSUPP; goto err_free_bin; } - context.ta_type = ta_type; - context.ta_load_type = GFX_CMD_ID_LOAD_TA; - context.bin_desc.fw_version = get_bin_version(ta_bin); - context.bin_desc.size_bytes = ta_bin_len; - context.bin_desc.start_addr = ta_bin; + /* + * Allocate TA shared buf in case shared buf was freed + * due to loading TA failed before. + */ + if (!context->mem_context.shared_buf) { + ret = psp_ta_init_shared_buf(psp, &context->mem_context); + if (ret) { + ret = -ENOMEM; + goto err_free_bin; + } + } + + ret = psp_fn_ta_terminate(psp); + if (ret || context->resp_status) { + dev_err(adev->dev, + "Failed to unload embedded TA (%d) and status (0x%X)\n", + ret, context->resp_status); + if (!ret) + ret = -EINVAL; + goto err_free_ta_shared_buf; + } + + /* Prepare TA context for TA initialization */ + context->ta_type = ta_type; + context->bin_desc.fw_version = get_bin_version(ta_bin); + context->bin_desc.size_bytes = ta_bin_len; + context->bin_desc.start_addr = ta_bin; - ret = psp_ta_load(psp, &context); + if (!psp->ta_funcs->fn_ta_initialize) { + dev_err(adev->dev, "Unsupported function to initialize TA\n"); + ret = -EOPNOTSUPP; + goto err_free_ta_shared_buf; + } - if (ret || context.resp_status) { - dev_err(adev->dev, "TA load via debugfs failed (%d) status %d\n", - ret, context.resp_status); + ret = psp_fn_ta_initialize(psp); + if (ret || context->resp_status) { + dev_err(adev->dev, "Failed to load TA via debugfs (%d) and status (0x%X)\n", + ret, context->resp_status); if (!ret) ret = -EINVAL; - goto err_free_bin; + goto err_free_ta_shared_buf; } - context.initialized = true; - if (copy_to_user((char *)buf, (void *)&context.session_id, sizeof(uint32_t))) + if (copy_to_user((char *)buf, (void *)&context->session_id, sizeof(uint32_t))) ret = -EFAULT; +err_free_ta_shared_buf: + /* Only free TA shared buf when returns error code */ + if (ret && context->mem_context.shared_buf) + psp_ta_free_shared_buf(&context->mem_context); err_free_bin: kfree(ta_bin); @@ -192,58 +243,85 @@ static ssize_t ta_if_load_debugfs_write(struct file *fp, const char *buf, size_t static ssize_t ta_if_unload_debugfs_write(struct file *fp, const char *buf, size_t len, loff_t *off) { - uint32_t ta_id = 0; - int ret = 0; + uint32_t ta_type = 0; + uint32_t ta_id = 0; + uint32_t copy_pos = 0; + int ret = 0; - struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(fp)->i_private; - struct psp_context *psp = &adev->psp; - struct ta_context context = {0}; + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(fp)->i_private; + struct psp_context *psp = &adev->psp; + struct ta_context *context = NULL; if (!buf) return -EINVAL; - ret = copy_from_user((void *)&ta_id, buf, sizeof(uint32_t)); + ret = copy_from_user((void *)&ta_type, &buf[copy_pos], sizeof(uint32_t)); + if (ret || (!is_ta_type_valid(ta_type))) + return -EFAULT; + + copy_pos += sizeof(uint32_t); + + ret = copy_from_user((void *)&ta_id, &buf[copy_pos], sizeof(uint32_t)); if (ret) - return -EINVAL; + return -EFAULT; - context.session_id = ta_id; + set_ta_context_funcs(psp, ta_type, &context); + context->session_id = ta_id; - ret = psp_ta_unload(psp, &context); - if (!ret) - context.initialized = false; + if (!psp->ta_funcs || !psp->ta_funcs->fn_ta_terminate) { + dev_err(adev->dev, "Unsupported function to terminate TA\n"); + return -EOPNOTSUPP; + } + + ret = psp_fn_ta_terminate(psp); + if (ret || context->resp_status) { + dev_err(adev->dev, "Failed to unload TA via debugfs (%d) and status (0x%X)\n", + ret, context->resp_status); + if (!ret) + ret = -EINVAL; + } + + if (context->mem_context.shared_buf) + psp_ta_free_shared_buf(&context->mem_context); return ret; } static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size_t len, loff_t *off) { + uint32_t ta_type = 0; uint32_t ta_id = 0; uint32_t cmd_id = 0; uint32_t shared_buf_len = 0; - uint8_t *shared_buf = NULL; + uint8_t *shared_buf = NULL; uint32_t copy_pos = 0; int ret = 0; - struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(fp)->i_private; - struct psp_context *psp = &adev->psp; - struct ta_context context = {0}; + struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(fp)->i_private; + struct psp_context *psp = &adev->psp; + struct ta_context *context = NULL; if (!buf) return -EINVAL; + ret = copy_from_user((void *)&ta_type, &buf[copy_pos], sizeof(uint32_t)); + if (ret) + return -EFAULT; + copy_pos += sizeof(uint32_t); + ret = copy_from_user((void *)&ta_id, &buf[copy_pos], sizeof(uint32_t)); if (ret) - return -EINVAL; + return -EFAULT; copy_pos += sizeof(uint32_t); ret = copy_from_user((void *)&cmd_id, &buf[copy_pos], sizeof(uint32_t)); if (ret) - return -EINVAL; + return -EFAULT; copy_pos += sizeof(uint32_t); ret = copy_from_user((void *)&shared_buf_len, &buf[copy_pos], sizeof(uint32_t)); if (ret) - return -EINVAL; + return -EFAULT; copy_pos += sizeof(uint32_t); shared_buf = kzalloc(shared_buf_len, GFP_KERNEL); @@ -254,26 +332,39 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size goto err_free_shared_buf; } - context.session_id = ta_id; + set_ta_context_funcs(psp, ta_type, &context); + + if (!context->initialized) { + dev_err(adev->dev, "TA is not initialized\n"); + ret = -EINVAL; + goto err_free_shared_buf; + } + + if (!psp->ta_funcs || !psp->ta_funcs->fn_ta_invoke) { + dev_err(adev->dev, "Unsupported function to invoke TA\n"); + ret = -EOPNOTSUPP; + goto err_free_shared_buf; + } - prep_ta_mem_context(psp, &context, shared_buf, shared_buf_len); + context->session_id = ta_id; - ret = psp_ta_invoke_indirect(psp, cmd_id, &context); + ret = prep_ta_mem_context(&context->mem_context, shared_buf, shared_buf_len); + if (ret) + goto err_free_shared_buf; - if (ret || context.resp_status) { - dev_err(adev->dev, "TA invoke via debugfs failed (%d) status %d\n", - ret, context.resp_status); - if (!ret) + ret = psp_fn_ta_invoke(psp, cmd_id); + if (ret || context->resp_status) { + dev_err(adev->dev, "Failed to invoke TA via debugfs (%d) and status (0x%X)\n", + ret, context->resp_status); + if (!ret) { ret = -EINVAL; - goto err_free_ta_shared_buf; + goto err_free_shared_buf; + } } - if (copy_to_user((char *)buf, context.mem_context.shared_buf, shared_buf_len)) + if (copy_to_user((char *)buf, context->mem_context.shared_buf, shared_buf_len)) ret = -EFAULT; -err_free_ta_shared_buf: - psp_ta_free_shared_buf(&context.mem_context); - err_free_shared_buf: kfree(shared_buf); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.h index cfc1542f63ef9..14cd1c81c3e6c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp_ta.h @@ -24,6 +24,11 @@ #ifndef __AMDGPU_PSP_TA_H__ #define __AMDGPU_PSP_TA_H__ +/* Calling set_ta_context_funcs is required before using the following macros */ +#define psp_fn_ta_initialize(psp) ((psp)->ta_funcs->fn_ta_initialize((psp))) +#define psp_fn_ta_invoke(psp, ta_cmd_id) ((psp)->ta_funcs->fn_ta_invoke((psp), (ta_cmd_id))) +#define psp_fn_ta_terminate(psp) ((psp)->ta_funcs->fn_ta_terminate((psp))) + void amdgpu_ta_if_debugfs_init(struct amdgpu_device *adev); #endif -- GitLab From bc22f8ec464af9e14263c3ed6a1c2be86618c804 Mon Sep 17 00:00:00 2001 From: Candice Li Date: Mon, 10 Oct 2022 16:26:37 +0800 Subject: [PATCH 0626/1927] drm/amdgpu: Update ras eeprom support for smu v13_0_0 and v13_0_10 Enable RAS EEPROM support for smu v13_0_0 and v13_0_10. Signed-off-by: Candice Li Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 84c241b9a2a13..7dc39154822c5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -90,6 +90,16 @@ static bool __is_ras_eeprom_supported(struct amdgpu_device *adev) { + if (adev->asic_type == CHIP_IP_DISCOVERY) { + switch (adev->ip_versions[MP1_HWIP][0]) { + case IP_VERSION(13, 0, 0): + case IP_VERSION(13, 0, 10): + return true; + default: + return false; + } + } + return adev->asic_type == CHIP_VEGA20 || adev->asic_type == CHIP_ARCTURUS || adev->asic_type == CHIP_SIENNA_CICHLID || -- GitLab From c9bdc6c3cf39df6db9c611d05fc512b1276b1cc8 Mon Sep 17 00:00:00 2001 From: Candice Li Date: Tue, 11 Oct 2022 15:08:38 +0800 Subject: [PATCH 0627/1927] drm/amdgpu: Add EEPROM I2C address support for ip discovery 1. Update EEPROM_I2C_MADDR_SMU_13_0_0 to EEPROM_I2C_MADDR_54H 2. Add EEPROM I2C address support for smu v13_0_0 and v13_0_10. Signed-off-by: Candice Li Reviewed-by: Tao Zhou Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 7dc39154822c5..7268ae65c140c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -38,7 +38,7 @@ #define EEPROM_I2C_MADDR_ARCTURUS_D342 0x0 #define EEPROM_I2C_MADDR_SIENNA_CICHLID 0x0 #define EEPROM_I2C_MADDR_ALDEBARAN 0x0 -#define EEPROM_I2C_MADDR_SMU_13_0_0 (0x54UL << 16) +#define EEPROM_I2C_MADDR_54H (0x54UL << 16) /* * The 2 macros bellow represent the actual size in bytes that @@ -124,6 +124,19 @@ static bool __get_eeprom_i2c_addr_arct(struct amdgpu_device *adev, return true; } +static bool __get_eeprom_i2c_addr_ip_discovery(struct amdgpu_device *adev, + struct amdgpu_ras_eeprom_control *control) +{ + switch (adev->ip_versions[MP1_HWIP][0]) { + case IP_VERSION(13, 0, 0): + case IP_VERSION(13, 0, 10): + control->i2c_address = EEPROM_I2C_MADDR_54H; + return true; + default: + return false; + } +} + static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, struct amdgpu_ras_eeprom_control *control) { @@ -163,13 +176,16 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, control->i2c_address = EEPROM_I2C_MADDR_ALDEBARAN; break; + case CHIP_IP_DISCOVERY: + return __get_eeprom_i2c_addr_ip_discovery(adev, control); + default: return false; } switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): - control->i2c_address = EEPROM_I2C_MADDR_SMU_13_0_0; + control->i2c_address = EEPROM_I2C_MADDR_54H; break; default: -- GitLab From aa193f7eff8ff753577351140b8af13b76cdc7c2 Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Fri, 21 Oct 2022 16:37:34 -0400 Subject: [PATCH 0628/1927] drm/edid: add a quirk for two LG monitors to get them to work on 10bpc The LG 27GP950 and LG 27GN950 have visible display corruption when trying to use 10bpc modes. So, to fix this, cap their maximum DSC target bitrate to 15bpp. Suggested-by: Roman Li Reviewed-by: Harry Wentland Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- drivers/gpu/drm/drm_edid.c | 12 ++++++++++++ include/drm/drm_connector.h | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c index 4005dab6147d9..b36abfa915813 100644 --- a/drivers/gpu/drm/drm_edid.c +++ b/drivers/gpu/drm/drm_edid.c @@ -87,6 +87,8 @@ static int oui(u8 first, u8 second, u8 third) #define EDID_QUIRK_FORCE_10BPC (1 << 11) /* Non desktop display (i.e. HMD) */ #define EDID_QUIRK_NON_DESKTOP (1 << 12) +/* Cap the DSC target bitrate to 15bpp */ +#define EDID_QUIRK_CAP_DSC_15BPP (1 << 13) #define MICROSOFT_IEEE_OUI 0xca125c @@ -147,6 +149,12 @@ static const struct edid_quirk { EDID_QUIRK('F', 'C', 'M', 13600, EDID_QUIRK_PREFER_LARGE_75 | EDID_QUIRK_DETAILED_IN_CM), + /* LG 27GP950 */ + EDID_QUIRK('G', 'S', 'M', 0x5bbf, EDID_QUIRK_CAP_DSC_15BPP), + + /* LG 27GN950 */ + EDID_QUIRK('G', 'S', 'M', 0x5b9a, EDID_QUIRK_CAP_DSC_15BPP), + /* LGD panel of HP zBook 17 G2, eDP 10 bpc, but reports unknown bpc */ EDID_QUIRK('L', 'G', 'D', 764, EDID_QUIRK_FORCE_10BPC), @@ -6166,6 +6174,7 @@ static void drm_reset_display_info(struct drm_connector *connector) info->mso_stream_count = 0; info->mso_pixel_overlap = 0; + info->max_dsc_bpp = 0; } static u32 update_display_info(struct drm_connector *connector, @@ -6252,6 +6261,9 @@ static u32 update_display_info(struct drm_connector *connector, info->non_desktop = true; } + if (quirks & EDID_QUIRK_CAP_DSC_15BPP) + info->max_dsc_bpp = 15; + return quirks; } diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h index 56aee949c6fa2..4d830fc55a3df 100644 --- a/include/drm/drm_connector.h +++ b/include/drm/drm_connector.h @@ -656,6 +656,12 @@ struct drm_display_info { * @mso_pixel_overlap: eDP MSO segment pixel overlap, 0-8 pixels. */ u8 mso_pixel_overlap; + + /** + * @max_dsc_bpp: Maximum DSC target bitrate, if it is set to 0 the + * monitor's default value is used instead. + */ + u32 max_dsc_bpp; }; int drm_display_info_set_bus_formats(struct drm_display_info *info, -- GitLab From 6e5abe94c6eb9b281398e39819217e8fdd1c336f Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Mon, 24 Oct 2022 14:53:24 -0400 Subject: [PATCH 0629/1927] drm/amd/display: use max_dsc_bpp in amdgpu_dm Since, the quirk is handled in the DRM core now, we can use that value instead of the internal value. Reviewed-by: Harry Wentland Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 6 ++---- .../drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c | 11 +++++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 9f2bf3488a786..94620f9b166c0 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -5607,16 +5607,14 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector, { struct drm_connector *drm_connector = &aconnector->base; uint32_t link_bandwidth_kbps; - uint32_t max_dsc_target_bpp_limit_override = 0; struct dc *dc = sink->ctx->dc; uint32_t max_supported_bw_in_kbps, timing_bw_in_kbps; uint32_t dsc_max_supported_bw_in_kbps; + uint32_t max_dsc_target_bpp_limit_override = + drm_connector->display_info.max_dsc_bpp; link_bandwidth_kbps = dc_link_bandwidth_kbps(aconnector->dc_link, dc_link_get_link_cap(aconnector->dc_link)); - if (stream->link && stream->link->local_sink) - max_dsc_target_bpp_limit_override = - stream->link->local_sink->edid_caps.panel_patch.max_dsc_target_bpp_limit; /* Set DSC policy according to dsc_clock_en */ dc_dsc_policy_set_enable_dsc_when_not_needed( diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c index 6ff96b4bdda5c..d7907974f25ad 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c @@ -642,15 +642,18 @@ static void set_dsc_configs_from_fairness_vars(struct dsc_mst_fairness_params *p int count, int k) { + struct drm_connector *drm_connector; int i; for (i = 0; i < count; i++) { + drm_connector = ¶ms[i].aconnector->base; + memset(¶ms[i].timing->dsc_cfg, 0, sizeof(params[i].timing->dsc_cfg)); if (vars[i + k].dsc_enabled && dc_dsc_compute_config( params[i].sink->ctx->dc->res_pool->dscs[0], ¶ms[i].sink->dsc_caps.dsc_dec_caps, params[i].sink->ctx->dc->debug.dsc_min_slice_height_override, - params[i].sink->edid_caps.panel_patch.max_dsc_target_bpp_limit, + drm_connector->display_info.max_dsc_bpp, 0, params[i].timing, ¶ms[i].timing->dsc_cfg)) { @@ -692,12 +695,16 @@ static int bpp_x16_from_pbn(struct dsc_mst_fairness_params param, int pbn) struct dc_dsc_config dsc_config; u64 kbps; + struct drm_connector *drm_connector = ¶m.aconnector->base; + uint32_t max_dsc_target_bpp_limit_override = + drm_connector->display_info.max_dsc_bpp; + kbps = div_u64((u64)pbn * 994 * 8 * 54, 64); dc_dsc_compute_config( param.sink->ctx->dc->res_pool->dscs[0], ¶m.sink->dsc_caps.dsc_dec_caps, param.sink->ctx->dc->debug.dsc_min_slice_height_override, - param.sink->edid_caps.panel_patch.max_dsc_target_bpp_limit, + max_dsc_target_bpp_limit_override, (int) kbps, param.timing, &dsc_config); return dsc_config.bits_per_pixel; -- GitLab From 6803dfd3a69ccb318772463a86e40929fd4fbac7 Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Tue, 25 Oct 2022 15:04:36 -0400 Subject: [PATCH 0630/1927] Revert "drm/amd/display: Limit max DSC target bpp for specific monitors" This reverts commit 55eea8ef98641f6e1e1c202bd3a49a57c1dd4059. This quirk is now handled in the DRM core, so we can drop all of the internal code that was added to handle it. Reviewed-by: Harry Wentland Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- .../amd/display/amdgpu_dm/amdgpu_dm_helpers.c | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index f0b01c8dc4a6b..f72c013d3a5b0 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -42,39 +42,6 @@ #include "dm_helpers.h" #include "ddc_service_types.h" -struct monitor_patch_info { - unsigned int manufacturer_id; - unsigned int product_id; - void (*patch_func)(struct dc_edid_caps *edid_caps, unsigned int param); - unsigned int patch_param; -}; -static void set_max_dsc_bpp_limit(struct dc_edid_caps *edid_caps, unsigned int param); - -static const struct monitor_patch_info monitor_patch_table[] = { -{0x6D1E, 0x5BBF, set_max_dsc_bpp_limit, 15}, -{0x6D1E, 0x5B9A, set_max_dsc_bpp_limit, 15}, -}; - -static void set_max_dsc_bpp_limit(struct dc_edid_caps *edid_caps, unsigned int param) -{ - if (edid_caps) - edid_caps->panel_patch.max_dsc_target_bpp_limit = param; -} - -static int amdgpu_dm_patch_edid_caps(struct dc_edid_caps *edid_caps) -{ - int i, ret = 0; - - for (i = 0; i < ARRAY_SIZE(monitor_patch_table); i++) - if ((edid_caps->manufacturer_id == monitor_patch_table[i].manufacturer_id) - && (edid_caps->product_id == monitor_patch_table[i].product_id)) { - monitor_patch_table[i].patch_func(edid_caps, monitor_patch_table[i].patch_param); - ret++; - } - - return ret; -} - /* dm_helpers_parse_edid_caps * * Parse edid caps @@ -149,8 +116,6 @@ enum dc_edid_status dm_helpers_parse_edid_caps( kfree(sads); kfree(sadb); - amdgpu_dm_patch_edid_caps(edid_caps); - return result; } -- GitLab From 7863c1552632223967f561654422cd2016c2520c Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 26 Oct 2022 14:03:55 -0500 Subject: [PATCH 0631/1927] drm/amd: Fail the suspend if resources can't be evicted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a system does not have swap and memory is under 100% usage, amdgpu will fail to evict resources. Currently the suspend carries on proceeding to reset the GPU: ``` [drm] evicting device resources failed [drm:amdgpu_device_ip_suspend_phase2 [amdgpu]] *ERROR* suspend of IP block failed -12 [drm] free PSP TMR buffer [TTM] Failed allocating page table [drm] evicting device resources failed amdgpu 0000:03:00.0: amdgpu: MODE1 reset amdgpu 0000:03:00.0: amdgpu: GPU mode1 reset amdgpu 0000:03:00.0: amdgpu: GPU smu mode1 reset ``` At this point if the suspend actually succeeded I think that amdgpu would have recovered because the GPU would have power cut off and restored. However the kernel fails to continue the suspend from the memory pressure and amdgpu fails to run the "resume" from the aborted suspend. ``` ACPI: PM: Preparing to enter system sleep state S3 SLUB: Unable to allocate memory on node -1, gfp=0xdc0(GFP_KERNEL|__GFP_ZERO) cache: Acpi-State, object size: 80, buffer size: 80, default order: 0, min order: 0 node 0: slabs: 22, objs: 1122, free: 0 ACPI Error: AE_NO_MEMORY, Could not update object reference count (20210730/utdelete-651) [drm:psp_hw_start [amdgpu]] *ERROR* PSP load kdb failed! [drm:psp_resume [amdgpu]] *ERROR* PSP resume failed [drm:amdgpu_device_fw_loading [amdgpu]] *ERROR* resume of IP block failed -62 amdgpu 0000:03:00.0: amdgpu: amdgpu_device_ip_resume failed (-62). PM: dpm_run_callback(): pci_pm_resume+0x0/0x100 returns -62 amdgpu 0000:03:00.0: PM: failed to resume async: error -62 ``` To avoid this series of unfortunate events, fail amdgpu's suspend when the memory eviction fails. This will let the system gracefully recover and the user can try suspend again when the memory pressure is relieved. Reported-by: post@davidak.de Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2223 Signed-off-by: Mario Limonciello Reviewed-by: Alex Deucher Acked-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 6f958603c8cc2..ae10acede495e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4060,15 +4060,18 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) * at suspend time. * */ -static void amdgpu_device_evict_resources(struct amdgpu_device *adev) +static int amdgpu_device_evict_resources(struct amdgpu_device *adev) { + int ret; + /* No need to evict vram on APUs for suspend to ram or s2idle */ if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) - return; + return 0; - if (amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM)) + ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); + if (ret) DRM_WARN("evicting device resources failed\n"); - + return ret; } /* @@ -4118,7 +4121,9 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) if (!adev->in_s0ix) amdgpu_amdkfd_suspend(adev, adev->in_runpm); - amdgpu_device_evict_resources(adev); + r = amdgpu_device_evict_resources(adev); + if (r) + return r; amdgpu_fence_driver_hw_fini(adev); -- GitLab From 087b8542c0b0d26066547bd2a2b08c58fd63df25 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 26 Oct 2022 15:08:24 -0400 Subject: [PATCH 0632/1927] drm/amdgpu: disable GFXOFF during compute for GFX11 Temporary workaround to fix issues observed in some compute applications when GFXOFF is enabled on GFX11. Signed-off-by: Graham Sider Acked-by: Alex Deucher Reviewed-by: Harish Kasiviswanathan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 37db39ba8718c..0d764c15f6bf8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -706,6 +706,13 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev, void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle) { + /* Temporary workaround to fix issues observed in some + * compute applications when GFXOFF is enabled on GFX11. + */ + if (IP_VERSION_MAJ(adev->ip_versions[GC_HWIP][0]) == 11) { + pr_debug("GFXOFF is %s\n", idle ? "enabled" : "disabled"); + amdgpu_gfx_off_ctrl(adev, idle); + } amdgpu_dpm_switch_power_profile(adev, PP_SMC_POWER_PROFILE_COMPUTE, !idle); -- GitLab From d69a3b762dc4c9fe257b89c2a9ae809770a81d38 Mon Sep 17 00:00:00 2001 From: Mukul Joshi Date: Mon, 9 May 2022 10:44:10 -0400 Subject: [PATCH 0633/1927] drm/amdkfd: Cleanup kfd_dev struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup kfd_dev struct by removing ddev and pdev as both drm_device and pci_dev can be fetched from amdgpu_device. Signed-off-by: Mukul Joshi Tested-by: Amber Lin Reviewed-by: Felix Kuehling Acked-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 3 +-- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 12 ++++----- drivers/gpu/drm/amd/amdkfd/kfd_device.c | 16 +++++------- drivers/gpu/drm/amd/amdkfd/kfd_iommu.c | 29 +++++++++++----------- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 +-- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 12 ++++----- drivers/gpu/drm/amd/amdkfd/kfd_svm.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 25 +++++++++---------- 9 files changed, 49 insertions(+), 56 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 0d764c15f6bf8..8816853e50c0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -195,7 +195,7 @@ void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) } adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev, - adev_to_drm(adev), &gpu_resources); + &gpu_resources); amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 647220a8762dc..f50e3ba4d7a58 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -353,7 +353,6 @@ int kgd2kfd_init(void); void kgd2kfd_exit(void); struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf); bool kgd2kfd_device_init(struct kfd_dev *kfd, - struct drm_device *ddev, const struct kgd2kfd_shared_resources *gpu_resources); void kgd2kfd_device_exit(struct kfd_dev *kfd); void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm); @@ -381,7 +380,7 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) } static inline -bool kgd2kfd_device_init(struct kfd_dev *kfd, struct drm_device *ddev, +bool kgd2kfd_device_init(struct kfd_dev *kfd, const struct kgd2kfd_shared_resources *gpu_resources) { return false; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 5e0bad7e0b774..339f60a4cdddf 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -2114,8 +2114,8 @@ static void kfd_find_numa_node_in_srat(struct kfd_dev *kdev) struct acpi_table_header *table_header = NULL; struct acpi_subtable_header *sub_header = NULL; unsigned long table_end, subtable_len; - u32 pci_id = pci_domain_nr(kdev->pdev->bus) << 16 | - pci_dev_id(kdev->pdev); + u32 pci_id = pci_domain_nr(kdev->adev->pdev->bus) << 16 | + pci_dev_id(kdev->adev->pdev); u32 bdf; acpi_status status; struct acpi_srat_cpu_affinity *cpu; @@ -2190,7 +2190,7 @@ static void kfd_find_numa_node_in_srat(struct kfd_dev *kdev) numa_node = 0; if (numa_node != NUMA_NO_NODE) - set_dev_node(&kdev->pdev->dev, numa_node); + set_dev_node(&kdev->adev->pdev->dev, numa_node); } #endif @@ -2251,14 +2251,14 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size, sub_type_hdr->proximity_domain_from = proximity_domain; #ifdef CONFIG_ACPI_NUMA - if (kdev->pdev->dev.numa_node == NUMA_NO_NODE) + if (kdev->adev->pdev->dev.numa_node == NUMA_NO_NODE) kfd_find_numa_node_in_srat(kdev); #endif #ifdef CONFIG_NUMA - if (kdev->pdev->dev.numa_node == NUMA_NO_NODE) + if (kdev->adev->pdev->dev.numa_node == NUMA_NO_NODE) sub_type_hdr->proximity_domain_to = 0; else - sub_type_hdr->proximity_domain_to = kdev->pdev->dev.numa_node; + sub_type_hdr->proximity_domain_to = kdev->adev->pdev->dev.numa_node; #else sub_type_hdr->proximity_domain_to = 0; #endif diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 65a1d4f9004ba..ee8e5f8b007d3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -227,7 +227,6 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) { struct kfd_dev *kfd = NULL; const struct kfd2kgd_calls *f2g = NULL; - struct pci_dev *pdev = adev->pdev; uint32_t gfx_target_version = 0; switch (adev->asic_type) { @@ -429,7 +428,6 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf) kfd->adev = adev; kfd_device_info_init(kfd, vf, gfx_target_version); - kfd->pdev = pdev; kfd->init_complete = false; kfd->kfd2kgd = f2g; atomic_set(&kfd->compute_profile, 0); @@ -511,12 +509,10 @@ static void kfd_smi_init(struct kfd_dev *dev) } bool kgd2kfd_device_init(struct kfd_dev *kfd, - struct drm_device *ddev, const struct kgd2kfd_shared_resources *gpu_resources) { unsigned int size, map_process_packet_size; - kfd->ddev = ddev; kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev, KGD_ENGINE_MEC1); kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev, @@ -541,7 +537,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, kfd->mec_fw_version < kfd->device_info.no_atomic_fw_version)) { dev_info(kfd_device, "skipped device %x:%x, PCI rejects atomics %d<%d\n", - kfd->pdev->vendor, kfd->pdev->device, + kfd->adev->pdev->vendor, kfd->adev->pdev->device, kfd->mec_fw_version, kfd->device_info.no_atomic_fw_version); return false; @@ -650,8 +646,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, kfd_smi_init(kfd); kfd->init_complete = true; - dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor, - kfd->pdev->device); + dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor, + kfd->adev->pdev->device); pr_debug("Starting kfd with the following scheduling policy %d\n", kfd->dqm->sched_policy); @@ -676,7 +672,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, amdgpu_amdkfd_free_gws(kfd->adev, kfd->gws); dev_err(kfd_device, "device %x:%x NOT added due to errors\n", - kfd->pdev->vendor, kfd->pdev->device); + kfd->adev->pdev->vendor, kfd->adev->pdev->device); out: return kfd->init_complete; } @@ -789,7 +785,7 @@ int kgd2kfd_resume_iommu(struct kfd_dev *kfd) if (err) dev_err(kfd_device, "Failed to resume IOMMU for device %x:%x\n", - kfd->pdev->vendor, kfd->pdev->device); + kfd->adev->pdev->vendor, kfd->adev->pdev->device); return err; } @@ -801,7 +797,7 @@ static int kfd_resume(struct kfd_dev *kfd) if (err) dev_err(kfd_device, "Error starting queue manager for device %x:%x\n", - kfd->pdev->vendor, kfd->pdev->device); + kfd->adev->pdev->vendor, kfd->adev->pdev->device); return err; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c index fbd0afe4da428..ec1bf611624ef 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_iommu.c @@ -49,7 +49,7 @@ int kfd_iommu_check_device(struct kfd_dev *kfd) return -ENODEV; iommu_info.flags = 0; - err = amd_iommu_device_info(kfd->pdev, &iommu_info); + err = amd_iommu_device_info(kfd->adev->pdev, &iommu_info); if (err) return err; @@ -71,7 +71,7 @@ int kfd_iommu_device_init(struct kfd_dev *kfd) return 0; iommu_info.flags = 0; - err = amd_iommu_device_info(kfd->pdev, &iommu_info); + err = amd_iommu_device_info(kfd->adev->pdev, &iommu_info); if (err < 0) { dev_err(kfd_device, "error getting iommu info. is the iommu enabled?\n"); @@ -121,7 +121,7 @@ int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd) return -EINVAL; } - err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread); + err = amd_iommu_bind_pasid(dev->adev->pdev, p->pasid, p->lead_thread); if (!err) pdd->bound = PDD_BOUND; @@ -139,7 +139,8 @@ void kfd_iommu_unbind_process(struct kfd_process *p) for (i = 0; i < p->n_pdds; i++) if (p->pdds[i]->bound == PDD_BOUND) - amd_iommu_unbind_pasid(p->pdds[i]->dev->pdev, p->pasid); + amd_iommu_unbind_pasid(p->pdds[i]->dev->adev->pdev, + p->pasid); } /* Callback for process shutdown invoked by the IOMMU driver */ @@ -222,7 +223,7 @@ static int kfd_bind_processes_to_device(struct kfd_dev *kfd) continue; } - err = amd_iommu_bind_pasid(kfd->pdev, p->pasid, + err = amd_iommu_bind_pasid(kfd->adev->pdev, p->pasid, p->lead_thread); if (err < 0) { pr_err("Unexpected pasid 0x%x binding failure\n", @@ -282,9 +283,9 @@ void kfd_iommu_suspend(struct kfd_dev *kfd) kfd_unbind_processes_from_device(kfd); - amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL); - amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL); - amd_iommu_free_device(kfd->pdev); + amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, NULL); + amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, NULL); + amd_iommu_free_device(kfd->adev->pdev); } /** kfd_iommu_resume - Restore IOMMU after resume @@ -302,20 +303,20 @@ int kfd_iommu_resume(struct kfd_dev *kfd) pasid_limit = kfd_get_pasid_limit(); - err = amd_iommu_init_device(kfd->pdev, pasid_limit); + err = amd_iommu_init_device(kfd->adev->pdev, pasid_limit); if (err) return -ENXIO; - amd_iommu_set_invalidate_ctx_cb(kfd->pdev, + amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, iommu_pasid_shutdown_callback); - amd_iommu_set_invalid_ppr_cb(kfd->pdev, + amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, iommu_invalid_ppr_cb); err = kfd_bind_processes_to_device(kfd); if (err) { - amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL); - amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL); - amd_iommu_free_device(kfd->pdev); + amd_iommu_set_invalidate_ctx_cb(kfd->adev->pdev, NULL); + amd_iommu_set_invalid_ppr_cb(kfd->adev->pdev, NULL); + amd_iommu_free_device(kfd->adev->pdev); return err; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index bf610e3b683bb..552c3ac85a132 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -254,8 +254,6 @@ struct kfd_dev { struct amdgpu_device *adev; struct kfd_device_info device_info; - struct pci_dev *pdev; - struct drm_device *ddev; unsigned int id; /* topology stub index */ @@ -1365,7 +1363,7 @@ void kfd_dec_compute_active(struct kfd_dev *dev); static inline int kfd_devcgroup_check_permission(struct kfd_dev *kfd) { #if defined(CONFIG_CGROUP_DEVICE) || defined(CONFIG_CGROUP_BPF) - struct drm_device *ddev = kfd->ddev; + struct drm_device *ddev = adev_to_drm(kfd->adev); return devcgroup_check_permission(DEVCG_DEV_CHAR, DRM_MAJOR, ddev->render->index, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 951b636772484..a26257171ab7c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1050,8 +1050,8 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) * for auto suspend */ if (pdd->runtime_inuse) { - pm_runtime_mark_last_busy(pdd->dev->ddev->dev); - pm_runtime_put_autosuspend(pdd->dev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev); pdd->runtime_inuse = false; } @@ -1633,9 +1633,9 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, * pdd is destroyed. */ if (!pdd->runtime_inuse) { - err = pm_runtime_get_sync(dev->ddev->dev); + err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev); if (err < 0) { - pm_runtime_put_autosuspend(dev->ddev->dev); + pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev); return ERR_PTR(err); } } @@ -1655,8 +1655,8 @@ struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, out: /* balance runpm reference count and exit with error */ if (!pdd->runtime_inuse) { - pm_runtime_mark_last_busy(dev->ddev->dev); - pm_runtime_put_autosuspend(dev->ddev->dev); + pm_runtime_mark_last_busy(adev_to_drm(dev->adev)->dev); + pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev); } return ERR_PTR(err); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index 0100812478b2e..afe7c49986768 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -259,7 +259,7 @@ void svm_range_free_dma_mappings(struct svm_range *prange) pr_debug("failed to find device idx %d\n", gpuidx); continue; } - dev = &pdd->dev->pdev->dev; + dev = &pdd->dev->adev->pdev->dev; svm_range_dma_unmap(dev, dma_addr, 0, prange->npages); kvfree(dma_addr); prange->dma_addr[gpuidx] = NULL; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 3f0a4a415907d..ec8c9c10d3488 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -115,7 +115,7 @@ struct kfd_dev *kfd_device_by_pci_dev(const struct pci_dev *pdev) down_read(&topology_lock); list_for_each_entry(top_dev, &topology_device_list, list) - if (top_dev->gpu && top_dev->gpu->pdev == pdev) { + if (top_dev->gpu && top_dev->gpu->adev->pdev == pdev) { device = top_dev->gpu; break; } @@ -1169,13 +1169,12 @@ static uint32_t kfd_generate_gpu_id(struct kfd_dev *gpu) local_mem_size = gpu->local_mem_info.local_mem_size_private + gpu->local_mem_info.local_mem_size_public; - - buf[0] = gpu->pdev->devfn; - buf[1] = gpu->pdev->subsystem_vendor | - (gpu->pdev->subsystem_device << 16); - buf[2] = pci_domain_nr(gpu->pdev->bus); - buf[3] = gpu->pdev->device; - buf[4] = gpu->pdev->bus->number; + buf[0] = gpu->adev->pdev->devfn; + buf[1] = gpu->adev->pdev->subsystem_vendor | + (gpu->adev->pdev->subsystem_device << 16); + buf[2] = pci_domain_nr(gpu->adev->pdev->bus); + buf[3] = gpu->adev->pdev->device; + buf[4] = gpu->adev->pdev->bus->number; buf[5] = lower_32_bits(local_mem_size); buf[6] = upper_32_bits(local_mem_size); @@ -1269,7 +1268,7 @@ static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev, if (target_gpu_dev) { uint32_t cap; - pcie_capability_read_dword(target_gpu_dev->gpu->pdev, + pcie_capability_read_dword(target_gpu_dev->gpu->adev->pdev, PCI_EXP_DEVCAP2, &cap); if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | @@ -1688,13 +1687,13 @@ int kfd_topology_add_device(struct kfd_dev *gpu) cu_info.num_shader_arrays_per_engine; dev->node_props.gfx_target_version = gpu->device_info.gfx_target_version; - dev->node_props.vendor_id = gpu->pdev->vendor; - dev->node_props.device_id = gpu->pdev->device; + dev->node_props.vendor_id = gpu->adev->pdev->vendor; + dev->node_props.device_id = gpu->adev->pdev->device; dev->node_props.capability |= ((dev->gpu->adev->rev_id << HSA_CAP_ASIC_REVISION_SHIFT) & HSA_CAP_ASIC_REVISION_MASK); - dev->node_props.location_id = pci_dev_id(gpu->pdev); - dev->node_props.domain = pci_domain_nr(gpu->pdev->bus); + dev->node_props.location_id = pci_dev_id(gpu->adev->pdev); + dev->node_props.domain = pci_domain_nr(gpu->adev->pdev->bus); dev->node_props.max_engine_clk_fcompute = amdgpu_amdkfd_get_max_engine_clock_in_mhz(dev->gpu->adev); dev->node_props.max_engine_clk_ccompute = -- GitLab From 5156133094dd6b57325fa107aca8be5be679452e Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 26 Oct 2022 15:01:09 +0800 Subject: [PATCH 0634/1927] drm/amd/display: make wake_up_aux_channel static This symbol is not used outside of dc_link_dp.c, so marks it static. drivers/gpu/drm/amd/amdgpu/../display/dc/core/dc_link_dp.c:5230:16: warning: no previous prototype for function 'wake_up_aux_channel'. Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=2581 Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index 420119efcf5f5..e5ab751a5ca1d 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -5227,7 +5227,7 @@ static void retrieve_cable_id(struct dc_link *link) &link->dpcd_caps.cable_id, &usbc_cable_id); } -enum dc_status wake_up_aux_channel(struct dc_link *link) +static enum dc_status wake_up_aux_channel(struct dc_link *link) { enum dc_status status = DC_ERROR_UNEXPECTED; uint32_t aux_channel_retry_cnt = 0; -- GitLab From 0645b7a64b908ef5cdbf9f117f53d283983fe285 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Wed, 26 Oct 2022 15:01:10 +0800 Subject: [PATCH 0635/1927] drm/amd/display: Modify mismatched function name No functional modification involved. drivers/gpu/drm/amd/amdgpu/../display/dc/dc_dmub_srv.c:615: warning: expecting prototype for setup_subvp_dmub_command(). Prototype was for populate_subvp_cmd_pipe_info() instead. Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=2587 Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index 774e069366617..67eef5beab951 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -595,7 +595,7 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc, } /** - * setup_subvp_dmub_command - Helper to populate the SubVP pipe info for the DMUB subvp command + * populate_subvp_cmd_pipe_info - Helper to populate the SubVP pipe info for the DMUB subvp command * * @dc: [in] current dc state * @context: [in] new dc state -- GitLab From 121affdf8a940555ceef6ab10a709030e52a4f91 Mon Sep 17 00:00:00 2001 From: Yushan Zhou Date: Tue, 18 Oct 2022 12:07:08 +0800 Subject: [PATCH 0636/1927] nfs: Remove redundant null checks before kfree Fix the following coccicheck warning: fs/nfs/dir.c:2494:2-7: WARNING: NULL check before some freeing functions is not needed. Signed-off-by: Yushan Zhou Signed-off-by: Anna Schumaker --- fs/nfs/dir.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 58036f6571268..f594dac436a7e 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2489,9 +2489,8 @@ int nfs_unlink(struct inode *dir, struct dentry *dentry) spin_unlock(&dentry->d_lock); goto out; } - if (dentry->d_fsdata) - /* old devname */ - kfree(dentry->d_fsdata); + /* old devname */ + kfree(dentry->d_fsdata); dentry->d_fsdata = NFS_FSDATA_BLOCKED; spin_unlock(&dentry->d_lock); -- GitLab From cf0d7e7f4520814f45e1313872ad5777ed504004 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Sun, 16 Oct 2022 21:36:50 -0700 Subject: [PATCH 0637/1927] NFS: Avoid memcpy() run-time warning for struct sockaddr overflows The 'nfs_server' and 'mount_server' structures include a union of 'struct sockaddr' (with the older 16 bytes max address size) and 'struct sockaddr_storage' which is large enough to hold all the supported sa_family types (128 bytes max size). The runtime memcpy() buffer overflow checker is seeing attempts to write beyond the 16 bytes as an overflow, but the actual expected size is that of 'struct sockaddr_storage'. Plumb the use of 'struct sockaddr_storage' more completely through-out NFS, which results in adjusting the memcpy() buffers to the correct union members. Avoids this false positive run-time warning under CONFIG_FORTIFY_SOURCE: memcpy: detected field-spanning write (size 28) of single field "&ctx->nfs_server.address" at fs/nfs/namespace.c:178 (size 16) Reported-by: kernel test robot Link: https://lore.kernel.org/all/202210110948.26b43120-yujie.liu@intel.com Cc: Trond Myklebust Cc: Anna Schumaker Cc: linux-nfs@vger.kernel.org Signed-off-by: Kees Cook Signed-off-by: Anna Schumaker --- fs/nfs/client.c | 4 ++-- fs/nfs/dns_resolve.c | 7 ++++--- fs/nfs/dns_resolve.h | 2 +- fs/nfs/fs_context.c | 14 +++++++------- fs/nfs/internal.h | 14 +++++++------- fs/nfs/mount_clnt.c | 4 ++-- fs/nfs/namespace.c | 2 +- fs/nfs/nfs3client.c | 4 ++-- fs/nfs/nfs4_fs.h | 2 +- fs/nfs/nfs4client.c | 18 +++++++++--------- fs/nfs/nfs4namespace.c | 16 ++++++++-------- fs/nfs/nfs4proc.c | 4 ++-- fs/nfs/pnfs_nfs.c | 6 +++--- fs/nfs/super.c | 5 ++--- 14 files changed, 51 insertions(+), 51 deletions(-) diff --git a/fs/nfs/client.c b/fs/nfs/client.c index da8da5cdbbc1f..f50e025ae4064 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -280,7 +280,7 @@ EXPORT_SYMBOL_GPL(nfs_put_client); static struct nfs_client *nfs_match_client(const struct nfs_client_initdata *data) { struct nfs_client *clp; - const struct sockaddr *sap = data->addr; + const struct sockaddr *sap = (struct sockaddr *)data->addr; struct nfs_net *nn = net_generic(data->net, nfs_net_id); int error; @@ -666,7 +666,7 @@ static int nfs_init_server(struct nfs_server *server, struct rpc_timeout timeparms; struct nfs_client_initdata cl_init = { .hostname = ctx->nfs_server.hostname, - .addr = (const struct sockaddr *)&ctx->nfs_server.address, + .addr = &ctx->nfs_server._address, .addrlen = ctx->nfs_server.addrlen, .nfs_mod = ctx->nfs_mod, .proto = ctx->nfs_server.protocol, diff --git a/fs/nfs/dns_resolve.c b/fs/nfs/dns_resolve.c index e87d500ad95aa..6603b5cee029c 100644 --- a/fs/nfs/dns_resolve.c +++ b/fs/nfs/dns_resolve.c @@ -16,8 +16,9 @@ #include "dns_resolve.h" ssize_t nfs_dns_resolve_name(struct net *net, char *name, size_t namelen, - struct sockaddr *sa, size_t salen) + struct sockaddr_storage *ss, size_t salen) { + struct sockaddr *sa = (struct sockaddr *)ss; ssize_t ret; char *ip_addr = NULL; int ip_len; @@ -341,7 +342,7 @@ static int do_cache_lookup_wait(struct cache_detail *cd, } ssize_t nfs_dns_resolve_name(struct net *net, char *name, - size_t namelen, struct sockaddr *sa, size_t salen) + size_t namelen, struct sockaddr_storage *ss, size_t salen) { struct nfs_dns_ent key = { .hostname = name, @@ -354,7 +355,7 @@ ssize_t nfs_dns_resolve_name(struct net *net, char *name, ret = do_cache_lookup_wait(nn->nfs_dns_resolve, &key, &item); if (ret == 0) { if (salen >= item->addrlen) { - memcpy(sa, &item->addr, item->addrlen); + memcpy(ss, &item->addr, item->addrlen); ret = item->addrlen; } else ret = -EOVERFLOW; diff --git a/fs/nfs/dns_resolve.h b/fs/nfs/dns_resolve.h index 576ff4b54c82a..fe3b172c4de1d 100644 --- a/fs/nfs/dns_resolve.h +++ b/fs/nfs/dns_resolve.h @@ -32,6 +32,6 @@ extern void nfs_dns_resolver_cache_destroy(struct net *net); #endif extern ssize_t nfs_dns_resolve_name(struct net *net, char *name, - size_t namelen, struct sockaddr *sa, size_t salen); + size_t namelen, struct sockaddr_storage *sa, size_t salen); #endif diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 4da701fd1424f..09833ec102fca 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -273,9 +273,9 @@ static const struct constant_table nfs_secflavor_tokens[] = { * Address family must be initialized, and address must not be * the ANY address for that family. */ -static int nfs_verify_server_address(struct sockaddr *addr) +static int nfs_verify_server_address(struct sockaddr_storage *addr) { - switch (addr->sa_family) { + switch (addr->ss_family) { case AF_INET: { struct sockaddr_in *sa = (struct sockaddr_in *)addr; return sa->sin_addr.s_addr != htonl(INADDR_ANY); @@ -969,7 +969,7 @@ static int nfs23_parse_monolithic(struct fs_context *fc, { struct nfs_fs_context *ctx = nfs_fc2context(fc); struct nfs_fh *mntfh = ctx->mntfh; - struct sockaddr *sap = (struct sockaddr *)&ctx->nfs_server.address; + struct sockaddr_storage *sap = &ctx->nfs_server._address; int extra_flags = NFS_MOUNT_LEGACY_INTERFACE; int ret; @@ -1044,7 +1044,7 @@ static int nfs23_parse_monolithic(struct fs_context *fc, memcpy(sap, &data->addr, sizeof(data->addr)); ctx->nfs_server.addrlen = sizeof(data->addr); ctx->nfs_server.port = ntohs(data->addr.sin_port); - if (sap->sa_family != AF_INET || + if (sap->ss_family != AF_INET || !nfs_verify_server_address(sap)) goto out_no_address; @@ -1200,7 +1200,7 @@ static int nfs4_parse_monolithic(struct fs_context *fc, struct nfs4_mount_data *data) { struct nfs_fs_context *ctx = nfs_fc2context(fc); - struct sockaddr *sap = (struct sockaddr *)&ctx->nfs_server.address; + struct sockaddr_storage *sap = &ctx->nfs_server._address; int ret; char *c; @@ -1314,7 +1314,7 @@ static int nfs_fs_context_validate(struct fs_context *fc) { struct nfs_fs_context *ctx = nfs_fc2context(fc); struct nfs_subversion *nfs_mod; - struct sockaddr *sap = (struct sockaddr *)&ctx->nfs_server.address; + struct sockaddr_storage *sap = &ctx->nfs_server._address; int max_namelen = PAGE_SIZE; int max_pathlen = NFS_MAXPATHLEN; int port = 0; @@ -1540,7 +1540,7 @@ static int nfs_init_fs_context(struct fs_context *fc) ctx->version = nfss->nfs_client->rpc_ops->version; ctx->minorversion = nfss->nfs_client->cl_minorversion; - memcpy(&ctx->nfs_server.address, &nfss->nfs_client->cl_addr, + memcpy(&ctx->nfs_server._address, &nfss->nfs_client->cl_addr, ctx->nfs_server.addrlen); if (fc->net_ns != net) { diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index d914d609b85b2..647fc3f547cbe 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -69,7 +69,7 @@ static inline fmode_t flags_to_mode(int flags) struct nfs_client_initdata { unsigned long init_flags; const char *hostname; /* Hostname of the server */ - const struct sockaddr *addr; /* Address of the server */ + const struct sockaddr_storage *addr; /* Address of the server */ const char *nodename; /* Hostname of the client */ const char *ip_addr; /* IP address of the client */ size_t addrlen; @@ -180,7 +180,7 @@ static inline struct nfs_fs_context *nfs_fc2context(const struct fs_context *fc) /* mount_clnt.c */ struct nfs_mount_request { - struct sockaddr *sap; + struct sockaddr_storage *sap; size_t salen; char *hostname; char *dirpath; @@ -223,7 +223,7 @@ extern void nfs4_server_set_init_caps(struct nfs_server *); extern struct nfs_server *nfs4_create_server(struct fs_context *); extern struct nfs_server *nfs4_create_referral_server(struct fs_context *); extern int nfs4_update_server(struct nfs_server *server, const char *hostname, - struct sockaddr *sap, size_t salen, + struct sockaddr_storage *sap, size_t salen, struct net *net); extern void nfs_free_server(struct nfs_server *server); extern struct nfs_server *nfs_clone_server(struct nfs_server *, @@ -235,7 +235,7 @@ extern int nfs_client_init_status(const struct nfs_client *clp); extern int nfs_wait_client_init_complete(const struct nfs_client *clp); extern void nfs_mark_client_ready(struct nfs_client *clp, int state); extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, - const struct sockaddr *ds_addr, + const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans, @@ -243,7 +243,7 @@ extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *, struct inode *); extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, - const struct sockaddr *ds_addr, int ds_addrlen, + const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans); #ifdef CONFIG_PROC_FS @@ -894,13 +894,13 @@ static inline bool nfs_error_is_fatal_on_server(int err) * Select between a default port value and a user-specified port value. * If a zero value is set, then autobind will be used. */ -static inline void nfs_set_port(struct sockaddr *sap, int *port, +static inline void nfs_set_port(struct sockaddr_storage *sap, int *port, const unsigned short default_port) { if (*port == NFS_UNSPEC_PORT) *port = default_port; - rpc_set_port(sap, *port); + rpc_set_port((struct sockaddr *)sap, *port); } struct nfs_direct_req { diff --git a/fs/nfs/mount_clnt.c b/fs/nfs/mount_clnt.c index c5e3b6b3366a6..68e76b6263710 100644 --- a/fs/nfs/mount_clnt.c +++ b/fs/nfs/mount_clnt.c @@ -158,7 +158,7 @@ int nfs_mount(struct nfs_mount_request *info, int timeo, int retrans) struct rpc_create_args args = { .net = info->net, .protocol = info->protocol, - .address = info->sap, + .address = (struct sockaddr *)info->sap, .addrsize = info->salen, .timeout = &mnt_timeout, .servername = info->hostname, @@ -245,7 +245,7 @@ void nfs_umount(const struct nfs_mount_request *info) struct rpc_create_args args = { .net = info->net, .protocol = IPPROTO_UDP, - .address = info->sap, + .address = (struct sockaddr *)info->sap, .addrsize = info->salen, .timeout = &nfs_umnt_timeout, .servername = info->hostname, diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c index 3295af4110f1b..2f336ace75554 100644 --- a/fs/nfs/namespace.c +++ b/fs/nfs/namespace.c @@ -175,7 +175,7 @@ struct vfsmount *nfs_d_automount(struct path *path) } /* for submounts we want the same server; referrals will reassign */ - memcpy(&ctx->nfs_server.address, &client->cl_addr, client->cl_addrlen); + memcpy(&ctx->nfs_server._address, &client->cl_addr, client->cl_addrlen); ctx->nfs_server.addrlen = client->cl_addrlen; ctx->nfs_server.port = server->port; diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c index b49359afac883..669cda757a5ce 100644 --- a/fs/nfs/nfs3client.c +++ b/fs/nfs/nfs3client.c @@ -78,7 +78,7 @@ struct nfs_server *nfs3_clone_server(struct nfs_server *source, * the MDS. */ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, - const struct sockaddr *ds_addr, int ds_addrlen, + const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans) { struct rpc_timeout ds_timeout; @@ -98,7 +98,7 @@ struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, char buf[INET6_ADDRSTRLEN + 1]; /* fake a hostname because lockd wants it */ - if (rpc_ntop(ds_addr, buf, sizeof(buf)) <= 0) + if (rpc_ntop((struct sockaddr *)ds_addr, buf, sizeof(buf)) <= 0) return ERR_PTR(-EINVAL); cl_init.hostname = buf; diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index 400a71e75238b..cfef738d765e0 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -281,7 +281,7 @@ struct rpc_clnt *nfs4_negotiate_security(struct rpc_clnt *, struct inode *, int nfs4_submount(struct fs_context *, struct nfs_server *); int nfs4_replace_transport(struct nfs_server *server, const struct nfs4_fs_locations *locations); -size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr *sa, +size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr_storage *ss, size_t salen, struct net *net, int port); /* nfs4proc.c */ extern int nfs4_handle_exception(struct nfs_server *, int, struct nfs4_exception *); diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 7a5162afa5c0d..3b9b534813095 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -889,7 +889,7 @@ nfs4_find_client_sessionid(struct net *net, const struct sockaddr *addr, */ static int nfs4_set_client(struct nfs_server *server, const char *hostname, - const struct sockaddr *addr, + const struct sockaddr_storage *addr, const size_t addrlen, const char *ip_addr, int proto, const struct rpc_timeout *timeparms, @@ -924,7 +924,7 @@ static int nfs4_set_client(struct nfs_server *server, __set_bit(NFS_CS_MIGRATION, &cl_init.init_flags); if (test_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status)) __set_bit(NFS_CS_TSM_POSSIBLE, &cl_init.init_flags); - server->port = rpc_get_port(addr); + server->port = rpc_get_port((struct sockaddr *)addr); /* Allocate or find a client reference we can use */ clp = nfs_get_client(&cl_init); @@ -960,7 +960,7 @@ static int nfs4_set_client(struct nfs_server *server, * the MDS. */ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, - const struct sockaddr *ds_addr, int ds_addrlen, + const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans, u32 minor_version) { @@ -980,7 +980,7 @@ struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, }; char buf[INET6_ADDRSTRLEN + 1]; - if (rpc_ntop(ds_addr, buf, sizeof(buf)) <= 0) + if (rpc_ntop((struct sockaddr *)ds_addr, buf, sizeof(buf)) <= 0) return ERR_PTR(-EINVAL); cl_init.hostname = buf; @@ -1148,7 +1148,7 @@ static int nfs4_init_server(struct nfs_server *server, struct fs_context *fc) /* Get a client record */ error = nfs4_set_client(server, ctx->nfs_server.hostname, - &ctx->nfs_server.address, + &ctx->nfs_server._address, ctx->nfs_server.addrlen, ctx->client_address, ctx->nfs_server.protocol, @@ -1238,7 +1238,7 @@ struct nfs_server *nfs4_create_referral_server(struct fs_context *fc) rpc_set_port(&ctx->nfs_server.address, NFS_RDMA_PORT); error = nfs4_set_client(server, ctx->nfs_server.hostname, - &ctx->nfs_server.address, + &ctx->nfs_server._address, ctx->nfs_server.addrlen, parent_client->cl_ipaddr, XPRT_TRANSPORT_RDMA, @@ -1254,7 +1254,7 @@ struct nfs_server *nfs4_create_referral_server(struct fs_context *fc) rpc_set_port(&ctx->nfs_server.address, NFS_PORT); error = nfs4_set_client(server, ctx->nfs_server.hostname, - &ctx->nfs_server.address, + &ctx->nfs_server._address, ctx->nfs_server.addrlen, parent_client->cl_ipaddr, XPRT_TRANSPORT_TCP, @@ -1303,14 +1303,14 @@ struct nfs_server *nfs4_create_referral_server(struct fs_context *fc) * Returns zero on success, or a negative errno value. */ int nfs4_update_server(struct nfs_server *server, const char *hostname, - struct sockaddr *sap, size_t salen, struct net *net) + struct sockaddr_storage *sap, size_t salen, struct net *net) { struct nfs_client *clp = server->nfs_client; struct rpc_clnt *clnt = server->client; struct xprt_create xargs = { .ident = clp->cl_proto, .net = net, - .dstaddr = sap, + .dstaddr = (struct sockaddr *)sap, .addrlen = salen, .servername = hostname, }; diff --git a/fs/nfs/nfs4namespace.c b/fs/nfs/nfs4namespace.c index f2dbf904c5989..9a98595bb1604 100644 --- a/fs/nfs/nfs4namespace.c +++ b/fs/nfs/nfs4namespace.c @@ -164,16 +164,17 @@ static int nfs4_validate_fspath(struct dentry *dentry, return 0; } -size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr *sa, +size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr_storage *ss, size_t salen, struct net *net, int port) { + struct sockaddr *sa = (struct sockaddr *)ss; ssize_t ret; ret = rpc_pton(net, string, len, sa, salen); if (ret == 0) { ret = rpc_uaddr2sockaddr(net, string, len, sa, salen); if (ret == 0) { - ret = nfs_dns_resolve_name(net, string, len, sa, salen); + ret = nfs_dns_resolve_name(net, string, len, ss, salen); if (ret < 0) ret = 0; } @@ -331,7 +332,7 @@ static int try_location(struct fs_context *fc, ctx->nfs_server.addrlen = nfs_parse_server_name(buf->data, buf->len, - &ctx->nfs_server.address, + &ctx->nfs_server._address, sizeof(ctx->nfs_server._address), fc->net_ns, 0); if (ctx->nfs_server.addrlen == 0) @@ -483,14 +484,13 @@ static int nfs4_try_replacing_one_location(struct nfs_server *server, char *page, char *page2, const struct nfs4_fs_location *location) { - const size_t addr_bufsize = sizeof(struct sockaddr_storage); struct net *net = rpc_net_ns(server->client); - struct sockaddr *sap; + struct sockaddr_storage *sap; unsigned int s; size_t salen; int error; - sap = kmalloc(addr_bufsize, GFP_KERNEL); + sap = kmalloc(sizeof(*sap), GFP_KERNEL); if (sap == NULL) return -ENOMEM; @@ -506,10 +506,10 @@ static int nfs4_try_replacing_one_location(struct nfs_server *server, continue; salen = nfs_parse_server_name(buf->data, buf->len, - sap, addr_bufsize, net, 0); + sap, sizeof(*sap), net, 0); if (salen == 0) continue; - rpc_set_port(sap, NFS_PORT); + rpc_set_port((struct sockaddr *)sap, NFS_PORT); error = -ENOMEM; hostname = kmemdup_nul(buf->data, buf->len, GFP_KERNEL); diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index e2efcd26336c0..4c4df7f213b6c 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -3951,7 +3951,7 @@ static void test_fs_location_for_trunking(struct nfs4_fs_location *location, for (i = 0; i < location->nservers; i++) { struct nfs4_string *srv_loc = &location->servers[i]; - struct sockaddr addr; + struct sockaddr_storage addr; size_t addrlen; struct xprt_create xprt_args = { .ident = 0, @@ -3974,7 +3974,7 @@ static void test_fs_location_for_trunking(struct nfs4_fs_location *location, clp->cl_net, server->port); if (!addrlen) return; - xprt_args.dstaddr = &addr; + xprt_args.dstaddr = (struct sockaddr *)&addr; xprt_args.addrlen = addrlen; servername = kmalloc(srv_loc->len + 1, GFP_KERNEL); if (!servername) diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 987c88ddeaf06..5d035dd2d7bf0 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -821,7 +821,7 @@ static void nfs4_clear_ds_conn_bit(struct nfs4_pnfs_ds *ds) static struct nfs_client *(*get_v3_ds_connect)( struct nfs_server *mds_srv, - const struct sockaddr *ds_addr, + const struct sockaddr_storage *ds_addr, int ds_addrlen, int ds_proto, unsigned int ds_timeo, @@ -882,7 +882,7 @@ static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv, continue; } clp = get_v3_ds_connect(mds_srv, - (struct sockaddr *)&da->da_addr, + &da->da_addr, da->da_addrlen, da->da_transport, timeo, retrans); if (IS_ERR(clp)) @@ -951,7 +951,7 @@ static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, put_cred(xprtdata.cred); } else { clp = nfs4_set_ds_client(mds_srv, - (struct sockaddr *)&da->da_addr, + &da->da_addr, da->da_addrlen, da->da_transport, timeo, retrans, minor_version); diff --git a/fs/nfs/super.c b/fs/nfs/super.c index ee66ffdb985e8..05ae23657527d 100644 --- a/fs/nfs/super.c +++ b/fs/nfs/super.c @@ -822,8 +822,7 @@ static int nfs_request_mount(struct fs_context *fc, { struct nfs_fs_context *ctx = nfs_fc2context(fc); struct nfs_mount_request request = { - .sap = (struct sockaddr *) - &ctx->mount_server.address, + .sap = &ctx->mount_server._address, .dirpath = ctx->nfs_server.export_path, .protocol = ctx->mount_server.protocol, .fh = root_fh, @@ -854,7 +853,7 @@ static int nfs_request_mount(struct fs_context *fc, * Construct the mount server's address. */ if (ctx->mount_server.address.sa_family == AF_UNSPEC) { - memcpy(request.sap, &ctx->nfs_server.address, + memcpy(request.sap, &ctx->nfs_server._address, ctx->nfs_server.addrlen); ctx->mount_server.addrlen = ctx->nfs_server.addrlen; } -- GitLab From 1ba04394e028ea8b45d92685cc0d6ab582cf7647 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 16 Oct 2022 14:44:31 -0400 Subject: [PATCH 0638/1927] NFSv4: Fix a potential state reclaim deadlock If the server reboots while we are engaged in a delegation return, and there is a pNFS layout with return-on-close set, then the current code can end up deadlocking in pnfs_roc() when nfs_inode_set_delegation() tries to return the old delegation. Now that delegreturn actually uses its own copy of the stateid, it should be safe to just always update the delegation stateid in place. Fixes: 078000d02d57 ("pNFS: We want return-on-close to complete when evicting the inode") Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/delegation.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 5c97cad741a73..ead8a0e06abf9 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -228,8 +228,7 @@ static int nfs_delegation_claim_opens(struct inode *inode, * */ void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, - fmode_t type, - const nfs4_stateid *stateid, + fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit) { struct nfs_delegation *delegation; @@ -239,25 +238,24 @@ void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, delegation = rcu_dereference(NFS_I(inode)->delegation); if (delegation != NULL) { spin_lock(&delegation->lock); - if (nfs4_is_valid_delegation(delegation, 0)) { - nfs4_stateid_copy(&delegation->stateid, stateid); - delegation->type = type; - delegation->pagemod_limit = pagemod_limit; - oldcred = delegation->cred; - delegation->cred = get_cred(cred); - clear_bit(NFS_DELEGATION_NEED_RECLAIM, - &delegation->flags); - spin_unlock(&delegation->lock); - rcu_read_unlock(); - put_cred(oldcred); - trace_nfs4_reclaim_delegation(inode, type); - return; - } - /* We appear to have raced with a delegation return. */ + nfs4_stateid_copy(&delegation->stateid, stateid); + delegation->type = type; + delegation->pagemod_limit = pagemod_limit; + oldcred = delegation->cred; + delegation->cred = get_cred(cred); + clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); + if (test_and_clear_bit(NFS_DELEGATION_REVOKED, + &delegation->flags)) + atomic_long_inc(&nfs_active_delegations); spin_unlock(&delegation->lock); + rcu_read_unlock(); + put_cred(oldcred); + trace_nfs4_reclaim_delegation(inode, type); + } else { + rcu_read_unlock(); + nfs_inode_set_delegation(inode, cred, type, stateid, + pagemod_limit); } - rcu_read_unlock(); - nfs_inode_set_delegation(inode, cred, type, stateid, pagemod_limit); } static int nfs_do_return_delegation(struct inode *inode, struct nfs_delegation *delegation, int issync) -- GitLab From 5d917cba3201e5c25059df96c29252fd99c4f6a7 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 16 Oct 2022 14:44:32 -0400 Subject: [PATCH 0639/1927] NFSv4.1: Handle RECLAIM_COMPLETE trunking errors If RECLAIM_COMPLETE sets the NFS4CLNT_BIND_CONN_TO_SESSION flag, then we need to loop back in order to handle it. Fixes: 0048fdd06614 ("NFSv4.1: RECLAIM_COMPLETE must handle NFS4ERR_CONN_NOT_BOUND_TO_SESSION") Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/nfs4state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index c3503fb26fa27..acdc74baec1fa 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -2670,6 +2670,7 @@ static void nfs4_state_manager(struct nfs_client *clp) if (status < 0) goto out_error; nfs4_state_end_reclaim_reboot(clp); + continue; } /* Detect expired delegations... */ -- GitLab From e59679f2b7e522ecad99974e5636291ffd47c184 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sun, 16 Oct 2022 14:44:33 -0400 Subject: [PATCH 0640/1927] NFSv4.1: We must always send RECLAIM_COMPLETE after a reboot Currently, we are only guaranteed to send RECLAIM_COMPLETE if we have open state to recover. Fix the client to always send RECLAIM_COMPLETE after setting up the lease. Fixes: fce5c838e133 ("nfs41: RECLAIM_COMPLETE functionality") Signed-off-by: Trond Myklebust Signed-off-by: Anna Schumaker --- fs/nfs/nfs4state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c index acdc74baec1fa..a2d2d5d1b0888 100644 --- a/fs/nfs/nfs4state.c +++ b/fs/nfs/nfs4state.c @@ -1786,6 +1786,7 @@ static void nfs4_state_mark_reclaim_helper(struct nfs_client *clp, static void nfs4_state_start_reclaim_reboot(struct nfs_client *clp) { + set_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state); /* Mark all delegations for reclaim */ nfs_delegation_mark_reclaim(clp); nfs4_state_mark_reclaim_helper(clp, nfs4_state_mark_reclaim_reboot); -- GitLab From cbdeaee94a415800c65a8c3fa04d9664a8b8fb3a Mon Sep 17 00:00:00 2001 From: Zhang Xiaoxu Date: Thu, 20 Oct 2022 11:42:17 +0800 Subject: [PATCH 0641/1927] SUNRPC: Fix null-ptr-deref when xps sysfs alloc failed There is a null-ptr-deref when xps sysfs alloc failed: BUG: KASAN: null-ptr-deref in sysfs_do_create_link_sd+0x40/0xd0 Read of size 8 at addr 0000000000000030 by task gssproxy/457 CPU: 5 PID: 457 Comm: gssproxy Not tainted 6.0.0-09040-g02357b27ee03 #9 Call Trace: dump_stack_lvl+0x34/0x44 kasan_report+0xa3/0x120 sysfs_do_create_link_sd+0x40/0xd0 rpc_sysfs_client_setup+0x161/0x1b0 rpc_new_client+0x3fc/0x6e0 rpc_create_xprt+0x71/0x220 rpc_create+0x1d4/0x350 gssp_rpc_create+0xc3/0x160 set_gssp_clnt+0xbc/0x140 write_gssp+0x116/0x1a0 proc_reg_write+0xd6/0x130 vfs_write+0x177/0x690 ksys_write+0xb9/0x150 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 When the xprt_switch sysfs alloc failed, should not add xprt and switch sysfs to it, otherwise, maybe null-ptr-deref; also initialize the 'xps_sysfs' to NULL to avoid oops when destroy it. Fixes: 2a338a543163 ("sunrpc: add a symlink from rpc-client directory to the xprt_switch") Fixes: d408ebe04ac5 ("sunrpc: add add sysfs directory per xprt under each xprt_switch") Fixes: baea99445dd4 ("sunrpc: add xprt_switch direcotry to sunrpc's sysfs") Signed-off-by: Zhang Xiaoxu Signed-off-by: Anna Schumaker --- net/sunrpc/sysfs.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/net/sunrpc/sysfs.c b/net/sunrpc/sysfs.c index c65c90ad626ad..c1f559892ae8a 100644 --- a/net/sunrpc/sysfs.c +++ b/net/sunrpc/sysfs.c @@ -518,13 +518,16 @@ void rpc_sysfs_client_setup(struct rpc_clnt *clnt, struct net *net) { struct rpc_sysfs_client *rpc_client; + struct rpc_sysfs_xprt_switch *xswitch = + (struct rpc_sysfs_xprt_switch *)xprt_switch->xps_sysfs; + + if (!xswitch) + return; rpc_client = rpc_sysfs_client_alloc(rpc_sunrpc_client_kobj, net, clnt->cl_clid); if (rpc_client) { char name[] = "switch"; - struct rpc_sysfs_xprt_switch *xswitch = - (struct rpc_sysfs_xprt_switch *)xprt_switch->xps_sysfs; int ret; clnt->cl_sysfs = rpc_client; @@ -558,6 +561,8 @@ void rpc_sysfs_xprt_switch_setup(struct rpc_xprt_switch *xprt_switch, rpc_xprt_switch->xprt_switch = xprt_switch; rpc_xprt_switch->xprt = xprt; kobject_uevent(&rpc_xprt_switch->kobject, KOBJ_ADD); + } else { + xprt_switch->xps_sysfs = NULL; } } @@ -569,6 +574,9 @@ void rpc_sysfs_xprt_setup(struct rpc_xprt_switch *xprt_switch, struct rpc_sysfs_xprt_switch *switch_obj = (struct rpc_sysfs_xprt_switch *)xprt_switch->xps_sysfs; + if (!switch_obj) + return; + rpc_xprt = rpc_sysfs_xprt_alloc(&switch_obj->kobject, xprt, gfp_flags); if (rpc_xprt) { xprt->xprt_sysfs = rpc_xprt; -- GitLab From f5ea16137a3fa2858620dc9084466491c128535f Mon Sep 17 00:00:00 2001 From: Benjamin Coddington Date: Wed, 19 Oct 2022 12:09:18 -0400 Subject: [PATCH 0642/1927] NFSv4: Retry LOCK on OLD_STATEID during delegation return There's a small window where a LOCK sent during a delegation return can race with another OPEN on client, but the open stateid has not yet been updated. In this case, the client doesn't handle the OLD_STATEID error from the server and will lose this lock, emitting: "NFS: nfs4_handle_delegation_recall_error: unhandled error -10024". Fix this by sending the task through the nfs4 error handling in nfs4_lock_done() when we may have to reconcile our stateid with what the server believes it to be. For this case, the result is a retry of the LOCK operation with the updated stateid. Reported-by: Gonzalo Siero Humet Signed-off-by: Benjamin Coddington Signed-off-by: Anna Schumaker --- fs/nfs/nfs4proc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 4c4df7f213b6c..86ed5c0142c3d 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7138,6 +7138,7 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata) { struct nfs4_lockdata *data = calldata; struct nfs4_lock_state *lsp = data->lsp; + struct nfs_server *server = NFS_SERVER(d_inode(data->ctx->dentry)); if (!nfs4_sequence_done(task, &data->res.seq_res)) return; @@ -7145,8 +7146,7 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata) data->rpc_status = task->tk_status; switch (task->tk_status) { case 0: - renew_lease(NFS_SERVER(d_inode(data->ctx->dentry)), - data->timestamp); + renew_lease(server, data->timestamp); if (data->arg.new_lock && !data->cancelled) { data->fl.fl_flags &= ~(FL_SLEEP | FL_ACCESS); if (locks_lock_inode_wait(lsp->ls_state->inode, &data->fl) < 0) @@ -7167,6 +7167,8 @@ static void nfs4_lock_done(struct rpc_task *task, void *calldata) if (!nfs4_stateid_match(&data->arg.open_stateid, &lsp->ls_state->open_stateid)) goto out_restart; + else if (nfs4_async_handle_error(task, server, lsp->ls_state, NULL) == -EAGAIN) + goto out_restart; } else if (!nfs4_stateid_match(&data->arg.lock_stateid, &lsp->ls_stateid)) goto out_restart; -- GitLab From 8a0fa3ff3b606b55c4edc71ad133e61529b64549 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 8 Oct 2022 14:58:29 -0400 Subject: [PATCH 0643/1927] SUNRPC: Fix crasher in gss_unwrap_resp_integ() If a zero length is passed to kmalloc() it returns 0x10, which is not a valid address. gss_unwrap_resp_integ() subsequently crashes when it attempts to dereference that pointer. Signed-off-by: Chuck Lever Signed-off-by: Anna Schumaker --- net/sunrpc/auth_gss/auth_gss.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index a31a27816cc0f..7bb247c51e2f6 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -1989,7 +1989,7 @@ gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred, goto unwrap_failed; mic.len = len; mic.data = kmalloc(len, GFP_KERNEL); - if (!mic.data) + if (ZERO_OR_NULL_PTR(mic.data)) goto unwrap_failed; if (read_bytes_from_xdr_buf(rcv_buf, offset, mic.data, mic.len)) goto unwrap_failed; -- GitLab From 038efb6348ce96228f6828354cb809c22a661681 Mon Sep 17 00:00:00 2001 From: Benjamin Coddington Date: Thu, 13 Oct 2022 11:58:01 -0400 Subject: [PATCH 0644/1927] NFSv4.2: Fixup CLONE dest file size for zero-length count When holding a delegation, the NFS client optimizes away setting the attributes of a file from the GETATTR in the compound after CLONE, and for a zero-length CLONE we will end up setting the inode's size to zero in nfs42_copy_dest_done(). Handle this case by computing the resulting count from the server's reported size after CLONE's GETATTR. Suggested-by: Trond Myklebust Signed-off-by: Benjamin Coddington Fixes: 94d202d5ca39 ("NFSv42: Copy offload should update the file size when appropriate") Signed-off-by: Anna Schumaker --- fs/nfs/nfs42proc.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index 13424f0d793b2..ecb428512fe1a 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -1093,6 +1093,9 @@ static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f, &args.seq_args, &res.seq_res, 0); trace_nfs4_clone(src_inode, dst_inode, &args, status); if (status == 0) { + /* a zero-length count means clone to EOF in src */ + if (count == 0 && res.dst_fattr->valid & NFS_ATTR_FATTR_SIZE) + count = nfs_size_to_loff_t(res.dst_fattr->size) - dst_offset; nfs42_copy_dest_done(dst_inode, dst_offset, count); status = nfs_post_op_update_inode(dst_inode, res.dst_fattr); } -- GitLab From 7e8436728e22181c3f12a5dbabd35ed3a8b8c593 Mon Sep 17 00:00:00 2001 From: Zhang Xiaoxu Date: Thu, 20 Oct 2022 11:20:54 +0800 Subject: [PATCH 0645/1927] nfs4: Fix kmemleak when allocate slot failed If one of the slot allocate failed, should cleanup all the other allocated slots, otherwise, the allocated slots will leak: unreferenced object 0xffff8881115aa100 (size 64): comm ""mount.nfs"", pid 679, jiffies 4294744957 (age 115.037s) hex dump (first 32 bytes): 00 cc 19 73 81 88 ff ff 00 a0 5a 11 81 88 ff ff ...s......Z..... 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<000000007a4c434a>] nfs4_find_or_create_slot+0x8e/0x130 [<000000005472a39c>] nfs4_realloc_slot_table+0x23f/0x270 [<00000000cd8ca0eb>] nfs40_init_client+0x4a/0x90 [<00000000128486db>] nfs4_init_client+0xce/0x270 [<000000008d2cacad>] nfs4_set_client+0x1a2/0x2b0 [<000000000e593b52>] nfs4_create_server+0x300/0x5f0 [<00000000e4425dd2>] nfs4_try_get_tree+0x65/0x110 [<00000000d3a6176f>] vfs_get_tree+0x41/0xf0 [<0000000016b5ad4c>] path_mount+0x9b3/0xdd0 [<00000000494cae71>] __x64_sys_mount+0x190/0x1d0 [<000000005d56bdec>] do_syscall_64+0x35/0x80 [<00000000687c9ae4>] entry_SYSCALL_64_after_hwframe+0x46/0xb0 Fixes: abf79bb341bf ("NFS: Add a slot table to struct nfs_client for NFSv4.0 transport blocking") Signed-off-by: Zhang Xiaoxu Signed-off-by: Anna Schumaker --- fs/nfs/nfs4client.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 3b9b534813095..d3051b051a564 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -346,6 +346,7 @@ int nfs40_init_client(struct nfs_client *clp) ret = nfs4_setup_slot_table(tbl, NFS4_MAX_SLOT_TABLE, "NFSv4.0 transport Slot table"); if (ret) { + nfs4_shutdown_slot_table(tbl); kfree(tbl); return ret; } -- GitLab From 9f2ac64d6ca60db99132e08628ac2899f956a0ec Mon Sep 17 00:00:00 2001 From: Qinglin Pan Date: Sun, 9 Oct 2022 16:30:50 +0800 Subject: [PATCH 0646/1927] riscv: mm: add missing memcpy in kasan_init Hi Atish, It seems that the panic is due to the missing memcpy during kasan_init. Could you please check whether this patch is helpful? When doing kasan_populate, the new allocated base_pud/base_p4d should contain kasan_early_shadow_{pud, p4d}'s content. Add the missing memcpy to avoid page fault when read/write kasan shadow region. Tested on: - qemu with sv57 and CONFIG_KASAN on. - qemu with sv48 and CONFIG_KASAN on. Signed-off-by: Qinglin Pan Tested-by: Atish Patra Fixes: 8fbdccd2b173 ("riscv: mm: Support kasan for sv57") Link: https://lore.kernel.org/r/20221009083050.3814850-1-panqinglin2020@iscas.ac.cn Signed-off-by: Palmer Dabbelt --- arch/riscv/mm/kasan_init.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index a22e418dbd82c..e1226709490fa 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -113,6 +113,8 @@ static void __init kasan_populate_pud(pgd_t *pgd, base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd))); } else if (pgd_none(*pgd)) { base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE); + memcpy(base_pud, (void *)kasan_early_shadow_pud, + sizeof(pud_t) * PTRS_PER_PUD); } else { base_pud = (pud_t *)pgd_page_vaddr(*pgd); if (base_pud == lm_alias(kasan_early_shadow_pud)) { @@ -173,8 +175,11 @@ static void __init kasan_populate_p4d(pgd_t *pgd, base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd))); } else { base_p4d = (p4d_t *)pgd_page_vaddr(*pgd); - if (base_p4d == lm_alias(kasan_early_shadow_p4d)) + if (base_p4d == lm_alias(kasan_early_shadow_p4d)) { base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE); + memcpy(base_p4d, (void *)kasan_early_shadow_p4d, + sizeof(p4d_t) * PTRS_PER_P4D); + } } p4dp = base_p4d + p4d_index(vaddr); -- GitLab From b8c86872d1dc171d8f1c137917d6913cae2fa4f2 Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Thu, 6 Oct 2022 18:35:20 +0100 Subject: [PATCH 0647/1927] riscv: fix detection of toolchain Zicbom support It is not sufficient to check if a toolchain supports a particular extension without checking if the linker supports that extension too. For example, Clang 15 supports Zicbom but GNU bintutils 2.35.2 does not, leading build errors like so: riscv64-linux-gnu-ld: -march=rv64i2p0_m2p0_a2p0_c2p0_zicbom1p0_zihintpause2p0: Invalid or unknown z ISA extension: 'zicbom' Convert CC_HAS_ZICBOM to TOOLCHAIN_HAS_ZICBOM & check if the linker also supports Zicbom. Reported-by: Kevin Hilman Link: https://github.com/ClangBuiltLinux/linux/issues/1714 Link: https://storage.kernelci.org/next/master/next-20220920/riscv/defconfig+CONFIG_EFI=n/clang-16/logs/kernel.log Fixes: 1631ba1259d6 ("riscv: Add support for non-coherent devices using zicbom extension") Signed-off-by: Conor Dooley Reviewed-by: Heiko Stuebner Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20221006173520.1785507-2-conor@kernel.org [Palmer: Check for ld-2.38, not 2.39, as 2.38 no longer errors.] Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 10 ++++++---- arch/riscv/Makefile | 3 +-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index ed66c31e46559..4c6a09f05348c 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -401,14 +401,16 @@ config RISCV_ISA_SVPBMT If you don't know what to do here, say Y. -config CC_HAS_ZICBOM +config TOOLCHAIN_HAS_ZICBOM bool - default y if 64BIT && $(cc-option,-mabi=lp64 -march=rv64ima_zicbom) - default y if 32BIT && $(cc-option,-mabi=ilp32 -march=rv32ima_zicbom) + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zicbom) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zicbom) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23800 config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" - depends on CC_HAS_ZICBOM + depends on TOOLCHAIN_HAS_ZICBOM depends on !XIP_KERNEL && MMU select RISCV_DMA_NONCOHERENT select RISCV_ALTERNATIVE diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 3fa8ef3368224..3607d38edb4f3 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -57,8 +57,7 @@ toolchain-need-zicsr-zifencei := $(call cc-option-yn, -march=$(riscv-march-y)_zi riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei # Check if the toolchain supports Zicbom extension -toolchain-supports-zicbom := $(call cc-option-yn, -march=$(riscv-march-y)_zicbom) -riscv-march-$(toolchain-supports-zicbom) := $(riscv-march-y)_zicbom +riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZICBOM) := $(riscv-march-y)_zicbom # Check if the toolchain supports Zihintpause extension toolchain-supports-zihintpause := $(call cc-option-yn, -march=$(riscv-march-y)_zihintpause) -- GitLab From aae538cd03bc8fc35979653d9180922d146da0ca Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Thu, 6 Oct 2022 18:35:21 +0100 Subject: [PATCH 0648/1927] riscv: fix detection of toolchain Zihintpause support It is not sufficient to check if a toolchain supports a particular extension without checking if the linker supports that extension too. For example, Clang 15 supports Zihintpause but GNU bintutils 2.35.2 does not, leading build errors like so: riscv64-linux-gnu-ld: -march=rv64i2p0_m2p0_a2p0_c2p0_zihintpause2p0: Invalid or unknown z ISA extension: 'zihintpause' Add a TOOLCHAIN_HAS_ZIHINTPAUSE which checks if each of the compiler, assembler and linker support the extension. Replace the ifdef in the vdso with one depending on this new symbol. Fixes: 8eb060e10185 ("arch/riscv: add Zihintpause support") Signed-off-by: Conor Dooley Reviewed-by: Heiko Stuebner Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20221006173520.1785507-3-conor@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/Kconfig | 7 +++++++ arch/riscv/Makefile | 3 +-- arch/riscv/include/asm/vdso/processor.h | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 4c6a09f05348c..d984f61d5eab6 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -425,6 +425,13 @@ config RISCV_ISA_ZICBOM If you don't know what to do here, say Y. +config TOOLCHAIN_HAS_ZIHINTPAUSE + bool + default y + depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zihintpause) + depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zihintpause) + depends on LLD_VERSION >= 150000 || LD_VERSION >= 23600 + config FPU bool "FPU support" default y diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile index 3607d38edb4f3..6651517f39623 100644 --- a/arch/riscv/Makefile +++ b/arch/riscv/Makefile @@ -60,8 +60,7 @@ riscv-march-$(toolchain-need-zicsr-zifencei) := $(riscv-march-y)_zicsr_zifencei riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZICBOM) := $(riscv-march-y)_zicbom # Check if the toolchain supports Zihintpause extension -toolchain-supports-zihintpause := $(call cc-option-yn, -march=$(riscv-march-y)_zihintpause) -riscv-march-$(toolchain-supports-zihintpause) := $(riscv-march-y)_zihintpause +riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE) := $(riscv-march-y)_zihintpause KBUILD_CFLAGS += -march=$(subst fd,,$(riscv-march-y)) KBUILD_AFLAGS += -march=$(riscv-march-y) diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h index 1e4f8b4aef79d..fa70cfe507aa1 100644 --- a/arch/riscv/include/asm/vdso/processor.h +++ b/arch/riscv/include/asm/vdso/processor.h @@ -21,7 +21,7 @@ static inline void cpu_relax(void) * Reduce instruction retirement. * This assumes the PC changes. */ -#ifdef __riscv_zihintpause +#ifdef CONFIG_TOOLCHAIN_HAS_ZIHINTPAUSE __asm__ __volatile__ ("pause"); #else /* Encoding of the pause instruction */ -- GitLab From d14e99bf95510fa2d6affc371ad68161afc1dc8e Mon Sep 17 00:00:00 2001 From: Andrew Jones Date: Fri, 14 Oct 2022 17:58:44 +0200 Subject: [PATCH 0649/1927] RISC-V: Fix /proc/cpuinfo cpumask warning Commit 78e5a3399421 ("cpumask: fix checking valid cpu range") has started issuing warnings[*] when cpu indices equal to nr_cpu_ids - 1 are passed to cpumask_next* functions. seq_read_iter() and cpuinfo's start and next seq operations implement a pattern like n = cpumask_next(n - 1, mask); show(n); while (1) { ++n; n = cpumask_next(n - 1, mask); if (n >= nr_cpu_ids) break; show(n); } which will issue the warning when reading /proc/cpuinfo. Ensure no warning is generated by validating the cpu index before calling cpumask_next(). [*] Warnings will only appear with DEBUG_PER_CPU_MAPS enabled. Signed-off-by: Andrew Jones Reviewed-by: Anup Patel Reviewed-by: Conor Dooley Tested-by: Conor Dooley Acked-by: Yury Norov Link: https://lore.kernel.org/r/20221014155845.1986223-2-ajones@ventanamicro.com/ Fixes: 78e5a3399421 ("cpumask: fix checking valid cpu range") Cc: stable@vger.kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/cpu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index fa427bdcf773d..852ecccd8920f 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -213,6 +213,9 @@ static void print_mmu(struct seq_file *f) static void *c_start(struct seq_file *m, loff_t *pos) { + if (*pos == nr_cpu_ids) + return NULL; + *pos = cpumask_next(*pos - 1, cpu_online_mask); if ((*pos) < nr_cpu_ids) return (void *)(uintptr_t)(1 + *pos); -- GitLab From 08a32902a56e1670850fe5d518d8203e9ce354b8 Mon Sep 17 00:00:00 2001 From: Maxime Ripard Date: Tue, 18 Oct 2022 15:52:56 +0200 Subject: [PATCH 0650/1927] clk: Remove WARN_ON NULL parent in clk_core_init_rate_req() If a clock has CLK_SET_RATE_PARENT, but core->parent is NULL (most likely because it's orphan), callers of clk_core_init_rate_req() will blindly call this function leading to a very verbose warning. Since it's a fairly common situation, let's just remove the WARN_ON but keep the check that prevents us from dereferencing the pointer. Interestingly, it fixes a regression on the Mediatek MT8195 where the GPU would stall during a clk_set_rate for its main clock. We couldn't come up with a proper explanation since the condition is essentially the same. It was then assumed that it could be timing related since printing the warning stacktrace takes a while, but we couldn't replicate the failure by using fairly large (10ms) mdelays. Fixes: 262ca38f4b6e ("clk: Stop forwarding clk_rate_requests to the parent") Reported-by: AngeloGioacchino Del Regno Signed-off-by: Maxime Ripard Link: https://lore.kernel.org/r/20221018-clk-range-checks-fixes-v1-1-f3ef80518140@cerno.tech Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Stephen Boyd --- drivers/clk/clk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index c3c3f8c072588..37d623c7b73b7 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -1459,7 +1459,7 @@ static void clk_core_init_rate_req(struct clk_core * const core, { struct clk_core *parent; - if (WARN_ON(!core || !req)) + if (!core || WARN_ON(!req)) return; memset(req, 0, sizeof(*req)); -- GitLab From 2079d029387adfc0cc123f01a6fcf9eb6540ee4d Mon Sep 17 00:00:00 2001 From: Maxime Ripard Date: Tue, 18 Oct 2022 15:52:57 +0200 Subject: [PATCH 0651/1927] clk: Initialize the clk_rate_request even if clk_core is NULL Since commit c35e84b09776 ("clk: Introduce clk_hw_init_rate_request()"), users that used to initialize their clk_rate_request by initializing their local structure now rely on clk_hw_init_rate_request(). This function is backed by clk_core_init_rate_req(), which will skip the initialization if either the pointer to struct clk_core or to struct clk_rate_request are NULL. However, the core->parent pointer might be NULL because the clock is orphan, and we will thus end up with our local struct clk_rate_request left untouched. And since clk_hw_init_rate_request() doesn't return an error, we will then call a determine_rate variant with that unitialized structure. In order to avoid this, let's clear our clk_rate_request if the pointer to it is valid but the pointer to struct clk_core isn't. Fixes: c35e84b09776 ("clk: Introduce clk_hw_init_rate_request()") Signed-off-by: Maxime Ripard Link: https://lore.kernel.org/r/20221018-clk-range-checks-fixes-v1-2-f3ef80518140@cerno.tech Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Stephen Boyd --- drivers/clk/clk.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index 37d623c7b73b7..eb2f9be9b9aa5 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -1459,11 +1459,14 @@ static void clk_core_init_rate_req(struct clk_core * const core, { struct clk_core *parent; - if (!core || WARN_ON(!req)) + if (WARN_ON(!req)) return; memset(req, 0, sizeof(*req)); + if (!core) + return; + req->rate = rate; clk_core_get_boundaries(core, &req->min_rate, &req->max_rate); -- GitLab From 774560cf28fe115f106b6176c54ec641491136ac Mon Sep 17 00:00:00 2001 From: Maxime Ripard Date: Tue, 18 Oct 2022 15:52:58 +0200 Subject: [PATCH 0652/1927] clk: Initialize max_rate in struct clk_rate_request Since commit b46fd8dbe8ad ("clk: Zero the clk_rate_request structure"), the clk_core_init_rate_req() function clears the struct clk_rate_request passed as argument. However, the default value for max_rate isn't 0 but ULONG_MAX, and we end up creating a clk_rate_request instance where the maximum rate is 0. Let's initialize max_rate to ULONG_MAX properly. Fixes: b46fd8dbe8ad ("clk: Zero the clk_rate_request structure") Signed-off-by: Maxime Ripard Link: https://lore.kernel.org/r/20221018-clk-range-checks-fixes-v1-3-f3ef80518140@cerno.tech Reviewed-by: AngeloGioacchino Del Regno Signed-off-by: Stephen Boyd --- drivers/clk/clk.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c index eb2f9be9b9aa5..57b83665e5c3a 100644 --- a/drivers/clk/clk.c +++ b/drivers/clk/clk.c @@ -1463,6 +1463,7 @@ static void clk_core_init_rate_req(struct clk_core * const core, return; memset(req, 0, sizeof(*req)); + req->max_rate = ULONG_MAX; if (!core) return; -- GitLab From ffa20aa581cf5377fc397b0d0ff9d67ea823629b Mon Sep 17 00:00:00 2001 From: Taniya Das Date: Wed, 19 Oct 2022 11:35:35 +0530 Subject: [PATCH 0653/1927] clk: qcom: Update the force mem core bit for GPU clocks There are few GPU clocks which are powering up the memories and thus enable the FORCE_MEM_PERIPH always for these clocks to force the periph_on signal to remain active during halt state of the clock. Fixes: a3cc092196ef ("clk: qcom: Add Global Clock controller (GCC) driver for SC7280") Fixes: 3e0f01d6c7e7 ("clk: qcom: Add graphics clock controller driver for SC7280") Signed-off-by: Taniya Das Signed-off-by: Satya Priya Link: https://lore.kernel.org/r/1666159535-6447-1-git-send-email-quic_c_skakit@quicinc.com Signed-off-by: Stephen Boyd --- drivers/clk/qcom/gcc-sc7280.c | 1 + drivers/clk/qcom/gpucc-sc7280.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/clk/qcom/gcc-sc7280.c b/drivers/clk/qcom/gcc-sc7280.c index 8afb7575e712f..46d41ebce2b08 100644 --- a/drivers/clk/qcom/gcc-sc7280.c +++ b/drivers/clk/qcom/gcc-sc7280.c @@ -3467,6 +3467,7 @@ static int gcc_sc7280_probe(struct platform_device *pdev) regmap_update_bits(regmap, 0x28004, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x28014, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x71004, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x7100C, BIT(13), BIT(13)); ret = qcom_cc_register_rcg_dfs(regmap, gcc_dfs_clocks, ARRAY_SIZE(gcc_dfs_clocks)); diff --git a/drivers/clk/qcom/gpucc-sc7280.c b/drivers/clk/qcom/gpucc-sc7280.c index 9a832f2bcf491..1490cd45a654a 100644 --- a/drivers/clk/qcom/gpucc-sc7280.c +++ b/drivers/clk/qcom/gpucc-sc7280.c @@ -463,6 +463,7 @@ static int gpu_cc_sc7280_probe(struct platform_device *pdev) */ regmap_update_bits(regmap, 0x1170, BIT(0), BIT(0)); regmap_update_bits(regmap, 0x1098, BIT(0), BIT(0)); + regmap_update_bits(regmap, 0x1098, BIT(13), BIT(13)); return qcom_cc_really_probe(pdev, &gpu_cc_sc7280_desc, regmap); } -- GitLab From 9a8c5b0d061554fedd7dbe894e63aa34d0bac7c4 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Thu, 27 Oct 2022 16:04:36 -0400 Subject: [PATCH 0654/1927] ext4: update the backup superblock's at the end of the online resize When expanding a file system using online resize, various fields in the superblock (e.g., s_blocks_count, s_inodes_count, etc.) change. To update the backup superblocks, the online resize uses the function update_backups() in fs/ext4/resize.c. This function was not updating the checksum field in the backup superblocks. This wasn't a big deal previously, because e2fsck didn't care about the checksum field in the backup superblock. (And indeed, update_backups() goes all the way back to the ext3 days, well before we had support for metadata checksums.) However, there is an alternate, more general way of updating superblock fields, ext4_update_primary_sb() in fs/ext4/ioctl.c. This function does check the checksum of the backup superblock, and if it doesn't match will mark the file system as corrupted. That was clearly not the intent, so avoid to aborting the resize when a bad superblock is found. In addition, teach update_backups() to properly update the checksum in the backup superblocks. We will eventually want to unify updapte_backups() with the infrasture in ext4_update_primary_sb(), but that's for another day. Note: The problem has been around for a while; it just didn't really matter until ext4_update_primary_sb() was added by commit bbc605cdb1e1 ("ext4: implement support for get/set fs label"). And it became trivially easy to reproduce after commit 827891a38acc ("ext4: update the s_overhead_clusters in the backup sb's when resizing") in v6.0. Cc: stable@kernel.org # 5.17+ Fixes: bbc605cdb1e1 ("ext4: implement support for get/set fs label") Signed-off-by: Theodore Ts'o --- fs/ext4/ioctl.c | 3 +-- fs/ext4/resize.c | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c index 4d49c5cfb690f..790d5ffe85596 100644 --- a/fs/ext4/ioctl.c +++ b/fs/ext4/ioctl.c @@ -145,9 +145,8 @@ static int ext4_update_backup_sb(struct super_block *sb, if (ext4_has_metadata_csum(sb) && es->s_checksum != ext4_superblock_csum(sb, es)) { ext4_msg(sb, KERN_ERR, "Invalid checksum for backup " - "superblock %llu\n", sb_block); + "superblock %llu", sb_block); unlock_buffer(bh); - err = -EFSBADCRC; goto out_bh; } func(es, arg); diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c index 6dfe9ccae0c50..46b87ffeb3045 100644 --- a/fs/ext4/resize.c +++ b/fs/ext4/resize.c @@ -1158,6 +1158,7 @@ static void update_backups(struct super_block *sb, sector_t blk_off, char *data, while (group < sbi->s_groups_count) { struct buffer_head *bh; ext4_fsblk_t backup_block; + struct ext4_super_block *es; /* Out of journal space, and can't get more - abort - so sad */ err = ext4_resize_ensure_credits_batch(handle, 1); @@ -1186,6 +1187,10 @@ static void update_backups(struct super_block *sb, sector_t blk_off, char *data, memcpy(bh->b_data, data, size); if (rest) memset(bh->b_data + size, 0, rest); + es = (struct ext4_super_block *) bh->b_data; + es->s_block_group_nr = cpu_to_le16(group); + if (ext4_has_metadata_csum(sb)) + es->s_checksum = ext4_superblock_csum(sb, es); set_buffer_uptodate(bh); unlock_buffer(bh); err = ext4_handle_dirty_metadata(handle, NULL, bh); -- GitLab From 153695d36ead0ccc4d0256953c751cabf673e621 Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Thu, 27 Oct 2022 20:45:28 +0800 Subject: [PATCH 0655/1927] cifs: fix use-after-free caused by invalid pointer `hostname` `hostname` needs to be set as null-pointer after free in `cifs_put_tcp_session` function, or when `cifsd` thread attempts to resolve hostname and reconnect the host, the thread would deref the invalid pointer. Here is one of practical backtrace examples as reference: Task 477 --------------------------- do_mount path_mount do_new_mount vfs_get_tree smb3_get_tree smb3_get_tree_common cifs_smb3_do_mount cifs_mount mount_put_conns cifs_put_tcp_session --> kfree(server->hostname) cifsd --------------------------- kthread cifs_demultiplex_thread cifs_reconnect reconn_set_ipaddr_from_hostname --> if (!server->hostname) --> if (server->hostname[0] == '\0') // !! UAF fault here CIFS: VFS: cifs_mount failed w/return code = -112 mount error(112): Host is down BUG: KASAN: use-after-free in reconn_set_ipaddr_from_hostname+0x2ba/0x310 Read of size 1 at addr ffff888108f35380 by task cifsd/480 CPU: 2 PID: 480 Comm: cifsd Not tainted 6.1.0-rc2-00106-gf705792f89dd-dirty #25 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.13.0-1ubuntu1.1 04/01/2014 Call Trace: dump_stack_lvl+0x68/0x85 print_report+0x16c/0x4a3 kasan_report+0x95/0x190 reconn_set_ipaddr_from_hostname+0x2ba/0x310 __cifs_reconnect.part.0+0x241/0x800 cifs_reconnect+0x65f/0xb60 cifs_demultiplex_thread+0x1570/0x2570 kthread+0x2c5/0x380 ret_from_fork+0x22/0x30 Allocated by task 477: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 __kasan_kmalloc+0x7e/0x90 __kmalloc_node_track_caller+0x52/0x1b0 kstrdup+0x3b/0x70 cifs_get_tcp_session+0xbc/0x19b0 mount_get_conns+0xa9/0x10c0 cifs_mount+0xdf/0x1970 cifs_smb3_do_mount+0x295/0x1660 smb3_get_tree+0x352/0x5e0 vfs_get_tree+0x8e/0x2e0 path_mount+0xf8c/0x1990 do_mount+0xee/0x110 __x64_sys_mount+0x14b/0x1f0 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 477: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_save_free_info+0x2a/0x50 __kasan_slab_free+0x10a/0x190 __kmem_cache_free+0xca/0x3f0 cifs_put_tcp_session+0x30c/0x450 cifs_mount+0xf95/0x1970 cifs_smb3_do_mount+0x295/0x1660 smb3_get_tree+0x352/0x5e0 vfs_get_tree+0x8e/0x2e0 path_mount+0xf8c/0x1990 do_mount+0xee/0x110 __x64_sys_mount+0x14b/0x1f0 do_syscall_64+0x3b/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd The buggy address belongs to the object at ffff888108f35380 which belongs to the cache kmalloc-16 of size 16 The buggy address is located 0 bytes inside of 16-byte region [ffff888108f35380, ffff888108f35390) The buggy address belongs to the physical page: page:00000000333f8e58 refcount:1 mapcount:0 mapping:0000000000000000 index:0xffff888108f350e0 pfn:0x108f35 flags: 0x200000000000200(slab|node=0|zone=2) raw: 0200000000000200 0000000000000000 dead000000000122 ffff8881000423c0 raw: ffff888108f350e0 000000008080007a 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected Memory state around the buggy address: ffff888108f35280: fa fb fc fc fa fb fc fc fa fb fc fc fa fb fc fc ffff888108f35300: fa fb fc fc fa fb fc fc fa fb fc fc fa fb fc fc >ffff888108f35380: fa fb fc fc fa fb fc fc fa fb fc fc fa fb fc fc ^ ffff888108f35400: fa fb fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff888108f35480: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc Fixes: 7be3248f3139 ("cifs: To match file servers, make sure the server hostname matches") Signed-off-by: Zeng Heng Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/connect.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/cifs/connect.c b/fs/cifs/connect.c index ffb291579bb9d..1cc47dd3b4d62 100644 --- a/fs/cifs/connect.c +++ b/fs/cifs/connect.c @@ -1584,6 +1584,7 @@ cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect) server->session_key.response = NULL; server->session_key.len = 0; kfree(server->hostname); + server->hostname = NULL; task = xchg(&server->tsk, NULL); if (task) -- GitLab From fb3041d61f6867158088c627c2790f94e208d1ea Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 28 Oct 2022 01:28:39 +0900 Subject: [PATCH 0656/1927] kbuild: fix SIGPIPE error message for AR=gcc-ar and AR=llvm-ar Jiri Slaby reported that building the kernel with AR=gcc-ar shows: /usr/bin/ar terminated with signal 13 [Broken pipe] Nathan Chancellor reported the latest AR=llvm-ar shows: error: write on a pipe with no reader The latter occurs since LLVM commit 51b557adc131 ("Add an error message to the default SIGPIPE handler"). The resulting vmlinux is correct, but it is better to silence it. 'head -n1' exits after reading the first line, so the pipe is closed. Use 'sed -n 1p' to eat the stream till the end. Fixes: 321648455061 ("kbuild: use obj-y instead extra-y for objects placed at the head") Link: https://github.com/ClangBuiltLinux/linux/issues/1651 Reported-by: Jiri Slaby Reported-by: Nathan Chancellor Signed-off-by: Masahiro Yamada Tested-by: Nick Desaulniers Reviewed-by: Nick Desaulniers Tested-by: Nathan Chancellor --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e90bb2b38607a..e9e7eff906a5e 100644 --- a/Makefile +++ b/Makefile @@ -1218,7 +1218,7 @@ quiet_cmd_ar_vmlinux.a = AR $@ cmd_ar_vmlinux.a = \ rm -f $@; \ $(AR) cDPrST $@ $(KBUILD_VMLINUX_OBJS); \ - $(AR) mPiT $$($(AR) t $@ | head -n1) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) + $(AR) mPiT $$($(AR) t $@ | sed -n 1p) $@ $$($(AR) t $@ | grep -F -f $(srctree)/scripts/head-object-list.txt) targets += vmlinux.a vmlinux.a: $(KBUILD_VMLINUX_OBJS) scripts/head-object-list.txt autoksyms_recursive FORCE -- GitLab From 633efc8b3dc96f56f5a57f2a49764853a2fa3f50 Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Wed, 26 Oct 2022 10:03:21 +0800 Subject: [PATCH 0657/1927] net: dsa: Fix possible memory leaks in dsa_loop_init() kmemleak reported memory leaks in dsa_loop_init(): kmemleak: 12 new suspected memory leaks unreferenced object 0xffff8880138ce000 (size 2048): comm "modprobe", pid 390, jiffies 4295040478 (age 238.976s) backtrace: [<000000006a94f1d5>] kmalloc_trace+0x26/0x60 [<00000000a9c44622>] phy_device_create+0x5d/0x970 [<00000000d0ee2afc>] get_phy_device+0xf3/0x2b0 [<00000000dca0c71f>] __fixed_phy_register.part.0+0x92/0x4e0 [<000000008a834798>] fixed_phy_register+0x84/0xb0 [<0000000055223fcb>] dsa_loop_init+0xa9/0x116 [dsa_loop] ... There are two reasons for memleak in dsa_loop_init(). First, fixed_phy_register() create and register phy_device: fixed_phy_register() get_phy_device() phy_device_create() # freed by phy_device_free() phy_device_register() # freed by phy_device_remove() But fixed_phy_unregister() only calls phy_device_remove(). So the memory allocated in phy_device_create() is leaked. Second, when mdio_driver_register() fail in dsa_loop_init(), it just returns and there is no cleanup for phydevs. Fix the problems by catching the error of mdio_driver_register() in dsa_loop_init(), then calling both fixed_phy_unregister() and phy_device_free() to release phydevs. Also add a function for phydevs cleanup to avoid duplacate. Fixes: 98cd1552ea27 ("net: dsa: Mock-up driver") Signed-off-by: Chen Zhongjin Signed-off-by: David S. Miller --- drivers/net/dsa/dsa_loop.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c index b9107fe400231..5b139f2206b6e 100644 --- a/drivers/net/dsa/dsa_loop.c +++ b/drivers/net/dsa/dsa_loop.c @@ -376,6 +376,17 @@ static struct mdio_driver dsa_loop_drv = { #define NUM_FIXED_PHYS (DSA_LOOP_NUM_PORTS - 2) +static void dsa_loop_phydevs_unregister(void) +{ + unsigned int i; + + for (i = 0; i < NUM_FIXED_PHYS; i++) + if (!IS_ERR(phydevs[i])) { + fixed_phy_unregister(phydevs[i]); + phy_device_free(phydevs[i]); + } +} + static int __init dsa_loop_init(void) { struct fixed_phy_status status = { @@ -383,23 +394,23 @@ static int __init dsa_loop_init(void) .speed = SPEED_100, .duplex = DUPLEX_FULL, }; - unsigned int i; + unsigned int i, ret; for (i = 0; i < NUM_FIXED_PHYS; i++) phydevs[i] = fixed_phy_register(PHY_POLL, &status, NULL); - return mdio_driver_register(&dsa_loop_drv); + ret = mdio_driver_register(&dsa_loop_drv); + if (ret) + dsa_loop_phydevs_unregister(); + + return ret; } module_init(dsa_loop_init); static void __exit dsa_loop_exit(void) { - unsigned int i; - mdio_driver_unregister(&dsa_loop_drv); - for (i = 0; i < NUM_FIXED_PHYS; i++) - if (!IS_ERR(phydevs[i])) - fixed_phy_unregister(phydevs[i]); + dsa_loop_phydevs_unregister(); } module_exit(dsa_loop_exit); -- GitLab From a51abbbf25317c07cb00b40ae7d04a209d2a3d54 Mon Sep 17 00:00:00 2001 From: Michal Luczaj Date: Thu, 13 Oct 2022 21:12:33 +0000 Subject: [PATCH 0658/1927] KVM: selftests: Add tests in xen_shinfo_test to detect lock races Tests for races between shinfo_cache (de)activation and hypercall+ioctl() processing. KVM has had bugs where activating the shared info cache multiple times and/or with concurrent users results in lock corruption, NULL pointer dereferences, and other fun. For the timer injection testcase (#22), re-arm the timer until the IRQ is successfully injected. If the timer expires while the shared info is deactivated (invalid), KVM will drop the event. Signed-off-by: Michal Luczaj Co-developed-by: Sean Christopherson Signed-off-by: Sean Christopherson Message-Id: <20221013211234.1318131-16-seanjc@google.com> Signed-off-by: Paolo Bonzini --- .../selftests/kvm/x86_64/xen_shinfo_test.c | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index 8a5cb800f50ed..caa3f5ab9e109 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -15,9 +15,13 @@ #include #include #include +#include #include +/* Defined in include/linux/kvm_types.h */ +#define GPA_INVALID (~(ulong)0) + #define SHINFO_REGION_GVA 0xc0000000ULL #define SHINFO_REGION_GPA 0xc0000000ULL #define SHINFO_REGION_SLOT 10 @@ -44,6 +48,8 @@ #define MIN_STEAL_TIME 50000 +#define SHINFO_RACE_TIMEOUT 2 /* seconds */ + #define __HYPERVISOR_set_timer_op 15 #define __HYPERVISOR_sched_op 29 #define __HYPERVISOR_event_channel_op 32 @@ -148,6 +154,7 @@ static void guest_wait_for_irq(void) static void guest_code(void) { struct vcpu_runstate_info *rs = (void *)RUNSTATE_VADDR; + int i; __asm__ __volatile__( "sti\n" @@ -325,6 +332,49 @@ static void guest_code(void) guest_wait_for_irq(); GUEST_SYNC(21); + /* Racing host ioctls */ + + guest_wait_for_irq(); + + GUEST_SYNC(22); + /* Racing vmcall against host ioctl */ + + ports[0] = 0; + + p = (struct sched_poll) { + .ports = ports, + .nr_ports = 1, + .timeout = 0 + }; + +wait_for_timer: + /* + * Poll for a timer wake event while the worker thread is mucking with + * the shared info. KVM XEN drops timer IRQs if the shared info is + * invalid when the timer expires. Arbitrarily poll 100 times before + * giving up and asking the VMM to re-arm the timer. 100 polls should + * consume enough time to beat on KVM without taking too long if the + * timer IRQ is dropped due to an invalid event channel. + */ + for (i = 0; i < 100 && !guest_saw_irq; i++) + asm volatile("vmcall" + : "=a" (rax) + : "a" (__HYPERVISOR_sched_op), + "D" (SCHEDOP_poll), + "S" (&p) + : "memory"); + + /* + * Re-send the timer IRQ if it was (likely) dropped due to the timer + * expiring while the event channel was invalid. + */ + if (!guest_saw_irq) { + GUEST_SYNC(23); + goto wait_for_timer; + } + guest_saw_irq = false; + + GUEST_SYNC(24); } static int cmp_timespec(struct timespec *a, struct timespec *b) @@ -352,11 +402,36 @@ static void handle_alrm(int sig) TEST_FAIL("IRQ delivery timed out"); } +static void *juggle_shinfo_state(void *arg) +{ + struct kvm_vm *vm = (struct kvm_vm *)arg; + + struct kvm_xen_hvm_attr cache_init = { + .type = KVM_XEN_ATTR_TYPE_SHARED_INFO, + .u.shared_info.gfn = SHINFO_REGION_GPA / PAGE_SIZE + }; + + struct kvm_xen_hvm_attr cache_destroy = { + .type = KVM_XEN_ATTR_TYPE_SHARED_INFO, + .u.shared_info.gfn = GPA_INVALID + }; + + for (;;) { + __vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &cache_init); + __vm_ioctl(vm, KVM_XEN_HVM_SET_ATTR, &cache_destroy); + pthread_testcancel(); + }; + + return NULL; +} + int main(int argc, char *argv[]) { struct timespec min_ts, max_ts, vm_ts; struct kvm_vm *vm; + pthread_t thread; bool verbose; + int ret; verbose = argc > 1 && (!strncmp(argv[1], "-v", 3) || !strncmp(argv[1], "--verbose", 10)); @@ -785,6 +860,71 @@ int main(int argc, char *argv[]) case 21: TEST_ASSERT(!evtchn_irq_expected, "Expected event channel IRQ but it didn't happen"); + alarm(0); + + if (verbose) + printf("Testing shinfo lock corruption (KVM_XEN_HVM_EVTCHN_SEND)\n"); + + ret = pthread_create(&thread, NULL, &juggle_shinfo_state, (void *)vm); + TEST_ASSERT(ret == 0, "pthread_create() failed: %s", strerror(ret)); + + struct kvm_irq_routing_xen_evtchn uxe = { + .port = 1, + .vcpu = vcpu->id, + .priority = KVM_IRQ_ROUTING_XEN_EVTCHN_PRIO_2LEVEL + }; + + evtchn_irq_expected = true; + for (time_t t = time(NULL) + SHINFO_RACE_TIMEOUT; time(NULL) < t;) + __vm_ioctl(vm, KVM_XEN_HVM_EVTCHN_SEND, &uxe); + break; + + case 22: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + + if (verbose) + printf("Testing shinfo lock corruption (SCHEDOP_poll)\n"); + + shinfo->evtchn_pending[0] = 1; + + evtchn_irq_expected = true; + tmr.u.timer.expires_ns = rs->state_entry_time + + SHINFO_RACE_TIMEOUT * 1000000000ULL; + vcpu_ioctl(vcpu, KVM_XEN_VCPU_SET_ATTR, &tmr); + break; + + case 23: + /* + * Optional and possibly repeated sync point. + * Injecting the timer IRQ may fail if the + * shinfo is invalid when the timer expires. + * If the timer has expired but the IRQ hasn't + * been delivered, rearm the timer and retry. + */ + vcpu_ioctl(vcpu, KVM_XEN_VCPU_GET_ATTR, &tmr); + + /* Resume the guest if the timer is still pending. */ + if (tmr.u.timer.expires_ns) + break; + + /* All done if the IRQ was delivered. */ + if (!evtchn_irq_expected) + break; + + tmr.u.timer.expires_ns = rs->state_entry_time + + SHINFO_RACE_TIMEOUT * 1000000000ULL; + vcpu_ioctl(vcpu, KVM_XEN_VCPU_SET_ATTR, &tmr); + break; + case 24: + TEST_ASSERT(!evtchn_irq_expected, + "Expected event channel IRQ but it didn't happen"); + + ret = pthread_cancel(thread); + TEST_ASSERT(ret == 0, "pthread_cancel() failed: %s", strerror(ret)); + + ret = pthread_join(thread, 0); + TEST_ASSERT(ret == 0, "pthread_join() failed: %s", strerror(ret)); goto done; case 0x20: -- GitLab From 5addaf530995ac203fa46efde0d1ded4c15ff98e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 13 Oct 2022 21:12:34 +0000 Subject: [PATCH 0659/1927] KVM: selftests: Mark "guest_saw_irq" as volatile in xen_shinfo_test Tag "guest_saw_irq" as "volatile" to ensure that the compiler will never optimize away lookups. Relying on the compiler thinking that the flag is global and thus might change also works, but it's subtle, less robust, and looks like a bug at first glance, e.g. risks being "fixed" and breaking the test. Make the flag "static" as well since convincing the compiler it's global is no longer necessary. Alternatively, the flag could be accessed with {READ,WRITE}_ONCE(), but literally every access would need the wrappers, and eking out performance isn't exactly top priority for selftests. Signed-off-by: Sean Christopherson Message-Id: <20221013211234.1318131-17-seanjc@google.com> Signed-off-by: Paolo Bonzini --- tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index caa3f5ab9e109..2a5727188c8d3 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -132,7 +132,7 @@ struct { struct kvm_irq_routing_entry entries[2]; } irq_routes; -bool guest_saw_irq; +static volatile bool guest_saw_irq; static void evtchn_handler(struct ex_regs *regs) { -- GitLab From 5015bb89b58225f97df6ac44383e7e8c8662c8c9 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 25 Oct 2022 15:47:28 +0300 Subject: [PATCH 0660/1927] KVM: x86: emulator: em_sysexit should update ctxt->mode SYSEXIT is one of the instructions that can change the processor mode, thus ctxt->mode should be updated after it. Note that this is likely a benign bug, because the only problematic mode change is from 32 bit to 64 bit which can lead to truncation of RIP, and it is not possible to do with sysexit, since sysexit running in 32 bit mode will be limited to 32 bit version. Signed-off-by: Maxim Levitsky Message-Id: <20221025124741.228045-11-mlevitsk@redhat.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/emulate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 3b27622d46425..261732957431b 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -2876,6 +2876,7 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt) ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); ctxt->_eip = rdx; + ctxt->mode = usermode; *reg_write(ctxt, VCPU_REGS_RSP) = rcx; return X86EMUL_CONTINUE; -- GitLab From d087e0f79fa0dd336a9a6b2f79ec23120f5eff73 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 25 Oct 2022 15:47:29 +0300 Subject: [PATCH 0661/1927] KVM: x86: emulator: introduce emulator_recalc_and_set_mode Some instructions update the cpu execution mode, which needs to update the emulation mode. Extract this code, and make assign_eip_far use it. assign_eip_far now reads CS, instead of getting it via a parameter, which is ok, because callers always assign CS to the same value before calling this function. No functional change is intended. Signed-off-by: Maxim Levitsky Message-Id: <20221025124741.228045-12-mlevitsk@redhat.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/emulate.c | 85 ++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 261732957431b..e5522a23d9856 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -791,8 +791,7 @@ static int linearize(struct x86_emulate_ctxt *ctxt, ctxt->mode, linear); } -static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst, - enum x86emul_mode mode) +static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst) { ulong linear; int rc; @@ -802,41 +801,71 @@ static inline int assign_eip(struct x86_emulate_ctxt *ctxt, ulong dst, if (ctxt->op_bytes != sizeof(unsigned long)) addr.ea = dst & ((1UL << (ctxt->op_bytes << 3)) - 1); - rc = __linearize(ctxt, addr, &max_size, 1, false, true, mode, &linear); + rc = __linearize(ctxt, addr, &max_size, 1, false, true, ctxt->mode, &linear); if (rc == X86EMUL_CONTINUE) ctxt->_eip = addr.ea; return rc; } +static inline int emulator_recalc_and_set_mode(struct x86_emulate_ctxt *ctxt) +{ + u64 efer; + struct desc_struct cs; + u16 selector; + u32 base3; + + ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); + + if (!(ctxt->ops->get_cr(ctxt, 0) & X86_CR0_PE)) { + /* Real mode. cpu must not have long mode active */ + if (efer & EFER_LMA) + return X86EMUL_UNHANDLEABLE; + ctxt->mode = X86EMUL_MODE_REAL; + return X86EMUL_CONTINUE; + } + + if (ctxt->eflags & X86_EFLAGS_VM) { + /* Protected/VM86 mode. cpu must not have long mode active */ + if (efer & EFER_LMA) + return X86EMUL_UNHANDLEABLE; + ctxt->mode = X86EMUL_MODE_VM86; + return X86EMUL_CONTINUE; + } + + if (!ctxt->ops->get_segment(ctxt, &selector, &cs, &base3, VCPU_SREG_CS)) + return X86EMUL_UNHANDLEABLE; + + if (efer & EFER_LMA) { + if (cs.l) { + /* Proper long mode */ + ctxt->mode = X86EMUL_MODE_PROT64; + } else if (cs.d) { + /* 32 bit compatibility mode*/ + ctxt->mode = X86EMUL_MODE_PROT32; + } else { + ctxt->mode = X86EMUL_MODE_PROT16; + } + } else { + /* Legacy 32 bit / 16 bit mode */ + ctxt->mode = cs.d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; + } + + return X86EMUL_CONTINUE; +} + static inline int assign_eip_near(struct x86_emulate_ctxt *ctxt, ulong dst) { - return assign_eip(ctxt, dst, ctxt->mode); + return assign_eip(ctxt, dst); } -static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst, - const struct desc_struct *cs_desc) +static int assign_eip_far(struct x86_emulate_ctxt *ctxt, ulong dst) { - enum x86emul_mode mode = ctxt->mode; - int rc; + int rc = emulator_recalc_and_set_mode(ctxt); -#ifdef CONFIG_X86_64 - if (ctxt->mode >= X86EMUL_MODE_PROT16) { - if (cs_desc->l) { - u64 efer = 0; + if (rc != X86EMUL_CONTINUE) + return rc; - ctxt->ops->get_msr(ctxt, MSR_EFER, &efer); - if (efer & EFER_LMA) - mode = X86EMUL_MODE_PROT64; - } else - mode = X86EMUL_MODE_PROT32; /* temporary value */ - } -#endif - if (mode == X86EMUL_MODE_PROT16 || mode == X86EMUL_MODE_PROT32) - mode = cs_desc->d ? X86EMUL_MODE_PROT32 : X86EMUL_MODE_PROT16; - rc = assign_eip(ctxt, dst, mode); - if (rc == X86EMUL_CONTINUE) - ctxt->mode = mode; - return rc; + return assign_eip(ctxt, dst); } static inline int jmp_rel(struct x86_emulate_ctxt *ctxt, int rel) @@ -2172,7 +2201,7 @@ static int em_jmp_far(struct x86_emulate_ctxt *ctxt) if (rc != X86EMUL_CONTINUE) return rc; - rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc); + rc = assign_eip_far(ctxt, ctxt->src.val); /* Error handling is not implemented. */ if (rc != X86EMUL_CONTINUE) return X86EMUL_UNHANDLEABLE; @@ -2250,7 +2279,7 @@ static int em_ret_far(struct x86_emulate_ctxt *ctxt) &new_desc); if (rc != X86EMUL_CONTINUE) return rc; - rc = assign_eip_far(ctxt, eip, &new_desc); + rc = assign_eip_far(ctxt, eip); /* Error handling is not implemented. */ if (rc != X86EMUL_CONTINUE) return X86EMUL_UNHANDLEABLE; @@ -3470,7 +3499,7 @@ static int em_call_far(struct x86_emulate_ctxt *ctxt) if (rc != X86EMUL_CONTINUE) return rc; - rc = assign_eip_far(ctxt, ctxt->src.val, &new_desc); + rc = assign_eip_far(ctxt, ctxt->src.val); if (rc != X86EMUL_CONTINUE) goto fail; -- GitLab From 055f37f84e304e59c046d1accfd8f08462f52c4c Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 25 Oct 2022 15:47:30 +0300 Subject: [PATCH 0662/1927] KVM: x86: emulator: update the emulation mode after rsm Update the emulation mode after RSM so that RIP will be correctly written back, because the RSM instruction can switch the CPU mode from 32 bit (or less) to 64 bit. This fixes a guest crash in case the #SMI is received while the guest runs a code from an address > 32 bit. Signed-off-by: Maxim Levitsky Message-Id: <20221025124741.228045-13-mlevitsk@redhat.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/emulate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index e5522a23d9856..33385ebae1004 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -2662,7 +2662,7 @@ static int em_rsm(struct x86_emulate_ctxt *ctxt) * those side effects need to be explicitly handled for both success * and shutdown. */ - return X86EMUL_CONTINUE; + return emulator_recalc_and_set_mode(ctxt); emulate_shutdown: ctxt->ops->triple_fault(ctxt); -- GitLab From ad8f9e69942c7db90758d9d774157e53bce94840 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 25 Oct 2022 15:47:31 +0300 Subject: [PATCH 0663/1927] KVM: x86: emulator: update the emulation mode after CR0 write Update the emulation mode when handling writes to CR0, because toggling CR0.PE switches between Real and Protected Mode, and toggling CR0.PG when EFER.LME=1 switches between Long and Protected Mode. This is likely a benign bug because there is no writeback of state, other than the RIP increment, and when toggling CR0.PE, the CPU has to execute code from a very low memory address. Signed-off-by: Maxim Levitsky Message-Id: <20221025124741.228045-14-mlevitsk@redhat.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/emulate.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 33385ebae1004..2954c046740bf 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3641,11 +3641,25 @@ static int em_movbe(struct x86_emulate_ctxt *ctxt) static int em_cr_write(struct x86_emulate_ctxt *ctxt) { - if (ctxt->ops->set_cr(ctxt, ctxt->modrm_reg, ctxt->src.val)) + int cr_num = ctxt->modrm_reg; + int r; + + if (ctxt->ops->set_cr(ctxt, cr_num, ctxt->src.val)) return emulate_gp(ctxt, 0); /* Disable writeback. */ ctxt->dst.type = OP_NONE; + + if (cr_num == 0) { + /* + * CR0 write might have updated CR0.PE and/or CR0.PG + * which can affect the cpu's execution mode. + */ + r = emulator_recalc_and_set_mode(ctxt); + if (r != X86EMUL_CONTINUE) + return r; + } + return X86EMUL_CONTINUE; } -- GitLab From 696db303e54f7352623d9f640e6c51d8fa9d5588 Mon Sep 17 00:00:00 2001 From: Maxim Levitsky Date: Tue, 25 Oct 2022 15:47:32 +0300 Subject: [PATCH 0664/1927] KVM: x86: smm: number of GPRs in the SMRAM image depends on the image format On 64 bit host, if the guest doesn't have X86_FEATURE_LM, KVM will access 16 gprs to 32-bit smram image, causing out-ouf-bound ram access. On 32 bit host, the rsm_load_state_64/enter_smm_save_state_64 is compiled out, thus access overflow can't happen. Fixes: b443183a25ab61 ("KVM: x86: Reduce the number of emulator GPRs to '8' for 32-bit KVM") Signed-off-by: Maxim Levitsky Reviewed-by: Sean Christopherson Message-Id: <20221025124741.228045-15-mlevitsk@redhat.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/emulate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 2954c046740bf..4a43261d25a2a 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -2461,7 +2461,7 @@ static int rsm_load_state_32(struct x86_emulate_ctxt *ctxt, ctxt->eflags = GET_SMSTATE(u32, smstate, 0x7ff4) | X86_EFLAGS_FIXED; ctxt->_eip = GET_SMSTATE(u32, smstate, 0x7ff0); - for (i = 0; i < NR_EMULATOR_GPRS; i++) + for (i = 0; i < 8; i++) *reg_write(ctxt, i) = GET_SMSTATE(u32, smstate, 0x7fd0 + i * 4); val = GET_SMSTATE(u32, smstate, 0x7fcc); @@ -2518,7 +2518,7 @@ static int rsm_load_state_64(struct x86_emulate_ctxt *ctxt, u16 selector; int i, r; - for (i = 0; i < NR_EMULATOR_GPRS; i++) + for (i = 0; i < 16; i++) *reg_write(ctxt, i) = GET_SMSTATE(u64, smstate, 0x7ff8 - i * 8); ctxt->_eip = GET_SMSTATE(u64, smstate, 0x7f78); -- GitLab From 8fdf3f6aba7cfa0c0e2bf66ecca7bb5783acd0d6 Mon Sep 17 00:00:00 2001 From: Radhey Shyam Pandey Date: Wed, 26 Oct 2022 20:45:24 +0530 Subject: [PATCH 0665/1927] net: emaclite: update reset_lock member documentation Instead of generic description, mention what reset_lock actually protects i.e. lock to serialize xmit and tx_timeout execution. Signed-off-by: Radhey Shyam Pandey Signed-off-by: David S. Miller --- drivers/net/ethernet/xilinx/xilinx_emaclite.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c index 05848ff15fb51..a3967f8de417d 100644 --- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c +++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c @@ -108,7 +108,7 @@ * @next_tx_buf_to_use: next Tx buffer to write to * @next_rx_buf_to_use: next Rx buffer to read from * @base_addr: base address of the Emaclite device - * @reset_lock: lock used for synchronization + * @reset_lock: lock to serialize xmit and tx_timeout execution * @deferred_skb: holds an skb (for transmission at a later time) when the * Tx buffer is not free * @phy_dev: pointer to the PHY device -- GitLab From 8cf0a1bc12870d148ae830a4ba88cfdf0e879cee Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Tue, 25 Oct 2022 21:33:57 +0800 Subject: [PATCH 0666/1927] capabilities: fix potential memleak on error path from vfs_getxattr_alloc() In cap_inode_getsecurity(), we will use vfs_getxattr_alloc() to complete the memory allocation of tmpbuf, if we have completed the memory allocation of tmpbuf, but failed to call handler->get(...), there will be a memleak in below logic: |-- ret = (int)vfs_getxattr_alloc(mnt_userns, ...) | /* ^^^ alloc for tmpbuf */ |-- value = krealloc(*xattr_value, error + 1, flags) | /* ^^^ alloc memory */ |-- error = handler->get(handler, ...) | /* error! */ |-- *xattr_value = value | /* xattr_value is &tmpbuf (memory leak!) */ So we will try to free(tmpbuf) after vfs_getxattr_alloc() fails to fix it. Cc: stable@vger.kernel.org Fixes: 8db6c34f1dbc ("Introduce v3 namespaced file capabilities") Signed-off-by: Gaosheng Cui Acked-by: Serge Hallyn [PM: subject line and backtrace tweaks] Signed-off-by: Paul Moore --- security/commoncap.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/security/commoncap.c b/security/commoncap.c index 5fc8986c3c77c..bc751fa5adad7 100644 --- a/security/commoncap.c +++ b/security/commoncap.c @@ -401,8 +401,10 @@ int cap_inode_getsecurity(struct user_namespace *mnt_userns, &tmpbuf, size, GFP_NOFS); dput(dentry); - if (ret < 0 || !tmpbuf) - return ret; + if (ret < 0 || !tmpbuf) { + size = ret; + goto out_free; + } fs_ns = inode->i_sb->s_user_ns; cap = (struct vfs_cap_data *) tmpbuf; -- GitLab From 7353633814f6e5b4899fb9ee1483709d6bb0e1cd Mon Sep 17 00:00:00 2001 From: Eiichi Tsukata Date: Fri, 28 Oct 2022 09:26:31 +0000 Subject: [PATCH 0667/1927] KVM: x86/xen: Fix eventfd error handling in kvm_xen_eventfd_assign() Should not call eventfd_ctx_put() in case of error. Fixes: 2fd6df2f2b47 ("KVM: x86/xen: intercept EVTCHNOP_send from guests") Reported-by: syzbot+6f0c896c5a9449a10ded@syzkaller.appspotmail.com Signed-off-by: Eiichi Tsukata Message-Id: <20221028092631.117438-1-eiichi.tsukata@nutanix.com> [Introduce new goto target instead. - Paolo] Signed-off-by: Paolo Bonzini --- arch/x86/kvm/xen.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index b2be60c6efa4b..2dae413bd62a6 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1666,18 +1666,18 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, case EVTCHNSTAT_ipi: /* IPI must map back to the same port# */ if (data->u.evtchn.deliver.port.port != data->u.evtchn.send_port) - goto out; /* -EINVAL */ + goto out_noeventfd; /* -EINVAL */ break; case EVTCHNSTAT_interdomain: if (data->u.evtchn.deliver.port.port) { if (data->u.evtchn.deliver.port.port >= max_evtchn_port(kvm)) - goto out; /* -EINVAL */ + goto out_noeventfd; /* -EINVAL */ } else { eventfd = eventfd_ctx_fdget(data->u.evtchn.deliver.eventfd.fd); if (IS_ERR(eventfd)) { ret = PTR_ERR(eventfd); - goto out; + goto out_noeventfd; } } break; @@ -1717,6 +1717,7 @@ static int kvm_xen_eventfd_assign(struct kvm *kvm, out: if (eventfd) eventfd_ctx_put(eventfd); +out_noeventfd: kfree(evtchnfd); return ret; } -- GitLab From 13c30a755847c7e804e1bf755e66e3ff7b7f9367 Mon Sep 17 00:00:00 2001 From: Sjoerd Simons Date: Thu, 20 Oct 2022 09:56:24 +0800 Subject: [PATCH 0668/1927] soundwire: intel: Initialize clock stop timeout The bus->clk_stop_timeout member is only initialized to a non-zero value during the codec driver probe. This can lead to corner cases where this value remains pegged at zero when the bus suspends, which results in an endless loop in sdw_bus_wait_for_clk_prep_deprep(). Corner cases include configurations with no codecs described in the firmware, or delays in probing codec drivers. Initializing the default timeout to the smallest non-zero value avoid this problem and allows for the existing logic to be preserved: the bus->clk_stop_timeout is set as the maximum required by all codecs connected on the bus. Fixes: 1f2dcf3a154ac ("soundwire: intel: set dev_num_ida_min") Signed-off-by: Sjoerd Simons Reviewed-by: Pierre-Louis Bossart Reviewed-by: Chao Song Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20221020015624.1703950-1-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- drivers/soundwire/intel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 244209358784f..8c76541d553f1 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -1513,6 +1513,7 @@ static int intel_link_probe(struct auxiliary_device *auxdev, bus->link_id = auxdev->id; bus->dev_num_ida_min = INTEL_DEV_NUM_IDA_MIN; + bus->clk_stop_timeout = 1; sdw_cdns_probe(cdns); -- GitLab From f936fa7a954b262cb3908bbc8f01ba19dfaf9fbf Mon Sep 17 00:00:00 2001 From: Srinivas Kandagatla Date: Wed, 26 Oct 2022 12:02:05 +0100 Subject: [PATCH 0669/1927] soundwire: qcom: reinit broadcast completion For some reason we never reinit the broadcast completion, there is a danger that broadcast commands could be treated as completed by driver from previous complete status. Fix this by reinitializing the completion before sending a broadcast command. Fixes: ddea6cf7b619 ("soundwire: qcom: update register read/write routine") Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20221026110210.6575-2-srinivas.kandagatla@linaro.org Signed-off-by: Vinod Koul --- drivers/soundwire/qcom.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index b33d5db494a54..8f1a1eb017a2e 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -344,6 +344,9 @@ static int qcom_swrm_cmd_fifo_wr_cmd(struct qcom_swrm_ctrl *swrm, u8 cmd_data, if (swrm_wait_for_wr_fifo_avail(swrm)) return SDW_CMD_FAIL_OTHER; + if (cmd_id == SWR_BROADCAST_CMD_ID) + reinit_completion(&swrm->broadcast); + /* Its assumed that write is okay as we do not get any status back */ swrm->reg_write(swrm, SWRM_CMD_FIFO_WR_CMD, val); -- GitLab From 49a467310dc4fae591a3547860ee04d8730780f4 Mon Sep 17 00:00:00 2001 From: Srinivas Kandagatla Date: Wed, 26 Oct 2022 12:02:06 +0100 Subject: [PATCH 0670/1927] soundwire: qcom: check for outanding writes before doing a read Reading will increase the fifo count, so check for outstanding cmd wrt. write fifo depth to avoid overflow as read will also increase write fifo cnt. Fixes: a661308c34de ("soundwire: qcom: wait for fifo space to be available before read/write") Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20221026110210.6575-3-srinivas.kandagatla@linaro.org Signed-off-by: Vinod Koul --- drivers/soundwire/qcom.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 8f1a1eb017a2e..cee2b22231410 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -380,6 +380,12 @@ static int qcom_swrm_cmd_fifo_rd_cmd(struct qcom_swrm_ctrl *swrm, val = swrm_get_packed_reg_val(&swrm->rcmd_id, len, dev_addr, reg_addr); + /* + * Check for outstanding cmd wrt. write fifo depth to avoid + * overflow as read will also increase write fifo cnt. + */ + swrm_wait_for_wr_fifo_avail(swrm); + /* wait for FIFO RD to complete to avoid overflow */ usleep_range(100, 105); swrm->reg_write(swrm, SWRM_CMD_FIFO_RD_CMD, val); -- GitLab From 6ec27c53886c8963729885bcf2dd996eba2767a7 Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Fri, 28 Oct 2022 11:16:03 +0800 Subject: [PATCH 0671/1927] ASoC: core: Fix use-after-free in snd_soc_exit() KASAN reports a use-after-free: BUG: KASAN: use-after-free in device_del+0xb5b/0xc60 Read of size 8 at addr ffff888008655050 by task rmmod/387 CPU: 2 PID: 387 Comm: rmmod Hardware name: QEMU Standard PC (i440FX + PIIX, 1996) Call Trace: dump_stack_lvl+0x79/0x9a print_report+0x17f/0x47b kasan_report+0xbb/0xf0 device_del+0xb5b/0xc60 platform_device_del.part.0+0x24/0x200 platform_device_unregister+0x2e/0x40 snd_soc_exit+0xa/0x22 [snd_soc_core] __do_sys_delete_module.constprop.0+0x34f/0x5b0 do_syscall_64+0x3a/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd ... It's bacause in snd_soc_init(), snd_soc_util_init() is possble to fail, but its ret is ignored, which makes soc_dummy_dev unregistered twice. snd_soc_init() snd_soc_util_init() platform_device_register_simple(soc_dummy_dev) platform_driver_register() # fail platform_device_unregister(soc_dummy_dev) platform_driver_register() # success ... snd_soc_exit() snd_soc_util_exit() # soc_dummy_dev will be unregistered for second time To fix it, handle error and stop snd_soc_init() when util_init() fail. Also clean debugfs when util_init() or driver_register() fail. Fixes: fb257897bf20 ("ASoC: Work around allmodconfig failure") Signed-off-by: Chen Zhongjin Link: https://lore.kernel.org/r/20221028031603.59416-1-chenzhongjin@huawei.com Signed-off-by: Mark Brown --- sound/soc/soc-core.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index 12a82f5a3ff64..a409fbed8f34c 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -3477,10 +3477,23 @@ EXPORT_SYMBOL_GPL(snd_soc_of_get_dai_link_cpus); static int __init snd_soc_init(void) { + int ret; + snd_soc_debugfs_init(); - snd_soc_util_init(); + ret = snd_soc_util_init(); + if (ret) + goto err_util_init; - return platform_driver_register(&soc_driver); + ret = platform_driver_register(&soc_driver); + if (ret) + goto err_register; + return 0; + +err_register: + snd_soc_util_exit(); +err_util_init: + snd_soc_debugfs_exit(); + return ret; } module_init(snd_soc_init); -- GitLab From c1de355e04e6ae8da21cb6bfbd5bbfd8deee0fe5 Mon Sep 17 00:00:00 2001 From: Jay Fang Date: Fri, 28 Oct 2022 10:37:39 +0800 Subject: [PATCH 0672/1927] MAINTAINERS: Update HiSilicon SFC Driver maintainer Add Jay Fang as the maintainer of the HiSilicon SFC Driver, replacing John Garry. Signed-off-by: Jay Fang Acked-by: John Garry Link: https://lore.kernel.org/r/20221028023739.4113998-1-f.fangjian@huawei.com Signed-off-by: Mark Brown --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 653501e7aa03b..6297d5d1dd901 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9260,7 +9260,7 @@ S: Maintained F: drivers/crypto/hisilicon/trng/trng.c HISILICON V3XX SPI NOR FLASH Controller Driver -M: John Garry +M: Jay Fang S: Maintained W: http://www.hisilicon.com F: drivers/spi/spi-hisi-sfc-v3xx.c -- GitLab From 4a6f278d4827b59ba26ceae0ff4529ee826aa258 Mon Sep 17 00:00:00 2001 From: Miklos Szeredi Date: Fri, 28 Oct 2022 14:25:20 +0200 Subject: [PATCH 0673/1927] fuse: add file_modified() to fallocate Add missing file_modified() call to fuse_file_fallocate(). Without this fallocate on fuse failed to clear privileges. Fixes: 05ba1f082300 ("fuse: add FALLOCATE operation") Cc: Signed-off-by: Miklos Szeredi --- fs/fuse/file.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 1a3afd469e3a9..71bfb663aac58 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3001,6 +3001,10 @@ static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, goto out; } + err = file_modified(file); + if (err) + goto out; + if (!(mode & FALLOC_FL_KEEP_SIZE)) set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); -- GitLab From 04948e757148f870a31f4887ea2239403f516c3c Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Wed, 26 Oct 2022 18:21:16 +0200 Subject: [PATCH 0674/1927] phy: qcom-qmp-combo: fix NULL-deref on runtime resume Commit fc64623637da ("phy: qcom-qmp-combo,usb: add support for separate PCS_USB region") started treating the PCS_USB registers as potentially separate from the PCS registers but used the wrong base when no PCS_USB offset has been provided. Fix the PCS_USB base used at runtime resume to prevent dereferencing a NULL pointer on platforms that do not provide a PCS_USB offset (e.g. SC7180). Fixes: fc64623637da ("phy: qcom-qmp-combo,usb: add support for separate PCS_USB region") Cc: stable@vger.kernel.org # 5.20 Signed-off-by: Johan Hovold Reviewed-by: Dmitry Baryshkov Reviewed-by: Andrew Halaney Link: https://lore.kernel.org/r/20221026162116.26462-1-johan+linaro@kernel.org Signed-off-by: Vinod Koul --- drivers/phy/qualcomm/phy-qcom-qmp-combo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index 9807c4d935cdb..ba9d761ec49a7 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -2240,7 +2240,7 @@ static void qmp_combo_enable_autonomous_mode(struct qmp_phy *qphy) static void qmp_combo_disable_autonomous_mode(struct qmp_phy *qphy) { const struct qmp_phy_cfg *cfg = qphy->cfg; - void __iomem *pcs_usb = qphy->pcs_usb ?: qphy->pcs_usb; + void __iomem *pcs_usb = qphy->pcs_usb ?: qphy->pcs; void __iomem *pcs_misc = qphy->pcs_misc; /* Disable i/o clamp_n on resume for normal mode */ -- GitLab From 8d06679b25fc6813eb2438fac7fa13f4f3c2ef37 Mon Sep 17 00:00:00 2001 From: Stefan Binding Date: Fri, 28 Oct 2022 11:27:42 +0100 Subject: [PATCH 0675/1927] ALSA: hda/realtek: Add quirk for ASUS Zenbook using CS35L41 This Asus Zenbook laptop use Realtek HDA codec combined with 2xCS35L41 Amplifiers using I2C with Internal Boost. Signed-off-by: Stefan Binding Cc: Link: https://lore.kernel.org/r/20221028102742.2588687-1-sbinding@opensource.cirrus.com Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_realtek.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 701a72ec5629a..b4f7ff8cfe41b 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -9404,6 +9404,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x1e8e, "ASUS Zephyrus G15", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1c52, "ASUS Zephyrus G15 2022", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x1f11, "ASUS Zephyrus G14", ALC289_FIXUP_ASUS_GA401), + SND_PCI_QUIRK(0x1043, 0x1f12, "ASUS UM5302", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x1043, 0x1f92, "ASUS ROG Flow X16", ALC289_FIXUP_ASUS_GA401), SND_PCI_QUIRK(0x1043, 0x3030, "ASUS ZN270IE", ALC256_FIXUP_ASUS_AIO_GPIO2), SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC), -- GitLab From e3c5a78cdb6237bfb9641b63cccf366325229eec Mon Sep 17 00:00:00 2001 From: John Garry Date: Wed, 26 Oct 2022 18:35:13 +0800 Subject: [PATCH 0676/1927] blk-mq: Properly init requests from blk_mq_alloc_request_hctx() Function blk_mq_alloc_request_hctx() is missing zeroing/init of rq->bio, biotail, __sector, and __data_len members, which blk_mq_alloc_request() has, so duplicate what we do in blk_mq_alloc_request(). Fixes: 1f5bd336b9150 ("blk-mq: add blk_mq_alloc_request_hctx") Signed-off-by: John Garry Reviewed-by: Christoph Hellwig Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/1666780513-121650-1-git-send-email-john.garry@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 33292c01875d5..75c8296b6feb3 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -611,6 +611,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, .nr_tags = 1, }; u64 alloc_time_ns = 0; + struct request *rq; unsigned int cpu; unsigned int tag; int ret; @@ -660,8 +661,12 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, tag = blk_mq_get_tag(&data); if (tag == BLK_MQ_NO_TAG) goto out_queue_exit; - return blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag, + rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag, alloc_time_ns); + rq->__data_len = 0; + rq->__sector = (sector_t) -1; + rq->bio = rq->biotail = NULL; + return rq; out_queue_exit: blk_queue_exit(q); -- GitLab From 9ed88fcfb1b08c41bde0381dece84d152d53774c Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Fri, 28 Oct 2022 08:15:45 +0300 Subject: [PATCH 0677/1927] MAINTAINERS: Change myself to a maintainer After some off-list discussion with Marek Vasut and Geert Uytterhoeven and finally a kx022a driver related discussion with Joe Perches https://lore.kernel.org/lkml/92c3f72e60bc99bf4a21da259b4d78c1bdca447d.camel@perches.com/ it seems that my status as a reviewer has been wrong. I do look after the ROHM/Kionix drivers I've authored and currently I am also paid to do so as is reflected by the 'S: Supported'. According to Joe, the reviewer entry in MAINTAINERS do not indicate such level of support and having a reviewer supporting an IC is a contradiction. Switch undersigned from a reviewer to a maintainer for IC drivers I am taking care of. Signed-off-by: Matti Vaittinen Reviewed-by: Geert Uytterhoeven Signed-off-by: Bartosz Golaszewski --- MAINTAINERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..c23e535e14d6c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -17801,7 +17801,7 @@ S: Odd Fixes F: drivers/tty/serial/rp2.* ROHM BD99954 CHARGER IC -R: Matti Vaittinen +M: Matti Vaittinen S: Supported F: drivers/power/supply/bd99954-charger.c F: drivers/power/supply/bd99954-charger.h @@ -17824,7 +17824,7 @@ F: drivers/regulator/bd9571mwv-regulator.c F: include/linux/mfd/bd9571mwv.h ROHM POWER MANAGEMENT IC DEVICE DRIVERS -R: Matti Vaittinen +M: Matti Vaittinen S: Supported F: drivers/clk/clk-bd718x7.c F: drivers/gpio/gpio-bd71815.c -- GitLab From 2124becad797245d49252d2d733aee0322233d7e Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Fri, 14 Oct 2022 07:11:36 -0500 Subject: [PATCH 0678/1927] ACPI: x86: Add another system to quirk list for forcing StorageD3Enable commit 018d6711c26e4 ("ACPI: x86: Add a quirk for Dell Inspiron 14 2-in-1 for StorageD3Enable") introduced a quirk to allow a system with ambiguous use of _ADR 0 to force StorageD3Enable. Julius Brockmann reports that Inspiron 16 5625 suffers that same symptoms. Add this other system to the list as well. Link: https://bugzilla.kernel.org/show_bug.cgi?id=216440 Reported-and-tested-by: Julius Brockmann Signed-off-by: Mario Limonciello Signed-off-by: Rafael J. Wysocki --- drivers/acpi/x86/utils.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c index f8a2cbdc0ce2b..d7d3f1669d4c0 100644 --- a/drivers/acpi/x86/utils.c +++ b/drivers/acpi/x86/utils.c @@ -219,6 +219,12 @@ static const struct dmi_system_id force_storage_d3_dmi[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 14 7425 2-in-1"), } }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 16 5625"), + } + }, {} }; -- GitLab From 07c0d131cc0fe1f3981a42958fc52d573d303d89 Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Tue, 25 Oct 2022 10:41:46 +0800 Subject: [PATCH 0679/1927] RDMA/core: Fix null-ptr-deref in ib_core_cleanup() KASAN reported a null-ptr-deref error: KASAN: null-ptr-deref in range [0x0000000000000118-0x000000000000011f] CPU: 1 PID: 379 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996) RIP: 0010:destroy_workqueue+0x2f/0x740 RSP: 0018:ffff888016137df8 EFLAGS: 00000202 ... Call Trace: ib_core_cleanup+0xa/0xa1 [ib_core] __do_sys_delete_module.constprop.0+0x34f/0x5b0 do_syscall_64+0x3a/0x90 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7fa1a0d221b7 ... It is because the fail of roce_gid_mgmt_init() is ignored: ib_core_init() roce_gid_mgmt_init() gid_cache_wq = alloc_ordered_workqueue # fail ... ib_core_cleanup() roce_gid_mgmt_cleanup() destroy_workqueue(gid_cache_wq) # destroy an unallocated wq Fix this by catching the fail of roce_gid_mgmt_init() in ib_core_init(). Fixes: 03db3a2d81e6 ("IB/core: Add RoCE GID table management") Signed-off-by: Chen Zhongjin Link: https://lore.kernel.org/r/20221025024146.109137-1-chenzhongjin@huawei.com Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/core/device.c | 10 +++++++++- drivers/infiniband/core/nldev.c | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index ae60c73babcc5..b69e2c4e4d2a4 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2815,10 +2815,18 @@ static int __init ib_core_init(void) nldev_init(); rdma_nl_register(RDMA_NL_LS, ibnl_ls_cb_table); - roce_gid_mgmt_init(); + ret = roce_gid_mgmt_init(); + if (ret) { + pr_warn("Couldn't init RoCE GID management\n"); + goto err_parent; + } return 0; +err_parent: + rdma_nl_unregister(RDMA_NL_LS); + nldev_exit(); + unregister_pernet_device(&rdma_dev_net_ops); err_compat: unregister_blocking_lsm_notifier(&ibdev_lsm_nb); err_sa: diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index b92358f606d00..12dc97067ed2b 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -2537,7 +2537,7 @@ void __init nldev_init(void) rdma_nl_register(RDMA_NL_NLDEV, nldev_cb_table); } -void __exit nldev_exit(void) +void nldev_exit(void) { rdma_nl_unregister(RDMA_NL_NLDEV); } -- GitLab From 7a47e077e503feb73d56e491ce89aa73b67a3972 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Tue, 25 Oct 2022 18:32:32 +0300 Subject: [PATCH 0680/1927] RDMA/qedr: clean up work queue on failure in qedr_alloc_resources() Add a check for if create_singlethread_workqueue() fails and also destroy the work queue on failure paths. Fixes: e411e0587e0d ("RDMA/qedr: Add iWARP connection management functions") Signed-off-by: Dan Carpenter Link: https://lore.kernel.org/r/Y1gBkDucQhhWj5YM@kili Signed-off-by: Leon Romanovsky Signed-off-by: Jason Gunthorpe --- drivers/infiniband/hw/qedr/main.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index 5152f10d2e6de..ba0c3e4c07d85 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -344,6 +344,10 @@ static int qedr_alloc_resources(struct qedr_dev *dev) if (IS_IWARP(dev)) { xa_init(&dev->qps); dev->iwarp_wq = create_singlethread_workqueue("qedr_iwarpq"); + if (!dev->iwarp_wq) { + rc = -ENOMEM; + goto err1; + } } /* Allocate Status blocks for CNQ */ @@ -351,7 +355,7 @@ static int qedr_alloc_resources(struct qedr_dev *dev) GFP_KERNEL); if (!dev->sb_array) { rc = -ENOMEM; - goto err1; + goto err_destroy_wq; } dev->cnq_array = kcalloc(dev->num_cnq, @@ -402,6 +406,9 @@ static int qedr_alloc_resources(struct qedr_dev *dev) kfree(dev->cnq_array); err2: kfree(dev->sb_array); +err_destroy_wq: + if (IS_IWARP(dev)) + destroy_workqueue(dev->iwarp_wq); err1: kfree(dev->sgid_tbl); return rc; -- GitLab From 6a564338a23cefcfc29c4a535b98402d13efdda6 Mon Sep 17 00:00:00 2001 From: Maarten Zanders Date: Fri, 28 Oct 2022 16:11:28 +0200 Subject: [PATCH 0681/1927] ASoC: fsl_asrc fsl_esai fsl_sai: allow CONFIG_PM=N When CONFIG_PM=N, pm_runtime_put_sync() returns -ENOSYS which breaks the probe function of these drivers. Other users of pm_runtime_put_sync() typically don't check the return value. In order to keep the program flow as intended, check for -ENOSYS. This commit is similar to commit 0434d3f (omap-mailbox.c). Fixes: cab04ab5900f ("ASoC: fsl_asrc: Don't use devm_regmap_init_mmio_clk") Fixes: 203773e39347 ("ASoC: fsl_esai: Don't use devm_regmap_init_mmio_clk") Fixes: 2277e7e36b4b ("ASoC: fsl_sai: Don't use devm_regmap_init_mmio_clk") Signed-off-by: Maarten Zanders Reviewed-by: Daniel Baluta Link: https://lore.kernel.org/r/20221028141129.100702-1-maarten.zanders@mind.be Signed-off-by: Mark Brown --- sound/soc/fsl/fsl_asrc.c | 2 +- sound/soc/fsl/fsl_esai.c | 2 +- sound/soc/fsl/fsl_sai.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c index 936aef5d2767c..e16e7b3fa96cc 100644 --- a/sound/soc/fsl/fsl_asrc.c +++ b/sound/soc/fsl/fsl_asrc.c @@ -1232,7 +1232,7 @@ static int fsl_asrc_probe(struct platform_device *pdev) } ret = pm_runtime_put_sync(&pdev->dev); - if (ret < 0) + if (ret < 0 && ret != -ENOSYS) goto err_pm_get_sync; ret = devm_snd_soc_register_component(&pdev->dev, &fsl_asrc_component, diff --git a/sound/soc/fsl/fsl_esai.c b/sound/soc/fsl/fsl_esai.c index 5c21fc490fce1..17fefd27ec90a 100644 --- a/sound/soc/fsl/fsl_esai.c +++ b/sound/soc/fsl/fsl_esai.c @@ -1069,7 +1069,7 @@ static int fsl_esai_probe(struct platform_device *pdev) regmap_write(esai_priv->regmap, REG_ESAI_RSMB, 0); ret = pm_runtime_put_sync(&pdev->dev); - if (ret < 0) + if (ret < 0 && ret != -ENOSYS) goto err_pm_get_sync; /* diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c index 81f89f6767a2d..e60c7b3445623 100644 --- a/sound/soc/fsl/fsl_sai.c +++ b/sound/soc/fsl/fsl_sai.c @@ -1446,7 +1446,7 @@ static int fsl_sai_probe(struct platform_device *pdev) } ret = pm_runtime_put_sync(dev); - if (ret < 0) + if (ret < 0 && ret != -ENOSYS) goto err_pm_get_sync; /* -- GitLab From f11a74b45d330ad1ab986852b099747161052526 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 27 Oct 2022 15:52:31 +0200 Subject: [PATCH 0682/1927] efi: efivars: Fix variable writes with unsupported query_variable_store() Commit 8a254d90a775 ("efi: efivars: Fix variable writes without query_variable_store()") addressed an issue that was introduced during the EFI variable store refactor, where alternative implementations of the efivars layer that lacked query_variable_store() would no longer work. Unfortunately, there is another case to consider here, which was missed: if the efivars layer is backed by the EFI runtime services as usual, but the EFI implementation predates the introduction of QueryVariableInfo(), we will return EFI_UNSUPPORTED, and this is no longer being dealt with correctly. So let's fix this, and while at it, clean up the code a bit, by merging the check_var_size() routines as well as their callers. Cc: # v6.0 Fixes: bbc6d2c6ef22 ("efi: vars: Switch to new wrapper layer") Signed-off-by: Ard Biesheuvel Tested-by: Aditya Garg --- drivers/firmware/efi/vars.c | 68 +++++++++++-------------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/drivers/firmware/efi/vars.c b/drivers/firmware/efi/vars.c index 433b615871395..0ba9f18312f5b 100644 --- a/drivers/firmware/efi/vars.c +++ b/drivers/firmware/efi/vars.c @@ -21,29 +21,22 @@ static struct efivars *__efivars; static DEFINE_SEMAPHORE(efivars_lock); -static efi_status_t check_var_size(u32 attributes, unsigned long size) -{ - const struct efivar_operations *fops; - - fops = __efivars->ops; - - if (!fops->query_variable_store) - return (size <= SZ_64K) ? EFI_SUCCESS : EFI_OUT_OF_RESOURCES; - - return fops->query_variable_store(attributes, size, false); -} - -static -efi_status_t check_var_size_nonblocking(u32 attributes, unsigned long size) +static efi_status_t check_var_size(bool nonblocking, u32 attributes, + unsigned long size) { const struct efivar_operations *fops; + efi_status_t status; fops = __efivars->ops; if (!fops->query_variable_store) + status = EFI_UNSUPPORTED; + else + status = fops->query_variable_store(attributes, size, + nonblocking); + if (status == EFI_UNSUPPORTED) return (size <= SZ_64K) ? EFI_SUCCESS : EFI_OUT_OF_RESOURCES; - - return fops->query_variable_store(attributes, size, true); + return status; } /** @@ -195,26 +188,6 @@ efi_status_t efivar_get_next_variable(unsigned long *name_size, } EXPORT_SYMBOL_NS_GPL(efivar_get_next_variable, EFIVAR); -/* - * efivar_set_variable_blocking() - local helper function for set_variable - * - * Must be called with efivars_lock held. - */ -static efi_status_t -efivar_set_variable_blocking(efi_char16_t *name, efi_guid_t *vendor, - u32 attr, unsigned long data_size, void *data) -{ - efi_status_t status; - - if (data_size > 0) { - status = check_var_size(attr, data_size + - ucs2_strsize(name, 1024)); - if (status != EFI_SUCCESS) - return status; - } - return __efivars->ops->set_variable(name, vendor, attr, data_size, data); -} - /* * efivar_set_variable_locked() - set a variable identified by name/vendor * @@ -228,23 +201,21 @@ efi_status_t efivar_set_variable_locked(efi_char16_t *name, efi_guid_t *vendor, efi_set_variable_t *setvar; efi_status_t status; - if (!nonblocking) - return efivar_set_variable_blocking(name, vendor, attr, - data_size, data); + if (data_size > 0) { + status = check_var_size(nonblocking, attr, + data_size + ucs2_strsize(name, 1024)); + if (status != EFI_SUCCESS) + return status; + } /* * If no _nonblocking variant exists, the ordinary one * is assumed to be non-blocking. */ - setvar = __efivars->ops->set_variable_nonblocking ?: - __efivars->ops->set_variable; + setvar = __efivars->ops->set_variable_nonblocking; + if (!setvar || !nonblocking) + setvar = __efivars->ops->set_variable; - if (data_size > 0) { - status = check_var_size_nonblocking(attr, data_size + - ucs2_strsize(name, 1024)); - if (status != EFI_SUCCESS) - return status; - } return setvar(name, vendor, attr, data_size, data); } EXPORT_SYMBOL_NS_GPL(efivar_set_variable_locked, EFIVAR); @@ -264,7 +235,8 @@ efi_status_t efivar_set_variable(efi_char16_t *name, efi_guid_t *vendor, if (efivar_lock()) return EFI_ABORTED; - status = efivar_set_variable_blocking(name, vendor, attr, data_size, data); + status = efivar_set_variable_locked(name, vendor, attr, data_size, + data, false); efivar_unlock(); return status; } -- GitLab From ae4b3c1252f0fd0951d2f072a02ba46cac8d6c92 Mon Sep 17 00:00:00 2001 From: Jon Hunter Date: Wed, 26 Oct 2022 16:56:33 +0100 Subject: [PATCH 0683/1927] spi: tegra210-quad: Don't initialise DMA if not supported The following error messages are observed on boot for Tegra234 ... ERR KERN tegra-qspi 3270000.spi: cannot use DMA: -19 ERR KERN tegra-qspi 3270000.spi: falling back to PIO Tegra234 does not support DMA for the QSPI and so initialising the DMA is expected to fail. The above error messages are misleading for devices that don't support DMA and so fix this by skipping the DMA initialisation for devices that don't support DMA. Signed-off-by: Jon Hunter Acked-by: Thierry Reding Link: https://lore.kernel.org/r/20221026155633.141792-1-jonathanh@nvidia.com Signed-off-by: Mark Brown --- drivers/spi/spi-tegra210-quad.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c index 904972606bd45..10f0c5a6e0dce 100644 --- a/drivers/spi/spi-tegra210-quad.c +++ b/drivers/spi/spi-tegra210-quad.c @@ -720,6 +720,9 @@ static int tegra_qspi_start_cpu_based_transfer(struct tegra_qspi *qspi, struct s static void tegra_qspi_deinit_dma(struct tegra_qspi *tqspi) { + if (!tqspi->soc_data->has_dma) + return; + if (tqspi->tx_dma_buf) { dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, tqspi->tx_dma_buf, tqspi->tx_dma_phys); @@ -750,6 +753,9 @@ static int tegra_qspi_init_dma(struct tegra_qspi *tqspi) u32 *dma_buf; int err; + if (!tqspi->soc_data->has_dma) + return 0; + dma_chan = dma_request_chan(tqspi->dev, "rx"); if (IS_ERR(dma_chan)) { err = PTR_ERR(dma_chan); -- GitLab From 9ef8eb6104527bfe9ed31f7a4ffa721390adf9a8 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Thu, 20 Oct 2022 23:36:14 +0100 Subject: [PATCH 0684/1927] squashfs: fix read regression introduced in readahead code Patch series "squashfs: fix some regressions introduced in the readahead code". This patchset fixes 3 regressions introduced by the recent readahead code changes. The first regression is causing "snaps" to randomly fail after a couple of hours or days, which how the regression came to light. This patch (of 3): If a file isn't a whole multiple of the page size, the last page will have trailing bytes unfilled. There was a mistake in the readahead code which did this. In particular it incorrectly assumed that the last page in the readahead page array (page[nr_pages - 1]) will always contain the last page in the block, which if we're at file end, will be the page that needs to be zero filled. But the readahead code may not return the last page in the block, which means it is unmapped and will be skipped by the decompressors (a temporary buffer used). In this case the zero filling code will zero out the wrong page, leading to data corruption. Fix this by by extending the "page actor" to return the last page if present, or NULL if a temporary buffer was used. Link: https://lkml.kernel.org/r/20221020223616.7571-1-phillip@squashfs.org.uk Link: https://lkml.kernel.org/r/20221020223616.7571-2-phillip@squashfs.org.uk Fixes: 8fc78b6fe24c ("squashfs: implement readahead") Link: https://lore.kernel.org/lkml/b0c258c3-6dcf-aade-efc4-d62a8b3a1ce2@alu.unizg.hr/ Signed-off-by: Phillip Lougher Reported-by: Mirsad Goran Todorovac Tested-by: Mirsad Goran Todorovac Tested-by: Slade Watkins Tested-by: Bagas Sanjaya Reported-by: Marc Miltenberger Cc: Dimitri John Ledkov Cc: Hsin-Yi Wang Cc: Thorsten Leemhuis Cc: Signed-off-by: Andrew Morton --- fs/squashfs/file.c | 7 ++++--- fs/squashfs/page_actor.c | 3 +++ fs/squashfs/page_actor.h | 6 +++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index e56510964b229..e526eb7a1658a 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -557,6 +557,7 @@ static void squashfs_readahead(struct readahead_control *ractl) int res, bsize; u64 block = 0; unsigned int expected; + struct page *last_page; nr_pages = __readahead_batch(ractl, pages, max_pages); if (!nr_pages) @@ -593,15 +594,15 @@ static void squashfs_readahead(struct readahead_control *ractl) res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor); - squashfs_page_actor_free(actor); + last_page = squashfs_page_actor_free(actor); if (res == expected) { int bytes; /* Last page (if present) may have trailing bytes not filled */ bytes = res % PAGE_SIZE; - if (pages[nr_pages - 1]->index == file_end && bytes) - memzero_page(pages[nr_pages - 1], bytes, + if (index == file_end && bytes && last_page) + memzero_page(last_page, bytes, PAGE_SIZE - bytes); for (i = 0; i < nr_pages; i++) { diff --git a/fs/squashfs/page_actor.c b/fs/squashfs/page_actor.c index 54b93bf4a25c1..81af6c4ca1157 100644 --- a/fs/squashfs/page_actor.c +++ b/fs/squashfs/page_actor.c @@ -71,11 +71,13 @@ static void *handle_next_page(struct squashfs_page_actor *actor) (actor->next_index != actor->page[actor->next_page]->index)) { actor->next_index++; actor->returned_pages++; + actor->last_page = NULL; return actor->alloc_buffer ? actor->tmp_buffer : ERR_PTR(-ENOMEM); } actor->next_index++; actor->returned_pages++; + actor->last_page = actor->page[actor->next_page]; return actor->pageaddr = kmap_local_page(actor->page[actor->next_page++]); } @@ -125,6 +127,7 @@ struct squashfs_page_actor *squashfs_page_actor_init_special(struct squashfs_sb_ actor->returned_pages = 0; actor->next_index = page[0]->index & ~((1 << (msblk->block_log - PAGE_SHIFT)) - 1); actor->pageaddr = NULL; + actor->last_page = NULL; actor->alloc_buffer = msblk->decompressor->alloc_buffer; actor->squashfs_first_page = direct_first_page; actor->squashfs_next_page = direct_next_page; diff --git a/fs/squashfs/page_actor.h b/fs/squashfs/page_actor.h index 95ffbb543d913..97d4983559b19 100644 --- a/fs/squashfs/page_actor.h +++ b/fs/squashfs/page_actor.h @@ -16,6 +16,7 @@ struct squashfs_page_actor { void *(*squashfs_first_page)(struct squashfs_page_actor *); void *(*squashfs_next_page)(struct squashfs_page_actor *); void (*squashfs_finish_page)(struct squashfs_page_actor *); + struct page *last_page; int pages; int length; int next_page; @@ -29,10 +30,13 @@ extern struct squashfs_page_actor *squashfs_page_actor_init(void **buffer, extern struct squashfs_page_actor *squashfs_page_actor_init_special( struct squashfs_sb_info *msblk, struct page **page, int pages, int length); -static inline void squashfs_page_actor_free(struct squashfs_page_actor *actor) +static inline struct page *squashfs_page_actor_free(struct squashfs_page_actor *actor) { + struct page *last_page = actor->last_page; + kfree(actor->tmp_buffer); kfree(actor); + return last_page; } static inline void *squashfs_first_page(struct squashfs_page_actor *actor) { -- GitLab From c9199de82bad03bceb94ec3c5195c879d7e11911 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Thu, 20 Oct 2022 23:36:15 +0100 Subject: [PATCH 0685/1927] squashfs: fix extending readahead beyond end of file The readahead code will try to extend readahead to the entire size of the Squashfs data block. But, it didn't take into account that the last block at the end of the file may not be a whole block. In this case, the code would extend readahead to beyond the end of the file, leaving trailing pages. Fix this by only requesting the expected number of pages. Link: https://lkml.kernel.org/r/20221020223616.7571-3-phillip@squashfs.org.uk Fixes: 8fc78b6fe24c ("squashfs: implement readahead") Signed-off-by: Phillip Lougher Tested-by: Bagas Sanjaya Reported-by: Marc Miltenberger Cc: Dimitri John Ledkov Cc: Hsin-Yi Wang Cc: Mirsad Goran Todorovac Cc: Slade Watkins Cc: Thorsten Leemhuis Cc: Signed-off-by: Andrew Morton --- fs/squashfs/file.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index e526eb7a1658a..f0afd4d6fd309 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -559,6 +559,12 @@ static void squashfs_readahead(struct readahead_control *ractl) unsigned int expected; struct page *last_page; + expected = start >> msblk->block_log == file_end ? + (i_size_read(inode) & (msblk->block_size - 1)) : + msblk->block_size; + + max_pages = (expected + PAGE_SIZE - 1) >> PAGE_SHIFT; + nr_pages = __readahead_batch(ractl, pages, max_pages); if (!nr_pages) break; @@ -567,13 +573,10 @@ static void squashfs_readahead(struct readahead_control *ractl) goto skip_pages; index = pages[0]->index >> shift; + if ((pages[nr_pages - 1]->index >> shift) != index) goto skip_pages; - expected = index == file_end ? - (i_size_read(inode) & (msblk->block_size - 1)) : - msblk->block_size; - if (index == file_end && squashfs_i(inode)->fragment_block != SQUASHFS_INVALID_BLK) { res = squashfs_readahead_fragment(pages, nr_pages, -- GitLab From e11c4e088be4c39d17f304fcf331670891905f42 Mon Sep 17 00:00:00 2001 From: Phillip Lougher Date: Thu, 20 Oct 2022 23:36:16 +0100 Subject: [PATCH 0686/1927] squashfs: fix buffer release race condition in readahead code Fix a buffer release race condition, where the error value was used after release. Link: https://lkml.kernel.org/r/20221020223616.7571-4-phillip@squashfs.org.uk Fixes: b09a7a036d20 ("squashfs: support reading fragments in readahead call") Signed-off-by: Phillip Lougher Tested-by: Bagas Sanjaya Reported-by: Marc Miltenberger Cc: Dimitri John Ledkov Cc: Hsin-Yi Wang Cc: Mirsad Goran Todorovac Cc: Slade Watkins Cc: Thorsten Leemhuis Cc: Signed-off-by: Andrew Morton --- fs/squashfs/file.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c index f0afd4d6fd309..8ba8c4c507707 100644 --- a/fs/squashfs/file.c +++ b/fs/squashfs/file.c @@ -506,8 +506,9 @@ static int squashfs_readahead_fragment(struct page **page, squashfs_i(inode)->fragment_size); struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; unsigned int n, mask = (1 << (msblk->block_log - PAGE_SHIFT)) - 1; + int error = buffer->error; - if (buffer->error) + if (error) goto out; expected += squashfs_i(inode)->fragment_offset; @@ -529,7 +530,7 @@ static int squashfs_readahead_fragment(struct page **page, out: squashfs_cache_put(buffer); - return buffer->error; + return error; } static void squashfs_readahead(struct readahead_control *ractl) -- GitLab From 984a608377cb623351b8a3670b285f32ebeb2d32 Mon Sep 17 00:00:00 2001 From: Waiman Long Date: Thu, 20 Oct 2022 13:56:19 -0400 Subject: [PATCH 0687/1927] mm/kmemleak: prevent soft lockup in kmemleak_scan()'s object iteration loops Commit 6edda04ccc7c ("mm/kmemleak: prevent soft lockup in first object iteration loop of kmemleak_scan()") adds cond_resched() in the first object iteration loop of kmemleak_scan(). However, it turns that the 2nd objection iteration loop can still cause soft lockup to happen in some cases. So add a cond_resched() call in the 2nd and 3rd loops as well to prevent that and for completeness. Link: https://lkml.kernel.org/r/20221020175619.366317-1-longman@redhat.com Fixes: 6edda04ccc7c ("mm/kmemleak: prevent soft lockup in first object iteration loop of kmemleak_scan()") Signed-off-by: Waiman Long Cc: Catalin Marinas Cc: Muchun Song Cc: Signed-off-by: Andrew Morton --- mm/kmemleak.c | 61 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 37af2dc8dac93..646e2979641fb 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1460,6 +1460,27 @@ static void scan_gray_list(void) WARN_ON(!list_empty(&gray_list)); } +/* + * Conditionally call resched() in a object iteration loop while making sure + * that the given object won't go away without RCU read lock by performing a + * get_object() if !pinned. + * + * Return: false if can't do a cond_resched() due to get_object() failure + * true otherwise + */ +static bool kmemleak_cond_resched(struct kmemleak_object *object, bool pinned) +{ + if (!pinned && !get_object(object)) + return false; + + rcu_read_unlock(); + cond_resched(); + rcu_read_lock(); + if (!pinned) + put_object(object); + return true; +} + /* * Scan data sections and all the referenced memory blocks allocated via the * kernel's standard allocators. This function must be called with the @@ -1471,7 +1492,7 @@ static void kmemleak_scan(void) struct zone *zone; int __maybe_unused i; int new_leaks = 0; - int loop1_cnt = 0; + int loop_cnt = 0; jiffies_last_scan = jiffies; @@ -1480,7 +1501,6 @@ static void kmemleak_scan(void) list_for_each_entry_rcu(object, &object_list, object_list) { bool obj_pinned = false; - loop1_cnt++; raw_spin_lock_irq(&object->lock); #ifdef DEBUG /* @@ -1514,24 +1534,11 @@ static void kmemleak_scan(void) raw_spin_unlock_irq(&object->lock); /* - * Do a cond_resched() to avoid soft lockup every 64k objects. - * Make sure a reference has been taken so that the object - * won't go away without RCU read lock. + * Do a cond_resched() every 64k objects to avoid soft lockup. */ - if (!(loop1_cnt & 0xffff)) { - if (!obj_pinned && !get_object(object)) { - /* Try the next object instead */ - loop1_cnt--; - continue; - } - - rcu_read_unlock(); - cond_resched(); - rcu_read_lock(); - - if (!obj_pinned) - put_object(object); - } + if (!(++loop_cnt & 0xffff) && + !kmemleak_cond_resched(object, obj_pinned)) + loop_cnt--; /* Try again on next object */ } rcu_read_unlock(); @@ -1598,7 +1605,15 @@ static void kmemleak_scan(void) * scan and color them gray until the next scan. */ rcu_read_lock(); + loop_cnt = 0; list_for_each_entry_rcu(object, &object_list, object_list) { + /* + * Do a cond_resched() every 64k objects to avoid soft lockup. + */ + if (!(++loop_cnt & 0xffff) && + !kmemleak_cond_resched(object, false)) + loop_cnt--; /* Try again on next object */ + /* * This is racy but we can save the overhead of lock/unlock * calls. The missed objects, if any, should be caught in @@ -1632,7 +1647,15 @@ static void kmemleak_scan(void) * Scanning result reporting. */ rcu_read_lock(); + loop_cnt = 0; list_for_each_entry_rcu(object, &object_list, object_list) { + /* + * Do a cond_resched() every 64k objects to avoid soft lockup. + */ + if (!(++loop_cnt & 0xffff) && + !kmemleak_cond_resched(object, false)) + loop_cnt--; /* Try again on next object */ + /* * This is racy but we can save the overhead of lock/unlock * calls. The missed objects, if any, should be caught in -- GitLab From b214fadff28d20f96456b73fe93340cb581ca891 Mon Sep 17 00:00:00 2001 From: Palmer Dabbelt Date: Thu, 20 Oct 2022 11:42:55 +0900 Subject: [PATCH 0688/1927] MAINTAINERS: git://github.com -> https://github.com for nilfs2 Github deprecated the git:// links about a year ago, so let's move to the https:// URLs instead. Link: https://lkml.kernel.org/r/20221020024255.5000-1-konishi.ryusuke@gmail.com Link: https://github.blog/2021-09-01-improving-git-protocol-security-github/ Link: https://lkml.kernel.org/r/20221013214638.30933-1-palmer@rivosinc.com Signed-off-by: Palmer Dabbelt Signed-off-by: Ryusuke Konishi Reported-by: Conor Dooley Signed-off-by: Andrew Morton --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index cf0f185023724..fe447637bf033 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14520,7 +14520,7 @@ L: linux-nilfs@vger.kernel.org S: Supported W: https://nilfs.sourceforge.io/ W: https://nilfs.osdn.jp/ -T: git git://github.com/konis/nilfs2.git +T: git https://github.com/konis/nilfs2.git F: Documentation/filesystems/nilfs2.rst F: fs/nilfs2/ F: include/trace/events/nilfs2.h -- GitLab From 27d676a1c2010450d00d514a8a6c1c780cb8d77f Mon Sep 17 00:00:00 2001 From: Huang Ying Date: Thu, 20 Oct 2022 09:51:22 +0800 Subject: [PATCH 0689/1927] memory tier, sysfs: rename attribute "nodes" to "nodelist" In sysfs, we use attribute name "cpumap" or "cpus" for cpu mask and "cpulist" or "cpus_list" for cpu list. For example, in my system, $ cat /sys/devices/system/node/node0/cpumap f,ffffffff $ cat /sys/devices/system/cpu/cpu2/topology/core_cpus 0,00100004 $ cat cat /sys/devices/system/node/node0/cpulist 0-35 $ cat /sys/devices/system/cpu/cpu2/topology/core_cpus_list 2,20 It looks reasonable to use "nodemap" for node mask and "nodelist" for node list. So, rename the attribute to follow the naming convention. Link: https://lkml.kernel.org/r/20221020015122.290097-1-ying.huang@intel.com Fixes: 9832fb87834e2b ("mm/demotion: expose memory tier details via sysfs") Signed-off-by: "Huang, Ying" Acked-by: Wei Xu Reviewed-by: Aneesh Kumar K.V Reviewed-by: Yang Shi Reviewed-by: Davidlohr Bueso Cc: Alistair Popple Cc: Bharata B Rao Cc: Dan Williams Cc: Dave Hansen Cc: Hesham Almatary Cc: Jagdish Gediya Cc: Johannes Weiner Cc: Jonathan Cameron Cc: Michal Hocko Cc: Tim Chen Signed-off-by: Andrew Morton --- Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers | 4 ++-- mm/memory-tiers.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers index 45985e411f13d..721a05b901097 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers +++ b/Documentation/ABI/testing/sysfs-kernel-mm-memory-tiers @@ -10,7 +10,7 @@ Description: A collection of all the memory tiers allocated. What: /sys/devices/virtual/memory_tiering/memory_tierN/ - /sys/devices/virtual/memory_tiering/memory_tierN/nodes + /sys/devices/virtual/memory_tiering/memory_tierN/nodelist Date: August 2022 Contact: Linux memory management mailing list Description: Directory with details of a specific memory tier @@ -21,5 +21,5 @@ Description: Directory with details of a specific memory tier A smaller value of N implies a higher (faster) memory tier in the hierarchy. - nodes: NUMA nodes that are part of this memory tier. + nodelist: NUMA nodes that are part of this memory tier. diff --git a/mm/memory-tiers.c b/mm/memory-tiers.c index f116b7b6333e5..fa8c9d07f9ce9 100644 --- a/mm/memory-tiers.c +++ b/mm/memory-tiers.c @@ -131,8 +131,8 @@ static void memory_tier_device_release(struct device *dev) kfree(tier); } -static ssize_t nodes_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t nodelist_show(struct device *dev, + struct device_attribute *attr, char *buf) { int ret; nodemask_t nmask; @@ -143,10 +143,10 @@ static ssize_t nodes_show(struct device *dev, mutex_unlock(&memory_tier_lock); return ret; } -static DEVICE_ATTR_RO(nodes); +static DEVICE_ATTR_RO(nodelist); static struct attribute *memtier_dev_attrs[] = { - &dev_attr_nodes.attr, + &dev_attr_nodelist.attr, NULL }; -- GitLab From 64b4c411a6c7a5f27555bfc2d6310b87bde3db67 Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Thu, 20 Oct 2022 21:19:22 -0700 Subject: [PATCH 0690/1927] ipc/msg.c: fix percpu_counter use after free These percpu counters are referenced in free_ipcs->freeque, so destroy them later. Fixes: 72d1e611082e ("ipc/msg: mitigate the lock contention with percpu counter") Reported-by: syzbot+96e659d35b9d6b541152@syzkaller.appspotmail.com Tested-by: Mark Rutland Cc: Jiebin Sun Signed-off-by: Andrew Morton --- ipc/msg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ipc/msg.c b/ipc/msg.c index e4e0990e08f75..fd08b3cb36d79 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -1329,11 +1329,11 @@ int msg_init_ns(struct ipc_namespace *ns) #ifdef CONFIG_IPC_NS void msg_exit_ns(struct ipc_namespace *ns) { - percpu_counter_destroy(&ns->percpu_msg_bytes); - percpu_counter_destroy(&ns->percpu_msg_hdrs); free_ipcs(ns, &msg_ids(ns), freeque); idr_destroy(&ns->ids[IPC_MSG_IDS].ipcs_idr); rhashtable_destroy(&ns->ids[IPC_MSG_IDS].key_ht); + percpu_counter_destroy(&ns->percpu_msg_bytes); + percpu_counter_destroy(&ns->percpu_msg_hdrs); } #endif -- GitLab From bb2282cf01fcae7379314dc026f3b534c83c186c Mon Sep 17 00:00:00 2001 From: Andrew Morton Date: Fri, 21 Oct 2022 08:05:49 -0700 Subject: [PATCH 0691/1927] fs/ext4/super.c: remove unused `deprecated_msg' fs/ext4/super.c:1744:19: warning: 'deprecated_msg' defined but not used [-Wunused-const-variable=] Reported-by: kernel test robot Cc: Theodore Ts'o Signed-off-by: Andrew Morton --- fs/ext4/super.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index 989365b878a67..7950904fbf04f 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -1741,10 +1741,6 @@ static const struct fs_parameter_spec ext4_param_specs[] = { #define DEFAULT_JOURNAL_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3)) -static const char deprecated_msg[] = - "Mount option \"%s\" will be removed by %s\n" - "Contact linux-ext4@vger.kernel.org if you think we should keep it.\n"; - #define MOPT_SET 0x0001 #define MOPT_CLEAR 0x0002 #define MOPT_NOSUPPORT 0x0004 -- GitLab From fba4eaf93164a6a6eb3cc12a3391b06f6187aa20 Mon Sep 17 00:00:00 2001 From: Maria Yu Date: Fri, 21 Oct 2022 18:15:55 +0800 Subject: [PATCH 0692/1927] mm/page_isolation: fix clang deadcode warning When !CONFIG_VM_BUG_ON, there is warning of clang-analyzer-deadcode.DeadStores: Value stored to 'mt' during its initialization is never read. Link: https://lkml.kernel.org/r/20221021101555.7992-2-quic_aiquny@quicinc.com Signed-off-by: Maria Yu Cc: David Hildenbrand Cc: Doug Berger Cc: Mike Kravetz Cc: Zi Yan Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- mm/page_isolation.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/page_isolation.c b/mm/page_isolation.c index 04141a9bea704..47fbc1696466f 100644 --- a/mm/page_isolation.c +++ b/mm/page_isolation.c @@ -330,7 +330,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags, zone->zone_start_pfn); if (skip_isolation) { - int mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock)); + int mt __maybe_unused = get_pageblock_migratetype(pfn_to_page(isolate_pageblock)); VM_BUG_ON(!is_migrate_isolate(mt)); } else { -- GitLab From 8ebe0a5eaaeb099de03d09ad20f54ed962e2261e Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Fri, 21 Oct 2022 19:28:05 -0400 Subject: [PATCH 0693/1927] mm,madvise,hugetlb: fix unexpected data loss with MADV_DONTNEED on hugetlbfs A common use case for hugetlbfs is for the application to create memory pools backed by huge pages, which then get handed over to some malloc library (eg. jemalloc) for further management. That malloc library may be doing MADV_DONTNEED calls on memory that is no longer needed, expecting those calls to happen on PAGE_SIZE boundaries. However, currently the MADV_DONTNEED code rounds up any such requests to HPAGE_PMD_SIZE boundaries. This leads to undesired outcomes when jemalloc expects a 4kB MADV_DONTNEED, but 2MB of memory get zeroed out, instead. Use of pre-built shared libraries means that user code does not always know the page size of every memory arena in use. Avoid unexpected data loss with MADV_DONTNEED by rounding up only to PAGE_SIZE (in do_madvise), and rounding down to huge page granularity. That way programs will only get as much memory zeroed out as they requested. Link: https://lkml.kernel.org/r/20221021192805.366ad573@imladris.surriel.com Fixes: 90e7e7f5ef3f ("mm: enable MADV_DONTNEED for hugetlb mappings") Signed-off-by: Rik van Riel Reviewed-by: Mike Kravetz Cc: David Hildenbrand Cc: Signed-off-by: Andrew Morton --- mm/madvise.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mm/madvise.c b/mm/madvise.c index 2baa93ca23109..c7105ec6d08c0 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -813,7 +813,14 @@ static bool madvise_dontneed_free_valid_vma(struct vm_area_struct *vma, if (start & ~huge_page_mask(hstate_vma(vma))) return false; - *end = ALIGN(*end, huge_page_size(hstate_vma(vma))); + /* + * Madvise callers expect the length to be rounded up to PAGE_SIZE + * boundaries, and may be unaware that this VMA uses huge pages. + * Avoid unexpected data loss by rounding down the number of + * huge pages freed. + */ + *end = ALIGN_DOWN(*end, huge_page_size(hstate_vma(vma))); + return true; } @@ -828,6 +835,9 @@ static long madvise_dontneed_free(struct vm_area_struct *vma, if (!madvise_dontneed_free_valid_vma(vma, start, &end, behavior)) return -EINVAL; + if (start == end) + return 0; + if (!userfaultfd_remove(vma, start, end)) { *prev = NULL; /* mmap_lock has been dropped, prev is stale */ -- GitLab From 5aae9265ee1a30cf716d6caf6b29fe99b9d55130 Mon Sep 17 00:00:00 2001 From: Hugh Dickins Date: Sat, 22 Oct 2022 00:51:06 -0700 Subject: [PATCH 0694/1927] mm: prep_compound_tail() clear page->private Although page allocation always clears page->private in the first page or head page of an allocation, it has never made a point of clearing page->private in the tails (though 0 is often what is already there). But now commit 71e2d666ef85 ("mm/huge_memory: do not clobber swp_entry_t during THP split") issues a warning when page_tail->private is found to be non-0 (unless it's swapcache). Change that warning to dump page_tail (which also dumps head), instead of just the head: so far we have seen dead000000000122, dead000000000003, dead000000000001 or 0000000000000002 in the raw output for tail private. We could just delete the warning, but today's consensus appears to want page->private to be 0, unless there's a good reason for it to be set: so now clear it in prep_compound_tail() (more general than just for THP; but not for high order allocation, which makes no pass down the tails). Link: https://lkml.kernel.org/r/1c4233bb-4e4d-5969-fbd4-96604268a285@google.com Fixes: 71e2d666ef85 ("mm/huge_memory: do not clobber swp_entry_t during THP split") Signed-off-by: Hugh Dickins Acked-by: Mel Gorman Cc: Matthew Wilcox (Oracle) Cc: Signed-off-by: Andrew Morton --- mm/huge_memory.c | 2 +- mm/page_alloc.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 03fc7e5edf075..561a42567477d 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2462,7 +2462,7 @@ static void __split_huge_page_tail(struct page *head, int tail, * Fix up and warn once if private is unexpectedly set. */ if (!folio_test_swapcache(page_folio(head))) { - VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, head); + VM_WARN_ON_ONCE_PAGE(page_tail->private != 0, page_tail); page_tail->private = 0; } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index b5a6c815ae284..218b28ee49ed3 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -807,6 +807,7 @@ static void prep_compound_tail(struct page *head, int tail_idx) p->mapping = TAIL_MAPPING; set_compound_head(p, head); + set_page_private(p, 0); } void prep_compound_page(struct page *page, unsigned int order) -- GitLab From 67eae54bc227b30dedcce9db68b063ba1adb7838 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Mon, 24 Oct 2022 15:33:35 -0400 Subject: [PATCH 0695/1927] mm/uffd: fix vma check on userfault for wp We used to have a report that pte-marker code can be reached even when uffd-wp is not compiled in for file memories, here: https://lore.kernel.org/all/YzeR+R6b4bwBlBHh@x1n/T/#u I just got time to revisit this and found that the root cause is we simply messed up with the vma check, so that for !PTE_MARKER_UFFD_WP system, we will allow UFFDIO_REGISTER of MINOR & WP upon shmem as the check was wrong: if (vm_flags & VM_UFFD_MINOR) return is_vm_hugetlb_page(vma) || vma_is_shmem(vma); Where we'll allow anything to pass on shmem as long as minor mode is requested. Axel did it right when introducing minor mode but I messed it up in b1f9e876862d when moving code around. Fix it. Link: https://lkml.kernel.org/r/20221024193336.1233616-1-peterx@redhat.com Link: https://lkml.kernel.org/r/20221024193336.1233616-2-peterx@redhat.com Fixes: b1f9e876862d ("mm/uffd: enable write protection for shmem & hugetlbfs") Signed-off-by: Peter Xu Cc: Axel Rasmussen Cc: Andrea Arcangeli Cc: Nadav Amit Cc: Signed-off-by: Andrew Morton --- include/linux/userfaultfd_k.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index f07e6998bb68e..9df0b9a762cc9 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -146,9 +146,9 @@ static inline bool userfaultfd_armed(struct vm_area_struct *vma) static inline bool vma_can_userfault(struct vm_area_struct *vma, unsigned long vm_flags) { - if (vm_flags & VM_UFFD_MINOR) - return is_vm_hugetlb_page(vma) || vma_is_shmem(vma); - + if ((vm_flags & VM_UFFD_MINOR) && + (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) + return false; #ifndef CONFIG_PTE_MARKER_UFFD_WP /* * If user requested uffd-wp but not enabled pte markers for -- GitLab From 03e5f82ea632af329e32ec03d952b2d99497eeaa Mon Sep 17 00:00:00 2001 From: Baolin Wang Date: Mon, 24 Oct 2022 16:34:21 +0800 Subject: [PATCH 0696/1927] mm: migrate: fix return value if all subpages of THPs are migrated successfully During THP migration, if THPs are not migrated but they are split and all subpages are migrated successfully, migrate_pages() will still return the number of THP pages that were not migrated. This will confuse the callers of migrate_pages(). For example, the longterm pinning will failed though all pages are migrated successfully. Thus we should return 0 to indicate that all pages are migrated in this case Link: https://lkml.kernel.org/r/de386aa864be9158d2f3b344091419ea7c38b2f7.1666599848.git.baolin.wang@linux.alibaba.com Fixes: b5bade978e9b ("mm: migrate: fix the return value of migrate_pages()") Signed-off-by: Baolin Wang Reviewed-by: Alistair Popple Reviewed-by: Yang Shi Cc: David Hildenbrand Cc: "Huang, Ying" Cc: Zi Yan Cc: Signed-off-by: Andrew Morton --- mm/migrate.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/migrate.c b/mm/migrate.c index 1379e1912772e..dff333593a8ae 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -1582,6 +1582,13 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, */ list_splice(&ret_pages, from); + /* + * Return 0 in case all subpages of fail-to-migrate THPs are + * migrated successfully. + */ + if (list_empty(from)) + rc = 0; + count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); count_vm_events(PGMIGRATE_FAIL, nr_failed_pages); count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded); -- GitLab From f59a3ee6912997fc56ecee78613fef53aae668d9 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 24 Oct 2022 23:21:40 +0200 Subject: [PATCH 0697/1927] mm: kmsan: export kmsan_copy_page_meta() Certain modules call copy_user_highpage(), which calls kmsan_copy_page_meta() under KMSAN, so we need to export the latter. Link: https://lkml.kernel.org/r/20221024212144.2852069-1-glider@google.com Link: https://github.com/google/kmsan/issues/89 Fixes: b073d7f8aee4 ("mm: kmsan: maintain KMSAN metadata for page operations") Signed-off-by: Alexander Potapenko Signed-off-by: Andrew Morton --- mm/kmsan/shadow.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/kmsan/shadow.c b/mm/kmsan/shadow.c index 21e3e196ec3cf..a787c04e9583c 100644 --- a/mm/kmsan/shadow.c +++ b/mm/kmsan/shadow.c @@ -167,6 +167,7 @@ void kmsan_copy_page_meta(struct page *dst, struct page *src) __memcpy(origin_ptr_for(dst), origin_ptr_for(src), PAGE_SIZE); kmsan_leave_runtime(); } +EXPORT_SYMBOL(kmsan_copy_page_meta); void kmsan_alloc_page(struct page *page, unsigned int order, gfp_t flags) { -- GitLab From 42855f588e187a6f22978e54422adbc010ac7630 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 24 Oct 2022 23:21:41 +0200 Subject: [PATCH 0698/1927] x86/purgatory: disable KMSAN instrumentation The stand-alone purgatory.ro does not contain the KMSAN runtime, therefore it can't be built with KMSAN compiler instrumentation. Link: https://lkml.kernel.org/r/20221024212144.2852069-2-glider@google.com Link: https://github.com/google/kmsan/issues/89 Signed-off-by: Alexander Potapenko Cc: Andrew Morton Cc: Thomas Gleixner Cc: Ingo Molnar Signed-off-by: Andrew Morton --- arch/x86/purgatory/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile index 58a200dc762d6..17f09dc263811 100644 --- a/arch/x86/purgatory/Makefile +++ b/arch/x86/purgatory/Makefile @@ -26,6 +26,7 @@ GCOV_PROFILE := n KASAN_SANITIZE := n UBSAN_SANITIZE := n KCSAN_SANITIZE := n +KMSAN_SANITIZE := n KCOV_INSTRUMENT := n # These are adjustments to the compiler flags used for objects that -- GitLab From 921757bc9b611efc483a548b86769934384e9c79 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 24 Oct 2022 23:21:42 +0200 Subject: [PATCH 0699/1927] Kconfig.debug: disable CONFIG_FRAME_WARN for KMSAN by default KMSAN adds a lot of instrumentation to the code, which results in increased stack usage (up to 2048 bytes and more in some cases). It's hard to predict how big the stack frames can be, so we disable the warnings for KMSAN instead. Link: https://lkml.kernel.org/r/20221024212144.2852069-3-glider@google.com Link: https://github.com/google/kmsan/issues/89 Signed-off-by: Alexander Potapenko Cc: Kees Cook Cc: Masahiro Yamada Cc: Nick Desaulniers Signed-off-by: Andrew Morton --- lib/Kconfig.debug | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 3fc7abffc7aa2..29280072dc0e4 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -400,8 +400,9 @@ config FRAME_WARN default 1536 if (!64BIT && XTENSA) default 1024 if !64BIT default 2048 if 64BIT + default 0 if KMSAN help - Tell gcc to warn at build time for stack frames larger than this. + Tell the compiler to warn at build time for stack frames larger than this. Setting this too low will cause a lot of warnings. Setting it to 0 disables the warning. -- GitLab From 59c8a02e24894e75639bcecc3cb1e768a2792220 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 24 Oct 2022 23:21:43 +0200 Subject: [PATCH 0700/1927] x86: asm: make sure __put_user_size() evaluates pointer once User access macros must ensure their arguments are evaluated only once if they are used more than once in the macro body. Adding instrument_put_user() to __put_user_size() resulted in double evaluation of the `ptr` argument, which led to correctness issues when performing e.g. unsafe_put_user(..., p++, ...). To fix those issues, evaluate the `ptr` argument of __put_user_size() at the beginning of the macro. Link: https://lkml.kernel.org/r/20221024212144.2852069-4-glider@google.com Fixes: 888f84a6da4d ("x86: asm: instrument usercopy in get_user() and put_user()") Signed-off-by: Alexander Potapenko Reported-by: youling257 Cc: Borislav Petkov Cc: Ingo Molnar Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/include/asm/uaccess.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index 8bc614cfe21b9..1cc756eafa447 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -254,24 +254,25 @@ extern void __put_user_nocheck_8(void); #define __put_user_size(x, ptr, size, label) \ do { \ __typeof__(*(ptr)) __x = (x); /* eval x once */ \ - __chk_user_ptr(ptr); \ + __typeof__(ptr) __ptr = (ptr); /* eval ptr once */ \ + __chk_user_ptr(__ptr); \ switch (size) { \ case 1: \ - __put_user_goto(__x, ptr, "b", "iq", label); \ + __put_user_goto(__x, __ptr, "b", "iq", label); \ break; \ case 2: \ - __put_user_goto(__x, ptr, "w", "ir", label); \ + __put_user_goto(__x, __ptr, "w", "ir", label); \ break; \ case 4: \ - __put_user_goto(__x, ptr, "l", "ir", label); \ + __put_user_goto(__x, __ptr, "l", "ir", label); \ break; \ case 8: \ - __put_user_goto_u64(__x, ptr, label); \ + __put_user_goto_u64(__x, __ptr, label); \ break; \ default: \ __put_user_bad(); \ } \ - instrument_put_user(__x, ptr, size); \ + instrument_put_user(__x, __ptr, size); \ } while (0) #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT -- GitLab From 78a498c3a227f2ac773a8234b2ce092a4403f2c3 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 24 Oct 2022 23:21:44 +0200 Subject: [PATCH 0701/1927] x86: fortify: kmsan: fix KMSAN fortify builds Ensure that KMSAN builds replace memset/memcpy/memmove calls with the respective __msan_XXX functions, and that none of the macros are redefined twice. This should allow building kernel with both CONFIG_KMSAN and CONFIG_FORTIFY_SOURCE. Link: https://lkml.kernel.org/r/20221024212144.2852069-5-glider@google.com Link: https://github.com/google/kmsan/issues/89 Signed-off-by: Alexander Potapenko Reported-by: Tamas K Lengyel Cc: Nathan Chancellor Cc: Nick Desaulniers Cc: Kees Cook Signed-off-by: Andrew Morton --- arch/x86/include/asm/string_64.h | 11 +++++++---- include/linux/fortify-string.h | 17 +++++++++++++++-- include/linux/kmsan_string.h | 21 +++++++++++++++++++++ mm/kmsan/instrumentation.c | 1 + 4 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 include/linux/kmsan_string.h diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string_64.h index 3b87d889b6e16..888731ccf1f67 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -10,10 +10,13 @@ /* Even with __builtin_ the compiler may decide to use the out of line function. */ +#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) +#include +#endif + #define __HAVE_ARCH_MEMCPY 1 -#if defined(__SANITIZE_MEMORY__) +#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) #undef memcpy -void *__msan_memcpy(void *dst, const void *src, size_t size); #define memcpy __msan_memcpy #else extern void *memcpy(void *to, const void *from, size_t len); @@ -21,7 +24,7 @@ extern void *memcpy(void *to, const void *from, size_t len); extern void *__memcpy(void *to, const void *from, size_t len); #define __HAVE_ARCH_MEMSET -#if defined(__SANITIZE_MEMORY__) +#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) extern void *__msan_memset(void *s, int c, size_t n); #undef memset #define memset __msan_memset @@ -67,7 +70,7 @@ static inline void *memset64(uint64_t *s, uint64_t v, size_t n) } #define __HAVE_ARCH_MEMMOVE -#if defined(__SANITIZE_MEMORY__) +#if defined(__SANITIZE_MEMORY__) && defined(__NO_FORTIFY) #undef memmove void *__msan_memmove(void *dest, const void *src, size_t len); #define memmove __msan_memmove diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 4029fe368a4f6..18a31b125f9d6 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -43,11 +43,24 @@ extern __kernel_size_t __underlying_strlen(const char *p) __RENAME(strlen); extern char *__underlying_strncat(char *p, const char *q, __kernel_size_t count) __RENAME(strncat); extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) __RENAME(strncpy); #else -#define __underlying_memchr __builtin_memchr -#define __underlying_memcmp __builtin_memcmp + +#if defined(__SANITIZE_MEMORY__) +/* + * For KMSAN builds all memcpy/memset/memmove calls should be replaced by the + * corresponding __msan_XXX functions. + */ +#include +#define __underlying_memcpy __msan_memcpy +#define __underlying_memmove __msan_memmove +#define __underlying_memset __msan_memset +#else #define __underlying_memcpy __builtin_memcpy #define __underlying_memmove __builtin_memmove #define __underlying_memset __builtin_memset +#endif + +#define __underlying_memchr __builtin_memchr +#define __underlying_memcmp __builtin_memcmp #define __underlying_strcat __builtin_strcat #define __underlying_strcpy __builtin_strcpy #define __underlying_strlen __builtin_strlen diff --git a/include/linux/kmsan_string.h b/include/linux/kmsan_string.h new file mode 100644 index 0000000000000..7287da6f52eff --- /dev/null +++ b/include/linux/kmsan_string.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * KMSAN string functions API used in other headers. + * + * Copyright (C) 2022 Google LLC + * Author: Alexander Potapenko + * + */ +#ifndef _LINUX_KMSAN_STRING_H +#define _LINUX_KMSAN_STRING_H + +/* + * KMSAN overrides the default memcpy/memset/memmove implementations in the + * kernel, which requires having __msan_XXX function prototypes in several other + * headers. Keep them in one place instead of open-coding. + */ +void *__msan_memcpy(void *dst, const void *src, size_t size); +void *__msan_memset(void *s, int c, size_t n); +void *__msan_memmove(void *dest, const void *src, size_t len); + +#endif /* _LINUX_KMSAN_STRING_H */ diff --git a/mm/kmsan/instrumentation.c b/mm/kmsan/instrumentation.c index 280d154132684..271f135f97a16 100644 --- a/mm/kmsan/instrumentation.c +++ b/mm/kmsan/instrumentation.c @@ -14,6 +14,7 @@ #include "kmsan.h" #include +#include #include #include -- GitLab From 5521de7dddd211e3a9403d7bde0b614fd0936ac6 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Sun, 23 Oct 2022 21:34:52 -0700 Subject: [PATCH 0702/1927] mm/userfaultfd: replace kmap/kmap_atomic() with kmap_local_page() kmap() and kmap_atomic() are being deprecated in favor of kmap_local_page() which is appropriate for any thread local context.[1] A recent locking bug report with userfaultfd showed that the conversion of the kmap_atomic()'s in those code flows requires care with regard to the prevention of deadlock.[2] git archaeology implied that the recursion may not be an actual bug.[3] However, depending on the implementation of the mmap_lock and the condition of the call there may still be a deadlock.[4] So this is not purely a lockdep issue. Considering a single threaded call stack there are 3 options. 1) Different mm's are in play (no issue) 2) Readlock implementation is recursive and same mm is in play (no issue) 3) Readlock implementation is _not_ recursive (issue) The mmap_lock is recursive so with a single thread there is no issue. However, Matthew pointed out a deadlock scenario when you consider additional process' and threads thusly. "The readlock implementation is only recursive if nobody else has taken a write lock. If you have a multithreaded process, one of the other threads can call mmap() and that will prevent recursion (due to fairness). Even if it's a different process that you're trying to acquire the mmap read lock on, you can still get into a deadly embrace. eg: process A thread 1 takes read lock on own mmap_lock process A thread 2 calls mmap, blocks taking write lock process B thread 1 takes page fault, read lock on own mmap lock process B thread 2 calls mmap, blocks taking write lock process A thread 1 blocks taking read lock on process B process B thread 1 blocks taking read lock on process A Now all four threads are blocked waiting for each other." Regardless using pagefault_disable() ensures that no matter what locking implementation is used a deadlock will not occur. Complete kmap conversion in userfaultfd by replacing the kmap() and kmap_atomic() calls with kmap_local_page(). When replacing the kmap_atomic() call ensure page faults continue to be disabled to support the correct fall back behavior and add a comment to inform future souls of the requirement. [1] https://lore.kernel.org/all/20220813220034.806698-1-ira.weiny@intel.com/ [2] https://lore.kernel.org/all/Y1Mh2S7fUGQ%2FiKFR@iweiny-desk3/ [3] https://lore.kernel.org/all/Y1MymJ%2FINb45AdaY@iweiny-desk3/ [4] https://lore.kernel.org/lkml/Y1bXBtGTCym77%2FoD@casper.infradead.org/ [ira.weiny@intel.com: v2] Link: https://lkml.kernel.org/r/20221025220136.2366143-1-ira.weiny@intel.com Link: https://lkml.kernel.org/r/20221024043452.1491677-1-ira.weiny@intel.com Signed-off-by: Ira Weiny Cc: Matthew Wilcox Cc: Andrew Morton Cc: Andrea Arcangeli Cc: Peter Xu Cc: Axel Rasmussen Signed-off-by: Andrew Morton --- mm/userfaultfd.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index e24e8a47ce8a2..3d0fef3980b3f 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -157,11 +157,28 @@ static int mcopy_atomic_pte(struct mm_struct *dst_mm, if (!page) goto out; - page_kaddr = kmap_atomic(page); + page_kaddr = kmap_local_page(page); + /* + * The read mmap_lock is held here. Despite the + * mmap_lock being read recursive a deadlock is still + * possible if a writer has taken a lock. For example: + * + * process A thread 1 takes read lock on own mmap_lock + * process A thread 2 calls mmap, blocks taking write lock + * process B thread 1 takes page fault, read lock on own mmap lock + * process B thread 2 calls mmap, blocks taking write lock + * process A thread 1 blocks taking read lock on process B + * process B thread 1 blocks taking read lock on process A + * + * Disable page faults to prevent potential deadlock + * and retry the copy outside the mmap_lock. + */ + pagefault_disable(); ret = copy_from_user(page_kaddr, (const void __user *) src_addr, PAGE_SIZE); - kunmap_atomic(page_kaddr); + pagefault_enable(); + kunmap_local(page_kaddr); /* fallback to copy_from_user outside mmap_lock */ if (unlikely(ret)) { @@ -646,11 +663,11 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm, mmap_read_unlock(dst_mm); BUG_ON(!page); - page_kaddr = kmap(page); + page_kaddr = kmap_local_page(page); err = copy_from_user(page_kaddr, (const void __user *) src_addr, PAGE_SIZE); - kunmap(page); + kunmap_local(page_kaddr); if (unlikely(err)) { err = -EFAULT; goto out; -- GitLab From 5dc21f0c0b1c02ea2c9014cbe7cd3b28884ff306 Mon Sep 17 00:00:00 2001 From: Ira Weiny Date: Tue, 25 Oct 2022 15:01:08 -0700 Subject: [PATCH 0703/1927] mm/shmem: ensure proper fallback if page faults The kernel test robot flagged a recursive lock as a result of a conversion from kmap_atomic() to kmap_local_folio()[Link] The cause was due to the code depending on the kmap_atomic() side effect of disabling page faults. In that case the code expects the fault to fail and take the fallback case. git archaeology implied that the recursion may not be an actual bug.[1] However, depending on the implementation of the mmap_lock and the condition of the call there may still be a deadlock.[2] So this is not purely a lockdep issue. Considering a single threaded call stack there are 3 options. 1) Different mm's are in play (no issue) 2) Readlock implementation is recursive and same mm is in play (no issue) 3) Readlock implementation is _not_ recursive (issue) The mmap_lock is recursive so with a single thread there is no issue. However, Matthew pointed out a deadlock scenario when you consider additional process' and threads thusly. "The readlock implementation is only recursive if nobody else has taken a write lock. If you have a multithreaded process, one of the other threads can call mmap() and that will prevent recursion (due to fairness). Even if it's a different process that you're trying to acquire the mmap read lock on, you can still get into a deadly embrace. eg: process A thread 1 takes read lock on own mmap_lock process A thread 2 calls mmap, blocks taking write lock process B thread 1 takes page fault, read lock on own mmap lock process B thread 2 calls mmap, blocks taking write lock process A thread 1 blocks taking read lock on process B process B thread 1 blocks taking read lock on process A Now all four threads are blocked waiting for each other." Regardless using pagefault_disable() ensures that no matter what locking implementation is used a deadlock will not occur. Add an explicit pagefault_disable() and a big comment to explain this for future souls looking at this code. [1] https://lore.kernel.org/all/Y1MymJ%2FINb45AdaY@iweiny-desk3/ [2] https://lore.kernel.org/lkml/Y1bXBtGTCym77%2FoD@casper.infradead.org/ Link: https://lkml.kernel.org/r/20221025220108.2366043-1-ira.weiny@intel.com Link: https://lore.kernel.org/r/202210211215.9dc6efb5-yujie.liu@intel.com Fixes: 7a7256d5f512 ("shmem: convert shmem_mfill_atomic_pte() to use a folio") Signed-off-by: Ira Weiny Reported-by: Matthew Wilcox (Oracle) Reported-by: kernel test robot Cc: Randy Dunlap Cc: Peter Xu Cc: Andrea Arcangeli Signed-off-by: Andrew Morton --- mm/shmem.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mm/shmem.c b/mm/shmem.c index 8280a5cb48dfc..c1d8b8a1aa3b4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -2424,9 +2424,26 @@ int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, if (!zeropage) { /* COPY */ page_kaddr = kmap_local_folio(folio, 0); + /* + * The read mmap_lock is held here. Despite the + * mmap_lock being read recursive a deadlock is still + * possible if a writer has taken a lock. For example: + * + * process A thread 1 takes read lock on own mmap_lock + * process A thread 2 calls mmap, blocks taking write lock + * process B thread 1 takes page fault, read lock on own mmap lock + * process B thread 2 calls mmap, blocks taking write lock + * process A thread 1 blocks taking read lock on process B + * process B thread 1 blocks taking read lock on process A + * + * Disable page faults to prevent potential deadlock + * and retry the copy outside the mmap_lock. + */ + pagefault_disable(); ret = copy_from_user(page_kaddr, (const void __user *)src_addr, PAGE_SIZE); + pagefault_enable(); kunmap_local(page_kaddr); /* fallback to copy_from_user outside mmap_lock */ -- GitLab From 1db43d3f3733351849ddca4b573c037c7821bfd8 Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Tue, 25 Oct 2022 16:12:49 +0000 Subject: [PATCH 0704/1927] mmap: fix remap_file_pages() regression When using the VMA iterator, the final execution will set the variable 'next' to NULL which causes the function to fail out. Restore the break in the loop to exit the VMA iterator early without clearing NULL fixes the issue. Link: https://lore.kernel.org/lkml/29344.1666681759@jrobl/ Link: https://lkml.kernel.org/r/20221025161222.2634030-1-Liam.Howlett@oracle.com Fixes: 763ecb035029 (mm: remove the vma linked list) Signed-off-by: Liam R. Howlett Reported-by: "J. R. Okajima" Tested-by: "J. R. Okajima" Signed-off-by: Andrew Morton --- mm/mmap.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm/mmap.c b/mm/mmap.c index e270057ed04eb..2def55555e05f 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2852,6 +2852,9 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, if (next->vm_flags != vma->vm_flags) goto out; + if (start + size <= next->vm_end) + break; + prev = next; } -- GitLab From 1b9c918318476b4441ddd754ee6699b5367bb5ee Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 26 Oct 2022 14:00:29 +0200 Subject: [PATCH 0705/1927] lib: maple_tree: remove unneeded initialization in mtree_range_walk() Before the do-while loop in mtree_range_walk(), the variables next, min, max need to be initialized. The variables last, prev_min and prev_max are set within the loop body before they are eventually used after exiting the loop body. As it is a do-while loop, the loop body is executed at least once, so the variables last, prev_min and prev_max do not need to be initialized before the loop body. Remove unneeded initialization of last and prev_min. The needless initialization was reported by clang-analyzer as Dead Stores. As the compiler already identifies these assignments as unneeded, it optimizes the assignments away. Hence: No functional change. No change in object code. Link: https://lkml.kernel.org/r/20221026120029.12555-2-lukas.bulwahn@gmail.com Signed-off-by: Lukas Bulwahn Reviewed-by: Liam R. Howlett Cc: Matthew Wilcox Signed-off-by: Andrew Morton --- lib/maple_tree.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index e1743803c8512..fbde494444b89 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -2903,8 +2903,8 @@ static inline void *mtree_range_walk(struct ma_state *mas) unsigned long max, min; unsigned long prev_max, prev_min; - last = next = mas->node; - prev_min = min = mas->min; + next = mas->node; + min = mas->min; max = mas->max; do { offset = 0; -- GitLab From dda1c41a07b4a4c3f99b5b28c1e8c485205fe860 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 26 Oct 2022 15:48:30 +0200 Subject: [PATCH 0706/1927] mm: multi-gen LRU: move lru_gen_add_mm() out of IRQ-off region lru_gen_add_mm() has been added within an IRQ-off region in the commit mentioned below. The other invocations of lru_gen_add_mm() are not within an IRQ-off region. The invocation within IRQ-off region is problematic on PREEMPT_RT because the function is using a spin_lock_t which must not be used within IRQ-disabled regions. The other invocations of lru_gen_add_mm() occur while task_struct::alloc_lock is acquired. Move lru_gen_add_mm() after interrupts are enabled and before task_unlock(). Link: https://lkml.kernel.org/r/20221026134830.711887-1-bigeasy@linutronix.de Fixes: bd74fdaea1460 ("mm: multi-gen LRU: support page table walks") Signed-off-by: Sebastian Andrzej Siewior Acked-by: Yu Zhao Cc: Al Viro Cc: "Eric W . Biederman" Cc: Kees Cook Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- fs/exec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/exec.c b/fs/exec.c index 349a5da91efe8..7ab1f27b805dc 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -1012,7 +1012,6 @@ static int exec_mmap(struct mm_struct *mm) active_mm = tsk->active_mm; tsk->active_mm = mm; tsk->mm = mm; - lru_gen_add_mm(mm); /* * This prevents preemption while active_mm is being loaded and * it and mm are being updated, which could cause problems for @@ -1025,6 +1024,7 @@ static int exec_mmap(struct mm_struct *mm) activate_mm(active_mm, mm); if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) local_irq_enable(); + lru_gen_add_mm(mm); task_unlock(tsk); lru_gen_use_mm(mm); if (old_mm) { -- GitLab From cadaa773bcf161184fa428180516bae33a7bc667 Mon Sep 17 00:00:00 2001 From: Ulf Hansson Date: Thu, 27 Oct 2022 13:57:45 +0200 Subject: [PATCH 0707/1927] arm64: dts: qcom: sm8250: Disable the not yet supported cluster idle state To support the deeper cluster idle state for sm8250 platforms, some additional synchronization is needed between the rpmh-rsc device and the CPU cluster PM domain. Until that is supported, let's disable the cluster idle state. This fixes a problem that has been reported for the Qcom RB5 platform (see below), but most likely other sm8250 platforms suffers from similar issues, so let's make the fix generic for sm8250. vreg_l11c_3p3: failed to enable: -ETIMEDOUT qcom-rpmh-regulator 18200000.rsc:pm8150l-rpmh-regulators: ldo11: devm_regulator_register() failed, ret=-110 qcom-rpmh-regulator: probe of 18200000.rsc:pm8150l-rpmh-regulators failed with error -110 Reported-by: Amit Pundir Fixes: 32bc936d7321 ("arm64: dts: qcom: sm8250: Add cpuidle states") Signed-off-by: Ulf Hansson Tested-by: Amit Pundir Reviewed-by: Sudeep Holla Signed-off-by: Bjorn Andersson Link: https://lore.kernel.org/r/20221027115745.240516-1-ulf.hansson@linaro.org --- arch/arm64/boot/dts/qcom/sm8250.dtsi | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index a5b62cadb1298..e276eed1f8e2c 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -334,6 +334,7 @@ CLUSTER_SLEEP_0: cluster-sleep-0 { exit-latency-us = <6562>; min-residency-us = <9987>; local-timer-stop; + status = "disabled"; }; }; }; -- GitLab From bfc3b0f05653a28c8d41067a2aa3875d1f982e3e Mon Sep 17 00:00:00 2001 From: Willy Tarreau Date: Sun, 9 Oct 2022 20:29:36 +0200 Subject: [PATCH 0708/1927] tools/nolibc: Fix missing strlen() definition and infinite loop with gcc-12 When built at -Os, gcc-12 recognizes an strlen() pattern in nolibc_strlen() and replaces it with a jump to strlen(), which is not defined as a symbol and breaks compilation. Worse, when the function is called strlen(), the function is simply replaced with a jump to itself, hence becomes an infinite loop. One way to avoid this is to always set -ffreestanding, but the calling code doesn't know this and there's no way (either via attributes or pragmas) to globally enable it from include files, effectively leaving a painful situation for the caller. Alexey suggested to place an empty asm() statement inside the loop to stop gcc from recognizing a well-known pattern, which happens to work pretty fine. At least it allows us to make sure our local definition is not replaced with a self jump. The function only needs to be renamed back to strlen() so that the symbol exists, which implies that nolibc_strlen() which is used on variable strings has to be declared as a macro that points back to it before the strlen() macro is redifined. It was verified to produce valid code with gcc 3.4 to 12.1 at different optimization levels, and both with constant and variable strings. In case this problem surfaces again in the future, an alternate approach consisting in adding an optimize("no-tree-loop-distribute-patterns") function attribute for gcc>=12 worked as well but is less pretty. Reported-by: kernel test robot Link: https://lore.kernel.org/r/202210081618.754a77db-yujie.liu@intel.com Fixes: 66b6f755ad45 ("rcutorture: Import a copy of nolibc") Fixes: 96980b833a21 ("tools/nolibc/string: do not use __builtin_strlen() at -O0") Cc: "Paul E. McKenney" Cc: Alexey Dobriyan Signed-off-by: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/include/nolibc/string.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/include/nolibc/string.h b/tools/include/nolibc/string.h index bef35bee9c44c..718a405ffbc38 100644 --- a/tools/include/nolibc/string.h +++ b/tools/include/nolibc/string.h @@ -125,14 +125,18 @@ char *strcpy(char *dst, const char *src) } /* this function is only used with arguments that are not constants or when - * it's not known because optimizations are disabled. + * it's not known because optimizations are disabled. Note that gcc 12 + * recognizes an strlen() pattern and replaces it with a jump to strlen(), + * thus itself, hence the asm() statement below that's meant to disable this + * confusing practice. */ static __attribute__((unused)) -size_t nolibc_strlen(const char *str) +size_t strlen(const char *str) { size_t len; - for (len = 0; str[len]; len++); + for (len = 0; str[len]; len++) + asm(""); return len; } @@ -140,13 +144,12 @@ size_t nolibc_strlen(const char *str) * the two branches, then will rely on an external definition of strlen(). */ #if defined(__OPTIMIZE__) +#define nolibc_strlen(x) strlen(x) #define strlen(str) ({ \ __builtin_constant_p((str)) ? \ __builtin_strlen((str)) : \ nolibc_strlen((str)); \ }) -#else -#define strlen(str) nolibc_strlen((str)) #endif static __attribute__((unused)) -- GitLab From b3f4f51ea68a495f8a5956064c33dce711a2df91 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 21 Oct 2022 08:01:53 +0200 Subject: [PATCH 0709/1927] tools/nolibc/string: Fix memcmp() implementation The C standard says that memcmp() must treat the buffers as consisting of "unsigned chars". If char happens to be unsigned, the casts are ok, but then obviously the c1 variable can never contain a negative value. And when char is signed, the casts are wrong, and there's still a problem with using an 8-bit quantity to hold the difference, because that can range from -255 to +255. For example, assuming char is signed, comparing two 1-byte buffers, one containing 0x00 and another 0x80, the current implementation would return -128 for both memcmp(a, b, 1) and memcmp(b, a, 1), whereas one of those should of course return something positive. Signed-off-by: Rasmus Villemoes Fixes: 66b6f755ad45 ("rcutorture: Import a copy of nolibc") Cc: stable@vger.kernel.org # v5.0+ Signed-off-by: Willy Tarreau Signed-off-by: Paul E. McKenney --- tools/include/nolibc/string.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/include/nolibc/string.h b/tools/include/nolibc/string.h index 718a405ffbc38..ad97c0d522b8e 100644 --- a/tools/include/nolibc/string.h +++ b/tools/include/nolibc/string.h @@ -19,9 +19,9 @@ static __attribute__((unused)) int memcmp(const void *s1, const void *s2, size_t n) { size_t ofs = 0; - char c1 = 0; + int c1 = 0; - while (ofs < n && !(c1 = ((char *)s1)[ofs] - ((char *)s2)[ofs])) { + while (ofs < n && !(c1 = ((unsigned char *)s1)[ofs] - ((unsigned char *)s2)[ofs])) { ofs++; } return c1; -- GitLab From f5e4ec155d145002fd9840868453d785fab86d42 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Brucker Date: Fri, 28 Oct 2022 17:00:42 +0100 Subject: [PATCH 0710/1927] random: use arch_get_random*_early() in random_init() While reworking the archrandom handling, commit d349ab99eec7 ("random: handle archrandom with multiple longs") switched to the non-early archrandom helpers in random_init(), which broke initialization of the entropy pool from the arm64 random generator. Indeed at that point the arm64 CPU features, which verify that all CPUs have compatible capabilities, are not finalized so arch_get_random_seed_longs() is unsuccessful. Instead random_init() should use the _early functions, which check only the boot CPU on arm64. On other architectures the _early functions directly call the normal ones. Fixes: d349ab99eec7 ("random: handle archrandom with multiple longs") Cc: stable@vger.kernel.org Signed-off-by: Jean-Philippe Brucker Signed-off-by: Jason A. Donenfeld --- drivers/char/random.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 2fe28eeb2f387..69754155300ea 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -791,13 +791,13 @@ void __init random_init_early(const char *command_line) #endif for (i = 0, arch_bits = sizeof(entropy) * 8; i < ARRAY_SIZE(entropy);) { - longs = arch_get_random_seed_longs(entropy, ARRAY_SIZE(entropy) - i); + longs = arch_get_random_seed_longs_early(entropy, ARRAY_SIZE(entropy) - i); if (longs) { _mix_pool_bytes(entropy, sizeof(*entropy) * longs); i += longs; continue; } - longs = arch_get_random_longs(entropy, ARRAY_SIZE(entropy) - i); + longs = arch_get_random_longs_early(entropy, ARRAY_SIZE(entropy) - i); if (longs) { _mix_pool_bytes(entropy, sizeof(*entropy) * longs); i += longs; -- GitLab From 8bc8824d30193eb7755043d5bb65fa7f0d11a595 Mon Sep 17 00:00:00 2001 From: Emil Renner Berthing Date: Wed, 12 Oct 2022 13:09:28 +0200 Subject: [PATCH 0711/1927] riscv: dts: sifive unleashed: Add PWM controlled LEDs This adds the 4 PWM controlled green LEDs to the HiFive Unleashed device tree. The schematic doesn't specify any special function for the LEDs, so they're added here without any default triggers and named d1, d2, d3 and d4 just like in the schematic. Signed-off-by: Emil Renner Berthing Reviewed-by: Conor Dooley Tested-by: Conor Dooley Link: https://lore.kernel.org/r/20221012110928.352910-1-emil.renner.berthing@canonical.com Signed-off-by: Palmer Dabbelt --- .../boot/dts/sifive/hifive-unleashed-a00.dts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts index ced0d4e479385..900a50526d771 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unleashed-a00.dts @@ -3,6 +3,8 @@ #include "fu540-c000.dtsi" #include +#include +#include /* Clock frequency (in Hz) of the PCB crystal for rtcclk */ #define RTCCLK_FREQ 1000000 @@ -42,6 +44,42 @@ gpio-restart { compatible = "gpio-restart"; gpios = <&gpio 10 GPIO_ACTIVE_LOW>; }; + + led-controller { + compatible = "pwm-leds"; + + led-d1 { + pwms = <&pwm0 0 7812500 PWM_POLARITY_INVERTED>; + active-low; + color = ; + max-brightness = <255>; + label = "d1"; + }; + + led-d2 { + pwms = <&pwm0 1 7812500 PWM_POLARITY_INVERTED>; + active-low; + color = ; + max-brightness = <255>; + label = "d2"; + }; + + led-d3 { + pwms = <&pwm0 2 7812500 PWM_POLARITY_INVERTED>; + active-low; + color = ; + max-brightness = <255>; + label = "d3"; + }; + + led-d4 { + pwms = <&pwm0 3 7812500 PWM_POLARITY_INVERTED>; + active-low; + color = ; + max-brightness = <255>; + label = "d4"; + }; + }; }; &uart0 { -- GitLab From 6f7630b1b5bc672b54c1285ee6aba752b446672c Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 28 Oct 2022 15:32:07 -0700 Subject: [PATCH 0712/1927] fortify: Capture __bos() results in const temp vars In two recent run-time memcpy() bound checking bug reports (NFS[1] and JFS[2]), the _detection_ was working correctly (in the sense that the requested copy size was larger than the destination field size), but the _warning text_ was showing the destination field size as SIZE_MAX ("unknown size"). This should be impossible, since the detection function will explicitly give up if the destination field size is unknown. For example, the JFS warning was: memcpy: detected field-spanning write (size 132) of single field "ip->i_link" at fs/jfs/namei.c:950 (size 18446744073709551615) Other cases of this warning (e.g.[3]) have reported correctly, and the reproducer only happens under GCC (at least 10.2 and 12.1), so this currently appears to be a GCC bug. Explicitly capturing the __builtin_object_size() results in const temporary variables fixes the report. For example, the JFS reproducer now correctly reports the field size (128): memcpy: detected field-spanning write (size 132) of single field "ip->i_link" at fs/jfs/namei.c:950 (size 128) Examination of the .text delta (which is otherwise identical), shows the literal value used in the report changing: - mov $0xffffffffffffffff,%rcx + mov $0x80,%ecx [1] https://lore.kernel.org/lkml/Y0zEzZwhOxTDcBTB@codemonkey.org.uk/ [2] https://syzkaller.appspot.com/bug?id=23d613df5259b977dac1696bec77f61a85890e3d [3] https://lore.kernel.org/all/202210110948.26b43120-yujie.liu@intel.com/ Cc: "Dr. David Alan Gilbert" Cc: llvm@lists.linux.dev Cc: linux-hardening@vger.kernel.org Signed-off-by: Kees Cook --- include/linux/fortify-string.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 4029fe368a4f6..0f00a551939a4 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -441,13 +441,18 @@ __FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size, #define __fortify_memcpy_chk(p, q, size, p_size, q_size, \ p_size_field, q_size_field, op) ({ \ - size_t __fortify_size = (size_t)(size); \ - WARN_ONCE(fortify_memcpy_chk(__fortify_size, p_size, q_size, \ - p_size_field, q_size_field, #op), \ + const size_t __fortify_size = (size_t)(size); \ + const size_t __p_size = (p_size); \ + const size_t __q_size = (q_size); \ + const size_t __p_size_field = (p_size_field); \ + const size_t __q_size_field = (q_size_field); \ + WARN_ONCE(fortify_memcpy_chk(__fortify_size, __p_size, \ + __q_size, __p_size_field, \ + __q_size_field, #op), \ #op ": detected field-spanning write (size %zu) of single %s (size %zu)\n", \ __fortify_size, \ "field \"" #p "\" at " __FILE__ ":" __stringify(__LINE__), \ - p_size_field); \ + __p_size_field); \ __underlying_##op(p, q, __fortify_size); \ }) -- GitLab From 7354c9024f2835f6122ed9612e21ab379df050f9 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 27 Oct 2022 14:21:07 -0700 Subject: [PATCH 0713/1927] netlink: hide validation union fields from kdoc Mark the validation fields as private, users shouldn't set them directly and they are too complicated to explain in a more succinct way (there's already a long explanation in the comment above). The strict_start_type field is set directly and has a dedicated comment so move that above the "private" section. Link: https://lore.kernel.org/r/20221027212107.2639255-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- include/net/netlink.h | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/include/net/netlink.h b/include/net/netlink.h index 4418b1981e318..7db13b3261fc2 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -317,19 +317,10 @@ struct nla_policy { u8 validation_type; u16 len; union { - const u32 bitfield32_valid; - const u32 mask; - const char *reject_message; - const struct nla_policy *nested_policy; - struct netlink_range_validation *range; - struct netlink_range_validation_signed *range_signed; - struct { - s16 min, max; - u8 network_byte_order:1; - }; - int (*validate)(const struct nlattr *attr, - struct netlink_ext_ack *extack); - /* This entry is special, and used for the attribute at index 0 + /** + * @strict_start_type: first attribute to validate strictly + * + * This entry is special, and used for the attribute at index 0 * only, and specifies special data about the policy, namely it * specifies the "boundary type" where strict length validation * starts for any attribute types >= this value, also, strict @@ -348,6 +339,20 @@ struct nla_policy { * was added to enforce strict validation from thereon. */ u16 strict_start_type; + + /* private: use NLA_POLICY_*() to set */ + const u32 bitfield32_valid; + const u32 mask; + const char *reject_message; + const struct nla_policy *nested_policy; + struct netlink_range_validation *range; + struct netlink_range_validation_signed *range_signed; + struct { + s16 min, max; + u8 network_byte_order:1; + }; + int (*validate)(const struct nlattr *attr, + struct netlink_ext_ack *extack); }; }; -- GitLab From e4ba4554209f626c52e2e57f26cba49a62663c8b Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Thu, 27 Oct 2022 20:25:01 -0700 Subject: [PATCH 0714/1927] net: openvswitch: add missing .resv_start_op I missed one of the families in OvS when annotating .resv_start_op. This triggers the warning added in commit ce48ebdd5651 ("genetlink: limit the use of validation workarounds to old ops"). Reported-by: syzbot+40eb8c0447c0e47a7e9b@syzkaller.appspotmail.com Fixes: 9c5d03d36251 ("genetlink: start to validate reserved header bytes") Link: https://lore.kernel.org/r/20221028032501.2724270-1-kuba@kernel.org Signed-off-by: Jakub Kicinski --- net/openvswitch/datapath.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 155263e735124..8b84869eb2ac7 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -2544,6 +2544,7 @@ struct genl_family dp_vport_genl_family __ro_after_init = { .parallel_ops = true, .small_ops = dp_vport_genl_ops, .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops), + .resv_start_op = OVS_VPORT_CMD_SET + 1, .mcgrps = &ovs_dp_vport_multicast_group, .n_mcgrps = 1, .module = THIS_MODULE, -- GitLab From 1208b93dd901bafe2526fa9db005bbc30e7ae83a Mon Sep 17 00:00:00 2001 From: Govindarajulu Varadarajan Date: Thu, 27 Oct 2022 21:21:59 -0700 Subject: [PATCH 0715/1927] enic: MAINTAINERS: Update enic maintainers Update enic maintainers. Signed-off-by: Govindarajulu Varadarajan Link: https://lore.kernel.org/r/20221028042159.735670-1-govind.varadar@gmail.com Signed-off-by: Jakub Kicinski --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 10c1344b44730..9e437612dd816 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5038,7 +5038,7 @@ F: drivers/scsi/snic/ CISCO VIC ETHERNET NIC DRIVER M: Christian Benvenuti -M: Govindarajulu Varadarajan <_govind@gmx.com> +M: Satish Kharat S: Supported F: drivers/net/ethernet/cisco/enic/ -- GitLab From 8f279fb00bb29def9ac79e28c5d6d8e07d21f3fb Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 27 Oct 2022 00:25:56 +0100 Subject: [PATCH 0716/1927] udp: advertise ipv6 udp support for msghdr::ubuf_info Mark udp ipv6 as supporting msghdr::ubuf_info. In the original commit SOCK_SUPPORT_ZC was supposed to be set by a udp_init_sock() call from udp6_init_sock(), but d38afeec26ed4 ("tcp/udp: Call inet6_destroy_sock() in IPv6 ...") removed it and so ipv6 udp misses the flag. Cc: # 6.0 Fixes: e993ffe3da4bc ("net: flag sockets supporting msghdr originated zerocopy") Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- net/ipv6/udp.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 129ec5a9b0eb7..bc65e5b7195b3 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -66,6 +66,7 @@ int udpv6_init_sock(struct sock *sk) { skb_queue_head_init(&udp_sk(sk)->reader_queue); sk->sk_destruct = udpv6_destruct_sock; + set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); return 0; } -- GitLab From fee9ac06647e59a69fb7aec58f25267c134264b4 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 27 Oct 2022 00:25:57 +0100 Subject: [PATCH 0717/1927] net: remove SOCK_SUPPORT_ZC from sockmap sockmap replaces ->sk_prot with its own callbacks, we should remove SOCK_SUPPORT_ZC as the new proto doesn't support msghdr::ubuf_info. Cc: # 6.0 Reported-by: Jakub Kicinski Fixes: e993ffe3da4bc ("net: flag sockets supporting msghdr originated zerocopy") Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- include/net/sock.h | 7 +++++++ net/ipv4/tcp_bpf.c | 4 ++-- net/ipv4/udp_bpf.c | 4 ++-- net/unix/unix_bpf.c | 8 ++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/include/net/sock.h b/include/net/sock.h index 22f8bab583ddd..5db02546941cf 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -1889,6 +1889,13 @@ void sock_kfree_s(struct sock *sk, void *mem, int size); void sock_kzfree_s(struct sock *sk, void *mem, int size); void sk_send_sigurg(struct sock *sk); +static inline void sock_replace_proto(struct sock *sk, struct proto *proto) +{ + if (sk->sk_socket) + clear_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); + WRITE_ONCE(sk->sk_prot, proto); +} + struct sockcm_cookie { u64 transmit_time; u32 mark; diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index a1626afe87a10..c501c329b1dbe 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -607,7 +607,7 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) } else { sk->sk_write_space = psock->saved_write_space; /* Pairs with lockless read in sk_clone_lock() */ - WRITE_ONCE(sk->sk_prot, psock->sk_proto); + sock_replace_proto(sk, psock->sk_proto); } return 0; } @@ -620,7 +620,7 @@ int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) } /* Pairs with lockless read in sk_clone_lock() */ - WRITE_ONCE(sk->sk_prot, &tcp_bpf_prots[family][config]); + sock_replace_proto(sk, &tcp_bpf_prots[family][config]); return 0; } EXPORT_SYMBOL_GPL(tcp_bpf_update_proto); diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c index ff15918b7bdc7..e5dc91d0e0793 100644 --- a/net/ipv4/udp_bpf.c +++ b/net/ipv4/udp_bpf.c @@ -141,14 +141,14 @@ int udp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) if (restore) { sk->sk_write_space = psock->saved_write_space; - WRITE_ONCE(sk->sk_prot, psock->sk_proto); + sock_replace_proto(sk, psock->sk_proto); return 0; } if (sk->sk_family == AF_INET6) udp_bpf_check_v6_needs_rebuild(psock->sk_proto); - WRITE_ONCE(sk->sk_prot, &udp_bpf_prots[family]); + sock_replace_proto(sk, &udp_bpf_prots[family]); return 0; } EXPORT_SYMBOL_GPL(udp_bpf_update_proto); diff --git a/net/unix/unix_bpf.c b/net/unix/unix_bpf.c index 7cf14c6b17254..e9bf155139612 100644 --- a/net/unix/unix_bpf.c +++ b/net/unix/unix_bpf.c @@ -145,12 +145,12 @@ int unix_dgram_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool re if (restore) { sk->sk_write_space = psock->saved_write_space; - WRITE_ONCE(sk->sk_prot, psock->sk_proto); + sock_replace_proto(sk, psock->sk_proto); return 0; } unix_dgram_bpf_check_needs_rebuild(psock->sk_proto); - WRITE_ONCE(sk->sk_prot, &unix_dgram_bpf_prot); + sock_replace_proto(sk, &unix_dgram_bpf_prot); return 0; } @@ -158,12 +158,12 @@ int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool r { if (restore) { sk->sk_write_space = psock->saved_write_space; - WRITE_ONCE(sk->sk_prot, psock->sk_proto); + sock_replace_proto(sk, psock->sk_proto); return 0; } unix_stream_bpf_check_needs_rebuild(psock->sk_proto); - WRITE_ONCE(sk->sk_prot, &unix_stream_bpf_prot); + sock_replace_proto(sk, &unix_stream_bpf_prot); return 0; } -- GitLab From e276d62dcfdee6582486e8b8344dd869518e14be Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 27 Oct 2022 00:25:58 +0100 Subject: [PATCH 0718/1927] net/ulp: remove SOCK_SUPPORT_ZC from tls sockets Remove SOCK_SUPPORT_ZC when we're setting ulp as it might not support msghdr::ubuf_info, e.g. like TLS replacing ->sk_prot with a new set of handlers. Cc: # 6.0 Reported-by: Jakub Kicinski Fixes: e993ffe3da4bc ("net: flag sockets supporting msghdr originated zerocopy") Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- net/ipv4/tcp_ulp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/ipv4/tcp_ulp.c b/net/ipv4/tcp_ulp.c index 7c27aa629af19..9ae50b1bd8444 100644 --- a/net/ipv4/tcp_ulp.c +++ b/net/ipv4/tcp_ulp.c @@ -136,6 +136,9 @@ static int __tcp_set_ulp(struct sock *sk, const struct tcp_ulp_ops *ulp_ops) if (icsk->icsk_ulp_ops) goto out_err; + if (sk->sk_socket) + clear_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags); + err = ulp_ops->init(sk); if (err) goto out_err; -- GitLab From 71b7786ea478f3c4611deff4d2b9676b0c17c56b Mon Sep 17 00:00:00 2001 From: Stefan Metzmacher Date: Thu, 27 Oct 2022 00:25:59 +0100 Subject: [PATCH 0719/1927] net: also flag accepted sockets supporting msghdr originated zerocopy Without this only the client initiated tcp sockets have SOCK_SUPPORT_ZC. The listening socket on the server also has it, but the accepted connections didn't, which meant IORING_OP_SEND[MSG]_ZC will always fails with -EOPNOTSUPP. Fixes: e993ffe3da4b ("net: flag sockets supporting msghdr originated zerocopy") Cc: # 6.0 CC: Jens Axboe Link: https://lore.kernel.org/io-uring/20221024141503.22b4e251@kernel.org/T/#m38aa19b0b825758fb97860a38ad13122051f9dda Signed-off-by: Stefan Metzmacher Signed-off-by: Pavel Begunkov Signed-off-by: Jakub Kicinski --- net/ipv4/af_inet.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 3dd02396517df..4728087c42a5c 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -754,6 +754,8 @@ int inet_accept(struct socket *sock, struct socket *newsock, int flags, (TCPF_ESTABLISHED | TCPF_SYN_RECV | TCPF_CLOSE_WAIT | TCPF_CLOSE))); + if (test_bit(SOCK_SUPPORT_ZC, &sock->flags)) + set_bit(SOCK_SUPPORT_ZC, &newsock->flags); sock_graft(sk2, newsock); newsock->state = SS_CONNECTED; -- GitLab From 21ce2c121fa07b00b0906bd781590ea362e82ea2 Mon Sep 17 00:00:00 2001 From: Alexandru Tachici Date: Thu, 27 Oct 2022 12:56:55 +0300 Subject: [PATCH 0720/1927] net: ethernet: adi: adin1110: Fix notifiers ADIN1110 was registering netdev_notifiers on each device probe. This leads to warnings/probe failures because of double registration of the same notifier when to adin1110/2111 devices are connected to the same system. Move the registration of netdev_notifiers in module init call, in this way multiple driver instances can use the same notifiers. Fixes: bc93e19d088b ("net: ethernet: adi: Add ADIN1110 support") Signed-off-by: Alexandru Tachici Link: https://lore.kernel.org/r/20221027095655.89890-2-alexandru.tachici@analog.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/adi/adin1110.c | 38 ++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/adi/adin1110.c b/drivers/net/ethernet/adi/adin1110.c index 1744d623999d0..606c976108085 100644 --- a/drivers/net/ethernet/adi/adin1110.c +++ b/drivers/net/ethernet/adi/adin1110.c @@ -1512,16 +1512,15 @@ static struct notifier_block adin1110_switchdev_notifier = { .notifier_call = adin1110_switchdev_event, }; -static void adin1110_unregister_notifiers(void *data) +static void adin1110_unregister_notifiers(void) { unregister_switchdev_blocking_notifier(&adin1110_switchdev_blocking_notifier); unregister_switchdev_notifier(&adin1110_switchdev_notifier); unregister_netdevice_notifier(&adin1110_netdevice_nb); } -static int adin1110_setup_notifiers(struct adin1110_priv *priv) +static int adin1110_setup_notifiers(void) { - struct device *dev = &priv->spidev->dev; int ret; ret = register_netdevice_notifier(&adin1110_netdevice_nb); @@ -1536,13 +1535,14 @@ static int adin1110_setup_notifiers(struct adin1110_priv *priv) if (ret < 0) goto err_sdev; - return devm_add_action_or_reset(dev, adin1110_unregister_notifiers, NULL); + return 0; err_sdev: unregister_switchdev_notifier(&adin1110_switchdev_notifier); err_netdev: unregister_netdevice_notifier(&adin1110_netdevice_nb); + return ret; } @@ -1613,10 +1613,6 @@ static int adin1110_probe_netdevs(struct adin1110_priv *priv) if (ret < 0) return ret; - ret = adin1110_setup_notifiers(priv); - if (ret < 0) - return ret; - for (i = 0; i < priv->cfg->ports_nr; i++) { ret = devm_register_netdev(dev, priv->ports[i]->netdev); if (ret < 0) { @@ -1693,7 +1689,31 @@ static struct spi_driver adin1110_driver = { .probe = adin1110_probe, .id_table = adin1110_spi_id, }; -module_spi_driver(adin1110_driver); + +static int __init adin1110_driver_init(void) +{ + int ret; + + ret = adin1110_setup_notifiers(); + if (ret < 0) + return ret; + + ret = spi_register_driver(&adin1110_driver); + if (ret < 0) { + adin1110_unregister_notifiers(); + return ret; + } + + return 0; +} + +static void __exit adin1110_exit(void) +{ + adin1110_unregister_notifiers(); + spi_unregister_driver(&adin1110_driver); +} +module_init(adin1110_driver_init); +module_exit(adin1110_exit); MODULE_DESCRIPTION("ADIN1110 Network driver"); MODULE_AUTHOR("Alexandru Tachici "); -- GitLab From a2c65a9d0568b6737c02b54f00b80716a53fac61 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Thu, 27 Oct 2022 17:54:39 +0300 Subject: [PATCH 0721/1927] net: dsa: fall back to default tagger if we can't load the one from DT DSA tagging protocol drivers can be changed at runtime through sysfs and at probe time through the device tree (support for the latter was added later). When changing through sysfs, it is assumed that the module for the new tagging protocol was already loaded into the kernel (in fact this is only a concern for Ocelot/Felix switches, where we have tag_ocelot.ko and tag_ocelot_8021q.ko; for every other switch, the default and alternative protocols are compiled within the same .ko, so there is nothing for the user to load). The kernel cannot currently call request_module(), because it has no way of constructing the modalias name of the tagging protocol driver ("dsa_tag-%d", where the number is one of DSA_TAG_PROTO_*_VALUE). The device tree only contains the string name of the tagging protocol ("ocelot-8021q"), and the only mapping between the string and the DSA_TAG_PROTO_OCELOT_8021Q_VALUE is present in tag_ocelot_8021q.ko. So this is a chicken-and-egg situation and dsa_core.ko has nothing based on which it can automatically request the insertion of the module. As a consequence, if CONFIG_NET_DSA_TAG_OCELOT_8021Q is built as module, the switch will forever defer probing. The long-term solution is to make DSA call request_module() somehow, but that probably needs some refactoring. What we can do to keep operating with existing device tree blobs is to cancel the attempt to change the tagging protocol with the one specified there, and to remain operating with the default one. Depending on the situation, the default protocol might still allow some functionality (in the case of ocelot, it does), and it's better to have that than to fail to probe. Fixes: deff710703d8 ("net: dsa: Allow default tag protocol to be overridden from DT") Link: https://lore.kernel.org/lkml/20221027113248.420216-1-michael@walle.cc/ Reported-by: Heiko Thiery Reported-by: Michael Walle Signed-off-by: Vladimir Oltean Tested-by: Michael Walle Reviewed-by: Florian Fainelli Link: https://lore.kernel.org/r/20221027145439.3086017-1-vladimir.oltean@nxp.com Signed-off-by: Jakub Kicinski --- net/dsa/dsa2.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index af0e2c0394ac3..e504a18fc1254 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -1409,9 +1409,9 @@ static enum dsa_tag_protocol dsa_get_tag_protocol(struct dsa_port *dp, static int dsa_port_parse_cpu(struct dsa_port *dp, struct net_device *master, const char *user_protocol) { + const struct dsa_device_ops *tag_ops = NULL; struct dsa_switch *ds = dp->ds; struct dsa_switch_tree *dst = ds->dst; - const struct dsa_device_ops *tag_ops; enum dsa_tag_protocol default_proto; /* Find out which protocol the switch would prefer. */ @@ -1434,10 +1434,17 @@ static int dsa_port_parse_cpu(struct dsa_port *dp, struct net_device *master, } tag_ops = dsa_find_tagger_by_name(user_protocol); - } else { - tag_ops = dsa_tag_driver_get(default_proto); + if (IS_ERR(tag_ops)) { + dev_warn(ds->dev, + "Failed to find a tagging driver for protocol %s, using default\n", + user_protocol); + tag_ops = NULL; + } } + if (!tag_ops) + tag_ops = dsa_tag_driver_get(default_proto); + if (IS_ERR(tag_ops)) { if (PTR_ERR(tag_ops) == -ENOPROTOOPT) return -EPROBE_DEFER; -- GitLab From 2f321fd6d89ad1e9525f5aa1f2be9202c2f3e724 Mon Sep 17 00:00:00 2001 From: Max Krummenacher Date: Thu, 22 Sep 2022 18:29:18 +0200 Subject: [PATCH 0722/1927] arm64: dts: verdin-imx8mp: fix ctrl_sleep_moci The GPIO signaling ctrl_sleep_moci is currently handled as a gpio hog. But the gpio-hog node is made a child of the wrong gpio controller. Move it to the node representing gpio4 so that it actually works. Without this carrier board components jumpered to use the signal are unconditionally switched off. Fixes: a39ed23bdf6e ("arm64: dts: freescale: add initial support for verdin imx8m plus") Signed-off-by: Max Krummenacher Signed-off-by: Marcel Ziswiler Signed-off-by: Shawn Guo --- .../boot/dts/freescale/imx8mp-verdin.dtsi | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi index 7b712d1888ead..5dcd1de586b52 100644 --- a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi @@ -354,16 +354,6 @@ &gpio2 { "SODIMM_82", "SODIMM_70", "SODIMM_72"; - - ctrl-sleep-moci-hog { - gpio-hog; - /* Verdin CTRL_SLEEP_MOCI# (SODIMM 256) */ - gpios = <29 GPIO_ACTIVE_HIGH>; - line-name = "CTRL_SLEEP_MOCI#"; - output-high; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_ctrl_sleep_moci>; - }; }; &gpio3 { @@ -432,6 +422,16 @@ &gpio4 { "SODIMM_256", "SODIMM_48", "SODIMM_44"; + + ctrl-sleep-moci-hog { + gpio-hog; + /* Verdin CTRL_SLEEP_MOCI# (SODIMM 256) */ + gpios = <29 GPIO_ACTIVE_HIGH>; + line-name = "CTRL_SLEEP_MOCI#"; + output-high; + pinctrl-names = "default"; + pinctrl-0 = <&pinctrl_ctrl_sleep_moci>; + }; }; /* On-module I2C */ -- GitLab From e1ec45b9a8127d9d31bb9fc1d802571a2ba8dd89 Mon Sep 17 00:00:00 2001 From: Li Jun Date: Mon, 26 Sep 2022 19:45:32 +0800 Subject: [PATCH 0723/1927] arm64: dts: imx8mm: remove otg1/2 power domain dependency on hsio pgc_otg1/2 are independent power domain of hsio, they for usb phy, so remove hsio power domain dependency from its node. Fixes: d39d4bb15310 ("arm64: dts: imx8mm: add GPC node") Signed-off-by: Li Jun Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx8mm.dtsi | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi index afb90f59c83c5..41204b871f4f3 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi @@ -674,13 +674,11 @@ pgc_pcie: power-domain@1 { pgc_otg1: power-domain@2 { #power-domain-cells = <0>; reg = ; - power-domains = <&pgc_hsiomix>; }; pgc_otg2: power-domain@3 { #power-domain-cells = <0>; reg = ; - power-domains = <&pgc_hsiomix>; }; pgc_gpumix: power-domain@4 { -- GitLab From 4585c79ff477f9517b7f384a4fce351417e8fa36 Mon Sep 17 00:00:00 2001 From: Li Jun Date: Mon, 26 Sep 2022 19:45:33 +0800 Subject: [PATCH 0724/1927] arm64: dts: imx8mm: correct usb power domains pgc_otg1/2 is actual the power domain of usb PHY, usb controller is in hsio power domain, and pgc_otg1/2 is required to be powered up to detect usb remote wakeup, so move the pgc_otg1/2 power domain to the usb phy node. Fixes: 01df28d80859 ("arm64: dts: imx8mm: put USB controllers into power-domains") Signed-off-by: Li Jun Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx8mm.dtsi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi index 41204b871f4f3..dabd94dc30c4b 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi @@ -276,6 +276,7 @@ usbphynop1: usbphynop1 { assigned-clocks = <&clk IMX8MM_CLK_USB_PHY_REF>; assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_100M>; clock-names = "main_clk"; + power-domains = <&pgc_otg1>; }; usbphynop2: usbphynop2 { @@ -285,6 +286,7 @@ usbphynop2: usbphynop2 { assigned-clocks = <&clk IMX8MM_CLK_USB_PHY_REF>; assigned-clock-parents = <&clk IMX8MM_SYS_PLL1_100M>; clock-names = "main_clk"; + power-domains = <&pgc_otg2>; }; soc: soc@0 { @@ -1184,7 +1186,7 @@ usbotg1: usb@32e40000 { assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_500M>; phys = <&usbphynop1>; fsl,usbmisc = <&usbmisc1 0>; - power-domains = <&pgc_otg1>; + power-domains = <&pgc_hsiomix>; status = "disabled"; }; @@ -1204,7 +1206,7 @@ usbotg2: usb@32e50000 { assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_500M>; phys = <&usbphynop2>; fsl,usbmisc = <&usbmisc2 0>; - power-domains = <&pgc_otg2>; + power-domains = <&pgc_hsiomix>; status = "disabled"; }; -- GitLab From 9e0bbb7a5218d856f1ccf8f1bf38c8869572b464 Mon Sep 17 00:00:00 2001 From: Li Jun Date: Mon, 26 Sep 2022 19:45:34 +0800 Subject: [PATCH 0725/1927] arm64: dts: imx8mn: remove otg1 power domain dependency on hsio pgc_otg1 is an independent power domain of hsio, it's for usb phy, so remove hsio power domain from its node. Fixes: 8b8ebec67360 ("arm64: dts: imx8mn: add GPC node") Signed-off-by: Li Jun Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx8mn.dtsi | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mn.dtsi b/arch/arm64/boot/dts/freescale/imx8mn.dtsi index cb2836bfbd95c..950f432627fe8 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn.dtsi @@ -662,7 +662,6 @@ pgc_hsiomix: power-domain@0 { pgc_otg1: power-domain@1 { #power-domain-cells = <0>; reg = ; - power-domains = <&pgc_hsiomix>; }; pgc_gpumix: power-domain@2 { -- GitLab From ee895139a761bdb7869f9f5b9ccc19a064d0d740 Mon Sep 17 00:00:00 2001 From: Li Jun Date: Mon, 26 Sep 2022 19:45:35 +0800 Subject: [PATCH 0726/1927] arm64: dts: imx8mn: Correct the usb power domain pgc_otg1 is actual the power domain of usb PHY, usb controller is in hsio power domain, and pgc_otg1 is required to be powered up to detect usb remote wakeup, so move the pgc_otg1 power domain to the usb phy node. Fixes: ea2b5af58ab2 ("arm64: dts: imx8mn: put USB controller into power-domains") Signed-off-by: Li Jun Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx8mn.dtsi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mn.dtsi b/arch/arm64/boot/dts/freescale/imx8mn.dtsi index 950f432627fe8..ad0b99adf6911 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn.dtsi @@ -1075,7 +1075,7 @@ usbotg1: usb@32e40000 { assigned-clock-parents = <&clk IMX8MN_SYS_PLL2_500M>; phys = <&usbphynop1>; fsl,usbmisc = <&usbmisc1 0>; - power-domains = <&pgc_otg1>; + power-domains = <&pgc_hsiomix>; status = "disabled"; }; @@ -1174,5 +1174,6 @@ usbphynop1: usbphynop1 { assigned-clocks = <&clk IMX8MN_CLK_USB_PHY_REF>; assigned-clock-parents = <&clk IMX8MN_SYS_PLL1_100M>; clock-names = "main_clk"; + power-domains = <&pgc_otg1>; }; }; -- GitLab From bb5ad73941dc3f4e3c2241348f385da6501d50ea Mon Sep 17 00:00:00 2001 From: Tim Harvey Date: Thu, 29 Sep 2022 12:52:22 -0700 Subject: [PATCH 0727/1927] ARM: dts: imx6qdl-gw59{10,13}: fix user pushbutton GPIO offset The GW5910 and GW5913 have a user pushbutton that is tied to the Gateworks System Controller GPIO offset 2. Fix the invalid offset of 0. Fixes: 64bf0a0af18d ("ARM: dts: imx6qdl-gw: add Gateworks System Controller support") Signed-off-by: Tim Harvey Signed-off-by: Shawn Guo --- arch/arm/boot/dts/imx6qdl-gw5910.dtsi | 2 +- arch/arm/boot/dts/imx6qdl-gw5913.dtsi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/boot/dts/imx6qdl-gw5910.dtsi b/arch/arm/boot/dts/imx6qdl-gw5910.dtsi index 68e5ab2e27e22..6bb4855d13ce5 100644 --- a/arch/arm/boot/dts/imx6qdl-gw5910.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw5910.dtsi @@ -29,7 +29,7 @@ gpio-keys { user-pb { label = "user_pb"; - gpios = <&gsc_gpio 0 GPIO_ACTIVE_LOW>; + gpios = <&gsc_gpio 2 GPIO_ACTIVE_LOW>; linux,code = ; }; diff --git a/arch/arm/boot/dts/imx6qdl-gw5913.dtsi b/arch/arm/boot/dts/imx6qdl-gw5913.dtsi index 8e23cec7149e5..696427b487f01 100644 --- a/arch/arm/boot/dts/imx6qdl-gw5913.dtsi +++ b/arch/arm/boot/dts/imx6qdl-gw5913.dtsi @@ -26,7 +26,7 @@ gpio-keys { user-pb { label = "user_pb"; - gpios = <&gsc_gpio 0 GPIO_ACTIVE_LOW>; + gpios = <&gsc_gpio 2 GPIO_ACTIVE_LOW>; linux,code = ; }; -- GitLab From 5e67d47d0b010f0704aca469d6d27637b1dcb2ce Mon Sep 17 00:00:00 2001 From: Petr Benes Date: Tue, 4 Oct 2022 17:39:20 +0200 Subject: [PATCH 0728/1927] ARM: dts: imx6dl-yapp4: Do not allow PM to switch PU regulator off on Q/QP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix our design flaw in supply voltage distribution on the Quad and QuadPlus based boards. The problem is that we supply the SoC cache (VDD_CACHE_CAP) from VDD_PU instead of VDD_SOC. The VDD_PU internal regulator can be disabled by PM if VPU or GPU is not used. If that happens the system freezes. To prevent that configure the reg_pu regulator to be always on. Fixes: 0de4ab81ab26 ("ARM: dts: imx6dl-yapp4: Add Y Soft IOTA Crux/Crux+ board") Cc: petrben@gmail.com Cc: stable@vger.kernel.org Signed-off-by: Petr Benes Signed-off-by: Michal Vokáč Signed-off-by: Shawn Guo --- arch/arm/boot/dts/imx6q-yapp4-crux.dts | 4 ++++ arch/arm/boot/dts/imx6qp-yapp4-crux-plus.dts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/arch/arm/boot/dts/imx6q-yapp4-crux.dts b/arch/arm/boot/dts/imx6q-yapp4-crux.dts index 15f4824a5142a..bddf3822ebf73 100644 --- a/arch/arm/boot/dts/imx6q-yapp4-crux.dts +++ b/arch/arm/boot/dts/imx6q-yapp4-crux.dts @@ -33,6 +33,10 @@ &oled_1309 { status = "okay"; }; +®_pu { + regulator-always-on; +}; + ®_usb_h1_vbus { status = "okay"; }; diff --git a/arch/arm/boot/dts/imx6qp-yapp4-crux-plus.dts b/arch/arm/boot/dts/imx6qp-yapp4-crux-plus.dts index cea165f2161a3..afaf4a6759d4b 100644 --- a/arch/arm/boot/dts/imx6qp-yapp4-crux-plus.dts +++ b/arch/arm/boot/dts/imx6qp-yapp4-crux-plus.dts @@ -33,6 +33,10 @@ &oled_1309 { status = "okay"; }; +®_pu { + regulator-always-on; +}; + ®_usb_h1_vbus { status = "okay"; }; -- GitLab From 06acb824d7d00a30e9400f67eee481b218371b5a Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 10 Oct 2022 18:07:47 +0800 Subject: [PATCH 0729/1927] arm64: dts: imx8: correct clock order Per bindings/mmc/fsl-imx-esdhc.yaml, the clock order is ipg, ahb, per, otherwise warning: " mmc@5b020000: clock-names:1: 'ahb' was expected mmc@5b020000: clock-names:2: 'per' was expected " Fixes: 16c4ea7501b1 ("arm64: dts: imx8: switch to new lpcg clock binding") Signed-off-by: Peng Fan Signed-off-by: Shawn Guo --- .../arm64/boot/dts/freescale/imx8-ss-conn.dtsi | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-conn.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-conn.dtsi index 82a1c44883786..10370d1a6c6de 100644 --- a/arch/arm64/boot/dts/freescale/imx8-ss-conn.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-ss-conn.dtsi @@ -38,9 +38,9 @@ usdhc1: mmc@5b010000 { interrupts = ; reg = <0x5b010000 0x10000>; clocks = <&sdhc0_lpcg IMX_LPCG_CLK_4>, - <&sdhc0_lpcg IMX_LPCG_CLK_5>, - <&sdhc0_lpcg IMX_LPCG_CLK_0>; - clock-names = "ipg", "per", "ahb"; + <&sdhc0_lpcg IMX_LPCG_CLK_0>, + <&sdhc0_lpcg IMX_LPCG_CLK_5>; + clock-names = "ipg", "ahb", "per"; power-domains = <&pd IMX_SC_R_SDHC_0>; status = "disabled"; }; @@ -49,9 +49,9 @@ usdhc2: mmc@5b020000 { interrupts = ; reg = <0x5b020000 0x10000>; clocks = <&sdhc1_lpcg IMX_LPCG_CLK_4>, - <&sdhc1_lpcg IMX_LPCG_CLK_5>, - <&sdhc1_lpcg IMX_LPCG_CLK_0>; - clock-names = "ipg", "per", "ahb"; + <&sdhc1_lpcg IMX_LPCG_CLK_0>, + <&sdhc1_lpcg IMX_LPCG_CLK_5>; + clock-names = "ipg", "ahb", "per"; power-domains = <&pd IMX_SC_R_SDHC_1>; fsl,tuning-start-tap = <20>; fsl,tuning-step = <2>; @@ -62,9 +62,9 @@ usdhc3: mmc@5b030000 { interrupts = ; reg = <0x5b030000 0x10000>; clocks = <&sdhc2_lpcg IMX_LPCG_CLK_4>, - <&sdhc2_lpcg IMX_LPCG_CLK_5>, - <&sdhc2_lpcg IMX_LPCG_CLK_0>; - clock-names = "ipg", "per", "ahb"; + <&sdhc2_lpcg IMX_LPCG_CLK_0>, + <&sdhc2_lpcg IMX_LPCG_CLK_5>; + clock-names = "ipg", "ahb", "per"; power-domains = <&pd IMX_SC_R_SDHC_2>; status = "disabled"; }; -- GitLab From ef370d8ceec62322dee24c960af8ca67a749f34d Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Mon, 10 Oct 2022 18:09:58 +0800 Subject: [PATCH 0730/1927] dt-bindings: power: gpcv2: add power-domains property Some pgc power-domain requires a parent power domain, so add an optional power-domains property, otherwise there will be dt check warning: gpc@303a0000: pgc:power-domain@1: 'power-domains' does not match any of the regexes: 'pinctrl-[0-9]+' Fixes: 30af8513bdb5 ("dt-bindings: power: add defines for i.MX8MM power domains") Signed-off-by: Peng Fan Acked-by: Krzysztof Kozlowski Signed-off-by: Shawn Guo --- Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml b/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml index 58022ae7d5ddc..dfdb8dfb6b653 100644 --- a/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml +++ b/Documentation/devicetree/bindings/power/fsl,imx-gpcv2.yaml @@ -81,6 +81,9 @@ properties: power-supply: true + power-domains: + maxItems: 1 + resets: description: | A number of phandles to resets that need to be asserted during -- GitLab From 82ce591967517b733c6e6e6882b5096d239b3afe Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Thu, 20 Oct 2022 18:16:56 +0800 Subject: [PATCH 0731/1927] arm64: dts: imx93: correct s4mu interrupt names Per binding doc, interrupt names should be tx and rx. Fixes: 0dfb380d2492 ("arm64: dts: imx93: add s4 mu node") Signed-off-by: Peng Fan Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx93.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/freescale/imx93.dtsi b/arch/arm64/boot/dts/freescale/imx93.dtsi index 3a5713bb4880e..55a70ee6f91af 100644 --- a/arch/arm64/boot/dts/freescale/imx93.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93.dtsi @@ -501,7 +501,7 @@ s4muap: mailbox@47520000 { reg = <0x47520000 0x10000>; interrupts = , ; - interrupt-names = "txirq", "rxirq"; + interrupt-names = "tx", "rx"; #mbox-cells = <2>; }; -- GitLab From d92a110130d492bd5eab81827ce3730581dc933a Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Thu, 20 Oct 2022 18:16:57 +0800 Subject: [PATCH 0732/1927] arm64: dts: imx93: correct gpio-ranges Per imx93-pinfunc.h and pinctrl-imx93.c, correct gpio-ranges. Fixes: ec8b5b5058ea ("arm64: dts: freescale: Add i.MX93 dtsi support") Reported-by: David Wolfe Reviewed-by: Haibo Chen Reviewed-by: Jacky Bai Signed-off-by: Peng Fan Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx93.dtsi | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx93.dtsi b/arch/arm64/boot/dts/freescale/imx93.dtsi index 55a70ee6f91af..0247866fc86b0 100644 --- a/arch/arm64/boot/dts/freescale/imx93.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93.dtsi @@ -451,7 +451,7 @@ gpio2: gpio@43810080 { clocks = <&clk IMX93_CLK_GPIO2_GATE>, <&clk IMX93_CLK_GPIO2_GATE>; clock-names = "gpio", "port"; - gpio-ranges = <&iomuxc 0 32 32>; + gpio-ranges = <&iomuxc 0 4 30>; }; gpio3: gpio@43820080 { @@ -465,7 +465,8 @@ gpio3: gpio@43820080 { clocks = <&clk IMX93_CLK_GPIO3_GATE>, <&clk IMX93_CLK_GPIO3_GATE>; clock-names = "gpio", "port"; - gpio-ranges = <&iomuxc 0 64 32>; + gpio-ranges = <&iomuxc 0 84 8>, <&iomuxc 8 66 18>, + <&iomuxc 26 34 2>, <&iomuxc 28 0 4>; }; gpio4: gpio@43830080 { @@ -479,7 +480,7 @@ gpio4: gpio@43830080 { clocks = <&clk IMX93_CLK_GPIO4_GATE>, <&clk IMX93_CLK_GPIO4_GATE>; clock-names = "gpio", "port"; - gpio-ranges = <&iomuxc 0 96 32>; + gpio-ranges = <&iomuxc 0 38 28>, <&iomuxc 28 36 2>; }; gpio1: gpio@47400080 { @@ -493,7 +494,7 @@ gpio1: gpio@47400080 { clocks = <&clk IMX93_CLK_GPIO1_GATE>, <&clk IMX93_CLK_GPIO1_GATE>; clock-names = "gpio", "port"; - gpio-ranges = <&iomuxc 0 0 32>; + gpio-ranges = <&iomuxc 0 92 16>; }; s4muap: mailbox@47520000 { -- GitLab From 0ba7b623f15d52fa056eca26573d8cf1b9c29fd1 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 22 Oct 2022 08:08:55 +0200 Subject: [PATCH 0733/1927] soc: imx: imx93-pd: Fix the error handling path of imx93_pd_probe() In imx93_pd_probe(); if an error occurs, some resources need to be released as done in the remove function. Fixes: 0a0f7cc25d4a ("soc: imx: add i.MX93 SRC power domain driver") Signed-off-by: Christophe JAILLET Signed-off-by: Shawn Guo --- drivers/soc/imx/imx93-pd.c | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/drivers/soc/imx/imx93-pd.c b/drivers/soc/imx/imx93-pd.c index 1f3d7039c1de9..4d235c8c4924d 100644 --- a/drivers/soc/imx/imx93-pd.c +++ b/drivers/soc/imx/imx93-pd.c @@ -135,11 +135,24 @@ static int imx93_pd_probe(struct platform_device *pdev) ret = pm_genpd_init(&domain->genpd, NULL, domain->init_off); if (ret) - return ret; + goto err_clk_unprepare; platform_set_drvdata(pdev, domain); - return of_genpd_add_provider_simple(np, &domain->genpd); + ret = of_genpd_add_provider_simple(np, &domain->genpd); + if (ret) + goto err_genpd_remove; + + return 0; + +err_genpd_remove: + pm_genpd_remove(&domain->genpd); + +err_clk_unprepare: + if (!domain->init_off) + clk_bulk_disable_unprepare(domain->num_clks, domain->clks); + + return ret; } static const struct of_device_id imx93_pd_ids[] = { -- GitLab From b40fa75e1542e069a4eb9b33d62061d4ae734537 Mon Sep 17 00:00:00 2001 From: Jinyang He Date: Sat, 29 Oct 2022 16:29:31 +0800 Subject: [PATCH 0734/1927] LoongArch: Remove unused kernel stack padding The current LoongArch kernel stack is padded as if obeying the MIPS o32 calling convention (32 bytes), signifying the port's MIPS lineage but no longer making sense. Remove the padding for clarity. Reviewed-by: WANG Xuerui Signed-off-by: Jinyang He Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/processor.h | 2 +- arch/loongarch/include/asm/ptrace.h | 2 +- arch/loongarch/kernel/head.S | 3 +-- arch/loongarch/kernel/process.c | 4 ++-- arch/loongarch/kernel/switch.S | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/loongarch/include/asm/processor.h b/arch/loongarch/include/asm/processor.h index 6954dc5d24e9d..7184f1dc61f27 100644 --- a/arch/loongarch/include/asm/processor.h +++ b/arch/loongarch/include/asm/processor.h @@ -191,7 +191,7 @@ static inline void flush_thread(void) unsigned long __get_wchan(struct task_struct *p); #define __KSTK_TOS(tsk) ((unsigned long)task_stack_page(tsk) + \ - THREAD_SIZE - 32 - sizeof(struct pt_regs)) + THREAD_SIZE - sizeof(struct pt_regs)) #define task_pt_regs(tsk) ((struct pt_regs *)__KSTK_TOS(tsk)) #define KSTK_EIP(tsk) (task_pt_regs(tsk)->csr_era) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->regs[3]) diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index 17838c6b7ccd5..82649a78fec1a 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -133,7 +133,7 @@ static inline void die_if_kernel(const char *str, struct pt_regs *regs) #define current_pt_regs() \ ({ \ unsigned long sp = (unsigned long)__builtin_frame_address(0); \ - (struct pt_regs *)((sp | (THREAD_SIZE - 1)) + 1 - 32) - 1; \ + (struct pt_regs *)((sp | (THREAD_SIZE - 1)) + 1) - 1; \ }) /* Helpers for working with the user stack pointer */ diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S index 97425779ce9f3..84970e2666588 100644 --- a/arch/loongarch/kernel/head.S +++ b/arch/loongarch/kernel/head.S @@ -84,10 +84,9 @@ SYM_CODE_START(kernel_entry) # kernel entry point la.pcrel tp, init_thread_union /* Set the SP after an empty pt_regs. */ - PTR_LI sp, (_THREAD_SIZE - 32 - PT_SIZE) + PTR_LI sp, (_THREAD_SIZE - PT_SIZE) PTR_ADD sp, sp, tp set_saved_sp sp, t0, t1 - PTR_ADDI sp, sp, -4 * SZREG # init stack pointer bl start_kernel ASM_BUG() diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c index 1256e3582475f..2526b68f1c0f0 100644 --- a/arch/loongarch/kernel/process.c +++ b/arch/loongarch/kernel/process.c @@ -129,7 +129,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) unsigned long clone_flags = args->flags; struct pt_regs *childregs, *regs = current_pt_regs(); - childksp = (unsigned long)task_stack_page(p) + THREAD_SIZE - 32; + childksp = (unsigned long)task_stack_page(p) + THREAD_SIZE; /* set up new TSS. */ childregs = (struct pt_regs *) childksp - 1; @@ -236,7 +236,7 @@ bool in_task_stack(unsigned long stack, struct task_struct *task, struct stack_info *info) { unsigned long begin = (unsigned long)task_stack_page(task); - unsigned long end = begin + THREAD_SIZE - 32; + unsigned long end = begin + THREAD_SIZE; if (stack < begin || stack >= end) return false; diff --git a/arch/loongarch/kernel/switch.S b/arch/loongarch/kernel/switch.S index 43ebbc3990f73..202a163cb32f6 100644 --- a/arch/loongarch/kernel/switch.S +++ b/arch/loongarch/kernel/switch.S @@ -26,7 +26,7 @@ SYM_FUNC_START(__switch_to) move tp, a2 cpu_restore_nonscratch a1 - li.w t0, _THREAD_SIZE - 32 + li.w t0, _THREAD_SIZE PTR_ADD t0, t0, tp set_saved_sp t0, t1, t2 -- GitLab From 4805a13d54be3f5e06436d41fdb13f24012a3c6c Mon Sep 17 00:00:00 2001 From: Yushan Zhou Date: Sat, 29 Oct 2022 16:29:31 +0800 Subject: [PATCH 0735/1927] LoongArch: Use flexible-array member instead of zero-length array Eliminate the following coccicheck warning: ./arch/loongarch/include/asm/ptrace.h:32:15-21: WARNING use flexible-array member instead Reviewed-by: WANG Xuerui Signed-off-by: Yushan Zhou Signed-off-by: Huacai Chen --- arch/loongarch/include/asm/ptrace.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/loongarch/include/asm/ptrace.h b/arch/loongarch/include/asm/ptrace.h index 82649a78fec1a..59c4608de91db 100644 --- a/arch/loongarch/include/asm/ptrace.h +++ b/arch/loongarch/include/asm/ptrace.h @@ -29,7 +29,7 @@ struct pt_regs { unsigned long csr_euen; unsigned long csr_ecfg; unsigned long csr_estat; - unsigned long __last[0]; + unsigned long __last[]; } __aligned(8); static inline int regs_irqs_disabled(struct pt_regs *regs) -- GitLab From bbfddb904df6f82a5948687a2d57766216b9bc0f Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sat, 29 Oct 2022 16:29:31 +0800 Subject: [PATCH 0736/1927] LoongArch: BPF: Avoid declare variables in switch-case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not all compilers support declare variables in switch-case, so move declarations to the beginning of a function. Otherwise we may get such build errors: arch/loongarch/net/bpf_jit.c: In function ‘emit_atomic’: arch/loongarch/net/bpf_jit.c:362:3: error: a label can only be part of a statement and a declaration is not a statement u8 r0 = regmap[BPF_REG_0]; ^~ arch/loongarch/net/bpf_jit.c: In function ‘build_insn’: arch/loongarch/net/bpf_jit.c:727:3: error: a label can only be part of a statement and a declaration is not a statement u8 t7 = -1; ^~ arch/loongarch/net/bpf_jit.c:778:3: error: a label can only be part of a statement and a declaration is not a statement int ret; ^~~ arch/loongarch/net/bpf_jit.c:779:3: error: expected expression before ‘u64’ u64 func_addr; ^~~ arch/loongarch/net/bpf_jit.c:780:3: warning: ISO C90 forbids mixed declarations and code [-Wdeclaration-after-statement] bool func_addr_fixed; ^~~~ arch/loongarch/net/bpf_jit.c:784:11: error: ‘func_addr’ undeclared (first use in this function); did you mean ‘in_addr’? &func_addr, &func_addr_fixed); ^~~~~~~~~ in_addr arch/loongarch/net/bpf_jit.c:784:11: note: each undeclared identifier is reported only once for each function it appears in arch/loongarch/net/bpf_jit.c:814:3: error: a label can only be part of a statement and a declaration is not a statement u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; ^~~ Signed-off-by: Huacai Chen --- arch/loongarch/net/bpf_jit.c | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index 43f0a98efe380..bdcd0c7719a9e 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -279,6 +279,7 @@ static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) const u8 t1 = LOONGARCH_GPR_T1; const u8 t2 = LOONGARCH_GPR_T2; const u8 t3 = LOONGARCH_GPR_T3; + const u8 r0 = regmap[BPF_REG_0]; const u8 src = regmap[insn->src_reg]; const u8 dst = regmap[insn->dst_reg]; const s16 off = insn->off; @@ -359,8 +360,6 @@ static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) break; /* r0 = atomic_cmpxchg(dst + off, r0, src); */ case BPF_CMPXCHG: - u8 r0 = regmap[BPF_REG_0]; - move_reg(ctx, t2, r0); if (isdw) { emit_insn(ctx, lld, r0, t1, 0); @@ -390,8 +389,11 @@ static bool is_signed_bpf_cond(u8 cond) static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass) { - const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || - BPF_CLASS(insn->code) == BPF_JMP32; + u8 tm = -1; + u64 func_addr; + bool func_addr_fixed; + int i = insn - ctx->prog->insnsi; + int ret, jmp_offset; const u8 code = insn->code; const u8 cond = BPF_OP(code); const u8 t1 = LOONGARCH_GPR_T1; @@ -400,8 +402,8 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext const u8 dst = regmap[insn->dst_reg]; const s16 off = insn->off; const s32 imm = insn->imm; - int jmp_offset; - int i = insn - ctx->prog->insnsi; + const u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; + const bool is32 = BPF_CLASS(insn->code) == BPF_ALU || BPF_CLASS(insn->code) == BPF_JMP32; switch (code) { /* dst = src */ @@ -724,24 +726,23 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: - u8 t7 = -1; jmp_offset = bpf2la_offset(i, off, ctx); if (imm) { move_imm(ctx, t1, imm, false); - t7 = t1; + tm = t1; } else { /* If imm is 0, simply use zero register. */ - t7 = LOONGARCH_GPR_ZERO; + tm = LOONGARCH_GPR_ZERO; } move_reg(ctx, t2, dst); if (is_signed_bpf_cond(BPF_OP(code))) { - emit_sext_32(ctx, t7, is32); + emit_sext_32(ctx, tm, is32); emit_sext_32(ctx, t2, is32); } else { - emit_zext_32(ctx, t7, is32); + emit_zext_32(ctx, tm, is32); emit_zext_32(ctx, t2, is32); } - if (emit_cond_jmp(ctx, cond, t2, t7, jmp_offset) < 0) + if (emit_cond_jmp(ctx, cond, t2, tm, jmp_offset) < 0) goto toofar; break; @@ -775,10 +776,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* function call */ case BPF_JMP | BPF_CALL: - int ret; - u64 func_addr; - bool func_addr_fixed; - mark_call(ctx); ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &func_addr, &func_addr_fixed); @@ -811,8 +808,6 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* dst = imm64 */ case BPF_LD | BPF_IMM | BPF_DW: - u64 imm64 = (u64)(insn + 1)->imm << 32 | (u32)insn->imm; - move_imm(ctx, dst, imm64, is32); return 1; -- GitLab From fbe605ab157b174385b3f19ce33928d3548a9b09 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Sat, 29 Oct 2022 16:29:31 +0800 Subject: [PATCH 0737/1927] platform/loongarch: laptop: Adjust resume order for loongson_hotkey_resume() Some laptops don't support SW_LID, but still have backlight control, move backlight resuming before SW_LID event handling so as to avoid backlight mistake due to early return. Signed-off-by: Huacai Chen --- drivers/platform/loongarch/loongson-laptop.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c index f0166ad5d2c28..0e6f4efe21e5d 100644 --- a/drivers/platform/loongarch/loongson-laptop.c +++ b/drivers/platform/loongarch/loongson-laptop.c @@ -199,6 +199,13 @@ static int loongson_hotkey_resume(struct device *dev) struct key_entry ke; struct backlight_device *bd; + bd = backlight_device_get_by_type(BACKLIGHT_PLATFORM); + if (bd) { + loongson_laptop_backlight_update(bd) ? + pr_warn("Loongson_backlight: resume brightness failed") : + pr_info("Loongson_backlight: resume brightness %d\n", bd->props.brightness); + } + /* * Only if the firmware supports SW_LID event model, we can handle the * event. This is for the consideration of development board without EC. @@ -228,13 +235,6 @@ static int loongson_hotkey_resume(struct device *dev) } } - bd = backlight_device_get_by_type(BACKLIGHT_PLATFORM); - if (bd) { - loongson_laptop_backlight_update(bd) ? - pr_warn("Loongson_backlight: resume brightness failed") : - pr_info("Loongson_backlight: resume brightness %d\n", bd->props.brightness); - } - return 0; } -- GitLab From d81916910f7498fe7a768697e0101d488f9fe665 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sat, 29 Oct 2022 16:29:31 +0800 Subject: [PATCH 0738/1927] platform/loongarch: laptop: Fix possible UAF and simplify generic_acpi_laptop_init() Currently the return value of 'sub_driver->init' is not checked. If sparse_keymap_setup() called in the init function fails, 'generic_ inputdev' is freed, then it will lead a UAF when using it in generic_ acpi_laptop_init(). Fix it by checking the return value and setting generic_inputdev to NULL after free, so as to avoid double free it. The error code in generic_subdriver_init() is always negative, so the return of generic_subdriver_init() can be simplified. Fixes: 6246ed09111f ("LoongArch: Add ACPI-based generic laptop driver") Signed-off-by: Yang Yingliang Signed-off-by: Huacai Chen --- drivers/platform/loongarch/loongson-laptop.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c index 0e6f4efe21e5d..99203584949da 100644 --- a/drivers/platform/loongarch/loongson-laptop.c +++ b/drivers/platform/loongarch/loongson-laptop.c @@ -448,6 +448,7 @@ static int __init event_init(struct generic_sub_driver *sub_driver) if (ret < 0) { pr_err("Failed to setup input device keymap\n"); input_free_device(generic_inputdev); + generic_inputdev = NULL; return ret; } @@ -502,8 +503,11 @@ static int __init generic_subdriver_init(struct generic_sub_driver *sub_driver) if (ret) return -EINVAL; - if (sub_driver->init) - sub_driver->init(sub_driver); + if (sub_driver->init) { + ret = sub_driver->init(sub_driver); + if (ret) + goto err_out; + } if (sub_driver->notify) { ret = setup_acpi_notify(sub_driver); @@ -519,7 +523,7 @@ static int __init generic_subdriver_init(struct generic_sub_driver *sub_driver) err_out: generic_subdriver_exit(sub_driver); - return (ret < 0) ? ret : 0; + return ret; } static void generic_subdriver_exit(struct generic_sub_driver *sub_driver) -- GitLab From 46cf1954de3f324dc7f9472c12c3bd03b268a11b Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sat, 29 Oct 2022 16:20:01 +0800 Subject: [PATCH 0739/1927] ALSA: arm: pxa: pxa2xx-ac97-lib: fix return value check of platform_get_irq() platform_get_irq() returns negative error number on failure, fix the return value check in pxa2xx_ac97_hw_probe() and assign the error code to 'ret'. Fixes: 2548e6c76ebf ("ARM: pxa: pxa2xx-ac97-lib: use IRQ resource") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221029082001.3207380-1-yangyingliang@huawei.com Signed-off-by: Takashi Iwai --- sound/arm/pxa2xx-ac97-lib.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sound/arm/pxa2xx-ac97-lib.c b/sound/arm/pxa2xx-ac97-lib.c index e55c0421718b3..2ca33fd5a5757 100644 --- a/sound/arm/pxa2xx-ac97-lib.c +++ b/sound/arm/pxa2xx-ac97-lib.c @@ -402,8 +402,10 @@ int pxa2xx_ac97_hw_probe(struct platform_device *dev) goto err_clk2; irq = platform_get_irq(dev, 0); - if (!irq) + if (irq < 0) { + ret = irq; goto err_irq; + } ret = request_irq(irq, pxa2xx_ac97_irq, 0, "AC97", NULL); if (ret < 0) -- GitLab From 57572cacd36e6d4be7722d7770d23f4430219827 Mon Sep 17 00:00:00 2001 From: Jonathan Cameron Date: Sun, 2 Oct 2022 15:41:33 +0100 Subject: [PATCH 0740/1927] iio: accel: bma400: Ensure VDDIO is enable defore reading the chip ID. The regulator enables were after the check on the chip variant, which was very unlikely to return a correct value when not powered. Presumably all the device anyone is testing on have a regulator that is already powered up when this code runs for reasons beyond the scope of this driver. Move the read call down a few lines. Fixes: 3cf7ded15e40 ("iio: accel: bma400: basic regulator support") Signed-off-by: Jonathan Cameron Reviewed-by: Dan Robertson Cc: Link: https://lore.kernel.org/r/20221002144133.3771029-1-jic23@kernel.org --- drivers/iio/accel/bma400_core.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/iio/accel/bma400_core.c b/drivers/iio/accel/bma400_core.c index ad8fce3e08cd0..490c342ef72a6 100644 --- a/drivers/iio/accel/bma400_core.c +++ b/drivers/iio/accel/bma400_core.c @@ -869,18 +869,6 @@ static int bma400_init(struct bma400_data *data) unsigned int val; int ret; - /* Try to read chip_id register. It must return 0x90. */ - ret = regmap_read(data->regmap, BMA400_CHIP_ID_REG, &val); - if (ret) { - dev_err(data->dev, "Failed to read chip id register\n"); - return ret; - } - - if (val != BMA400_ID_REG_VAL) { - dev_err(data->dev, "Chip ID mismatch\n"); - return -ENODEV; - } - data->regulators[BMA400_VDD_REGULATOR].supply = "vdd"; data->regulators[BMA400_VDDIO_REGULATOR].supply = "vddio"; ret = devm_regulator_bulk_get(data->dev, @@ -906,6 +894,18 @@ static int bma400_init(struct bma400_data *data) if (ret) return ret; + /* Try to read chip_id register. It must return 0x90. */ + ret = regmap_read(data->regmap, BMA400_CHIP_ID_REG, &val); + if (ret) { + dev_err(data->dev, "Failed to read chip id register\n"); + return ret; + } + + if (val != BMA400_ID_REG_VAL) { + dev_err(data->dev, "Chip ID mismatch\n"); + return -ENODEV; + } + ret = bma400_get_power_mode(data); if (ret) { dev_err(data->dev, "Failed to get the initial power-mode\n"); -- GitLab From 17f442e7e47579d3881fc4d47354eaef09302e6f Mon Sep 17 00:00:00 2001 From: Mitja Spes Date: Fri, 21 Oct 2022 15:58:20 +0200 Subject: [PATCH 0741/1927] iio: pressure: ms5611: fixed value compensation bug When using multiple instances of this driver the compensation PROM was overwritten by the last initialized sensor. Now each sensor has own PROM storage. Signed-off-by: Mitja Spes Fixes: 9690d81a02dc ("iio: pressure: ms5611: add support for MS5607 temperature and pressure sensor") Link: https://lore.kernel.org/r/20221021135827.1444793-2-mitja@lxnav.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/pressure/ms5611.h | 12 +++---- drivers/iio/pressure/ms5611_core.c | 51 ++++++++++++++++-------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/drivers/iio/pressure/ms5611.h b/drivers/iio/pressure/ms5611.h index cbc9349c342a9..550b75b7186fb 100644 --- a/drivers/iio/pressure/ms5611.h +++ b/drivers/iio/pressure/ms5611.h @@ -25,13 +25,6 @@ enum { MS5607, }; -struct ms5611_chip_info { - u16 prom[MS5611_PROM_WORDS_NB]; - - int (*temp_and_pressure_compensate)(struct ms5611_chip_info *chip_info, - s32 *temp, s32 *pressure); -}; - /* * OverSampling Rate descriptor. * Warning: cmd MUST be kept aligned on a word boundary (see @@ -50,12 +43,15 @@ struct ms5611_state { const struct ms5611_osr *pressure_osr; const struct ms5611_osr *temp_osr; + u16 prom[MS5611_PROM_WORDS_NB]; + int (*reset)(struct ms5611_state *st); int (*read_prom_word)(struct ms5611_state *st, int index, u16 *word); int (*read_adc_temp_and_pressure)(struct ms5611_state *st, s32 *temp, s32 *pressure); - struct ms5611_chip_info *chip_info; + int (*compensate_temp_and_pressure)(struct ms5611_state *st, s32 *temp, + s32 *pressure); struct regulator *vdd; }; diff --git a/drivers/iio/pressure/ms5611_core.c b/drivers/iio/pressure/ms5611_core.c index 717521de66c47..c564a1d6cafe8 100644 --- a/drivers/iio/pressure/ms5611_core.c +++ b/drivers/iio/pressure/ms5611_core.c @@ -85,7 +85,7 @@ static int ms5611_read_prom(struct iio_dev *indio_dev) struct ms5611_state *st = iio_priv(indio_dev); for (i = 0; i < MS5611_PROM_WORDS_NB; i++) { - ret = st->read_prom_word(st, i, &st->chip_info->prom[i]); + ret = st->read_prom_word(st, i, &st->prom[i]); if (ret < 0) { dev_err(&indio_dev->dev, "failed to read prom at %d\n", i); @@ -93,7 +93,7 @@ static int ms5611_read_prom(struct iio_dev *indio_dev) } } - if (!ms5611_prom_is_valid(st->chip_info->prom, MS5611_PROM_WORDS_NB)) { + if (!ms5611_prom_is_valid(st->prom, MS5611_PROM_WORDS_NB)) { dev_err(&indio_dev->dev, "PROM integrity check failed\n"); return -ENODEV; } @@ -114,21 +114,20 @@ static int ms5611_read_temp_and_pressure(struct iio_dev *indio_dev, return ret; } - return st->chip_info->temp_and_pressure_compensate(st->chip_info, - temp, pressure); + return st->compensate_temp_and_pressure(st, temp, pressure); } -static int ms5611_temp_and_pressure_compensate(struct ms5611_chip_info *chip_info, +static int ms5611_temp_and_pressure_compensate(struct ms5611_state *st, s32 *temp, s32 *pressure) { s32 t = *temp, p = *pressure; s64 off, sens, dt; - dt = t - (chip_info->prom[5] << 8); - off = ((s64)chip_info->prom[2] << 16) + ((chip_info->prom[4] * dt) >> 7); - sens = ((s64)chip_info->prom[1] << 15) + ((chip_info->prom[3] * dt) >> 8); + dt = t - (st->prom[5] << 8); + off = ((s64)st->prom[2] << 16) + ((st->prom[4] * dt) >> 7); + sens = ((s64)st->prom[1] << 15) + ((st->prom[3] * dt) >> 8); - t = 2000 + ((chip_info->prom[6] * dt) >> 23); + t = 2000 + ((st->prom[6] * dt) >> 23); if (t < 2000) { s64 off2, sens2, t2; @@ -154,17 +153,17 @@ static int ms5611_temp_and_pressure_compensate(struct ms5611_chip_info *chip_inf return 0; } -static int ms5607_temp_and_pressure_compensate(struct ms5611_chip_info *chip_info, +static int ms5607_temp_and_pressure_compensate(struct ms5611_state *st, s32 *temp, s32 *pressure) { s32 t = *temp, p = *pressure; s64 off, sens, dt; - dt = t - (chip_info->prom[5] << 8); - off = ((s64)chip_info->prom[2] << 17) + ((chip_info->prom[4] * dt) >> 6); - sens = ((s64)chip_info->prom[1] << 16) + ((chip_info->prom[3] * dt) >> 7); + dt = t - (st->prom[5] << 8); + off = ((s64)st->prom[2] << 17) + ((st->prom[4] * dt) >> 6); + sens = ((s64)st->prom[1] << 16) + ((st->prom[3] * dt) >> 7); - t = 2000 + ((chip_info->prom[6] * dt) >> 23); + t = 2000 + ((st->prom[6] * dt) >> 23); if (t < 2000) { s64 off2, sens2, t2, tmp; @@ -342,15 +341,6 @@ static int ms5611_write_raw(struct iio_dev *indio_dev, static const unsigned long ms5611_scan_masks[] = {0x3, 0}; -static struct ms5611_chip_info chip_info_tbl[] = { - [MS5611] = { - .temp_and_pressure_compensate = ms5611_temp_and_pressure_compensate, - }, - [MS5607] = { - .temp_and_pressure_compensate = ms5607_temp_and_pressure_compensate, - } -}; - static const struct iio_chan_spec ms5611_channels[] = { { .type = IIO_PRESSURE, @@ -433,7 +423,20 @@ int ms5611_probe(struct iio_dev *indio_dev, struct device *dev, struct ms5611_state *st = iio_priv(indio_dev); mutex_init(&st->lock); - st->chip_info = &chip_info_tbl[type]; + + switch (type) { + case MS5611: + st->compensate_temp_and_pressure = + ms5611_temp_and_pressure_compensate; + break; + case MS5607: + st->compensate_temp_and_pressure = + ms5607_temp_and_pressure_compensate; + break; + default: + return -EINVAL; + } + st->temp_osr = &ms5611_avail_temp_osr[ARRAY_SIZE(ms5611_avail_temp_osr) - 1]; st->pressure_osr = -- GitLab From 741cec30cc52058d1c10d415f3b98319887e4f73 Mon Sep 17 00:00:00 2001 From: Mitja Spes Date: Fri, 21 Oct 2022 15:58:21 +0200 Subject: [PATCH 0742/1927] iio: pressure: ms5611: changed hardcoded SPI speed to value limited Don't hardcode the ms5611 SPI speed, limit it instead. Signed-off-by: Mitja Spes Fixes: c0644160a8b5 ("iio: pressure: add support for MS5611 pressure and temperature sensor") Link: https://lore.kernel.org/r/20221021135827.1444793-3-mitja@lxnav.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/pressure/ms5611_spi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iio/pressure/ms5611_spi.c b/drivers/iio/pressure/ms5611_spi.c index 432e912096f42..a0a7205c9c3a4 100644 --- a/drivers/iio/pressure/ms5611_spi.c +++ b/drivers/iio/pressure/ms5611_spi.c @@ -91,7 +91,7 @@ static int ms5611_spi_probe(struct spi_device *spi) spi_set_drvdata(spi, indio_dev); spi->mode = SPI_MODE_0; - spi->max_speed_hz = 20000000; + spi->max_speed_hz = min(spi->max_speed_hz, 20000000U); spi->bits_per_word = 8; ret = spi_setup(spi); if (ret < 0) -- GitLab From 1eb20332a082fa801fb89c347c5e62de916a4001 Mon Sep 17 00:00:00 2001 From: Saravanan Sekar Date: Sat, 29 Oct 2022 11:29:53 +0200 Subject: [PATCH 0743/1927] iio: adc: mp2629: fix wrong comparison of channel Input voltage channel enum is compared against iio address instead of the channel. Fixes: 7abd9fb64682 ("iio: adc: mp2629: Add support for mp2629 ADC driver") Signed-off-by: Saravanan Sekar Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20221029093000.45451-2-sravanhome@gmail.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/adc/mp2629_adc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iio/adc/mp2629_adc.c b/drivers/iio/adc/mp2629_adc.c index 30a31f185d08d..f7af9af1665d9 100644 --- a/drivers/iio/adc/mp2629_adc.c +++ b/drivers/iio/adc/mp2629_adc.c @@ -74,7 +74,7 @@ static int mp2629_read_raw(struct iio_dev *indio_dev, if (ret) return ret; - if (chan->address == MP2629_INPUT_VOLT) + if (chan->channel == MP2629_INPUT_VOLT) rval &= GENMASK(6, 0); *val = rval; return IIO_VAL_INT; -- GitLab From c126a0abc5dadd7df236f20aae6d8c3d103f095c Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Tue, 25 Oct 2022 17:41:15 +0300 Subject: [PATCH 0744/1927] arm64: dts: lx2160a: specify clock frequencies for the MDIO controllers Up until now, the external MDIO controller frequency values relied either on the default ones out of reset or on those setup by u-boot. Let's just properly specify the MDC frequency in the DTS so that even without u-boot's intervention Linux can drive the MDIO bus. Fixes: 6e1b8fae892d ("arm64: dts: lx2160a: add emdio1 node") Fixes: 5705b9dcda57 ("arm64: dts: lx2160a: add emdio2 node") Signed-off-by: Ioana Ciornei Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi index 6680fb2a6dc92..8c76d86cb7566 100644 --- a/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-lx2160a.dtsi @@ -1385,6 +1385,9 @@ emdio1: mdio@8b96000 { #address-cells = <1>; #size-cells = <0>; little-endian; + clock-frequency = <2500000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(2)>; status = "disabled"; }; @@ -1395,6 +1398,9 @@ emdio2: mdio@8b97000 { little-endian; #address-cells = <1>; #size-cells = <0>; + clock-frequency = <2500000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(2)>; status = "disabled"; }; -- GitLab From d78a57426e64fc4c61e6189e450a0432d24536ca Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Tue, 25 Oct 2022 17:41:16 +0300 Subject: [PATCH 0745/1927] arm64: dts: ls1088a: specify clock frequencies for the MDIO controllers Up until now, the external MDIO controller frequency values relied either on the default ones out of reset or on those setup by u-boot. Let's just properly specify the MDC frequency in the DTS so that even without u-boot's intervention Linux can drive the MDIO bus. Fixes: bbe75af7b092 ("arm64: dts: ls1088a: add external MDIO device nodes") Signed-off-by: Ioana Ciornei Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi index 421d879013d7f..260d045dbd9a8 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls1088a.dtsi @@ -779,6 +779,9 @@ emdio1: mdio@8b96000 { little-endian; #address-cells = <1>; #size-cells = <0>; + clock-frequency = <2500000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(1)>; status = "disabled"; }; @@ -788,6 +791,9 @@ emdio2: mdio@8b97000 { little-endian; #address-cells = <1>; #size-cells = <0>; + clock-frequency = <2500000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(1)>; status = "disabled"; }; -- GitLab From d5c921a53c80dfa942f6dff36253db5a50775a5f Mon Sep 17 00:00:00 2001 From: Ioana Ciornei Date: Tue, 25 Oct 2022 17:41:17 +0300 Subject: [PATCH 0746/1927] arm64: dts: ls208xa: specify clock frequencies for the MDIO controllers Up until now, the external MDIO controller frequency values relied either on the default ones out of reset or on those setup by u-boot. Let's just properly specify the MDC frequency in the DTS so that even without u-boot's intervention Linux can drive the MDIO bus. Fixes: 0420dde30a90 ("arm64: dts: ls208xa: add the external MDIO nodes") Signed-off-by: Ioana Ciornei Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi index f1b9cc8714dc0..348d9e3a91252 100644 --- a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi +++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi @@ -532,6 +532,9 @@ emdio1: mdio@8b96000 { little-endian; #address-cells = <1>; #size-cells = <0>; + clock-frequency = <2500000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(2)>; status = "disabled"; }; @@ -541,6 +544,9 @@ emdio2: mdio@8b97000 { little-endian; #address-cells = <1>; #size-cells = <0>; + clock-frequency = <2500000>; + clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL + QORIQ_CLK_PLL_DIV(2)>; status = "disabled"; }; -- GitLab From bfab00b94bd8569cdb84a6511d6615e6a8104e9c Mon Sep 17 00:00:00 2001 From: Aurelien Jarno Date: Mon, 26 Sep 2022 22:37:52 +0200 Subject: [PATCH 0747/1927] drm/rockchip: dw_hdmi: filter regulator -EPROBE_DEFER error messages When the avdd-0v9 or avdd-1v8 supply are not yet available, EPROBE_DEFER is returned by rockchip_hdmi_parse_dt(). This causes the following error message to be printed multiple times: dwhdmi-rockchip fe0a0000.hdmi: [drm:dw_hdmi_rockchip_bind [rockchipdrm]] *ERROR* Unable to parse OF data Fix that by not printing the message when rockchip_hdmi_parse_dt() returns -EPROBE_DEFER. Fixes: ca80c4eb4b01 ("drm/rockchip: dw_hdmi: add regulator support") Signed-off-by: Aurelien Jarno Reviewed-by: Dmitry Osipenko Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20220926203752.5430-1-aurelien@aurel32.net --- drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c index c14f888938688..2f4b8f64cbad3 100644 --- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c @@ -565,7 +565,8 @@ static int dw_hdmi_rockchip_bind(struct device *dev, struct device *master, ret = rockchip_hdmi_parse_dt(hdmi); if (ret) { - DRM_DEV_ERROR(hdmi->dev, "Unable to parse OF data\n"); + if (ret != -EPROBE_DEFER) + DRM_DEV_ERROR(hdmi->dev, "Unable to parse OF data\n"); return ret; } -- GitLab From 0be67e0556e469c57100ffe3c90df90abc796f3b Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 19 Oct 2022 17:03:48 -0700 Subject: [PATCH 0748/1927] drm/rockchip: dsi: Clean up 'usage_mode' when failing to attach If we fail to attach the first time (especially: EPROBE_DEFER), we fail to clean up 'usage_mode', and thus will fail to attach on any subsequent attempts, with "dsi controller already in use". Re-set to DW_DSI_USAGE_IDLE on attach failure. This is especially common to hit when enabling asynchronous probe on a duel-DSI system (such as RK3399 Gru/Scarlet), such that we're more likely to fail dw_mipi_dsi_rockchip_find_second() the first time. Fixes: 71f68fe7f121 ("drm/rockchip: dsi: add ability to work as a phy instead of full dsi") Cc: Signed-off-by: Brian Norris Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20221019170255.1.Ia68dfb27b835d31d22bfe23812baf366ee1c6eac@changeid --- drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c index bf6948125b841..d222c68112076 100644 --- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c @@ -1051,23 +1051,31 @@ static int dw_mipi_dsi_rockchip_host_attach(void *priv_data, if (ret) { DRM_DEV_ERROR(dsi->dev, "Failed to register component: %d\n", ret); - return ret; + goto out; } second = dw_mipi_dsi_rockchip_find_second(dsi); - if (IS_ERR(second)) - return PTR_ERR(second); + if (IS_ERR(second)) { + ret = PTR_ERR(second); + goto out; + } if (second) { ret = component_add(second, &dw_mipi_dsi_rockchip_ops); if (ret) { DRM_DEV_ERROR(second, "Failed to register component: %d\n", ret); - return ret; + goto out; } } return 0; + +out: + mutex_lock(&dsi->usage_mutex); + dsi->usage_mode = DW_DSI_USAGE_IDLE; + mutex_unlock(&dsi->usage_mutex); + return ret; } static int dw_mipi_dsi_rockchip_host_detach(void *priv_data, -- GitLab From 81e592f86f7afdb76d655e7fbd7803d7b8f985d8 Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 19 Oct 2022 17:03:49 -0700 Subject: [PATCH 0749/1927] drm/rockchip: dsi: Force synchronous probe We can't safely probe a dual-DSI display asynchronously (driver_async_probe='*' or driver_async_probe='dw-mipi-dsi-rockchip' cmdline), because dw_mipi_dsi_rockchip_find_second() pokes one DSI device's drvdata from the other device without any locking. Request synchronous probe, at least until this driver learns some appropriate locking for dual-DSI initialization. Cc: Signed-off-by: Brian Norris Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20221019170255.2.I6b985b0ca372b7e35c6d9ea970b24bcb262d4fc1@changeid --- drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c index d222c68112076..528ddce144e54 100644 --- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c @@ -1689,5 +1689,11 @@ struct platform_driver dw_mipi_dsi_rockchip_driver = { .of_match_table = dw_mipi_dsi_rockchip_dt_ids, .pm = &dw_mipi_dsi_rockchip_pm_ops, .name = "dw-mipi-dsi-rockchip", + /* + * For dual-DSI display, one DSI pokes at the other DSI's + * drvdata in dw_mipi_dsi_rockchip_find_second(). This is not + * safe for asynchronous probe. + */ + .probe_type = PROBE_FORCE_SYNCHRONOUS, }, }; -- GitLab From ab78c74cfc5a3caa2bbb7627cb8f3bca40bb5fb0 Mon Sep 17 00:00:00 2001 From: John Keeping Date: Thu, 20 Oct 2022 19:12:47 +0100 Subject: [PATCH 0750/1927] drm/rockchip: fix fbdev on non-IOMMU devices When switching to the generic fbdev infrastructure, it was missed that framebuffers were created with the alloc_kmap parameter to rockchip_gem_create_object() set to true. The generic infrastructure calls this via the .dumb_create() driver operation and thus creates a buffer without an associated kmap. alloc_kmap only makes a difference on devices without an IOMMU, but when it is missing rockchip_gem_prime_vmap() fails and the framebuffer cannot be used. Detect the case where a buffer is being allocated for the framebuffer and ensure a kernel mapping is created in this case. Fixes: 24af7c34b290 ("drm/rockchip: use generic fbdev setup") Reported-by: Johan Jonker Cc: Thomas Zimmermann Signed-off-by: John Keeping Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20221020181248.2497065-1-john@metanate.com --- drivers/gpu/drm/rockchip/rockchip_drm_gem.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c index 614e97aaac805..da8a69953706d 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c @@ -364,9 +364,12 @@ rockchip_gem_create_with_handle(struct drm_file *file_priv, { struct rockchip_gem_object *rk_obj; struct drm_gem_object *obj; + bool is_framebuffer; int ret; - rk_obj = rockchip_gem_create_object(drm, size, false); + is_framebuffer = drm->fb_helper && file_priv == drm->fb_helper->client.file; + + rk_obj = rockchip_gem_create_object(drm, size, is_framebuffer); if (IS_ERR(rk_obj)) return ERR_CAST(rk_obj); -- GitLab From 553c5a429aee26c9cfaf37ae158a8915540270fe Mon Sep 17 00:00:00 2001 From: Ondrej Jirman Date: Sun, 23 Oct 2022 18:07:47 +0200 Subject: [PATCH 0751/1927] drm/rockchip: dsi: Fix VOP selection on SoCs that support it lcdsel_grf_reg is defined as u32, so "< 0" comaprison is always false, which breaks VOP selection on eg. RK3399. Compare against 0. Fixes: f3aaa6125b6f ("drm/rockchip: dsi: add rk3568 support") Signed-off-by: Ondrej Jirman Tested-by: Chris Morgan Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20221023160747.607943-1-megi@xff.cz --- drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c index 528ddce144e54..f4df9820b295d 100644 --- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c @@ -752,7 +752,7 @@ static void dw_mipi_dsi_rockchip_config(struct dw_mipi_dsi_rockchip *dsi) static void dw_mipi_dsi_rockchip_set_lcdsel(struct dw_mipi_dsi_rockchip *dsi, int mux) { - if (dsi->cdata->lcdsel_grf_reg < 0) + if (dsi->cdata->lcdsel_grf_reg) regmap_write(dsi->grf_regmap, dsi->cdata->lcdsel_grf_reg, mux ? dsi->cdata->lcdsel_lit : dsi->cdata->lcdsel_big); } @@ -1643,7 +1643,6 @@ static const struct rockchip_dw_dsi_chip_data rk3399_chip_data[] = { static const struct rockchip_dw_dsi_chip_data rk3568_chip_data[] = { { .reg = 0xfe060000, - .lcdsel_grf_reg = -1, .lanecfg1_grf_reg = RK3568_GRF_VO_CON2, .lanecfg1 = HIWORD_UPDATE(0, RK3568_DSI0_SKEWCALHS | RK3568_DSI0_FORCETXSTOPMODE | @@ -1653,7 +1652,6 @@ static const struct rockchip_dw_dsi_chip_data rk3568_chip_data[] = { }, { .reg = 0xfe070000, - .lcdsel_grf_reg = -1, .lanecfg1_grf_reg = RK3568_GRF_VO_CON3, .lanecfg1 = HIWORD_UPDATE(0, RK3568_DSI1_SKEWCALHS | RK3568_DSI1_FORCETXSTOPMODE | -- GitLab From 471bf2406c043491b1a8288e5f04bc278f7d7ca1 Mon Sep 17 00:00:00 2001 From: Michael Tretter Date: Fri, 28 Oct 2022 11:52:05 +0200 Subject: [PATCH 0752/1927] drm/rockchip: vop2: fix null pointer in plane_atomic_disable If the vop2_plane_atomic_disable function is called with NULL as a state, accessing the old_pstate runs into a null pointer exception. However, the drm_atomic_helper_disable_planes_on_crtc function calls the atomic_disable callback with state NULL. Allow to disable a plane without passing a plane state by checking the old_pstate only if a state is passed. Signed-off-by: Michael Tretter Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20221028095206.2136601-2-m.tretter@pengutronix.de --- drivers/gpu/drm/rockchip/rockchip_drm_vop2.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index aac20be5ac082..26f8a8489ded4 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -996,13 +996,15 @@ static int vop2_plane_atomic_check(struct drm_plane *plane, static void vop2_plane_atomic_disable(struct drm_plane *plane, struct drm_atomic_state *state) { - struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(state, plane); + struct drm_plane_state *old_pstate = NULL; struct vop2_win *win = to_vop2_win(plane); struct vop2 *vop2 = win->vop2; drm_dbg(vop2->drm, "%s disable\n", win->data->name); - if (!old_pstate->crtc) + if (state) + old_pstate = drm_atomic_get_old_plane_state(state, plane); + if (old_pstate && !old_pstate->crtc) return; vop2_win_disable(win); -- GitLab From 447fb14bf07905b880c9ed1ea92c53d6dd0649d7 Mon Sep 17 00:00:00 2001 From: Michael Tretter Date: Fri, 28 Oct 2022 11:52:06 +0200 Subject: [PATCH 0753/1927] drm/rockchip: vop2: disable planes when disabling the crtc The vop2 driver needs to explicitly disable the planes if the crtc is disabled. Unless the planes are explicitly disabled, the address of the last framebuffer is kept in the registers of the VOP2. When re-enabling the encoder after it has been disabled by the driver, the VOP2 will start and read the framebuffer that has been freed but is still pointed to by the register. The iommu will catch these read accesses and print errors. Explicitly disable the planes when the crtc is disabled to reset the registers. Signed-off-by: Michael Tretter Signed-off-by: Heiko Stuebner Link: https://patchwork.freedesktop.org/patch/msgid/20221028095206.2136601-3-m.tretter@pengutronix.de --- drivers/gpu/drm/rockchip/rockchip_drm_vop2.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index 26f8a8489ded4..105a548d0abeb 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -877,10 +877,14 @@ static void vop2_crtc_atomic_disable(struct drm_crtc *crtc, { struct vop2_video_port *vp = to_vop2_video_port(crtc); struct vop2 *vop2 = vp->vop2; + struct drm_crtc_state *old_crtc_state; int ret; vop2_lock(vop2); + old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc); + drm_atomic_helper_disable_planes_on_crtc(old_crtc_state, false); + drm_crtc_vblank_off(crtc); /* -- GitLab From 88e1f16ba58665e9edfce437ea487da2fa759af9 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Thu, 1 Sep 2022 03:10:07 +0000 Subject: [PATCH 0754/1927] ksefltests: pidfd: Fix wait_states: Test terminated by timeout 0Day/LKP observed that the kselftest blocks forever since one of the pidfd_wait doesn't terminate in 1 of 30 runs. After digging into the source, we found that it blocks at: ASSERT_EQ(sys_waitid(P_PIDFD, pidfd, &info, WCONTINUED, NULL), 0); wait_states has below testing flow: CHILD PARENT ---------------+-------------- 1 STOP itself 2 WAIT for CHILD STOPPED 3 SIGNAL CHILD to CONT 4 CONT 5 STOP itself 5' WAIT for CHILD CONT 6 WAIT for CHILD STOPPED The problem is that the kernel cannot ensure the order of 5 and 5', once 5 goes first, the test will fail. we can reproduce it by: $ while true; do make run_tests -C pidfd; done Introduce a blocking read in child process to make sure the parent can check its WCONTINUED. CC: Philip Li Reported-by: kernel test robot Signed-off-by: Li Zhijian Reviewed-by: Christian Brauner (Microsoft) Signed-off-by: Shuah Khan --- tools/testing/selftests/pidfd/pidfd_wait.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/testing/selftests/pidfd/pidfd_wait.c b/tools/testing/selftests/pidfd/pidfd_wait.c index 070c1c876df15..c3e2a3041f55f 100644 --- a/tools/testing/selftests/pidfd/pidfd_wait.c +++ b/tools/testing/selftests/pidfd/pidfd_wait.c @@ -95,20 +95,28 @@ TEST(wait_states) .flags = CLONE_PIDFD | CLONE_PARENT_SETTID, .exit_signal = SIGCHLD, }; + int pfd[2]; pid_t pid; siginfo_t info = { .si_signo = 0, }; + ASSERT_EQ(pipe(pfd), 0); pid = sys_clone3(&args); ASSERT_GE(pid, 0); if (pid == 0) { + char buf[2]; + + close(pfd[1]); kill(getpid(), SIGSTOP); + ASSERT_EQ(read(pfd[0], buf, 1), 1); + close(pfd[0]); kill(getpid(), SIGSTOP); exit(EXIT_SUCCESS); } + close(pfd[0]); ASSERT_EQ(sys_waitid(P_PIDFD, pidfd, &info, WSTOPPED, NULL), 0); ASSERT_EQ(info.si_signo, SIGCHLD); ASSERT_EQ(info.si_code, CLD_STOPPED); @@ -117,6 +125,8 @@ TEST(wait_states) ASSERT_EQ(sys_pidfd_send_signal(pidfd, SIGCONT, NULL, 0), 0); ASSERT_EQ(sys_waitid(P_PIDFD, pidfd, &info, WCONTINUED, NULL), 0); + ASSERT_EQ(write(pfd[1], "C", 1), 1); + close(pfd[1]); ASSERT_EQ(info.si_signo, SIGCHLD); ASSERT_EQ(info.si_code, CLD_CONTINUED); ASSERT_EQ(info.si_pid, parent_tid); -- GitLab From 3d982441308ebdf713771c8a85c23d9b8b66b4d4 Mon Sep 17 00:00:00 2001 From: Li Zhijian Date: Thu, 1 Sep 2022 03:17:34 +0000 Subject: [PATCH 0755/1927] selftests: pidfd: Fix compling warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix warnings and enable Wall. pidfd_wait.c: In function ‘wait_nonblock’: pidfd_wait.c:150:13: warning: unused variable ‘status’ [-Wunused-variable] 150 | int pidfd, status = 0; | ^~~~~~ ... pidfd_test.c: In function ‘child_poll_exec_test’: pidfd_test.c:438:1: warning: no return statement in function returning non-void [-Wreturn-type] 438 | } | ^ Signed-off-by: Li Zhijian v2: fix mistake assignment to pidfd Signed-off-by: Shuah Khan --- tools/testing/selftests/pidfd/Makefile | 2 +- tools/testing/selftests/pidfd/pidfd_test.c | 2 ++ tools/testing/selftests/pidfd/pidfd_wait.c | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/pidfd/Makefile b/tools/testing/selftests/pidfd/Makefile index f4a2f28f926bb..778b6cdc8aed8 100644 --- a/tools/testing/selftests/pidfd/Makefile +++ b/tools/testing/selftests/pidfd/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -CFLAGS += -g -I../../../../usr/include/ -pthread +CFLAGS += -g -I../../../../usr/include/ -pthread -Wall TEST_GEN_PROGS := pidfd_test pidfd_fdinfo_test pidfd_open_test \ pidfd_poll_test pidfd_wait pidfd_getfd_test pidfd_setns_test diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c index 9a2d64901d591..d36654265b7a5 100644 --- a/tools/testing/selftests/pidfd/pidfd_test.c +++ b/tools/testing/selftests/pidfd/pidfd_test.c @@ -435,6 +435,8 @@ static int child_poll_exec_test(void *args) */ while (1) sleep(1); + + return 0; } static void test_pidfd_poll_exec(int use_waitpid) diff --git a/tools/testing/selftests/pidfd/pidfd_wait.c b/tools/testing/selftests/pidfd/pidfd_wait.c index c3e2a3041f55f..0dcb8365ddc39 100644 --- a/tools/testing/selftests/pidfd/pidfd_wait.c +++ b/tools/testing/selftests/pidfd/pidfd_wait.c @@ -148,7 +148,7 @@ TEST(wait_states) TEST(wait_nonblock) { - int pidfd, status = 0; + int pidfd; unsigned int flags = 0; pid_t parent_tid = -1; struct clone_args args = { -- GitLab From 8e4aae6b8ca76afb1fb64dcb24be44ba814e7f8a Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Thu, 27 Oct 2022 22:03:29 +0800 Subject: [PATCH 0756/1927] nfc: fdp: Fix potential memory leak in fdp_nci_send() fdp_nci_send() will call fdp_nci_i2c_write that will not free skb in the function. As a result, when fdp_nci_i2c_write() finished, the skb will memleak. fdp_nci_send() should free skb after fdp_nci_i2c_write() finished. Fixes: a06347c04c13 ("NFC: Add Intel Fields Peak NFC solution driver") Signed-off-by: Shang XiaoJing Signed-off-by: David S. Miller --- drivers/nfc/fdp/fdp.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/drivers/nfc/fdp/fdp.c b/drivers/nfc/fdp/fdp.c index c6b3334f24c9e..f12f903a9dd13 100644 --- a/drivers/nfc/fdp/fdp.c +++ b/drivers/nfc/fdp/fdp.c @@ -249,11 +249,19 @@ static int fdp_nci_close(struct nci_dev *ndev) static int fdp_nci_send(struct nci_dev *ndev, struct sk_buff *skb) { struct fdp_nci_info *info = nci_get_drvdata(ndev); + int ret; if (atomic_dec_and_test(&info->data_pkt_counter)) info->data_pkt_counter_cb(ndev); - return info->phy_ops->write(info->phy, skb); + ret = info->phy_ops->write(info->phy, skb); + if (ret < 0) { + kfree_skb(skb); + return ret; + } + + consume_skb(skb); + return 0; } static int fdp_nci_request_firmware(struct nci_dev *ndev) -- GitLab From 7bf1ed6aff0f70434bd0cdd45495e83f1dffb551 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Thu, 27 Oct 2022 22:03:30 +0800 Subject: [PATCH 0757/1927] nfc: nxp-nci: Fix potential memory leak in nxp_nci_send() nxp_nci_send() will call nxp_nci_i2c_write(), and only free skb when nxp_nci_i2c_write() failed. However, even if the nxp_nci_i2c_write() run succeeds, the skb will not be freed in nxp_nci_i2c_write(). As the result, the skb will memleak. nxp_nci_send() should also free the skb when nxp_nci_i2c_write() succeeds. Fixes: dece45855a8b ("NFC: nxp-nci: Add support for NXP NCI chips") Signed-off-by: Shang XiaoJing Signed-off-by: David S. Miller --- drivers/nfc/nxp-nci/core.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/nfc/nxp-nci/core.c b/drivers/nfc/nxp-nci/core.c index 7c93d484dc1bc..580cb6ecffee4 100644 --- a/drivers/nfc/nxp-nci/core.c +++ b/drivers/nfc/nxp-nci/core.c @@ -80,10 +80,13 @@ static int nxp_nci_send(struct nci_dev *ndev, struct sk_buff *skb) return -EINVAL; r = info->phy_ops->write(info->phy_id, skb); - if (r < 0) + if (r < 0) { kfree_skb(skb); + return r; + } - return r; + consume_skb(skb); + return 0; } static int nxp_nci_rf_pll_unlocked_ntf(struct nci_dev *ndev, -- GitLab From 3a146b7e3099dc7cf3114f627d9b79291e2d2203 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Thu, 27 Oct 2022 22:03:31 +0800 Subject: [PATCH 0758/1927] nfc: s3fwrn5: Fix potential memory leak in s3fwrn5_nci_send() s3fwrn5_nci_send() will call s3fwrn5_i2c_write() or s3fwrn82_uart_write(), and free the skb if write() failed. However, even if the write() run succeeds, the skb will not be freed in write(). As the result, the skb will memleak. s3fwrn5_nci_send() should also free the skb when write() succeeds. Fixes: c04c674fadeb ("nfc: s3fwrn5: Add driver for Samsung S3FWRN5 NFC Chip") Signed-off-by: Shang XiaoJing Signed-off-by: David S. Miller --- drivers/nfc/s3fwrn5/core.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/nfc/s3fwrn5/core.c b/drivers/nfc/s3fwrn5/core.c index 1c412007fabb6..0270e05b68dff 100644 --- a/drivers/nfc/s3fwrn5/core.c +++ b/drivers/nfc/s3fwrn5/core.c @@ -110,11 +110,15 @@ static int s3fwrn5_nci_send(struct nci_dev *ndev, struct sk_buff *skb) } ret = s3fwrn5_write(info, skb); - if (ret < 0) + if (ret < 0) { kfree_skb(skb); + mutex_unlock(&info->mutex); + return ret; + } + consume_skb(skb); mutex_unlock(&info->mutex); - return ret; + return 0; } static int s3fwrn5_nci_post_setup(struct nci_dev *ndev) -- GitLab From 93d904a734a74c54d945a9884b4962977f1176cd Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Thu, 27 Oct 2022 22:03:32 +0800 Subject: [PATCH 0759/1927] nfc: nfcmrvl: Fix potential memory leak in nfcmrvl_i2c_nci_send() nfcmrvl_i2c_nci_send() will be called by nfcmrvl_nci_send(), and skb should be freed in nfcmrvl_i2c_nci_send(). However, nfcmrvl_nci_send() will only free skb when i2c_master_send() return >=0, which means skb will memleak when i2c_master_send() failed. Free skb no matter whether i2c_master_send() succeeds. Fixes: b5b3e23e4cac ("NFC: nfcmrvl: add i2c driver") Signed-off-by: Shang XiaoJing Signed-off-by: David S. Miller --- drivers/nfc/nfcmrvl/i2c.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/nfc/nfcmrvl/i2c.c b/drivers/nfc/nfcmrvl/i2c.c index acef0cfd76af3..24436c9e54c99 100644 --- a/drivers/nfc/nfcmrvl/i2c.c +++ b/drivers/nfc/nfcmrvl/i2c.c @@ -132,10 +132,15 @@ static int nfcmrvl_i2c_nci_send(struct nfcmrvl_private *priv, ret = -EREMOTEIO; } else ret = 0; + } + + if (ret) { kfree_skb(skb); + return ret; } - return ret; + consume_skb(skb); + return 0; } static void nfcmrvl_i2c_nci_update_config(struct nfcmrvl_private *priv, -- GitLab From 30a0b95b1335e12efef89dd78518ed3e4a71a763 Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 30 Oct 2022 15:19:28 -0700 Subject: [PATCH 0760/1927] Linux 6.1-rc3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d148a55bfd0f5..28026d1ebb9d8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ VERSION = 6 PATCHLEVEL = 1 SUBLEVEL = 0 -EXTRAVERSION = -rc2 +EXTRAVERSION = -rc3 NAME = Hurr durr I'ma ninja sloth # *DOCUMENTATION* -- GitLab From 2153fc9623e5465f503d793d4c94ad65e9ec9b5f Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Thu, 27 Oct 2022 23:56:26 +1100 Subject: [PATCH 0761/1927] powerpc/64e: Fix amdgpu build on Book3E w/o AltiVec There's a build failure for Book3E without AltiVec: Error: cc1: error: AltiVec not supported in this target make[6]: *** [/linux/scripts/Makefile.build:250: drivers/gpu/drm/amd/amdgpu/../display/dc/dml/display_mode_lib.o] Error 1 This happens because the amdgpu build is only gated by PPC_LONG_DOUBLE_128, but that symbol can be enabled even though AltiVec is disabled. The only user of PPC_LONG_DOUBLE_128 is amdgpu, so just add a dependency on AltiVec to that symbol to fix the build. Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221027125626.1383092-1-mpe@ellerman.id.au --- arch/powerpc/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 699df27b0e2fc..20fb1765238c4 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -285,7 +285,7 @@ config PPC # config PPC_LONG_DOUBLE_128 - depends on PPC64 + depends on PPC64 && ALTIVEC def_bool $(success,test "$(shell,echo __LONG_DOUBLE_128__ | $(CC) -E -P -)" = 1) config PPC_BARRIER_NOSPEC -- GitLab From 0873509ea64739a9be02e8ee7b4ff573e503ab8e Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Wed, 26 Oct 2022 19:43:03 +0200 Subject: [PATCH 0762/1927] ARM: dts: lan966x: Enable sgpio on pcb8291 Enable sgpio node on pcb8291 as this is needed to be able to control the LEDs on this board. Otherwise the LEDs support on the board will not be available. On the other board pcb8309 the sgpio is already enabled because it needed to access the SFP ports. Fixes: 0b7baa1a307f ("ARM: dts: lan966x: add led configuration") Signed-off-by: Horatiu Vultur Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20221026174303.702919-1-horatiu.vultur@microchip.com --- arch/arm/boot/dts/lan966x-pcb8291.dts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/arch/arm/boot/dts/lan966x-pcb8291.dts b/arch/arm/boot/dts/lan966x-pcb8291.dts index f4f054cdf2a87..3a3d76af86122 100644 --- a/arch/arm/boot/dts/lan966x-pcb8291.dts +++ b/arch/arm/boot/dts/lan966x-pcb8291.dts @@ -69,6 +69,12 @@ can0_b_pins: can0-b-pins { pins = "GPIO_35", "GPIO_36"; function = "can0_b"; }; + + sgpio_a_pins: sgpio-a-pins { + /* SCK, D0, D1, LD */ + pins = "GPIO_32", "GPIO_33", "GPIO_34", "GPIO_35"; + function = "sgpio_a"; + }; }; &can0 { @@ -118,6 +124,20 @@ &serdes { status = "okay"; }; +&sgpio { + pinctrl-0 = <&sgpio_a_pins>; + pinctrl-names = "default"; + microchip,sgpio-port-ranges = <0 3>, <8 11>; + status = "okay"; + + gpio@0 { + ngpios = <64>; + }; + gpio@1 { + ngpios = <64>; + }; +}; + &switch { status = "okay"; }; -- GitLab From 6fdaed8c79887680bc46cb0a51775bd7c8645528 Mon Sep 17 00:00:00 2001 From: Hector Martin Date: Thu, 27 Oct 2022 22:57:11 +0900 Subject: [PATCH 0763/1927] drm/format-helper: Only advertise supported formats for conversion drm_fb_build_fourcc_list() currently returns all emulated formats unconditionally as long as the native format is among them, even though not all combinations have conversion helpers. Although the list is arguably provided to userspace in precedence order, userspace can pick something out-of-order (and thus break when it shouldn't), or simply only support a format that is unsupported (and thus think it can work, which results in the appearance of a hang as FB blits fail later on, instead of the initialization error you'd expect in this case). Add checks to filter the list of emulated formats to only those supported for conversion to the native format. This presumes that there is a single native format (only the first is checked, if there are multiple). Refactoring this API to drop the native list or support it properly (by returning the appropriate emulated->native mapping table) is left for a future patch. The simpledrm driver is left as-is with a full table of emulated formats. This keeps all currently working conversions available and drops all the broken ones (i.e. this a strict bugfix patch, adding no new supported formats nor removing any actually working ones). In order to avoid proliferation of emulated formats, future drivers should advertise only XRGB8888 as the sole emulated format (since some userspace assumes its presence). This fixes a real user regression where the ?RGB2101010 support commit started advertising it unconditionally where not supported, and KWin decided to start to use it over the native format and broke, but also the fixes the spurious RGB565/RGB888 formats which have been wrongly unconditionally advertised since the dawn of simpledrm. Fixes: 6ea966fca084 ("drm/simpledrm: Add [AX]RGB2101010 formats") Fixes: 11e8f5fd223b ("drm: Add simpledrm driver") Cc: stable@vger.kernel.org Signed-off-by: Hector Martin Acked-by: Pekka Paalanen Reviewed-by: Thomas Zimmermann Signed-off-by: Thomas Zimmermann Link: https://patchwork.freedesktop.org/patch/msgid/20221027135711.24425-1-marcan@marcan.st --- drivers/gpu/drm/drm_format_helper.c | 66 ++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/drm_format_helper.c b/drivers/gpu/drm/drm_format_helper.c index e2f76621453c7..3ee59bae9d2ff 100644 --- a/drivers/gpu/drm/drm_format_helper.c +++ b/drivers/gpu/drm/drm_format_helper.c @@ -807,6 +807,38 @@ static bool is_listed_fourcc(const uint32_t *fourccs, size_t nfourccs, uint32_t return false; } +static const uint32_t conv_from_xrgb8888[] = { + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, + DRM_FORMAT_XRGB2101010, + DRM_FORMAT_ARGB2101010, + DRM_FORMAT_RGB565, + DRM_FORMAT_RGB888, +}; + +static const uint32_t conv_from_rgb565_888[] = { + DRM_FORMAT_XRGB8888, + DRM_FORMAT_ARGB8888, +}; + +static bool is_conversion_supported(uint32_t from, uint32_t to) +{ + switch (from) { + case DRM_FORMAT_XRGB8888: + case DRM_FORMAT_ARGB8888: + return is_listed_fourcc(conv_from_xrgb8888, ARRAY_SIZE(conv_from_xrgb8888), to); + case DRM_FORMAT_RGB565: + case DRM_FORMAT_RGB888: + return is_listed_fourcc(conv_from_rgb565_888, ARRAY_SIZE(conv_from_rgb565_888), to); + case DRM_FORMAT_XRGB2101010: + return to == DRM_FORMAT_ARGB2101010; + case DRM_FORMAT_ARGB2101010: + return to == DRM_FORMAT_XRGB2101010; + default: + return false; + } +} + /** * drm_fb_build_fourcc_list - Filters a list of supported color formats against * the device's native formats @@ -827,7 +859,9 @@ static bool is_listed_fourcc(const uint32_t *fourccs, size_t nfourccs, uint32_t * be handed over to drm_universal_plane_init() et al. Native formats * will go before emulated formats. Other heuristics might be applied * to optimize the order. Formats near the beginning of the list are - * usually preferred over formats near the end of the list. + * usually preferred over formats near the end of the list. Formats + * without conversion helpers will be skipped. New drivers should only + * pass in XRGB8888 and avoid exposing additional emulated formats. * * Returns: * The number of color-formats 4CC codes returned in @fourccs_out. @@ -839,7 +873,7 @@ size_t drm_fb_build_fourcc_list(struct drm_device *dev, { u32 *fourccs = fourccs_out; const u32 *fourccs_end = fourccs_out + nfourccs_out; - bool found_native = false; + uint32_t native_format = 0; size_t i; /* @@ -858,26 +892,18 @@ size_t drm_fb_build_fourcc_list(struct drm_device *dev, drm_dbg_kms(dev, "adding native format %p4cc\n", &fourcc); - if (!found_native) - found_native = is_listed_fourcc(driver_fourccs, driver_nfourccs, fourcc); + /* + * There should only be one native format with the current API. + * This API needs to be refactored to correctly support arbitrary + * sets of native formats, since it needs to report which native + * format to use for each emulated format. + */ + if (!native_format) + native_format = fourcc; *fourccs = fourcc; ++fourccs; } - /* - * The plane's atomic_update helper converts the framebuffer's color format - * to a native format when copying to device memory. - * - * If there is not a single format supported by both, device and - * driver, the native formats are likely not supported by the conversion - * helpers. Therefore *only* support the native formats and add a - * conversion helper ASAP. - */ - if (!found_native) { - drm_warn(dev, "Format conversion helpers required to add extra formats.\n"); - goto out; - } - /* * The extra formats, emulated by the driver, go second. */ @@ -890,6 +916,9 @@ size_t drm_fb_build_fourcc_list(struct drm_device *dev, } else if (fourccs == fourccs_end) { drm_warn(dev, "Ignoring emulated format %p4cc\n", &fourcc); continue; /* end of available output buffer */ + } else if (!is_conversion_supported(fourcc, native_format)) { + drm_dbg_kms(dev, "Unsupported emulated format %p4cc\n", &fourcc); + continue; /* format is not supported for conversion */ } drm_dbg_kms(dev, "adding emulated format %p4cc\n", &fourcc); @@ -898,7 +927,6 @@ size_t drm_fb_build_fourcc_list(struct drm_device *dev, ++fourccs; } -out: return fourccs - fourccs_out; } EXPORT_SYMBOL(drm_fb_build_fourcc_list); -- GitLab From 06a4df5863f73af193a4ff7abf7cb04058584f06 Mon Sep 17 00:00:00 2001 From: Zhang Changzhong Date: Fri, 28 Oct 2022 10:09:11 +0800 Subject: [PATCH 0764/1927] net: fec: fix improper use of NETDEV_TX_BUSY The ndo_start_xmit() method must not free skb when returning NETDEV_TX_BUSY, since caller is going to requeue freed skb. Fix it by returning NETDEV_TX_OK in case of dma_map_single() fails. Fixes: 79f339125ea3 ("net: fec: Add software TSO support") Signed-off-by: Zhang Changzhong Signed-off-by: David S. Miller --- drivers/net/ethernet/freescale/fec_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 28ef4d3c18789..f623c12eaf951 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -713,7 +713,7 @@ fec_enet_txq_put_data_tso(struct fec_enet_priv_tx_q *txq, struct sk_buff *skb, dev_kfree_skb_any(skb); if (net_ratelimit()) netdev_err(ndev, "Tx DMA memory map failed\n"); - return NETDEV_TX_BUSY; + return NETDEV_TX_OK; } bdp->cbd_datlen = cpu_to_fec16(size); @@ -775,7 +775,7 @@ fec_enet_txq_put_hdr_tso(struct fec_enet_priv_tx_q *txq, dev_kfree_skb_any(skb); if (net_ratelimit()) netdev_err(ndev, "Tx DMA memory map failed\n"); - return NETDEV_TX_BUSY; + return NETDEV_TX_OK; } } -- GitLab From 171a93182eccd6e6835d2c86b40787f9f832efaa Mon Sep 17 00:00:00 2001 From: Sergey Shtylyov Date: Sat, 29 Oct 2022 00:07:06 +0300 Subject: [PATCH 0765/1927] ata: pata_legacy: fix pdc20230_set_piomode() Clang gives a warning when compiling pata_legacy.c with 'make W=1' about the 'rt' local variable in pdc20230_set_piomode() being set but unused. Quite obviously, there is an outb() call missing to write back the updated variable. Moreover, checking the docs by Petr Soucek revealed that bitwise AND should have been done with a negated timing mask and the master/slave timing masks were swapped while updating... Fixes: 669a5db411d8 ("[libata] Add a bunch of PATA drivers.") Reported-by: Damien Le Moal Signed-off-by: Sergey Shtylyov Signed-off-by: Damien Le Moal --- drivers/ata/pata_legacy.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/ata/pata_legacy.c b/drivers/ata/pata_legacy.c index 0a8bf09a5c19e..03c580625c2cc 100644 --- a/drivers/ata/pata_legacy.c +++ b/drivers/ata/pata_legacy.c @@ -315,9 +315,10 @@ static void pdc20230_set_piomode(struct ata_port *ap, struct ata_device *adev) outb(inb(0x1F4) & 0x07, 0x1F4); rt = inb(0x1F3); - rt &= 0x07 << (3 * adev->devno); + rt &= ~(0x07 << (3 * !adev->devno)); if (pio) - rt |= (1 + 3 * pio) << (3 * adev->devno); + rt |= (1 + 3 * pio) << (3 * !adev->devno); + outb(rt, 0x1F3); udelay(100); outb(inb(0x1F2) | 0x01, 0x1F2); -- GitLab From 015618c3ec19584c83ff179fa631be8cec906aaf Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sat, 29 Oct 2022 15:49:31 +0800 Subject: [PATCH 0766/1927] ata: palmld: fix return value check in palmld_pata_probe() If devm_platform_ioremap_resource() fails, it never return NULL pointer, replace the check with IS_ERR(). Fixes: 57bf0f5a162d ("ARM: pxa: use pdev resource for palmld mmio") Signed-off-by: Yang Yingliang Reviewed-by: Sergey Shtylyov Signed-off-by: Damien Le Moal --- drivers/ata/pata_palmld.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/ata/pata_palmld.c b/drivers/ata/pata_palmld.c index 400e65190904f..51caa2a427dd8 100644 --- a/drivers/ata/pata_palmld.c +++ b/drivers/ata/pata_palmld.c @@ -63,8 +63,8 @@ static int palmld_pata_probe(struct platform_device *pdev) /* remap drive's physical memory address */ mem = devm_platform_ioremap_resource(pdev, 0); - if (!mem) - return -ENOMEM; + if (IS_ERR(mem)) + return PTR_ERR(mem); /* request and activate power and reset GPIOs */ lda->power = devm_gpiod_get(dev, "power", GPIOD_OUT_HIGH); -- GitLab From 986d93f55bdeab1cac858d1e47b41fac10b2d7f6 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Mon, 31 Oct 2022 10:10:21 +0800 Subject: [PATCH 0767/1927] audit: fix undefined behavior in bit shift for AUDIT_BIT Shifting signed 32-bit value by 31 bits is undefined, so changing significant bit to unsigned. The UBSAN warning calltrace like below: UBSAN: shift-out-of-bounds in kernel/auditfilter.c:179:23 left shift of 1 by 31 places cannot be represented in type 'int' Call Trace: dump_stack_lvl+0x7d/0xa5 dump_stack+0x15/0x1b ubsan_epilogue+0xe/0x4e __ubsan_handle_shift_out_of_bounds+0x1e7/0x20c audit_register_class+0x9d/0x137 audit_classes_init+0x4d/0xb8 do_one_initcall+0x76/0x430 kernel_init_freeable+0x3b3/0x422 kernel_init+0x24/0x1e0 ret_from_fork+0x1f/0x30 Signed-off-by: Gaosheng Cui [PM: remove bad 'Fixes' tag as issue predates git, added in v2.6.6-rc1] Signed-off-by: Paul Moore --- include/uapi/linux/audit.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/audit.h b/include/uapi/linux/audit.h index 7c1dc818b1d56..d676ed2b246ec 100644 --- a/include/uapi/linux/audit.h +++ b/include/uapi/linux/audit.h @@ -187,7 +187,7 @@ #define AUDIT_MAX_KEY_LEN 256 #define AUDIT_BITMASK_SIZE 64 #define AUDIT_WORD(nr) ((__u32)((nr)/32)) -#define AUDIT_BIT(nr) (1 << ((nr) - AUDIT_WORD(nr)*32)) +#define AUDIT_BIT(nr) (1U << ((nr) - AUDIT_WORD(nr)*32)) #define AUDIT_SYSCALL_CLASSES 16 #define AUDIT_CLASS_DIR_WRITE 0 -- GitLab From 8bdc2acd420c6f3dd1f1c78750ec989f02a1e2b9 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 28 Oct 2022 18:05:00 +0300 Subject: [PATCH 0768/1927] net: sched: Fix use after free in red_enqueue() We can't use "skb" again after passing it to qdisc_enqueue(). This is basically identical to commit 2f09707d0c97 ("sch_sfb: Also store skb len before calling child enqueue"). Fixes: d7f4f332f082 ("sch_red: update backlog as well") Signed-off-by: Dan Carpenter Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- net/sched/sch_red.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index a5a401f93c1a2..98129324e1573 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -72,6 +72,7 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct red_sched_data *q = qdisc_priv(sch); struct Qdisc *child = q->qdisc; + unsigned int len; int ret; q->vars.qavg = red_calc_qavg(&q->parms, @@ -126,9 +127,10 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch, break; } + len = qdisc_pkt_len(skb); ret = qdisc_enqueue(skb, child, to_free); if (likely(ret == NET_XMIT_SUCCESS)) { - qdisc_qstats_backlog_inc(sch, skb); + sch->qstats.backlog += len; sch->q.qlen++; } else if (net_xmit_drop_count(ret)) { q->stats.pdrop++; -- GitLab From d7164a5048e8a6afe2cc4aaf7f12643c14e7f241 Mon Sep 17 00:00:00 2001 From: Imre Deak Date: Tue, 25 Oct 2022 14:44:55 +0300 Subject: [PATCH 0769/1927] drm/i915/tgl+: Add locking around DKL PHY register accesses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the TypeC DKL PHY registers during modeset-commit, -verification, DP link-retraining and AUX power well toggling is racy due to these code paths being concurrent and the PHY register bank selection register (HIP_INDEX_REG) being shared between PHY instances (aka TC ports) and the bank selection being not atomic wrt. the actual PHY register access. Add the required locking around each PHY register bank selection-> register access sequence. Kudos to Ville for noticing the race conditions. v2: - Add the DKL PHY register accessors to intel_dkl_phy.[ch]. (Jani) - Make the DKL_REG_TC_PORT macro independent of PHY internals. - Move initing the DKL PHY lock to a more logical place. v3: - Fix parameter reuse in the DKL_REG_TC_PORT definition. - Document the usage of phy_lock. v4: - Fix adding TC_PORT_1 offset in the DKL_REG_TC_PORT definition. Cc: Ville Syrjälä Cc: Jani Nikula Cc: # v5.5+ Acked-by: Jani Nikula Signed-off-by: Imre Deak Link: https://patchwork.freedesktop.org/patch/msgid/20221025114457.2191004-1-imre.deak@intel.com (cherry picked from commit 89cb0ba4ceee6bed1059904859c5723b3f39da68) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/display/intel_ddi.c | 68 +++++------ .../gpu/drm/i915/display/intel_display_core.h | 8 ++ .../i915/display/intel_display_power_well.c | 7 +- drivers/gpu/drm/i915/display/intel_dkl_phy.c | 109 ++++++++++++++++++ drivers/gpu/drm/i915/display/intel_dkl_phy.h | 24 ++++ drivers/gpu/drm/i915/display/intel_dpll_mgr.c | 59 +++++----- drivers/gpu/drm/i915/i915_driver.c | 1 + drivers/gpu/drm/i915/i915_reg.h | 3 + 9 files changed, 204 insertions(+), 76 deletions(-) create mode 100644 drivers/gpu/drm/i915/display/intel_dkl_phy.c create mode 100644 drivers/gpu/drm/i915/display/intel_dkl_phy.h diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index a26edcdadc217..cea00aaca04b6 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -282,6 +282,7 @@ i915-y += \ display/intel_ddi.o \ display/intel_ddi_buf_trans.o \ display/intel_display_trace.o \ + display/intel_dkl_phy.o \ display/intel_dp.o \ display/intel_dp_aux.o \ display/intel_dp_aux_backlight.o \ diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index da8472cdc1357..69ecf2a3d6c65 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -43,6 +43,7 @@ #include "intel_de.h" #include "intel_display_power.h" #include "intel_display_types.h" +#include "intel_dkl_phy.h" #include "intel_dp.h" #include "intel_dp_link_training.h" #include "intel_dp_mst.h" @@ -1262,33 +1263,30 @@ static void tgl_dkl_phy_set_signal_levels(struct intel_encoder *encoder, for (ln = 0; ln < 2; ln++) { int level; - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, ln)); - - intel_de_write(dev_priv, DKL_TX_PMD_LANE_SUS(tc_port), 0); + intel_dkl_phy_write(dev_priv, DKL_TX_PMD_LANE_SUS(tc_port), ln, 0); level = intel_ddi_level(encoder, crtc_state, 2*ln+0); - intel_de_rmw(dev_priv, DKL_TX_DPCNTL0(tc_port), - DKL_TX_PRESHOOT_COEFF_MASK | - DKL_TX_DE_EMPAHSIS_COEFF_MASK | - DKL_TX_VSWING_CONTROL_MASK, - DKL_TX_PRESHOOT_COEFF(trans->entries[level].dkl.preshoot) | - DKL_TX_DE_EMPHASIS_COEFF(trans->entries[level].dkl.de_emphasis) | - DKL_TX_VSWING_CONTROL(trans->entries[level].dkl.vswing)); + intel_dkl_phy_rmw(dev_priv, DKL_TX_DPCNTL0(tc_port), ln, + DKL_TX_PRESHOOT_COEFF_MASK | + DKL_TX_DE_EMPAHSIS_COEFF_MASK | + DKL_TX_VSWING_CONTROL_MASK, + DKL_TX_PRESHOOT_COEFF(trans->entries[level].dkl.preshoot) | + DKL_TX_DE_EMPHASIS_COEFF(trans->entries[level].dkl.de_emphasis) | + DKL_TX_VSWING_CONTROL(trans->entries[level].dkl.vswing)); level = intel_ddi_level(encoder, crtc_state, 2*ln+1); - intel_de_rmw(dev_priv, DKL_TX_DPCNTL1(tc_port), - DKL_TX_PRESHOOT_COEFF_MASK | - DKL_TX_DE_EMPAHSIS_COEFF_MASK | - DKL_TX_VSWING_CONTROL_MASK, - DKL_TX_PRESHOOT_COEFF(trans->entries[level].dkl.preshoot) | - DKL_TX_DE_EMPHASIS_COEFF(trans->entries[level].dkl.de_emphasis) | - DKL_TX_VSWING_CONTROL(trans->entries[level].dkl.vswing)); + intel_dkl_phy_rmw(dev_priv, DKL_TX_DPCNTL1(tc_port), ln, + DKL_TX_PRESHOOT_COEFF_MASK | + DKL_TX_DE_EMPAHSIS_COEFF_MASK | + DKL_TX_VSWING_CONTROL_MASK, + DKL_TX_PRESHOOT_COEFF(trans->entries[level].dkl.preshoot) | + DKL_TX_DE_EMPHASIS_COEFF(trans->entries[level].dkl.de_emphasis) | + DKL_TX_VSWING_CONTROL(trans->entries[level].dkl.vswing)); - intel_de_rmw(dev_priv, DKL_TX_DPCNTL2(tc_port), - DKL_TX_DP20BITMODE, 0); + intel_dkl_phy_rmw(dev_priv, DKL_TX_DPCNTL2(tc_port), ln, + DKL_TX_DP20BITMODE, 0); if (IS_ALDERLAKE_P(dev_priv)) { u32 val; @@ -1306,10 +1304,10 @@ static void tgl_dkl_phy_set_signal_levels(struct intel_encoder *encoder, val |= DKL_TX_DPCNTL2_CFG_LOADGENSELECT_TX2(0); } - intel_de_rmw(dev_priv, DKL_TX_DPCNTL2(tc_port), - DKL_TX_DPCNTL2_CFG_LOADGENSELECT_TX1_MASK | - DKL_TX_DPCNTL2_CFG_LOADGENSELECT_TX2_MASK, - val); + intel_dkl_phy_rmw(dev_priv, DKL_TX_DPCNTL2(tc_port), ln, + DKL_TX_DPCNTL2_CFG_LOADGENSELECT_TX1_MASK | + DKL_TX_DPCNTL2_CFG_LOADGENSELECT_TX2_MASK, + val); } } } @@ -2019,12 +2017,8 @@ icl_program_mg_dp_mode(struct intel_digital_port *dig_port, return; if (DISPLAY_VER(dev_priv) >= 12) { - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x0)); - ln0 = intel_de_read(dev_priv, DKL_DP_MODE(tc_port)); - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x1)); - ln1 = intel_de_read(dev_priv, DKL_DP_MODE(tc_port)); + ln0 = intel_dkl_phy_read(dev_priv, DKL_DP_MODE(tc_port), 0); + ln1 = intel_dkl_phy_read(dev_priv, DKL_DP_MODE(tc_port), 1); } else { ln0 = intel_de_read(dev_priv, MG_DP_MODE(0, tc_port)); ln1 = intel_de_read(dev_priv, MG_DP_MODE(1, tc_port)); @@ -2085,12 +2079,8 @@ icl_program_mg_dp_mode(struct intel_digital_port *dig_port, } if (DISPLAY_VER(dev_priv) >= 12) { - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x0)); - intel_de_write(dev_priv, DKL_DP_MODE(tc_port), ln0); - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x1)); - intel_de_write(dev_priv, DKL_DP_MODE(tc_port), ln1); + intel_dkl_phy_write(dev_priv, DKL_DP_MODE(tc_port), 0, ln0); + intel_dkl_phy_write(dev_priv, DKL_DP_MODE(tc_port), 1, ln1); } else { intel_de_write(dev_priv, MG_DP_MODE(0, tc_port), ln0); intel_de_write(dev_priv, MG_DP_MODE(1, tc_port), ln1); @@ -3094,10 +3084,8 @@ static void adlp_tbt_to_dp_alt_switch_wa(struct intel_encoder *encoder) enum tc_port tc_port = intel_port_to_tc(i915, encoder->port); int ln; - for (ln = 0; ln < 2; ln++) { - intel_de_write(i915, HIP_INDEX_REG(tc_port), HIP_INDEX_VAL(tc_port, ln)); - intel_de_rmw(i915, DKL_PCS_DW5(tc_port), DKL_PCS_DW5_CORE_SOFTRESET, 0); - } + for (ln = 0; ln < 2; ln++) + intel_dkl_phy_rmw(i915, DKL_PCS_DW5(tc_port), ln, DKL_PCS_DW5_CORE_SOFTRESET, 0); } static void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp, diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h index 96cf994b0ad1f..9b51148e8ba56 100644 --- a/drivers/gpu/drm/i915/display/intel_display_core.h +++ b/drivers/gpu/drm/i915/display/intel_display_core.h @@ -315,6 +315,14 @@ struct intel_display { struct intel_global_obj obj; } dbuf; + struct { + /* + * dkl.phy_lock protects against concurrent access of the + * Dekel TypeC PHYs. + */ + spinlock_t phy_lock; + } dkl; + struct { /* VLV/CHV/BXT/GLK DSI MMIO register base address */ u32 mmio_base; diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c index df7ee4969ef17..1d18eee562534 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_well.c +++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c @@ -12,6 +12,7 @@ #include "intel_de.h" #include "intel_display_power_well.h" #include "intel_display_types.h" +#include "intel_dkl_phy.h" #include "intel_dmc.h" #include "intel_dpio_phy.h" #include "intel_dpll.h" @@ -529,11 +530,9 @@ icl_tc_phy_aux_power_well_enable(struct drm_i915_private *dev_priv, enum tc_port tc_port; tc_port = TGL_AUX_PW_TO_TC_PORT(i915_power_well_instance(power_well)->hsw.idx); - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x2)); - if (intel_de_wait_for_set(dev_priv, DKL_CMN_UC_DW_27(tc_port), - DKL_CMN_UC_DW27_UC_HEALTH, 1)) + if (wait_for(intel_dkl_phy_read(dev_priv, DKL_CMN_UC_DW_27(tc_port), 2) & + DKL_CMN_UC_DW27_UC_HEALTH, 1)) drm_warn(&dev_priv->drm, "Timeout waiting TC uC health\n"); } diff --git a/drivers/gpu/drm/i915/display/intel_dkl_phy.c b/drivers/gpu/drm/i915/display/intel_dkl_phy.c new file mode 100644 index 0000000000000..710b030c7ed54 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_dkl_phy.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "i915_drv.h" +#include "i915_reg.h" + +#include "intel_de.h" +#include "intel_display.h" +#include "intel_dkl_phy.h" + +static void +dkl_phy_set_hip_idx(struct drm_i915_private *i915, i915_reg_t reg, int idx) +{ + enum tc_port tc_port = DKL_REG_TC_PORT(reg); + + drm_WARN_ON(&i915->drm, tc_port < TC_PORT_1 || tc_port >= I915_MAX_TC_PORTS); + + intel_de_write(i915, + HIP_INDEX_REG(tc_port), + HIP_INDEX_VAL(tc_port, idx)); +} + +/** + * intel_dkl_phy_read - read a Dekel PHY register + * @i915: i915 device instance + * @reg: Dekel PHY register + * @ln: lane instance of @reg + * + * Read the @reg Dekel PHY register. + * + * Returns the read value. + */ +u32 +intel_dkl_phy_read(struct drm_i915_private *i915, i915_reg_t reg, int ln) +{ + u32 val; + + spin_lock(&i915->display.dkl.phy_lock); + + dkl_phy_set_hip_idx(i915, reg, ln); + val = intel_de_read(i915, reg); + + spin_unlock(&i915->display.dkl.phy_lock); + + return val; +} + +/** + * intel_dkl_phy_write - write a Dekel PHY register + * @i915: i915 device instance + * @reg: Dekel PHY register + * @ln: lane instance of @reg + * @val: value to write + * + * Write @val to the @reg Dekel PHY register. + */ +void +intel_dkl_phy_write(struct drm_i915_private *i915, i915_reg_t reg, int ln, u32 val) +{ + spin_lock(&i915->display.dkl.phy_lock); + + dkl_phy_set_hip_idx(i915, reg, ln); + intel_de_write(i915, reg, val); + + spin_unlock(&i915->display.dkl.phy_lock); +} + +/** + * intel_dkl_phy_rmw - read-modify-write a Dekel PHY register + * @i915: i915 device instance + * @reg: Dekel PHY register + * @ln: lane instance of @reg + * @clear: mask to clear + * @set: mask to set + * + * Read the @reg Dekel PHY register, clearing then setting the @clear/@set bits in it, and writing + * this value back to the register if the value differs from the read one. + */ +void +intel_dkl_phy_rmw(struct drm_i915_private *i915, i915_reg_t reg, int ln, u32 clear, u32 set) +{ + spin_lock(&i915->display.dkl.phy_lock); + + dkl_phy_set_hip_idx(i915, reg, ln); + intel_de_rmw(i915, reg, clear, set); + + spin_unlock(&i915->display.dkl.phy_lock); +} + +/** + * intel_dkl_phy_posting_read - do a posting read from a Dekel PHY register + * @i915: i915 device instance + * @reg: Dekel PHY register + * @ln: lane instance of @reg + * + * Read the @reg Dekel PHY register without returning the read value. + */ +void +intel_dkl_phy_posting_read(struct drm_i915_private *i915, i915_reg_t reg, int ln) +{ + spin_lock(&i915->display.dkl.phy_lock); + + dkl_phy_set_hip_idx(i915, reg, ln); + intel_de_posting_read(i915, reg); + + spin_unlock(&i915->display.dkl.phy_lock); +} diff --git a/drivers/gpu/drm/i915/display/intel_dkl_phy.h b/drivers/gpu/drm/i915/display/intel_dkl_phy.h new file mode 100644 index 0000000000000..260ad121a0b18 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_dkl_phy.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_DKL_PHY_H__ +#define __INTEL_DKL_PHY_H__ + +#include + +#include "i915_reg_defs.h" + +struct drm_i915_private; + +u32 +intel_dkl_phy_read(struct drm_i915_private *i915, i915_reg_t reg, int ln); +void +intel_dkl_phy_write(struct drm_i915_private *i915, i915_reg_t reg, int ln, u32 val); +void +intel_dkl_phy_rmw(struct drm_i915_private *i915, i915_reg_t reg, int ln, u32 clear, u32 set); +void +intel_dkl_phy_posting_read(struct drm_i915_private *i915, i915_reg_t reg, int ln); + +#endif /* __INTEL_DKL_PHY_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index e5fb66a5dd025..64dd603dc69aa 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -25,6 +25,7 @@ #include "intel_de.h" #include "intel_display_types.h" +#include "intel_dkl_phy.h" #include "intel_dpio_phy.h" #include "intel_dpll.h" #include "intel_dpll_mgr.h" @@ -3508,15 +3509,12 @@ static bool dkl_pll_get_hw_state(struct drm_i915_private *dev_priv, * All registers read here have the same HIP_INDEX_REG even though * they are on different building blocks */ - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x2)); - - hw_state->mg_refclkin_ctl = intel_de_read(dev_priv, - DKL_REFCLKIN_CTL(tc_port)); + hw_state->mg_refclkin_ctl = intel_dkl_phy_read(dev_priv, + DKL_REFCLKIN_CTL(tc_port), 2); hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK; hw_state->mg_clktop2_hsclkctl = - intel_de_read(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port)); + intel_dkl_phy_read(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), 2); hw_state->mg_clktop2_hsclkctl &= MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK | MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK | @@ -3524,32 +3522,32 @@ static bool dkl_pll_get_hw_state(struct drm_i915_private *dev_priv, MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK; hw_state->mg_clktop2_coreclkctl1 = - intel_de_read(dev_priv, DKL_CLKTOP2_CORECLKCTL1(tc_port)); + intel_dkl_phy_read(dev_priv, DKL_CLKTOP2_CORECLKCTL1(tc_port), 2); hw_state->mg_clktop2_coreclkctl1 &= MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; - hw_state->mg_pll_div0 = intel_de_read(dev_priv, DKL_PLL_DIV0(tc_port)); + hw_state->mg_pll_div0 = intel_dkl_phy_read(dev_priv, DKL_PLL_DIV0(tc_port), 2); val = DKL_PLL_DIV0_MASK; if (dev_priv->display.vbt.override_afc_startup) val |= DKL_PLL_DIV0_AFC_STARTUP_MASK; hw_state->mg_pll_div0 &= val; - hw_state->mg_pll_div1 = intel_de_read(dev_priv, DKL_PLL_DIV1(tc_port)); + hw_state->mg_pll_div1 = intel_dkl_phy_read(dev_priv, DKL_PLL_DIV1(tc_port), 2); hw_state->mg_pll_div1 &= (DKL_PLL_DIV1_IREF_TRIM_MASK | DKL_PLL_DIV1_TDC_TARGET_CNT_MASK); - hw_state->mg_pll_ssc = intel_de_read(dev_priv, DKL_PLL_SSC(tc_port)); + hw_state->mg_pll_ssc = intel_dkl_phy_read(dev_priv, DKL_PLL_SSC(tc_port), 2); hw_state->mg_pll_ssc &= (DKL_PLL_SSC_IREF_NDIV_RATIO_MASK | DKL_PLL_SSC_STEP_LEN_MASK | DKL_PLL_SSC_STEP_NUM_MASK | DKL_PLL_SSC_EN); - hw_state->mg_pll_bias = intel_de_read(dev_priv, DKL_PLL_BIAS(tc_port)); + hw_state->mg_pll_bias = intel_dkl_phy_read(dev_priv, DKL_PLL_BIAS(tc_port), 2); hw_state->mg_pll_bias &= (DKL_PLL_BIAS_FRAC_EN_H | DKL_PLL_BIAS_FBDIV_FRAC_MASK); hw_state->mg_pll_tdc_coldst_bias = - intel_de_read(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port)); + intel_dkl_phy_read(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port), 2); hw_state->mg_pll_tdc_coldst_bias &= (DKL_PLL_TDC_SSC_STEP_SIZE_MASK | DKL_PLL_TDC_FEED_FWD_GAIN_MASK); @@ -3737,61 +3735,58 @@ static void dkl_pll_write(struct drm_i915_private *dev_priv, * All registers programmed here have the same HIP_INDEX_REG even * though on different building block */ - intel_de_write(dev_priv, HIP_INDEX_REG(tc_port), - HIP_INDEX_VAL(tc_port, 0x2)); - /* All the registers are RMW */ - val = intel_de_read(dev_priv, DKL_REFCLKIN_CTL(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_REFCLKIN_CTL(tc_port), 2); val &= ~MG_REFCLKIN_CTL_OD_2_MUX_MASK; val |= hw_state->mg_refclkin_ctl; - intel_de_write(dev_priv, DKL_REFCLKIN_CTL(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_REFCLKIN_CTL(tc_port), 2, val); - val = intel_de_read(dev_priv, DKL_CLKTOP2_CORECLKCTL1(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_CLKTOP2_CORECLKCTL1(tc_port), 2); val &= ~MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK; val |= hw_state->mg_clktop2_coreclkctl1; - intel_de_write(dev_priv, DKL_CLKTOP2_CORECLKCTL1(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_CLKTOP2_CORECLKCTL1(tc_port), 2, val); - val = intel_de_read(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), 2); val &= ~(MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK | MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK | MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK | MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK); val |= hw_state->mg_clktop2_hsclkctl; - intel_de_write(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), 2, val); val = DKL_PLL_DIV0_MASK; if (dev_priv->display.vbt.override_afc_startup) val |= DKL_PLL_DIV0_AFC_STARTUP_MASK; - intel_de_rmw(dev_priv, DKL_PLL_DIV0(tc_port), val, - hw_state->mg_pll_div0); + intel_dkl_phy_rmw(dev_priv, DKL_PLL_DIV0(tc_port), 2, val, + hw_state->mg_pll_div0); - val = intel_de_read(dev_priv, DKL_PLL_DIV1(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_PLL_DIV1(tc_port), 2); val &= ~(DKL_PLL_DIV1_IREF_TRIM_MASK | DKL_PLL_DIV1_TDC_TARGET_CNT_MASK); val |= hw_state->mg_pll_div1; - intel_de_write(dev_priv, DKL_PLL_DIV1(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_PLL_DIV1(tc_port), 2, val); - val = intel_de_read(dev_priv, DKL_PLL_SSC(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_PLL_SSC(tc_port), 2); val &= ~(DKL_PLL_SSC_IREF_NDIV_RATIO_MASK | DKL_PLL_SSC_STEP_LEN_MASK | DKL_PLL_SSC_STEP_NUM_MASK | DKL_PLL_SSC_EN); val |= hw_state->mg_pll_ssc; - intel_de_write(dev_priv, DKL_PLL_SSC(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_PLL_SSC(tc_port), 2, val); - val = intel_de_read(dev_priv, DKL_PLL_BIAS(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_PLL_BIAS(tc_port), 2); val &= ~(DKL_PLL_BIAS_FRAC_EN_H | DKL_PLL_BIAS_FBDIV_FRAC_MASK); val |= hw_state->mg_pll_bias; - intel_de_write(dev_priv, DKL_PLL_BIAS(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_PLL_BIAS(tc_port), 2, val); - val = intel_de_read(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port)); + val = intel_dkl_phy_read(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port), 2); val &= ~(DKL_PLL_TDC_SSC_STEP_SIZE_MASK | DKL_PLL_TDC_FEED_FWD_GAIN_MASK); val |= hw_state->mg_pll_tdc_coldst_bias; - intel_de_write(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port), val); + intel_dkl_phy_write(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port), 2, val); - intel_de_posting_read(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port)); + intel_dkl_phy_posting_read(dev_priv, DKL_PLL_TDC_COLDST_BIAS(tc_port), 2); } static void icl_pll_power_enable(struct drm_i915_private *dev_priv, diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c index c459eb362c47f..f2a15d8155f4a 100644 --- a/drivers/gpu/drm/i915/i915_driver.c +++ b/drivers/gpu/drm/i915/i915_driver.c @@ -353,6 +353,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) mutex_init(&dev_priv->display.wm.wm_mutex); mutex_init(&dev_priv->display.pps.mutex); mutex_init(&dev_priv->display.hdcp.comp_mutex); + spin_lock_init(&dev_priv->display.dkl.phy_lock); i915_memcpy_init_early(dev_priv); intel_runtime_pm_init_early(&dev_priv->runtime_pm); diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 0b287a59dc2f4..da35bb2db26b6 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -7420,6 +7420,9 @@ enum skl_power_gate { #define _DKL_PHY5_BASE 0x16C000 #define _DKL_PHY6_BASE 0x16D000 +#define DKL_REG_TC_PORT(__reg) \ + (TC_PORT_1 + ((__reg).reg - _DKL_PHY1_BASE) / (_DKL_PHY2_BASE - _DKL_PHY1_BASE)) + /* DEKEL PHY MMIO Address = Phy base + (internal address & ~index_mask) */ #define _DKL_PCS_DW5 0x14 #define DKL_PCS_DW5(tc_port) _MMIO(_PORT(tc_port, _DKL_PHY1_BASE, \ -- GitLab From d3f6bacfca86f6cf6bf85be1e8b54083d68d8195 Mon Sep 17 00:00:00 2001 From: Robert Beckett Date: Thu, 20 Oct 2022 13:03:08 +0200 Subject: [PATCH 0770/1927] drm/i915: stop abusing swiotlb_max_segment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit swiotlb_max_segment used to return either the maximum size that swiotlb could bounce, or for Xen PV PAGE_SIZE even if swiotlb could bounce buffer larger mappings. This made i915 on Xen PV work as it bypasses the coherency aspect of the DMA API and can't cope with bounce buffering and this avoided bounce buffering for the Xen/PV case. So instead of adding this hack back, check for Xen/PV directly in i915 for the Xen case and otherwise use the proper DMA API helper to query the maximum mapping size. Replace swiotlb_max_segment() calls with dma_max_mapping_size(). In i915_gem_object_get_pages_internal() no longer consider max_segment only if CONFIG_SWIOTLB is enabled. There can be other (iommu related) causes of specific max segment sizes. Fixes: a2daa27c0c61 ("swiotlb: simplify swiotlb_max_segment") Reported-by: Marek Marczykowski-Górecki Signed-off-by: Robert Beckett Signed-off-by: Christoph Hellwig [hch: added the Xen hack, rewrote the changelog] Reviewed-by: Tvrtko Ursulin Signed-off-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20221020110308.1582518-1-hch@lst.de (cherry picked from commit 78a07fe777c42800bd1adaec12abe5dcee43919e) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/gem/i915_gem_internal.c | 19 +++-------- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 4 +-- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 2 +- drivers/gpu/drm/i915/i915_scatterlist.h | 34 ++++++++++++-------- 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index c698f95af15fe..629acb403a2c9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -6,7 +6,6 @@ #include #include -#include #include "i915_drv.h" #include "i915_gem.h" @@ -38,22 +37,12 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) struct scatterlist *sg; unsigned int sg_page_sizes; unsigned int npages; - int max_order; + int max_order = MAX_ORDER; + unsigned int max_segment; gfp_t gfp; - max_order = MAX_ORDER; -#ifdef CONFIG_SWIOTLB - if (is_swiotlb_active(obj->base.dev->dev)) { - unsigned int max_segment; - - max_segment = swiotlb_max_segment(); - if (max_segment) { - max_segment = max_t(unsigned int, max_segment, - PAGE_SIZE) >> PAGE_SHIFT; - max_order = min(max_order, ilog2(max_segment)); - } - } -#endif + max_segment = i915_sg_segment_size(i915->drm.dev) >> PAGE_SHIFT; + max_order = min(max_order, get_order(max_segment)); gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; if (IS_I965GM(i915) || IS_I965G(i915)) { diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index f42ca1179f373..11125c32dd35d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -194,7 +194,7 @@ static int shmem_get_pages(struct drm_i915_gem_object *obj) struct intel_memory_region *mem = obj->mm.region; struct address_space *mapping = obj->base.filp->f_mapping; const unsigned long page_count = obj->base.size / PAGE_SIZE; - unsigned int max_segment = i915_sg_segment_size(); + unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); struct sg_table *st; struct sgt_iter sgt_iter; struct page *page; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 4f861782c3e85..a4aa9500fa179 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -189,7 +189,7 @@ static int i915_ttm_tt_shmem_populate(struct ttm_device *bdev, struct drm_i915_private *i915 = container_of(bdev, typeof(*i915), bdev); struct intel_memory_region *mr = i915->mm.regions[INTEL_MEMORY_SYSTEM]; struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm); - const unsigned int max_segment = i915_sg_segment_size(); + const unsigned int max_segment = i915_sg_segment_size(i915->drm.dev); const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT; struct file *filp = i915_tt->filp; struct sgt_iter sgt_iter; @@ -538,7 +538,7 @@ static struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm) ret = sg_alloc_table_from_pages_segment(st, ttm->pages, ttm->num_pages, 0, (unsigned long)ttm->num_pages << PAGE_SHIFT, - i915_sg_segment_size(), GFP_KERNEL); + i915_sg_segment_size(i915_tt->dev), GFP_KERNEL); if (ret) { st->sgl = NULL; return ERR_PTR(ret); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index d4398948f0162..f34e01a7fefb9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -129,7 +129,7 @@ static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj) static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) { const unsigned long num_pages = obj->base.size >> PAGE_SHIFT; - unsigned int max_segment = i915_sg_segment_size(); + unsigned int max_segment = i915_sg_segment_size(obj->base.dev->dev); struct sg_table *st; unsigned int sg_page_sizes; struct page **pvec; diff --git a/drivers/gpu/drm/i915/i915_scatterlist.h b/drivers/gpu/drm/i915/i915_scatterlist.h index 9ddb3e743a3e5..b0a1db44f8950 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.h +++ b/drivers/gpu/drm/i915/i915_scatterlist.h @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include "i915_gem.h" @@ -127,19 +128,26 @@ static inline unsigned int i915_sg_dma_sizes(struct scatterlist *sg) return page_sizes; } -static inline unsigned int i915_sg_segment_size(void) +static inline unsigned int i915_sg_segment_size(struct device *dev) { - unsigned int size = swiotlb_max_segment(); - - if (size == 0) - size = UINT_MAX; - - size = rounddown(size, PAGE_SIZE); - /* swiotlb_max_segment_size can return 1 byte when it means one page. */ - if (size < PAGE_SIZE) - size = PAGE_SIZE; - - return size; + size_t max = min_t(size_t, UINT_MAX, dma_max_mapping_size(dev)); + + /* + * For Xen PV guests pages aren't contiguous in DMA (machine) address + * space. The DMA API takes care of that both in dma_alloc_* (by + * calling into the hypervisor to make the pages contiguous) and in + * dma_map_* (by bounce buffering). But i915 abuses ignores the + * coherency aspects of the DMA API and thus can't cope with bounce + * buffering actually happening, so add a hack here to force small + * allocations and mappings when running in PV mode on Xen. + * + * Note this will still break if bounce buffering is required for other + * reasons, like confidential computing hypervisors or PCIe root ports + * with addressing limitations. + */ + if (xen_pv_domain()) + max = PAGE_SIZE; + return round_down(max, PAGE_SIZE); } bool i915_sg_trim(struct sg_table *orig_st); -- GitLab From 3e206b6aa6df7eed4297577e0cf8403169b800a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 13:11:27 +0300 Subject: [PATCH 0771/1927] drm/i915/sdvo: Filter out invalid outputs more sensibly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We try to filter out the corresponding xxx1 output if the xxx0 output is not present. But the way that is being done is pretty awkward. Make it less so. Cc: stable@vger.kernel.org Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026101134.20865-2-ville.syrjala@linux.intel.com Reviewed-by: Jani Nikula (cherry picked from commit cc1e66394daaa7e9f005e2487a84e34a39f9308b) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/display/intel_sdvo.c | 27 ++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index f5b744bef18ff..30d3778c61366 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -2926,16 +2926,33 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) return false; } +static u16 intel_sdvo_filter_output_flags(u16 flags) +{ + flags &= SDVO_OUTPUT_MASK; + + /* SDVO requires XXX1 function may not exist unless it has XXX0 function.*/ + if (!(flags & SDVO_OUTPUT_TMDS0)) + flags &= ~SDVO_OUTPUT_TMDS1; + + if (!(flags & SDVO_OUTPUT_RGB0)) + flags &= ~SDVO_OUTPUT_RGB1; + + if (!(flags & SDVO_OUTPUT_LVDS0)) + flags &= ~SDVO_OUTPUT_LVDS1; + + return flags; +} + static bool intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags) { - /* SDVO requires XXX1 function may not exist unless it has XXX0 function.*/ + flags = intel_sdvo_filter_output_flags(flags); if (flags & SDVO_OUTPUT_TMDS0) if (!intel_sdvo_dvi_init(intel_sdvo, 0)) return false; - if ((flags & SDVO_TMDS_MASK) == SDVO_TMDS_MASK) + if (flags & SDVO_OUTPUT_TMDS1) if (!intel_sdvo_dvi_init(intel_sdvo, 1)) return false; @@ -2956,7 +2973,7 @@ intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags) if (!intel_sdvo_analog_init(intel_sdvo, 0)) return false; - if ((flags & SDVO_RGB_MASK) == SDVO_RGB_MASK) + if (flags & SDVO_OUTPUT_RGB1) if (!intel_sdvo_analog_init(intel_sdvo, 1)) return false; @@ -2964,11 +2981,11 @@ intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags) if (!intel_sdvo_lvds_init(intel_sdvo, 0)) return false; - if ((flags & SDVO_LVDS_MASK) == SDVO_LVDS_MASK) + if (flags & SDVO_OUTPUT_LVDS1) if (!intel_sdvo_lvds_init(intel_sdvo, 1)) return false; - if ((flags & SDVO_OUTPUT_MASK) == 0) { + if (flags == 0) { unsigned char bytes[2]; intel_sdvo->controlled_output = 0; -- GitLab From e79762512120f11c51317570519a1553c70805d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 13:11:28 +0300 Subject: [PATCH 0772/1927] drm/i915/sdvo: Setup DDC fully before output init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Call intel_sdvo_select_ddc_bus() before initializing any of the outputs. And before that is functional (assuming no VBT) we have to set up the controlled_outputs thing. Otherwise DDC won't be functional during the output init but LVDS really needs it for the fixed mode setup. Note that the whole multi output support still looks very bogus, and more work will be needed to make it correct. But for now this should at least fix the LVDS EDID fixed mode setup. Cc: stable@vger.kernel.org Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7301 Fixes: aa2b88074a56 ("drm/i915/sdvo: Fix multi function encoder stuff") Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026101134.20865-3-ville.syrjala@linux.intel.com Reviewed-by: Jani Nikula (cherry picked from commit 64b7b557dc8a96d9cfed6aedbf81de2df80c025d) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/display/intel_sdvo.c | 31 +++++++++-------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 30d3778c61366..8046d02a8ad01 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -2747,13 +2747,10 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device) if (!intel_sdvo_connector) return false; - if (device == 0) { - intel_sdvo->controlled_output |= SDVO_OUTPUT_TMDS0; + if (device == 0) intel_sdvo_connector->output_flag = SDVO_OUTPUT_TMDS0; - } else if (device == 1) { - intel_sdvo->controlled_output |= SDVO_OUTPUT_TMDS1; + else if (device == 1) intel_sdvo_connector->output_flag = SDVO_OUTPUT_TMDS1; - } intel_connector = &intel_sdvo_connector->base; connector = &intel_connector->base; @@ -2808,7 +2805,6 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type) encoder->encoder_type = DRM_MODE_ENCODER_TVDAC; connector->connector_type = DRM_MODE_CONNECTOR_SVIDEO; - intel_sdvo->controlled_output |= type; intel_sdvo_connector->output_flag = type; if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { @@ -2849,13 +2845,10 @@ intel_sdvo_analog_init(struct intel_sdvo *intel_sdvo, int device) encoder->encoder_type = DRM_MODE_ENCODER_DAC; connector->connector_type = DRM_MODE_CONNECTOR_VGA; - if (device == 0) { - intel_sdvo->controlled_output |= SDVO_OUTPUT_RGB0; + if (device == 0) intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB0; - } else if (device == 1) { - intel_sdvo->controlled_output |= SDVO_OUTPUT_RGB1; + else if (device == 1) intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB1; - } if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { kfree(intel_sdvo_connector); @@ -2885,13 +2878,10 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) encoder->encoder_type = DRM_MODE_ENCODER_LVDS; connector->connector_type = DRM_MODE_CONNECTOR_LVDS; - if (device == 0) { - intel_sdvo->controlled_output |= SDVO_OUTPUT_LVDS0; + if (device == 0) intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS0; - } else if (device == 1) { - intel_sdvo->controlled_output |= SDVO_OUTPUT_LVDS1; + else if (device == 1) intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS1; - } if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) { kfree(intel_sdvo_connector); @@ -2946,8 +2936,14 @@ static u16 intel_sdvo_filter_output_flags(u16 flags) static bool intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags) { + struct drm_i915_private *i915 = to_i915(intel_sdvo->base.base.dev); + flags = intel_sdvo_filter_output_flags(flags); + intel_sdvo->controlled_output = flags; + + intel_sdvo_select_ddc_bus(i915, intel_sdvo); + if (flags & SDVO_OUTPUT_TMDS0) if (!intel_sdvo_dvi_init(intel_sdvo, 0)) return false; @@ -2988,7 +2984,6 @@ intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, u16 flags) if (flags == 0) { unsigned char bytes[2]; - intel_sdvo->controlled_output = 0; memcpy(bytes, &intel_sdvo->caps.output_flags, 2); DRM_DEBUG_KMS("%s: Unknown SDVO output type (0x%02x%02x)\n", SDVO_NAME(intel_sdvo), @@ -3400,8 +3395,6 @@ bool intel_sdvo_init(struct drm_i915_private *dev_priv, */ intel_sdvo->base.cloneable = 0; - intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo); - /* Set the input timing to the screen. Assume always input 0. */ if (!intel_sdvo_set_target_input(intel_sdvo)) goto err_output; -- GitLab From e59bf547a7dd366f93bfebb7487959580ca6c0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Thu, 27 Oct 2022 11:57:58 +0200 Subject: [PATCH 0773/1927] ASoC: tas2770: Fix set_tdm_slot in case of single slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's a special branch in the set_tdm_slot op for the case of nslots being 1, but: (1) That branch can never work (there's a check for tx_mask being non-zero, later there's another check for it *being* zero; one or the other always throws -EINVAL). (2) The intention of the branch seems to be what the general other branch reduces to in case of nslots being 1. For those reasons remove the 'nslots being 1' special case. Fixes: 1a476abc723e ("tas2770: add tas2770 smart PA kernel driver") Suggested-by: Jos Dehaes Signed-off-by: Martin Povišer Link: https://lore.kernel.org/r/20221027095800.16094-1-povik+lin@cutebit.org Signed-off-by: Mark Brown --- sound/soc/codecs/tas2770.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c index b6765235a4b3d..8557759acb1f2 100644 --- a/sound/soc/codecs/tas2770.c +++ b/sound/soc/codecs/tas2770.c @@ -395,21 +395,13 @@ static int tas2770_set_dai_tdm_slot(struct snd_soc_dai *dai, if (tx_mask == 0 || rx_mask != 0) return -EINVAL; - if (slots == 1) { - if (tx_mask != 1) - return -EINVAL; - - left_slot = 0; - right_slot = 0; + left_slot = __ffs(tx_mask); + tx_mask &= ~(1 << left_slot); + if (tx_mask == 0) { + right_slot = left_slot; } else { - left_slot = __ffs(tx_mask); - tx_mask &= ~(1 << left_slot); - if (tx_mask == 0) { - right_slot = left_slot; - } else { - right_slot = __ffs(tx_mask); - tx_mask &= ~(1 << right_slot); - } + right_slot = __ffs(tx_mask); + tx_mask &= ~(1 << right_slot); } if (tx_mask != 0 || left_slot >= slots || right_slot >= slots) -- GitLab From faac764ea1ea6898d93e46c403271fb105c0906e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Thu, 27 Oct 2022 11:57:59 +0200 Subject: [PATCH 0774/1927] ASoC: tas2764: Fix set_tdm_slot in case of single slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's a special branch in the set_tdm_slot op for the case of nslots being 1, but: (1) That branch can never work (there's a check for tx_mask being non-zero, later there's another check for it *being* zero; one or the other always throws -EINVAL). (2) The intention of the branch seems to be what the general other branch reduces to in case of nslots being 1. For those reasons remove the 'nslots being 1' special case. Fixes: 827ed8a0fa50 ("ASoC: tas2764: Add the driver for the TAS2764") Suggested-by: Jos Dehaes Signed-off-by: Martin Povišer Link: https://lore.kernel.org/r/20221027095800.16094-2-povik+lin@cutebit.org Signed-off-by: Mark Brown --- sound/soc/codecs/tas2764.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c index 51b87a9361798..2e0ed3e68fa54 100644 --- a/sound/soc/codecs/tas2764.c +++ b/sound/soc/codecs/tas2764.c @@ -438,20 +438,13 @@ static int tas2764_set_dai_tdm_slot(struct snd_soc_dai *dai, if (tx_mask == 0 || rx_mask != 0) return -EINVAL; - if (slots == 1) { - if (tx_mask != 1) - return -EINVAL; - left_slot = 0; - right_slot = 0; + left_slot = __ffs(tx_mask); + tx_mask &= ~(1 << left_slot); + if (tx_mask == 0) { + right_slot = left_slot; } else { - left_slot = __ffs(tx_mask); - tx_mask &= ~(1 << left_slot); - if (tx_mask == 0) { - right_slot = left_slot; - } else { - right_slot = __ffs(tx_mask); - tx_mask &= ~(1 << right_slot); - } + right_slot = __ffs(tx_mask); + tx_mask &= ~(1 << right_slot); } if (tx_mask != 0 || left_slot >= slots || right_slot >= slots) -- GitLab From 6f934afa6a980bb8d3ce73836b1a9922685e50d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Thu, 27 Oct 2022 11:58:00 +0200 Subject: [PATCH 0775/1927] ASoC: tas2780: Fix set_tdm_slot in case of single slot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There's a special branch in the set_tdm_slot op for the case of nslots being 1, but: (1) That branch can never work (there's a check for tx_mask being non-zero, later there's another check for it *being* zero; one or the other always throws -EINVAL). (2) The intention of the branch seems to be what the general other branch reduces to in case of nslots being 1. For those reasons remove the 'nslots being 1' special case. Fixes: eae9f9ce181b ("ASoC: add tas2780 driver") Suggested-by: Jos Dehaes Signed-off-by: Martin Povišer Link: https://lore.kernel.org/r/20221027095800.16094-3-povik+lin@cutebit.org Signed-off-by: Mark Brown --- sound/soc/codecs/tas2780.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/sound/soc/codecs/tas2780.c b/sound/soc/codecs/tas2780.c index a6db6f0e5431f..afdf0c863aa10 100644 --- a/sound/soc/codecs/tas2780.c +++ b/sound/soc/codecs/tas2780.c @@ -380,20 +380,13 @@ static int tas2780_set_dai_tdm_slot(struct snd_soc_dai *dai, if (tx_mask == 0 || rx_mask != 0) return -EINVAL; - if (slots == 1) { - if (tx_mask != 1) - return -EINVAL; - left_slot = 0; - right_slot = 0; + left_slot = __ffs(tx_mask); + tx_mask &= ~(1 << left_slot); + if (tx_mask == 0) { + right_slot = left_slot; } else { - left_slot = __ffs(tx_mask); - tx_mask &= ~(1 << left_slot); - if (tx_mask == 0) { - right_slot = left_slot; - } else { - right_slot = __ffs(tx_mask); - tx_mask &= ~(1 << right_slot); - } + right_slot = __ffs(tx_mask); + tx_mask &= ~(1 << right_slot); } if (tx_mask != 0 || left_slot >= slots || right_slot >= slots) -- GitLab From cf82d0ecb84e8ef9958721193f901609b408655b Mon Sep 17 00:00:00 2001 From: "zhichao.liu" Date: Fri, 21 Oct 2022 17:16:53 +0800 Subject: [PATCH 0776/1927] spi: mediatek: Fix package division error Commit 7e963fb2a33ce ("spi: mediatek: add ipm design support for MT7986") makes a mistake on package dividing operation (one change is missing), need to fix it. Background: Ipm design is expanding the HW capability of dma (adjust package length from 1KB to 64KB), and using "dev_comp->ipm_support" flag to indicate it. Issue description: Ipm support patch (said above) is missing to handle remainder at package dividing operation. One case, a transmission length is 65KB, is will divide to 1K (package length) * 65(package loop) in non-ipm desgin case, and will divide to 64K(package length) * 1(package loop) + 1K(remainder) in ipm design case. And the 1K remainder will be lost with the current SW flow, and the transmission will be failure. So, it should be fixed. Solution: Add "ipm_design" flag in function "mtk_spi_get_mult_delta()" to indicate HW capability, and modify the parameters corespondingly. fixes: 7e963fb2a33ce ("spi: mediatek: add ipm design support for MT7986") Signed-off-by: zhichao.liu Link: https://lore.kernel.org/r/20221021091653.18297-1-zhichao.liu@mediatek.com Signed-off-by: Mark Brown --- drivers/spi/spi-mt65xx.c | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c index 11aeae7fe7fc9..a33c9a3de395f 100644 --- a/drivers/spi/spi-mt65xx.c +++ b/drivers/spi/spi-mt65xx.c @@ -551,14 +551,17 @@ static void mtk_spi_enable_transfer(struct spi_master *master) writel(cmd, mdata->base + SPI_CMD_REG); } -static int mtk_spi_get_mult_delta(u32 xfer_len) +static int mtk_spi_get_mult_delta(struct mtk_spi *mdata, u32 xfer_len) { - u32 mult_delta; + u32 mult_delta = 0; - if (xfer_len > MTK_SPI_PACKET_SIZE) - mult_delta = xfer_len % MTK_SPI_PACKET_SIZE; - else - mult_delta = 0; + if (mdata->dev_comp->ipm_design) { + if (xfer_len > MTK_SPI_IPM_PACKET_SIZE) + mult_delta = xfer_len % MTK_SPI_IPM_PACKET_SIZE; + } else { + if (xfer_len > MTK_SPI_PACKET_SIZE) + mult_delta = xfer_len % MTK_SPI_PACKET_SIZE; + } return mult_delta; } @@ -570,22 +573,22 @@ static void mtk_spi_update_mdata_len(struct spi_master *master) if (mdata->tx_sgl_len && mdata->rx_sgl_len) { if (mdata->tx_sgl_len > mdata->rx_sgl_len) { - mult_delta = mtk_spi_get_mult_delta(mdata->rx_sgl_len); + mult_delta = mtk_spi_get_mult_delta(mdata, mdata->rx_sgl_len); mdata->xfer_len = mdata->rx_sgl_len - mult_delta; mdata->rx_sgl_len = mult_delta; mdata->tx_sgl_len -= mdata->xfer_len; } else { - mult_delta = mtk_spi_get_mult_delta(mdata->tx_sgl_len); + mult_delta = mtk_spi_get_mult_delta(mdata, mdata->tx_sgl_len); mdata->xfer_len = mdata->tx_sgl_len - mult_delta; mdata->tx_sgl_len = mult_delta; mdata->rx_sgl_len -= mdata->xfer_len; } } else if (mdata->tx_sgl_len) { - mult_delta = mtk_spi_get_mult_delta(mdata->tx_sgl_len); + mult_delta = mtk_spi_get_mult_delta(mdata, mdata->tx_sgl_len); mdata->xfer_len = mdata->tx_sgl_len - mult_delta; mdata->tx_sgl_len = mult_delta; } else if (mdata->rx_sgl_len) { - mult_delta = mtk_spi_get_mult_delta(mdata->rx_sgl_len); + mult_delta = mtk_spi_get_mult_delta(mdata, mdata->rx_sgl_len); mdata->xfer_len = mdata->rx_sgl_len - mult_delta; mdata->rx_sgl_len = mult_delta; } -- GitLab From 224e858f215a3d6304f95a92357a1753475ca9cf Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sat, 29 Oct 2022 09:04:29 +0800 Subject: [PATCH 0777/1927] ublk_drv: return flag of UBLK_F_URING_CMD_COMP_IN_TASK in case of module UBLK_F_URING_CMD_COMP_IN_TASK needs to be set and returned to userspace if ublk driver is built as module, otherwise userspace may get wrong flags shown. Fixes: 71f28f3136af ("ublk_drv: add io_uring based userspace block driver") Signed-off-by: Ming Lei Reviewed-by: ZiyangZhang Link: https://lore.kernel.org/r/20221029010432.598367-2-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 5afce6ffaadfa..6b2f214f0d5c6 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1658,6 +1658,9 @@ static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd) */ ub->dev_info.flags &= UBLK_F_ALL; + if (!IS_BUILTIN(CONFIG_BLK_DEV_UBLK)) + ub->dev_info.flags |= UBLK_F_URING_CMD_COMP_IN_TASK; + /* We are not ready to support zero copy */ ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY; -- GitLab From d57c2c6c1145148bb23d68db73de0b52d482d4ba Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sat, 29 Oct 2022 09:04:30 +0800 Subject: [PATCH 0778/1927] ublk_drv: comment on ublk_driver entry of Kconfig Add help info for choosing to build ublk_drv as module or builtin. Signed-off-by: Ming Lei Reviewed-by: ZiyangZhang Link: https://lore.kernel.org/r/20221029010432.598367-3-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/Kconfig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/block/Kconfig b/drivers/block/Kconfig index db1b4b202646e..a41145d52de94 100644 --- a/drivers/block/Kconfig +++ b/drivers/block/Kconfig @@ -408,6 +408,12 @@ config BLK_DEV_UBLK definition isn't finalized yet, and might change according to future requirement, so mark is as experimental now. + Say Y if you want to get better performance because task_work_add() + can be used in IO path for replacing io_uring cmd, which will become + shared between IO tasks and ubq daemon, meantime task_work_add() can + can handle batch more effectively, but task_work_add() isn't exported + for module, so ublk has to be built to kernel. + source "drivers/block/rnbd/Kconfig" endif # BLK_DEV -- GitLab From 3ab6e94ca539242247d4f00414a1bde584d001ed Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sat, 29 Oct 2022 09:04:31 +0800 Subject: [PATCH 0779/1927] ublk_drv: avoid to touch io_uring cmd in blk_mq io path io_uring cmd is supposed to be used in ubq daemon context mainly, and we should try to avoid to touch it in ublk io submission context, otherwise this data could become shared between the two contexts, and performance is hurt. So link request into one per-queue list, and use same batching policy of io_uring command, just avoid to touch ucmd in blk-mq io context. Signed-off-by: Ming Lei Reviewed-by: ZiyangZhang Link: https://lore.kernel.org/r/20221029010432.598367-4-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 83 +++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 6b2f214f0d5c6..3a59271dafe4d 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -57,11 +57,14 @@ #define UBLK_PARAM_TYPE_ALL (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD) struct ublk_rq_data { - struct callback_head work; + union { + struct callback_head work; + struct llist_node node; + }; }; struct ublk_uring_cmd_pdu { - struct request *req; + struct ublk_queue *ubq; }; /* @@ -119,6 +122,8 @@ struct ublk_queue { struct task_struct *ubq_daemon; char *io_cmd_buf; + struct llist_head io_cmds; + unsigned long io_addr; /* mapped vm address */ unsigned int max_io_sz; bool force_abort; @@ -764,8 +769,12 @@ static inline void __ublk_rq_task_work(struct request *req) static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd) { struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); + struct ublk_queue *ubq = pdu->ubq; + struct llist_node *io_cmds = llist_del_all(&ubq->io_cmds); + struct ublk_rq_data *data; - __ublk_rq_task_work(pdu->req); + llist_for_each_entry(data, io_cmds, node) + __ublk_rq_task_work(blk_mq_rq_from_pdu(data)); } static void ublk_rq_task_work_fn(struct callback_head *work) @@ -777,17 +786,50 @@ static void ublk_rq_task_work_fn(struct callback_head *work) __ublk_rq_task_work(req); } +static void ublk_submit_cmd(struct ublk_queue *ubq, const struct request *rq) +{ + struct ublk_io *io = &ubq->ios[rq->tag]; + + /* + * If the check pass, we know that this is a re-issued request aborted + * previously in monitor_work because the ubq_daemon(cmd's task) is + * PF_EXITING. We cannot call io_uring_cmd_complete_in_task() anymore + * because this ioucmd's io_uring context may be freed now if no inflight + * ioucmd exists. Otherwise we may cause null-deref in ctx->fallback_work. + * + * Note: monitor_work sets UBLK_IO_FLAG_ABORTED and ends this request(releasing + * the tag). Then the request is re-started(allocating the tag) and we are here. + * Since releasing/allocating a tag implies smp_mb(), finding UBLK_IO_FLAG_ABORTED + * guarantees that here is a re-issued request aborted previously. + */ + if (unlikely(io->flags & UBLK_IO_FLAG_ABORTED)) { + struct llist_node *io_cmds = llist_del_all(&ubq->io_cmds); + struct ublk_rq_data *data; + + llist_for_each_entry(data, io_cmds, node) + __ublk_abort_rq(ubq, blk_mq_rq_from_pdu(data)); + } else { + struct io_uring_cmd *cmd = io->cmd; + struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); + + pdu->ubq = ubq; + io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb); + } +} + static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct ublk_queue *ubq = hctx->driver_data; struct request *rq = bd->rq; + struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq); blk_status_t res; /* fill iod to slot in io cmd buffer */ res = ublk_setup_iod(ubq, rq); if (unlikely(res != BLK_STS_OK)) return BLK_STS_IOERR; + /* With recovery feature enabled, force_abort is set in * ublk_stop_dev() before calling del_gendisk(). We have to * abort all requeued and new rqs here to let del_gendisk() @@ -809,36 +851,15 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, } if (ublk_can_use_task_work(ubq)) { - struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq); enum task_work_notify_mode notify_mode = bd->last ? TWA_SIGNAL_NO_IPI : TWA_NONE; if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode)) goto fail; } else { - struct ublk_io *io = &ubq->ios[rq->tag]; - struct io_uring_cmd *cmd = io->cmd; - struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); - - /* - * If the check pass, we know that this is a re-issued request aborted - * previously in monitor_work because the ubq_daemon(cmd's task) is - * PF_EXITING. We cannot call io_uring_cmd_complete_in_task() anymore - * because this ioucmd's io_uring context may be freed now if no inflight - * ioucmd exists. Otherwise we may cause null-deref in ctx->fallback_work. - * - * Note: monitor_work sets UBLK_IO_FLAG_ABORTED and ends this request(releasing - * the tag). Then the request is re-started(allocating the tag) and we are here. - * Since releasing/allocating a tag implies smp_mb(), finding UBLK_IO_FLAG_ABORTED - * guarantees that here is a re-issued request aborted previously. - */ - if ((io->flags & UBLK_IO_FLAG_ABORTED)) - goto fail; - - pdu->req = rq; - io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb); + if (llist_add(&data->node, &ubq->io_cmds)) + ublk_submit_cmd(ubq, rq); } - return BLK_STS_OK; } @@ -1168,17 +1189,19 @@ static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id, { struct ublk_queue *ubq = ublk_get_queue(ub, q_id); struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag); + struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); if (ublk_can_use_task_work(ubq)) { - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); - /* should not fail since we call it just in ubq->ubq_daemon */ task_work_add(ubq->ubq_daemon, &data->work, TWA_SIGNAL_NO_IPI); } else { struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); - pdu->req = req; - io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb); + if (llist_add(&data->node, &ubq->io_cmds)) { + pdu->ubq = ubq; + io_uring_cmd_complete_in_task(cmd, + ublk_rq_task_work_cb); + } } } -- GitLab From fee32f312405726eec6b35b5740c48acda0315e9 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sat, 29 Oct 2022 09:04:32 +0800 Subject: [PATCH 0780/1927] ublk_drv: add ublk_queue_cmd() for cleanup Add helper of ublk_queue_cmd() so that both ublk_queue_rq() and ublk_handle_need_get_data() can reuse this helper. Signed-off-by: Ming Lei Reviewed-by: ZiyangZhang Link: https://lore.kernel.org/r/20221029010432.598367-5-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 47 ++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 3a59271dafe4d..f96cb01e9604d 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -817,12 +817,28 @@ static void ublk_submit_cmd(struct ublk_queue *ubq, const struct request *rq) } } +static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq, + bool last) +{ + struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq); + + if (ublk_can_use_task_work(ubq)) { + enum task_work_notify_mode notify_mode = last ? + TWA_SIGNAL_NO_IPI : TWA_NONE; + + if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode)) + __ublk_abort_rq(ubq, rq); + } else { + if (llist_add(&data->node, &ubq->io_cmds)) + ublk_submit_cmd(ubq, rq); + } +} + static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { struct ublk_queue *ubq = hctx->driver_data; struct request *rq = bd->rq; - struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq); blk_status_t res; /* fill iod to slot in io cmd buffer */ @@ -845,21 +861,12 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, blk_mq_start_request(bd->rq); if (unlikely(ubq_daemon_is_dying(ubq))) { - fail: __ublk_abort_rq(ubq, rq); return BLK_STS_OK; } - if (ublk_can_use_task_work(ubq)) { - enum task_work_notify_mode notify_mode = bd->last ? - TWA_SIGNAL_NO_IPI : TWA_NONE; + ublk_queue_cmd(ubq, rq, bd->last); - if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode)) - goto fail; - } else { - if (llist_add(&data->node, &ubq->io_cmds)) - ublk_submit_cmd(ubq, rq); - } return BLK_STS_OK; } @@ -1185,24 +1192,12 @@ static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq) } static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id, - int tag, struct io_uring_cmd *cmd) + int tag) { struct ublk_queue *ubq = ublk_get_queue(ub, q_id); struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag); - struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); - if (ublk_can_use_task_work(ubq)) { - /* should not fail since we call it just in ubq->ubq_daemon */ - task_work_add(ubq->ubq_daemon, &data->work, TWA_SIGNAL_NO_IPI); - } else { - struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); - - if (llist_add(&data->node, &ubq->io_cmds)) { - pdu->ubq = ubq; - io_uring_cmd_complete_in_task(cmd, - ublk_rq_task_work_cb); - } - } + ublk_queue_cmd(ubq, req, true); } static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) @@ -1290,7 +1285,7 @@ static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) io->addr = ub_cmd->addr; io->cmd = cmd; io->flags |= UBLK_IO_FLAG_ACTIVE; - ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag, cmd); + ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag); break; default: goto out; -- GitLab From fa81cbafbf5764ad5053512152345fab37a1fe18 Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Sat, 29 Oct 2022 15:13:55 +0800 Subject: [PATCH 0781/1927] block: Fix possible memory leak for rq_wb on add_disk failure kmemleak reported memory leaks in device_add_disk(): kmemleak: 3 new suspected memory leaks unreferenced object 0xffff88800f420800 (size 512): comm "modprobe", pid 4275, jiffies 4295639067 (age 223.512s) hex dump (first 32 bytes): 04 00 00 00 08 00 00 00 01 00 00 00 00 00 00 00 ................ 00 e1 f5 05 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<00000000d3662699>] kmalloc_trace+0x26/0x60 [<00000000edc7aadc>] wbt_init+0x50/0x6f0 [<0000000069601d16>] wbt_enable_default+0x157/0x1c0 [<0000000028fc393f>] blk_register_queue+0x2a4/0x420 [<000000007345a042>] device_add_disk+0x6fd/0xe40 [<0000000060e6aab0>] nbd_dev_add+0x828/0xbf0 [nbd] ... It is because the memory allocated in wbt_enable_default() is not released in device_add_disk() error path. Normally, these memory are freed in: del_gendisk() rq_qos_exit() rqos->ops->exit(rqos); wbt_exit() So rq_qos_exit() is called to free the rq_wb memory for wbt_init(). However in the error path of device_add_disk(), only blk_unregister_queue() is called and make rq_wb memory leaked. Add rq_qos_exit() to the error path to fix it. Fixes: 83cbce957446 ("block: add error handling for device_add_disk / add_disk") Signed-off-by: Chen Zhongjin Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20221029071355.35462-1-chenzhongjin@huawei.com Signed-off-by: Jens Axboe --- block/genhd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block/genhd.c b/block/genhd.c index fee90eb98b4a5..0f9769db2de83 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -527,6 +527,7 @@ int __must_check device_add_disk(struct device *parent, struct gendisk *disk, bdi_unregister(disk->bdi); out_unregister_queue: blk_unregister_queue(disk); + rq_qos_exit(disk->queue); out_put_slave_dir: kobject_put(disk->slave_dir); out_put_holder_dir: -- GitLab From d372ec94a018c3a19dad71e2ee3478126394d9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Tue, 27 Sep 2022 21:06:13 +0300 Subject: [PATCH 0782/1927] drm/i915: Simplify intel_panel_add_edid_alt_fixed_modes() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit a5810f551d0a ("drm/i915: Allow more varied alternate fixed modes for panels") intel_panel_add_edid_alt_fixed_modes() no longer considers vrr vs. drrs separately. So no reason to pass them as separate parameters either. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20220927180615.25476-2-ville.syrjala@linux.intel.com Reviewed-by: Jani Nikula (cherry picked from commit eb89e83c152b122a94e79527d63cb7c79823c37e) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/display/intel_dp.c | 2 +- drivers/gpu/drm/i915/display/intel_lvds.c | 3 +-- drivers/gpu/drm/i915/display/intel_panel.c | 4 ++-- drivers/gpu/drm/i915/display/intel_panel.h | 2 +- drivers/gpu/drm/i915/display/intel_sdvo.c | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 47419d162f302..2b5bc95a8b0df 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -5276,7 +5276,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp, encoder->devdata, IS_ERR(edid) ? NULL : edid); intel_panel_add_edid_fixed_modes(intel_connector, - intel_connector->panel.vbt.drrs_type != DRRS_TYPE_NONE, + intel_connector->panel.vbt.drrs_type != DRRS_TYPE_NONE || intel_vrr_is_capable(intel_connector)); /* MSO requires information from the EDID */ diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c index 9aa38e8141b52..e5352239b2a2f 100644 --- a/drivers/gpu/drm/i915/display/intel_lvds.c +++ b/drivers/gpu/drm/i915/display/intel_lvds.c @@ -972,8 +972,7 @@ void intel_lvds_init(struct drm_i915_private *dev_priv) /* Try EDID first */ intel_panel_add_edid_fixed_modes(intel_connector, - intel_connector->panel.vbt.drrs_type != DRRS_TYPE_NONE, - false); + intel_connector->panel.vbt.drrs_type != DRRS_TYPE_NONE); /* Failed to get EDID, what about VBT? */ if (!intel_panel_preferred_fixed_mode(intel_connector)) diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c index a3a3f9fe4342f..41cec9dc42231 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.c +++ b/drivers/gpu/drm/i915/display/intel_panel.c @@ -254,10 +254,10 @@ static void intel_panel_destroy_probed_modes(struct intel_connector *connector) } void intel_panel_add_edid_fixed_modes(struct intel_connector *connector, - bool has_drrs, bool has_vrr) + bool use_alt_fixed_modes) { intel_panel_add_edid_preferred_mode(connector); - if (intel_panel_preferred_fixed_mode(connector) && (has_drrs || has_vrr)) + if (intel_panel_preferred_fixed_mode(connector) && use_alt_fixed_modes) intel_panel_add_edid_alt_fixed_modes(connector); intel_panel_destroy_probed_modes(connector); } diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h index eff3ffd3d0825..5c5b5b7f95b6c 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.h +++ b/drivers/gpu/drm/i915/display/intel_panel.h @@ -44,7 +44,7 @@ int intel_panel_fitting(struct intel_crtc_state *crtc_state, int intel_panel_compute_config(struct intel_connector *connector, struct drm_display_mode *adjusted_mode); void intel_panel_add_edid_fixed_modes(struct intel_connector *connector, - bool has_drrs, bool has_vrr); + bool use_alt_fixed_modes); void intel_panel_add_vbt_lfp_fixed_mode(struct intel_connector *connector); void intel_panel_add_vbt_sdvo_fixed_mode(struct intel_connector *connector); void intel_panel_add_encoder_fixed_mode(struct intel_connector *connector, diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 8046d02a8ad01..8ee7b05ab7333 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -2901,7 +2901,7 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) if (!intel_panel_preferred_fixed_mode(intel_connector)) { intel_ddc_get_modes(connector, &intel_sdvo->ddc); - intel_panel_add_edid_fixed_modes(intel_connector, false, false); + intel_panel_add_edid_fixed_modes(intel_connector, false); } intel_panel_init(intel_connector); -- GitLab From 12caf46cf4fc92b1c3884cb363ace2e12732fd2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 13:11:29 +0300 Subject: [PATCH 0783/1927] drm/i915/sdvo: Grab mode_config.mutex during LVDS init to avoid WARNs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drm_mode_probed_add() is unhappy about being called w/o mode_config.mutex. Grab it during LVDS fixed mode setup to silence the WARNs. Cc: stable@vger.kernel.org Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7301 Fixes: aa2b88074a56 ("drm/i915/sdvo: Fix multi function encoder stuff") Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026101134.20865-4-ville.syrjala@linux.intel.com Reviewed-by: Jani Nikula (cherry picked from commit a3cd4f447281c56377de2ee109327400eb00668d) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/display/intel_sdvo.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 8ee7b05ab7333..774c1dc31a521 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -2900,8 +2900,12 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device) intel_panel_add_vbt_sdvo_fixed_mode(intel_connector); if (!intel_panel_preferred_fixed_mode(intel_connector)) { + mutex_lock(&i915->drm.mode_config.mutex); + intel_ddc_get_modes(connector, &intel_sdvo->ddc); intel_panel_add_edid_fixed_modes(intel_connector, false); + + mutex_unlock(&i915->drm.mode_config.mutex); } intel_panel_init(intel_connector); -- GitLab From 943f45b9399ed8b2b5190cbc797995edaa97f58f Mon Sep 17 00:00:00 2001 From: Chen Jun Date: Mon, 31 Oct 2022 03:12:42 +0000 Subject: [PATCH 0784/1927] blk-mq: Fix kmemleak in blk_mq_init_allocated_queue There is a kmemleak caused by modprobe null_blk.ko unreferenced object 0xffff8881acb1f000 (size 1024): comm "modprobe", pid 836, jiffies 4294971190 (age 27.068s) hex dump (first 32 bytes): 00 00 00 00 ad 4e ad de ff ff ff ff 00 00 00 00 .....N.......... ff ff ff ff ff ff ff ff 00 53 99 9e ff ff ff ff .........S...... backtrace: [<000000004a10c249>] kmalloc_node_trace+0x22/0x60 [<00000000648f7950>] blk_mq_alloc_and_init_hctx+0x289/0x350 [<00000000af06de0e>] blk_mq_realloc_hw_ctxs+0x2fe/0x3d0 [<00000000e00c1872>] blk_mq_init_allocated_queue+0x48c/0x1440 [<00000000d16b4e68>] __blk_mq_alloc_disk+0xc8/0x1c0 [<00000000d10c98c3>] 0xffffffffc450d69d [<00000000b9299f48>] 0xffffffffc4538392 [<0000000061c39ed6>] do_one_initcall+0xd0/0x4f0 [<00000000b389383b>] do_init_module+0x1a4/0x680 [<0000000087cf3542>] load_module+0x6249/0x7110 [<00000000beba61b8>] __do_sys_finit_module+0x140/0x200 [<00000000fdcfff51>] do_syscall_64+0x35/0x80 [<000000003c0f1f71>] entry_SYSCALL_64_after_hwframe+0x46/0xb0 That is because q->ma_ops is set to NULL before blk_release_queue is called. blk_mq_init_queue_data blk_mq_init_allocated_queue blk_mq_realloc_hw_ctxs for (i = 0; i < set->nr_hw_queues; i++) { old_hctx = xa_load(&q->hctx_table, i); if (!blk_mq_alloc_and_init_hctx(.., i, ..)) [1] if (!old_hctx) break; xa_for_each_start(&q->hctx_table, j, hctx, j) blk_mq_exit_hctx(q, set, hctx, j); [2] if (!q->nr_hw_queues) [3] goto err_hctxs; err_exit: q->mq_ops = NULL; [4] blk_put_queue blk_release_queue if (queue_is_mq(q)) [5] blk_mq_release(q); [1]: blk_mq_alloc_and_init_hctx failed at i != 0. [2]: The hctxs allocated by [1] are moved to q->unused_hctx_list and will be cleaned up in blk_mq_release. [3]: q->nr_hw_queues is 0. [4]: Set q->mq_ops to NULL. [5]: queue_is_mq returns false due to [4]. And blk_mq_release will not be called. The hctxs in q->unused_hctx_list are leaked. To fix it, call blk_release_queue in exception path. Fixes: 2f8f1336a48b ("blk-mq: always free hctx after request queue is freed") Signed-off-by: Yuan Can Signed-off-by: Chen Jun Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20221031031242.94107-1-chenjun102@huawei.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 75c8296b6feb3..21cc7c2da0f96 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4193,9 +4193,7 @@ int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, return 0; err_hctxs: - xa_destroy(&q->hctx_table); - q->nr_hw_queues = 0; - blk_mq_sysfs_deinit(q); + blk_mq_release(q); err_poll: blk_stat_free_callback(q->poll_cb); q->poll_cb = NULL; -- GitLab From 2b6ae0962b421103feb41a80406732944b0665b3 Mon Sep 17 00:00:00 2001 From: Helge Deller Date: Fri, 28 Oct 2022 18:12:49 +0200 Subject: [PATCH 0785/1927] parisc: Avoid printing the hardware path twice Avoid that the hardware path is shown twice in the kernel log, and clean up the output of the version numbers to show up in the same order as they are listed in the hardware database in the hardware.c file. Additionally, optimize the memory footprint of the hardware database and mark some code as init code. Fixes: cab56b51ec0e ("parisc: Fix device names in /proc/iomem") Signed-off-by: Helge Deller Cc: # v4.9+ --- arch/parisc/include/asm/hardware.h | 12 ++++++------ arch/parisc/kernel/drivers.c | 14 ++++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/arch/parisc/include/asm/hardware.h b/arch/parisc/include/asm/hardware.h index 9d3d7737c58b1..a005ebc547793 100644 --- a/arch/parisc/include/asm/hardware.h +++ b/arch/parisc/include/asm/hardware.h @@ -10,12 +10,12 @@ #define SVERSION_ANY_ID PA_SVERSION_ANY_ID struct hp_hardware { - unsigned short hw_type:5; /* HPHW_xxx */ - unsigned short hversion; - unsigned long sversion:28; - unsigned short opt; - const char name[80]; /* The hardware description */ -}; + unsigned int hw_type:8; /* HPHW_xxx */ + unsigned int hversion:12; + unsigned int sversion:12; + unsigned char opt; + unsigned char name[59]; /* The hardware description */ +} __packed; struct parisc_device; diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index d126e78e101ae..e7ee0c0c91d35 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -882,15 +882,13 @@ void __init walk_central_bus(void) &root); } -static void print_parisc_device(struct parisc_device *dev) +static __init void print_parisc_device(struct parisc_device *dev) { - char hw_path[64]; - static int count; + static int count __initdata; - print_pa_hwpath(dev, hw_path); - pr_info("%d. %s at %pap [%s] { %d, 0x%x, 0x%.3x, 0x%.5x }", - ++count, dev->name, &(dev->hpa.start), hw_path, dev->id.hw_type, - dev->id.hversion_rev, dev->id.hversion, dev->id.sversion); + pr_info("%d. %s at %pap { type:%d, hv:%#x, sv:%#x, rev:%#x }", + ++count, dev->name, &(dev->hpa.start), dev->id.hw_type, + dev->id.hversion, dev->id.sversion, dev->id.hversion_rev); if (dev->num_addrs) { int k; @@ -1079,7 +1077,7 @@ static __init int qemu_print_iodc_data(struct device *lin_dev, void *data) -static int print_one_device(struct device * dev, void * data) +static __init int print_one_device(struct device * dev, void * data) { struct parisc_device * pdev = to_parisc_device(dev); -- GitLab From 063b1f21cc9be07291a1f5e227436f353c6d1695 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sun, 30 Oct 2022 08:35:28 +0100 Subject: [PATCH 0786/1927] btrfs: fix a memory allocation failure test in btrfs_submit_direct After allocation 'dip' is tested instead of 'dip->csums'. Fix it. Fixes: 642c5d34da53 ("btrfs: allocate the btrfs_dio_private as part of the iomap dio bio") CC: stable@vger.kernel.org # 5.19+ Reviewed-by: Nikolay Borisov Signed-off-by: Christophe JAILLET Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/inode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 45ebef8d3ea8d..f0ebc17aeb672 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -7980,7 +7980,7 @@ static void btrfs_submit_direct(const struct iomap_iter *iter, */ status = BLK_STS_RESOURCE; dip->csums = kcalloc(nr_sectors, fs_info->csum_size, GFP_NOFS); - if (!dip) + if (!dip->csums) goto out_err; status = btrfs_lookup_bio_sums(inode, dio_bio, dip->csums); -- GitLab From 47ba8cc7b4f82c927cec3ad7c7392e4c45c81c56 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 24 Oct 2022 10:11:02 -0700 Subject: [PATCH 0787/1927] xfs: fix incorrect return type for fsdax fault handlers The kernel robot complained about this: >> fs/xfs/xfs_file.c:1266:31: sparse: sparse: incorrect type in return expression (different base types) @@ expected int @@ got restricted vm_fault_t @@ fs/xfs/xfs_file.c:1266:31: sparse: expected int fs/xfs/xfs_file.c:1266:31: sparse: got restricted vm_fault_t fs/xfs/xfs_file.c:1314:21: sparse: sparse: incorrect type in assignment (different base types) @@ expected restricted vm_fault_t [usertype] ret @@ got int @@ fs/xfs/xfs_file.c:1314:21: sparse: expected restricted vm_fault_t [usertype] ret fs/xfs/xfs_file.c:1314:21: sparse: got int Fix the incorrect return type for these two functions. While we're at it, make the !fsdax version return VM_FAULT_SIGBUS because a zero return value will cause some callers to try to lock vmf->page, which we never set here. Fixes: ea6c49b784f0 ("xfs: support CoW in fsdax mode") Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/xfs_file.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index c6c80265c0b25..e462d39c840e6 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -1261,7 +1261,7 @@ xfs_file_llseek( } #ifdef CONFIG_FS_DAX -static int +static inline vm_fault_t xfs_dax_fault( struct vm_fault *vmf, enum page_entry_size pe_size, @@ -1274,14 +1274,15 @@ xfs_dax_fault( &xfs_read_iomap_ops); } #else -static int +static inline vm_fault_t xfs_dax_fault( struct vm_fault *vmf, enum page_entry_size pe_size, bool write_fault, pfn_t *pfn) { - return 0; + ASSERT(0); + return VM_FAULT_SIGBUS; } #endif -- GitLab From 8184620ae21213d51eaf2e0bd4186baacb928172 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Fri, 28 Oct 2022 13:15:35 +0100 Subject: [PATCH 0788/1927] btrfs: fix lost file sync on direct IO write with nowait and dsync iocb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When doing a direct IO write using a iocb with nowait and dsync set, we end up not syncing the file once the write completes. This is because we tell iomap to not call generic_write_sync(), which would result in calling btrfs_sync_file(), in order to avoid a deadlock since iomap can call it while we are holding the inode's lock and btrfs_sync_file() needs to acquire the inode's lock. The deadlock happens only if the write happens synchronously, when iomap_dio_rw() calls iomap_dio_complete() before it returns. Instead we do the sync ourselves at btrfs_do_write_iter(). For a nowait write however we can end up not doing the sync ourselves at at btrfs_do_write_iter() because the write could have been queued, and therefore we get -EIOCBQUEUED returned from iomap in such case. That makes us skip the sync call at btrfs_do_write_iter(), as we don't do it for any error returned from btrfs_direct_write(). We can't simply do the call even if -EIOCBQUEUED is returned, since that would block the task waiting for IO, both for the data since there are bios still in progress as well as potentially blocking when joining a log transaction and when syncing the log (writing log trees, super blocks, etc). So let iomap do the sync call itself and in order to avoid deadlocks for the case of synchronous writes (without nowait), use __iomap_dio_rw() and have ourselves call iomap_dio_complete() after unlocking the inode. A test case will later be sent for fstests, after this is fixed in Linus' tree. Fixes: 51bd9563b678 ("btrfs: fix deadlock due to page faults during direct IO reads and writes") Reported-by: Марк Коренберг Link: https://lore.kernel.org/linux-btrfs/CAEmTpZGRKbzc16fWPvxbr6AfFsQoLmz-Lcg-7OgJOZDboJ+SGQ@mail.gmail.com/ CC: stable@vger.kernel.org # 6.0+ Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/ctree.h | 5 ++++- fs/btrfs/file.c | 22 ++++++++++++++++------ fs/btrfs/inode.c | 14 +++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 727595eee9732..f677b49df8ae0 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3462,7 +3462,10 @@ ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, const struct btrfs_ioctl_encoded_io_args *encoded); -ssize_t btrfs_dio_rw(struct kiocb *iocb, struct iov_iter *iter, size_t done_before); +ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, + size_t done_before); +struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter, + size_t done_before); extern const struct dentry_operations btrfs_dentry_operations; diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 176b432035aea..d55ad46384d17 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1765,6 +1765,7 @@ static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) loff_t endbyte; ssize_t err; unsigned int ilock_flags = 0; + struct iomap_dio *dio; if (iocb->ki_flags & IOCB_NOWAIT) ilock_flags |= BTRFS_ILOCK_TRY; @@ -1825,11 +1826,22 @@ static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) * So here we disable page faults in the iov_iter and then retry if we * got -EFAULT, faulting in the pages before the retry. */ -again: from->nofault = true; - err = btrfs_dio_rw(iocb, from, written); + dio = btrfs_dio_write(iocb, from, written); from->nofault = false; + /* + * iomap_dio_complete() will call btrfs_sync_file() if we have a dsync + * iocb, and that needs to lock the inode. So unlock it before calling + * iomap_dio_complete() to avoid a deadlock. + */ + btrfs_inode_unlock(inode, ilock_flags); + + if (IS_ERR_OR_NULL(dio)) + err = PTR_ERR_OR_ZERO(dio); + else + err = iomap_dio_complete(dio); + /* No increment (+=) because iomap returns a cumulative value. */ if (err > 0) written = err; @@ -1855,12 +1867,10 @@ static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) } else { fault_in_iov_iter_readable(from, left); prev_left = left; - goto again; + goto relock; } } - btrfs_inode_unlock(inode, ilock_flags); - /* * If 'err' is -ENOTBLK or we have not written all data, then it means * we must fallback to buffered IO. @@ -4035,7 +4045,7 @@ static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to) */ pagefault_disable(); to->nofault = true; - ret = btrfs_dio_rw(iocb, to, read); + ret = btrfs_dio_read(iocb, to, read); to->nofault = false; pagefault_enable(); diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index f0ebc17aeb672..d70f85b73169b 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -8078,13 +8078,21 @@ static const struct iomap_dio_ops btrfs_dio_ops = { .bio_set = &btrfs_dio_bioset, }; -ssize_t btrfs_dio_rw(struct kiocb *iocb, struct iov_iter *iter, size_t done_before) +ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, size_t done_before) { struct btrfs_dio_data data; return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, - IOMAP_DIO_PARTIAL | IOMAP_DIO_NOSYNC, - &data, done_before); + IOMAP_DIO_PARTIAL, &data, done_before); +} + +struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter, + size_t done_before) +{ + struct btrfs_dio_data data; + + return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + IOMAP_DIO_PARTIAL, &data, done_before); } static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, -- GitLab From 59da7ff49d67a1b63b1b81c7f53dcb6a84cdad2b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 20 Oct 2022 16:08:11 -0700 Subject: [PATCH 0789/1927] xfs: fix validation in attr log item recovery Before we start fixing all the complaints about memcpy'ing log items around, let's fix some inadequate validation in the xattr log item recovery code and get rid of the (now trivial) copy_format function. Signed-off-by: Darrick J. Wong Reviewed-by: Kees Cook Reviewed-by: Allison Henderson Reviewed-by: Dave Chinner --- fs/xfs/xfs_attr_item.c | 54 ++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index cf5ce607dc051..ee8f678a10a17 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -245,28 +245,6 @@ xfs_attri_init( return attrip; } -/* - * Copy an attr format buffer from the given buf, and into the destination attr - * format structure. - */ -STATIC int -xfs_attri_copy_format( - struct xfs_log_iovec *buf, - struct xfs_attri_log_format *dst_attr_fmt) -{ - struct xfs_attri_log_format *src_attr_fmt = buf->i_addr; - size_t len; - - len = sizeof(struct xfs_attri_log_format); - if (buf->i_len != len) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); - return -EFSCORRUPTED; - } - - memcpy((char *)dst_attr_fmt, (char *)src_attr_fmt, len); - return 0; -} - static inline struct xfs_attrd_log_item *ATTRD_ITEM(struct xfs_log_item *lip) { return container_of(lip, struct xfs_attrd_log_item, attrd_item); @@ -731,24 +709,44 @@ xlog_recover_attri_commit_pass2( struct xfs_attri_log_nameval *nv; const void *attr_value = NULL; const void *attr_name; - int error; + size_t len; attri_formatp = item->ri_buf[0].i_addr; attr_name = item->ri_buf[1].i_addr; /* Validate xfs_attri_log_format before the large memory allocation */ + len = sizeof(struct xfs_attri_log_format); + if (item->ri_buf[0].i_len != len) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + return -EFSCORRUPTED; + } + if (!xfs_attri_validate(mp, attri_formatp)) { XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); return -EFSCORRUPTED; } + /* Validate the attr name */ + if (item->ri_buf[1].i_len != + xlog_calc_iovec_len(attri_formatp->alfi_name_len)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + return -EFSCORRUPTED; + } + if (!xfs_attr_namecheck(attr_name, attri_formatp->alfi_name_len)) { XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); return -EFSCORRUPTED; } - if (attri_formatp->alfi_value_len) + /* Validate the attr value, if present */ + if (attri_formatp->alfi_value_len != 0) { + if (item->ri_buf[2].i_len != xlog_calc_iovec_len(attri_formatp->alfi_value_len)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + return -EFSCORRUPTED; + } + attr_value = item->ri_buf[2].i_addr; + } /* * Memory alloc failure will cause replay to abort. We attach the @@ -760,9 +758,7 @@ xlog_recover_attri_commit_pass2( attri_formatp->alfi_value_len); attrip = xfs_attri_init(mp, nv); - error = xfs_attri_copy_format(&item->ri_buf[0], &attrip->attri_format); - if (error) - goto out; + memcpy(&attrip->attri_format, attri_formatp, len); /* * The ATTRI has two references. One for the ATTRD and one for ATTRI to @@ -774,10 +770,6 @@ xlog_recover_attri_commit_pass2( xfs_attri_release(attrip); xfs_attri_log_nameval_put(nv); return 0; -out: - xfs_attri_item_free(attrip); - xfs_attri_log_nameval_put(nv); - return error; } /* -- GitLab From a38ebce1da271f480e47c3def4f810c6106b74a1 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 20 Oct 2022 16:17:41 -0700 Subject: [PATCH 0790/1927] xfs: fix memcpy fortify errors in BUI log format copying Starting in 6.1, CONFIG_FORTIFY_SOURCE checks the length parameter of memcpy. Unfortunately, it doesn't handle flex arrays correctly: ------------[ cut here ]------------ memcpy: detected field-spanning write (size 48) of single field "dst_bui_fmt" at fs/xfs/xfs_bmap_item.c:628 (size 16) Fix this by refactoring the xfs_bui_copy_format function to handle the copying of the head and the flex array members separately. While we're at it, fix a minor validation deficiency in the recovery function. Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Dave Chinner --- fs/xfs/xfs_bmap_item.c | 46 ++++++++++++++++++++---------------------- fs/xfs/xfs_ondisk.h | 5 +++++ 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index 51f66e9824846..a1da6205252ba 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -608,28 +608,18 @@ static const struct xfs_item_ops xfs_bui_item_ops = { .iop_relog = xfs_bui_item_relog, }; -/* - * Copy an BUI format buffer from the given buf, and into the destination - * BUI format structure. The BUI/BUD items were designed not to need any - * special alignment handling. - */ -static int +static inline void xfs_bui_copy_format( - struct xfs_log_iovec *buf, - struct xfs_bui_log_format *dst_bui_fmt) + struct xfs_bui_log_format *dst, + const struct xfs_bui_log_format *src) { - struct xfs_bui_log_format *src_bui_fmt; - uint len; + unsigned int i; - src_bui_fmt = buf->i_addr; - len = xfs_bui_log_format_sizeof(src_bui_fmt->bui_nextents); + memcpy(dst, src, offsetof(struct xfs_bui_log_format, bui_extents)); - if (buf->i_len == len) { - memcpy(dst_bui_fmt, src_bui_fmt, len); - return 0; - } - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); - return -EFSCORRUPTED; + for (i = 0; i < src->bui_nextents; i++) + memcpy(&dst->bui_extents[i], &src->bui_extents[i], + sizeof(struct xfs_map_extent)); } /* @@ -646,23 +636,31 @@ xlog_recover_bui_commit_pass2( struct xlog_recover_item *item, xfs_lsn_t lsn) { - int error; struct xfs_mount *mp = log->l_mp; struct xfs_bui_log_item *buip; struct xfs_bui_log_format *bui_formatp; + size_t len; bui_formatp = item->ri_buf[0].i_addr; + if (item->ri_buf[0].i_len < xfs_bui_log_format_sizeof(0)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + if (bui_formatp->bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); return -EFSCORRUPTED; } - buip = xfs_bui_init(mp); - error = xfs_bui_copy_format(&item->ri_buf[0], &buip->bui_format); - if (error) { - xfs_bui_item_free(buip); - return error; + + len = xfs_bui_log_format_sizeof(bui_formatp->bui_nextents); + if (item->ri_buf[0].i_len != len) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; } + + buip = xfs_bui_init(mp); + xfs_bui_copy_format(&buip->bui_format, bui_formatp); atomic_set(&buip->bui_next_extent, bui_formatp->bui_nextents); /* * Insert the intent into the AIL directly and drop one reference so diff --git a/fs/xfs/xfs_ondisk.h b/fs/xfs/xfs_ondisk.h index 758702b9495ff..56917e2363702 100644 --- a/fs/xfs/xfs_ondisk.h +++ b/fs/xfs/xfs_ondisk.h @@ -134,6 +134,11 @@ xfs_check_ondisk_structs(void) XFS_CHECK_STRUCT_SIZE(struct xfs_trans_header, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_attri_log_format, 40); XFS_CHECK_STRUCT_SIZE(struct xfs_attrd_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_bui_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_bud_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_map_extent, 32); + + XFS_CHECK_OFFSET(struct xfs_bui_log_format, bui_extents, 16); /* * The v5 superblock format extended several v4 header structures with -- GitLab From a38935c03c7914a6ab22eefb750b259868ed5a4b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 20 Oct 2022 16:23:16 -0700 Subject: [PATCH 0791/1927] xfs: fix memcpy fortify errors in CUI log format copying Starting in 6.1, CONFIG_FORTIFY_SOURCE checks the length parameter of memcpy. Since we're already fixing problems with BUI item copying, we should fix it everything else. Refactor the xfs_cui_copy_format function to handle the copying of the head and the flex array members separately. While we're at it, fix a minor validation deficiency in the recovery function. Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Dave Chinner --- fs/xfs/xfs_ondisk.h | 4 ++++ fs/xfs/xfs_refcount_item.c | 45 ++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/fs/xfs/xfs_ondisk.h b/fs/xfs/xfs_ondisk.h index 56917e2363702..e20d2844b0c53 100644 --- a/fs/xfs/xfs_ondisk.h +++ b/fs/xfs/xfs_ondisk.h @@ -136,9 +136,13 @@ xfs_check_ondisk_structs(void) XFS_CHECK_STRUCT_SIZE(struct xfs_attrd_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_bui_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_bud_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_cui_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_cud_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_map_extent, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_phys_extent, 16); XFS_CHECK_OFFSET(struct xfs_bui_log_format, bui_extents, 16); + XFS_CHECK_OFFSET(struct xfs_cui_log_format, cui_extents, 16); /* * The v5 superblock format extended several v4 header structures with diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 7e97bf19793df..24cf4c64ebaab 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -622,28 +622,18 @@ static const struct xfs_item_ops xfs_cui_item_ops = { .iop_relog = xfs_cui_item_relog, }; -/* - * Copy an CUI format buffer from the given buf, and into the destination - * CUI format structure. The CUI/CUD items were designed not to need any - * special alignment handling. - */ -static int +static inline void xfs_cui_copy_format( - struct xfs_log_iovec *buf, - struct xfs_cui_log_format *dst_cui_fmt) + struct xfs_cui_log_format *dst, + const struct xfs_cui_log_format *src) { - struct xfs_cui_log_format *src_cui_fmt; - uint len; + unsigned int i; - src_cui_fmt = buf->i_addr; - len = xfs_cui_log_format_sizeof(src_cui_fmt->cui_nextents); + memcpy(dst, src, offsetof(struct xfs_cui_log_format, cui_extents)); - if (buf->i_len == len) { - memcpy(dst_cui_fmt, src_cui_fmt, len); - return 0; - } - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); - return -EFSCORRUPTED; + for (i = 0; i < src->cui_nextents; i++) + memcpy(&dst->cui_extents[i], &src->cui_extents[i], + sizeof(struct xfs_phys_extent)); } /* @@ -660,19 +650,26 @@ xlog_recover_cui_commit_pass2( struct xlog_recover_item *item, xfs_lsn_t lsn) { - int error; struct xfs_mount *mp = log->l_mp; struct xfs_cui_log_item *cuip; struct xfs_cui_log_format *cui_formatp; + size_t len; cui_formatp = item->ri_buf[0].i_addr; - cuip = xfs_cui_init(mp, cui_formatp->cui_nextents); - error = xfs_cui_copy_format(&item->ri_buf[0], &cuip->cui_format); - if (error) { - xfs_cui_item_free(cuip); - return error; + if (item->ri_buf[0].i_len < xfs_cui_log_format_sizeof(0)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; } + + len = xfs_cui_log_format_sizeof(cui_formatp->cui_nextents); + if (item->ri_buf[0].i_len != len) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + cuip = xfs_cui_init(mp, cui_formatp->cui_nextents); + xfs_cui_copy_format(&cuip->cui_format, cui_formatp); atomic_set(&cuip->cui_next_extent, cui_formatp->cui_nextents); /* * Insert the intent into the AIL directly and drop one reference so -- GitLab From b45ca961e94673df83ab1900802afe82776966e6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 20 Oct 2022 16:26:36 -0700 Subject: [PATCH 0792/1927] xfs: fix memcpy fortify errors in RUI log format copying Starting in 6.1, CONFIG_FORTIFY_SOURCE checks the length parameter of memcpy. Since we're already fixing problems with BUI item copying, we should fix it everything else. Refactor the xfs_rui_copy_format function to handle the copying of the head and the flex array members separately. While we're at it, fix a minor validation deficiency in the recovery function. Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Dave Chinner --- fs/xfs/xfs_ondisk.h | 3 +++ fs/xfs/xfs_rmap_item.c | 58 ++++++++++++++++++++---------------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/fs/xfs/xfs_ondisk.h b/fs/xfs/xfs_ondisk.h index e20d2844b0c53..19c1df00b48e8 100644 --- a/fs/xfs/xfs_ondisk.h +++ b/fs/xfs/xfs_ondisk.h @@ -138,11 +138,14 @@ xfs_check_ondisk_structs(void) XFS_CHECK_STRUCT_SIZE(struct xfs_bud_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_cui_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_cud_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_rui_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_rud_log_format, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_map_extent, 32); XFS_CHECK_STRUCT_SIZE(struct xfs_phys_extent, 16); XFS_CHECK_OFFSET(struct xfs_bui_log_format, bui_extents, 16); XFS_CHECK_OFFSET(struct xfs_cui_log_format, cui_extents, 16); + XFS_CHECK_OFFSET(struct xfs_rui_log_format, rui_extents, 16); /* * The v5 superblock format extended several v4 header structures with diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index fef92e02f3bb6..27047e73f5829 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -155,31 +155,6 @@ xfs_rui_init( return ruip; } -/* - * Copy an RUI format buffer from the given buf, and into the destination - * RUI format structure. The RUI/RUD items were designed not to need any - * special alignment handling. - */ -STATIC int -xfs_rui_copy_format( - struct xfs_log_iovec *buf, - struct xfs_rui_log_format *dst_rui_fmt) -{ - struct xfs_rui_log_format *src_rui_fmt; - uint len; - - src_rui_fmt = buf->i_addr; - len = xfs_rui_log_format_sizeof(src_rui_fmt->rui_nextents); - - if (buf->i_len != len) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); - return -EFSCORRUPTED; - } - - memcpy(dst_rui_fmt, src_rui_fmt, len); - return 0; -} - static inline struct xfs_rud_log_item *RUD_ITEM(struct xfs_log_item *lip) { return container_of(lip, struct xfs_rud_log_item, rud_item); @@ -652,6 +627,20 @@ static const struct xfs_item_ops xfs_rui_item_ops = { .iop_relog = xfs_rui_item_relog, }; +static inline void +xfs_rui_copy_format( + struct xfs_rui_log_format *dst, + const struct xfs_rui_log_format *src) +{ + unsigned int i; + + memcpy(dst, src, offsetof(struct xfs_rui_log_format, rui_extents)); + + for (i = 0; i < src->rui_nextents; i++) + memcpy(&dst->rui_extents[i], &src->rui_extents[i], + sizeof(struct xfs_map_extent)); +} + /* * This routine is called to create an in-core extent rmap update * item from the rui format structure which was logged on disk. @@ -666,19 +655,26 @@ xlog_recover_rui_commit_pass2( struct xlog_recover_item *item, xfs_lsn_t lsn) { - int error; struct xfs_mount *mp = log->l_mp; struct xfs_rui_log_item *ruip; struct xfs_rui_log_format *rui_formatp; + size_t len; rui_formatp = item->ri_buf[0].i_addr; - ruip = xfs_rui_init(mp, rui_formatp->rui_nextents); - error = xfs_rui_copy_format(&item->ri_buf[0], &ruip->rui_format); - if (error) { - xfs_rui_item_free(ruip); - return error; + if (item->ri_buf[0].i_len < xfs_rui_log_format_sizeof(0)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; } + + len = xfs_rui_log_format_sizeof(rui_formatp->rui_nextents); + if (item->ri_buf[0].i_len != len) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + ruip = xfs_rui_init(mp, rui_formatp->rui_nextents); + xfs_rui_copy_format(&ruip->rui_format, rui_formatp); atomic_set(&ruip->rui_next_extent, rui_formatp->rui_nextents); /* * Insert the intent into the AIL directly and drop one reference so -- GitLab From f850995f60e49818093ef5e477cdb0ff2c11a0a4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 25 Oct 2022 18:18:21 -0700 Subject: [PATCH 0793/1927] xfs: make sure aglen never goes negative in xfs_refcount_adjust_extents Prior to calling xfs_refcount_adjust_extents, we trimmed agbno/aglen such that the end of the range would not be in the middle of a refcount record. If this is no longer the case, something is seriously wrong with the btree. Bail out with a corruption error. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 64b910caafaad..831353ba96dc8 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -986,15 +986,29 @@ xfs_refcount_adjust_extents( (*agbno) += tmp.rc_blockcount; (*aglen) -= tmp.rc_blockcount; + /* Stop if there's nothing left to modify */ + if (*aglen == 0 || !xfs_refcount_still_have_space(cur)) + break; + + /* Move the cursor to the start of ext. */ error = xfs_refcount_lookup_ge(cur, *agbno, &found_rec); if (error) goto out_error; } - /* Stop if there's nothing left to modify */ - if (*aglen == 0 || !xfs_refcount_still_have_space(cur)) - break; + /* + * A previous step trimmed agbno/aglen such that the end of the + * range would not be in the middle of the record. If this is + * no longer the case, something is seriously wrong with the + * btree. Make sure we never feed the synthesized record into + * the processing loop below. + */ + if (XFS_IS_CORRUPT(cur->bc_mp, ext.rc_blockcount == 0) || + XFS_IS_CORRUPT(cur->bc_mp, ext.rc_blockcount > *aglen)) { + error = -EFSCORRUPTED; + goto out_error; + } /* * Adjust the reference count and either update the tree -- GitLab From 03a7485cd701e1c08baadcf39d9592d83715e224 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 20 Oct 2022 16:39:59 -0700 Subject: [PATCH 0794/1927] xfs: fix memcpy fortify errors in EFI log format copying Starting in 6.1, CONFIG_FORTIFY_SOURCE checks the length parameter of memcpy. Since we're already fixing problems with BUI item copying, we should fix it everything else. An extra difficulty here is that the ef[id]_extents arrays are declared as single-element arrays. This is not the convention for flex arrays in the modern kernel, and it causes all manner of problems with static checking tools, since they often cannot tell the difference between a single element array and a flex array. So for starters, change those array[1] declarations to array[] declarations to signal that they are proper flex arrays and adjust all the "size-1" expressions to fit the new declaration style. Next, refactor the xfs_efi_copy_format function to handle the copying of the head and the flex array members separately. While we're at it, fix a minor validation deficiency in the recovery function. Signed-off-by: Darrick J. Wong Reviewed-by: Kees Cook Reviewed-by: Allison Henderson Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_log_format.h | 12 ++++++------ fs/xfs/xfs_extfree_item.c | 31 +++++++++++++++++++++---------- fs/xfs/xfs_ondisk.h | 11 +++++++---- fs/xfs/xfs_super.c | 4 ++-- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index b351b9dc65618..2f41fa8477c9d 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -613,7 +613,7 @@ typedef struct xfs_efi_log_format { uint16_t efi_size; /* size of this item */ uint32_t efi_nextents; /* # extents to free */ uint64_t efi_id; /* efi identifier */ - xfs_extent_t efi_extents[1]; /* array of extents to free */ + xfs_extent_t efi_extents[]; /* array of extents to free */ } xfs_efi_log_format_t; typedef struct xfs_efi_log_format_32 { @@ -621,7 +621,7 @@ typedef struct xfs_efi_log_format_32 { uint16_t efi_size; /* size of this item */ uint32_t efi_nextents; /* # extents to free */ uint64_t efi_id; /* efi identifier */ - xfs_extent_32_t efi_extents[1]; /* array of extents to free */ + xfs_extent_32_t efi_extents[]; /* array of extents to free */ } __attribute__((packed)) xfs_efi_log_format_32_t; typedef struct xfs_efi_log_format_64 { @@ -629,7 +629,7 @@ typedef struct xfs_efi_log_format_64 { uint16_t efi_size; /* size of this item */ uint32_t efi_nextents; /* # extents to free */ uint64_t efi_id; /* efi identifier */ - xfs_extent_64_t efi_extents[1]; /* array of extents to free */ + xfs_extent_64_t efi_extents[]; /* array of extents to free */ } xfs_efi_log_format_64_t; /* @@ -642,7 +642,7 @@ typedef struct xfs_efd_log_format { uint16_t efd_size; /* size of this item */ uint32_t efd_nextents; /* # of extents freed */ uint64_t efd_efi_id; /* id of corresponding efi */ - xfs_extent_t efd_extents[1]; /* array of extents freed */ + xfs_extent_t efd_extents[]; /* array of extents freed */ } xfs_efd_log_format_t; typedef struct xfs_efd_log_format_32 { @@ -650,7 +650,7 @@ typedef struct xfs_efd_log_format_32 { uint16_t efd_size; /* size of this item */ uint32_t efd_nextents; /* # of extents freed */ uint64_t efd_efi_id; /* id of corresponding efi */ - xfs_extent_32_t efd_extents[1]; /* array of extents freed */ + xfs_extent_32_t efd_extents[]; /* array of extents freed */ } __attribute__((packed)) xfs_efd_log_format_32_t; typedef struct xfs_efd_log_format_64 { @@ -658,7 +658,7 @@ typedef struct xfs_efd_log_format_64 { uint16_t efd_size; /* size of this item */ uint32_t efd_nextents; /* # of extents freed */ uint64_t efd_efi_id; /* id of corresponding efi */ - xfs_extent_64_t efd_extents[1]; /* array of extents freed */ + xfs_extent_64_t efd_extents[]; /* array of extents freed */ } xfs_efd_log_format_64_t; /* diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 27ccfcd82f042..466cc5c5cd33f 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -76,7 +76,7 @@ xfs_efi_item_sizeof( struct xfs_efi_log_item *efip) { return sizeof(struct xfs_efi_log_format) + - (efip->efi_format.efi_nextents - 1) * sizeof(xfs_extent_t); + efip->efi_format.efi_nextents * sizeof(xfs_extent_t); } STATIC void @@ -160,7 +160,7 @@ xfs_efi_init( ASSERT(nextents > 0); if (nextents > XFS_EFI_MAX_FAST_EXTENTS) { size = (uint)(sizeof(struct xfs_efi_log_item) + - ((nextents - 1) * sizeof(xfs_extent_t))); + (nextents * sizeof(xfs_extent_t))); efip = kmem_zalloc(size, 0); } else { efip = kmem_cache_zalloc(xfs_efi_cache, @@ -189,14 +189,19 @@ xfs_efi_copy_format(xfs_log_iovec_t *buf, xfs_efi_log_format_t *dst_efi_fmt) xfs_efi_log_format_t *src_efi_fmt = buf->i_addr; uint i; uint len = sizeof(xfs_efi_log_format_t) + - (src_efi_fmt->efi_nextents - 1) * sizeof(xfs_extent_t); + src_efi_fmt->efi_nextents * sizeof(xfs_extent_t); uint len32 = sizeof(xfs_efi_log_format_32_t) + - (src_efi_fmt->efi_nextents - 1) * sizeof(xfs_extent_32_t); + src_efi_fmt->efi_nextents * sizeof(xfs_extent_32_t); uint len64 = sizeof(xfs_efi_log_format_64_t) + - (src_efi_fmt->efi_nextents - 1) * sizeof(xfs_extent_64_t); + src_efi_fmt->efi_nextents * sizeof(xfs_extent_64_t); if (buf->i_len == len) { - memcpy((char *)dst_efi_fmt, (char*)src_efi_fmt, len); + memcpy(dst_efi_fmt, src_efi_fmt, + offsetof(struct xfs_efi_log_format, efi_extents)); + for (i = 0; i < src_efi_fmt->efi_nextents; i++) + memcpy(&dst_efi_fmt->efi_extents[i], + &src_efi_fmt->efi_extents[i], + sizeof(struct xfs_extent)); return 0; } else if (buf->i_len == len32) { xfs_efi_log_format_32_t *src_efi_fmt_32 = buf->i_addr; @@ -256,7 +261,7 @@ xfs_efd_item_sizeof( struct xfs_efd_log_item *efdp) { return sizeof(xfs_efd_log_format_t) + - (efdp->efd_format.efd_nextents - 1) * sizeof(xfs_extent_t); + efdp->efd_format.efd_nextents * sizeof(xfs_extent_t); } STATIC void @@ -341,7 +346,7 @@ xfs_trans_get_efd( if (nextents > XFS_EFD_MAX_FAST_EXTENTS) { efdp = kmem_zalloc(sizeof(struct xfs_efd_log_item) + - (nextents - 1) * sizeof(struct xfs_extent), + nextents * sizeof(struct xfs_extent), 0); } else { efdp = kmem_cache_zalloc(xfs_efd_cache, @@ -733,6 +738,12 @@ xlog_recover_efi_commit_pass2( efi_formatp = item->ri_buf[0].i_addr; + if (item->ri_buf[0].i_len < + offsetof(struct xfs_efi_log_format, efi_extents)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + efip = xfs_efi_init(mp, efi_formatp->efi_nextents); error = xfs_efi_copy_format(&item->ri_buf[0], &efip->efi_format); if (error) { @@ -772,9 +783,9 @@ xlog_recover_efd_commit_pass2( efd_formatp = item->ri_buf[0].i_addr; ASSERT((item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_32_t) + - ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_32_t)))) || + (efd_formatp->efd_nextents * sizeof(xfs_extent_32_t)))) || (item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_64_t) + - ((efd_formatp->efd_nextents - 1) * sizeof(xfs_extent_64_t))))); + (efd_formatp->efd_nextents * sizeof(xfs_extent_64_t))))); xlog_recover_release_intent(log, XFS_LI_EFI, efd_formatp->efd_efi_id); return 0; diff --git a/fs/xfs/xfs_ondisk.h b/fs/xfs/xfs_ondisk.h index 19c1df00b48e8..9737b5a9f405e 100644 --- a/fs/xfs/xfs_ondisk.h +++ b/fs/xfs/xfs_ondisk.h @@ -118,10 +118,10 @@ xfs_check_ondisk_structs(void) /* log structures */ XFS_CHECK_STRUCT_SIZE(struct xfs_buf_log_format, 88); XFS_CHECK_STRUCT_SIZE(struct xfs_dq_logformat, 24); - XFS_CHECK_STRUCT_SIZE(struct xfs_efd_log_format_32, 28); - XFS_CHECK_STRUCT_SIZE(struct xfs_efd_log_format_64, 32); - XFS_CHECK_STRUCT_SIZE(struct xfs_efi_log_format_32, 28); - XFS_CHECK_STRUCT_SIZE(struct xfs_efi_log_format_64, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_efd_log_format_32, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_efd_log_format_64, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_efi_log_format_32, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_efi_log_format_64, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_extent_32, 12); XFS_CHECK_STRUCT_SIZE(struct xfs_extent_64, 16); XFS_CHECK_STRUCT_SIZE(struct xfs_log_dinode, 176); @@ -146,6 +146,9 @@ xfs_check_ondisk_structs(void) XFS_CHECK_OFFSET(struct xfs_bui_log_format, bui_extents, 16); XFS_CHECK_OFFSET(struct xfs_cui_log_format, cui_extents, 16); XFS_CHECK_OFFSET(struct xfs_rui_log_format, rui_extents, 16); + XFS_CHECK_OFFSET(struct xfs_efi_log_format, efi_extents, 16); + XFS_CHECK_OFFSET(struct xfs_efi_log_format_32, efi_extents, 16); + XFS_CHECK_OFFSET(struct xfs_efi_log_format_64, efi_extents, 16); /* * The v5 superblock format extended several v4 header structures with diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index f029c6702dda1..8485e3b37ca01 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -2029,7 +2029,7 @@ xfs_init_caches(void) xfs_efd_cache = kmem_cache_create("xfs_efd_item", (sizeof(struct xfs_efd_log_item) + - (XFS_EFD_MAX_FAST_EXTENTS - 1) * + XFS_EFD_MAX_FAST_EXTENTS * sizeof(struct xfs_extent)), 0, 0, NULL); if (!xfs_efd_cache) @@ -2037,7 +2037,7 @@ xfs_init_caches(void) xfs_efi_cache = kmem_cache_create("xfs_efi_item", (sizeof(struct xfs_efi_log_item) + - (XFS_EFI_MAX_FAST_EXTENTS - 1) * + XFS_EFI_MAX_FAST_EXTENTS * sizeof(struct xfs_extent)), 0, 0, NULL); if (!xfs_efi_cache) -- GitLab From b65e08f83b119ae9345ed23d4da357a72b3cb55c Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 28 Oct 2022 15:48:58 -0700 Subject: [PATCH 0795/1927] xfs: create a predicate to verify per-AG extents Create a predicate function to verify that a given agbno/blockcount pair fit entirely within a single allocation group and don't suffer mathematical overflows. Refactor the existng open-coded logic; we're going to add more calls to this function in the next patch. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_ag.h | 15 +++++++++++++++ fs/xfs/libxfs/xfs_alloc.c | 6 +----- fs/xfs/libxfs/xfs_refcount.c | 6 +----- fs/xfs/libxfs/xfs_rmap.c | 9 ++------- fs/xfs/scrub/alloc.c | 4 +--- fs/xfs/scrub/ialloc.c | 5 ++--- fs/xfs/scrub/refcount.c | 5 ++--- 7 files changed, 24 insertions(+), 26 deletions(-) diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h index 517a138faa669..191b22b9a35bf 100644 --- a/fs/xfs/libxfs/xfs_ag.h +++ b/fs/xfs/libxfs/xfs_ag.h @@ -133,6 +133,21 @@ xfs_verify_agbno(struct xfs_perag *pag, xfs_agblock_t agbno) return true; } +static inline bool +xfs_verify_agbext( + struct xfs_perag *pag, + xfs_agblock_t agbno, + xfs_agblock_t len) +{ + if (agbno + len <= agbno) + return false; + + if (!xfs_verify_agbno(pag, agbno)) + return false; + + return xfs_verify_agbno(pag, agbno + len - 1); +} + /* * Verify that an AG inode number pointer neither points outside the AG * nor points at static metadata. diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index 6261599bb389a..de79f5d07f651 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -263,11 +263,7 @@ xfs_alloc_get_rec( goto out_bad_rec; /* check for valid extent range, including overflow */ - if (!xfs_verify_agbno(pag, *bno)) - goto out_bad_rec; - if (*bno > *bno + *len) - goto out_bad_rec; - if (!xfs_verify_agbno(pag, *bno + *len - 1)) + if (!xfs_verify_agbext(pag, *bno, *len)) goto out_bad_rec; return 0; diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 831353ba96dc8..1a50ca53304a2 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -135,11 +135,7 @@ xfs_refcount_get_rec( } /* check for valid extent range, including overflow */ - if (!xfs_verify_agbno(pag, realstart)) - goto out_bad_rec; - if (realstart > realstart + irec->rc_blockcount) - goto out_bad_rec; - if (!xfs_verify_agbno(pag, realstart + irec->rc_blockcount - 1)) + if (!xfs_verify_agbext(pag, realstart, irec->rc_blockcount)) goto out_bad_rec; if (irec->rc_refcount == 0 || irec->rc_refcount > MAXREFCOUNT) diff --git a/fs/xfs/libxfs/xfs_rmap.c b/fs/xfs/libxfs/xfs_rmap.c index 094dfc897ebcd..b56aca1e7c66c 100644 --- a/fs/xfs/libxfs/xfs_rmap.c +++ b/fs/xfs/libxfs/xfs_rmap.c @@ -235,13 +235,8 @@ xfs_rmap_get_rec( goto out_bad_rec; } else { /* check for valid extent range, including overflow */ - if (!xfs_verify_agbno(pag, irec->rm_startblock)) - goto out_bad_rec; - if (irec->rm_startblock > - irec->rm_startblock + irec->rm_blockcount) - goto out_bad_rec; - if (!xfs_verify_agbno(pag, - irec->rm_startblock + irec->rm_blockcount - 1)) + if (!xfs_verify_agbext(pag, irec->rm_startblock, + irec->rm_blockcount)) goto out_bad_rec; } diff --git a/fs/xfs/scrub/alloc.c b/fs/xfs/scrub/alloc.c index ab427b4d7fe0b..3b38f4e2a5373 100644 --- a/fs/xfs/scrub/alloc.c +++ b/fs/xfs/scrub/alloc.c @@ -100,9 +100,7 @@ xchk_allocbt_rec( bno = be32_to_cpu(rec->alloc.ar_startblock); len = be32_to_cpu(rec->alloc.ar_blockcount); - if (bno + len <= bno || - !xfs_verify_agbno(pag, bno) || - !xfs_verify_agbno(pag, bno + len - 1)) + if (!xfs_verify_agbext(pag, bno, len)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); xchk_allocbt_xref(bs->sc, bno, len); diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index e1026e07bf946..e312be7cd3751 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -108,9 +108,8 @@ xchk_iallocbt_chunk( xfs_agblock_t bno; bno = XFS_AGINO_TO_AGBNO(mp, agino); - if (bno + len <= bno || - !xfs_verify_agbno(pag, bno) || - !xfs_verify_agbno(pag, bno + len - 1)) + + if (!xfs_verify_agbext(pag, bno, len)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); xchk_iallocbt_chunk_xref(bs->sc, irec, agino, bno, len); diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index c68b767dc08fe..9959397f797f1 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -354,9 +354,8 @@ xchk_refcountbt_rec( /* Check the extent. */ bno &= ~XFS_REFC_COW_START; - if (bno + len <= bno || - !xfs_verify_agbno(pag, bno) || - !xfs_verify_agbno(pag, bno + len - 1)) + + if (!xfs_verify_agbext(pag, bno, len)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); if (refcount == 0) -- GitLab From 3c5aaaced99912c9fb3352fc5af5b104df67d4aa Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Fri, 21 Oct 2022 09:10:05 -0700 Subject: [PATCH 0796/1927] xfs: refactor all the EFI/EFD log item sizeof logic Refactor all the open-coded sizeof logic for EFI/EFD log item and log format structures into common helper functions whose names reflect the struct names. Signed-off-by: Darrick J. Wong Reviewed-by: Allison Henderson Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_log_format.h | 48 +++++++++++++++++++++++ fs/xfs/xfs_extfree_item.c | 69 ++++++++++------------------------ fs/xfs/xfs_extfree_item.h | 16 ++++++++ fs/xfs/xfs_super.c | 12 ++---- 4 files changed, 88 insertions(+), 57 deletions(-) diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h index 2f41fa8477c9d..f13e0809dc63f 100644 --- a/fs/xfs/libxfs/xfs_log_format.h +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -616,6 +616,14 @@ typedef struct xfs_efi_log_format { xfs_extent_t efi_extents[]; /* array of extents to free */ } xfs_efi_log_format_t; +static inline size_t +xfs_efi_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efi_log_format) + + nr * sizeof(struct xfs_extent); +} + typedef struct xfs_efi_log_format_32 { uint16_t efi_type; /* efi log item type */ uint16_t efi_size; /* size of this item */ @@ -624,6 +632,14 @@ typedef struct xfs_efi_log_format_32 { xfs_extent_32_t efi_extents[]; /* array of extents to free */ } __attribute__((packed)) xfs_efi_log_format_32_t; +static inline size_t +xfs_efi_log_format32_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efi_log_format_32) + + nr * sizeof(struct xfs_extent_32); +} + typedef struct xfs_efi_log_format_64 { uint16_t efi_type; /* efi log item type */ uint16_t efi_size; /* size of this item */ @@ -632,6 +648,14 @@ typedef struct xfs_efi_log_format_64 { xfs_extent_64_t efi_extents[]; /* array of extents to free */ } xfs_efi_log_format_64_t; +static inline size_t +xfs_efi_log_format64_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efi_log_format_64) + + nr * sizeof(struct xfs_extent_64); +} + /* * This is the structure used to lay out an efd log item in the * log. The efd_extents array is a variable size array whose @@ -645,6 +669,14 @@ typedef struct xfs_efd_log_format { xfs_extent_t efd_extents[]; /* array of extents freed */ } xfs_efd_log_format_t; +static inline size_t +xfs_efd_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efd_log_format) + + nr * sizeof(struct xfs_extent); +} + typedef struct xfs_efd_log_format_32 { uint16_t efd_type; /* efd log item type */ uint16_t efd_size; /* size of this item */ @@ -653,6 +685,14 @@ typedef struct xfs_efd_log_format_32 { xfs_extent_32_t efd_extents[]; /* array of extents freed */ } __attribute__((packed)) xfs_efd_log_format_32_t; +static inline size_t +xfs_efd_log_format32_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efd_log_format_32) + + nr * sizeof(struct xfs_extent_32); +} + typedef struct xfs_efd_log_format_64 { uint16_t efd_type; /* efd log item type */ uint16_t efd_size; /* size of this item */ @@ -661,6 +701,14 @@ typedef struct xfs_efd_log_format_64 { xfs_extent_64_t efd_extents[]; /* array of extents freed */ } xfs_efd_log_format_64_t; +static inline size_t +xfs_efd_log_format64_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efd_log_format_64) + + nr * sizeof(struct xfs_extent_64); +} + /* * RUI/RUD (reverse mapping) log format definitions */ diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 466cc5c5cd33f..f7e52db8da667 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -66,27 +66,16 @@ xfs_efi_release( xfs_efi_item_free(efip); } -/* - * This returns the number of iovecs needed to log the given efi item. - * We only need 1 iovec for an efi item. It just logs the efi_log_format - * structure. - */ -static inline int -xfs_efi_item_sizeof( - struct xfs_efi_log_item *efip) -{ - return sizeof(struct xfs_efi_log_format) + - efip->efi_format.efi_nextents * sizeof(xfs_extent_t); -} - STATIC void xfs_efi_item_size( struct xfs_log_item *lip, int *nvecs, int *nbytes) { + struct xfs_efi_log_item *efip = EFI_ITEM(lip); + *nvecs += 1; - *nbytes += xfs_efi_item_sizeof(EFI_ITEM(lip)); + *nbytes += xfs_efi_log_format_sizeof(efip->efi_format.efi_nextents); } /* @@ -112,7 +101,7 @@ xfs_efi_item_format( xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_EFI_FORMAT, &efip->efi_format, - xfs_efi_item_sizeof(efip)); + xfs_efi_log_format_sizeof(efip->efi_format.efi_nextents)); } @@ -155,13 +144,11 @@ xfs_efi_init( { struct xfs_efi_log_item *efip; - uint size; ASSERT(nextents > 0); if (nextents > XFS_EFI_MAX_FAST_EXTENTS) { - size = (uint)(sizeof(struct xfs_efi_log_item) + - (nextents * sizeof(xfs_extent_t))); - efip = kmem_zalloc(size, 0); + efip = kzalloc(xfs_efi_log_item_sizeof(nextents), + GFP_KERNEL | __GFP_NOFAIL); } else { efip = kmem_cache_zalloc(xfs_efi_cache, GFP_KERNEL | __GFP_NOFAIL); @@ -188,12 +175,9 @@ xfs_efi_copy_format(xfs_log_iovec_t *buf, xfs_efi_log_format_t *dst_efi_fmt) { xfs_efi_log_format_t *src_efi_fmt = buf->i_addr; uint i; - uint len = sizeof(xfs_efi_log_format_t) + - src_efi_fmt->efi_nextents * sizeof(xfs_extent_t); - uint len32 = sizeof(xfs_efi_log_format_32_t) + - src_efi_fmt->efi_nextents * sizeof(xfs_extent_32_t); - uint len64 = sizeof(xfs_efi_log_format_64_t) + - src_efi_fmt->efi_nextents * sizeof(xfs_extent_64_t); + uint len = xfs_efi_log_format_sizeof(src_efi_fmt->efi_nextents); + uint len32 = xfs_efi_log_format32_sizeof(src_efi_fmt->efi_nextents); + uint len64 = xfs_efi_log_format64_sizeof(src_efi_fmt->efi_nextents); if (buf->i_len == len) { memcpy(dst_efi_fmt, src_efi_fmt, @@ -251,27 +235,16 @@ xfs_efd_item_free(struct xfs_efd_log_item *efdp) kmem_cache_free(xfs_efd_cache, efdp); } -/* - * This returns the number of iovecs needed to log the given efd item. - * We only need 1 iovec for an efd item. It just logs the efd_log_format - * structure. - */ -static inline int -xfs_efd_item_sizeof( - struct xfs_efd_log_item *efdp) -{ - return sizeof(xfs_efd_log_format_t) + - efdp->efd_format.efd_nextents * sizeof(xfs_extent_t); -} - STATIC void xfs_efd_item_size( struct xfs_log_item *lip, int *nvecs, int *nbytes) { + struct xfs_efd_log_item *efdp = EFD_ITEM(lip); + *nvecs += 1; - *nbytes += xfs_efd_item_sizeof(EFD_ITEM(lip)); + *nbytes += xfs_efd_log_format_sizeof(efdp->efd_format.efd_nextents); } /* @@ -296,7 +269,7 @@ xfs_efd_item_format( xlog_copy_iovec(lv, &vecp, XLOG_REG_TYPE_EFD_FORMAT, &efdp->efd_format, - xfs_efd_item_sizeof(efdp)); + xfs_efd_log_format_sizeof(efdp->efd_format.efd_nextents)); } /* @@ -345,9 +318,8 @@ xfs_trans_get_efd( ASSERT(nextents > 0); if (nextents > XFS_EFD_MAX_FAST_EXTENTS) { - efdp = kmem_zalloc(sizeof(struct xfs_efd_log_item) + - nextents * sizeof(struct xfs_extent), - 0); + efdp = kzalloc(xfs_efd_log_item_sizeof(nextents), + GFP_KERNEL | __GFP_NOFAIL); } else { efdp = kmem_cache_zalloc(xfs_efd_cache, GFP_KERNEL | __GFP_NOFAIL); @@ -738,8 +710,7 @@ xlog_recover_efi_commit_pass2( efi_formatp = item->ri_buf[0].i_addr; - if (item->ri_buf[0].i_len < - offsetof(struct xfs_efi_log_format, efi_extents)) { + if (item->ri_buf[0].i_len < xfs_efi_log_format_sizeof(0)) { XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); return -EFSCORRUPTED; } @@ -782,10 +753,10 @@ xlog_recover_efd_commit_pass2( struct xfs_efd_log_format *efd_formatp; efd_formatp = item->ri_buf[0].i_addr; - ASSERT((item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_32_t) + - (efd_formatp->efd_nextents * sizeof(xfs_extent_32_t)))) || - (item->ri_buf[0].i_len == (sizeof(xfs_efd_log_format_64_t) + - (efd_formatp->efd_nextents * sizeof(xfs_extent_64_t))))); + ASSERT(item->ri_buf[0].i_len == xfs_efd_log_format32_sizeof( + efd_formatp->efd_nextents) || + item->ri_buf[0].i_len == xfs_efd_log_format64_sizeof( + efd_formatp->efd_nextents)); xlog_recover_release_intent(log, XFS_LI_EFI, efd_formatp->efd_efi_id); return 0; diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h index 186d0f2137f1f..da6a5afa607cf 100644 --- a/fs/xfs/xfs_extfree_item.h +++ b/fs/xfs/xfs_extfree_item.h @@ -52,6 +52,14 @@ struct xfs_efi_log_item { xfs_efi_log_format_t efi_format; }; +static inline size_t +xfs_efi_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_efi_log_item, efi_format) + + xfs_efi_log_format_sizeof(nr); +} + /* * This is the "extent free done" log item. It is used to log * the fact that some extents earlier mentioned in an efi item @@ -64,6 +72,14 @@ struct xfs_efd_log_item { xfs_efd_log_format_t efd_format; }; +static inline size_t +xfs_efd_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_efd_log_item, efd_format) + + xfs_efd_log_format_sizeof(nr); +} + /* * Max number of extents in fast allocation path. */ diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c index 8485e3b37ca01..ee4b429a2f2c9 100644 --- a/fs/xfs/xfs_super.c +++ b/fs/xfs/xfs_super.c @@ -2028,18 +2028,14 @@ xfs_init_caches(void) goto out_destroy_trans_cache; xfs_efd_cache = kmem_cache_create("xfs_efd_item", - (sizeof(struct xfs_efd_log_item) + - XFS_EFD_MAX_FAST_EXTENTS * - sizeof(struct xfs_extent)), - 0, 0, NULL); + xfs_efd_log_item_sizeof(XFS_EFD_MAX_FAST_EXTENTS), + 0, 0, NULL); if (!xfs_efd_cache) goto out_destroy_buf_item_cache; xfs_efi_cache = kmem_cache_create("xfs_efi_item", - (sizeof(struct xfs_efi_log_item) + - XFS_EFI_MAX_FAST_EXTENTS * - sizeof(struct xfs_extent)), - 0, 0, NULL); + xfs_efi_log_item_sizeof(XFS_EFI_MAX_FAST_EXTENTS), + 0, 0, NULL); if (!xfs_efi_cache) goto out_destroy_efd_cache; -- GitLab From 8edbe0cf8b4bbe2cf47513998641797b0aca8ee2 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 10 Oct 2022 11:33:47 -0700 Subject: [PATCH 0797/1927] xfs: check deferred refcount op continuation parameters If we're in the middle of a deferred refcount operation and decide to roll the transaction to avoid overflowing the transaction space, we need to check the new agbno/aglen parameters that we're about to record in the new intent. Specifically, we need to check that the new extent is completely within the filesystem, and that continuation does not put us into a different AG. If the keys of a node block are wrong, the lookup to resume an xfs_refcount_adjust_extents operation can put us into the wrong record block. If this happens, we might not find that we run out of aglen at an exact record boundary, which will cause the loop control to do the wrong thing. The previous patch should take care of that problem, but let's add this extra sanity check to stop corruption problems sooner than later. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 38 ++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 1a50ca53304a2..542f749d0c6a0 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -1134,6 +1134,32 @@ xfs_refcount_finish_one_cleanup( xfs_trans_brelse(tp, agbp); } +/* + * Set up a continuation a deferred refcount operation by updating the intent. + * Checks to make sure we're not going to run off the end of the AG. + */ +static inline int +xfs_refcount_continue_op( + struct xfs_btree_cur *cur, + xfs_fsblock_t startblock, + xfs_agblock_t new_agbno, + xfs_extlen_t new_len, + xfs_fsblock_t *new_fsbno) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_perag *pag = cur->bc_ag.pag; + + if (XFS_IS_CORRUPT(mp, !xfs_verify_agbext(pag, new_agbno, new_len))) + return -EFSCORRUPTED; + + *new_fsbno = XFS_AGB_TO_FSB(mp, pag->pag_agno, new_agbno); + + ASSERT(xfs_verify_fsbext(mp, *new_fsbno, new_len)); + ASSERT(pag->pag_agno == XFS_FSB_TO_AGNO(mp, *new_fsbno)); + + return 0; +} + /* * Process one of the deferred refcount operations. We pass back the * btree cursor to maintain our lock on the btree between calls. @@ -1201,12 +1227,20 @@ xfs_refcount_finish_one( case XFS_REFCOUNT_INCREASE: error = xfs_refcount_adjust(rcur, bno, blockcount, &new_agbno, new_len, XFS_REFCOUNT_ADJUST_INCREASE); - *new_fsb = XFS_AGB_TO_FSB(mp, pag->pag_agno, new_agbno); + if (error) + goto out_drop; + if (*new_len > 0) + error = xfs_refcount_continue_op(rcur, startblock, + new_agbno, *new_len, new_fsb); break; case XFS_REFCOUNT_DECREASE: error = xfs_refcount_adjust(rcur, bno, blockcount, &new_agbno, new_len, XFS_REFCOUNT_ADJUST_DECREASE); - *new_fsb = XFS_AGB_TO_FSB(mp, pag->pag_agno, new_agbno); + if (error) + goto out_drop; + if (*new_len > 0) + error = xfs_refcount_continue_op(rcur, startblock, + new_agbno, *new_len, new_fsb); break; case XFS_REFCOUNT_ALLOC_COW: *new_fsb = startblock + blockcount; -- GitLab From 921ed96b4f4e3bd19da7f775f39234226e6647e7 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 25 Oct 2022 15:14:06 -0700 Subject: [PATCH 0798/1927] xfs: actually abort log recovery on corrupt intent-done log items If log recovery picks up intent-done log items that are not of the correct size it needs to abort recovery and fail the mount. Debug assertions are not good enough. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/xfs_extfree_item.c | 20 ++++++++++++++++---- fs/xfs/xfs_rmap_item.c | 6 +++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index f7e52db8da667..18c2243513434 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -751,12 +751,24 @@ xlog_recover_efd_commit_pass2( xfs_lsn_t lsn) { struct xfs_efd_log_format *efd_formatp; + int buflen = item->ri_buf[0].i_len; efd_formatp = item->ri_buf[0].i_addr; - ASSERT(item->ri_buf[0].i_len == xfs_efd_log_format32_sizeof( - efd_formatp->efd_nextents) || - item->ri_buf[0].i_len == xfs_efd_log_format64_sizeof( - efd_formatp->efd_nextents)); + + if (buflen < sizeof(struct xfs_efd_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + efd_formatp, buflen); + return -EFSCORRUPTED; + } + + if (item->ri_buf[0].i_len != xfs_efd_log_format32_sizeof( + efd_formatp->efd_nextents) && + item->ri_buf[0].i_len != xfs_efd_log_format64_sizeof( + efd_formatp->efd_nextents)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + efd_formatp, buflen); + return -EFSCORRUPTED; + } xlog_recover_release_intent(log, XFS_LI_EFI, efd_formatp->efd_efi_id); return 0; diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 27047e73f5829..5a360c384ea5c 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -707,7 +707,11 @@ xlog_recover_rud_commit_pass2( struct xfs_rud_log_format *rud_formatp; rud_formatp = item->ri_buf[0].i_addr; - ASSERT(item->ri_buf[0].i_len == sizeof(struct xfs_rud_log_format)); + if (item->ri_buf[0].i_len != sizeof(struct xfs_rud_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + rud_formatp, item->ri_buf[0].i_len); + return -EFSCORRUPTED; + } xlog_recover_release_intent(log, XFS_LI_RUI, rud_formatp->rud_rui_id); return 0; -- GitLab From 9e7e2436c159490fbbadbc4b5a4ee6bc30dae02e Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 10 Oct 2022 08:47:59 -0700 Subject: [PATCH 0799/1927] xfs: move _irec structs to xfs_types.h Structure definitions for incore objects do not belong in the ondisk format header. Move them to the incore types header where they belong. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_format.h | 20 -------------------- fs/xfs/libxfs/xfs_types.h | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index b55bdfa9c8a8c..005dd65b71cd6 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1564,20 +1564,6 @@ struct xfs_rmap_rec { #define RMAPBT_UNUSED_OFFSET_BITLEN 7 #define RMAPBT_OFFSET_BITLEN 54 -#define XFS_RMAP_ATTR_FORK (1 << 0) -#define XFS_RMAP_BMBT_BLOCK (1 << 1) -#define XFS_RMAP_UNWRITTEN (1 << 2) -#define XFS_RMAP_KEY_FLAGS (XFS_RMAP_ATTR_FORK | \ - XFS_RMAP_BMBT_BLOCK) -#define XFS_RMAP_REC_FLAGS (XFS_RMAP_UNWRITTEN) -struct xfs_rmap_irec { - xfs_agblock_t rm_startblock; /* extent start block */ - xfs_extlen_t rm_blockcount; /* extent length */ - uint64_t rm_owner; /* extent owner */ - uint64_t rm_offset; /* offset within the owner */ - unsigned int rm_flags; /* state flags */ -}; - /* * Key structure * @@ -1640,12 +1626,6 @@ struct xfs_refcount_key { __be32 rc_startblock; /* starting block number */ }; -struct xfs_refcount_irec { - xfs_agblock_t rc_startblock; /* starting block number */ - xfs_extlen_t rc_blockcount; /* count of free blocks */ - xfs_nlink_t rc_refcount; /* number of inodes linked here */ -}; - #define MAXREFCOUNT ((xfs_nlink_t)~0U) #define MAXREFCEXTLEN ((xfs_extlen_t)~0U) diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h index a6b7d98cf68fa..2d9ebc7338b11 100644 --- a/fs/xfs/libxfs/xfs_types.h +++ b/fs/xfs/libxfs/xfs_types.h @@ -166,6 +166,26 @@ typedef struct xfs_bmbt_irec xfs_exntst_t br_state; /* extent state */ } xfs_bmbt_irec_t; +struct xfs_refcount_irec { + xfs_agblock_t rc_startblock; /* starting block number */ + xfs_extlen_t rc_blockcount; /* count of free blocks */ + xfs_nlink_t rc_refcount; /* number of inodes linked here */ +}; + +#define XFS_RMAP_ATTR_FORK (1 << 0) +#define XFS_RMAP_BMBT_BLOCK (1 << 1) +#define XFS_RMAP_UNWRITTEN (1 << 2) +#define XFS_RMAP_KEY_FLAGS (XFS_RMAP_ATTR_FORK | \ + XFS_RMAP_BMBT_BLOCK) +#define XFS_RMAP_REC_FLAGS (XFS_RMAP_UNWRITTEN) +struct xfs_rmap_irec { + xfs_agblock_t rm_startblock; /* extent start block */ + xfs_extlen_t rm_blockcount; /* extent length */ + uint64_t rm_owner; /* extent owner */ + uint64_t rm_offset; /* offset within the owner */ + unsigned int rm_flags; /* state flags */ +}; + /* per-AG block reservation types */ enum xfs_ag_resv_type { XFS_AG_RESV_NONE = 0, -- GitLab From 950f0d50ee7138d7e631aefea8528d485426eda6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 25 Oct 2022 15:07:14 -0700 Subject: [PATCH 0800/1927] xfs: dump corrupt recovered log intent items to dmesg consistently If log recovery decides that an intent item is corrupt and wants to abort the mount, capture a hexdump of the corrupt log item in the kernel log for further analysis. Some of the log item code already did this, so we're fixing the rest to do it consistently. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/xfs_attr_item.c | 19 +++++++++++++------ fs/xfs/xfs_bmap_item.c | 12 ++++++++---- fs/xfs/xfs_extfree_item.c | 6 ++++-- fs/xfs/xfs_refcount_item.c | 16 +++++++++++----- fs/xfs/xfs_rmap_item.c | 10 +++++++--- 5 files changed, 43 insertions(+), 20 deletions(-) diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c index ee8f678a10a17..2788a6f2edcdb 100644 --- a/fs/xfs/xfs_attr_item.c +++ b/fs/xfs/xfs_attr_item.c @@ -717,31 +717,37 @@ xlog_recover_attri_commit_pass2( /* Validate xfs_attri_log_format before the large memory allocation */ len = sizeof(struct xfs_attri_log_format); if (item->ri_buf[0].i_len != len) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } if (!xfs_attri_validate(mp, attri_formatp)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } /* Validate the attr name */ if (item->ri_buf[1].i_len != xlog_calc_iovec_len(attri_formatp->alfi_name_len)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } if (!xfs_attr_namecheck(attr_name, attri_formatp->alfi_name_len)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[1].i_addr, item->ri_buf[1].i_len); return -EFSCORRUPTED; } /* Validate the attr value, if present */ if (attri_formatp->alfi_value_len != 0) { if (item->ri_buf[2].i_len != xlog_calc_iovec_len(attri_formatp->alfi_value_len)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, + item->ri_buf[0].i_len); return -EFSCORRUPTED; } @@ -834,7 +840,8 @@ xlog_recover_attrd_commit_pass2( attrd_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len != sizeof(struct xfs_attrd_log_format)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c index a1da6205252ba..41323da523d1a 100644 --- a/fs/xfs/xfs_bmap_item.c +++ b/fs/xfs/xfs_bmap_item.c @@ -644,18 +644,21 @@ xlog_recover_bui_commit_pass2( bui_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len < xfs_bui_log_format_sizeof(0)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } if (bui_formatp->bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } len = xfs_bui_log_format_sizeof(bui_formatp->bui_nextents); if (item->ri_buf[0].i_len != len) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } @@ -694,7 +697,8 @@ xlog_recover_bud_commit_pass2( bud_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len != sizeof(struct xfs_bud_log_format)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c index 18c2243513434..d5130d1fcfaea 100644 --- a/fs/xfs/xfs_extfree_item.c +++ b/fs/xfs/xfs_extfree_item.c @@ -216,7 +216,8 @@ xfs_efi_copy_format(xfs_log_iovec_t *buf, xfs_efi_log_format_t *dst_efi_fmt) } return 0; } - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, NULL, buf->i_addr, + buf->i_len); return -EFSCORRUPTED; } @@ -711,7 +712,8 @@ xlog_recover_efi_commit_pass2( efi_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len < xfs_efi_log_format_sizeof(0)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c index 24cf4c64ebaab..858e3e9eb4a85 100644 --- a/fs/xfs/xfs_refcount_item.c +++ b/fs/xfs/xfs_refcount_item.c @@ -523,7 +523,9 @@ xfs_cui_item_recover( type = refc_type; break; default: - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &cuip->cui_format, + sizeof(cuip->cui_format)); error = -EFSCORRUPTED; goto abort_error; } @@ -536,7 +538,8 @@ xfs_cui_item_recover( &new_fsb, &new_len, &rcur); if (error == -EFSCORRUPTED) XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, - refc, sizeof(*refc)); + &cuip->cui_format, + sizeof(cuip->cui_format)); if (error) goto abort_error; @@ -658,13 +661,15 @@ xlog_recover_cui_commit_pass2( cui_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len < xfs_cui_log_format_sizeof(0)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } len = xfs_cui_log_format_sizeof(cui_formatp->cui_nextents); if (item->ri_buf[0].i_len != len) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } @@ -703,7 +708,8 @@ xlog_recover_cud_commit_pass2( cud_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len != sizeof(struct xfs_cud_log_format)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c index 5a360c384ea5c..534504ede1a33 100644 --- a/fs/xfs/xfs_rmap_item.c +++ b/fs/xfs/xfs_rmap_item.c @@ -557,7 +557,9 @@ xfs_rui_item_recover( type = XFS_RMAP_FREE; break; default: - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &ruip->rui_format, + sizeof(ruip->rui_format)); error = -EFSCORRUPTED; goto abort_error; } @@ -663,13 +665,15 @@ xlog_recover_rui_commit_pass2( rui_formatp = item->ri_buf[0].i_addr; if (item->ri_buf[0].i_len < xfs_rui_log_format_sizeof(0)) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } len = xfs_rui_log_format_sizeof(rui_formatp->rui_nextents); if (item->ri_buf[0].i_len != len) { - XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].i_addr, item->ri_buf[0].i_len); return -EFSCORRUPTED; } -- GitLab From 5a8c345ca8b99a9f54b89991f2f6a20521cb05f4 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Tue, 11 Oct 2022 11:22:54 -0700 Subject: [PATCH 0801/1927] xfs: refactor refcount record usage in xchk_refcountbt_rec Consolidate the open-coded xfs_refcount_irec fields into an actual struct and use the existing _btrec_to_irec to decode the ondisk record. This will reduce code churn in the next patch. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/scrub/refcount.c | 54 ++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index 9959397f797f1..9e6b36ac80798 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -269,15 +269,13 @@ xchk_refcountbt_process_rmap_fragments( STATIC void xchk_refcountbt_xref_rmap( struct xfs_scrub *sc, - xfs_agblock_t bno, - xfs_extlen_t len, - xfs_nlink_t refcount) + const struct xfs_refcount_irec *irec) { struct xchk_refcnt_check refchk = { - .sc = sc, - .bno = bno, - .len = len, - .refcount = refcount, + .sc = sc, + .bno = irec->rc_startblock, + .len = irec->rc_blockcount, + .refcount = irec->rc_refcount, .seen = 0, }; struct xfs_rmap_irec low; @@ -291,9 +289,9 @@ xchk_refcountbt_xref_rmap( /* Cross-reference with the rmapbt to confirm the refcount. */ memset(&low, 0, sizeof(low)); - low.rm_startblock = bno; + low.rm_startblock = irec->rc_startblock; memset(&high, 0xFF, sizeof(high)); - high.rm_startblock = bno + len - 1; + high.rm_startblock = irec->rc_startblock + irec->rc_blockcount - 1; INIT_LIST_HEAD(&refchk.fragments); error = xfs_rmap_query_range(sc->sa.rmap_cur, &low, &high, @@ -302,7 +300,7 @@ xchk_refcountbt_xref_rmap( goto out_free; xchk_refcountbt_process_rmap_fragments(&refchk); - if (refcount != refchk.seen) + if (irec->rc_refcount != refchk.seen) xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); out_free: @@ -315,17 +313,16 @@ xchk_refcountbt_xref_rmap( /* Cross-reference with the other btrees. */ STATIC void xchk_refcountbt_xref( - struct xfs_scrub *sc, - xfs_agblock_t agbno, - xfs_extlen_t len, - xfs_nlink_t refcount) + struct xfs_scrub *sc, + const struct xfs_refcount_irec *irec) { if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) return; - xchk_xref_is_used_space(sc, agbno, len); - xchk_xref_is_not_inode_chunk(sc, agbno, len); - xchk_refcountbt_xref_rmap(sc, agbno, len, refcount); + xchk_xref_is_used_space(sc, irec->rc_startblock, irec->rc_blockcount); + xchk_xref_is_not_inode_chunk(sc, irec->rc_startblock, + irec->rc_blockcount); + xchk_refcountbt_xref_rmap(sc, irec); } /* Scrub a refcountbt record. */ @@ -334,34 +331,31 @@ xchk_refcountbt_rec( struct xchk_btree *bs, const union xfs_btree_rec *rec) { + struct xfs_refcount_irec irec; xfs_agblock_t *cow_blocks = bs->private; struct xfs_perag *pag = bs->cur->bc_ag.pag; - xfs_agblock_t bno; - xfs_extlen_t len; - xfs_nlink_t refcount; bool has_cowflag; - bno = be32_to_cpu(rec->refc.rc_startblock); - len = be32_to_cpu(rec->refc.rc_blockcount); - refcount = be32_to_cpu(rec->refc.rc_refcount); + xfs_refcount_btrec_to_irec(rec, &irec); /* Only CoW records can have refcount == 1. */ - has_cowflag = (bno & XFS_REFC_COW_START); - if ((refcount == 1 && !has_cowflag) || (refcount != 1 && has_cowflag)) + has_cowflag = (irec.rc_startblock & XFS_REFC_COW_START); + if ((irec.rc_refcount == 1 && !has_cowflag) || + (irec.rc_refcount != 1 && has_cowflag)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); if (has_cowflag) - (*cow_blocks) += len; + (*cow_blocks) += irec.rc_blockcount; /* Check the extent. */ - bno &= ~XFS_REFC_COW_START; + irec.rc_startblock &= ~XFS_REFC_COW_START; - if (!xfs_verify_agbext(pag, bno, len)) + if (!xfs_verify_agbext(pag, irec.rc_startblock, irec.rc_blockcount)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); - if (refcount == 0) + if (irec.rc_refcount == 0) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); - xchk_refcountbt_xref(bs->sc, bno, len, refcount); + xchk_refcountbt_xref(bs->sc, &irec); return 0; } -- GitLab From 9a50ee4f8db6e4dd0d8d757b7adaf0591776860a Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 10 Oct 2022 09:06:24 -0700 Subject: [PATCH 0802/1927] xfs: track cow/shared record domains explicitly in xfs_refcount_irec Just prior to committing the reflink code into upstream, the xfs maintainer at the time requested that I find a way to shard the refcount records into two domains -- one for records tracking shared extents, and a second for tracking CoW staging extents. The idea here was to minimize mount time CoW reclamation by pushing all the CoW records to the right edge of the keyspace, and it was accomplished by setting the upper bit in rc_startblock. We don't allow AGs to have more than 2^31 blocks, so the bit was free. Unfortunately, this was a very late addition to the codebase, so most of the refcount record processing code still treats rc_startblock as a u32 and pays no attention to whether or not the upper bit (the cow flag) is set. This is a weakness is theoretically exploitable, since we're not fully validating the incoming metadata records. Fuzzing demonstrates practical exploits of this weakness. If the cow flag of a node block key record is corrupted, a lookup operation can go to the wrong record block and start returning records from the wrong cow/shared domain. This causes the math to go all wrong (since cow domain is still implicit in the upper bit of rc_startblock) and we can crash the kernel by tricking xfs into jumping into a nonexistent AG and tripping over xfs_perag_get(mp, ) returning NULL. To fix this, start tracking the domain as an explicit part of struct xfs_refcount_irec, adjust all refcount functions to check the domain of a returned record, and alter the function definitions to accept them where necessary. Found by fuzzing keys[2].cowflag = add in xfs/464. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 146 +++++++++++++++++++---------- fs/xfs/libxfs/xfs_refcount.h | 28 +++++- fs/xfs/libxfs/xfs_refcount_btree.c | 15 ++- fs/xfs/libxfs/xfs_types.h | 6 ++ fs/xfs/scrub/refcount.c | 23 ++--- 5 files changed, 151 insertions(+), 67 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 542f749d0c6a0..0f920eff34c4a 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -46,13 +46,16 @@ STATIC int __xfs_refcount_cow_free(struct xfs_btree_cur *rcur, int xfs_refcount_lookup_le( struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat) { - trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.pag->pag_agno, bno, + trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.pag->pag_agno, + xfs_refcount_encode_startblock(bno, domain), XFS_LOOKUP_LE); cur->bc_rec.rc.rc_startblock = bno; cur->bc_rec.rc.rc_blockcount = 0; + cur->bc_rec.rc.rc_domain = domain; return xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat); } @@ -63,13 +66,16 @@ xfs_refcount_lookup_le( int xfs_refcount_lookup_ge( struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat) { - trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.pag->pag_agno, bno, + trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.pag->pag_agno, + xfs_refcount_encode_startblock(bno, domain), XFS_LOOKUP_GE); cur->bc_rec.rc.rc_startblock = bno; cur->bc_rec.rc.rc_blockcount = 0; + cur->bc_rec.rc.rc_domain = domain; return xfs_btree_lookup(cur, XFS_LOOKUP_GE, stat); } @@ -80,13 +86,16 @@ xfs_refcount_lookup_ge( int xfs_refcount_lookup_eq( struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat) { - trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.pag->pag_agno, bno, + trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_ag.pag->pag_agno, + xfs_refcount_encode_startblock(bno, domain), XFS_LOOKUP_LE); cur->bc_rec.rc.rc_startblock = bno; cur->bc_rec.rc.rc_blockcount = 0; + cur->bc_rec.rc.rc_domain = domain; return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat); } @@ -96,7 +105,17 @@ xfs_refcount_btrec_to_irec( const union xfs_btree_rec *rec, struct xfs_refcount_irec *irec) { - irec->rc_startblock = be32_to_cpu(rec->refc.rc_startblock); + uint32_t start; + + start = be32_to_cpu(rec->refc.rc_startblock); + if (start & XFS_REFC_COW_START) { + start &= ~XFS_REFC_COW_START; + irec->rc_domain = XFS_REFC_DOMAIN_COW; + } else { + irec->rc_domain = XFS_REFC_DOMAIN_SHARED; + } + + irec->rc_startblock = start; irec->rc_blockcount = be32_to_cpu(rec->refc.rc_blockcount); irec->rc_refcount = be32_to_cpu(rec->refc.rc_refcount); } @@ -114,7 +133,6 @@ xfs_refcount_get_rec( struct xfs_perag *pag = cur->bc_ag.pag; union xfs_btree_rec *rec; int error; - xfs_agblock_t realstart; error = xfs_btree_get_rec(cur, &rec, stat); if (error || !*stat) @@ -124,18 +142,14 @@ xfs_refcount_get_rec( if (irec->rc_blockcount == 0 || irec->rc_blockcount > MAXREFCEXTLEN) goto out_bad_rec; - /* handle special COW-staging state */ - realstart = irec->rc_startblock; - if (realstart & XFS_REFC_COW_START) { - if (irec->rc_refcount != 1) - goto out_bad_rec; - realstart &= ~XFS_REFC_COW_START; - } else if (irec->rc_refcount < 2) { + /* handle special COW-staging domain */ + if (irec->rc_domain == XFS_REFC_DOMAIN_COW && irec->rc_refcount != 1) + goto out_bad_rec; + if (irec->rc_domain == XFS_REFC_DOMAIN_SHARED && irec->rc_refcount < 2) goto out_bad_rec; - } /* check for valid extent range, including overflow */ - if (!xfs_verify_agbext(pag, realstart, irec->rc_blockcount)) + if (!xfs_verify_agbext(pag, irec->rc_startblock, irec->rc_blockcount)) goto out_bad_rec; if (irec->rc_refcount == 0 || irec->rc_refcount > MAXREFCOUNT) @@ -165,12 +179,17 @@ xfs_refcount_update( struct xfs_refcount_irec *irec) { union xfs_btree_rec rec; + uint32_t start; int error; trace_xfs_refcount_update(cur->bc_mp, cur->bc_ag.pag->pag_agno, irec); - rec.refc.rc_startblock = cpu_to_be32(irec->rc_startblock); + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + rec.refc.rc_startblock = cpu_to_be32(start); rec.refc.rc_blockcount = cpu_to_be32(irec->rc_blockcount); rec.refc.rc_refcount = cpu_to_be32(irec->rc_refcount); + error = xfs_btree_update(cur, &rec); if (error) trace_xfs_refcount_update_error(cur->bc_mp, @@ -192,9 +211,12 @@ xfs_refcount_insert( int error; trace_xfs_refcount_insert(cur->bc_mp, cur->bc_ag.pag->pag_agno, irec); + cur->bc_rec.rc.rc_startblock = irec->rc_startblock; cur->bc_rec.rc.rc_blockcount = irec->rc_blockcount; cur->bc_rec.rc.rc_refcount = irec->rc_refcount; + cur->bc_rec.rc.rc_domain = irec->rc_domain; + error = xfs_btree_insert(cur, i); if (error) goto out_error; @@ -240,7 +262,8 @@ xfs_refcount_delete( } if (error) goto out_error; - error = xfs_refcount_lookup_ge(cur, irec.rc_startblock, &found_rec); + error = xfs_refcount_lookup_ge(cur, irec.rc_domain, irec.rc_startblock, + &found_rec); out_error: if (error) trace_xfs_refcount_delete_error(cur->bc_mp, @@ -339,6 +362,7 @@ xfs_refc_next( STATIC int xfs_refcount_split_extent( struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t agbno, bool *shape_changed) { @@ -347,7 +371,7 @@ xfs_refcount_split_extent( int error; *shape_changed = false; - error = xfs_refcount_lookup_le(cur, agbno, &found_rec); + error = xfs_refcount_lookup_le(cur, domain, agbno, &found_rec); if (error) goto out_error; if (!found_rec) @@ -419,8 +443,8 @@ xfs_refcount_merge_center_extents( * call removes the center and the second one removes the right * extent. */ - error = xfs_refcount_lookup_ge(cur, center->rc_startblock, - &found_rec); + error = xfs_refcount_lookup_ge(cur, center->rc_domain, + center->rc_startblock, &found_rec); if (error) goto out_error; if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { @@ -447,8 +471,8 @@ xfs_refcount_merge_center_extents( } /* Enlarge the left extent. */ - error = xfs_refcount_lookup_le(cur, left->rc_startblock, - &found_rec); + error = xfs_refcount_lookup_le(cur, left->rc_domain, + left->rc_startblock, &found_rec); if (error) goto out_error; if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { @@ -489,8 +513,8 @@ xfs_refcount_merge_left_extent( /* If the extent at agbno (cleft) wasn't synthesized, remove it. */ if (cleft->rc_refcount > 1) { - error = xfs_refcount_lookup_le(cur, cleft->rc_startblock, - &found_rec); + error = xfs_refcount_lookup_le(cur, cleft->rc_domain, + cleft->rc_startblock, &found_rec); if (error) goto out_error; if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { @@ -508,8 +532,8 @@ xfs_refcount_merge_left_extent( } /* Enlarge the left extent. */ - error = xfs_refcount_lookup_le(cur, left->rc_startblock, - &found_rec); + error = xfs_refcount_lookup_le(cur, left->rc_domain, + left->rc_startblock, &found_rec); if (error) goto out_error; if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { @@ -553,8 +577,8 @@ xfs_refcount_merge_right_extent( * remove it. */ if (cright->rc_refcount > 1) { - error = xfs_refcount_lookup_le(cur, cright->rc_startblock, - &found_rec); + error = xfs_refcount_lookup_le(cur, cright->rc_domain, + cright->rc_startblock, &found_rec); if (error) goto out_error; if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { @@ -572,8 +596,8 @@ xfs_refcount_merge_right_extent( } /* Enlarge the right extent. */ - error = xfs_refcount_lookup_le(cur, right->rc_startblock, - &found_rec); + error = xfs_refcount_lookup_le(cur, right->rc_domain, + right->rc_startblock, &found_rec); if (error) goto out_error; if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { @@ -612,11 +636,17 @@ xfs_refcount_find_left_extents( int flags) { struct xfs_refcount_irec tmp; + enum xfs_refc_domain domain; int error; int found_rec; + if (flags & XFS_FIND_RCEXT_SHARED) + domain = XFS_REFC_DOMAIN_SHARED; + else + domain = XFS_REFC_DOMAIN_COW; + left->rc_startblock = cleft->rc_startblock = NULLAGBLOCK; - error = xfs_refcount_lookup_le(cur, agbno - 1, &found_rec); + error = xfs_refcount_lookup_le(cur, domain, agbno - 1, &found_rec); if (error) goto out_error; if (!found_rec) @@ -667,6 +697,7 @@ xfs_refcount_find_left_extents( cleft->rc_blockcount = min(aglen, tmp.rc_startblock - agbno); cleft->rc_refcount = 1; + cleft->rc_domain = domain; } } else { /* @@ -676,6 +707,7 @@ xfs_refcount_find_left_extents( cleft->rc_startblock = agbno; cleft->rc_blockcount = aglen; cleft->rc_refcount = 1; + cleft->rc_domain = domain; } trace_xfs_refcount_find_left_extent(cur->bc_mp, cur->bc_ag.pag->pag_agno, left, cleft, agbno); @@ -701,11 +733,17 @@ xfs_refcount_find_right_extents( int flags) { struct xfs_refcount_irec tmp; + enum xfs_refc_domain domain; int error; int found_rec; + if (flags & XFS_FIND_RCEXT_SHARED) + domain = XFS_REFC_DOMAIN_SHARED; + else + domain = XFS_REFC_DOMAIN_COW; + right->rc_startblock = cright->rc_startblock = NULLAGBLOCK; - error = xfs_refcount_lookup_ge(cur, agbno + aglen, &found_rec); + error = xfs_refcount_lookup_ge(cur, domain, agbno + aglen, &found_rec); if (error) goto out_error; if (!found_rec) @@ -756,6 +794,7 @@ xfs_refcount_find_right_extents( cright->rc_blockcount = right->rc_startblock - cright->rc_startblock; cright->rc_refcount = 1; + cright->rc_domain = domain; } } else { /* @@ -765,6 +804,7 @@ xfs_refcount_find_right_extents( cright->rc_startblock = agbno; cright->rc_blockcount = aglen; cright->rc_refcount = 1; + cright->rc_domain = domain; } trace_xfs_refcount_find_right_extent(cur->bc_mp, cur->bc_ag.pag->pag_agno, cright, right, agbno + aglen); @@ -929,7 +969,8 @@ xfs_refcount_adjust_extents( if (*aglen == 0) return 0; - error = xfs_refcount_lookup_ge(cur, *agbno, &found_rec); + error = xfs_refcount_lookup_ge(cur, XFS_REFC_DOMAIN_SHARED, *agbno, + &found_rec); if (error) goto out_error; @@ -941,6 +982,7 @@ xfs_refcount_adjust_extents( ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks; ext.rc_blockcount = 0; ext.rc_refcount = 0; + ext.rc_domain = XFS_REFC_DOMAIN_SHARED; } /* @@ -953,6 +995,8 @@ xfs_refcount_adjust_extents( tmp.rc_blockcount = min(*aglen, ext.rc_startblock - *agbno); tmp.rc_refcount = 1 + adj; + tmp.rc_domain = XFS_REFC_DOMAIN_SHARED; + trace_xfs_refcount_modify_extent(cur->bc_mp, cur->bc_ag.pag->pag_agno, &tmp); @@ -987,7 +1031,8 @@ xfs_refcount_adjust_extents( break; /* Move the cursor to the start of ext. */ - error = xfs_refcount_lookup_ge(cur, *agbno, + error = xfs_refcount_lookup_ge(cur, + XFS_REFC_DOMAIN_SHARED, *agbno, &found_rec); if (error) goto out_error; @@ -1080,13 +1125,15 @@ xfs_refcount_adjust( /* * Ensure that no rcextents cross the boundary of the adjustment range. */ - error = xfs_refcount_split_extent(cur, agbno, &shape_changed); + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_SHARED, + agbno, &shape_changed); if (error) goto out_error; if (shape_changed) shape_changes++; - error = xfs_refcount_split_extent(cur, agbno + aglen, &shape_changed); + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_SHARED, + agbno + aglen, &shape_changed); if (error) goto out_error; if (shape_changed) @@ -1351,7 +1398,8 @@ xfs_refcount_find_shared( *flen = 0; /* Try to find a refcount extent that crosses the start */ - error = xfs_refcount_lookup_le(cur, agbno, &have); + error = xfs_refcount_lookup_le(cur, XFS_REFC_DOMAIN_SHARED, agbno, + &have); if (error) goto out_error; if (!have) { @@ -1499,17 +1547,18 @@ xfs_refcount_adjust_cow_extents( return 0; /* Find any overlapping refcount records */ - error = xfs_refcount_lookup_ge(cur, agbno, &found_rec); + error = xfs_refcount_lookup_ge(cur, XFS_REFC_DOMAIN_COW, agbno, + &found_rec); if (error) goto out_error; error = xfs_refcount_get_rec(cur, &ext, &found_rec); if (error) goto out_error; if (!found_rec) { - ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks + - XFS_REFC_COW_START; + ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks; ext.rc_blockcount = 0; ext.rc_refcount = 0; + ext.rc_domain = XFS_REFC_DOMAIN_COW; } switch (adj) { @@ -1524,6 +1573,8 @@ xfs_refcount_adjust_cow_extents( tmp.rc_startblock = agbno; tmp.rc_blockcount = aglen; tmp.rc_refcount = 1; + tmp.rc_domain = XFS_REFC_DOMAIN_COW; + trace_xfs_refcount_modify_extent(cur->bc_mp, cur->bc_ag.pag->pag_agno, &tmp); @@ -1586,16 +1637,16 @@ xfs_refcount_adjust_cow( bool shape_changed; int error; - agbno += XFS_REFC_COW_START; - /* * Ensure that no rcextents cross the boundary of the adjustment range. */ - error = xfs_refcount_split_extent(cur, agbno, &shape_changed); + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_COW, + agbno, &shape_changed); if (error) goto out_error; - error = xfs_refcount_split_extent(cur, agbno + aglen, &shape_changed); + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_COW, + agbno + aglen, &shape_changed); if (error) goto out_error; @@ -1731,7 +1782,6 @@ xfs_refcount_recover_cow_leftovers( union xfs_btree_irec low; union xfs_btree_irec high; xfs_fsblock_t fsb; - xfs_agblock_t agbno; int error; if (mp->m_sb.sb_agblocks >= XFS_REFC_COW_START) @@ -1761,7 +1811,7 @@ xfs_refcount_recover_cow_leftovers( /* Find all the leftover CoW staging extents. */ memset(&low, 0, sizeof(low)); memset(&high, 0, sizeof(high)); - low.rc.rc_startblock = XFS_REFC_COW_START; + low.rc.rc_domain = high.rc.rc_domain = XFS_REFC_DOMAIN_COW; high.rc.rc_startblock = -1U; error = xfs_btree_query_range(cur, &low, &high, xfs_refcount_recover_extent, &debris); @@ -1782,8 +1832,8 @@ xfs_refcount_recover_cow_leftovers( &rr->rr_rrec); /* Free the orphan record */ - agbno = rr->rr_rrec.rc_startblock - XFS_REFC_COW_START; - fsb = XFS_AGB_TO_FSB(mp, pag->pag_agno, agbno); + fsb = XFS_AGB_TO_FSB(mp, pag->pag_agno, + rr->rr_rrec.rc_startblock); xfs_refcount_free_cow_extent(tp, fsb, rr->rr_rrec.rc_blockcount); @@ -1814,6 +1864,7 @@ xfs_refcount_recover_cow_leftovers( int xfs_refcount_has_record( struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, xfs_extlen_t len, bool *exists) @@ -1825,6 +1876,7 @@ xfs_refcount_has_record( low.rc.rc_startblock = bno; memset(&high, 0xFF, sizeof(high)); high.rc.rc_startblock = bno + len - 1; + low.rc.rc_domain = high.rc.rc_domain = domain; return xfs_btree_has_record(cur, &low, &high, exists); } diff --git a/fs/xfs/libxfs/xfs_refcount.h b/fs/xfs/libxfs/xfs_refcount.h index e8b322de7f3d9..3beb5a30a9c9e 100644 --- a/fs/xfs/libxfs/xfs_refcount.h +++ b/fs/xfs/libxfs/xfs_refcount.h @@ -14,14 +14,33 @@ struct xfs_bmbt_irec; struct xfs_refcount_irec; extern int xfs_refcount_lookup_le(struct xfs_btree_cur *cur, - xfs_agblock_t bno, int *stat); + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat); extern int xfs_refcount_lookup_ge(struct xfs_btree_cur *cur, - xfs_agblock_t bno, int *stat); + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat); extern int xfs_refcount_lookup_eq(struct xfs_btree_cur *cur, - xfs_agblock_t bno, int *stat); + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat); extern int xfs_refcount_get_rec(struct xfs_btree_cur *cur, struct xfs_refcount_irec *irec, int *stat); +static inline uint32_t +xfs_refcount_encode_startblock( + xfs_agblock_t startblock, + enum xfs_refc_domain domain) +{ + uint32_t start; + + /* + * low level btree operations need to handle the generic btree range + * query functions (which set rc_domain == -1U), so we check that the + * domain is /not/ shared. + */ + start = startblock & ~XFS_REFC_COW_START; + if (domain != XFS_REFC_DOMAIN_SHARED) + start |= XFS_REFC_COW_START; + + return start; +} + enum xfs_refcount_intent_type { XFS_REFCOUNT_INCREASE = 1, XFS_REFCOUNT_DECREASE, @@ -79,7 +98,8 @@ extern int xfs_refcount_recover_cow_leftovers(struct xfs_mount *mp, #define XFS_REFCOUNT_ITEM_OVERHEAD 32 extern int xfs_refcount_has_record(struct xfs_btree_cur *cur, - xfs_agblock_t bno, xfs_extlen_t len, bool *exists); + enum xfs_refc_domain domain, xfs_agblock_t bno, + xfs_extlen_t len, bool *exists); union xfs_btree_rec; extern void xfs_refcount_btrec_to_irec(const union xfs_btree_rec *rec, struct xfs_refcount_irec *irec); diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c index 316c1ec0c3c26..e1f7898666831 100644 --- a/fs/xfs/libxfs/xfs_refcount_btree.c +++ b/fs/xfs/libxfs/xfs_refcount_btree.c @@ -13,6 +13,7 @@ #include "xfs_btree.h" #include "xfs_btree_staging.h" #include "xfs_refcount_btree.h" +#include "xfs_refcount.h" #include "xfs_alloc.h" #include "xfs_error.h" #include "xfs_trace.h" @@ -160,7 +161,12 @@ xfs_refcountbt_init_rec_from_cur( struct xfs_btree_cur *cur, union xfs_btree_rec *rec) { - rec->refc.rc_startblock = cpu_to_be32(cur->bc_rec.rc.rc_startblock); + const struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + uint32_t start; + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + rec->refc.rc_startblock = cpu_to_be32(start); rec->refc.rc_blockcount = cpu_to_be32(cur->bc_rec.rc.rc_blockcount); rec->refc.rc_refcount = cpu_to_be32(cur->bc_rec.rc.rc_refcount); } @@ -182,10 +188,13 @@ xfs_refcountbt_key_diff( struct xfs_btree_cur *cur, const union xfs_btree_key *key) { - struct xfs_refcount_irec *rec = &cur->bc_rec.rc; const struct xfs_refcount_key *kp = &key->refc; + const struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + uint32_t start; - return (int64_t)be32_to_cpu(kp->rc_startblock) - rec->rc_startblock; + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + return (int64_t)be32_to_cpu(kp->rc_startblock) - start; } STATIC int64_t diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h index 2d9ebc7338b11..eb9a98338bb98 100644 --- a/fs/xfs/libxfs/xfs_types.h +++ b/fs/xfs/libxfs/xfs_types.h @@ -166,10 +166,16 @@ typedef struct xfs_bmbt_irec xfs_exntst_t br_state; /* extent state */ } xfs_bmbt_irec_t; +enum xfs_refc_domain { + XFS_REFC_DOMAIN_SHARED = 0, + XFS_REFC_DOMAIN_COW, +}; + struct xfs_refcount_irec { xfs_agblock_t rc_startblock; /* starting block number */ xfs_extlen_t rc_blockcount; /* count of free blocks */ xfs_nlink_t rc_refcount; /* number of inodes linked here */ + enum xfs_refc_domain rc_domain; /* shared or cow staging extent? */ }; #define XFS_RMAP_ATTR_FORK (1 << 0) diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index 9e6b36ac80798..af5b796ec9ecb 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -334,21 +334,19 @@ xchk_refcountbt_rec( struct xfs_refcount_irec irec; xfs_agblock_t *cow_blocks = bs->private; struct xfs_perag *pag = bs->cur->bc_ag.pag; - bool has_cowflag; xfs_refcount_btrec_to_irec(rec, &irec); /* Only CoW records can have refcount == 1. */ - has_cowflag = (irec.rc_startblock & XFS_REFC_COW_START); - if ((irec.rc_refcount == 1 && !has_cowflag) || - (irec.rc_refcount != 1 && has_cowflag)) + if (irec.rc_domain == XFS_REFC_DOMAIN_SHARED && irec.rc_refcount == 1) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); - if (has_cowflag) + if (irec.rc_domain == XFS_REFC_DOMAIN_COW) { + if (irec.rc_refcount != 1) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); (*cow_blocks) += irec.rc_blockcount; + } /* Check the extent. */ - irec.rc_startblock &= ~XFS_REFC_COW_START; - if (!xfs_verify_agbext(pag, irec.rc_startblock, irec.rc_blockcount)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); @@ -419,7 +417,6 @@ xchk_xref_is_cow_staging( xfs_extlen_t len) { struct xfs_refcount_irec rc; - bool has_cowflag; int has_refcount; int error; @@ -427,8 +424,8 @@ xchk_xref_is_cow_staging( return; /* Find the CoW staging extent. */ - error = xfs_refcount_lookup_le(sc->sa.refc_cur, - agbno + XFS_REFC_COW_START, &has_refcount); + error = xfs_refcount_lookup_le(sc->sa.refc_cur, XFS_REFC_DOMAIN_COW, + agbno, &has_refcount); if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) return; if (!has_refcount) { @@ -445,8 +442,7 @@ xchk_xref_is_cow_staging( } /* CoW flag must be set, refcount must be 1. */ - has_cowflag = (rc.rc_startblock & XFS_REFC_COW_START); - if (!has_cowflag || rc.rc_refcount != 1) + if (rc.rc_domain != XFS_REFC_DOMAIN_COW || rc.rc_refcount != 1) xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); /* Must be at least as long as what was passed in */ @@ -470,7 +466,8 @@ xchk_xref_is_not_shared( if (!sc->sa.refc_cur || xchk_skip_xref(sc->sm)) return; - error = xfs_refcount_has_record(sc->sa.refc_cur, agbno, len, &shared); + error = xfs_refcount_has_record(sc->sa.refc_cur, XFS_REFC_DOMAIN_SHARED, + agbno, len, &shared); if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) return; if (shared) -- GitLab From 571423a162cd86acb1b010a01c6203369586daa6 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 26 Oct 2022 14:23:58 -0700 Subject: [PATCH 0803/1927] xfs: report refcount domain in tracepoints Now that we've broken out the startblock and shared/cow domain in the incore refcount extent record structure, update the tracepoints to report the domain. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_types.h | 4 ++++ fs/xfs/xfs_trace.h | 48 +++++++++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h index eb9a98338bb98..5ebdda7e10780 100644 --- a/fs/xfs/libxfs/xfs_types.h +++ b/fs/xfs/libxfs/xfs_types.h @@ -171,6 +171,10 @@ enum xfs_refc_domain { XFS_REFC_DOMAIN_COW, }; +#define XFS_REFC_DOMAIN_STRINGS \ + { XFS_REFC_DOMAIN_SHARED, "shared" }, \ + { XFS_REFC_DOMAIN_COW, "cow" } + struct xfs_refcount_irec { xfs_agblock_t rc_startblock; /* starting block number */ xfs_extlen_t rc_blockcount; /* count of free blocks */ diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h index cb7c81ba7fa38..372d871bccc5e 100644 --- a/fs/xfs/xfs_trace.h +++ b/fs/xfs/xfs_trace.h @@ -799,6 +799,9 @@ TRACE_DEFINE_ENUM(PE_SIZE_PTE); TRACE_DEFINE_ENUM(PE_SIZE_PMD); TRACE_DEFINE_ENUM(PE_SIZE_PUD); +TRACE_DEFINE_ENUM(XFS_REFC_DOMAIN_SHARED); +TRACE_DEFINE_ENUM(XFS_REFC_DOMAIN_COW); + TRACE_EVENT(xfs_filemap_fault, TP_PROTO(struct xfs_inode *ip, enum page_entry_size pe_size, bool write_fault), @@ -2925,6 +2928,7 @@ DECLARE_EVENT_CLASS(xfs_refcount_extent_class, TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, domain) __field(xfs_agblock_t, startblock) __field(xfs_extlen_t, blockcount) __field(xfs_nlink_t, refcount) @@ -2932,13 +2936,15 @@ DECLARE_EVENT_CLASS(xfs_refcount_extent_class, TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->agno = agno; + __entry->domain = irec->rc_domain; __entry->startblock = irec->rc_startblock; __entry->blockcount = irec->rc_blockcount; __entry->refcount = irec->rc_refcount; ), - TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x refcount %u", + TP_printk("dev %d:%d agno 0x%x dom %s agbno 0x%x fsbcount 0x%x refcount %u", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, + __print_symbolic(__entry->domain, XFS_REFC_DOMAIN_STRINGS), __entry->startblock, __entry->blockcount, __entry->refcount) @@ -2958,6 +2964,7 @@ DECLARE_EVENT_CLASS(xfs_refcount_extent_at_class, TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, domain) __field(xfs_agblock_t, startblock) __field(xfs_extlen_t, blockcount) __field(xfs_nlink_t, refcount) @@ -2966,14 +2973,16 @@ DECLARE_EVENT_CLASS(xfs_refcount_extent_at_class, TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->agno = agno; + __entry->domain = irec->rc_domain; __entry->startblock = irec->rc_startblock; __entry->blockcount = irec->rc_blockcount; __entry->refcount = irec->rc_refcount; __entry->agbno = agbno; ), - TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x refcount %u @ agbno 0x%x", + TP_printk("dev %d:%d agno 0x%x dom %s agbno 0x%x fsbcount 0x%x refcount %u @ agbno 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, + __print_symbolic(__entry->domain, XFS_REFC_DOMAIN_STRINGS), __entry->startblock, __entry->blockcount, __entry->refcount, @@ -2994,9 +3003,11 @@ DECLARE_EVENT_CLASS(xfs_refcount_double_extent_class, TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, i1_domain) __field(xfs_agblock_t, i1_startblock) __field(xfs_extlen_t, i1_blockcount) __field(xfs_nlink_t, i1_refcount) + __field(enum xfs_refc_domain, i2_domain) __field(xfs_agblock_t, i2_startblock) __field(xfs_extlen_t, i2_blockcount) __field(xfs_nlink_t, i2_refcount) @@ -3004,20 +3015,24 @@ DECLARE_EVENT_CLASS(xfs_refcount_double_extent_class, TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->agno = agno; + __entry->i1_domain = i1->rc_domain; __entry->i1_startblock = i1->rc_startblock; __entry->i1_blockcount = i1->rc_blockcount; __entry->i1_refcount = i1->rc_refcount; + __entry->i2_domain = i2->rc_domain; __entry->i2_startblock = i2->rc_startblock; __entry->i2_blockcount = i2->rc_blockcount; __entry->i2_refcount = i2->rc_refcount; ), - TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x refcount %u -- " - "agbno 0x%x fsbcount 0x%x refcount %u", + TP_printk("dev %d:%d agno 0x%x dom %s agbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s agbno 0x%x fsbcount 0x%x refcount %u", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, + __print_symbolic(__entry->i1_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i1_startblock, __entry->i1_blockcount, __entry->i1_refcount, + __print_symbolic(__entry->i2_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i2_startblock, __entry->i2_blockcount, __entry->i2_refcount) @@ -3038,9 +3053,11 @@ DECLARE_EVENT_CLASS(xfs_refcount_double_extent_at_class, TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, i1_domain) __field(xfs_agblock_t, i1_startblock) __field(xfs_extlen_t, i1_blockcount) __field(xfs_nlink_t, i1_refcount) + __field(enum xfs_refc_domain, i2_domain) __field(xfs_agblock_t, i2_startblock) __field(xfs_extlen_t, i2_blockcount) __field(xfs_nlink_t, i2_refcount) @@ -3049,21 +3066,25 @@ DECLARE_EVENT_CLASS(xfs_refcount_double_extent_at_class, TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->agno = agno; + __entry->i1_domain = i1->rc_domain; __entry->i1_startblock = i1->rc_startblock; __entry->i1_blockcount = i1->rc_blockcount; __entry->i1_refcount = i1->rc_refcount; + __entry->i2_domain = i2->rc_domain; __entry->i2_startblock = i2->rc_startblock; __entry->i2_blockcount = i2->rc_blockcount; __entry->i2_refcount = i2->rc_refcount; __entry->agbno = agbno; ), - TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x refcount %u -- " - "agbno 0x%x fsbcount 0x%x refcount %u @ agbno 0x%x", + TP_printk("dev %d:%d agno 0x%x dom %s agbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s agbno 0x%x fsbcount 0x%x refcount %u @ agbno 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, + __print_symbolic(__entry->i1_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i1_startblock, __entry->i1_blockcount, __entry->i1_refcount, + __print_symbolic(__entry->i2_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i2_startblock, __entry->i2_blockcount, __entry->i2_refcount, @@ -3086,12 +3107,15 @@ DECLARE_EVENT_CLASS(xfs_refcount_triple_extent_class, TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, i1_domain) __field(xfs_agblock_t, i1_startblock) __field(xfs_extlen_t, i1_blockcount) __field(xfs_nlink_t, i1_refcount) + __field(enum xfs_refc_domain, i2_domain) __field(xfs_agblock_t, i2_startblock) __field(xfs_extlen_t, i2_blockcount) __field(xfs_nlink_t, i2_refcount) + __field(enum xfs_refc_domain, i3_domain) __field(xfs_agblock_t, i3_startblock) __field(xfs_extlen_t, i3_blockcount) __field(xfs_nlink_t, i3_refcount) @@ -3099,27 +3123,33 @@ DECLARE_EVENT_CLASS(xfs_refcount_triple_extent_class, TP_fast_assign( __entry->dev = mp->m_super->s_dev; __entry->agno = agno; + __entry->i1_domain = i1->rc_domain; __entry->i1_startblock = i1->rc_startblock; __entry->i1_blockcount = i1->rc_blockcount; __entry->i1_refcount = i1->rc_refcount; + __entry->i2_domain = i2->rc_domain; __entry->i2_startblock = i2->rc_startblock; __entry->i2_blockcount = i2->rc_blockcount; __entry->i2_refcount = i2->rc_refcount; + __entry->i3_domain = i3->rc_domain; __entry->i3_startblock = i3->rc_startblock; __entry->i3_blockcount = i3->rc_blockcount; __entry->i3_refcount = i3->rc_refcount; ), - TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x refcount %u -- " - "agbno 0x%x fsbcount 0x%x refcount %u -- " - "agbno 0x%x fsbcount 0x%x refcount %u", + TP_printk("dev %d:%d agno 0x%x dom %s agbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s agbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s agbno 0x%x fsbcount 0x%x refcount %u", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, + __print_symbolic(__entry->i1_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i1_startblock, __entry->i1_blockcount, __entry->i1_refcount, + __print_symbolic(__entry->i2_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i2_startblock, __entry->i2_blockcount, __entry->i2_refcount, + __print_symbolic(__entry->i3_domain, XFS_REFC_DOMAIN_STRINGS), __entry->i3_startblock, __entry->i3_blockcount, __entry->i3_refcount) -- GitLab From f492135df0aa0417337f9b8b1cc6d6a994d61d25 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 26 Oct 2022 14:31:27 -0700 Subject: [PATCH 0804/1927] xfs: refactor domain and refcount checking Create a helper function to ensure that CoW staging extent records have a single refcount and that shared extent records have more than 1 refcount. We'll put this to more use in the next patch. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 5 +---- fs/xfs/libxfs/xfs_refcount.h | 12 ++++++++++++ fs/xfs/scrub/refcount.c | 10 ++++------ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 0f920eff34c4a..8eaa11470f469 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -142,10 +142,7 @@ xfs_refcount_get_rec( if (irec->rc_blockcount == 0 || irec->rc_blockcount > MAXREFCEXTLEN) goto out_bad_rec; - /* handle special COW-staging domain */ - if (irec->rc_domain == XFS_REFC_DOMAIN_COW && irec->rc_refcount != 1) - goto out_bad_rec; - if (irec->rc_domain == XFS_REFC_DOMAIN_SHARED && irec->rc_refcount < 2) + if (!xfs_refcount_check_domain(irec)) goto out_bad_rec; /* check for valid extent range, including overflow */ diff --git a/fs/xfs/libxfs/xfs_refcount.h b/fs/xfs/libxfs/xfs_refcount.h index 3beb5a30a9c9e..ee32e8eb5a993 100644 --- a/fs/xfs/libxfs/xfs_refcount.h +++ b/fs/xfs/libxfs/xfs_refcount.h @@ -55,6 +55,18 @@ struct xfs_refcount_intent { xfs_fsblock_t ri_startblock; }; +/* Check that the refcount is appropriate for the record domain. */ +static inline bool +xfs_refcount_check_domain( + const struct xfs_refcount_irec *irec) +{ + if (irec->rc_domain == XFS_REFC_DOMAIN_COW && irec->rc_refcount != 1) + return false; + if (irec->rc_domain == XFS_REFC_DOMAIN_SHARED && irec->rc_refcount < 2) + return false; + return true; +} + void xfs_refcount_increase_extent(struct xfs_trans *tp, struct xfs_bmbt_irec *irec); void xfs_refcount_decrease_extent(struct xfs_trans *tp, diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index af5b796ec9ecb..fe5ffe4f478d8 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -337,14 +337,12 @@ xchk_refcountbt_rec( xfs_refcount_btrec_to_irec(rec, &irec); - /* Only CoW records can have refcount == 1. */ - if (irec.rc_domain == XFS_REFC_DOMAIN_SHARED && irec.rc_refcount == 1) + /* Check the domain and refcount are not incompatible. */ + if (!xfs_refcount_check_domain(&irec)) xchk_btree_set_corrupt(bs->sc, bs->cur, 0); - if (irec.rc_domain == XFS_REFC_DOMAIN_COW) { - if (irec.rc_refcount != 1) - xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + if (irec.rc_domain == XFS_REFC_DOMAIN_COW) (*cow_blocks) += irec.rc_blockcount; - } /* Check the extent. */ if (!xfs_verify_agbext(pag, irec.rc_startblock, irec.rc_blockcount)) -- GitLab From 68d0f389179a52555cfd8fa3254e4adcd7576904 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 26 Oct 2022 14:42:48 -0700 Subject: [PATCH 0805/1927] xfs: remove XFS_FIND_RCEXT_SHARED and _COW Now that we have an explicit enum for shared and CoW staging extents, we can get rid of the old FIND_RCEXT flags. Omit a couple of conversions that disappear in the next patches. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 48 +++++++++++++----------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 8eaa11470f469..ba2ddf177a498 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -617,8 +617,6 @@ xfs_refcount_merge_right_extent( return error; } -#define XFS_FIND_RCEXT_SHARED 1 -#define XFS_FIND_RCEXT_COW 2 /* * Find the left extent and the one after it (cleft). This function assumes * that we've already split any extent crossing agbno. @@ -628,20 +626,14 @@ xfs_refcount_find_left_extents( struct xfs_btree_cur *cur, struct xfs_refcount_irec *left, struct xfs_refcount_irec *cleft, + enum xfs_refc_domain domain, xfs_agblock_t agbno, - xfs_extlen_t aglen, - int flags) + xfs_extlen_t aglen) { struct xfs_refcount_irec tmp; - enum xfs_refc_domain domain; int error; int found_rec; - if (flags & XFS_FIND_RCEXT_SHARED) - domain = XFS_REFC_DOMAIN_SHARED; - else - domain = XFS_REFC_DOMAIN_COW; - left->rc_startblock = cleft->rc_startblock = NULLAGBLOCK; error = xfs_refcount_lookup_le(cur, domain, agbno - 1, &found_rec); if (error) @@ -659,9 +651,9 @@ xfs_refcount_find_left_extents( if (xfs_refc_next(&tmp) != agbno) return 0; - if ((flags & XFS_FIND_RCEXT_SHARED) && tmp.rc_refcount < 2) + if (domain == XFS_REFC_DOMAIN_SHARED && tmp.rc_refcount < 2) return 0; - if ((flags & XFS_FIND_RCEXT_COW) && tmp.rc_refcount > 1) + if (domain == XFS_REFC_DOMAIN_COW && tmp.rc_refcount > 1) return 0; /* We have a left extent; retrieve (or invent) the next right one */ *left = tmp; @@ -725,20 +717,14 @@ xfs_refcount_find_right_extents( struct xfs_btree_cur *cur, struct xfs_refcount_irec *right, struct xfs_refcount_irec *cright, + enum xfs_refc_domain domain, xfs_agblock_t agbno, - xfs_extlen_t aglen, - int flags) + xfs_extlen_t aglen) { struct xfs_refcount_irec tmp; - enum xfs_refc_domain domain; int error; int found_rec; - if (flags & XFS_FIND_RCEXT_SHARED) - domain = XFS_REFC_DOMAIN_SHARED; - else - domain = XFS_REFC_DOMAIN_COW; - right->rc_startblock = cright->rc_startblock = NULLAGBLOCK; error = xfs_refcount_lookup_ge(cur, domain, agbno + aglen, &found_rec); if (error) @@ -756,9 +742,9 @@ xfs_refcount_find_right_extents( if (tmp.rc_startblock != agbno + aglen) return 0; - if ((flags & XFS_FIND_RCEXT_SHARED) && tmp.rc_refcount < 2) + if (domain == XFS_REFC_DOMAIN_SHARED && tmp.rc_refcount < 2) return 0; - if ((flags & XFS_FIND_RCEXT_COW) && tmp.rc_refcount > 1) + if (domain == XFS_REFC_DOMAIN_COW && tmp.rc_refcount > 1) return 0; /* We have a right extent; retrieve (or invent) the next left one */ *right = tmp; @@ -827,10 +813,10 @@ xfs_refc_valid( STATIC int xfs_refcount_merge_extents( struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t *agbno, xfs_extlen_t *aglen, enum xfs_refc_adjust_op adjust, - int flags, bool *shape_changed) { struct xfs_refcount_irec left = {0}, cleft = {0}; @@ -845,12 +831,12 @@ xfs_refcount_merge_extents( * just below (agbno + aglen) [cright], and just above (agbno + aglen) * [right]. */ - error = xfs_refcount_find_left_extents(cur, &left, &cleft, *agbno, - *aglen, flags); + error = xfs_refcount_find_left_extents(cur, &left, &cleft, domain, + *agbno, *aglen); if (error) return error; - error = xfs_refcount_find_right_extents(cur, &right, &cright, *agbno, - *aglen, flags); + error = xfs_refcount_find_right_extents(cur, &right, &cright, domain, + *agbno, *aglen); if (error) return error; @@ -1139,8 +1125,8 @@ xfs_refcount_adjust( /* * Try to merge with the left or right extents of the range. */ - error = xfs_refcount_merge_extents(cur, new_agbno, new_aglen, adj, - XFS_FIND_RCEXT_SHARED, &shape_changed); + error = xfs_refcount_merge_extents(cur, XFS_REFC_DOMAIN_SHARED, + new_agbno, new_aglen, adj, &shape_changed); if (error) goto out_error; if (shape_changed) @@ -1650,8 +1636,8 @@ xfs_refcount_adjust_cow( /* * Try to merge with the left or right extents of the range. */ - error = xfs_refcount_merge_extents(cur, &agbno, &aglen, adj, - XFS_FIND_RCEXT_COW, &shape_changed); + error = xfs_refcount_merge_extents(cur, XFS_REFC_DOMAIN_COW, &agbno, + &aglen, adj, &shape_changed); if (error) goto out_error; -- GitLab From f62ac3e0ac33d366fe81e194fee81de9be2cd886 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 26 Oct 2022 14:16:36 -0700 Subject: [PATCH 0806/1927] xfs: check record domain when accessing refcount records Now that we've separated the startblock and CoW/shared extent domain in the incore refcount record structure, check the domain whenever we retrieve a record to ensure that it's still in the domain that we want. Depending on the circumstances, a change in domain either means we're done processing or that we've found a corruption and need to fail out. The refcount check in xchk_xref_is_cow_staging is redundant since _get_rec has done that for a long time now, so we can get rid of it. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 53 ++++++++++++++++++++++++++++-------- fs/xfs/scrub/refcount.c | 4 +-- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index ba2ddf177a498..27ed4c10d0d0d 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -381,6 +381,8 @@ xfs_refcount_split_extent( error = -EFSCORRUPTED; goto out_error; } + if (rcext.rc_domain != domain) + return 0; if (rcext.rc_startblock == agbno || xfs_refc_next(&rcext) <= agbno) return 0; @@ -432,6 +434,9 @@ xfs_refcount_merge_center_extents( trace_xfs_refcount_merge_center_extents(cur->bc_mp, cur->bc_ag.pag->pag_agno, left, center, right); + ASSERT(left->rc_domain == center->rc_domain); + ASSERT(right->rc_domain == center->rc_domain); + /* * Make sure the center and right extents are not in the btree. * If the center extent was synthesized, the first delete call @@ -508,6 +513,8 @@ xfs_refcount_merge_left_extent( trace_xfs_refcount_merge_left_extent(cur->bc_mp, cur->bc_ag.pag->pag_agno, left, cleft); + ASSERT(left->rc_domain == cleft->rc_domain); + /* If the extent at agbno (cleft) wasn't synthesized, remove it. */ if (cleft->rc_refcount > 1) { error = xfs_refcount_lookup_le(cur, cleft->rc_domain, @@ -569,6 +576,8 @@ xfs_refcount_merge_right_extent( trace_xfs_refcount_merge_right_extent(cur->bc_mp, cur->bc_ag.pag->pag_agno, cright, right); + ASSERT(right->rc_domain == cright->rc_domain); + /* * If the extent ending at agbno+aglen (cright) wasn't synthesized, * remove it. @@ -649,11 +658,9 @@ xfs_refcount_find_left_extents( goto out_error; } - if (xfs_refc_next(&tmp) != agbno) - return 0; - if (domain == XFS_REFC_DOMAIN_SHARED && tmp.rc_refcount < 2) + if (tmp.rc_domain != domain) return 0; - if (domain == XFS_REFC_DOMAIN_COW && tmp.rc_refcount > 1) + if (xfs_refc_next(&tmp) != agbno) return 0; /* We have a left extent; retrieve (or invent) the next right one */ *left = tmp; @@ -670,6 +677,9 @@ xfs_refcount_find_left_extents( goto out_error; } + if (tmp.rc_domain != domain) + goto not_found; + /* if tmp starts at the end of our range, just use that */ if (tmp.rc_startblock == agbno) *cleft = tmp; @@ -689,6 +699,7 @@ xfs_refcount_find_left_extents( cleft->rc_domain = domain; } } else { +not_found: /* * No extents, so pretend that there's one covering the whole * range. @@ -740,11 +751,9 @@ xfs_refcount_find_right_extents( goto out_error; } - if (tmp.rc_startblock != agbno + aglen) - return 0; - if (domain == XFS_REFC_DOMAIN_SHARED && tmp.rc_refcount < 2) + if (tmp.rc_domain != domain) return 0; - if (domain == XFS_REFC_DOMAIN_COW && tmp.rc_refcount > 1) + if (tmp.rc_startblock != agbno + aglen) return 0; /* We have a right extent; retrieve (or invent) the next left one */ *right = tmp; @@ -761,6 +770,9 @@ xfs_refcount_find_right_extents( goto out_error; } + if (tmp.rc_domain != domain) + goto not_found; + /* if tmp ends at the end of our range, just use that */ if (xfs_refc_next(&tmp) == agbno + aglen) *cright = tmp; @@ -780,6 +792,7 @@ xfs_refcount_find_right_extents( cright->rc_domain = domain; } } else { +not_found: /* * No extents, so pretend that there's one covering the whole * range. @@ -889,7 +902,7 @@ xfs_refcount_merge_extents( aglen); } - return error; + return 0; } /* @@ -961,7 +974,7 @@ xfs_refcount_adjust_extents( error = xfs_refcount_get_rec(cur, &ext, &found_rec); if (error) goto out_error; - if (!found_rec) { + if (!found_rec || ext.rc_domain != XFS_REFC_DOMAIN_SHARED) { ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks; ext.rc_blockcount = 0; ext.rc_refcount = 0; @@ -1400,6 +1413,8 @@ xfs_refcount_find_shared( error = -EFSCORRUPTED; goto out_error; } + if (tmp.rc_domain != XFS_REFC_DOMAIN_SHARED) + goto done; /* If the extent ends before the start, look at the next one */ if (tmp.rc_startblock + tmp.rc_blockcount <= agbno) { @@ -1415,6 +1430,8 @@ xfs_refcount_find_shared( error = -EFSCORRUPTED; goto out_error; } + if (tmp.rc_domain != XFS_REFC_DOMAIN_SHARED) + goto done; } /* If the extent starts after the range we want, bail out */ @@ -1446,7 +1463,8 @@ xfs_refcount_find_shared( error = -EFSCORRUPTED; goto out_error; } - if (tmp.rc_startblock >= agbno + aglen || + if (tmp.rc_domain != XFS_REFC_DOMAIN_SHARED || + tmp.rc_startblock >= agbno + aglen || tmp.rc_startblock != *fbno + *flen) break; *flen = min(*flen + tmp.rc_blockcount, agbno + aglen - *fbno); @@ -1537,6 +1555,11 @@ xfs_refcount_adjust_cow_extents( error = xfs_refcount_get_rec(cur, &ext, &found_rec); if (error) goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec && + ext.rc_domain != XFS_REFC_DOMAIN_COW)) { + error = -EFSCORRUPTED; + goto out_error; + } if (!found_rec) { ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks; ext.rc_blockcount = 0; @@ -1746,8 +1769,14 @@ xfs_refcount_recover_extent( rr = kmem_alloc(sizeof(struct xfs_refcount_recovery), 0); xfs_refcount_btrec_to_irec(rec, &rr->rr_rrec); - list_add_tail(&rr->rr_list, debris); + if (XFS_IS_CORRUPT(cur->bc_mp, + rr->rr_rrec.rc_domain != XFS_REFC_DOMAIN_COW)) { + kmem_free(rr); + return -EFSCORRUPTED; + } + + list_add_tail(&rr->rr_list, debris); return 0; } diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index fe5ffe4f478d8..a26ee0f24ef2a 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -439,8 +439,8 @@ xchk_xref_is_cow_staging( return; } - /* CoW flag must be set, refcount must be 1. */ - if (rc.rc_domain != XFS_REFC_DOMAIN_COW || rc.rc_refcount != 1) + /* CoW lookup returned a shared extent record? */ + if (rc.rc_domain != XFS_REFC_DOMAIN_COW) xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); /* Must be at least as long as what was passed in */ -- GitLab From f1fdc8207840672a46f26414f2c989ec078a153b Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Thu, 27 Oct 2022 09:48:59 -0700 Subject: [PATCH 0807/1927] xfs: fix agblocks check in the cow leftover recovery function As we've seen, refcount records use the upper bit of the rc_startblock field to ensure that all the refcount records are at the right side of the refcount btree. This works because an AG is never allowed to have more than (1U << 31) blocks in it. If we ever encounter a filesystem claiming to have that many blocks, we absolutely do not want reflink touching it at all. However, this test at the start of xfs_refcount_recover_cow_leftovers is slightly incorrect -- it /should/ be checking that agblocks isn't larger than the XFS_MAX_CRC_AG_BLOCKS constant, and it should check that the constant is never large enough to conflict with that CoW flag. Note that the V5 superblock verifier has not historically rejected filesystems where agblocks >= XFS_MAX_CRC_AG_BLOCKS, which is why this ended up in the COW recovery routine. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 27ed4c10d0d0d..ad0fb6a7177b4 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -1796,7 +1796,9 @@ xfs_refcount_recover_cow_leftovers( xfs_fsblock_t fsb; int error; - if (mp->m_sb.sb_agblocks >= XFS_REFC_COW_START) + /* reflink filesystems mustn't have AGs larger than 2^31-1 blocks */ + BUILD_BUG_ON(XFS_MAX_CRC_AG_BLOCKS >= XFS_REFC_COW_START); + if (mp->m_sb.sb_agblocks > XFS_MAX_CRC_AG_BLOCKS) return -EOPNOTSUPP; INIT_LIST_HEAD(&debris); -- GitLab From c1ccf967bf962b998f0c096e06a658ece27d10a0 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Wed, 26 Oct 2022 14:55:04 -0700 Subject: [PATCH 0808/1927] xfs: fix uninitialized list head in struct xfs_refcount_recovery We're supposed to initialize the list head of an object before adding it to another list. Fix that, and stop using the kmem_{alloc,free} calls from the Irix days. Fixes: 174edb0e46e5 ("xfs: store in-progress CoW allocations in the refcount btree") Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_refcount.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index ad0fb6a7177b4..44d4667d43016 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -1767,12 +1767,14 @@ xfs_refcount_recover_extent( be32_to_cpu(rec->refc.rc_refcount) != 1)) return -EFSCORRUPTED; - rr = kmem_alloc(sizeof(struct xfs_refcount_recovery), 0); + rr = kmalloc(sizeof(struct xfs_refcount_recovery), + GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&rr->rr_list); xfs_refcount_btrec_to_irec(rec, &rr->rr_rrec); if (XFS_IS_CORRUPT(cur->bc_mp, rr->rr_rrec.rc_domain != XFS_REFC_DOMAIN_COW)) { - kmem_free(rr); + kfree(rr); return -EFSCORRUPTED; } @@ -1859,7 +1861,7 @@ xfs_refcount_recover_cow_leftovers( goto out_free; list_del(&rr->rr_list); - kmem_free(rr); + kfree(rr); } return error; @@ -1869,7 +1871,7 @@ xfs_refcount_recover_cow_leftovers( /* Free the leftover list */ list_for_each_entry_safe(rr, n, &debris, rr_list) { list_del(&rr->rr_list); - kmem_free(rr); + kfree(rr); } return error; } -- GitLab From 8b972158afcaa66c538c3ee1d394f096fcd238a8 Mon Sep 17 00:00:00 2001 From: "Darrick J. Wong" Date: Mon, 10 Oct 2022 11:13:20 -0700 Subject: [PATCH 0809/1927] xfs: rename XFS_REFC_COW_START to _COWFLAG We've been (ab)using XFS_REFC_COW_START as both an integer quantity and a bit flag, even though it's *only* a bit flag. Rename the variable to reflect its nature and update the cast target since we're not supposed to be comparing it to xfs_agblock_t now. Signed-off-by: Darrick J. Wong Reviewed-by: Dave Chinner --- fs/xfs/libxfs/xfs_format.h | 2 +- fs/xfs/libxfs/xfs_refcount.c | 6 +++--- fs/xfs/libxfs/xfs_refcount.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h index 005dd65b71cd6..371dc07233e05 100644 --- a/fs/xfs/libxfs/xfs_format.h +++ b/fs/xfs/libxfs/xfs_format.h @@ -1612,7 +1612,7 @@ unsigned int xfs_refc_block(struct xfs_mount *mp); * on the startblock. This speeds up mount time deletion of stale * staging extents because they're all at the right side of the tree. */ -#define XFS_REFC_COW_START ((xfs_agblock_t)(1U << 31)) +#define XFS_REFC_COWFLAG (1U << 31) #define REFCNTBT_COWFLAG_BITLEN 1 #define REFCNTBT_AGBLOCK_BITLEN 31 diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c index 44d4667d43016..3f34bafe18dd1 100644 --- a/fs/xfs/libxfs/xfs_refcount.c +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -108,8 +108,8 @@ xfs_refcount_btrec_to_irec( uint32_t start; start = be32_to_cpu(rec->refc.rc_startblock); - if (start & XFS_REFC_COW_START) { - start &= ~XFS_REFC_COW_START; + if (start & XFS_REFC_COWFLAG) { + start &= ~XFS_REFC_COWFLAG; irec->rc_domain = XFS_REFC_DOMAIN_COW; } else { irec->rc_domain = XFS_REFC_DOMAIN_SHARED; @@ -1799,7 +1799,7 @@ xfs_refcount_recover_cow_leftovers( int error; /* reflink filesystems mustn't have AGs larger than 2^31-1 blocks */ - BUILD_BUG_ON(XFS_MAX_CRC_AG_BLOCKS >= XFS_REFC_COW_START); + BUILD_BUG_ON(XFS_MAX_CRC_AG_BLOCKS >= XFS_REFC_COWFLAG); if (mp->m_sb.sb_agblocks > XFS_MAX_CRC_AG_BLOCKS) return -EOPNOTSUPP; diff --git a/fs/xfs/libxfs/xfs_refcount.h b/fs/xfs/libxfs/xfs_refcount.h index ee32e8eb5a993..452f30556f5a9 100644 --- a/fs/xfs/libxfs/xfs_refcount.h +++ b/fs/xfs/libxfs/xfs_refcount.h @@ -34,9 +34,9 @@ xfs_refcount_encode_startblock( * query functions (which set rc_domain == -1U), so we check that the * domain is /not/ shared. */ - start = startblock & ~XFS_REFC_COW_START; + start = startblock & ~XFS_REFC_COWFLAG; if (domain != XFS_REFC_DOMAIN_SHARED) - start |= XFS_REFC_COW_START; + start |= XFS_REFC_COWFLAG; return start; } -- GitLab From 7a2726ec3290c52f52ce8d5f5af73ab8c7681bc1 Mon Sep 17 00:00:00 2001 From: Gavin Shan Date: Mon, 31 Oct 2022 08:36:15 +0800 Subject: [PATCH 0810/1927] KVM: Check KVM_CAP_DIRTY_LOG_{RING, RING_ACQ_REL} prior to enabling them There are two capabilities related to ring-based dirty page tracking: KVM_CAP_DIRTY_LOG_RING and KVM_CAP_DIRTY_LOG_RING_ACQ_REL. Both are supported by x86. However, arm64 supports KVM_CAP_DIRTY_LOG_RING_ACQ_REL only when the feature is supported on arm64. The userspace doesn't have to enable the advertised capability, meaning KVM_CAP_DIRTY_LOG_RING can be enabled on arm64 by userspace and it's wrong. Fix it by double checking if the capability has been advertised prior to enabling it. It's rejected to enable the capability if it hasn't been advertised. Fixes: 17601bfed909 ("KVM: Add KVM_CAP_DIRTY_LOG_RING_ACQ_REL capability and config option") Reported-by: Sean Christopherson Suggested-by: Sean Christopherson Signed-off-by: Gavin Shan Reviewed-by: Oliver Upton Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20221031003621.164306-4-gshan@redhat.com --- virt/kvm/kvm_main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 1376a47fedeed..46e8ed1ae6470 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4585,6 +4585,9 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, } case KVM_CAP_DIRTY_LOG_RING: case KVM_CAP_DIRTY_LOG_RING_ACQ_REL: + if (!kvm_vm_ioctl_check_extension_generic(kvm, cap->cap)) + return -EINVAL; + return kvm_vm_ioctl_enable_dirty_log_ring(kvm, cap->args[0]); default: return kvm_vm_ioctl_enable_cap(kvm, cap); -- GitLab From 9a1d248bb4beaf1b43d17ba12481ee0629fa29b9 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Mon, 31 Oct 2022 15:58:36 -0400 Subject: [PATCH 0811/1927] ASoC: Intel: soc-acpi: add ES83x6 support to IceLake Missing entry to find a machine driver for ES83x6-based platforms. Link: https://github.com/thesofproject/linux/issues/3873 Signed-off-by: Pierre-Louis Bossart Reviewed-by: Bard Liao Link: https://lore.kernel.org/r/20221031195836.250193-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/intel/common/soc-acpi-intel-icl-match.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sound/soc/intel/common/soc-acpi-intel-icl-match.c b/sound/soc/intel/common/soc-acpi-intel-icl-match.c index b032bc07de8bf..d0062f2cd2566 100644 --- a/sound/soc/intel/common/soc-acpi-intel-icl-match.c +++ b/sound/soc/intel/common/soc-acpi-intel-icl-match.c @@ -10,6 +10,11 @@ #include #include "../skylake/skl.h" +static const struct snd_soc_acpi_codecs essx_83x6 = { + .num_codecs = 3, + .codecs = { "ESSX8316", "ESSX8326", "ESSX8336"}, +}; + static struct skl_machine_pdata icl_pdata = { .use_tplg_pcm = true, }; @@ -27,6 +32,14 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_icl_machines[] = { .drv_name = "sof_rt5682", .sof_tplg_filename = "sof-icl-rt5682.tplg", }, + { + .comp_ids = &essx_83x6, + .drv_name = "sof-essx8336", + .sof_tplg_filename = "sof-icl-es8336", /* the tplg suffix is added at run time */ + .tplg_quirk_mask = SND_SOC_ACPI_TPLG_INTEL_SSP_NUMBER | + SND_SOC_ACPI_TPLG_INTEL_SSP_MSB | + SND_SOC_ACPI_TPLG_INTEL_DMIC_NUMBER, + }, {}, }; EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_icl_machines); -- GitLab From 40ff21432883216aa440b6619d559ad8f7d7a7d9 Mon Sep 17 00:00:00 2001 From: Andreas Schwab Date: Mon, 31 Oct 2022 14:23:13 +0100 Subject: [PATCH 0812/1927] asm-generic: compat: fix compat_arg_u64() and compat_arg_u64_dual() The macros are defined backwards. This affects the following compat syscalls: - compat_sys_truncate64() - compat_sys_ftruncate64() - compat_sys_fallocate() - compat_sys_sync_file_range() - compat_sys_fadvise64_64() - compat_sys_readahead() - compat_sys_pread64() - compat_sys_pwrite64() Fixes: 43d5de2b67d7 ("asm-generic: compat: Support BE for long long args in 32-bit ABIs") Signed-off-by: Andreas Schwab [mpe: Add list of affected syscalls] Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/871qqoyvni.fsf_-_@igel.home --- include/asm-generic/compat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/asm-generic/compat.h b/include/asm-generic/compat.h index aeb257ad3d1a6..8392caea398f4 100644 --- a/include/asm-generic/compat.h +++ b/include/asm-generic/compat.h @@ -15,7 +15,7 @@ #endif #ifndef compat_arg_u64 -#ifdef CONFIG_CPU_BIG_ENDIAN +#ifndef CONFIG_CPU_BIG_ENDIAN #define compat_arg_u64(name) u32 name##_lo, u32 name##_hi #define compat_arg_u64_dual(name) u32, name##_lo, u32, name##_hi #else -- GitLab From ce883a2ba310cd7c291bb66ce5d207965fca6003 Mon Sep 17 00:00:00 2001 From: Andreas Schwab Date: Mon, 31 Oct 2022 15:47:35 +0100 Subject: [PATCH 0813/1927] powerpc/32: fix syscall wrappers with 64-bit arguments With the introduction of syscall wrappers all wrappers for syscalls with 64-bit arguments must be handled specially, not only those that have unaligned 64-bit arguments. This left out the fallocate() and sync_file_range2() syscalls. Fixes: 7e92e01b7245 ("powerpc: Provide syscall wrapper") Fixes: e23750623835 ("powerpc/32: fix syscall wrappers with 64-bit arguments of unaligned register-pairs") Signed-off-by: Andreas Schwab Reviewed-by: Arnd Bergmann Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/87mt9cxd6g.fsf_-_@igel.home --- arch/powerpc/include/asm/syscalls.h | 7 +++++++ arch/powerpc/kernel/sys_ppc32.c | 13 ++++++++++++- arch/powerpc/kernel/syscalls/syscall.tbl | 7 +++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/include/asm/syscalls.h b/arch/powerpc/include/asm/syscalls.h index a1142496cd588..6d51b007b59e5 100644 --- a/arch/powerpc/include/asm/syscalls.h +++ b/arch/powerpc/include/asm/syscalls.h @@ -104,6 +104,13 @@ long sys_ppc_ftruncate64(unsigned int fd, u32 reg4, unsigned long len1, unsigned long len2); long sys_ppc32_fadvise64(int fd, u32 unused, u32 offset1, u32 offset2, size_t len, int advice); +long sys_ppc_sync_file_range2(int fd, unsigned int flags, + unsigned int offset1, + unsigned int offset2, + unsigned int nbytes1, + unsigned int nbytes2); +long sys_ppc_fallocate(int fd, int mode, u32 offset1, u32 offset2, + u32 len1, u32 len2); #endif #ifdef CONFIG_COMPAT long compat_sys_mmap2(unsigned long addr, size_t len, diff --git a/arch/powerpc/kernel/sys_ppc32.c b/arch/powerpc/kernel/sys_ppc32.c index 1ab4a4d95abaf..d451a8229223a 100644 --- a/arch/powerpc/kernel/sys_ppc32.c +++ b/arch/powerpc/kernel/sys_ppc32.c @@ -112,7 +112,7 @@ PPC32_SYSCALL_DEFINE6(ppc32_fadvise64, advice); } -COMPAT_SYSCALL_DEFINE6(ppc_sync_file_range2, +PPC32_SYSCALL_DEFINE6(ppc_sync_file_range2, int, fd, unsigned int, flags, unsigned int, offset1, unsigned int, offset2, unsigned int, nbytes1, unsigned int, nbytes2) @@ -122,3 +122,14 @@ COMPAT_SYSCALL_DEFINE6(ppc_sync_file_range2, return ksys_sync_file_range(fd, offset, nbytes, flags); } + +#ifdef CONFIG_PPC32 +SYSCALL_DEFINE6(ppc_fallocate, + int, fd, int, mode, + u32, offset1, u32, offset2, u32, len1, u32, len2) +{ + return ksys_fallocate(fd, mode, + merge_64(offset1, offset2), + merge_64(len1, len2)); +} +#endif diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index e9e0df4f9a61a..a0be127475b1f 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -394,8 +394,11 @@ 305 common signalfd sys_signalfd compat_sys_signalfd 306 common timerfd_create sys_timerfd_create 307 common eventfd sys_eventfd -308 common sync_file_range2 sys_sync_file_range2 compat_sys_ppc_sync_file_range2 -309 nospu fallocate sys_fallocate compat_sys_fallocate +308 32 sync_file_range2 sys_ppc_sync_file_range2 compat_sys_ppc_sync_file_range2 +308 64 sync_file_range2 sys_sync_file_range2 +308 spu sync_file_range2 sys_sync_file_range2 +309 32 fallocate sys_ppc_fallocate compat_sys_fallocate +309 64 fallocate sys_fallocate 310 nospu subpage_prot sys_subpage_prot 311 32 timerfd_settime sys_timerfd_settime32 311 64 timerfd_settime sys_timerfd_settime -- GitLab From 878eb6e48f240d02ed1c9298020a0b6370695f24 Mon Sep 17 00:00:00 2001 From: Al Viro Date: Tue, 1 Nov 2022 00:54:13 +0000 Subject: [PATCH 0814/1927] block: blk_add_rq_to_plug(): clear stale 'last' after flush blk_mq_flush_plug_list() empties ->mq_list and request we'd peeked there before that call is gone; in any case, we are not dealing with a mix of requests for different queues now - there's no requests left in the plug. Signed-off-by: Al Viro Signed-off-by: Jens Axboe --- block/blk-mq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block/blk-mq.c b/block/blk-mq.c index 21cc7c2da0f96..6a789cda68a5b 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1262,6 +1262,7 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq) (!blk_queue_nomerges(rq->q) && blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { blk_mq_flush_plug_list(plug, false); + last = NULL; trace_block_plug(rq->q); } -- GitLab From e230d36f7d4cf1b89614e2bb4c2f0c55a16d3259 Mon Sep 17 00:00:00 2001 From: Rick Lindsley Date: Fri, 28 Oct 2022 13:35:11 -0700 Subject: [PATCH 0815/1927] ibmvnic: change maintainers for vnic driver Changed maintainers for vnic driver, since Dany has new responsibilities. Also added Nick Child as reviewer. Signed-off-by: Rick Lindsley Link: https://lore.kernel.org/r/20221028203509.4070154-1-ricklind@us.ibm.com Signed-off-by: Jakub Kicinski --- MAINTAINERS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 9e437612dd816..9b297be5c6ed4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9775,7 +9775,10 @@ S: Supported F: drivers/pci/hotplug/rpaphp* IBM Power SRIOV Virtual NIC Device Driver -M: Dany Madden +M: Haren Myneni +M: Rick Lindsley +R: Nick Child +R: Dany Madden R: Thomas Falcon L: netdev@vger.kernel.org S: Supported -- GitLab From 363a5328f4b0517e59572118ccfb7c626d81dca9 Mon Sep 17 00:00:00 2001 From: Ziyang Xuan Date: Sat, 29 Oct 2022 17:41:01 +0800 Subject: [PATCH 0816/1927] net: tun: fix bugs for oversize packet when napi frags enabled Recently, we got two syzkaller problems because of oversize packet when napi frags enabled. One of the problems is because the first seg size of the iov_iter from user space is very big, it is 2147479538 which is bigger than the threshold value for bail out early in __alloc_pages(). And skb->pfmemalloc is true, __kmalloc_reserve() would use pfmemalloc reserves without __GFP_NOWARN flag. Thus we got a warning as following: ======================================================== WARNING: CPU: 1 PID: 17965 at mm/page_alloc.c:5295 __alloc_pages+0x1308/0x16c4 mm/page_alloc.c:5295 ... Call trace: __alloc_pages+0x1308/0x16c4 mm/page_alloc.c:5295 __alloc_pages_node include/linux/gfp.h:550 [inline] alloc_pages_node include/linux/gfp.h:564 [inline] kmalloc_large_node+0x94/0x350 mm/slub.c:4038 __kmalloc_node_track_caller+0x620/0x8e4 mm/slub.c:4545 __kmalloc_reserve.constprop.0+0x1e4/0x2b0 net/core/skbuff.c:151 pskb_expand_head+0x130/0x8b0 net/core/skbuff.c:1654 __skb_grow include/linux/skbuff.h:2779 [inline] tun_napi_alloc_frags+0x144/0x610 drivers/net/tun.c:1477 tun_get_user+0x31c/0x2010 drivers/net/tun.c:1835 tun_chr_write_iter+0x98/0x100 drivers/net/tun.c:2036 The other problem is because odd IPv6 packets without NEXTHDR_NONE extension header and have big packet length, it is 2127925 which is bigger than ETH_MAX_MTU(65535). After ipv6_gso_pull_exthdrs() in ipv6_gro_receive(), network_header offset and transport_header offset are all bigger than U16_MAX. That would trigger skb->network_header and skb->transport_header overflow error, because they are all '__u16' type. Eventually, it would affect the value for __skb_push(skb, value), and make it be a big value. After __skb_push() in ipv6_gro_receive(), skb->data would less than skb->head, an out of bounds memory bug occurred. That would trigger the problem as following: ================================================================== BUG: KASAN: use-after-free in eth_type_trans+0x100/0x260 ... Call trace: dump_backtrace+0xd8/0x130 show_stack+0x1c/0x50 dump_stack_lvl+0x64/0x7c print_address_description.constprop.0+0xbc/0x2e8 print_report+0x100/0x1e4 kasan_report+0x80/0x120 __asan_load8+0x78/0xa0 eth_type_trans+0x100/0x260 napi_gro_frags+0x164/0x550 tun_get_user+0xda4/0x1270 tun_chr_write_iter+0x74/0x130 do_iter_readv_writev+0x130/0x1ec do_iter_write+0xbc/0x1e0 vfs_writev+0x13c/0x26c To fix the problems, restrict the packet size less than (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN) which has considered reserved skb space in napi_alloc_skb() because transport_header is an offset from skb->head. Add len check in tun_napi_alloc_frags() simply. Fixes: 90e33d459407 ("tun: enable napi_gro_frags() for TUN/TAP driver") Signed-off-by: Ziyang Xuan Reviewed-by: Eric Dumazet Link: https://lore.kernel.org/r/20221029094101.1653855-1-william.xuanziyang@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/tun.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 27c6d235cbda3..946628050f282 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1459,7 +1459,8 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, int err; int i; - if (it->nr_segs > MAX_SKB_FRAGS + 1) + if (it->nr_segs > MAX_SKB_FRAGS + 1 || + len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN)) return ERR_PTR(-EMSGSIZE); local_bh_disable(); -- GitLab From 02a771c9a68a9f08cce4ec5e324fb1bc4dce7202 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Tue, 1 Nov 2022 14:48:52 +1100 Subject: [PATCH 0817/1927] powerpc/32: Select ARCH_SPLIT_ARG64 On 32-bit kernels, 64-bit syscall arguments are split into two registers. For that to work with syscall wrappers, the prototype of the syscall must have the argument split so that the wrapper macro properly unpacks the arguments from pt_regs. The fanotify_mark() syscall is one such syscall, which already has a split prototype, guarded behind ARCH_SPLIT_ARG64. So select ARCH_SPLIT_ARG64 to get that prototype and fix fanotify_mark() on 32-bit kernels with syscall wrappers. Note also that fanotify_mark() is the only usage of ARCH_SPLIT_ARG64. Fixes: 7e92e01b7245 ("powerpc: Provide syscall wrapper") Signed-off-by: Michael Ellerman Link: https://lore.kernel.org/r/20221101034852.2340319-1-mpe@ellerman.id.au --- arch/powerpc/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 20fb1765238c4..2ca5418457ed2 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -147,6 +147,7 @@ config PPC select ARCH_MIGHT_HAVE_PC_SERIO select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT + select ARCH_SPLIT_ARG64 if PPC32 select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x -- GitLab From ca1547ab15f48dc81624183ae17a2fd1bad06dfc Mon Sep 17 00:00:00 2001 From: Saravanan Sekar Date: Sat, 29 Oct 2022 11:29:55 +0200 Subject: [PATCH 0818/1927] iio: adc: mp2629: fix potential array out of bound access Add sentinel at end of maps to avoid potential array out of bound access in iio core. Fixes: 7abd9fb64682 ("iio: adc: mp2629: Add support for mp2629 ADC driver") Signed-off-by: Saravanan Sekar Link: https://lore.kernel.org/r/20221029093000.45451-4-sravanhome@gmail.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/adc/mp2629_adc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/iio/adc/mp2629_adc.c b/drivers/iio/adc/mp2629_adc.c index f7af9af1665d9..88e947f300cfd 100644 --- a/drivers/iio/adc/mp2629_adc.c +++ b/drivers/iio/adc/mp2629_adc.c @@ -57,7 +57,8 @@ static struct iio_map mp2629_adc_maps[] = { MP2629_MAP(SYSTEM_VOLT, "system-volt"), MP2629_MAP(INPUT_VOLT, "input-volt"), MP2629_MAP(BATT_CURRENT, "batt-current"), - MP2629_MAP(INPUT_CURRENT, "input-current") + MP2629_MAP(INPUT_CURRENT, "input-current"), + { } }; static int mp2629_read_raw(struct iio_dev *indio_dev, -- GitLab From 65f20301607d07ee279b0804d11a05a62a6c1a1c Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 24 Oct 2022 16:45:11 +0800 Subject: [PATCH 0819/1927] iio: adc: at91_adc: fix possible memory leak in at91_adc_allocate_trigger() If iio_trigger_register() returns error, it should call iio_trigger_free() to give up the reference that hold in iio_trigger_alloc(), so that it can call iio_trig_release() to free memory when the refcount hit to 0. Fixes: 0e589d5fb317 ("ARM: AT91: IIO: Add AT91 ADC driver.") Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221024084511.815096-1-yangyingliang@huawei.com Cc: Signed-off-by: Jonathan Cameron --- drivers/iio/adc/at91_adc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/iio/adc/at91_adc.c b/drivers/iio/adc/at91_adc.c index 532daaa6f943c..366e252ebeb07 100644 --- a/drivers/iio/adc/at91_adc.c +++ b/drivers/iio/adc/at91_adc.c @@ -634,8 +634,10 @@ static struct iio_trigger *at91_adc_allocate_trigger(struct iio_dev *idev, trig->ops = &at91_adc_trigger_ops; ret = iio_trigger_register(trig); - if (ret) + if (ret) { + iio_trigger_free(trig); return NULL; + } return trig; } -- GitLab From dd4753f88f242f46d0d8726f5936f64e754a47f2 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Fri, 14 Oct 2022 12:39:52 +0300 Subject: [PATCH 0820/1927] iio: imu: bno055: uninitialized variable bug in bno055_trigger_handler() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This bug is basically harmless, although it will trigger a runtime warning if you use KMSan. On the first iteration through the loop, the "best_delta" variable is uninitialized so re-order the condition to prevent reading uninitialized memory. Fixes: 4aefe1c2bd0c ("iio: imu: add Bosch Sensortec BNO055 core driver") Signed-off-by: Dan Carpenter Acked-by: Nuno Sá Link: https://lore.kernel.org/r/Y0kuaO9PQkSQja+A@kili Signed-off-by: Jonathan Cameron --- drivers/iio/imu/bno055/bno055.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c index 307557a609e3f..52744dd98e65b 100644 --- a/drivers/iio/imu/bno055/bno055.c +++ b/drivers/iio/imu/bno055/bno055.c @@ -632,7 +632,7 @@ static int bno055_set_regmask(struct bno055_priv *priv, int val, int val2, return -EINVAL; } delta = abs(tbl_val - req_val); - if (delta < best_delta || first) { + if (first || delta < best_delta) { best_delta = delta; hwval = i; first = false; -- GitLab From 7c919b619bcc68158921b1bd968f0e704549bbb6 Mon Sep 17 00:00:00 2001 From: Matti Vaittinen Date: Fri, 14 Oct 2022 10:15:19 +0300 Subject: [PATCH 0821/1927] tools: iio: iio_generic_buffer: Fix read size When noevents is true and small buffer is used the allocated memory for holding the data may be smaller than the hard-coded 64 bytes. This can cause the iio_generic_buffer to crash. Following was recorded on beagle bone black with v6.0 kernel and the digit fix patch: https://lore.kernel.org/all/Y0f+tKCz+ZAIoroQ@dc75zzyyyyyyyyyyyyycy-3.rev.dnainternet.fi/ using valgrind; ==339== Using Valgrind-3.18.1 and LibVEX; rerun with -h for copyright info ==339== Command: /iio_generic_buffer -n kx022-accel -T0 -e -l 10 -a -w 2000000 ==339== Parent PID: 307 ==339== ==339== Syscall param read(buf) points to unaddressable byte(s) ==339== at 0x496BFA4: read (read.c:26) ==339== by 0x11699: main (iio_generic_buffer.c:724) ==339== Address 0x4ab3518 is 0 bytes after a block of size 160 alloc'd ==339== at 0x4864B70: malloc (vg_replace_malloc.c:381) ==339== by 0x115BB: main (iio_generic_buffer.c:677) Fix this by always using the same size for reading as was used for data storage allocation. Signed-off-by: Matti Vaittinen Link: https://lore.kernel.org/r/Y0kMh0t5qUXJw3nQ@dc75zzyyyyyyyyyyyyycy-3.rev.dnainternet.fi Signed-off-by: Jonathan Cameron --- tools/iio/iio_generic_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/iio/iio_generic_buffer.c b/tools/iio/iio_generic_buffer.c index 2491c54a5e4fb..f8deae4e26a15 100644 --- a/tools/iio/iio_generic_buffer.c +++ b/tools/iio/iio_generic_buffer.c @@ -715,12 +715,12 @@ int main(int argc, char **argv) continue; } - toread = buf_len; } else { usleep(timedelay); - toread = 64; } + toread = buf_len; + read_size = read(buf_fd, data, toread * scan_size); if (read_size < 0) { if (errno == EAGAIN) { -- GitLab From e6cb8769452e8236b52134e5cb4a18b8f5986932 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 24 Oct 2022 18:28:43 +0200 Subject: [PATCH 0822/1927] wifi: airo: do not assign -1 to unsigned char With char becoming unsigned by default, and with `char` alone being ambiguous and based on architecture, we get a warning when assigning the unchecked output of hex_to_bin() to that unsigned char. Mark `key` as a `u8`, which matches the struct's type, and then check each call to hex_to_bin() before casting. Cc: Kalle Valo Cc: linux-wireless@vger.kernel.org Signed-off-by: Jason A. Donenfeld Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20221024162843.535921-1-Jason@zx2c4.com --- drivers/net/wireless/cisco/airo.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/net/wireless/cisco/airo.c b/drivers/net/wireless/cisco/airo.c index 10daef81c3553..fb2c35bd73bb1 100644 --- a/drivers/net/wireless/cisco/airo.c +++ b/drivers/net/wireless/cisco/airo.c @@ -5232,7 +5232,7 @@ static int get_wep_tx_idx(struct airo_info *ai) return -1; } -static int set_wep_key(struct airo_info *ai, u16 index, const char *key, +static int set_wep_key(struct airo_info *ai, u16 index, const u8 *key, u16 keylen, int perm, int lock) { static const unsigned char macaddr[ETH_ALEN] = { 0x01, 0, 0, 0, 0, 0 }; @@ -5283,7 +5283,7 @@ static void proc_wepkey_on_close(struct inode *inode, struct file *file) struct net_device *dev = pde_data(inode); struct airo_info *ai = dev->ml_priv; int i, rc; - char key[16]; + u8 key[16]; u16 index = 0; int j = 0; @@ -5311,12 +5311,22 @@ static void proc_wepkey_on_close(struct inode *inode, struct file *file) } for (i = 0; i < 16*3 && data->wbuffer[i+j]; i++) { + int val; + + if (i % 3 == 2) + continue; + + val = hex_to_bin(data->wbuffer[i+j]); + if (val < 0) { + airo_print_err(ai->dev->name, "WebKey passed invalid key hex"); + return; + } switch(i%3) { case 0: - key[i/3] = hex_to_bin(data->wbuffer[i+j])<<4; + key[i/3] = (u8)val << 4; break; case 1: - key[i/3] |= hex_to_bin(data->wbuffer[i+j]); + key[i/3] |= (u8)val; break; } } -- GitLab From cef8cdc0d0e7c701fe4dcfba4ed3fd25d28a6020 Mon Sep 17 00:00:00 2001 From: Claudiu Beznea Date: Wed, 26 Oct 2022 15:41:04 +0300 Subject: [PATCH 0823/1927] ARM: at91: pm: avoid soft resetting AC DLL Do not soft reset AC DLL as controller is buggy and this operation my introduce glitches in the controller leading to undefined behavior. Fixes: f0bbf17958e8 ("ARM: at91: pm: add self-refresh support for sama7g5") Depends-on: a02875c4cbd6 ("ARM: at91: pm: fix self-refresh for sama7g5") Signed-off-by: Claudiu Beznea Link: https://lore.kernel.org/r/20221026124114.985876-2-claudiu.beznea@microchip.com --- arch/arm/mach-at91/pm_suspend.S | 7 ++++++- include/soc/at91/sama7-ddr.h | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/arch/arm/mach-at91/pm_suspend.S b/arch/arm/mach-at91/pm_suspend.S index ffed4d9490428..e4904faf17532 100644 --- a/arch/arm/mach-at91/pm_suspend.S +++ b/arch/arm/mach-at91/pm_suspend.S @@ -169,10 +169,15 @@ sr_ena_2: cmp tmp1, #UDDRC_STAT_SELFREF_TYPE_SW bne sr_ena_2 - /* Put DDR PHY's DLL in bypass mode for non-backup modes. */ + /* Disable DX DLLs for non-backup modes. */ cmp r7, #AT91_PM_BACKUP beq sr_ena_3 + /* Do not soft reset the AC DLL. */ + ldr tmp1, [r3, DDR3PHY_ACDLLCR] + bic tmp1, tmp1, DDR3PHY_ACDLLCR_DLLSRST + str tmp1, [r3, DDR3PHY_ACDLLCR] + /* Disable DX DLLs. */ ldr tmp1, [r3, #DDR3PHY_DX0DLLCR] orr tmp1, tmp1, #DDR3PHY_DXDLLCR_DLLDIS diff --git a/include/soc/at91/sama7-ddr.h b/include/soc/at91/sama7-ddr.h index 6ce3bd22f6c69..5ad7ac2e3a7ce 100644 --- a/include/soc/at91/sama7-ddr.h +++ b/include/soc/at91/sama7-ddr.h @@ -26,7 +26,10 @@ #define DDR3PHY_PGSR (0x0C) /* DDR3PHY PHY General Status Register */ #define DDR3PHY_PGSR_IDONE (1 << 0) /* Initialization Done */ -#define DDR3PHY_ACIOCR (0x24) /* DDR3PHY AC I/O Configuration Register */ +#define DDR3PHY_ACDLLCR (0x14) /* DDR3PHY AC DLL Control Register */ +#define DDR3PHY_ACDLLCR_DLLSRST (1 << 30) /* DLL Soft Reset */ + +#define DDR3PHY_ACIOCR (0x24) /* DDR3PHY AC I/O Configuration Register */ #define DDR3PHY_ACIOCR_CSPDD_CS0 (1 << 18) /* CS#[0] Power Down Driver */ #define DDR3PHY_ACIOCR_CKPDD_CK0 (1 << 8) /* CK[0] Power Down Driver */ #define DDR3PHY_ACIORC_ACPDD (1 << 3) /* AC Power Down Driver */ -- GitLab From 10e794bdba2734bad0d61864e7e526b13b75aa6b Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Mon, 31 Oct 2022 15:55:05 -0400 Subject: [PATCH 0824/1927] ALSA: hda: clarify comments on SCF changes The commit 1f9d3d98694b1 ("ALSA: hda - set intel audio clock to a proper value") added a number of misleading comments. There is no ability to detect if an SCF value was set or not, what the code does is prevent the use of the 6MHz audio clock represented by the value 0 in LCTL.SCF. Changing the SCF settings does require the link to be power-cycled, but in all other cases the link is powered automatically when exiting reset. In other words, the power-cycle is an exception to the rule that the HDaudio legacy driver does not need to program SPA/CPA bits. In addition, the SCF related changes are only relevant for the first link. No functionality change, only comment clarifications. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Reviewed-by: Rander Wang Link: https://lore.kernel.org/r/20221031195505.249929-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Takashi Iwai --- sound/pci/hda/hda_intel.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index 6ff19dd0d10c2..92f66bbc5c91e 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -485,8 +485,8 @@ static int intel_ml_lctl_set_power(struct azx *chip, int state) int timeout; /* - * the codecs are sharing the first link setting by default - * If other links are enabled for stream, they need similar fix + * Changes to LCTL.SCF are only needed for the first multi-link dealing + * with external codecs */ val = readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); val &= ~AZX_ML_LCTL_SPA; @@ -513,7 +513,7 @@ static void intel_init_lctl(struct azx *chip) /* 0. check lctl register value is correct or not */ val = readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); - /* if SCF is already set, let's use it */ + /* only perform additional configurations if the SCF is initially based on 6MHz */ if ((val & AZX_ML_LCTL_SCF) != 0) return; @@ -531,7 +531,7 @@ static void intel_init_lctl(struct azx *chip) if (ret) goto set_spa; - /* 2. update SCF to select a properly audio clock*/ + /* 2. update SCF to select an audio clock different from 6MHz */ val &= ~AZX_ML_LCTL_SCF; val |= intel_get_lctl_scf(chip); writel(val, bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL); -- GitLab From 5d73263f9e7c54ccb20814dc50809b9deb9e2bc7 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Mon, 31 Oct 2022 15:56:39 -0400 Subject: [PATCH 0825/1927] ASoC: hda: intel-dsp-config: add ES83x6 quirk for IceLake Yet another hardware variant we need to handle. Link: https://github.com/thesofproject/linux/issues/3873 Signed-off-by: Pierre-Louis Bossart Reviewed-by: Bard Liao Reviewed-by: Takashi Iwai Link: https://lore.kernel.org/r/20221031195639.250062-1-pierre-louis.bossart@linux.intel.com Signed-off-by: Mark Brown --- sound/hda/intel-dsp-config.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sound/hda/intel-dsp-config.c b/sound/hda/intel-dsp-config.c index b9eb3208f2888..ae31bb1275940 100644 --- a/sound/hda/intel-dsp-config.c +++ b/sound/hda/intel-dsp-config.c @@ -320,6 +320,11 @@ static const struct config_entry config_table[] = { {} } }, + { + .flags = FLAG_SOF, + .device = 0x34c8, + .codec_hid = &essx_83x6, + }, { .flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE, .device = 0x34c8, -- GitLab From 6788ba8aed4e28e90f72d68a9d794e34eac17295 Mon Sep 17 00:00:00 2001 From: Dokyung Song Date: Fri, 21 Oct 2022 15:13:59 +0900 Subject: [PATCH 0826/1927] wifi: brcmfmac: Fix potential buffer overflow in brcmf_fweh_event_worker() This patch fixes an intra-object buffer overflow in brcmfmac that occurs when the device provides a 'bsscfgidx' equal to or greater than the buffer size. The patch adds a check that leads to a safe failure if that is the case. This fixes CVE-2022-3628. UBSAN: array-index-out-of-bounds in drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index 52 is out of range for type 'brcmf_if *[16]' CPU: 0 PID: 1898 Comm: kworker/0:2 Tainted: G O 5.14.0+ #132 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.1-0-ga5cab58e9a3f-prebuilt.qemu.org 04/01/2014 Workqueue: events brcmf_fweh_event_worker Call Trace: dump_stack_lvl+0x57/0x7d ubsan_epilogue+0x5/0x40 __ubsan_handle_out_of_bounds+0x69/0x80 ? memcpy+0x39/0x60 brcmf_fweh_event_worker+0xae1/0xc00 ? brcmf_fweh_call_event_handler.isra.0+0x100/0x100 ? rcu_read_lock_sched_held+0xa1/0xd0 ? rcu_read_lock_bh_held+0xb0/0xb0 ? lockdep_hardirqs_on_prepare+0x273/0x3e0 process_one_work+0x873/0x13e0 ? lock_release+0x640/0x640 ? pwq_dec_nr_in_flight+0x320/0x320 ? rwlock_bug.part.0+0x90/0x90 worker_thread+0x8b/0xd10 ? __kthread_parkme+0xd9/0x1d0 ? process_one_work+0x13e0/0x13e0 kthread+0x379/0x450 ? _raw_spin_unlock_irq+0x24/0x30 ? set_kthread_struct+0x100/0x100 ret_from_fork+0x1f/0x30 ================================================================================ general protection fault, probably for non-canonical address 0xe5601c0020023fff: 0000 [#1] SMP KASAN KASAN: maybe wild-memory-access in range [0x2b0100010011fff8-0x2b0100010011ffff] CPU: 0 PID: 1898 Comm: kworker/0:2 Tainted: G O 5.14.0+ #132 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.12.1-0-ga5cab58e9a3f-prebuilt.qemu.org 04/01/2014 Workqueue: events brcmf_fweh_event_worker RIP: 0010:brcmf_fweh_call_event_handler.isra.0+0x42/0x100 Code: 89 f5 53 48 89 fb 48 83 ec 08 e8 79 0b 38 fe 48 85 ed 74 7e e8 6f 0b 38 fe 48 89 ea 48 b8 00 00 00 00 00 fc ff df 48 c1 ea 03 <80> 3c 02 00 0f 85 8b 00 00 00 4c 8b 7d 00 44 89 e0 48 ba 00 00 00 RSP: 0018:ffffc9000259fbd8 EFLAGS: 00010207 RAX: dffffc0000000000 RBX: ffff888115d8cd50 RCX: 0000000000000000 RDX: 0560200020023fff RSI: ffffffff8304bc91 RDI: ffff888115d8cd50 RBP: 2b0100010011ffff R08: ffff888112340050 R09: ffffed1023549809 R10: ffff88811aa4c047 R11: ffffed1023549808 R12: 0000000000000045 R13: ffffc9000259fca0 R14: ffff888112340050 R15: ffff888112340000 FS: 0000000000000000(0000) GS:ffff88811aa00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000000004053ccc0 CR3: 0000000112740000 CR4: 0000000000750ef0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Call Trace: brcmf_fweh_event_worker+0x117/0xc00 ? brcmf_fweh_call_event_handler.isra.0+0x100/0x100 ? rcu_read_lock_sched_held+0xa1/0xd0 ? rcu_read_lock_bh_held+0xb0/0xb0 ? lockdep_hardirqs_on_prepare+0x273/0x3e0 process_one_work+0x873/0x13e0 ? lock_release+0x640/0x640 ? pwq_dec_nr_in_flight+0x320/0x320 ? rwlock_bug.part.0+0x90/0x90 worker_thread+0x8b/0xd10 ? __kthread_parkme+0xd9/0x1d0 ? process_one_work+0x13e0/0x13e0 kthread+0x379/0x450 ? _raw_spin_unlock_irq+0x24/0x30 ? set_kthread_struct+0x100/0x100 ret_from_fork+0x1f/0x30 Modules linked in: 88XXau(O) 88x2bu(O) ---[ end trace 41d302138f3ff55a ]--- RIP: 0010:brcmf_fweh_call_event_handler.isra.0+0x42/0x100 Code: 89 f5 53 48 89 fb 48 83 ec 08 e8 79 0b 38 fe 48 85 ed 74 7e e8 6f 0b 38 fe 48 89 ea 48 b8 00 00 00 00 00 fc ff df 48 c1 ea 03 <80> 3c 02 00 0f 85 8b 00 00 00 4c 8b 7d 00 44 89 e0 48 ba 00 00 00 RSP: 0018:ffffc9000259fbd8 EFLAGS: 00010207 RAX: dffffc0000000000 RBX: ffff888115d8cd50 RCX: 0000000000000000 RDX: 0560200020023fff RSI: ffffffff8304bc91 RDI: ffff888115d8cd50 RBP: 2b0100010011ffff R08: ffff888112340050 R09: ffffed1023549809 R10: ffff88811aa4c047 R11: ffffed1023549808 R12: 0000000000000045 R13: ffffc9000259fca0 R14: ffff888112340050 R15: ffff888112340000 FS: 0000000000000000(0000) GS:ffff88811aa00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000000004053ccc0 CR3: 0000000112740000 CR4: 0000000000750ef0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Kernel panic - not syncing: Fatal exception Reported-by: Dokyung Song Reported-by: Jisoo Jang Reported-by: Minsuk Kang Reviewed-by: Arend van Spriel Cc: Signed-off-by: Dokyung Song Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20221021061359.GA550858@laguna --- drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c index bc3f4e4edcdf9..dac7eb77799bd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fweh.c @@ -228,6 +228,10 @@ static void brcmf_fweh_event_worker(struct work_struct *work) brcmf_fweh_event_name(event->code), event->code, event->emsg.ifidx, event->emsg.bsscfgidx, event->emsg.addr); + if (event->emsg.bsscfgidx >= BRCMF_MAX_IFS) { + bphy_err(drvr, "invalid bsscfg index: %u\n", event->emsg.bsscfgidx); + goto event_free; + } /* convert event message */ emsg_be = &event->emsg; -- GitLab From d4bc8271db21ea9f1c86a1ca4d64999f184d4aae Mon Sep 17 00:00:00 2001 From: Pablo Neira Ayuso Date: Wed, 26 Oct 2022 09:52:36 +0200 Subject: [PATCH 0827/1927] netfilter: nf_tables: netlink notifier might race to release objects commit release path is invoked via call_rcu and it runs lockless to release the objects after rcu grace period. The netlink notifier handler might win race to remove objects that the transaction context is still referencing from the commit release path. Call rcu_barrier() to ensure pending rcu callbacks run to completion if the list of transactions to be destroyed is not empty. Fixes: 6001a930ce03 ("netfilter: nftables: introduce table ownership") Reported-by: syzbot+8f747f62763bc6c32916@syzkaller.appspotmail.com Signed-off-by: Pablo Neira Ayuso --- net/netfilter/nf_tables_api.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 58d9cbc9ccdc7..2197118aa7b09 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -10030,6 +10030,8 @@ static int nft_rcv_nl_event(struct notifier_block *this, unsigned long event, nft_net = nft_pernet(net); deleted = 0; mutex_lock(&nft_net->commit_mutex); + if (!list_empty(&nf_tables_destroy_list)) + rcu_barrier(); again: list_for_each_entry(table, &nft_net->tables, list) { if (nft_table_has_owner(table) && -- GitLab From 26b5934ff4194e13196bedcba373cd4915071d0e Mon Sep 17 00:00:00 2001 From: Pablo Neira Ayuso Date: Wed, 26 Oct 2022 09:54:45 +0200 Subject: [PATCH 0828/1927] netfilter: nf_tables: release flow rule object from commit path No need to postpone this to the commit release path, since no packets are walking over this object, this is accessed from control plane only. This helped uncovered UAF triggered by races with the netlink notifier. Fixes: 9dd732e0bdf5 ("netfilter: nf_tables: memleak flow rule from commit path") Reported-by: syzbot+8f747f62763bc6c32916@syzkaller.appspotmail.com Signed-off-by: Pablo Neira Ayuso --- net/netfilter/nf_tables_api.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 2197118aa7b09..76bd4d03dbda4 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -8465,9 +8465,6 @@ static void nft_commit_release(struct nft_trans *trans) nf_tables_chain_destroy(&trans->ctx); break; case NFT_MSG_DELRULE: - if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) - nft_flow_rule_destroy(nft_trans_flow_rule(trans)); - nf_tables_rule_destroy(&trans->ctx, nft_trans_rule(trans)); break; case NFT_MSG_DELSET: @@ -8973,6 +8970,9 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) nft_rule_expr_deactivate(&trans->ctx, nft_trans_rule(trans), NFT_TRANS_COMMIT); + + if (trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) + nft_flow_rule_destroy(nft_trans_flow_rule(trans)); break; case NFT_MSG_NEWSET: nft_clear(net, nft_trans_set(trans)); -- GitLab From cd73adcdbad3d9e9923b045d3643409e9c148d17 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Sun, 30 Oct 2022 22:08:54 +0100 Subject: [PATCH 0829/1927] ARM: dts: ux500: Add trips to battery thermal zones Recent changes to the thermal framework has made the trip points (trips) for thermal zones compulsory, which made the Ux500 DTS files break validation and also stopped probing because of similar changes to the code. Fix this by adding an "outer bounding box": battery thermal zones should not get warmer than 70 degress, then we will shut down. Fixes: 8c596324232d ("dt-bindings: thermal: Fix missing required property") Fixes: 3fd6d6e2b4e8 ("thermal/of: Rework the thermal device tree initialization") Signed-off-by: Linus Walleij Cc: Daniel Lezcano Cc: linux-pm@vger.kernel.org Link: https://lore.kernel.org/r/20221030210854.346662-1-linus.walleij@linaro.org' Signed-off-by: Arnd Bergmann --- arch/arm/boot/dts/ste-href.dtsi | 8 ++++++++ arch/arm/boot/dts/ste-snowball.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-codina-tmo.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-codina.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-gavini.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-golden.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-janice.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-kyle.dts | 8 ++++++++ arch/arm/boot/dts/ste-ux500-samsung-skomer.dts | 8 ++++++++ 9 files changed, 72 insertions(+) diff --git a/arch/arm/boot/dts/ste-href.dtsi b/arch/arm/boot/dts/ste-href.dtsi index fbaa0ce464271..8f1bb78fc1e48 100644 --- a/arch/arm/boot/dts/ste-href.dtsi +++ b/arch/arm/boot/dts/ste-href.dtsi @@ -24,6 +24,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-snowball.dts b/arch/arm/boot/dts/ste-snowball.dts index 1c9094f248939..e2f0cdacba7d1 100644 --- a/arch/arm/boot/dts/ste-snowball.dts +++ b/arch/arm/boot/dts/ste-snowball.dts @@ -28,6 +28,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-codina-tmo.dts b/arch/arm/boot/dts/ste-ux500-samsung-codina-tmo.dts index d6940e0afa863..27a3ab7e25e13 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-codina-tmo.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-codina-tmo.dts @@ -44,6 +44,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-codina.dts b/arch/arm/boot/dts/ste-ux500-samsung-codina.dts index 5f41256d7f4b4..b88f0c07873dd 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-codina.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-codina.dts @@ -57,6 +57,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-gavini.dts b/arch/arm/boot/dts/ste-ux500-samsung-gavini.dts index 806da3fc33cd7..7231bc7452000 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-gavini.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-gavini.dts @@ -30,6 +30,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-golden.dts b/arch/arm/boot/dts/ste-ux500-samsung-golden.dts index b0dce91aff4be..9604695edf530 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-golden.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-golden.dts @@ -35,6 +35,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-janice.dts b/arch/arm/boot/dts/ste-ux500-samsung-janice.dts index ed5c79c3d04b0..69387e8754a95 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-janice.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-janice.dts @@ -30,6 +30,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-kyle.dts b/arch/arm/boot/dts/ste-ux500-samsung-kyle.dts index c57676faf181b..167846df31045 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-kyle.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-kyle.dts @@ -34,6 +34,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; diff --git a/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts b/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts index 81b341a5ae451..93e5f5ed888d1 100644 --- a/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts +++ b/arch/arm/boot/dts/ste-ux500-samsung-skomer.dts @@ -30,6 +30,14 @@ battery-thermal { polling-delay = <0>; polling-delay-passive = <0>; thermal-sensors = <&bat_therm>; + + trips { + battery-crit-hi { + temperature = <70000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; }; -- GitLab From a0e215088ef4955f442af58822da5d2bcc3fbbe7 Mon Sep 17 00:00:00 2001 From: Jay Fang Date: Fri, 28 Oct 2022 18:54:34 +0800 Subject: [PATCH 0830/1927] MAINTAINERS: Update HiSilicon LPC BUS Driver maintainer Add Jay Fang as the maintainer of the HiSilicon LPC BUS Driver, replacing John Garry. Signed-off-by: Jay Fang Link: https://lore.kernel.org/r/20221028105434.1661264-1-f.fangjian@huawei.com' Signed-off-by: Arnd Bergmann --- MAINTAINERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index 8ae6d1a81824f..d3b729b3cbfcf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9214,7 +9214,7 @@ W: https://www.hisilicon.com F: drivers/i2c/busses/i2c-hisi.c HISILICON LPC BUS DRIVER -M: john.garry@huawei.com +M: Jay Fang S: Maintained W: http://www.hisilicon.com F: Documentation/devicetree/bindings/arm/hisilicon/low-pin-count.yaml -- GitLab From 3f4071cbd2063b917486d1047a4da47718215fee Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:26 +0100 Subject: [PATCH 0831/1927] firmware: arm_scmi: Cleanup the core driver removal callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Platform drivers .remove callbacks are not supposed to fail and report errors. Such errors are indeed ignored by the core platform drivers and the driver unbind process is anyway completed. The SCMI core platform driver as it is now, instead, bails out reporting an error in case of an explicit unbind request. Fix the removal path by adding proper device links between the core SCMI device and the SCMI protocol devices so that a full SCMI stack unbind is triggered when the core driver is removed. The remove process does not bail out anymore on the anomalous conditions triggered by an explicit unbind but the user is still warned. Reported-by: Uwe Kleine-König Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20221028140833.280091-1-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/bus.c | 11 +++++++++++ drivers/firmware/arm_scmi/common.h | 1 + drivers/firmware/arm_scmi/driver.c | 31 ++++++++++++++++++------------ 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c index d4e23101448ae..35bb70724d44b 100644 --- a/drivers/firmware/arm_scmi/bus.c +++ b/drivers/firmware/arm_scmi/bus.c @@ -216,9 +216,20 @@ void scmi_device_destroy(struct scmi_device *scmi_dev) device_unregister(&scmi_dev->dev); } +void scmi_device_link_add(struct device *consumer, struct device *supplier) +{ + struct device_link *link; + + link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER); + + WARN_ON(!link); +} + void scmi_set_handle(struct scmi_device *scmi_dev) { scmi_dev->handle = scmi_handle_get(&scmi_dev->dev); + if (scmi_dev->handle) + scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev); } int scmi_protocol_register(const struct scmi_protocol *proto) diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index 61aba7447c32a..9b87b5b695355 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -97,6 +97,7 @@ static inline void unpack_scmi_header(u32 msg_hdr, struct scmi_msg_hdr *hdr) struct scmi_revision_info * scmi_revision_area_get(const struct scmi_protocol_handle *ph); int scmi_handle_put(const struct scmi_handle *handle); +void scmi_device_link_add(struct device *consumer, struct device *supplier); struct scmi_handle *scmi_handle_get(struct device *dev); void scmi_set_handle(struct scmi_device *scmi_dev); void scmi_setup_protocol_implemented(const struct scmi_protocol_handle *ph, diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 609ebedee9cb6..7e19b6055d759 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -2273,10 +2273,16 @@ int scmi_protocol_device_request(const struct scmi_device_id *id_table) sdev = scmi_get_protocol_device(child, info, id_table->protocol_id, id_table->name); - /* Set handle if not already set: device existed */ - if (sdev && !sdev->handle) - sdev->handle = - scmi_handle_get_from_info_unlocked(info); + if (sdev) { + /* Set handle if not already set: device existed */ + if (!sdev->handle) + sdev->handle = + scmi_handle_get_from_info_unlocked(info); + /* Relink consumer and suppliers */ + if (sdev->handle) + scmi_device_link_add(&sdev->dev, + sdev->handle->dev); + } } else { dev_err(info->dev, "Failed. SCMI protocol %d not active.\n", @@ -2475,20 +2481,17 @@ void scmi_free_channel(struct scmi_chan_info *cinfo, struct idr *idr, int id) static int scmi_remove(struct platform_device *pdev) { - int ret = 0, id; + int ret, id; struct scmi_info *info = platform_get_drvdata(pdev); struct device_node *child; mutex_lock(&scmi_list_mutex); if (info->users) - ret = -EBUSY; - else - list_del(&info->node); + dev_warn(&pdev->dev, + "Still active SCMI users will be forcibly unbound.\n"); + list_del(&info->node); mutex_unlock(&scmi_list_mutex); - if (ret) - return ret; - scmi_notification_exit(&info->handle); mutex_lock(&info->protocols_mtx); @@ -2500,7 +2503,11 @@ static int scmi_remove(struct platform_device *pdev) idr_destroy(&info->active_protocols); /* Safe to free channels since no more users */ - return scmi_cleanup_txrx_channels(info); + ret = scmi_cleanup_txrx_channels(info); + if (ret) + dev_warn(&pdev->dev, "Failed to cleanup SCMI channels.\n"); + + return 0; } static ssize_t protocol_version_show(struct device *dev, -- GitLab From fd96fbc8fad35d6b1872c90df8a2f5d721f14d91 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:27 +0100 Subject: [PATCH 0832/1927] firmware: arm_scmi: Suppress the driver's bind attributes Suppress the capability to unbind the core SCMI driver since all the SCMI stack protocol drivers depend on it. Fixes: aa4f886f3893 ("firmware: arm_scmi: add basic driver infrastructure for SCMI") Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20221028140833.280091-2-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/driver.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 7e19b6055d759..94be633b55a08 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -2578,6 +2578,7 @@ MODULE_DEVICE_TABLE(of, scmi_of_match); static struct platform_driver scmi_driver = { .driver = { .name = "arm-scmi", + .suppress_bind_attrs = true, .of_match_table = scmi_of_match, .dev_groups = versions_groups, }, -- GitLab From 59172b212ec0dbb97ceb5671d912e6e61fa802d5 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:28 +0100 Subject: [PATCH 0833/1927] firmware: arm_scmi: Make tx_prepare time out eventually SCMI transports based on shared memory, at start of transmissions, have to wait for the shared Tx channel area to be eventually freed by the SCMI platform before accessing the channel. In fact the channel is owned by the SCMI platform until marked as free by the platform itself and, as such, cannot be used by the agent until relinquished. As a consequence a badly misbehaving SCMI platform firmware could lock the channel indefinitely and make the kernel side SCMI stack loop forever waiting for such channel to be freed, possibly hanging the whole boot sequence. Add a timeout to the existent Tx waiting spin-loop so that, when the system ends up in this situation, the SCMI stack can at least bail-out, nosily warn the user, and abort the transmission. Reported-by: YaxiongTian Suggested-by: YaxiongTian Cc: Vincent Guittot Cc: Etienne Carriere Cc: Florian Fainelli Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20221028140833.280091-3-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/common.h | 4 +++- drivers/firmware/arm_scmi/driver.c | 1 + drivers/firmware/arm_scmi/mailbox.c | 2 +- drivers/firmware/arm_scmi/optee.c | 2 +- drivers/firmware/arm_scmi/shmem.c | 31 +++++++++++++++++++++++++---- drivers/firmware/arm_scmi/smc.c | 2 +- 6 files changed, 34 insertions(+), 8 deletions(-) diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index 9b87b5b695355..a1c0154c31c6f 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -118,6 +118,7 @@ void scmi_protocol_release(const struct scmi_handle *handle, u8 protocol_id); * * @dev: Reference to device in the SCMI hierarchy corresponding to this * channel + * @rx_timeout_ms: The configured RX timeout in milliseconds. * @handle: Pointer to SCMI entity handle * @no_completion_irq: Flag to indicate that this channel has no completion * interrupt mechanism for synchronous commands. @@ -127,6 +128,7 @@ void scmi_protocol_release(const struct scmi_handle *handle, u8 protocol_id); */ struct scmi_chan_info { struct device *dev; + unsigned int rx_timeout_ms; struct scmi_handle *handle; bool no_completion_irq; void *transport_info; @@ -233,7 +235,7 @@ void scmi_free_channel(struct scmi_chan_info *cinfo, struct idr *idr, int id); struct scmi_shared_mem; void shmem_tx_prepare(struct scmi_shared_mem __iomem *shmem, - struct scmi_xfer *xfer); + struct scmi_xfer *xfer, struct scmi_chan_info *cinfo); u32 shmem_read_header(struct scmi_shared_mem __iomem *shmem); void shmem_fetch_response(struct scmi_shared_mem __iomem *shmem, struct scmi_xfer *xfer); diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 94be633b55a08..985775f210f99 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -2013,6 +2013,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device *dev, return -ENOMEM; cinfo->dev = dev; + cinfo->rx_timeout_ms = info->desc->max_rx_timeout_ms; ret = info->desc->ops->chan_setup(cinfo, info->dev, tx); if (ret) diff --git a/drivers/firmware/arm_scmi/mailbox.c b/drivers/firmware/arm_scmi/mailbox.c index 08ff4d110beb4..1e40cb035044d 100644 --- a/drivers/firmware/arm_scmi/mailbox.c +++ b/drivers/firmware/arm_scmi/mailbox.c @@ -36,7 +36,7 @@ static void tx_prepare(struct mbox_client *cl, void *m) { struct scmi_mailbox *smbox = client_to_scmi_mailbox(cl); - shmem_tx_prepare(smbox->shmem, m); + shmem_tx_prepare(smbox->shmem, m, smbox->cinfo); } static void rx_callback(struct mbox_client *cl, void *m) diff --git a/drivers/firmware/arm_scmi/optee.c b/drivers/firmware/arm_scmi/optee.c index f42dad997ac9a..2a7aeab40e543 100644 --- a/drivers/firmware/arm_scmi/optee.c +++ b/drivers/firmware/arm_scmi/optee.c @@ -498,7 +498,7 @@ static int scmi_optee_send_message(struct scmi_chan_info *cinfo, msg_tx_prepare(channel->req.msg, xfer); ret = invoke_process_msg_channel(channel, msg_command_size(xfer)); } else { - shmem_tx_prepare(channel->req.shmem, xfer); + shmem_tx_prepare(channel->req.shmem, xfer, cinfo); ret = invoke_process_smt_channel(channel); } diff --git a/drivers/firmware/arm_scmi/shmem.c b/drivers/firmware/arm_scmi/shmem.c index 0e3eaea5d8526..1dfe534b85184 100644 --- a/drivers/firmware/arm_scmi/shmem.c +++ b/drivers/firmware/arm_scmi/shmem.c @@ -5,10 +5,13 @@ * Copyright (C) 2019 ARM Ltd. */ +#include #include #include #include +#include + #include "common.h" /* @@ -30,16 +33,36 @@ struct scmi_shared_mem { }; void shmem_tx_prepare(struct scmi_shared_mem __iomem *shmem, - struct scmi_xfer *xfer) + struct scmi_xfer *xfer, struct scmi_chan_info *cinfo) { + ktime_t stop; + /* * Ideally channel must be free by now unless OS timeout last * request and platform continued to process the same, wait * until it releases the shared memory, otherwise we may endup - * overwriting its response with new message payload or vice-versa + * overwriting its response with new message payload or vice-versa. + * Giving up anyway after twice the expected channel timeout so as + * not to bail-out on intermittent issues where the platform is + * occasionally a bit slower to answer. + * + * Note that after a timeout is detected we bail-out and carry on but + * the transport functionality is probably permanently compromised: + * this is just to ease debugging and avoid complete hangs on boot + * due to a misbehaving SCMI firmware. */ - spin_until_cond(ioread32(&shmem->channel_status) & - SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE); + stop = ktime_add_ms(ktime_get(), 2 * cinfo->rx_timeout_ms); + spin_until_cond((ioread32(&shmem->channel_status) & + SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE) || + ktime_after(ktime_get(), stop)); + if (!(ioread32(&shmem->channel_status) & + SCMI_SHMEM_CHAN_STAT_CHANNEL_FREE)) { + WARN_ON_ONCE(1); + dev_err(cinfo->dev, + "Timeout waiting for a free TX channel !\n"); + return; + } + /* Mark channel busy + clear error */ iowrite32(0x0, &shmem->channel_status); iowrite32(xfer->hdr.poll_completion ? 0 : SCMI_SHMEM_FLAG_INTR_ENABLED, diff --git a/drivers/firmware/arm_scmi/smc.c b/drivers/firmware/arm_scmi/smc.c index 745acfdd0b3df..87a7b13cf868b 100644 --- a/drivers/firmware/arm_scmi/smc.c +++ b/drivers/firmware/arm_scmi/smc.c @@ -188,7 +188,7 @@ static int smc_send_message(struct scmi_chan_info *cinfo, */ smc_channel_lock_acquire(scmi_info, xfer); - shmem_tx_prepare(scmi_info->shmem, xfer); + shmem_tx_prepare(scmi_info->shmem, xfer, cinfo); arm_smccc_1_1_invoke(scmi_info->func_id, 0, 0, 0, 0, 0, 0, 0, &res); -- GitLab From be9ba1f7f9e0b565b19f4294f5871da9d654bc6d Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:29 +0100 Subject: [PATCH 0834/1927] firmware: arm_scmi: Make Rx chan_setup fail on memory errors SCMI Rx channels are optional and they can fail to be setup when not present but anyway channels setup routines must bail-out on memory errors. Make channels setup, and related probing, fail when memory errors are reported on Rx channels. Fixes: 5c8a47a5a91d ("firmware: arm_scmi: Make scmi core independent of the transport type") Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20221028140833.280091-4-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/driver.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 985775f210f99..f818d00bb2c69 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -2045,8 +2045,12 @@ scmi_txrx_setup(struct scmi_info *info, struct device *dev, int prot_id) { int ret = scmi_chan_setup(info, dev, prot_id, true); - if (!ret) /* Rx is optional, hence no error check */ - scmi_chan_setup(info, dev, prot_id, false); + if (!ret) { + /* Rx is optional, report only memory errors */ + ret = scmi_chan_setup(info, dev, prot_id, false); + if (ret && ret != -ENOMEM) + ret = 0; + } return ret; } -- GitLab From 5ffc1c4cb896f8d2cf10309422da3633a616d60f Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:30 +0100 Subject: [PATCH 0835/1927] firmware: arm_scmi: Fix devres allocation device in virtio transport SCMI virtio transport device managed allocations must use the main platform device in devres operations instead of the channel devices. Cc: Peter Hilber Fixes: 46abe13b5e3d ("firmware: arm_scmi: Add virtio transport") Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20221028140833.280091-5-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/virtio.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/firmware/arm_scmi/virtio.c b/drivers/firmware/arm_scmi/virtio.c index 14709dbc96a1a..36b7686843a42 100644 --- a/drivers/firmware/arm_scmi/virtio.c +++ b/drivers/firmware/arm_scmi/virtio.c @@ -444,12 +444,12 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, for (i = 0; i < vioch->max_msg; i++) { struct scmi_vio_msg *msg; - msg = devm_kzalloc(cinfo->dev, sizeof(*msg), GFP_KERNEL); + msg = devm_kzalloc(dev, sizeof(*msg), GFP_KERNEL); if (!msg) return -ENOMEM; if (tx) { - msg->request = devm_kzalloc(cinfo->dev, + msg->request = devm_kzalloc(dev, VIRTIO_SCMI_MAX_PDU_SIZE, GFP_KERNEL); if (!msg->request) @@ -458,7 +458,7 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, refcount_set(&msg->users, 1); } - msg->input = devm_kzalloc(cinfo->dev, VIRTIO_SCMI_MAX_PDU_SIZE, + msg->input = devm_kzalloc(dev, VIRTIO_SCMI_MAX_PDU_SIZE, GFP_KERNEL); if (!msg->input) return -ENOMEM; -- GitLab From 1eff6929aff594fba3182660f7b6213ec0ceda0c Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:31 +0100 Subject: [PATCH 0836/1927] firmware: arm_scmi: Fix deferred_tx_wq release on error paths Use devres to allocate the dedicated deferred_tx_wq polling workqueue so as to automatically trigger the proper resource release on error path. Reported-by: Dan Carpenter Fixes: 5a3b7185c47c ("firmware: arm_scmi: Add atomic mode support to virtio transport") Signed-off-by: Cristian Marussi Link: https://lore.kernel.org/r/20221028140833.280091-6-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- drivers/firmware/arm_scmi/virtio.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/firmware/arm_scmi/virtio.c b/drivers/firmware/arm_scmi/virtio.c index 36b7686843a42..33c9b81a55cd1 100644 --- a/drivers/firmware/arm_scmi/virtio.c +++ b/drivers/firmware/arm_scmi/virtio.c @@ -148,7 +148,6 @@ static void scmi_vio_channel_cleanup_sync(struct scmi_vio_channel *vioch) { unsigned long flags; DECLARE_COMPLETION_ONSTACK(vioch_shutdown_done); - void *deferred_wq = NULL; /* * Prepare to wait for the last release if not already released @@ -162,16 +161,11 @@ static void scmi_vio_channel_cleanup_sync(struct scmi_vio_channel *vioch) vioch->shutdown_done = &vioch_shutdown_done; virtio_break_device(vioch->vqueue->vdev); - if (!vioch->is_rx && vioch->deferred_tx_wq) { - deferred_wq = vioch->deferred_tx_wq; + if (!vioch->is_rx && vioch->deferred_tx_wq) /* Cannot be kicked anymore after this...*/ vioch->deferred_tx_wq = NULL; - } spin_unlock_irqrestore(&vioch->lock, flags); - if (deferred_wq) - destroy_workqueue(deferred_wq); - scmi_vio_channel_release(vioch); /* Let any possibly concurrent RX path release the channel */ @@ -416,6 +410,11 @@ static bool virtio_chan_available(struct device *dev, int idx) return vioch && !vioch->cinfo; } +static void scmi_destroy_tx_workqueue(void *deferred_tx_wq) +{ + destroy_workqueue(deferred_tx_wq); +} + static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, bool tx) { @@ -430,6 +429,8 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, /* Setup a deferred worker for polling. */ if (tx && !vioch->deferred_tx_wq) { + int ret; + vioch->deferred_tx_wq = alloc_workqueue(dev_name(&scmi_vdev->dev), WQ_UNBOUND | WQ_FREEZABLE | WQ_SYSFS, @@ -437,6 +438,11 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, if (!vioch->deferred_tx_wq) return -ENOMEM; + ret = devm_add_action_or_reset(dev, scmi_destroy_tx_workqueue, + vioch->deferred_tx_wq); + if (ret) + return ret; + INIT_WORK(&vioch->deferred_tx_work, scmi_vio_deferred_tx_worker); } -- GitLab From c4a7b9b587ca1bb4678d48d8be7132492b23a81c Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Fri, 28 Oct 2022 15:08:33 +0100 Subject: [PATCH 0837/1927] arm64: dts: juno: Add thermal critical trip points When thermnal zones are defined, trip points definitions are mandatory. Define a couple of critical trip points for monitoring of existing PMIC and SOC thermal zones. This was lost between txt to yaml conversion and was re-enforced recently via the commit 8c596324232d ("dt-bindings: thermal: Fix missing required property") Cc: Rob Herring Cc: Krzysztof Kozlowski Cc: devicetree@vger.kernel.org Signed-off-by: Cristian Marussi Fixes: f7b636a8d83c ("arm64: dts: juno: add thermal zones for scpi sensors") Link: https://lore.kernel.org/r/20221028140833.280091-8-cristian.marussi@arm.com Signed-off-by: Sudeep Holla --- arch/arm64/boot/dts/arm/juno-base.dtsi | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/boot/dts/arm/juno-base.dtsi b/arch/arm64/boot/dts/arm/juno-base.dtsi index 2f27619d8abd5..8b4d280b1e7e7 100644 --- a/arch/arm64/boot/dts/arm/juno-base.dtsi +++ b/arch/arm64/boot/dts/arm/juno-base.dtsi @@ -751,12 +751,26 @@ pmic { polling-delay = <1000>; polling-delay-passive = <100>; thermal-sensors = <&scpi_sensors0 0>; + trips { + pmic_crit0: trip0 { + temperature = <90000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; soc { polling-delay = <1000>; polling-delay-passive = <100>; thermal-sensors = <&scpi_sensors0 3>; + trips { + soc_crit0: trip0 { + temperature = <80000>; + hysteresis = <2000>; + type = "critical"; + }; + }; }; big_cluster_thermal_zone: big-cluster { -- GitLab From 569bea74c94d37785682b11bab76f557520477cd Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Thu, 27 Oct 2022 20:13:53 +0800 Subject: [PATCH 0838/1927] i2c: piix4: Fix adapter not be removed in piix4_remove() In piix4_probe(), the piix4 adapter will be registered in: piix4_probe() piix4_add_adapters_sb800() / piix4_add_adapter() i2c_add_adapter() Based on the probed device type, piix4_add_adapters_sb800() or single piix4_add_adapter() will be called. For the former case, piix4_adapter_count is set as the number of adapters, while for antoher case it is not set and kept default *zero*. When piix4 is removed, piix4_remove() removes the adapters added in piix4_probe(), basing on the piix4_adapter_count value. Because the count is zero for the single adapter case, the adapter won't be removed and makes the sources allocated for adapter leaked, such as the i2c client and device. These sources can still be accessed by i2c or bus and cause problems. An easily reproduced case is that if a new adapter is registered, i2c will get the leaked adapter and try to call smbus_algorithm, which was already freed: Triggered by: rmmod i2c_piix4 && modprobe max31730 BUG: unable to handle page fault for address: ffffffffc053d860 #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page Oops: 0000 [#1] PREEMPT SMP KASAN CPU: 0 PID: 3752 Comm: modprobe Tainted: G Hardware name: QEMU Standard PC (i440FX + PIIX, 1996) RIP: 0010:i2c_default_probe (drivers/i2c/i2c-core-base.c:2259) i2c_core RSP: 0018:ffff888107477710 EFLAGS: 00000246 ... i2c_detect (drivers/i2c/i2c-core-base.c:2302) i2c_core __process_new_driver (drivers/i2c/i2c-core-base.c:1336) i2c_core bus_for_each_dev (drivers/base/bus.c:301) i2c_for_each_dev (drivers/i2c/i2c-core-base.c:1823) i2c_core i2c_register_driver (drivers/i2c/i2c-core-base.c:1861) i2c_core do_one_initcall (init/main.c:1296) do_init_module (kernel/module/main.c:2455) ... ---[ end trace 0000000000000000 ]--- Fix this problem by correctly set piix4_adapter_count as 1 for the single adapter so it can be normally removed. Fixes: 528d53a1592b ("i2c: piix4: Fix probing of reserved ports on AMD Family 16h Model 30h") Signed-off-by: Chen Zhongjin Reviewed-by: Jean Delvare Signed-off-by: Wolfram Sang --- drivers/i2c/busses/i2c-piix4.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c index 39cb1b7bb8656..809fbd014cd68 100644 --- a/drivers/i2c/busses/i2c-piix4.c +++ b/drivers/i2c/busses/i2c-piix4.c @@ -1080,6 +1080,7 @@ static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id) "", &piix4_main_adapters[0]); if (retval < 0) return retval; + piix4_adapter_count = 1; } /* Check for auxiliary SMBus on some AMD chipsets */ -- GitLab From cdbf26251d3b35c4ccaea0c3a6de4318f727d3d2 Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Thu, 20 Oct 2022 16:39:33 +0200 Subject: [PATCH 0839/1927] i2c: tegra: Allocate DMA memory for DMA engine When the I2C controllers are running in DMA mode, it is the DMA engine that performs the memory accesses rather than the I2C controller. Pass the DMA engine's struct device pointer to the DMA API to make sure the correct DMA operations are used. This fixes an issue where the DMA engine's SMMU stream ID needs to be misleadingly set for the I2C controllers in device tree. Suggested-by: Robin Murphy Signed-off-by: Thierry Reding Signed-off-by: Wolfram Sang --- drivers/i2c/busses/i2c-tegra.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 954022c04cc42..3869c258a5296 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -284,6 +284,7 @@ struct tegra_i2c_dev { struct dma_chan *tx_dma_chan; struct dma_chan *rx_dma_chan; unsigned int dma_buf_size; + struct device *dma_dev; dma_addr_t dma_phys; void *dma_buf; @@ -420,7 +421,7 @@ static int tegra_i2c_dma_submit(struct tegra_i2c_dev *i2c_dev, size_t len) static void tegra_i2c_release_dma(struct tegra_i2c_dev *i2c_dev) { if (i2c_dev->dma_buf) { - dma_free_coherent(i2c_dev->dev, i2c_dev->dma_buf_size, + dma_free_coherent(i2c_dev->dma_dev, i2c_dev->dma_buf_size, i2c_dev->dma_buf, i2c_dev->dma_phys); i2c_dev->dma_buf = NULL; } @@ -472,10 +473,13 @@ static int tegra_i2c_init_dma(struct tegra_i2c_dev *i2c_dev) i2c_dev->tx_dma_chan = chan; + WARN_ON(i2c_dev->tx_dma_chan->device != i2c_dev->rx_dma_chan->device); + i2c_dev->dma_dev = chan->device->dev; + i2c_dev->dma_buf_size = i2c_dev->hw->quirks->max_write_len + I2C_PACKET_HEADER_SIZE; - dma_buf = dma_alloc_coherent(i2c_dev->dev, i2c_dev->dma_buf_size, + dma_buf = dma_alloc_coherent(i2c_dev->dma_dev, i2c_dev->dma_buf_size, &dma_phys, GFP_KERNEL | __GFP_NOWARN); if (!dma_buf) { dev_err(i2c_dev->dev, "failed to allocate DMA buffer\n"); @@ -1272,7 +1276,7 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, if (i2c_dev->dma_mode) { if (i2c_dev->msg_read) { - dma_sync_single_for_device(i2c_dev->dev, + dma_sync_single_for_device(i2c_dev->dma_dev, i2c_dev->dma_phys, xfer_size, DMA_FROM_DEVICE); @@ -1280,7 +1284,7 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, if (err) return err; } else { - dma_sync_single_for_cpu(i2c_dev->dev, + dma_sync_single_for_cpu(i2c_dev->dma_dev, i2c_dev->dma_phys, xfer_size, DMA_TO_DEVICE); } @@ -1293,7 +1297,7 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, memcpy(i2c_dev->dma_buf + I2C_PACKET_HEADER_SIZE, msg->buf, msg->len); - dma_sync_single_for_device(i2c_dev->dev, + dma_sync_single_for_device(i2c_dev->dma_dev, i2c_dev->dma_phys, xfer_size, DMA_TO_DEVICE); @@ -1344,7 +1348,7 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, } if (i2c_dev->msg_read && i2c_dev->msg_err == I2C_ERR_NONE) { - dma_sync_single_for_cpu(i2c_dev->dev, + dma_sync_single_for_cpu(i2c_dev->dma_dev, i2c_dev->dma_phys, xfer_size, DMA_FROM_DEVICE); -- GitLab From d6643d7207c572c1b0305ed505101f15502c6c87 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Thu, 6 Oct 2022 16:54:40 +0200 Subject: [PATCH 0840/1927] i2c: i801: add lis3lv02d's I2C address for Vostro 5568 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dell Vostro 5568 laptop has lis3lv02d, but its i2c address is not known to the kernel. Add this address. Output of "cat /sys/devices/platform/lis3lv02d/position" on Dell Vostro 5568 laptop: - Horizontal: (-18,0,1044) - Front elevated: (522,-18,1080) - Left elevated: (-18,-360,1080) - Upside down: (36,108,-1134) Signed-off-by: Nam Cao Reviewed-by: Jean Delvare Reviewed-by: Pali Rohár Signed-off-by: Wolfram Sang --- drivers/i2c/busses/i2c-i801.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index e06509edc5f39..1fda1eaa6d6ab 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -1243,6 +1243,7 @@ static const struct { */ { "Latitude 5480", 0x29 }, { "Vostro V131", 0x1d }, + { "Vostro 5568", 0x29 }, }; static void register_dell_lis3lv02d_i2c_device(struct i801_priv *priv) -- GitLab From d755f89220a2b49bc90b7b520bb6edeb4adb5f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jouni=20H=C3=B6gander?= Date: Mon, 24 Oct 2022 08:46:49 +0300 Subject: [PATCH 0841/1927] drm/i915/psr: Send update also on invalidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we are observing mouse cursor stuttering when using xrandr --scaling=1.2x1.2. X scaling/transformation seems to be doing fronbuffer rendering. When moving mouse cursor X seems to perform several invalidates and only one DirtyFB. I.e. it seems to be assuming updates are sent to panel while drawing is done. Earlier we were disabling PSR in frontbuffer invalidate call back (when drawing in X started). PSR was re-enabled in frontbuffer flush callback (dirtyfb ioctl). This was working fine with X scaling/transformation. Now we are just enabling continuous full frame (cff) in PSR invalidate callback. Enabling cff doesn't trigger any updates. It just configures PSR to send full frame when updates are sent. I.e. there are no updates on screen before PSR flush callback is made. X seems to be doing several updates in frontbuffer before doing dirtyfb ioctl. Fix this by sending single update on every invalidate callback. Cc: José Roberto de Souza Cc: Ville Syrjälä Cc: Mika Kahola Fixes: 805f04d42a6b ("drm/i915/display/psr: Use continuos full frame to handle frontbuffer invalidations") Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/6679 Signed-off-by: Jouni Högander Reported-by: Brian J. Tarricone Tested-by: Brian J. Tarricone Reviewed-by: Mika Kahola Reviewed-by: José Roberto de Souza Signed-off-by: José Roberto de Souza Link: https://patchwork.freedesktop.org/patch/msgid/20221024054649.31299-1-jouni.hogander@intel.com --- drivers/gpu/drm/i915/display/intel_psr.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 904a1049eff39..564d4fd94048c 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -2209,8 +2209,11 @@ static void _psr_invalidate_handle(struct intel_dp *intel_dp) if (intel_dp->psr.psr2_sel_fetch_enabled) { u32 val; - if (intel_dp->psr.psr2_sel_fetch_cff_enabled) + if (intel_dp->psr.psr2_sel_fetch_cff_enabled) { + /* Send one update otherwise lag is observed in screen */ + intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0); return; + } val = man_trk_ctl_enable_bit_get(dev_priv) | man_trk_ctl_partial_frame_bit_get(dev_priv) | -- GitLab From 42378a9ca55347102bbf86708776061d8fe3ece2 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 28 Oct 2022 19:54:30 -0700 Subject: [PATCH 0842/1927] bpf, verifier: Fix memory leak in array reallocation for stack state If an error (NULL) is returned by krealloc(), callers of realloc_array() were setting their allocation pointers to NULL, but on error krealloc() does not touch the original allocation. This would result in a memory resource leak. Instead, free the old allocation on the error handling path. The memory leak information is as follows as also reported by Zhengchao: unreferenced object 0xffff888019801800 (size 256): comm "bpf_repo", pid 6490, jiffies 4294959200 (age 17.170s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<00000000b211474b>] __kmalloc_node_track_caller+0x45/0xc0 [<0000000086712a0b>] krealloc+0x83/0xd0 [<00000000139aab02>] realloc_array+0x82/0xe2 [<00000000b1ca41d1>] grow_stack_state+0xfb/0x186 [<00000000cd6f36d2>] check_mem_access.cold+0x141/0x1341 [<0000000081780455>] do_check_common+0x5358/0xb350 [<0000000015f6b091>] bpf_check.cold+0xc3/0x29d [<000000002973c690>] bpf_prog_load+0x13db/0x2240 [<00000000028d1644>] __sys_bpf+0x1605/0x4ce0 [<00000000053f29bd>] __x64_sys_bpf+0x75/0xb0 [<0000000056fedaf5>] do_syscall_64+0x35/0x80 [<000000002bd58261>] entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: c69431aab67a ("bpf: verifier: Improve function state reallocation") Reported-by: Zhengchao Shao Reported-by: Kees Cook Signed-off-by: Kees Cook Signed-off-by: Daniel Borkmann Reviewed-by: Bill Wendling Cc: Lorenz Bauer Link: https://lore.kernel.org/bpf/20221029025433.2533810-1-keescook@chromium.org --- kernel/bpf/verifier.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7f0a9f6cb8897..dd9019c8b0db0 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -1027,12 +1027,17 @@ static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t */ static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size) { + void *new_arr; + if (!new_n || old_n == new_n) goto out; - arr = krealloc_array(arr, new_n, size, GFP_KERNEL); - if (!arr) + new_arr = krealloc_array(arr, new_n, size, GFP_KERNEL); + if (!new_arr) { + kfree(arr); return NULL; + } + arr = new_arr; if (new_n > old_n) memset(arr + old_n * size, 0, (new_n - old_n) * size); -- GitLab From ff52fe006fdeacee49745dabed26154db52a6343 Mon Sep 17 00:00:00 2001 From: Liu Ying Date: Sun, 9 Oct 2022 10:35:27 +0800 Subject: [PATCH 0843/1927] drm/imx: Kconfig: Remove duplicated 'select DRM_KMS_HELPER' line A duplicated line 'select DRM_KMS_HELPER' was introduced in Kconfig file by commit 09717af7d13d ("drm: Remove CONFIG_DRM_KMS_CMA_HELPER option"), so remove it. Fixes: 09717af7d13d ("drm: Remove CONFIG_DRM_KMS_CMA_HELPER option") Signed-off-by: Liu Ying Reviewed-by: Philipp Zabel Signed-off-by: Philipp Zabel Link: https://patchwork.freedesktop.org/patch/msgid/20221009023527.3669647-1-victor.liu@nxp.com --- drivers/gpu/drm/imx/Kconfig | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/imx/Kconfig b/drivers/gpu/drm/imx/Kconfig index 975de4ff7313c..fd5b2471fdf0a 100644 --- a/drivers/gpu/drm/imx/Kconfig +++ b/drivers/gpu/drm/imx/Kconfig @@ -4,7 +4,6 @@ config DRM_IMX select DRM_KMS_HELPER select VIDEOMODE_HELPERS select DRM_GEM_DMA_HELPER - select DRM_KMS_HELPER depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM || COMPILE_TEST) depends on IMX_IPUV3_CORE help -- GitLab From fc007fb815ab5395c3962c09b79a1630b0fbed9c Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Tue, 13 Sep 2022 13:55:44 -0700 Subject: [PATCH 0844/1927] drm/imx: imx-tve: Fix return type of imx_tve_connector_mode_valid The mode_valid field in drm_connector_helper_funcs is expected to be of type: enum drm_mode_status (* mode_valid) (struct drm_connector *connector, struct drm_display_mode *mode); The mismatched return type breaks forward edge kCFI since the underlying function definition does not match the function hook definition. The return type of imx_tve_connector_mode_valid should be changed from int to enum drm_mode_status. Reported-by: Dan Carpenter Link: https://github.com/ClangBuiltLinux/linux/issues/1703 Cc: llvm@lists.linux.dev Signed-off-by: Nathan Huckleberry Reviewed-by: Nathan Chancellor Reviewed-by: Fabio Estevam Reviewed-by: Philipp Zabel Signed-off-by: Philipp Zabel Link: https://patchwork.freedesktop.org/patch/msgid/20220913205544.155106-1-nhuck@google.com --- drivers/gpu/drm/imx/imx-tve.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/imx/imx-tve.c b/drivers/gpu/drm/imx/imx-tve.c index 6b34fac3f73a0..ab4d1c878fda3 100644 --- a/drivers/gpu/drm/imx/imx-tve.c +++ b/drivers/gpu/drm/imx/imx-tve.c @@ -218,8 +218,9 @@ static int imx_tve_connector_get_modes(struct drm_connector *connector) return ret; } -static int imx_tve_connector_mode_valid(struct drm_connector *connector, - struct drm_display_mode *mode) +static enum drm_mode_status +imx_tve_connector_mode_valid(struct drm_connector *connector, + struct drm_display_mode *mode) { struct imx_tve *tve = con_to_tve(connector); unsigned long rate; -- GitLab From 9d1b073d018bcb1701a00cc3d4b8c6998cc34a4e Mon Sep 17 00:00:00 2001 From: Candice Li Date: Fri, 22 Jul 2022 20:39:12 +0800 Subject: [PATCH 0845/1927] drm/amdgpu: Enable GFX RAS feature for gfx v11_0_3 v1: Support gfx ras feature enablement for gfx v11_0_3. v2: Update function name and error message. Signed-off-by: Candice Li Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 5eefba2948a55..443ce664b6630 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -4673,6 +4673,26 @@ static int gfx_v11_0_early_init(void *handle) return 0; } +static int gfx_v11_0_ras_late_init(void *handle) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)handle; + struct ras_common_if *gfx_common_if; + int ret; + + gfx_common_if = kzalloc(sizeof(struct ras_common_if), GFP_KERNEL); + if (!gfx_common_if) + return -ENOMEM; + + gfx_common_if->block = AMDGPU_RAS_BLOCK__GFX; + + ret = amdgpu_ras_feature_enable(adev, gfx_common_if, true); + if (ret) + dev_err(adev->dev, "Failed to enable gfx11 ras feature\n"); + + kfree(gfx_common_if); + return ret; +} + static int gfx_v11_0_late_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -4686,6 +4706,12 @@ static int gfx_v11_0_late_init(void *handle) if (r) return r; + if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3)) { + r = gfx_v11_0_ras_late_init(handle); + if (r) + return r; + } + return 0; } -- GitLab From 2103c42198d38d6c02f8b9720c2a19cf3b4adddd Mon Sep 17 00:00:00 2001 From: Gavin Wan Date: Wed, 26 Oct 2022 13:45:25 -0400 Subject: [PATCH 0846/1927] drm/amdgpu: Disable GPU reset on SRIOV before remove pci. The recent change brought a bug on SRIOV envrionment. It caused unloading amdgpu failed on Guest VM. The reason is that the VF FLR was requested while unloading amdgpu driver, but the VF FLR of SRIOV sequence is wrong while removing PCI device. For SRIOV, the guest driver should not trigger the whole XGMI hive to do the reset. Host driver control how the device been reset. Fixes: f5c7e7797060 ("drm/amdgpu: Adjust removal control flow for smu v13_0_2") Acked-by: Alex Deucher Reviewed-by: Shaoyun Liu Signed-off-by: Gavin Wan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 3c9fecdd6b2f3..bf2d50c8c92ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2201,7 +2201,8 @@ amdgpu_pci_remove(struct pci_dev *pdev) pm_runtime_forbid(dev->dev); } - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)) { + if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2) && + !amdgpu_sriov_vf(adev)) { bool need_to_reset_gpu = false; if (adev->gmc.xgmi.num_physical_nodes > 1) { -- GitLab From 47a7470bb2ca2d2ccbd945082e5ebf0279a33fe5 Mon Sep 17 00:00:00 2001 From: Yifan Zha Date: Tue, 25 Oct 2022 16:55:05 +0800 Subject: [PATCH 0847/1927] drm/amdgpu: Skip access GRBM_CNTL under SRIOV on gfx_v11 [Why] GRBM_CNTL is a PF_only register on gfx_v11. RLCG interface will return "out of range" under SRIOV VF. [How] Skip access GRBM_CNTL under gfx_v11 SRIOV VF. Signed-off-by: Yifan Zha Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 443ce664b6630..f68e13b6282ca 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -1626,7 +1626,8 @@ static void gfx_v11_0_constants_init(struct amdgpu_device *adev) u32 tmp; int i; - WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); + if (!amdgpu_sriov_vf(adev)) + WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); gfx_v11_0_setup_rb(adev); gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info); -- GitLab From e1a29b28e70457549149d3b2c82e4333d68ac49d Mon Sep 17 00:00:00 2001 From: Yifan Zha Date: Tue, 25 Oct 2022 14:46:11 +0800 Subject: [PATCH 0848/1927] drm/amdgpu: Skip access SDMA0_F32_CNTL in sdma_v6_0_enable under SRIOV [Why] SDMA0_F32_CNTL is a PF_only regitser which will be blocked by L1. RLCG will not program the register as well. [How] Skip to program SDMA0_F32_CNTL under SRIOV VF. Signed-off-by: Yifan Zha Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index d9ae69be5ea4a..049c26a45d851 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -455,6 +455,9 @@ static void sdma_v6_0_enable(struct amdgpu_device *adev, bool enable) sdma_v6_0_rlc_stop(adev); } + if (amdgpu_sriov_vf(adev)) + return; + for (i = 0; i < adev->sdma.num_instances; i++) { f32_cntl = RREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_F32_CNTL)); f32_cntl = REG_SET_FIELD(f32_cntl, SDMA0_F32_CNTL, HALT, enable ? 0 : 1); -- GitLab From 2c763f37d0a08d39c1a0b1318fa82896c31a6c9d Mon Sep 17 00:00:00 2001 From: Yifan Zha Date: Tue, 25 Oct 2022 15:47:19 +0800 Subject: [PATCH 0849/1927] drm/amdgpu: Skip program gfxhub_v3_0_3 system aperture registers under SRIOV [Why] gfxhub_v3_0_3 system aperture registers are removed from RLCG register access range. [How] Skip access gfxhub_v3_0_3 system aperture registers under SRIOV VF. These registers will be programmed on host side. Signed-off-by: Yifan Zha Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c index 5d3fffd4929ff..716ae6f2aefed 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c @@ -154,6 +154,9 @@ static void gfxhub_v3_0_3_init_system_aperture_regs(struct amdgpu_device *adev) { uint64_t value; + if (amdgpu_sriov_vf(adev)) + return; + /* Disable AGP. */ WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0); WREG32_SOC15(GC, 0, regGCMC_VM_AGP_TOP, 0); -- GitLab From ee22f63b1085fd5cf1649e8915b1e2617cd33ca0 Mon Sep 17 00:00:00 2001 From: Ma Jun Date: Wed, 26 Oct 2022 17:29:18 +0800 Subject: [PATCH 0850/1927] drm/amdkfd: Remove unused variable kfd_topology_device->cache_count is not used by other fucntions, so remove it. Signed-off-by: Ma Jun Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 1 - drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 1 - 2 files changed, 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 339f60a4cdddf..568cbf1d8259c 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -1131,7 +1131,6 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache, if (cache->flags & CRAT_CACHE_FLAGS_SIMD_CACHE) props->cache_type |= HSA_CACHE_TYPE_HSACU; - dev->cache_count++; dev->node_props.caches_count++; list_add_tail(&props->list, &dev->cache_props); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h index 9f6c949186c19..90c069fbd581e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h @@ -128,7 +128,6 @@ struct kfd_topology_device { uint32_t proximity_domain; struct kfd_node_properties node_props; struct list_head mem_props; - uint32_t cache_count; struct list_head cache_props; struct list_head io_link_props; struct list_head p2p_link_props; -- GitLab From 26f2fe8608374e3dda39d01a6acd42d1f67b8ee1 Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Mon, 31 Oct 2022 10:47:36 -0400 Subject: [PATCH 0851/1927] drm/amd/display: drop vblank_lock from struct amdgpu_display_manager As of commit 09a5df6c444c ("drm/amd/display: Fix multi-display support for idle opt workqueue"), vblank_lock is no longer being used. So, don't init it in amdgpu_dm_init() and remove it from struct amdgpu_display_manager. Reviewed-by: Harry Wentland Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 1 - drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h | 7 ------- 2 files changed, 8 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 94620f9b166c0..3130da83e17b9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1399,7 +1399,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) mutex_init(&adev->dm.dc_lock); mutex_init(&adev->dm.audio_lock); - spin_lock_init(&adev->dm.vblank_lock); if(amdgpu_dm_irq_init(adev)) { DRM_ERROR("amdgpu: failed to initialize DM IRQ support.\n"); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index b5ce15c43bcc1..b618b2586e7bb 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -365,13 +365,6 @@ struct amdgpu_display_manager { */ struct mutex audio_lock; - /** - * @vblank_lock: - * - * Guards access to deferred vblank work state. - */ - spinlock_t vblank_lock; - /** * @audio_component: * -- GitLab From 7fbf451e7639f4ce41e2e3e802e5c521c1827b48 Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Mon, 17 Oct 2022 18:51:15 -0400 Subject: [PATCH 0852/1927] drm/amd/display: Reinit DPG when exiting dynamic ODM [WHY] DPG must be returned to initialized state when pipe is disabled. [HOW] Reinit DPG on unused pipes when exiting dynamic ODM. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: Dillon Varone Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/dcn20/dcn20_hwseq.c | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index feff834d5410e..b465a83bde6f0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1079,6 +1079,29 @@ void dcn20_blank_pixel_data( 0); } + if (!blank && dc->debug.enable_single_display_2to1_odm_policy) { + /* when exiting dynamic ODM need to reinit DPG state for unused pipes */ + struct pipe_ctx *old_odm_pipe = dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx].next_odm_pipe; + + odm_pipe = pipe_ctx->next_odm_pipe; + + while (old_odm_pipe) { + if (!odm_pipe || old_odm_pipe->pipe_idx != odm_pipe->pipe_idx) + dc->hwss.set_disp_pattern_generator(dc, + old_odm_pipe, + CONTROLLER_DP_TEST_PATTERN_VIDEOMODE, + CONTROLLER_DP_COLOR_SPACE_UDEFINED, + COLOR_DEPTH_888, + NULL, + 0, + 0, + 0); + old_odm_pipe = old_odm_pipe->next_odm_pipe; + if (odm_pipe) + odm_pipe = odm_pipe->next_odm_pipe; + } + } + if (!blank) if (stream_res->abm) { dc->hwss.set_pipe(pipe_ctx); -- GitLab From 368307cef69ccd9bf5511f25e58e3a103be169fb Mon Sep 17 00:00:00 2001 From: Eric Bernstein Date: Tue, 11 Oct 2022 14:28:47 -0400 Subject: [PATCH 0853/1927] drm/amd/display: Include virtual signal to set k1 and k2 values Reviewed-by: Charlene Liu Acked-by: Alex Hung Signed-off-by: Eric Bernstein Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c index cf5bd9713f54f..aeb4822cebc86 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c @@ -1186,7 +1186,7 @@ unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsign *k2_div = PIXEL_RATE_DIV_BY_2; else *k2_div = PIXEL_RATE_DIV_BY_4; - } else if (dc_is_dp_signal(pipe_ctx->stream->signal)) { + } else if (dc_is_dp_signal(pipe_ctx->stream->signal) || dc_is_virtual_signal(pipe_ctx->stream->signal)) { if (two_pix_per_container) { *k1_div = PIXEL_RATE_DIV_BY_1; *k2_div = PIXEL_RATE_DIV_BY_2; -- GitLab From 667be3ef1bb93a599fd425acb914dfbbf433dda0 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 20 Oct 2022 16:07:07 -0400 Subject: [PATCH 0854/1927] drm/amd/display: Don't enable ODM + MPO [Description] Driver doesn't support ODM + MPO Reviewed-by: Dillon Varone Acked-by: Alex Hung Signed-off-by: Alvin Lee Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c index aeb4822cebc86..5e0018efe055d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c @@ -989,10 +989,6 @@ void dcn32_init_hw(struct dc *dc) dc_dmub_srv_query_caps_cmd(dc->ctx->dmub_srv->dmub); dc->caps.dmub_caps.psr = dc->ctx->dmub_srv->dmub->feature_caps.psr; } - - /* Enable support for ODM and windowed MPO if policy flag is set */ - if (dc->debug.enable_single_display_2to1_odm_policy) - dc->config.enable_windowed_mpo_odm = true; } static int calc_mpc_flow_ctrl_cnt(const struct dc_stream_state *stream, -- GitLab From 6139ab41f26848c92fd20fd678e1b7e5fb17959c Mon Sep 17 00:00:00 2001 From: Charlene Liu Date: Thu, 20 Oct 2022 16:21:13 -0400 Subject: [PATCH 0855/1927] drm/amd/display: Update DML formula [why] This is to update SW DML implementation. Reviewed-by: Dmytro Laktyushkin Reviewed-by: Ariel Bernstein Acked-by: Alex Hung Signed-off-by: Charlene Liu Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c | 3 ++- .../gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c index b612edb144172..45ab0ce508602 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c @@ -1052,7 +1052,8 @@ static bool CalculatePrefetchSchedule( else bytes_pp = myPipe->BytePerPixelY + myPipe->BytePerPixelC; /*rev 99*/ - prefetch_bw_pr = dml_min(1, bytes_pp * myPipe->PixelClock / (double) myPipe->DPPPerPlane); + prefetch_bw_pr = bytes_pp * myPipe->PixelClock / (double) myPipe->DPPPerPlane; + prefetch_bw_pr = dml_min(1, myPipe->VRatio) * prefetch_bw_pr; max_Tsw = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime; prefetch_sw_bytes = PrefetchSourceLinesY * swath_width_luma_ub * myPipe->BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * myPipe->BytePerPixelC; prefetch_bw_oto = dml_max(bytes_pp * myPipe->PixelClock / myPipe->DPPPerPlane, prefetch_sw_bytes / (dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime)); diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c index 0d12fd079cd61..53e3e7364ec6f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c @@ -1074,7 +1074,8 @@ static bool CalculatePrefetchSchedule( else bytes_pp = myPipe->BytePerPixelY + myPipe->BytePerPixelC; /*rev 99*/ - prefetch_bw_pr = dml_min(1, bytes_pp * myPipe->PixelClock / (double) myPipe->DPPPerPlane); + prefetch_bw_pr = bytes_pp * myPipe->PixelClock / (double) myPipe->DPPPerPlane; + prefetch_bw_pr = dml_min(1, myPipe->VRatio) * prefetch_bw_pr; max_Tsw = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime; prefetch_sw_bytes = PrefetchSourceLinesY * swath_width_luma_ub * myPipe->BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * myPipe->BytePerPixelC; prefetch_bw_oto = dml_max(bytes_pp * myPipe->PixelClock / myPipe->DPPPerPlane, prefetch_sw_bytes / (dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime)); -- GitLab From c40f8b132e6014e3034926cc492f113d9d5ca85b Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Wed, 19 Oct 2022 18:42:49 -0400 Subject: [PATCH 0856/1927] drm/amd/display: Check validation passed after applying pipe split changes [WHY?] Validation can fail for configurations that were previously supported, by only changing parameters such as the DET allocations, which is currently unexpected. [HOW?] Add a check that validation passes after applying pipe split related changes. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: Dillon Varone Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index b6cd10f3181c3..9c03f62b9d197 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -1736,10 +1736,12 @@ bool dcn32_internal_validate_bw(struct dc *dc, * ensure all the params are calculated correctly. We do not need to run the * pipe split check again after this call (pipes are already split / merged). * */ - if (!fast_validate) { - context->bw_ctx.dml.soc.allow_for_pstate_or_stutter_in_vblank_final = - dm_prefetch_support_uclk_fclk_and_stutter_if_possible; - vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt); + context->bw_ctx.dml.soc.allow_for_pstate_or_stutter_in_vblank_final = + dm_prefetch_support_uclk_fclk_and_stutter_if_possible; + vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt); + if (vlevel == context->bw_ctx.dml.soc.num_states) { + /* failed after DET size changes */ + goto validate_fail; } } *vlevel_out = vlevel; -- GitLab From 14e1f089d5ee0519af0c646efbc3196d3e1674ba Mon Sep 17 00:00:00 2001 From: Taimur Hassan Date: Fri, 21 Oct 2022 10:54:42 -0400 Subject: [PATCH 0857/1927] drm/amd/display: Remove rate check from pixel rate divider update [Why] This check is not needed, and can cause CRC mismatch. [How] Remove check and early exit from divider update. Reviewed-by: Nicholas Kazlauskas Acked-by: Alex Hung Signed-off-by: Taimur Hassan Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c index 1bd7e0f327d81..367cb6e6d074c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c @@ -97,7 +97,7 @@ static void dccg314_set_pixel_rate_div( enum pixel_rate_div cur_k1 = PIXEL_RATE_DIV_NA, cur_k2 = PIXEL_RATE_DIV_NA; dccg314_get_pixel_rate_div(dccg, otg_inst, &cur_k1, &cur_k2); - if (k1 == PIXEL_RATE_DIV_NA || k2 == PIXEL_RATE_DIV_NA || (k1 == cur_k1 && k2 == cur_k2)) + if (k1 == cur_k1 && k2 == cur_k2) return; switch (otg_inst) { -- GitLab From 361edc61a05cdbc99c6b53264d42af1533e9c20c Mon Sep 17 00:00:00 2001 From: Iswara Nagulendran Date: Fri, 21 Oct 2022 12:27:12 -0400 Subject: [PATCH 0858/1927] drm/amd/display: Modify checks to enable TPS3 pattern when required [HOW&WHY] Checking if both DSC and FEC supported from sink and source before going with TPS3 pattern during link training. Reviewed-by: Anthony Koo Acked-by: Alex Hung Signed-off-by: Iswara Nagulendran Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index bec5e9f787fc0..cda1592c3a5be 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -399,7 +399,11 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, link->psr_settings.force_ffu_mode = 0; copy_settings_data->force_ffu_mode = link->psr_settings.force_ffu_mode; - if (link->fec_state == dc_link_fec_enabled && + if (((link->dpcd_caps.fec_cap.bits.FEC_CAPABLE && + !link->dc->debug.disable_fec) && + (link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT && + !link->panel_config.dsc.disable_dsc_edp && + link->dc->caps.edp_dsc_support)) && link->dpcd_caps.sink_dev_id == DP_DEVICE_ID_38EC11 && (!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_1, sizeof(DP_SINK_DEVICE_STR_ID_1)) || -- GitLab From 935e35d8dbde76a02d00290cf4d9eb420514eaf3 Mon Sep 17 00:00:00 2001 From: Martin Leung Date: Fri, 21 Oct 2022 15:28:52 -0400 Subject: [PATCH 0859/1927] drm/amd/display: Stop waiting for vblank during pipe programming [why and how] This line was originally removed for a compliance issue, but then reverted as it caused a fringe underflow case. However, the addition of this line caused a underflow regression when subVP is on, and it needs to be removed again. We plan to fix subvp underflow and then re-add in this line. After that, we will investigate what to do next for the compliance issue. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: Martin Leung Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index b465a83bde6f0..f3334f513eb4f 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1663,7 +1663,6 @@ static void dcn20_program_pipe( pipe_ctx->pipe_dlg_param.vupdate_width); if (pipe_ctx->stream->mall_stream_config.type != SUBVP_PHANTOM) { - pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); } -- GitLab From da339aa4573de3526890627490ee5f61c1de9c31 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Fri, 21 Oct 2022 18:24:02 -0400 Subject: [PATCH 0860/1927] drm/amd/display: Add a debug option HBR2CP2520 over TPS4 [Why & How] There are cases when we may need to override the hardcoded TPS4 test pattern. Added parameters and config option to be able to allow this. Reviewed-by: Charlene Liu Acked-by: Alex Hung Signed-off-by: Leo Chen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 + drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c | 2 ++ drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c | 2 ++ drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c | 2 ++ drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c | 2 ++ drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c | 2 ++ 6 files changed, 11 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 2a64064400326..6be110b00b242 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -393,6 +393,7 @@ struct dc_config { bool disable_dmcu; bool enable_4to1MPC; bool enable_windowed_mpo_odm; + bool forceHBR2CP2520; // Used for switching between test patterns TPS4 and CP2520 uint32_t allow_edp_hotplug_detection; bool clamp_min_dcfclk; uint64_t vblank_alignment_dto_params; diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c index 8729d6305996a..8f5e89cb9d3ec 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c @@ -1902,6 +1902,8 @@ static bool dcn31_resource_construct( dc->caps.max_slave_rgb_planes = 2; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; + if (dc->config.forceHBR2CP2520) + dc->caps.force_dp_tps4_for_cp2520 = false; dc->caps.dp_hpo = true; dc->caps.dp_hdmi21_pcon_support = true; dc->caps.edp_dsc_support = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c index 85e2d87a8e44b..3b3e093e94477 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c @@ -1770,6 +1770,8 @@ static bool dcn314_resource_construct( dc->caps.max_slave_rgb_planes = 2; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; + if (dc->config.forceHBR2CP2520) + dc->caps.force_dp_tps4_for_cp2520 = false; dc->caps.dp_hpo = true; dc->caps.dp_hdmi21_pcon_support = true; dc->caps.edp_dsc_support = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c index 31cbc5762eab3..96a3d41febffe 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c @@ -1783,6 +1783,8 @@ static bool dcn315_resource_construct( dc->caps.max_slave_rgb_planes = 2; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; + if (dc->config.forceHBR2CP2520) + dc->caps.force_dp_tps4_for_cp2520 = false; dc->caps.dp_hpo = true; dc->caps.dp_hdmi21_pcon_support = true; dc->caps.edp_dsc_support = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c index af3eddc0cf32e..2f643cdaf59fe 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c @@ -1785,6 +1785,8 @@ static bool dcn316_resource_construct( dc->caps.max_slave_rgb_planes = 2; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; + if (dc->config.forceHBR2CP2520) + dc->caps.force_dp_tps4_for_cp2520 = false; dc->caps.dp_hpo = true; dc->caps.dp_hdmi21_pcon_support = true; dc->caps.edp_dsc_support = true; diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index 287b7fa9bf419..4ba9a86621854 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -2126,6 +2126,8 @@ static bool dcn32_resource_construct( dc->caps.max_slave_rgb_planes = 2; dc->caps.post_blend_color_processing = true; dc->caps.force_dp_tps4_for_cp2520 = true; + if (dc->config.forceHBR2CP2520) + dc->caps.force_dp_tps4_for_cp2520 = false; dc->caps.dp_hpo = true; dc->caps.dp_hdmi21_pcon_support = true; dc->caps.edp_dsc_support = true; -- GitLab From cf472dbdd7204e1aaaf587df067e8e986616788b Mon Sep 17 00:00:00 2001 From: Anthony Koo Date: Sat, 22 Oct 2022 22:18:19 -0400 Subject: [PATCH 0861/1927] drm/amd/display: [FW Promotion] Release 0.0.141.0 - Add flag as a status read back that indicates back to back flips detected during psr. Reviewed-by: Aric Cyr Acked-by: Alex Hung Signed-off-by: Anthony Koo Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index bc9344af19957..9df330c86a55b 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -225,6 +225,12 @@ union dmub_psr_debug_flags { * Use TPS3 signal when restore main link. */ uint32_t force_wakeup_by_tps3 : 1; + + /** + * Back to back flip, therefore cannot power down PHY + */ + uint32_t back_to_back_flip : 1; + } bitfields; /** -- GitLab From 86975c34b6e2ce85b5b03f3b0a83333c97890ad7 Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 7 Oct 2022 12:43:26 -0400 Subject: [PATCH 0862/1927] drm/amd/display: Fix DCN32 DSC delay calculation [Why] DCN32 DSC delay calculation had an unintentional integer division, resulting in a mismatch against the DML spreadsheet. [How] Cast numerator to double before performing the division. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index ad66e241f9ae2..4a3e7a5d2758e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -1746,7 +1746,7 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, } DSCDelayRequirement_val = DSCDelayRequirement_val + (HTotal - HActive) * - dml_ceil(DSCDelayRequirement_val / HActive, 1); + dml_ceil((double)DSCDelayRequirement_val / HActive, 1); DSCDelayRequirement_val = DSCDelayRequirement_val * PixelClock / PixelClockBackEnd; -- GitLab From e70a70d0c03f5594cd09a1a2732c7e136f9c8edd Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 14 Oct 2022 17:36:32 -0400 Subject: [PATCH 0863/1927] drm/amd/display: Use forced DSC bpp in DML [Why] DSC config is calculated separately from DML calculations. DML should use these separately calculated DSC params. The issue is that the calculated bpp is not properly propagated into DML. [How] Correctly used forced_bpp value in DML. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 2 +- drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 5b91660a6496b..60351b2891d07 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -1627,7 +1627,7 @@ static void mode_support_configuration(struct vba_vars_st *v, && !mode_lib->vba.MSOOrODMSplitWithNonDPLink && !mode_lib->vba.NotEnoughLanesForMSO && mode_lib->vba.LinkCapacitySupport[i] == true && !mode_lib->vba.P2IWith420 - && !mode_lib->vba.DSCOnlyIfNecessaryWithBPP + //&& !mode_lib->vba.DSCOnlyIfNecessaryWithBPP && !mode_lib->vba.DSC422NativeNotSupported && !mode_lib->vba.MPCCombineMethodIncompatible && mode_lib->vba.ODMCombine2To1SupportCheckOK[i] == true diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index 03924aed8d5c7..8e6585dab20ef 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -625,7 +625,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) mode_lib->vba.skip_dio_check[mode_lib->vba.NumberOfActivePlanes] = dout->is_virtual; - if (!dout->dsc_enable) + if (dout->dsc_enable) mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = dout->output_bpp; else mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = 0.0; -- GitLab From fa7e2b031faf84a607c2b81c15328314b286bc6a Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 14 Oct 2022 17:40:08 -0400 Subject: [PATCH 0864/1927] drm/amd/display: Round up DST_after_scaler to nearest int [Why] The DST_after_scaler value that DML spreadsheet outputs is generally the driver value round up to the nearest int. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c index a1276f6b9581b..395ae8761980f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c @@ -291,8 +291,8 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib, dml_print("DML_DLG: %s: vready_after_vcount0 = %d\n", __func__, dlg_regs->vready_after_vcount0); - dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); - dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); + dst_x_after_scaler = dml_ceil(get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1); + dst_y_after_scaler = dml_ceil(get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1); // do some adjustment on the dst_after scaler to account for odm combine mode dml_print("DML_DLG: %s: input dst_x_after_scaler = %d\n", __func__, dst_x_after_scaler); -- GitLab From f30508e918b4011f44f617f13f8251e72f1a5661 Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 14 Oct 2022 17:46:03 -0400 Subject: [PATCH 0865/1927] drm/amd/display: Add DSC delay factor workaround [Why] Certain 4K high refresh rate modes requiring DSC are exhibiting top of screen underflow corruption. Increasing the DSC delay by a factor of 6 percent stops the underflow for most use cases. [How] Multiply DSC delay requirement in DML by a factor. Add debug option to make this DSC delay factor configurable. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 + drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 4 +++- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 5 +++-- .../drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c | 5 +++-- .../drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h | 3 ++- drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c | 5 ++++- drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h | 3 +++ 7 files changed, 19 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 6be110b00b242..b2c1dcae4ca79 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -863,6 +863,7 @@ struct dc_debug_options { bool enable_double_buffered_dsc_pg_support; bool enable_dp_dig_pixel_rate_div_policy; enum lttpr_mode lttpr_mode_override; + unsigned int dsc_delay_factor_wa_x1000; }; struct gpu_info_soc_bounding_box_v1_0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index 9c03f62b9d197..0d704e302d035 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -2346,9 +2346,11 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) dcn3_2_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes; - } + /* DML DSC delay factor workaround */ + dcn3_2_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_2_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 60351b2891d07..47ff0a8bd1052 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -367,7 +367,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman mode_lib->vba.OutputBpp[k], mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], mode_lib->vba.NumberOfDSCSlices[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], mode_lib->vba.PixelClock[k], - mode_lib->vba.PixelClockBackEnd[k]); + mode_lib->vba.PixelClockBackEnd[k], mode_lib->vba.ip.dsc_delay_factor_wa); } for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) @@ -2475,7 +2475,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.OutputBppPerState[i][k], mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], mode_lib->vba.NumberOfDSCSlices[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], - mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k]); + mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k], + mode_lib->vba.ip.dsc_delay_factor_wa); } for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index 4a3e7a5d2758e..968924c491c18 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -1726,7 +1726,8 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, enum output_format_class OutputFormat, enum output_encoder_class Output, double PixelClock, - double PixelClockBackEnd) + double PixelClockBackEnd, + double dsc_delay_factor_wa) { unsigned int DSCDelayRequirement_val; @@ -1764,7 +1765,7 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, dml_print("DML::%s: DSCDelayRequirement_val = %d\n", __func__, DSCDelayRequirement_val); #endif - return DSCDelayRequirement_val; + return dml_ceil(DSCDelayRequirement_val * dsc_delay_factor_wa, 1); } void dml32_CalculateSurfaceSizeInMall( diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index 55cead0d42374..2c3827546ac77 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -327,7 +327,8 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, enum output_format_class OutputFormat, enum output_encoder_class Output, double PixelClock, - double PixelClockBackEnd); + double PixelClockBackEnd, + double dsc_delay_factor_wa); void dml32_CalculateSurfaceSizeInMall( unsigned int NumberOfActiveSurfaces, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index 7352f75144101..ec0486efab147 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -29,6 +29,7 @@ #include "dcn321_fpu.h" #include "dcn32/dcn32_resource.h" #include "dcn321/dcn321_resource.h" +#include "dml/dcn32/display_mode_vba_util_32.h" #define DCN3_2_DEFAULT_DET_SIZE 256 @@ -538,9 +539,11 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) dcn3_21_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes; - } + /* DML DSC delay factor workaround */ + dcn3_21_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_21_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index f33a8879b05ad..d7be01ac07514 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -364,6 +364,9 @@ struct _vcs_dpi_ip_params_st { unsigned int max_num_dp2p0_outputs; unsigned int max_num_dp2p0_streams; unsigned int VBlankNomDefaultUS; + + /* DM workarounds */ + double dsc_delay_factor_wa; // TODO: Remove after implementing root cause fix }; struct _vcs_dpi_display_xfc_params_st { -- GitLab From 6818f755f737758211d196cf7015628a88622d85 Mon Sep 17 00:00:00 2001 From: Charlene Liu Date: Fri, 21 Oct 2022 15:39:09 -0400 Subject: [PATCH 0866/1927] drm/amd/display: Fix null pointer issues found in emulation [why] fix null point issues found in emulation Reviewed-by: Chris Park Acked-by: Alex Hung Signed-off-by: Charlene Liu Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c index bdf101547484a..165c920ca7766 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c @@ -89,7 +89,8 @@ static void enable_memory_low_power(struct dc *dc) REG_UPDATE(MMHUBBUB_MEM_PWR_CNTL, VGA_MEM_PWR_FORCE, 1); } - if (dc->debug.enable_mem_low_power.bits.mpc) + if (dc->debug.enable_mem_low_power.bits.mpc && + dc->res_pool->mpc->funcs->set_mpc_mem_lp_mode) dc->res_pool->mpc->funcs->set_mpc_mem_lp_mode(dc->res_pool->mpc); @@ -141,7 +142,8 @@ void dcn31_init_hw(struct dc *dc) if (!dcb->funcs->is_accelerated_mode(dcb)) { hws->funcs.bios_golden_init(dc); - hws->funcs.disable_vga(dc->hwseq); + if (hws->funcs.disable_vga) + hws->funcs.disable_vga(dc->hwseq); } // Initialize the dccg if (res_pool->dccg->funcs->dccg_init) -- GitLab From 9ee8b902b891723237e3aace0c2ec640327f31d2 Mon Sep 17 00:00:00 2001 From: Nevenko Stupar Date: Fri, 6 May 2022 16:32:38 -0400 Subject: [PATCH 0867/1927] drm/amd/display: Investigate tool reported FCLK P-state deviations [Why] Fix for some of the tool reported modes for FCLK P-state deviations and UCLK P-state deviations that are coming from DSC terms and/or Scaling terms causing MinActiveFCLKChangeLatencySupported and MaxActiveDRAMClockChangeLatencySupported incorrectly calculated in DML for these configurations. Reviewed-by: Chaitanya Dhere Acked-by: Jasdeep Dhillon Acked-by: Alex Hung Signed-off-by: Nevenko Stupar Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 47ff0a8bd1052..3d184679f129e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -364,7 +364,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) { v->DSCDelay[k] = dml32_DSCDelayRequirement(mode_lib->vba.DSCEnabled[k], mode_lib->vba.ODMCombineEnabled[k], mode_lib->vba.DSCInputBitPerComponent[k], - mode_lib->vba.OutputBpp[k], mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], + mode_lib->vba.OutputBppPerState[mode_lib->vba.VoltageLevel][k], + mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], mode_lib->vba.NumberOfDSCSlices[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k], mode_lib->vba.ip.dsc_delay_factor_wa); -- GitLab From ba6029726854e43e77fec05af5299a671e8483a0 Mon Sep 17 00:00:00 2001 From: Aric Cyr Date: Sun, 23 Oct 2022 13:41:03 -0400 Subject: [PATCH 0868/1927] drm/amd/display: 3.2.210 DC version 3.2.210 brings along the following: - Investigate tool reported FCLK P-state deviations - Fix null pointer issues found in emulation - Add DSC delay factor workaround - Round up DST_after_scaler to nearest int - Use forced DSC bpp in DML - Fix DCN32 DSC delay calculation - Add a debug option HBR2CP2520 over TPS4 - Stop waiting for vblank during pipe programming - Modify checks to enable TPS3 pattern when required - Remove rate check from pixel rate divider update - Check validation passed after applying pipe split changes - Update DML formula - Don't enable ODM + MPO - Include virtual signal to set k1 and k2 values - Reinit DPG when exiting dynamic ODM Acked-by: Alex Hung Signed-off-by: Aric Cyr Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index b2c1dcae4ca79..84c82d3a6761d 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -47,7 +47,7 @@ struct aux_payload; struct set_config_cmd_payload; struct dmub_notification; -#define DC_VER "3.2.209" +#define DC_VER "3.2.210" #define MAX_SURFACES 3 #define MAX_PLANES 6 -- GitLab From 545094d993f4639482018becda5f2a47d126f0ab Mon Sep 17 00:00:00 2001 From: "Jiri Slaby (SUSE)" Date: Mon, 31 Oct 2022 12:42:47 +0100 Subject: [PATCH 0869/1927] drm/amd/display (gcc13): fix enum mismatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rn_vbios_smu_set_dcn_low_power_state() produces a valid warning with gcc-13: drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.c:237:6: error: conflicting types for 'rn_vbios_smu_set_dcn_low_power_state' due to enum/integer mismatch; have 'void(struct clk_mgr_internal *, enum dcn_pwr_state)' drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.h:36:6: note: previous declaration of 'rn_vbios_smu_set_dcn_low_power_state' with type 'void(struct clk_mgr_internal *, int)' I.e. the type of the 2nd parameter of rn_vbios_smu_set_dcn_low_power_state() in the declaration is int, while the definition spells enum dcn_pwr_state. Synchronize them to the latter (and add a forward enum declaration). Cc: Martin Liska Cc: Harry Wentland Cc: Leo Li Cc: Rodrigo Siqueira Cc: Alex Deucher Cc: "Christian König" Cc: "Pan, Xinhui" Cc: David Airlie Cc: Daniel Vetter Cc: amd-gfx@lists.freedesktop.org Cc: dri-devel@lists.freedesktop.org Reviewed-by: Harry Wentland Signed-off-by: Jiri Slaby (SUSE) Signed-off-by: Alex Deucher --- .../drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.h index 3e5df27aa96fc..1ce19d875358f 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.h +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr_vbios_smu.h @@ -26,6 +26,8 @@ #ifndef DAL_DC_RN_CLK_MGR_VBIOS_SMU_H_ #define DAL_DC_RN_CLK_MGR_VBIOS_SMU_H_ +enum dcn_pwr_state; + int rn_vbios_smu_get_smu_version(struct clk_mgr_internal *clk_mgr); int rn_vbios_smu_set_dispclk(struct clk_mgr_internal *clk_mgr, int requested_dispclk_khz); int rn_vbios_smu_set_dprefclk(struct clk_mgr_internal *clk_mgr); @@ -33,7 +35,7 @@ int rn_vbios_smu_set_hard_min_dcfclk(struct clk_mgr_internal *clk_mgr, int reque int rn_vbios_smu_set_min_deep_sleep_dcfclk(struct clk_mgr_internal *clk_mgr, int requested_min_ds_dcfclk_khz); void rn_vbios_smu_set_phyclk(struct clk_mgr_internal *clk_mgr, int requested_phyclk_khz); int rn_vbios_smu_set_dppclk(struct clk_mgr_internal *clk_mgr, int requested_dpp_khz); -void rn_vbios_smu_set_dcn_low_power_state(struct clk_mgr_internal *clk_mgr, int display_count); +void rn_vbios_smu_set_dcn_low_power_state(struct clk_mgr_internal *clk_mgr, enum dcn_pwr_state); void rn_vbios_smu_enable_48mhz_tmdp_refclk_pwrdwn(struct clk_mgr_internal *clk_mgr, bool enable); void rn_vbios_smu_enable_pme_wa(struct clk_mgr_internal *clk_mgr); int rn_vbios_smu_is_periodic_retraining_disabled(struct clk_mgr_internal *clk_mgr); -- GitLab From 4151bb636acf32bb2e6126cec8216b023117c0e9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 1 Nov 2022 12:19:51 +0000 Subject: [PATCH 0870/1927] KVM: arm64: Fix SMPRI_EL1/TPIDR2_EL0 trapping on VHE The trapping of SMPRI_EL1 and TPIDR2_EL0 currently only really work on nVHE, as only this mode uses the fine-grained trapping that controls these two registers. Move the trapping enable/disable code into __{de,}activate_traps_common(), allowing it to be called when it actually matters on VHE, and remove the flipping of EL2 control for TPIDR2_EL0, which only affects the host access of this register. Fixes: 861262ab8627 ("KVM: arm64: Handle SME host state when running guests") Reported-by: Mark Brown Reviewed-by: Mark Brown Signed-off-by: Marc Zyngier Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/86bkpqer4z.wl-maz@kernel.org --- arch/arm64/kvm/hyp/include/hyp/switch.h | 20 +++++++++++++++++++ arch/arm64/kvm/hyp/nvhe/switch.c | 26 ------------------------- arch/arm64/kvm/hyp/vhe/switch.c | 8 -------- 3 files changed, 20 insertions(+), 34 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 6cbbb6c02f663..3330d1b76bdd2 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -87,6 +87,17 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu) vcpu->arch.mdcr_el2_host = read_sysreg(mdcr_el2); write_sysreg(vcpu->arch.mdcr_el2, mdcr_el2); + + if (cpus_have_final_cap(ARM64_SME)) { + sysreg_clear_set_s(SYS_HFGRTR_EL2, + HFGxTR_EL2_nSMPRI_EL1_MASK | + HFGxTR_EL2_nTPIDR2_EL0_MASK, + 0); + sysreg_clear_set_s(SYS_HFGWTR_EL2, + HFGxTR_EL2_nSMPRI_EL1_MASK | + HFGxTR_EL2_nTPIDR2_EL0_MASK, + 0); + } } static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) @@ -96,6 +107,15 @@ static inline void __deactivate_traps_common(struct kvm_vcpu *vcpu) write_sysreg(0, hstr_el2); if (kvm_arm_support_pmu_v3()) write_sysreg(0, pmuserenr_el0); + + if (cpus_have_final_cap(ARM64_SME)) { + sysreg_clear_set_s(SYS_HFGRTR_EL2, 0, + HFGxTR_EL2_nSMPRI_EL1_MASK | + HFGxTR_EL2_nTPIDR2_EL0_MASK); + sysreg_clear_set_s(SYS_HFGWTR_EL2, 0, + HFGxTR_EL2_nSMPRI_EL1_MASK | + HFGxTR_EL2_nTPIDR2_EL0_MASK); + } } static inline void ___activate_traps(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 8e9d49a964be6..c2cb46ca4fb66 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -55,18 +55,6 @@ static void __activate_traps(struct kvm_vcpu *vcpu) write_sysreg(val, cptr_el2); write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el2); - if (cpus_have_final_cap(ARM64_SME)) { - val = read_sysreg_s(SYS_HFGRTR_EL2); - val &= ~(HFGxTR_EL2_nTPIDR2_EL0_MASK | - HFGxTR_EL2_nSMPRI_EL1_MASK); - write_sysreg_s(val, SYS_HFGRTR_EL2); - - val = read_sysreg_s(SYS_HFGWTR_EL2); - val &= ~(HFGxTR_EL2_nTPIDR2_EL0_MASK | - HFGxTR_EL2_nSMPRI_EL1_MASK); - write_sysreg_s(val, SYS_HFGWTR_EL2); - } - if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) { struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt; @@ -110,20 +98,6 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2); - if (cpus_have_final_cap(ARM64_SME)) { - u64 val; - - val = read_sysreg_s(SYS_HFGRTR_EL2); - val |= HFGxTR_EL2_nTPIDR2_EL0_MASK | - HFGxTR_EL2_nSMPRI_EL1_MASK; - write_sysreg_s(val, SYS_HFGRTR_EL2); - - val = read_sysreg_s(SYS_HFGWTR_EL2); - val |= HFGxTR_EL2_nTPIDR2_EL0_MASK | - HFGxTR_EL2_nSMPRI_EL1_MASK; - write_sysreg_s(val, SYS_HFGWTR_EL2); - } - cptr = CPTR_EL2_DEFAULT; if (vcpu_has_sve(vcpu) && (vcpu->arch.fp_state == FP_STATE_GUEST_OWNED)) cptr |= CPTR_EL2_TZ; diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 7acb87eaa0925..1a97391fedd29 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -63,10 +63,6 @@ static void __activate_traps(struct kvm_vcpu *vcpu) __activate_traps_fpsimd32(vcpu); } - if (cpus_have_final_cap(ARM64_SME)) - write_sysreg(read_sysreg(sctlr_el2) & ~SCTLR_ELx_ENTP2, - sctlr_el2); - write_sysreg(val, cpacr_el1); write_sysreg(__this_cpu_read(kvm_hyp_vector), vbar_el1); @@ -88,10 +84,6 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) */ asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT)); - if (cpus_have_final_cap(ARM64_SME)) - write_sysreg(read_sysreg(sctlr_el2) | SCTLR_ELx_ENTP2, - sctlr_el2); - write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1); if (!arm64_kernel_unmapped_at_el0()) -- GitLab From 6c412da54c80a54b1a8b7f89677f6e82f0fabec4 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Sat, 29 Oct 2022 22:57:11 +0200 Subject: [PATCH 0871/1927] sfc: Fix an error handling path in efx_pci_probe() If an error occurs after the first kzalloc() the corresponding memory allocation is never freed. Add the missing kfree() in the error handling path, as already done in the remove() function. Fixes: 7e773594dada ("sfc: Separate efx_nic memory from net_device memory") Signed-off-by: Christophe JAILLET Acked-by: Martin Habets Link: https://lore.kernel.org/r/dc114193121c52c8fa3779e49bdd99d4b41344a9.1667077009.git.christophe.jaillet@wanadoo.fr Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/sfc/efx.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 054d5ce6029e4..0556542d7a6b6 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -1059,8 +1059,10 @@ static int efx_pci_probe(struct pci_dev *pci_dev, /* Allocate and initialise a struct net_device */ net_dev = alloc_etherdev_mq(sizeof(probe_data), EFX_MAX_CORE_TX_QUEUES); - if (!net_dev) - return -ENOMEM; + if (!net_dev) { + rc = -ENOMEM; + goto fail0; + } probe_ptr = netdev_priv(net_dev); *probe_ptr = probe_data; efx->net_dev = net_dev; @@ -1132,6 +1134,8 @@ static int efx_pci_probe(struct pci_dev *pci_dev, WARN_ON(rc > 0); netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc); free_netdev(net_dev); + fail0: + kfree(probe_data); return rc; } -- GitLab From a6dd6f39008bb3ef7c73ef0a2acc2a4209555bd8 Mon Sep 17 00:00:00 2001 From: Dave Hansen Date: Fri, 28 Oct 2022 17:12:19 +0300 Subject: [PATCH 0872/1927] x86/tdx: Prepare for using "INFO" call for a second purpose The TDG.VP.INFO TDCALL provides the guest with various details about the TDX system that the guest needs to run. Only one field is currently used: 'gpa_width' which tells the guest which PTE bits mark pages shared or private. A second field is now needed: the guest "TD attributes" to tell if virtualization exceptions are configured in a way that can harm the guest. Make the naming and calling convention more generic and discrete from the mask-centric one. Thanks to Sathya for the inspiration here, but there's no code, comments or changelogs left from where he started. Signed-off-by: Dave Hansen Acked-by: Kirill A. Shutemov Tested-by: Kirill A. Shutemov Cc: stable@vger.kernel.org --- arch/x86/coco/tdx/tdx.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 928dcf7a20d98..3fee96931ff56 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -98,7 +98,7 @@ static inline void tdx_module_call(u64 fn, u64 rcx, u64 rdx, u64 r8, u64 r9, panic("TDCALL %lld failed (Buggy TDX module!)\n", fn); } -static u64 get_cc_mask(void) +static void tdx_parse_tdinfo(u64 *cc_mask) { struct tdx_module_output out; unsigned int gpa_width; @@ -121,7 +121,7 @@ static u64 get_cc_mask(void) * The highest bit of a guest physical address is the "sharing" bit. * Set it for shared pages and clear it for private pages. */ - return BIT_ULL(gpa_width - 1); + *cc_mask = BIT_ULL(gpa_width - 1); } /* @@ -758,7 +758,7 @@ void __init tdx_early_init(void) setup_force_cpu_cap(X86_FEATURE_TDX_GUEST); cc_set_vendor(CC_VENDOR_INTEL); - cc_mask = get_cc_mask(); + tdx_parse_tdinfo(&cc_mask); cc_set_mask(cc_mask); /* -- GitLab From 024f4b2e1f874934943eb2d3d288ebc52c79f55c Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 17 Oct 2022 10:01:57 +0100 Subject: [PATCH 0873/1927] arm64: entry: avoid kprobe recursion The cortex_a76_erratum_1463225_debug_handler() function is called when handling debug exceptions (and synchronous exceptions from BRK instructions), and so is called when a probed function executes. If the compiler does not inline cortex_a76_erratum_1463225_debug_handler(), it can be probed. If cortex_a76_erratum_1463225_debug_handler() is probed, any debug exception or software breakpoint exception will result in recursive exceptions leading to a stack overflow. This can be triggered with the ftrace multiple_probes selftest, and as per the example splat below. This is a regression caused by commit: 6459b8469753e9fe ("arm64: entry: consolidate Cortex-A76 erratum 1463225 workaround") ... which removed the NOKPROBE_SYMBOL() annotation associated with the function. My intent was that cortex_a76_erratum_1463225_debug_handler() would be inlined into its caller, el1_dbg(), which is marked noinstr and cannot be probed. Mark cortex_a76_erratum_1463225_debug_handler() as __always_inline to ensure this. Example splat prior to this patch (with recursive entries elided): | # echo p cortex_a76_erratum_1463225_debug_handler > /sys/kernel/debug/tracing/kprobe_events | # echo p do_el0_svc >> /sys/kernel/debug/tracing/kprobe_events | # echo 1 > /sys/kernel/debug/tracing/events/kprobes/enable | Insufficient stack space to handle exception! | ESR: 0x0000000096000047 -- DABT (current EL) | FAR: 0xffff800009cefff0 | Task stack: [0xffff800009cf0000..0xffff800009cf4000] | IRQ stack: [0xffff800008000000..0xffff800008004000] | Overflow stack: [0xffff00007fbc00f0..0xffff00007fbc10f0] | CPU: 0 PID: 145 Comm: sh Not tainted 6.0.0 #2 | Hardware name: linux,dummy-virt (DT) | pstate: 604003c5 (nZCv DAIF +PAN -UAO -TCO -DIT -SSBS BTYPE=--) | pc : arm64_enter_el1_dbg+0x4/0x20 | lr : el1_dbg+0x24/0x5c | sp : ffff800009cf0000 | x29: ffff800009cf0000 x28: ffff000002c74740 x27: 0000000000000000 | x26: 0000000000000000 x25: 0000000000000000 x24: 0000000000000000 | x23: 00000000604003c5 x22: ffff80000801745c x21: 0000aaaac95ac068 | x20: 00000000f2000004 x19: ffff800009cf0040 x18: 0000000000000000 | x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000 | x14: 0000000000000000 x13: 0000000000000000 x12: 0000000000000000 | x11: 0000000000000010 x10: ffff800008c87190 x9 : ffff800008ca00d0 | x8 : 000000000000003c x7 : 0000000000000000 x6 : 0000000000000000 | x5 : 0000000000000000 x4 : 0000000000000000 x3 : 00000000000043a4 | x2 : 00000000f2000004 x1 : 00000000f2000004 x0 : ffff800009cf0040 | Kernel panic - not syncing: kernel stack overflow | CPU: 0 PID: 145 Comm: sh Not tainted 6.0.0 #2 | Hardware name: linux,dummy-virt (DT) | Call trace: | dump_backtrace+0xe4/0x104 | show_stack+0x18/0x4c | dump_stack_lvl+0x64/0x7c | dump_stack+0x18/0x38 | panic+0x14c/0x338 | test_taint+0x0/0x2c | panic_bad_stack+0x104/0x118 | handle_bad_stack+0x34/0x48 | __bad_stack+0x78/0x7c | arm64_enter_el1_dbg+0x4/0x20 | el1h_64_sync_handler+0x40/0x98 | el1h_64_sync+0x64/0x68 | cortex_a76_erratum_1463225_debug_handler+0x0/0x34 ... | el1h_64_sync_handler+0x40/0x98 | el1h_64_sync+0x64/0x68 | cortex_a76_erratum_1463225_debug_handler+0x0/0x34 ... | el1h_64_sync_handler+0x40/0x98 | el1h_64_sync+0x64/0x68 | cortex_a76_erratum_1463225_debug_handler+0x0/0x34 | el1h_64_sync_handler+0x40/0x98 | el1h_64_sync+0x64/0x68 | do_el0_svc+0x0/0x28 | el0t_64_sync_handler+0x84/0xf0 | el0t_64_sync+0x18c/0x190 | Kernel Offset: disabled | CPU features: 0x0080,00005021,19001080 | Memory Limit: none | ---[ end Kernel panic - not syncing: kernel stack overflow ]--- With this patch, cortex_a76_erratum_1463225_debug_handler() is inlined into el1_dbg(), and el1_dbg() cannot be probed: | # echo p cortex_a76_erratum_1463225_debug_handler > /sys/kernel/debug/tracing/kprobe_events | sh: write error: No such file or directory | # grep -w cortex_a76_erratum_1463225_debug_handler /proc/kallsyms | wc -l | 0 | # echo p el1_dbg > /sys/kernel/debug/tracing/kprobe_events | sh: write error: Invalid argument | # grep -w el1_dbg /proc/kallsyms | wc -l | 1 Fixes: 6459b8469753 ("arm64: entry: consolidate Cortex-A76 erratum 1463225 workaround") Cc: # 5.12.x Signed-off-by: Mark Rutland Cc: Will Deacon Link: https://lore.kernel.org/r/20221017090157.2881408-1-mark.rutland@arm.com Signed-off-by: Catalin Marinas --- arch/arm64/kernel/entry-common.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 9173fad279af9..27369fa1c032c 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -329,7 +329,8 @@ static void cortex_a76_erratum_1463225_svc_handler(void) __this_cpu_write(__in_cortex_a76_erratum_1463225_wa, 0); } -static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) +static __always_inline bool +cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) { if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa)) return false; -- GitLab From 003b786b678919e072c2b12ffa73901ef840963e Mon Sep 17 00:00:00 2001 From: Kai Vehmanen Date: Tue, 1 Nov 2022 13:49:13 +0200 Subject: [PATCH 0874/1927] ASoC: SOF: ipc3-topology: use old pipeline teardown flow with SOF2.1 and older MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Originally in commit b2ebcf42a48f ("ASoC: SOF: free widgets in sof_tear_down_pipelines() for static pipelines"), freeing of pipeline components at suspend was only done with recent FW as there were known limitations in older firmware versions. Tests show that if static pipelines are used, i.e. all pipelines are setup whenever firmware is powered up, the reverse action of freeing all components at power down, leads to firmware failures with also SOF2.0 and SOF2.1 based firmware. The problems can be specific to certain topologies with e.g. components not prepared to be freed at suspend (as this did not happen with older SOF kernels). To avoid hitting these problems when kernel is upgraded and used with an older firmware, bump the firmware requirement to SOF2.2 or newer. If an older firmware is used, and pipeline is a static one, do not free the components at suspend. This ensures the suspend flow remains backwards compatible with older firmware versions. This limitation does not apply if the product configuration is updated to dynamic pipelines. The limitation is not linked to firmware ABI, as the interface to free pipeline components has been available already before ABI3.19. The problem is in the implementation, so firmware version should be used to decide whether it is safe to use the newer flow or not. This patch adds a new SOF_FW_VER() macro to compare SOF firmware release versions. Link: https://github.com/thesofproject/sof/issues/6475 Signed-off-by: Kai Vehmanen Reviewed-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Reviewed-by: Péter Ujfalusi Link: https://lore.kernel.org/r/20221101114913.1292671-1-kai.vehmanen@linux.intel.com Signed-off-by: Mark Brown --- include/sound/sof/info.h | 4 ++++ sound/soc/sof/ipc3-topology.c | 15 ++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/sound/sof/info.h b/include/sound/sof/info.h index 65e86e4e9fd8e..75193850ead0c 100644 --- a/include/sound/sof/info.h +++ b/include/sound/sof/info.h @@ -36,6 +36,10 @@ enum sof_ipc_ext_data { SOF_IPC_EXT_USER_ABI_INFO = 4, }; +/* Build u32 number in format MMmmmppp */ +#define SOF_FW_VER(MAJOR, MINOR, PATCH) ((uint32_t)( \ + ((MAJOR) << 24) | ((MINOR) << 12) | (PATCH))) + /* FW version - SOF_IPC_GLB_VERSION */ struct sof_ipc_fw_version { struct sof_ipc_hdr hdr; diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c index c148715aa0f91..0720e1eae0843 100644 --- a/sound/soc/sof/ipc3-topology.c +++ b/sound/soc/sof/ipc3-topology.c @@ -2275,6 +2275,7 @@ static int sof_ipc3_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verif struct sof_ipc_fw_version *v = &sdev->fw_ready.version; struct snd_sof_widget *swidget; struct snd_sof_route *sroute; + bool dyn_widgets = false; int ret; /* @@ -2284,12 +2285,14 @@ static int sof_ipc3_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verif * topology loading the sound card unavailable to open PCMs. */ list_for_each_entry(swidget, &sdev->widget_list, list) { - if (swidget->dynamic_pipeline_widget) + if (swidget->dynamic_pipeline_widget) { + dyn_widgets = true; continue; + } - /* Do not free widgets for static pipelines with FW ABI older than 3.19 */ + /* Do not free widgets for static pipelines with FW older than SOF2.2 */ if (!verify && !swidget->dynamic_pipeline_widget && - v->abi_version < SOF_ABI_VER(3, 19, 0)) { + SOF_FW_VER(v->major, v->minor, v->micro) < SOF_FW_VER(2, 2, 0)) { swidget->use_count = 0; swidget->complete = 0; continue; @@ -2303,9 +2306,11 @@ static int sof_ipc3_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verif /* * Tear down all pipelines associated with PCMs that did not get suspended * and unset the prepare flag so that they can be set up again during resume. - * Skip this step for older firmware. + * Skip this step for older firmware unless topology has any + * dynamic pipeline (in which case the step is mandatory). */ - if (!verify && v->abi_version >= SOF_ABI_VER(3, 19, 0)) { + if (!verify && (dyn_widgets || SOF_FW_VER(v->major, v->minor, v->micro) >= + SOF_FW_VER(2, 2, 0))) { ret = sof_tear_down_left_over_pipelines(sdev); if (ret < 0) { dev_err(sdev->dev, "failed to tear down paused pipelines\n"); -- GitLab From 92a66cbf6b30eda5719fbdfb24cd15fb341bba32 Mon Sep 17 00:00:00 2001 From: Mika Westerberg Date: Tue, 25 Oct 2022 09:28:00 +0300 Subject: [PATCH 0875/1927] spi: intel: Use correct mask for flash and protected regions The flash and protected region mask is actually 0x7fff (30:16 and 14:0) and not 0x3fff so fix this accordingly. While there use GENMASK() instead. Cc: stable@vger.kernel.org Signed-off-by: Mika Westerberg Link: https://lore.kernel.org/r/20221025062800.22357-1-mika.westerberg@linux.intel.com Signed-off-by: Mark Brown --- drivers/spi/spi-intel.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c index 605acb1bf4b08..3ac73691fbb54 100644 --- a/drivers/spi/spi-intel.c +++ b/drivers/spi/spi-intel.c @@ -52,17 +52,17 @@ #define FRACC 0x50 #define FREG(n) (0x54 + ((n) * 4)) -#define FREG_BASE_MASK 0x3fff +#define FREG_BASE_MASK GENMASK(14, 0) #define FREG_LIMIT_SHIFT 16 -#define FREG_LIMIT_MASK (0x03fff << FREG_LIMIT_SHIFT) +#define FREG_LIMIT_MASK GENMASK(30, 16) /* Offset is from @ispi->pregs */ #define PR(n) ((n) * 4) #define PR_WPE BIT(31) #define PR_LIMIT_SHIFT 16 -#define PR_LIMIT_MASK (0x3fff << PR_LIMIT_SHIFT) +#define PR_LIMIT_MASK GENMASK(30, 16) #define PR_RPE BIT(15) -#define PR_BASE_MASK 0x3fff +#define PR_BASE_MASK GENMASK(14, 0) /* Offsets are from @ispi->sregs */ #define SSFSTS_CTL 0x00 -- GitLab From be0ddf5293a7895a8c9096e1a8560930c6a0ab3f Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Tue, 1 Nov 2022 11:27:14 +0000 Subject: [PATCH 0876/1927] arm64: booting: Document our requirements for fine grained traps with SME With SME we require that fine grained traps on access to TPIDR2_EL0 and SMPRI_EL1 are disabled but did not document that fact. Add the relevant register bits. Signed-off-by: Mark Brown Reviewed-by: Oliver Upton Acked-by: Catalin Marinas Signed-off-by: Marc Zyngier Link: https://lore.kernel.org/r/20221101112716.52035-2-broonie@kernel.org --- Documentation/arm64/booting.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Documentation/arm64/booting.rst b/Documentation/arm64/booting.rst index 8aefa1001ae52..8c324ad638de2 100644 --- a/Documentation/arm64/booting.rst +++ b/Documentation/arm64/booting.rst @@ -340,6 +340,14 @@ Before jumping into the kernel, the following conditions must be met: - SMCR_EL2.LEN must be initialised to the same value for all CPUs the kernel will execute on. + - HWFGRTR_EL2.nTPIDR2_EL0 (bit 55) must be initialised to 0b01. + + - HWFGWTR_EL2.nTPIDR2_EL0 (bit 55) must be initialised to 0b01. + + - HWFGRTR_EL2.nSMPRI_EL1 (bit 54) must be initialised to 0b01. + + - HWFGWTR_EL2.nSMPRI_EL1 (bit 54) must be initialised to 0b01. + For CPUs with the Scalable Matrix Extension FA64 feature (FEAT_SME_FA64) - If EL3 is present: -- GitLab From 57cadf5b77b5ff21ffba4b82bc1f24be62d361ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jouni=20H=C3=B6gander?= Date: Tue, 1 Nov 2022 13:53:42 +0200 Subject: [PATCH 0877/1927] drm/i915/mtl: Fix PSR2_MAN_TRK_CTL bit getter functions for MTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MTL shares PSR2_MAN_TRK_CTL bits with ADL. Currently some bit getter functions are incorrect for MTL. This patch fixes those. Bspec: 49274 Cc: José Roberto de Souza Cc: Mika Kahola Cc: Radhakrishna Sripada Fixes: 47d4ae2192cb ("drm/i915/mtl: Extend PSR support") Signed-off-by: Jouni Högander Reviewed-by: José Roberto de Souza Signed-off-by: José Roberto de Souza Link: https://patchwork.freedesktop.org/patch/msgid/20221101115342.1136720-1-jouni.hogander@intel.com --- drivers/gpu/drm/i915/display/intel_psr.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 564d4fd94048c..e11b0592055fb 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -1470,7 +1470,8 @@ void intel_psr_resume(struct intel_dp *intel_dp) static u32 man_trk_ctl_enable_bit_get(struct drm_i915_private *dev_priv) { - return IS_ALDERLAKE_P(dev_priv) ? 0 : PSR2_MAN_TRK_CTL_ENABLE; + return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ? 0 : + PSR2_MAN_TRK_CTL_ENABLE; } static u32 man_trk_ctl_single_full_frame_bit_get(struct drm_i915_private *dev_priv) @@ -1482,14 +1483,14 @@ static u32 man_trk_ctl_single_full_frame_bit_get(struct drm_i915_private *dev_pr static u32 man_trk_ctl_partial_frame_bit_get(struct drm_i915_private *dev_priv) { - return IS_ALDERLAKE_P(dev_priv) ? + return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ? ADLP_PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE : PSR2_MAN_TRK_CTL_SF_PARTIAL_FRAME_UPDATE; } static u32 man_trk_ctl_continuos_full_frame(struct drm_i915_private *dev_priv) { - return IS_ALDERLAKE_P(dev_priv) ? + return IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14 ? ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME : PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME; } -- GitLab From 8ec95b94716a1e4d126edc3fb2bc426a717e2dba Mon Sep 17 00:00:00 2001 From: Wang Yufen Date: Tue, 1 Nov 2022 09:31:36 +0800 Subject: [PATCH 0878/1927] bpf, sockmap: Fix the sk->sk_forward_alloc warning of sk_stream_kill_queues When running `test_sockmap` selftests, the following warning appears: WARNING: CPU: 2 PID: 197 at net/core/stream.c:205 sk_stream_kill_queues+0xd3/0xf0 Call Trace: inet_csk_destroy_sock+0x55/0x110 tcp_rcv_state_process+0xd28/0x1380 ? tcp_v4_do_rcv+0x77/0x2c0 tcp_v4_do_rcv+0x77/0x2c0 __release_sock+0x106/0x130 __tcp_close+0x1a7/0x4e0 tcp_close+0x20/0x70 inet_release+0x3c/0x80 __sock_release+0x3a/0xb0 sock_close+0x14/0x20 __fput+0xa3/0x260 task_work_run+0x59/0xb0 exit_to_user_mode_prepare+0x1b3/0x1c0 syscall_exit_to_user_mode+0x19/0x50 do_syscall_64+0x48/0x90 entry_SYSCALL_64_after_hwframe+0x44/0xae The root case is in commit 84472b436e76 ("bpf, sockmap: Fix more uncharged while msg has more_data"), where I used msg->sg.size to replace the tosend, causing breakage: if (msg->apply_bytes && msg->apply_bytes < tosend) tosend = psock->apply_bytes; Fixes: 84472b436e76 ("bpf, sockmap: Fix more uncharged while msg has more_data") Reported-by: Jakub Sitnicki Signed-off-by: Wang Yufen Signed-off-by: Daniel Borkmann Acked-by: John Fastabend Acked-by: Jakub Sitnicki Link: https://lore.kernel.org/bpf/1667266296-8794-1-git-send-email-wangyufen@huawei.com --- net/ipv4/tcp_bpf.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index c501c329b1dbe..cf9c3e8f7ccbf 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -278,7 +278,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, { bool cork = false, enospc = sk_msg_full(msg); struct sock *sk_redir; - u32 tosend, delta = 0; + u32 tosend, origsize, sent, delta = 0; u32 eval = __SK_NONE; int ret; @@ -333,10 +333,12 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, cork = true; psock->cork = NULL; } - sk_msg_return(sk, msg, msg->sg.size); + sk_msg_return(sk, msg, tosend); release_sock(sk); + origsize = msg->sg.size; ret = tcp_bpf_sendmsg_redir(sk_redir, msg, tosend, flags); + sent = origsize - msg->sg.size; if (eval == __SK_REDIRECT) sock_put(sk_redir); @@ -375,7 +377,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, msg->sg.data[msg->sg.start].page_link && msg->sg.data[msg->sg.start].length) { if (eval == __SK_REDIRECT) - sk_mem_charge(sk, msg->sg.size); + sk_mem_charge(sk, tosend - sent); goto more_data; } } -- GitLab From d3aefd2b29ff5ffdeb5c06a7d3191a027a18cdb8 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Mon, 31 Oct 2022 11:49:21 -0400 Subject: [PATCH 0879/1927] nfsd: fix net-namespace logic in __nfsd_file_cache_purge If the namespace doesn't match the one in "net", then we'll continue, but that doesn't cause another rhashtable_walk_next call, so it will loop infinitely. Fixes: ce502f81ba88 ("NFSD: Convert the filecache to use rhashtable") Reported-by: Petr Vorel Link: https://lore.kernel.org/ltp/Y1%2FP8gDAcWC%2F+VR3@pevik/ Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index 29a62db155fba..adc4e87a71d25 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -893,9 +893,8 @@ __nfsd_file_cache_purge(struct net *net) nf = rhashtable_walk_next(&iter); while (!IS_ERR_OR_NULL(nf)) { - if (net && nf->nf_net != net) - continue; - nfsd_file_unhash_and_dispose(nf, &dispose); + if (!net || nf->nf_net == net) + nfsd_file_unhash_and_dispose(nf, &dispose); nf = rhashtable_walk_next(&iter); } -- GitLab From 71ee71d7adcba648077997a29a91158d20c40b09 Mon Sep 17 00:00:00 2001 From: Vishal Verma Date: Tue, 1 Nov 2022 01:41:00 -0600 Subject: [PATCH 0880/1927] cxl/region: Fix decoder allocation crash When an intermediate port's decoders have been exhausted by existing regions, and creating a new region with the port in question in it's hierarchical path is attempted, cxl_port_attach_region() fails to find a port decoder (as would be expected), and drops into the failure / cleanup path. However, during cleanup of the region reference, a sanity check attempts to dereference the decoder, which in the above case didn't exist. This causes a NULL pointer dereference BUG. To fix this, refactor the decoder allocation and de-allocation into helper routines, and in this 'free' routine, check that the decoder, @cxld, is valid before attempting any operations on it. Cc: Suggested-by: Dan Williams Signed-off-by: Vishal Verma Reviewed-by: Dave Jiang Fixes: 384e624bb211 ("cxl/region: Attach endpoint decoders") Link: https://lore.kernel.org/r/20221101074100.1732003-1-vishal.l.verma@intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/region.c | 67 ++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index c49d9a5f1091c..bb6f4fc84a3ff 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -687,18 +687,27 @@ static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port, return cxl_rr; } -static void free_region_ref(struct cxl_region_ref *cxl_rr) +static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) { - struct cxl_port *port = cxl_rr->port; struct cxl_region *cxlr = cxl_rr->region; struct cxl_decoder *cxld = cxl_rr->decoder; + if (!cxld) + return; + dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); if (cxld->region == cxlr) { cxld->region = NULL; put_device(&cxlr->dev); } +} +static void free_region_ref(struct cxl_region_ref *cxl_rr) +{ + struct cxl_port *port = cxl_rr->port; + struct cxl_region *cxlr = cxl_rr->region; + + cxl_rr_free_decoder(cxl_rr); xa_erase(&port->regions, (unsigned long)cxlr); xa_destroy(&cxl_rr->endpoints); kfree(cxl_rr); @@ -729,6 +738,33 @@ static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, return 0; } +static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, + struct cxl_endpoint_decoder *cxled, + struct cxl_region_ref *cxl_rr) +{ + struct cxl_decoder *cxld; + + if (port == cxled_to_port(cxled)) + cxld = &cxled->cxld; + else + cxld = cxl_region_find_decoder(port, cxlr); + if (!cxld) { + dev_dbg(&cxlr->dev, "%s: no decoder available\n", + dev_name(&port->dev)); + return -EBUSY; + } + + if (cxld->region) { + dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", + dev_name(&port->dev), dev_name(&cxld->dev), + dev_name(&cxld->region->dev)); + return -EBUSY; + } + + cxl_rr->decoder = cxld; + return 0; +} + /** * cxl_port_attach_region() - track a region's interest in a port by endpoint * @port: port to add a new region reference 'struct cxl_region_ref' @@ -795,12 +831,6 @@ static int cxl_port_attach_region(struct cxl_port *port, cxl_rr->nr_targets++; nr_targets_inc = true; } - - /* - * The decoder for @cxlr was allocated when the region was first - * attached to @port. - */ - cxld = cxl_rr->decoder; } else { cxl_rr = alloc_region_ref(port, cxlr); if (IS_ERR(cxl_rr)) { @@ -811,26 +841,11 @@ static int cxl_port_attach_region(struct cxl_port *port, } nr_targets_inc = true; - if (port == cxled_to_port(cxled)) - cxld = &cxled->cxld; - else - cxld = cxl_region_find_decoder(port, cxlr); - if (!cxld) { - dev_dbg(&cxlr->dev, "%s: no decoder available\n", - dev_name(&port->dev)); - goto out_erase; - } - - if (cxld->region) { - dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", - dev_name(&port->dev), dev_name(&cxld->dev), - dev_name(&cxld->region->dev)); - rc = -EBUSY; + rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); + if (rc) goto out_erase; - } - - cxl_rr->decoder = cxld; } + cxld = cxl_rr->decoder; rc = cxl_rr_ep_add(cxl_rr, cxled); if (rc) { -- GitLab From 373e715e31bf4e0f129befe87613a278fac228d3 Mon Sep 17 00:00:00 2001 From: "Kirill A. Shutemov" Date: Fri, 28 Oct 2022 17:12:20 +0300 Subject: [PATCH 0881/1927] x86/tdx: Panic on bad configs that #VE on "private" memory access All normal kernel memory is "TDX private memory". This includes everything from kernel stacks to kernel text. Handling exceptions on arbitrary accesses to kernel memory is essentially impossible because they can happen in horribly nasty places like kernel entry/exit. But, TDX hardware can theoretically _deliver_ a virtualization exception (#VE) on any access to private memory. But, it's not as bad as it sounds. TDX can be configured to never deliver these exceptions on private memory with a "TD attribute" called ATTR_SEPT_VE_DISABLE. The guest has no way to *set* this attribute, but it can check it. Ensure ATTR_SEPT_VE_DISABLE is set in early boot. panic() if it is unset. There is no sane way for Linux to run with this attribute clear so a panic() is appropriate. There's small window during boot before the check where kernel has an early #VE handler. But the handler is only for port I/O and will also panic() as soon as it sees any other #VE, such as a one generated by a private memory access. [ dhansen: Rewrite changelog and rebase on new tdx_parse_tdinfo(). Add Kirill's tested-by because I made changes since he wrote this. ] Fixes: 9a22bf6debbf ("x86/traps: Add #VE support for TDX guest") Reported-by: ruogui.ygr@alibaba-inc.com Signed-off-by: Kirill A. Shutemov Signed-off-by: Dave Hansen Tested-by: Kirill A. Shutemov Cc: stable@vger.kernel.org Link: https://lore.kernel.org/all/20221028141220.29217-3-kirill.shutemov%40linux.intel.com --- arch/x86/coco/tdx/tdx.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 3fee96931ff56..b8998cf0508a6 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -34,6 +34,8 @@ #define VE_GET_PORT_NUM(e) ((e) >> 16) #define VE_IS_IO_STRING(e) ((e) & BIT(4)) +#define ATTR_SEPT_VE_DISABLE BIT(28) + /* * Wrapper for standard use of __tdx_hypercall with no output aside from * return code. @@ -102,6 +104,7 @@ static void tdx_parse_tdinfo(u64 *cc_mask) { struct tdx_module_output out; unsigned int gpa_width; + u64 td_attr; /* * TDINFO TDX module call is used to get the TD execution environment @@ -109,19 +112,27 @@ static void tdx_parse_tdinfo(u64 *cc_mask) * information, etc. More details about the ABI can be found in TDX * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL * [TDG.VP.INFO]. - * - * The GPA width that comes out of this call is critical. TDX guests - * can not meaningfully run without it. */ tdx_module_call(TDX_GET_INFO, 0, 0, 0, 0, &out); - gpa_width = out.rcx & GENMASK(5, 0); - /* * The highest bit of a guest physical address is the "sharing" bit. * Set it for shared pages and clear it for private pages. + * + * The GPA width that comes out of this call is critical. TDX guests + * can not meaningfully run without it. */ + gpa_width = out.rcx & GENMASK(5, 0); *cc_mask = BIT_ULL(gpa_width - 1); + + /* + * The kernel can not handle #VE's when accessing normal kernel + * memory. Ensure that no #VE will be delivered for accesses to + * TD-private memory. Only VMM-shared memory (MMIO) will #VE. + */ + td_attr = out.rdx; + if (!(td_attr & ATTR_SEPT_VE_DISABLE)) + panic("TD misconfiguration: SEPT_VE_DISABLE attibute must be set.\n"); } /* -- GitLab From 486c292230166c2d61701d3c984bf9143588ea28 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Sun, 30 Oct 2022 22:36:34 +0100 Subject: [PATCH 0882/1927] net: lan966x: Fix the MTU calculation When the MTU was changed, the lan966x didn't take in consideration the L2 header and the FCS. So the HW was configured with a smaller value than what was desired. Therefore the correct value to configure the HW would be new_mtu + ETH_HLEN + ETH_FCS_LEN. The vlan tag is not considered here, because at the time when the blamed commit was added, there was no vlan filtering support. The vlan fix will be part of the next patch. Fixes: d28d6d2e37d1 ("net: lan966x: add port module support") Signed-off-by: Horatiu Vultur Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/microchip/lan966x/lan966x_main.c | 2 +- drivers/net/ethernet/microchip/lan966x/lan966x_main.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index be2fd030cccbe..b3070c3fcad0a 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -386,7 +386,7 @@ static int lan966x_port_change_mtu(struct net_device *dev, int new_mtu) int old_mtu = dev->mtu; int err; - lan_wr(DEV_MAC_MAXLEN_CFG_MAX_LEN_SET(new_mtu), + lan_wr(DEV_MAC_MAXLEN_CFG_MAX_LEN_SET(LAN966X_HW_MTU(new_mtu)), lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); dev->mtu = new_mtu; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index 9656071b8289e..4ec33999e4df6 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -26,6 +26,8 @@ #define LAN966X_BUFFER_MEMORY (160 * 1024) #define LAN966X_BUFFER_MIN_SZ 60 +#define LAN966X_HW_MTU(mtu) ((mtu) + ETH_HLEN + ETH_FCS_LEN) + #define PGID_AGGR 64 #define PGID_SRC 80 #define PGID_ENTRIES 89 -- GitLab From 25f28bb1b4a7717a9df3aa574d210374ebb6bb23 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Sun, 30 Oct 2022 22:36:35 +0100 Subject: [PATCH 0883/1927] net: lan966x: Adjust maximum frame size when vlan is enabled/disabled When vlan filtering is enabled/disabled, it is required to adjust the maximum received frame size that it can received. When vlan filtering is enabled, it would all to receive extra 4 bytes, that are the vlan tag. So the maximum frame size would be 1522 with a vlan tag. If vlan filtering is disabled then the maximum frame size would be 1518 regardless if there is or not a vlan tag. Fixes: 6d2c186afa5d ("net: lan966x: Add vlan support.") Signed-off-by: Horatiu Vultur Signed-off-by: Jakub Kicinski --- .../net/ethernet/microchip/lan966x/lan966x_regs.h | 15 +++++++++++++++ .../net/ethernet/microchip/lan966x/lan966x_vlan.c | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h index 1d90b93dd417a..fb5087fef22e1 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h @@ -585,6 +585,21 @@ enum lan966x_target { #define DEV_MAC_MAXLEN_CFG_MAX_LEN_GET(x)\ FIELD_GET(DEV_MAC_MAXLEN_CFG_MAX_LEN, x) +/* DEV:MAC_CFG_STATUS:MAC_TAGS_CFG */ +#define DEV_MAC_TAGS_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 12, 0, 1, 4) + +#define DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA BIT(1) +#define DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA_SET(x)\ + FIELD_PREP(DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA, x) +#define DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA_GET(x)\ + FIELD_GET(DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA, x) + +#define DEV_MAC_TAGS_CFG_VLAN_AWR_ENA BIT(0) +#define DEV_MAC_TAGS_CFG_VLAN_AWR_ENA_SET(x)\ + FIELD_PREP(DEV_MAC_TAGS_CFG_VLAN_AWR_ENA, x) +#define DEV_MAC_TAGS_CFG_VLAN_AWR_ENA_GET(x)\ + FIELD_GET(DEV_MAC_TAGS_CFG_VLAN_AWR_ENA, x) + /* DEV:MAC_CFG_STATUS:MAC_IFG_CFG */ #define DEV_MAC_IFG_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 20, 0, 1, 4) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c b/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c index 8d7260cd7da9c..3c44660128dae 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_vlan.c @@ -169,6 +169,12 @@ void lan966x_vlan_port_apply(struct lan966x_port *port) ANA_VLAN_CFG_VLAN_POP_CNT, lan966x, ANA_VLAN_CFG(port->chip_port)); + lan_rmw(DEV_MAC_TAGS_CFG_VLAN_AWR_ENA_SET(port->vlan_aware) | + DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA_SET(port->vlan_aware), + DEV_MAC_TAGS_CFG_VLAN_AWR_ENA | + DEV_MAC_TAGS_CFG_VLAN_DBL_AWR_ENA, + lan966x, DEV_MAC_TAGS_CFG(port->chip_port)); + /* Drop frames with multicast source address */ val = ANA_DROP_CFG_DROP_MC_SMAC_ENA_SET(1); if (port->vlan_aware && !pvid) -- GitLab From 872ad758f9b7fb4eb42aebaf64e50c5b29b7ffe5 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Sun, 30 Oct 2022 22:36:36 +0100 Subject: [PATCH 0884/1927] net: lan966x: Fix FDMA when MTU is changed When MTU is changed, FDMA is required to calculate what is the maximum size of the frame that it can received. So it can calculate what is the page order needed to allocate for the received frames. The first problem was that, when the max MTU was calculated it was reading the value from dev and not from HW, so in this way it was missing L2 header + the FCS. The other problem was that once the skb is created using __build_skb_around, it would reserve some space for skb_shared_info. So if we received a frame which size is at the limit of the page order then the creating will failed because it would not have space to put all the data. Fixes: 2ea1cbac267e ("net: lan966x: Update FDMA to change MTU.") Signed-off-by: Horatiu Vultur Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c | 8 ++++++-- drivers/net/ethernet/microchip/lan966x/lan966x_main.c | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index a42035cec611c..c235edd2b182a 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -668,12 +668,14 @@ static int lan966x_fdma_get_max_mtu(struct lan966x *lan966x) int i; for (i = 0; i < lan966x->num_phys_ports; ++i) { + struct lan966x_port *port; int mtu; - if (!lan966x->ports[i]) + port = lan966x->ports[i]; + if (!port) continue; - mtu = lan966x->ports[i]->dev->mtu; + mtu = lan_rd(lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); if (mtu > max_mtu) max_mtu = mtu; } @@ -733,6 +735,8 @@ int lan966x_fdma_change_mtu(struct lan966x *lan966x) max_mtu = lan966x_fdma_get_max_mtu(lan966x); max_mtu += IFH_LEN * sizeof(u32); + max_mtu += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + max_mtu += VLAN_HLEN * 2; if (round_up(max_mtu, PAGE_SIZE) / PAGE_SIZE - 1 == lan966x->rx.page_order) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index b3070c3fcad0a..20ee5b28f70a5 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -395,7 +395,7 @@ static int lan966x_port_change_mtu(struct net_device *dev, int new_mtu) err = lan966x_fdma_change_mtu(lan966x); if (err) { - lan_wr(DEV_MAC_MAXLEN_CFG_MAX_LEN_SET(old_mtu), + lan_wr(DEV_MAC_MAXLEN_CFG_MAX_LEN_SET(LAN966X_HW_MTU(old_mtu)), lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); dev->mtu = old_mtu; } -- GitLab From fc57062f98b0b0ae52bc584d8fd5ac77c50df607 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Mon, 31 Oct 2022 14:34:21 +0100 Subject: [PATCH 0885/1927] net: lan966x: Fix unmapping of received frames using FDMA When lan966x was receiving a frame, then it was building the skb and after that it was calling dma_unmap_single with frame size as the length. This actually has 2 issues: 1. It is using a length to map and a different length to unmap. 2. When the unmap was happening, the data was sync for cpu but it could be that this will overwrite what build_skb was initializing. The fix for these two problems is to change the order of operations. First to sync the frame for cpu, then to build the skb and in the end to unmap using the correct size but without sync the frame again for cpu. Fixes: c8349639324a ("net: lan966x: Add FDMA functionality") Signed-off-by: Horatiu Vultur Link: https://lore.kernel.org/r/20221031133421.1283196-1-horatiu.vultur@microchip.com Signed-off-by: Jakub Kicinski --- .../ethernet/microchip/lan966x/lan966x_fdma.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index c235edd2b182a..e6948939ccc2b 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -414,13 +414,15 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) /* Get the received frame and unmap it */ db = &rx->dcbs[rx->dcb_index].db[rx->db_index]; page = rx->page[rx->dcb_index][rx->db_index]; + + dma_sync_single_for_cpu(lan966x->dev, (dma_addr_t)db->dataptr, + FDMA_DCB_STATUS_BLOCKL(db->status), + DMA_FROM_DEVICE); + skb = build_skb(page_address(page), PAGE_SIZE << rx->page_order); if (unlikely(!skb)) goto unmap_page; - dma_unmap_single(lan966x->dev, (dma_addr_t)db->dataptr, - FDMA_DCB_STATUS_BLOCKL(db->status), - DMA_FROM_DEVICE); skb_put(skb, FDMA_DCB_STATUS_BLOCKL(db->status)); lan966x_ifh_get_src_port(skb->data, &src_port); @@ -429,6 +431,10 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) if (WARN_ON(src_port >= lan966x->num_phys_ports)) goto free_skb; + dma_unmap_single_attrs(lan966x->dev, (dma_addr_t)db->dataptr, + PAGE_SIZE << rx->page_order, DMA_FROM_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); + skb->dev = lan966x->ports[src_port]->dev; skb_pull(skb, IFH_LEN * sizeof(u32)); @@ -454,9 +460,9 @@ static struct sk_buff *lan966x_fdma_rx_get_frame(struct lan966x_rx *rx) free_skb: kfree_skb(skb); unmap_page: - dma_unmap_page(lan966x->dev, (dma_addr_t)db->dataptr, - FDMA_DCB_STATUS_BLOCKL(db->status), - DMA_FROM_DEVICE); + dma_unmap_single_attrs(lan966x->dev, (dma_addr_t)db->dataptr, + PAGE_SIZE << rx->page_order, DMA_FROM_DEVICE, + DMA_ATTR_SKIP_CPU_SYNC); __free_pages(page, rx->page_order); return NULL; -- GitLab From ecaf75ffd5f5db320d8b1da0198eef5a5ce64a3f Mon Sep 17 00:00:00 2001 From: Florian Westphal Date: Mon, 31 Oct 2022 13:34:07 +0100 Subject: [PATCH 0886/1927] netlink: introduce bigendian integer types Jakub reported that the addition of the "network_byte_order" member in struct nla_policy increases size of 32bit platforms. Instead of scraping the bit from elsewhere Johannes suggested to add explicit NLA_BE types instead, so do this here. NLA_POLICY_MAX_BE() macro is removed again, there is no need for it: NLA_POLICY_MAX(NLA_BE.., ..) will do the right thing. NLA_BE64 can be added later. Fixes: 08724ef69907 ("netlink: introduce NLA_POLICY_MAX_BE") Reported-by: Jakub Kicinski Suggested-by: Johannes Berg Signed-off-by: Florian Westphal Link: https://lore.kernel.org/r/20221031123407.9158-1-fw@strlen.de Signed-off-by: Jakub Kicinski --- include/net/netlink.h | 19 +++++++++-------- lib/nlattr.c | 41 ++++++++++++++----------------------- net/netfilter/nft_payload.c | 6 +++--- 3 files changed, 28 insertions(+), 38 deletions(-) diff --git a/include/net/netlink.h b/include/net/netlink.h index 7db13b3261fc2..6bfa972f2fbf2 100644 --- a/include/net/netlink.h +++ b/include/net/netlink.h @@ -181,6 +181,8 @@ enum { NLA_S64, NLA_BITFIELD32, NLA_REJECT, + NLA_BE16, + NLA_BE32, __NLA_TYPE_MAX, }; @@ -231,6 +233,7 @@ enum nla_policy_validation { * NLA_U32, NLA_U64, * NLA_S8, NLA_S16, * NLA_S32, NLA_S64, + * NLA_BE16, NLA_BE32, * NLA_MSECS Leaving the length field zero will verify the * given type fits, using it verifies minimum length * just like "All other" @@ -261,6 +264,8 @@ enum nla_policy_validation { * NLA_U16, * NLA_U32, * NLA_U64, + * NLA_BE16, + * NLA_BE32, * NLA_S8, * NLA_S16, * NLA_S32, @@ -349,7 +354,6 @@ struct nla_policy { struct netlink_range_validation_signed *range_signed; struct { s16 min, max; - u8 network_byte_order:1; }; int (*validate)(const struct nlattr *attr, struct netlink_ext_ack *extack); @@ -374,6 +378,8 @@ struct nla_policy { (tp == NLA_U8 || tp == NLA_U16 || tp == NLA_U32 || tp == NLA_U64) #define __NLA_IS_SINT_TYPE(tp) \ (tp == NLA_S8 || tp == NLA_S16 || tp == NLA_S32 || tp == NLA_S64) +#define __NLA_IS_BEINT_TYPE(tp) \ + (tp == NLA_BE16 || tp == NLA_BE32) #define __NLA_ENSURE(condition) BUILD_BUG_ON_ZERO(!(condition)) #define NLA_ENSURE_UINT_TYPE(tp) \ @@ -387,6 +393,7 @@ struct nla_policy { #define NLA_ENSURE_INT_OR_BINARY_TYPE(tp) \ (__NLA_ENSURE(__NLA_IS_UINT_TYPE(tp) || \ __NLA_IS_SINT_TYPE(tp) || \ + __NLA_IS_BEINT_TYPE(tp) || \ tp == NLA_MSECS || \ tp == NLA_BINARY) + tp) #define NLA_ENSURE_NO_VALIDATION_PTR(tp) \ @@ -394,6 +401,8 @@ struct nla_policy { tp != NLA_REJECT && \ tp != NLA_NESTED && \ tp != NLA_NESTED_ARRAY) + tp) +#define NLA_ENSURE_BEINT_TYPE(tp) \ + (__NLA_ENSURE(__NLA_IS_BEINT_TYPE(tp)) + tp) #define NLA_POLICY_RANGE(tp, _min, _max) { \ .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \ @@ -424,14 +433,6 @@ struct nla_policy { .type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp), \ .validation_type = NLA_VALIDATE_MAX, \ .max = _max, \ - .network_byte_order = 0, \ -} - -#define NLA_POLICY_MAX_BE(tp, _max) { \ - .type = NLA_ENSURE_UINT_TYPE(tp), \ - .validation_type = NLA_VALIDATE_MAX, \ - .max = _max, \ - .network_byte_order = 1, \ } #define NLA_POLICY_MASK(tp, _mask) { \ diff --git a/lib/nlattr.c b/lib/nlattr.c index 40f22b177d690..b67a53e29b8fe 100644 --- a/lib/nlattr.c +++ b/lib/nlattr.c @@ -124,10 +124,12 @@ void nla_get_range_unsigned(const struct nla_policy *pt, range->max = U8_MAX; break; case NLA_U16: + case NLA_BE16: case NLA_BINARY: range->max = U16_MAX; break; case NLA_U32: + case NLA_BE32: range->max = U32_MAX; break; case NLA_U64: @@ -159,31 +161,6 @@ void nla_get_range_unsigned(const struct nla_policy *pt, } } -static u64 nla_get_attr_bo(const struct nla_policy *pt, - const struct nlattr *nla) -{ - switch (pt->type) { - case NLA_U16: - if (pt->network_byte_order) - return ntohs(nla_get_be16(nla)); - - return nla_get_u16(nla); - case NLA_U32: - if (pt->network_byte_order) - return ntohl(nla_get_be32(nla)); - - return nla_get_u32(nla); - case NLA_U64: - if (pt->network_byte_order) - return be64_to_cpu(nla_get_be64(nla)); - - return nla_get_u64(nla); - } - - WARN_ON_ONCE(1); - return 0; -} - static int nla_validate_range_unsigned(const struct nla_policy *pt, const struct nlattr *nla, struct netlink_ext_ack *extack, @@ -197,9 +174,13 @@ static int nla_validate_range_unsigned(const struct nla_policy *pt, value = nla_get_u8(nla); break; case NLA_U16: + value = nla_get_u16(nla); + break; case NLA_U32: + value = nla_get_u32(nla); + break; case NLA_U64: - value = nla_get_attr_bo(pt, nla); + value = nla_get_u64(nla); break; case NLA_MSECS: value = nla_get_u64(nla); @@ -207,6 +188,12 @@ static int nla_validate_range_unsigned(const struct nla_policy *pt, case NLA_BINARY: value = nla_len(nla); break; + case NLA_BE16: + value = ntohs(nla_get_be16(nla)); + break; + case NLA_BE32: + value = ntohl(nla_get_be32(nla)); + break; default: return -EINVAL; } @@ -334,6 +321,8 @@ static int nla_validate_int_range(const struct nla_policy *pt, case NLA_U64: case NLA_MSECS: case NLA_BINARY: + case NLA_BE16: + case NLA_BE32: return nla_validate_range_unsigned(pt, nla, extack, validate); case NLA_S8: case NLA_S16: diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index 088244f9d8383..4edd899aeb9bb 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -173,10 +173,10 @@ static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = { [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 }, [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 }, [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 }, - [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX_BE(NLA_U32, 255), - [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX_BE(NLA_U32, 255), + [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), + [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 }, - [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX_BE(NLA_U32, 255), + [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 }, }; -- GitLab From 7a263a0402561035199cd9049baadb908a92b6b4 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 24 Oct 2022 04:10:55 +0900 Subject: [PATCH 0887/1927] kconfig: fix segmentation fault in menuconfig search Since commit d05377e184fc ("kconfig: Create links to main menu items in search"), menuconfig shows a jump key next to "Main menu" if the nearest visible parent is the rootmenu. If you press that jump key, menuconfig crashes with a segmentation fault. For example, do this: $ make ARCH=arm64 allnoconfig menuconfig Press '/' to search for the string "ACPI". Press '1' to choose "(1) Main menu". Then, menuconfig crashed with a segmentation fault. The following code in search_conf() conf(targets[i]->parent, targets[i]); results in NULL pointer dereference because targets[i] is the rootmenu, which does not have a parent. Commit d05377e184fc tried to fix the issue of top-level items not having a jump key, but adding the "Main menu" was not the right fix. The correct fix is to show the searched item itself. This fixes another weird behavior described in the comment block. Fixes: d05377e184fc ("kconfig: Create links to main menu items in search") Reported-by: Johannes Zink Signed-off-by: Masahiro Yamada Tested-by: Bagas Sanjaya Tested-by: Johannes Zink --- scripts/kconfig/menu.c | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/scripts/kconfig/menu.c b/scripts/kconfig/menu.c index 62b6313f51c8b..109325f31bef3 100644 --- a/scripts/kconfig/menu.c +++ b/scripts/kconfig/menu.c @@ -722,8 +722,8 @@ static void get_prompt_str(struct gstr *r, struct property *prop, if (!expr_eq(prop->menu->dep, prop->visible.expr)) get_dep_str(r, prop->visible.expr, " Visible if: "); - menu = prop->menu->parent; - for (i = 0; menu && i < 8; menu = menu->parent) { + menu = prop->menu; + for (i = 0; menu != &rootmenu && i < 8; menu = menu->parent) { bool accessible = menu_is_visible(menu); submenu[i++] = menu; @@ -733,16 +733,7 @@ static void get_prompt_str(struct gstr *r, struct property *prop, if (head && location) { jump = xmalloc(sizeof(struct jump_key)); - if (menu_is_visible(prop->menu)) { - /* - * There is not enough room to put the hint at the - * beginning of the "Prompt" line. Put the hint on the - * last "Location" line even when it would belong on - * the former. - */ - jump->target = prop->menu; - } else - jump->target = location; + jump->target = location; if (list_empty(head)) jump->index = 0; @@ -758,13 +749,7 @@ static void get_prompt_str(struct gstr *r, struct property *prop, menu = submenu[i]; if (jump && menu == location) jump->offset = strlen(r->s); - - if (menu == &rootmenu) - /* The real rootmenu prompt is ugly */ - str_printf(r, "%*cMain menu", j, ' '); - else - str_printf(r, "%*c-> %s", j, ' ', menu_get_prompt(menu)); - + str_printf(r, "%*c-> %s", j, ' ', menu_get_prompt(menu)); if (menu->sym) { str_printf(r, " (%s [=%s])", menu->sym->name ? menu->sym->name : "", -- GitLab From 5c26159c97b324dc5174a5713eafb8c855cf8106 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 26 Oct 2022 14:32:16 +0200 Subject: [PATCH 0888/1927] ipvs: use explicitly signed chars The `char` type with no explicit sign is sometimes signed and sometimes unsigned. This code will break on platforms such as arm, where char is unsigned. So mark it here as explicitly signed, so that the todrop_counter decrement and subsequent comparison is correct. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Jason A. Donenfeld Acked-by: Julian Anastasov Signed-off-by: Pablo Neira Ayuso --- net/netfilter/ipvs/ip_vs_conn.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 8c04bb57dd6fe..7c4866c043437 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1265,8 +1265,8 @@ static inline int todrop_entry(struct ip_vs_conn *cp) * The drop rate array needs tuning for real environments. * Called from timer bh only => no locking */ - static const char todrop_rate[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; - static char todrop_counter[9] = {0}; + static const signed char todrop_rate[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + static signed char todrop_counter[9] = {0}; int i; /* if the conn entry hasn't lasted for 60 seconds, don't drop it. -- GitLab From 3d00c6a0da8ddcf75213e004765e4a42acc71d5d Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 31 Oct 2022 20:07:04 +0800 Subject: [PATCH 0889/1927] ipvs: fix WARNING in __ip_vs_cleanup_batch() During the initialization of ip_vs_conn_net_init(), if file ip_vs_conn or ip_vs_conn_sync fails to be created, the initialization is successful by default. Therefore, the ip_vs_conn or ip_vs_conn_sync file doesn't be found during the remove. The following is the stack information: name 'ip_vs_conn_sync' WARNING: CPU: 3 PID: 9 at fs/proc/generic.c:712 remove_proc_entry+0x389/0x460 Modules linked in: Workqueue: netns cleanup_net RIP: 0010:remove_proc_entry+0x389/0x460 Call Trace: __ip_vs_cleanup_batch+0x7d/0x120 ops_exit_list+0x125/0x170 cleanup_net+0x4ea/0xb00 process_one_work+0x9bf/0x1710 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Fixes: 61b1ab4583e2 ("IPVS: netns, add basic init per netns.") Signed-off-by: Zhengchao Shao Acked-by: Julian Anastasov Signed-off-by: Pablo Neira Ayuso --- net/netfilter/ipvs/ip_vs_conn.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 7c4866c043437..13534e02346cc 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -1447,20 +1447,36 @@ int __net_init ip_vs_conn_net_init(struct netns_ipvs *ipvs) { atomic_set(&ipvs->conn_count, 0); - proc_create_net("ip_vs_conn", 0, ipvs->net->proc_net, - &ip_vs_conn_seq_ops, sizeof(struct ip_vs_iter_state)); - proc_create_net("ip_vs_conn_sync", 0, ipvs->net->proc_net, - &ip_vs_conn_sync_seq_ops, - sizeof(struct ip_vs_iter_state)); +#ifdef CONFIG_PROC_FS + if (!proc_create_net("ip_vs_conn", 0, ipvs->net->proc_net, + &ip_vs_conn_seq_ops, + sizeof(struct ip_vs_iter_state))) + goto err_conn; + + if (!proc_create_net("ip_vs_conn_sync", 0, ipvs->net->proc_net, + &ip_vs_conn_sync_seq_ops, + sizeof(struct ip_vs_iter_state))) + goto err_conn_sync; +#endif + return 0; + +#ifdef CONFIG_PROC_FS +err_conn_sync: + remove_proc_entry("ip_vs_conn", ipvs->net->proc_net); +err_conn: + return -ENOMEM; +#endif } void __net_exit ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs) { /* flush all the connection entries first */ ip_vs_conn_flush(ipvs); +#ifdef CONFIG_PROC_FS remove_proc_entry("ip_vs_conn", ipvs->net->proc_net); remove_proc_entry("ip_vs_conn_sync", ipvs->net->proc_net); +#endif } int __init ip_vs_conn_init(void) -- GitLab From 5663ed63adb9619c98ab7479aa4606fa9b7a548c Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 31 Oct 2022 20:07:05 +0800 Subject: [PATCH 0890/1927] ipvs: fix WARNING in ip_vs_app_net_cleanup() During the initialization of ip_vs_app_net_init(), if file ip_vs_app fails to be created, the initialization is successful by default. Therefore, the ip_vs_app file doesn't be found during the remove in ip_vs_app_net_cleanup(). It will cause WRNING. The following is the stack information: name 'ip_vs_app' WARNING: CPU: 1 PID: 9 at fs/proc/generic.c:712 remove_proc_entry+0x389/0x460 Modules linked in: Workqueue: netns cleanup_net RIP: 0010:remove_proc_entry+0x389/0x460 Call Trace: ops_exit_list+0x125/0x170 cleanup_net+0x4ea/0xb00 process_one_work+0x9bf/0x1710 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Fixes: 457c4cbc5a3d ("[NET]: Make /proc/net per network namespace") Signed-off-by: Zhengchao Shao Acked-by: Julian Anastasov Signed-off-by: Pablo Neira Ayuso --- net/netfilter/ipvs/ip_vs_app.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/net/netfilter/ipvs/ip_vs_app.c b/net/netfilter/ipvs/ip_vs_app.c index f9b16f2b22191..fdacbc3c15bef 100644 --- a/net/netfilter/ipvs/ip_vs_app.c +++ b/net/netfilter/ipvs/ip_vs_app.c @@ -599,13 +599,19 @@ static const struct seq_operations ip_vs_app_seq_ops = { int __net_init ip_vs_app_net_init(struct netns_ipvs *ipvs) { INIT_LIST_HEAD(&ipvs->app_list); - proc_create_net("ip_vs_app", 0, ipvs->net->proc_net, &ip_vs_app_seq_ops, - sizeof(struct seq_net_private)); +#ifdef CONFIG_PROC_FS + if (!proc_create_net("ip_vs_app", 0, ipvs->net->proc_net, + &ip_vs_app_seq_ops, + sizeof(struct seq_net_private))) + return -ENOMEM; +#endif return 0; } void __net_exit ip_vs_app_net_cleanup(struct netns_ipvs *ipvs) { unregister_ip_vs_app(ipvs, NULL /* all */); +#ifdef CONFIG_PROC_FS remove_proc_entry("ip_vs_app", ipvs->net->proc_net); +#endif } -- GitLab From 780854186946e0de2be192ee7fa5125666533b3a Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 26 Oct 2022 14:39:59 +0800 Subject: [PATCH 0891/1927] wifi: mac80211: fix general-protection-fault in ieee80211_subif_start_xmit() When device is running and the interface status is changed, the gpf issue is triggered. The problem triggering process is as follows: Thread A: Thread B ieee80211_runtime_change_iftype() process_one_work() ... ... ieee80211_do_stop() ... ... ... sdata->bss = NULL ... ... ieee80211_subif_start_xmit() ieee80211_multicast_to_unicast //!sdata->bss->multicast_to_unicast cause gpf issue When the interface status is changed, the sending queue continues to send packets. After the bss is set to NULL, the bss is accessed. As a result, this causes a general-protection-fault issue. The following is the stack information: general protection fault, probably for non-canonical address 0xdffffc000000002f: 0000 [#1] PREEMPT SMP KASAN KASAN: null-ptr-deref in range [0x0000000000000178-0x000000000000017f] Workqueue: mld mld_ifc_work RIP: 0010:ieee80211_subif_start_xmit+0x25b/0x1310 Call Trace: dev_hard_start_xmit+0x1be/0x990 __dev_queue_xmit+0x2c9a/0x3b60 ip6_finish_output2+0xf92/0x1520 ip6_finish_output+0x6af/0x11e0 ip6_output+0x1ed/0x540 mld_sendpack+0xa09/0xe70 mld_ifc_work+0x71c/0xdb0 process_one_work+0x9bf/0x1710 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Fixes: f856373e2f31 ("wifi: mac80211: do not wake queues on a vif that is being stopped") Reported-by: syzbot+c6e8fca81c294fd5620a@syzkaller.appspotmail.com Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221026063959.177813-1-shaozhengchao@huawei.com Signed-off-by: Johannes Berg --- net/mac80211/tx.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index a364148149f94..874f2a4d831d0 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -4418,6 +4418,11 @@ netdev_tx_t ieee80211_subif_start_xmit(struct sk_buff *skb, if (likely(!is_multicast_ether_addr(eth->h_dest))) goto normal; + if (unlikely(!ieee80211_sdata_running(sdata))) { + kfree_skb(skb); + return NETDEV_TX_OK; + } + if (unlikely(ieee80211_multicast_to_unicast(skb, dev))) { struct sk_buff_head queue; -- GitLab From 39e7b5de9853bd92ddbfa4b14165babacd7da0ba Mon Sep 17 00:00:00 2001 From: Nicolas Cavallari Date: Thu, 27 Oct 2022 16:01:33 +0200 Subject: [PATCH 0892/1927] wifi: mac80211: Fix ack frame idr leak when mesh has no route When trying to transmit an data frame with tx_status to a destination that have no route in the mesh, then it is dropped without recrediting the ack_status_frames idr. Once it is exhausted, wpa_supplicant starts failing to do SAE with NL80211_CMD_FRAME and logs "nl80211: Frame command failed". Use ieee80211_free_txskb() instead of kfree_skb() to fix it. Signed-off-by: Nicolas Cavallari Link: https://lore.kernel.org/r/20221027140133.1504-1-nicolas.cavallari@green-communications.fr Signed-off-by: Johannes Berg --- net/mac80211/mesh_pathtbl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/mac80211/mesh_pathtbl.c b/net/mac80211/mesh_pathtbl.c index acc1c299f1ae5..69d5e1ec6edef 100644 --- a/net/mac80211/mesh_pathtbl.c +++ b/net/mac80211/mesh_pathtbl.c @@ -710,7 +710,7 @@ int mesh_path_send_to_gates(struct mesh_path *mpath) void mesh_path_discard_frame(struct ieee80211_sub_if_data *sdata, struct sk_buff *skb) { - kfree_skb(skb); + ieee80211_free_txskb(&sdata->local->hw, skb); sdata->u.mesh.mshstats.dropped_frames_no_route++; } -- GitLab From 30ac96f7cc973bb850c718c9bbe1fdcedfbe826b Mon Sep 17 00:00:00 2001 From: Howard Hsu Date: Thu, 27 Oct 2022 09:56:53 +0800 Subject: [PATCH 0893/1927] wifi: mac80211: Set TWT Information Frame Disabled bit as 1 The TWT Information Frame Disabled bit of control field of TWT Setup frame shall be set to 1 since handling TWT Information frame is not supported by current mac80211 implementation. Fixes: f5a4c24e689f ("mac80211: introduce individual TWT support in AP mode") Signed-off-by: Howard Hsu Link: https://lore.kernel.org/r/20221027015653.1448-1-howard-yh.hsu@mediatek.com Signed-off-by: Johannes Berg --- net/mac80211/s1g.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c index 8ca7d45d6daae..c1f964e9991cd 100644 --- a/net/mac80211/s1g.c +++ b/net/mac80211/s1g.c @@ -112,6 +112,9 @@ ieee80211_s1g_rx_twt_setup(struct ieee80211_sub_if_data *sdata, goto out; } + /* TWT Information not supported yet */ + twt->control |= IEEE80211_TWT_CONTROL_RX_DISABLED; + drv_add_twt_setup(sdata->local, sdata, &sta->sta, twt); out: ieee80211_s1g_send_twt_setup(sdata, mgmt->sa, sdata->vif.addr, twt); -- GitLab From 89c1017aac67ca81973b7c8eac5d021315811a93 Mon Sep 17 00:00:00 2001 From: Zhao Gongyi Date: Tue, 1 Nov 2022 11:56:02 +0800 Subject: [PATCH 0894/1927] selftests/pidfd_test: Remove the erroneous ',' Remove the erroneous ',', otherwise it might result in wrong output and report: ... Bail out! (errno %d) test: Unexpected epoll_wait result (c=4208480, events=2) ... Fixes: 740378dc7834 ("pidfd: add polling selftests") Signed-off-by: Zhao Gongyi Signed-off-by: Shuah Khan --- tools/testing/selftests/pidfd/pidfd_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c index d36654265b7a5..e2dd4ed849846 100644 --- a/tools/testing/selftests/pidfd/pidfd_test.c +++ b/tools/testing/selftests/pidfd/pidfd_test.c @@ -413,7 +413,7 @@ static void poll_pidfd(const char *test_name, int pidfd) c = epoll_wait(epoll_fd, events, MAX_EVENTS, 5000); if (c != 1 || !(events[0].events & EPOLLIN)) - ksft_exit_fail_msg("%s test: Unexpected epoll_wait result (c=%d, events=%x) ", + ksft_exit_fail_msg("%s test: Unexpected epoll_wait result (c=%d, events=%x) " "(errno %d)\n", test_name, c, events[0].events, errno); -- GitLab From cbc1dd5b659f5a2c3cba88b197b7443679bb35a0 Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Tue, 1 Nov 2022 19:52:52 +0800 Subject: [PATCH 0895/1927] netfilter: nf_nat: Fix possible memory leak in nf_nat_init() In nf_nat_init(), register_nf_nat_bpf() can fail and return directly without any error handling. Then nf_nat_bysource will leak and registering of &nat_net_ops, &follow_master_nat and nf_nat_hook won't be reverted. This leaves wild ops in linkedlists and when another module tries to call register_pernet_operations() or nf_ct_helper_expectfn_register() it triggers page fault: BUG: unable to handle page fault for address: fffffbfff81b964c RIP: 0010:register_pernet_operations+0x1b9/0x5f0 Call Trace: register_pernet_subsys+0x29/0x40 ebtables_init+0x58/0x1000 [ebtables] ... Fixes: 820dc0523e05 ("net: netfilter: move bpf_ct_set_nat_info kfunc in nf_nat_bpf.c") Signed-off-by: Chen Zhongjin Signed-off-by: Pablo Neira Ayuso --- net/netfilter/nf_nat_core.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 18319a6e68062..e29e4ccb5c5a3 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -1152,7 +1152,16 @@ static int __init nf_nat_init(void) WARN_ON(nf_nat_hook != NULL); RCU_INIT_POINTER(nf_nat_hook, &nat_hook); - return register_nf_nat_bpf(); + ret = register_nf_nat_bpf(); + if (ret < 0) { + RCU_INIT_POINTER(nf_nat_hook, NULL); + nf_ct_helper_expectfn_unregister(&follow_master_nat); + synchronize_net(); + unregister_pernet_subsys(&nat_net_ops); + kvfree(nf_nat_bysource); + } + + return ret; } static void __exit nf_nat_cleanup(void) -- GitLab From 34de8e6e0e1f66e431abf4123934a2581cb5f133 Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Wed, 2 Nov 2022 16:40:34 +0800 Subject: [PATCH 0896/1927] bpftool: Fix NULL pointer dereference when pin {PROG, MAP, LINK} without FILE When using bpftool to pin {PROG, MAP, LINK} without FILE, segmentation fault will occur. The reson is that the lack of FILE will cause strlen to trigger NULL pointer dereference. The corresponding stacktrace is shown below: do_pin do_pin_any do_pin_fd mount_bpffs_for_pin strlen(name) <- NULL pointer dereference Fix it by adding validation to the common process. Fixes: 75a1e792c335 ("tools: bpftool: Allow all prog/map handles for pinning objects") Signed-off-by: Pu Lehui Signed-off-by: Daniel Borkmann Reviewed-by: Quentin Monnet Link: https://lore.kernel.org/bpf/20221102084034.3342995-1-pulehui@huaweicloud.com --- tools/bpf/bpftool/common.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 8727765add888..0cdb4f7115101 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -300,6 +300,9 @@ int do_pin_any(int argc, char **argv, int (*get_fd)(int *, char ***)) int err; int fd; + if (!REQ_ARGS(3)) + return -EINVAL; + fd = get_fd(&argc, &argv); if (fd < 0) return fd; -- GitLab From 4b18cb3f74dcfc183c2434e17bfce09ce6302e37 Mon Sep 17 00:00:00 2001 From: David Gow Date: Wed, 26 Oct 2022 22:10:40 +0800 Subject: [PATCH 0897/1927] perf/hw_breakpoint: test: Skip the test if dependencies unmet Running the test currently fails on non-SMP systems, despite being enabled by default. This means that running the test with: ./tools/testing/kunit/kunit.py run --arch x86_64 hw_breakpoint results in every hw_breakpoint test failing with: # test_one_cpu: failed to initialize: -22 not ok 1 - test_one_cpu Instead, use kunit_skip(), which will mark the test as skipped, and give a more comprehensible message: ok 1 - test_one_cpu # SKIP not enough cpus This makes it more obvious that the test is not suited to the test environment, and so wasn't run, rather than having run and failed. Signed-off-by: David Gow Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Daniel Latypov Acked-by: Marco Elver Link: https://lore.kernel.org/r/20221026141040.1609203-1-davidgow@google.com --- kernel/events/hw_breakpoint_test.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/events/hw_breakpoint_test.c b/kernel/events/hw_breakpoint_test.c index 5ced822df7889..c57610f52bb4d 100644 --- a/kernel/events/hw_breakpoint_test.c +++ b/kernel/events/hw_breakpoint_test.c @@ -295,11 +295,11 @@ static int test_init(struct kunit *test) { /* Most test cases want 2 distinct CPUs. */ if (num_online_cpus() < 2) - return -EINVAL; + kunit_skip(test, "not enough cpus"); /* Want the system to not use breakpoints elsewhere. */ if (hw_breakpoint_is_used()) - return -EBUSY; + kunit_skip(test, "hw breakpoint already in use"); return 0; } -- GitLab From 80275ca9e525c198c7efe045c4a6cdb68a2ea763 Mon Sep 17 00:00:00 2001 From: Zhang Rui Date: Sat, 24 Sep 2022 13:47:37 +0800 Subject: [PATCH 0898/1927] perf/x86/rapl: Use standard Energy Unit for SPR Dram RAPL domain Intel Xeon servers used to use a fixed energy resolution (15.3uj) for Dram RAPL domain. But on SPR, Dram RAPL domain follows the standard energy resolution as described in MSR_RAPL_POWER_UNIT. Remove the SPR Dram energy unit quirk. Fixes: bcfd218b6679 ("perf/x86/rapl: Add support for Intel SPR platform") Signed-off-by: Zhang Rui Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Kan Liang Tested-by: Wang Wendy Link: https://lkml.kernel.org/r/20220924054738.12076-3-rui.zhang@intel.com --- arch/x86/events/rapl.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/x86/events/rapl.c b/arch/x86/events/rapl.c index fea544e5842a1..a829492bca4c1 100644 --- a/arch/x86/events/rapl.c +++ b/arch/x86/events/rapl.c @@ -619,12 +619,8 @@ static int rapl_check_hw_unit(struct rapl_model *rm) case RAPL_UNIT_QUIRK_INTEL_HSW: rapl_hw_unit[PERF_RAPL_RAM] = 16; break; - /* - * SPR shares the same DRAM domain energy unit as HSW, plus it - * also has a fixed energy unit for Psys domain. - */ + /* SPR uses a fixed energy unit for Psys domain. */ case RAPL_UNIT_QUIRK_INTEL_SPR: - rapl_hw_unit[PERF_RAPL_RAM] = 16; rapl_hw_unit[PERF_RAPL_PSYS] = 0; break; default: -- GitLab From acc5568b90c19ac6375508a93b9676cd18a92a35 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Mon, 31 Oct 2022 08:41:18 -0700 Subject: [PATCH 0899/1927] perf/x86/intel: Fix pebs event constraints for ICL According to the latest event list, update the MEM_INST_RETIRED events which support the DataLA facility. Fixes: 6017608936c1 ("perf/x86/intel: Add Icelake support") Reported-by: Jannis Klinkenberg Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20221031154119.571386-1-kan.liang@linux.intel.com --- arch/x86/events/intel/ds.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 7839507b38448..41e8d6591777b 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -982,8 +982,13 @@ struct event_constraint intel_icl_pebs_event_constraints[] = { INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), /* SLOTS */ INTEL_PLD_CONSTRAINT(0x1cd, 0xff), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x1d0, 0xf), /* MEM_INST_RETIRED.LOAD */ - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x2d0, 0xf), /* MEM_INST_RETIRED.STORE */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), /* MEM_LOAD_*_RETIRED.* */ -- GitLab From 0916886bb978e7eae1ca3955ba07f51c020da20c Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Mon, 31 Oct 2022 08:41:19 -0700 Subject: [PATCH 0900/1927] perf/x86/intel: Fix pebs event constraints for SPR According to the latest event list, update the MEM_INST_RETIRED events which support the DataLA facility for SPR. Fixes: 61b985e3e775 ("perf/x86/intel: Add perf core PMU support for Sapphire Rapids") Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20221031154119.571386-2-kan.liang@linux.intel.com --- arch/x86/events/intel/ds.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 41e8d6591777b..446d2833efa76 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1009,8 +1009,13 @@ struct event_constraint intel_spr_pebs_event_constraints[] = { INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xfe), INTEL_PLD_CONSTRAINT(0x1cd, 0xfe), INTEL_PSD_CONSTRAINT(0x2cd, 0x1), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x1d0, 0xf), - INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x2d0, 0xf), + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x21d0, 0xf), /* MEM_INST_RETIRED.LOCK_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x41d0, 0xf), /* MEM_INST_RETIRED.SPLIT_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf), /* MEM_INST_RETIRED.SPLIT_STORES */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf), /* MEM_INST_RETIRED.ALL_LOADS */ + INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf), /* MEM_INST_RETIRED.ALL_STORES */ INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf), -- GitLab From 6f8faf471446844bb9c318e0340221049d5c19f4 Mon Sep 17 00:00:00 2001 From: Kan Liang Date: Mon, 31 Oct 2022 08:45:50 -0700 Subject: [PATCH 0901/1927] perf/x86/intel: Add Cooper Lake stepping to isolation_ucodes[] The intel_pebs_isolation quirk checks both model number and stepping. Cooper Lake has a different stepping (11) than the other Skylake Xeon. It cannot benefit from the optimization in commit 9b545c04abd4f ("perf/x86/kvm: Avoid unnecessary work in guest filtering"). Add the stepping of Cooper Lake into the isolation_ucodes[] table. Signed-off-by: Kan Liang Signed-off-by: Peter Zijlstra (Intel) Cc: stable@vger.kernel.org Link: https://lkml.kernel.org/r/20221031154550.571663-1-kan.liang@linux.intel.com --- arch/x86/events/intel/core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index a646a5f9a235c..1b92bf05fd652 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -4911,6 +4911,7 @@ static const struct x86_cpu_desc isolation_ucodes[] = { INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 5, 0x00000000), INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 6, 0x00000000), INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 7, 0x00000000), + INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 11, 0x00000000), INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_L, 3, 0x0000007c), INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE, 3, 0x0000007c), INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE, 9, 0x0000004e), -- GitLab From e97c089d7a49f67027395ddf70bf327eeac2611e Mon Sep 17 00:00:00 2001 From: Zhang Qilong Date: Sat, 29 Oct 2022 00:10:49 +0800 Subject: [PATCH 0902/1927] rose: Fix NULL pointer dereference in rose_send_frame() The syzkaller reported an issue: KASAN: null-ptr-deref in range [0x0000000000000380-0x0000000000000387] CPU: 0 PID: 4069 Comm: kworker/0:15 Not tainted 6.0.0-syzkaller-02734-g0326074ff465 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/22/2022 Workqueue: rcu_gp srcu_invoke_callbacks RIP: 0010:rose_send_frame+0x1dd/0x2f0 net/rose/rose_link.c:101 Call Trace: rose_transmit_clear_request+0x1d5/0x290 net/rose/rose_link.c:255 rose_rx_call_request+0x4c0/0x1bc0 net/rose/af_rose.c:1009 rose_loopback_timer+0x19e/0x590 net/rose/rose_loopback.c:111 call_timer_fn+0x1a0/0x6b0 kernel/time/timer.c:1474 expire_timers kernel/time/timer.c:1519 [inline] __run_timers.part.0+0x674/0xa80 kernel/time/timer.c:1790 __run_timers kernel/time/timer.c:1768 [inline] run_timer_softirq+0xb3/0x1d0 kernel/time/timer.c:1803 __do_softirq+0x1d0/0x9c8 kernel/softirq.c:571 [...] It triggers NULL pointer dereference when 'neigh->dev->dev_addr' is called in the rose_send_frame(). It's the first occurrence of the `neigh` is in rose_loopback_timer() as `rose_loopback_neigh', and the 'dev' in 'rose_loopback_neigh' is initialized sa nullptr. It had been fixed by commit 3b3fd068c56e3fbea30090859216a368398e39bf ("rose: Fix Null pointer dereference in rose_send_frame()") ever. But it's introduced by commit 3c53cd65dece47dd1f9d3a809f32e59d1d87b2b8 ("rose: check NULL rose_loopback_neigh->loopback") again. We fix it by add NULL check in rose_transmit_clear_request(). When the 'dev' in 'neigh' is NULL, we don't reply the request and just clear it. syzkaller don't provide repro, and I provide a syz repro like: r0 = syz_init_net_socket$bt_sco(0x1f, 0x5, 0x2) ioctl$sock_inet_SIOCSIFFLAGS(r0, 0x8914, &(0x7f0000000180)={'rose0\x00', 0x201}) r1 = syz_init_net_socket$rose(0xb, 0x5, 0x0) bind$rose(r1, &(0x7f00000000c0)=@full={0xb, @dev, @null, 0x0, [@null, @null, @netrom, @netrom, @default, @null]}, 0x40) connect$rose(r1, &(0x7f0000000240)=@short={0xb, @dev={0xbb, 0xbb, 0xbb, 0x1, 0x0}, @remote={0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0xcc, 0x1}, 0x1, @netrom={0xbb, 0xbb, 0xbb, 0xbb, 0xbb, 0x0, 0x0}}, 0x1c) Fixes: 3c53cd65dece ("rose: check NULL rose_loopback_neigh->loopback") Signed-off-by: Zhang Qilong Signed-off-by: David S. Miller --- net/rose/rose_link.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/rose/rose_link.c b/net/rose/rose_link.c index 8b96a56d3a49b..0f77ae8ef944a 100644 --- a/net/rose/rose_link.c +++ b/net/rose/rose_link.c @@ -236,6 +236,9 @@ void rose_transmit_clear_request(struct rose_neigh *neigh, unsigned int lci, uns unsigned char *dptr; int len; + if (!neigh->dev) + return; + len = AX25_BPQ_HEADER_LEN + AX25_MAX_HEADER_LEN + ROSE_MIN_LEN + 3; if ((skb = alloc_skb(len, GFP_ATOMIC)) == NULL) -- GitLab From 134af9aa88453aeb9224e407092530ebba366c6c Mon Sep 17 00:00:00 2001 From: Neil Armstrong Date: Wed, 2 Nov 2022 09:46:01 +0100 Subject: [PATCH 0903/1927] spi: meson-spicc: fix do_div build error on non-arm64 This fixes : error: passing argument 1 of '__div64_32' from incompatible pointer type By passing an uint64_t as first variable to do_div(). Reported-by: kernel test robot Fixes: 04694e50020b ("spi: meson-spicc: move wait completion in driver to take bursts delay in account") Acked-by: Martin Blumenstingl Signed-off-by: Neil Armstrong Link: https://lore.kernel.org/r/20221027-b4-spicc-burst-delay-fix-v2-0-8cc2bab3417a@linaro.org Signed-off-by: Mark Brown --- drivers/spi/spi-meson-spicc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c index 52bffab183298..1b4195c54ee26 100644 --- a/drivers/spi/spi-meson-spicc.c +++ b/drivers/spi/spi-meson-spicc.c @@ -387,7 +387,7 @@ static int meson_spicc_transfer_one(struct spi_master *master, struct spi_transfer *xfer) { struct meson_spicc_device *spicc = spi_master_get_devdata(master); - unsigned long timeout; + uint64_t timeout; /* Store current transfer */ spicc->xfer = xfer; -- GitLab From e7d1d4d9ac0dfa40be4c2c8abd0731659869b297 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 31 Oct 2022 20:13:40 +0800 Subject: [PATCH 0904/1927] mISDN: fix possible memory leak in mISDN_register_device() Afer commit 1fa5ae857bb1 ("driver core: get rid of struct device's bus_id string array"), the name of device is allocated dynamically, add put_device() to give up the reference, so that the name can be freed in kobject_cleanup() when the refcount is 0. Set device class before put_device() to avoid null release() function WARN message in device_release(). Fixes: 1fa5ae857bb1 ("driver core: get rid of struct device's bus_id string array") Signed-off-by: Yang Yingliang Signed-off-by: David S. Miller --- drivers/isdn/mISDN/core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/isdn/mISDN/core.c b/drivers/isdn/mISDN/core.c index a41b4b2645941..7ea0100f218a0 100644 --- a/drivers/isdn/mISDN/core.c +++ b/drivers/isdn/mISDN/core.c @@ -233,11 +233,12 @@ mISDN_register_device(struct mISDNdevice *dev, if (debug & DEBUG_CORE) printk(KERN_DEBUG "mISDN_register %s %d\n", dev_name(&dev->dev), dev->id); + dev->dev.class = &mISDN_class; + err = create_stack(dev); if (err) goto error1; - dev->dev.class = &mISDN_class; dev->dev.platform_data = dev; dev->dev.parent = parent; dev_set_drvdata(&dev->dev, dev); @@ -249,8 +250,8 @@ mISDN_register_device(struct mISDNdevice *dev, error3: delete_stack(dev); - return err; error1: + put_device(&dev->dev); return err; } -- GitLab From bf00f5426074249058a106a6edbb89e4b25a4d79 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 31 Oct 2022 20:13:41 +0800 Subject: [PATCH 0905/1927] isdn: mISDN: netjet: fix wrong check of device registration The class is set in mISDN_register_device(), but if device_add() returns error, it will lead to delete a device without added, fix this by using device_is_registered() to check if the device is registered. Fixes: a900845e5661 ("mISDN: Add support for Traverse Technologies NETJet PCI cards") Signed-off-by: Yang Yingliang Signed-off-by: David S. Miller --- drivers/isdn/hardware/mISDN/netjet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/isdn/hardware/mISDN/netjet.c b/drivers/isdn/hardware/mISDN/netjet.c index a52f275f82634..f8447135a9022 100644 --- a/drivers/isdn/hardware/mISDN/netjet.c +++ b/drivers/isdn/hardware/mISDN/netjet.c @@ -956,7 +956,7 @@ nj_release(struct tiger_hw *card) } if (card->irq > 0) free_irq(card->irq, card); - if (card->isac.dch.dev.dev.class) + if (device_is_registered(&card->isac.dch.dev.dev)) mISDN_unregister_device(&card->isac.dch.dev); for (i = 0; i < 2; i++) { -- GitLab From 7f9dbf54c3dc7170df86f7753abb41b5d7e8779e Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 24 Oct 2022 17:11:02 +0100 Subject: [PATCH 0906/1927] HID: asus: Remove unused variable in asus_report_tool_width() Variable count is just being incremented and it's never used anywhere else. The variable and the increment are redundant so remove it. Signed-off-by: Colin Ian King Signed-off-by: Jiri Kosina --- drivers/hid/hid-asus.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index b59c3dafa6a48..f99752b998f3d 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -219,14 +219,13 @@ static void asus_report_tool_width(struct asus_drvdata *drvdat) { struct input_mt *mt = drvdat->input->mt; struct input_mt_slot *oldest; - int oldid, count, i; + int oldid, i; if (drvdat->tp->contact_size < 5) return; oldest = NULL; oldid = mt->trkid; - count = 0; for (i = 0; i < mt->num_slots; ++i) { struct input_mt_slot *ps = &mt->slots[i]; @@ -238,7 +237,6 @@ static void asus_report_tool_width(struct asus_drvdata *drvdat) oldest = ps; oldid = id; } - count++; } if (oldest) { -- GitLab From b5bcb94b0954a026bbd671741fdb00e7141f9c91 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Fri, 28 Oct 2022 21:40:43 +0800 Subject: [PATCH 0907/1927] HID: hyperv: fix possible memory leak in mousevsc_probe() If hid_add_device() returns error, it should call hid_destroy_device() to free hid_dev which is allocated in hid_allocate_device(). Fixes: 74c4fb058083 ("HID: hv_mouse: Properly add the hid device") Signed-off-by: Yang Yingliang Reviewed-by: Wei Liu Signed-off-by: Jiri Kosina --- drivers/hid/hid-hyperv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/hid/hid-hyperv.c b/drivers/hid/hid-hyperv.c index e0bc731241960..ab57b49a44ed9 100644 --- a/drivers/hid/hid-hyperv.c +++ b/drivers/hid/hid-hyperv.c @@ -499,7 +499,7 @@ static int mousevsc_probe(struct hv_device *device, ret = hid_add_device(hid_dev); if (ret) - goto probe_err1; + goto probe_err2; ret = hid_parse(hid_dev); -- GitLab From 9921d5013a6e51892623bf2f1c5b49eaecda55ac Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 27 Oct 2022 00:11:53 +0100 Subject: [PATCH 0908/1927] selftests/net: don't tests batched TCP io_uring zc It doesn't make sense batch submitting io_uring requests to a single TCP socket without linking or some other kind of ordering. Moreover, it causes spurious -EINTR fails due to interaction with task_work. Disable it for now and keep queue depth=1. Signed-off-by: Pavel Begunkov Link: https://lore.kernel.org/r/b547698d5938b1b1a898af1c260188d8546ded9a.1666700897.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- tools/testing/selftests/net/io_uring_zerocopy_tx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/net/io_uring_zerocopy_tx.sh b/tools/testing/selftests/net/io_uring_zerocopy_tx.sh index 32aa6e9dacc26..9ac4456d48fcc 100755 --- a/tools/testing/selftests/net/io_uring_zerocopy_tx.sh +++ b/tools/testing/selftests/net/io_uring_zerocopy_tx.sh @@ -29,7 +29,7 @@ if [[ "$#" -eq "0" ]]; then for IP in "${IPs[@]}"; do for mode in $(seq 1 3); do $0 "$IP" udp -m "$mode" -t 1 -n 32 - $0 "$IP" tcp -m "$mode" -t 1 -n 32 + $0 "$IP" tcp -m "$mode" -t 1 -n 1 done done -- GitLab From 5614dc3a47e3310fbc77ea3b67eaadd1c6417bf1 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 1 Nov 2022 16:15:37 +0000 Subject: [PATCH 0909/1927] btrfs: fix inode list leak during backref walking at resolve_indirect_refs() During backref walking, at resolve_indirect_refs(), if we get an error we jump to the 'out' label and call ulist_free() on the 'parents' ulist, which frees all the elements in the ulist - however that does not free any inode lists that may be attached to elements, through the 'aux' field of a ulist node, so we end up leaking lists if we have any attached to the unodes. Fix this by calling free_leaf_list() instead of ulist_free() when we exit from resolve_indirect_refs(). The static function free_leaf_list() is moved up for this to be possible and it's slightly simplified by removing unnecessary code. Fixes: 3301958b7c1d ("Btrfs: add inodes before dropping the extent lock in find_all_leafs") Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/backref.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 4ec18ceb2f21d..40afae0af4e67 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -648,6 +648,18 @@ unode_aux_to_inode_list(struct ulist_node *node) return (struct extent_inode_elem *)(uintptr_t)node->aux; } +static void free_leaf_list(struct ulist *ulist) +{ + struct ulist_node *node; + struct ulist_iterator uiter; + + ULIST_ITER_INIT(&uiter); + while ((node = ulist_next(ulist, &uiter))) + free_inode_elem_list(unode_aux_to_inode_list(node)); + + ulist_free(ulist); +} + /* * We maintain three separate rbtrees: one for direct refs, one for * indirect refs which have a key, and one for indirect refs which do not @@ -762,7 +774,11 @@ static int resolve_indirect_refs(struct btrfs_fs_info *fs_info, cond_resched(); } out: - ulist_free(parents); + /* + * We may have inode lists attached to refs in the parents ulist, so we + * must free them before freeing the ulist and its refs. + */ + free_leaf_list(parents); return ret; } @@ -1409,24 +1425,6 @@ static int find_parent_nodes(struct btrfs_trans_handle *trans, return ret; } -static void free_leaf_list(struct ulist *blocks) -{ - struct ulist_node *node = NULL; - struct extent_inode_elem *eie; - struct ulist_iterator uiter; - - ULIST_ITER_INIT(&uiter); - while ((node = ulist_next(blocks, &uiter))) { - if (!node->aux) - continue; - eie = unode_aux_to_inode_list(node); - free_inode_elem_list(eie); - node->aux = 0; - } - - ulist_free(blocks); -} - /* * Finds all leafs with a reference to the specified combination of bytenr and * offset. key_list_head will point to a list of corresponding keys (caller must -- GitLab From 92876eec382a0f19f33d09d2c939e9ca49038ae5 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 1 Nov 2022 16:15:38 +0000 Subject: [PATCH 0910/1927] btrfs: fix inode list leak during backref walking at find_parent_nodes() During backref walking, at find_parent_nodes(), if we are dealing with a data extent and we get an error while resolving the indirect backrefs, at resolve_indirect_refs(), or in the while loop that iterates over the refs in the direct refs rbtree, we end up leaking the inode lists attached to the direct refs we have in the direct refs rbtree that were not yet added to the refs ulist passed as argument to find_parent_nodes(). Since they were not yet added to the refs ulist and prelim_release() does not free the lists, on error the caller can only free the lists attached to the refs that were added to the refs ulist, all the remaining refs get their inode lists never freed, therefore leaking their memory. Fix this by having prelim_release() always free any attached inode list to each ref found in the rbtree, and have find_parent_nodes() set the ref's inode list to NULL once it transfers ownership of the inode list to a ref added to the refs ulist passed to find_parent_nodes(). Fixes: 86d5f9944252 ("btrfs: convert prelimary reference tracking to use rbtrees") Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/backref.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 40afae0af4e67..18374a6d05bdf 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -289,8 +289,10 @@ static void prelim_release(struct preftree *preftree) struct prelim_ref *ref, *next_ref; rbtree_postorder_for_each_entry_safe(ref, next_ref, - &preftree->root.rb_root, rbnode) + &preftree->root.rb_root, rbnode) { + free_inode_elem_list(ref->inode_list); free_pref(ref); + } preftree->root = RB_ROOT_CACHED; preftree->count = 0; @@ -1384,6 +1386,12 @@ static int find_parent_nodes(struct btrfs_trans_handle *trans, if (ret < 0) goto out; ref->inode_list = eie; + /* + * We transferred the list ownership to the ref, + * so set to NULL to avoid a double free in case + * an error happens after this. + */ + eie = NULL; } ret = ulist_add_merge_ptr(refs, ref->parent, ref->inode_list, @@ -1409,6 +1417,14 @@ static int find_parent_nodes(struct btrfs_trans_handle *trans, eie->next = ref->inode_list; } eie = NULL; + /* + * We have transferred the inode list ownership from + * this ref to the ref we added to the 'refs' ulist. + * So set this ref's inode list to NULL to avoid + * use-after-free when our caller uses it or double + * frees in case an error happens before we return. + */ + ref->inode_list = NULL; } cond_resched(); } -- GitLab From d37de92b38932d40e4a251e876cc388f9aee5f42 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 1 Nov 2022 16:15:39 +0000 Subject: [PATCH 0911/1927] btrfs: fix ulist leaks in error paths of qgroup self tests In the test_no_shared_qgroup() and test_multiple_refs() qgroup self tests, if we fail to add the tree ref, remove the extent item or remove the extent ref, we are returning from the test function without freeing the "old_roots" ulist that was allocated by the previous calls to btrfs_find_all_roots(). Fix that by calling ulist_free() before returning. Fixes: 442244c96332 ("btrfs: qgroup: Switch self test to extent-oriented qgroup mechanism.") Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tests/qgroup-tests.c | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/fs/btrfs/tests/qgroup-tests.c b/fs/btrfs/tests/qgroup-tests.c index eee1e44595410..843dd3d3adbe7 100644 --- a/fs/btrfs/tests/qgroup-tests.c +++ b/fs/btrfs/tests/qgroup-tests.c @@ -232,8 +232,10 @@ static int test_no_shared_qgroup(struct btrfs_root *root, ret = insert_normal_tree_ref(root, nodesize, nodesize, 0, BTRFS_FS_TREE_OBJECTID); - if (ret) + if (ret) { + ulist_free(old_roots); return ret; + } ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { @@ -266,8 +268,10 @@ static int test_no_shared_qgroup(struct btrfs_root *root, } ret = remove_extent_item(root, nodesize, nodesize); - if (ret) + if (ret) { + ulist_free(old_roots); return -EINVAL; + } ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { @@ -329,8 +333,10 @@ static int test_multiple_refs(struct btrfs_root *root, ret = insert_normal_tree_ref(root, nodesize, nodesize, 0, BTRFS_FS_TREE_OBJECTID); - if (ret) + if (ret) { + ulist_free(old_roots); return ret; + } ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { @@ -362,8 +368,10 @@ static int test_multiple_refs(struct btrfs_root *root, ret = add_tree_ref(root, nodesize, nodesize, 0, BTRFS_FIRST_FREE_OBJECTID); - if (ret) + if (ret) { + ulist_free(old_roots); return ret; + } ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { @@ -401,8 +409,10 @@ static int test_multiple_refs(struct btrfs_root *root, ret = remove_extent_ref(root, nodesize, nodesize, 0, BTRFS_FIRST_FREE_OBJECTID); - if (ret) + if (ret) { + ulist_free(old_roots); return ret; + } ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { -- GitLab From d0ea17aec12ea0f7b9d2ed727d8ef8169d1e7699 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Tue, 1 Nov 2022 16:15:40 +0000 Subject: [PATCH 0912/1927] btrfs: remove pointless and double ulist frees in error paths of qgroup tests Several places in the qgroup self tests follow the pattern of freeing the ulist pointer they passed to btrfs_find_all_roots() if the call to that function returned an error. That is pointless because that function always frees the ulist in case it returns an error. Also In some places like at test_multiple_refs(), after a call to btrfs_qgroup_account_extent() we also leave "old_roots" and "new_roots" pointing to ulists that were freed, because btrfs_qgroup_account_extent() has freed those ulists, and if after that the next call to btrfs_find_all_roots() fails, we call ulist_free() on the "old_roots" ulist again, resulting in a double free. So remove those calls to reduce the code size and avoid double ulist free in case of an error. Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/tests/qgroup-tests.c | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/fs/btrfs/tests/qgroup-tests.c b/fs/btrfs/tests/qgroup-tests.c index 843dd3d3adbe7..63676ea19f29e 100644 --- a/fs/btrfs/tests/qgroup-tests.c +++ b/fs/btrfs/tests/qgroup-tests.c @@ -225,7 +225,6 @@ static int test_no_shared_qgroup(struct btrfs_root *root, */ ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &old_roots, false); if (ret) { - ulist_free(old_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -240,7 +239,6 @@ static int test_no_shared_qgroup(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { ulist_free(old_roots); - ulist_free(new_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -252,17 +250,18 @@ static int test_no_shared_qgroup(struct btrfs_root *root, return ret; } + /* btrfs_qgroup_account_extent() always frees the ulists passed to it. */ + old_roots = NULL; + new_roots = NULL; + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FS_TREE_OBJECTID, nodesize, nodesize)) { test_err("qgroup counts didn't match expected values"); return -EINVAL; } - old_roots = NULL; - new_roots = NULL; ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &old_roots, false); if (ret) { - ulist_free(old_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -276,7 +275,6 @@ static int test_no_shared_qgroup(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { ulist_free(old_roots); - ulist_free(new_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -326,7 +324,6 @@ static int test_multiple_refs(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &old_roots, false); if (ret) { - ulist_free(old_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -341,7 +338,6 @@ static int test_multiple_refs(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { ulist_free(old_roots); - ulist_free(new_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -361,7 +357,6 @@ static int test_multiple_refs(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &old_roots, false); if (ret) { - ulist_free(old_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -376,7 +371,6 @@ static int test_multiple_refs(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { ulist_free(old_roots); - ulist_free(new_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -402,7 +396,6 @@ static int test_multiple_refs(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &old_roots, false); if (ret) { - ulist_free(old_roots); test_err("couldn't find old roots: %d", ret); return ret; } @@ -417,7 +410,6 @@ static int test_multiple_refs(struct btrfs_root *root, ret = btrfs_find_all_roots(&trans, fs_info, nodesize, 0, &new_roots, false); if (ret) { ulist_free(old_roots); - ulist_free(new_roots); test_err("couldn't find old roots: %d", ret); return ret; } -- GitLab From a348c8d4f6cf23ef04b0edaccdfe9d94c2d335db Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 2 Nov 2022 12:46:35 +0000 Subject: [PATCH 0913/1927] btrfs: fix nowait buffered write returning -ENOSPC If we are doing a buffered write in NOWAIT context and we can't reserve metadata space due to -ENOSPC, then we should return -EAGAIN so that we retry the write in a context allowed to block and do metadata reservation with flushing, which might succeed this time due to the allowed flushing. Returning -ENOSPC while in NOWAIT context simply makes some writes fail with -ENOSPC when they would likely succeed after switching from NOWAIT context to blocking context. That is unexpected behaviour and even fio complains about it with a warning like this: fio: io_u error on file /mnt/sdi/task_0.0.0: No space left on device: write offset=1535705088, buflen=65536 fio: pid=592630, err=28/file:io_u.c:1846, func=io_u error, error=No space left on device The fio's job config is this: [global] bs=64K ioengine=io_uring iodepth=1 size=2236962133 nr_files=1 filesize=2236962133 direct=0 runtime=10 fallocate=posix io_size=2236962133 group_reporting time_based [task_0] rw=randwrite directory=/mnt/sdi numjobs=4 So fix this by returning -EAGAIN if we are in NOWAIT context and the metadata reservation failed with -ENOSPC. Fixes: 304e45acdb8f ("btrfs: plumb NOWAIT through the write path") Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index d55ad46384d17..5ce2ae9d4f723 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1598,6 +1598,9 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, write_bytes); else btrfs_check_nocow_unlock(BTRFS_I(inode)); + + if (nowait && ret == -ENOSPC) + ret = -EAGAIN; break; } -- GitLab From eb81b682b131642405a05c627ab08cf0967b3dd8 Mon Sep 17 00:00:00 2001 From: Filipe Manana Date: Wed, 2 Nov 2022 12:46:36 +0000 Subject: [PATCH 0914/1927] btrfs: fix inode reserve space leak due to nowait buffered write During a nowait buffered write, if we fail to balance dirty pages we exit btrfs_buffered_write() without releasing the delalloc space reserved for an extent, resulting in leaking space from the inode's block reserve. So fix that by releasing the delalloc space for the extent when balancing dirty pages fails. Reported-by: kernel test robot Link: https://lore.kernel.org/all/202210111304.d369bc32-yujie.liu@intel.com Fixes: 965f47aeb5de ("btrfs: make btrfs_buffered_write nowait compatible") Reviewed-by: Josef Bacik Signed-off-by: Filipe Manana Signed-off-by: David Sterba --- fs/btrfs/file.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 5ce2ae9d4f723..d01631d478067 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -1607,8 +1607,10 @@ static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb, release_bytes = reserve_bytes; again: ret = balance_dirty_pages_ratelimited_flags(inode->i_mapping, bdp_flags); - if (ret) + if (ret) { + btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes); break; + } /* * This is going to setup the pages array with the number of -- GitLab From 3a89b6dec9920026eaa90fe8457f4348d3388a98 Mon Sep 17 00:00:00 2001 From: "Tyler J. Stachecki" Date: Wed, 2 Nov 2022 18:56:39 +0200 Subject: [PATCH 0915/1927] wifi: ath11k: Fix QCN9074 firmware boot on x86 The 2.7.0 series of QCN9074's firmware requests 5 segments of memory instead of 3 (as in the 2.5.0 series). The first segment (11M) is too large to be kalloc'd in one go on x86 and requires piecemeal 1MB allocations, as was the case with the prior public firmware (2.5.0, 15M). Since f6f92968e1e5, ath11k will break the memory requests, but only if there were fewer than 3 segments requested by the firmware. It seems that 5 segments works fine and allows QCN9074 to boot on x86 with firmware 2.7.0, so change things accordingly. Tested-on: QCN9074 hw1.0 PCI WLAN.HK.2.7.0.1-01744-QCAHKSWPL_SILICONZ-1 Tested-on: QCN9074 hw1.0 PCI WLAN.HK.2.5.0.1-01208-QCAHKSWPL_SILICONZ-1 Tested-on: WCN6855 hw2.0 PCI WLAN.HSP.1.1-03125-QCAHSPSWPL_V1_V2_SILICONZ_LITE-3.6510.16 Signed-off-by: Tyler J. Stachecki Signed-off-by: Kalle Valo Link: https://lore.kernel.org/r/20221022042728.43015-1-stachecki.tyler@gmail.com --- drivers/net/wireless/ath/ath11k/qmi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wireless/ath/ath11k/qmi.h b/drivers/net/wireless/ath/ath11k/qmi.h index 2ec56a34fa810..0909d53cefebc 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.h +++ b/drivers/net/wireless/ath/ath11k/qmi.h @@ -27,7 +27,7 @@ #define ATH11K_QMI_WLANFW_MAX_NUM_MEM_SEG_V01 52 #define ATH11K_QMI_CALDB_SIZE 0x480000 #define ATH11K_QMI_BDF_EXT_STR_LENGTH 0x20 -#define ATH11K_QMI_FW_MEM_REQ_SEGMENT_CNT 3 +#define ATH11K_QMI_FW_MEM_REQ_SEGMENT_CNT 5 #define QMI_WLFW_REQUEST_MEM_IND_V01 0x0035 #define QMI_WLFW_FW_MEM_READY_IND_V01 0x0037 -- GitLab From f45cb6b29cd36514e13f7519770873d8c0457008 Mon Sep 17 00:00:00 2001 From: Wen Gong Date: Wed, 2 Nov 2022 13:48:03 +0200 Subject: [PATCH 0916/1927] wifi: ath11k: avoid deadlock during regulatory update in ath11k_regd_update() (cherry picked from commit d99884ad9e3673a12879bc2830f6e5a66cccbd78 in ath-next as users are seeing this bug more now, also cc stable) Running this test in a loop it is easy to reproduce an rtnl deadlock: iw reg set FI ifconfig wlan0 down What happens is that thread A (workqueue) tries to update the regulatory: try to acquire the rtnl_lock of ar->regd_update_work rtnl_lock+0x17/0x20 ath11k_regd_update+0x15a/0x260 [ath11k] ath11k_regd_update_work+0x15/0x20 [ath11k] process_one_work+0x228/0x670 worker_thread+0x4d/0x440 kthread+0x16d/0x1b0 ret_from_fork+0x22/0x30 And thread B (ifconfig) tries to stop the interface: try to cancel_work_sync(&ar->regd_update_work) in ath11k_mac_op_stop(). ifconfig 3109 [003] 2414.232506: probe: ath11k_mac_op_stop: (ffffffffc14187a0) drv_stop+0x30 ([mac80211]) ieee80211_do_stop+0x5d2 ([mac80211]) ieee80211_stop+0x3e ([mac80211]) __dev_close_many+0x9e ([kernel.kallsyms]) __dev_change_flags+0xbe ([kernel.kallsyms]) dev_change_flags+0x23 ([kernel.kallsyms]) devinet_ioctl+0x5e3 ([kernel.kallsyms]) inet_ioctl+0x197 ([kernel.kallsyms]) sock_do_ioctl+0x4d ([kernel.kallsyms]) sock_ioctl+0x264 ([kernel.kallsyms]) __x64_sys_ioctl+0x92 ([kernel.kallsyms]) do_syscall_64+0x3a ([kernel.kallsyms]) entry_SYSCALL_64_after_hwframe+0x63 ([kernel.kallsyms]) __GI___ioctl+0x7 (/lib/x86_64-linux-gnu/libc-2.23.so) The sequence of deadlock is: 1. Thread B calls rtnl_lock(). 2. Thread A starts to run and calls rtnl_lock() from within ath11k_regd_update_work(), then enters wait state because the lock is owned by thread B. 3. Thread B continues to run and tries to call cancel_work_sync(&ar->regd_update_work), but thread A is in ath11k_regd_update_work() waiting for rtnl_lock(). So cancel_work_sync() forever waits for ath11k_regd_update_work() to finish and we have a deadlock. Fix this by switching from using regulatory_set_wiphy_regd_sync() to regulatory_set_wiphy_regd(). Now cfg80211 will schedule another workqueue which handles the locking on it's own. So the ath11k workqueue can simply exit without taking any locks, avoiding the deadlock. Tested-on: WCN6855 hw2.0 PCI WLAN.HSP.1.1-03125-QCAHSPSWPL_V1_V2_SILICONZ_LITE-3 Cc: Signed-off-by: Wen Gong [kvalo: improve commit log] Signed-off-by: Kalle Valo --- drivers/net/wireless/ath/ath11k/reg.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c index 7ee3ff69dfc85..6fae4e61ede7f 100644 --- a/drivers/net/wireless/ath/ath11k/reg.c +++ b/drivers/net/wireless/ath/ath11k/reg.c @@ -287,11 +287,7 @@ int ath11k_regd_update(struct ath11k *ar) goto err; } - rtnl_lock(); - wiphy_lock(ar->hw->wiphy); - ret = regulatory_set_wiphy_regd_sync(ar->hw->wiphy, regd_copy); - wiphy_unlock(ar->hw->wiphy); - rtnl_unlock(); + ret = regulatory_set_wiphy_regd(ar->hw->wiphy, regd_copy); kfree(regd_copy); -- GitLab From 145dfad998eac74abc59219d936e905766ba2d98 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Oct 2022 00:03:08 +0000 Subject: [PATCH 0917/1927] KVM: VMX: Advertise PMU LBRs if and only if perf supports LBRs Advertise LBR support to userspace via MSR_IA32_PERF_CAPABILITIES if and only if perf fully supports LBRs. Perf may disable LBRs (by zeroing the number of LBRs) even on platforms the allegedly support LBRs, e.g. if probing any LBR MSRs during setup fails. Fixes: be635e34c284 ("KVM: vmx/pmu: Expose LBR_FMT in the MSR_IA32_PERF_CAPABILITIES") Reported-by: Like Xu Signed-off-by: Sean Christopherson Message-Id: <20221006000314.73240-3-seanjc@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/capabilities.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 87c4e46daf372..3bd7a89706187 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -400,6 +400,7 @@ static inline bool vmx_pebs_supported(void) static inline u64 vmx_get_perf_capabilities(void) { u64 perf_cap = PMU_CAP_FW_WRITES; + struct x86_pmu_lbr lbr; u64 host_perf_cap = 0; if (!enable_pmu) @@ -408,7 +409,8 @@ static inline u64 vmx_get_perf_capabilities(void) if (boot_cpu_has(X86_FEATURE_PDCM)) rdmsrl(MSR_IA32_PERF_CAPABILITIES, host_perf_cap); - perf_cap |= host_perf_cap & PMU_CAP_LBR_FMT; + if (x86_perf_get_lbr(&lbr) >= 0 && lbr.nr) + perf_cap |= host_perf_cap & PMU_CAP_LBR_FMT; if (vmx_pebs_supported()) { perf_cap |= host_perf_cap & PERF_CAP_PEBS_MASK; -- GitLab From 18e897d213cb152c786abab14919196bd9dc3a9f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Oct 2022 00:03:09 +0000 Subject: [PATCH 0918/1927] KVM: VMX: Fold vmx_supported_debugctl() into vcpu_supported_debugctl() Fold vmx_supported_debugctl() into vcpu_supported_debugctl(), its only caller. Setting bits only to clear them a few instructions later is rather silly, and splitting the logic makes things seem more complicated than they actually are. Opportunistically drop DEBUGCTLMSR_LBR_MASK now that there's a single reference to the pair of bits. The extra layer of indirection provides no meaningful value and makes it unnecessarily tedious to understand what KVM is doing. No functional change. Signed-off-by: Sean Christopherson Message-Id: <20221006000314.73240-4-seanjc@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/capabilities.h | 15 --------------- arch/x86/kvm/vmx/vmx.c | 12 +++++++----- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index 3bd7a89706187..07254314f3dd5 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -24,8 +24,6 @@ extern int __read_mostly pt_mode; #define PMU_CAP_FW_WRITES (1ULL << 13) #define PMU_CAP_LBR_FMT 0x3f -#define DEBUGCTLMSR_LBR_MASK (DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI) - struct nested_vmx_msrs { /* * We only store the "true" versions of the VMX capability MSRs. We @@ -421,19 +419,6 @@ static inline u64 vmx_get_perf_capabilities(void) return perf_cap; } -static inline u64 vmx_supported_debugctl(void) -{ - u64 debugctl = 0; - - if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) - debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; - - if (vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT) - debugctl |= DEBUGCTLMSR_LBR_MASK; - - return debugctl; -} - static inline bool cpu_has_notify_vmexit(void) { return vmcs_config.cpu_based_2nd_exec_ctrl & diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 65f092e4a81b3..981b38355066a 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2023,13 +2023,15 @@ static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu) { - u64 debugctl = vmx_supported_debugctl(); + u64 debugctl = 0; - if (!intel_pmu_lbr_is_enabled(vcpu)) - debugctl &= ~DEBUGCTLMSR_LBR_MASK; + if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && + guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; - if (!guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) - debugctl &= ~DEBUGCTLMSR_BUS_LOCK_DETECT; + if ((vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT) && + intel_pmu_lbr_is_enabled(vcpu)) + debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; return debugctl; } -- GitLab From b333b8ebb85d62469f32b52fa03fd7d1522afc03 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 6 Oct 2022 00:03:10 +0000 Subject: [PATCH 0919/1927] KVM: VMX: Ignore guest CPUID for host userspace writes to DEBUGCTL Ignore guest CPUID for host userspace writes to the DEBUGCTL MSR, KVM's ABI is that setting CPUID vs. state can be done in any order, i.e. KVM allows userspace to stuff MSRs prior to setting the guest's CPUID that makes the new MSR "legal". Keep the vmx_get_perf_capabilities() check for guest writes, even though it's technically unnecessary since the vCPU's PERF_CAPABILITIES is consulted when refreshing LBR support. A future patch will clean up vmx_get_perf_capabilities() to avoid the RDMSR on every call, at which point the paranoia will incur no meaningful overhead. Note, prior to vmx_get_perf_capabilities() checking that the host fully supports LBRs via x86_perf_get_lbr(), KVM effectively relied on intel_pmu_lbr_is_enabled() to guard against host userspace enabling LBRs on platforms without full support. Fixes: c646236344e9 ("KVM: vmx/pmu: Add PMU_CAP_LBR_FMT check when guest LBR is enabled") Signed-off-by: Sean Christopherson Message-Id: <20221006000314.73240-5-seanjc@google.com> Cc: stable@vger.kernel.org Signed-off-by: Paolo Bonzini --- arch/x86/kvm/vmx/vmx.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 981b38355066a..63247c57c72cc 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2021,16 +2021,16 @@ static u64 nested_vmx_truncate_sysenter_addr(struct kvm_vcpu *vcpu, return (unsigned long)data; } -static u64 vcpu_supported_debugctl(struct kvm_vcpu *vcpu) +static u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated) { u64 debugctl = 0; if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT) && - guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + (host_initiated || guest_cpuid_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT))) debugctl |= DEBUGCTLMSR_BUS_LOCK_DETECT; if ((vmx_get_perf_capabilities() & PMU_CAP_LBR_FMT) && - intel_pmu_lbr_is_enabled(vcpu)) + (host_initiated || intel_pmu_lbr_is_enabled(vcpu))) debugctl |= DEBUGCTLMSR_LBR | DEBUGCTLMSR_FREEZE_LBRS_ON_PMI; return debugctl; @@ -2105,7 +2105,9 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmcs_writel(GUEST_SYSENTER_ESP, data); break; case MSR_IA32_DEBUGCTLMSR: { - u64 invalid = data & ~vcpu_supported_debugctl(vcpu); + u64 invalid; + + invalid = data & ~vmx_get_supported_debugctl(vcpu, msr_info->host_initiated); if (invalid & (DEBUGCTLMSR_BTF|DEBUGCTLMSR_LBR)) { if (report_ignored_msrs) vcpu_unimpl(vcpu, "%s: BTF|LBR in IA32_DEBUGCTLMSR 0x%llx, nop\n", -- GitLab From 510841da1fcc16f702440ab58ef0b4d82a9056b7 Mon Sep 17 00:00:00 2001 From: Jozsef Kadlecsik Date: Wed, 2 Nov 2022 10:40:47 +0100 Subject: [PATCH 0920/1927] netfilter: ipset: enforce documented limit to prevent allocating huge memory Daniel Xu reported that the hash:net,iface type of the ipset subsystem does not limit adding the same network with different interfaces to a set, which can lead to huge memory usage or allocation failure. The quick reproducer is $ ipset create ACL.IN.ALL_PERMIT hash:net,iface hashsize 1048576 timeout 0 $ for i in $(seq 0 100); do /sbin/ipset add ACL.IN.ALL_PERMIT 0.0.0.0/0,kaf_$i timeout 0 -exist; done The backtrace when vmalloc fails: [Tue Oct 25 00:13:08 2022] ipset: vmalloc error: size 1073741848, exceeds total pages <...> [Tue Oct 25 00:13:08 2022] Call Trace: [Tue Oct 25 00:13:08 2022] [Tue Oct 25 00:13:08 2022] dump_stack_lvl+0x48/0x60 [Tue Oct 25 00:13:08 2022] warn_alloc+0x155/0x180 [Tue Oct 25 00:13:08 2022] __vmalloc_node_range+0x72a/0x760 [Tue Oct 25 00:13:08 2022] ? hash_netiface4_add+0x7c0/0xb20 [Tue Oct 25 00:13:08 2022] ? __kmalloc_large_node+0x4a/0x90 [Tue Oct 25 00:13:08 2022] kvmalloc_node+0xa6/0xd0 [Tue Oct 25 00:13:08 2022] ? hash_netiface4_resize+0x99/0x710 <...> The fix is to enforce the limit documented in the ipset(8) manpage: > The internal restriction of the hash:net,iface set type is that the same > network prefix cannot be stored with more than 64 different interfaces > in a single set. Fixes: ccf0a4b7fc68 ("netfilter: ipset: Add bucketsize parameter to all hash types") Reported-by: Daniel Xu Signed-off-by: Jozsef Kadlecsik Signed-off-by: Pablo Neira Ayuso --- net/netfilter/ipset/ip_set_hash_gen.h | 30 ++++++--------------------- 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 6e391308431da..3adc291d9ce18 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -42,31 +42,8 @@ #define AHASH_MAX_SIZE (6 * AHASH_INIT_SIZE) /* Max muber of elements in the array block when tuned */ #define AHASH_MAX_TUNED 64 - #define AHASH_MAX(h) ((h)->bucketsize) -/* Max number of elements can be tuned */ -#ifdef IP_SET_HASH_WITH_MULTI -static u8 -tune_bucketsize(u8 curr, u32 multi) -{ - u32 n; - - if (multi < curr) - return curr; - - n = curr + AHASH_INIT_SIZE; - /* Currently, at listing one hash bucket must fit into a message. - * Therefore we have a hard limit here. - */ - return n > curr && n <= AHASH_MAX_TUNED ? n : curr; -} -#define TUNE_BUCKETSIZE(h, multi) \ - ((h)->bucketsize = tune_bucketsize((h)->bucketsize, multi)) -#else -#define TUNE_BUCKETSIZE(h, multi) -#endif - /* A hash bucket */ struct hbucket { struct rcu_head rcu; /* for call_rcu */ @@ -936,7 +913,12 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, goto set_full; /* Create a new slot */ if (n->pos >= n->size) { - TUNE_BUCKETSIZE(h, multi); +#ifdef IP_SET_HASH_WITH_MULTI + if (h->bucketsize >= AHASH_MAX_TUNED) + goto set_full; + else if (h->bucketsize < multi) + h->bucketsize += AHASH_INIT_SIZE; +#endif if (n->size >= AHASH_MAX(h)) { /* Trigger rehashing */ mtype_data_next(&h->next, d); -- GitLab From 8e987f1f4da92d9f1dd020418bfab9fe04b1c54c Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Wed, 2 Nov 2022 21:45:59 +0800 Subject: [PATCH 0921/1927] Documentation: devres: add missing I2C helper Add missing devm_i2c_add_adapter() to devres.rst. It's introduced by commit 07740c92ae57 ("i2c: core: add managed function for adding i2c adapters"). Fixes: 07740c92ae57 ("i2c: core: add managed function for adding i2c adapters") Signed-off-by: Yang Yingliang Acked-by: Yicong Yang Reviewed-by: Andy Shevchenko Signed-off-by: Wolfram Sang --- Documentation/driver-api/driver-model/devres.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst index 687adb58048ec..56082265e8e50 100644 --- a/Documentation/driver-api/driver-model/devres.rst +++ b/Documentation/driver-api/driver-model/devres.rst @@ -279,6 +279,7 @@ GPIO devm_gpio_request_one() I2C + devm_i2c_add_adapter() devm_i2c_new_dummy_device() IIO -- GitLab From 341421084d705475817f7f0d68e130370d10b20d Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Thu, 20 Oct 2022 11:46:40 -0400 Subject: [PATCH 0922/1927] drm/amd/display: Update DSC capabilitie for DCN314 dcn314 has 4 DSC - conflicted hardware document updated and confirmed. Tested-by: Mark Broadworth Reviewed-by: Charlene Liu Acked-by: Rodrigo Siqueira Signed-off-by: Leo Chen Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c index d0ad72caead28..9066c511a0529 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c @@ -847,7 +847,7 @@ static const struct resource_caps res_cap_dcn314 = { .num_ddc = 5, .num_vmid = 16, .num_mpc_3dlut = 2, - .num_dsc = 3, + .num_dsc = 4, }; static const struct dc_plane_cap plane_cap = { -- GitLab From 14aed119942f6c2f1286022323139f7404db5d2b Mon Sep 17 00:00:00 2001 From: Fangzhi Zuo Date: Thu, 20 Oct 2022 11:46:41 -0400 Subject: [PATCH 0923/1927] drm/amd/display: Ignore Cable ID Feature Ignore cable ID for DP2 receivers that does not support the feature. Tested-by: Mark Broadworth Reviewed-by: Roman Li Acked-by: Rodrigo Siqueira Signed-off-by: Fangzhi Zuo Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index c053cb79cd063..589bee9acf162 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -1549,6 +1549,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm; + /* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */ + adev->dm.dc->debug.ignore_cable_id = true; + r = dm_dmub_hw_init(adev); if (r) { DRM_ERROR("DMUB interface failed to initialize: status=%d\n", r); -- GitLab From e59843c4cdd68a369591630088171eeacce9859f Mon Sep 17 00:00:00 2001 From: Jun Lei Date: Thu, 20 Oct 2022 11:46:44 -0400 Subject: [PATCH 0924/1927] drm/amd/display: Limit dcn32 to 1950Mhz display clock [why] Hardware team recommends we limit dispclock to 1950Mhz for all DCN3.2.x [how] Limit to 1950 when initializing clocks. Tested-by: Mark Broadworth Reviewed-by: Alvin Lee Acked-by: Rodrigo Siqueira Signed-off-by: Jun Lei Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- .../gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c index 1c612ccf1944a..fd0313468fdbc 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c @@ -157,6 +157,7 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base) struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base); unsigned int num_levels; struct clk_limit_num_entries *num_entries_per_clk = &clk_mgr_base->bw_params->clk_table.num_entries_per_clk; + unsigned int i; memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks)); clk_mgr_base->clks.p_state_change_support = true; @@ -205,18 +206,17 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base) clk_mgr->dpm_present = true; if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) { - unsigned int i; - for (i = 0; i < num_levels; i++) if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz < khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz)) clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz = khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz); } + for (i = 0; i < num_levels; i++) + if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz > 1950) + clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz = 1950; if (clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz) { - unsigned int i; - for (i = 0; i < num_levels; i++) if (clk_mgr_base->bw_params->clk_table.entries[i].dppclk_mhz < khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz)) -- GitLab From c580d758ba1b79de9ea7a475d95a6278736ae462 Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Thu, 20 Oct 2022 11:46:47 -0400 Subject: [PATCH 0925/1927] drm/amd/display: Update latencies on DCN321 Update DF related latencies based on new measurements. Tested-by: Mark Broadworth Reviewed-by: Jun Lei Acked-by: Rodrigo Siqueira Signed-off-by: Dillon Varone Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index dd90f241e9065..7352f75144101 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -119,15 +119,15 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_21_soc = { }, }, .num_states = 1, - .sr_exit_time_us = 12.36, - .sr_enter_plus_exit_time_us = 16.72, + .sr_exit_time_us = 19.95, + .sr_enter_plus_exit_time_us = 24.36, .sr_exit_z8_time_us = 285.0, .sr_enter_plus_exit_z8_time_us = 320, .writeback_latency_us = 12.0, .round_trip_ping_latency_dcfclk_cycles = 263, - .urgent_latency_pixel_data_only_us = 4.0, - .urgent_latency_pixel_mixed_with_vm_data_us = 4.0, - .urgent_latency_vm_data_only_us = 4.0, + .urgent_latency_pixel_data_only_us = 9.35, + .urgent_latency_pixel_mixed_with_vm_data_us = 9.35, + .urgent_latency_vm_data_only_us = 9.35, .fclk_change_latency_us = 20, .usr_retraining_latency_us = 2, .smn_latency_us = 2, -- GitLab From 6cb5cec16c380be4cf9776a8c23b72e9fe742fd1 Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Thu, 20 Oct 2022 11:46:48 -0400 Subject: [PATCH 0926/1927] drm/amd/display: Set memclk levels to be at least 1 for dcn32 [Why] Cannot report 0 memclk levels even when SMU does not provide any. [How] When memclk levels reported by SMU is 0, set levels to 1. Tested-by: Mark Broadworth Reviewed-by: Martin Leung Acked-by: Rodrigo Siqueira Signed-off-by: Dillon Varone Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c index fd0313468fdbc..6f77d8e538ab1 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c @@ -669,6 +669,9 @@ static void dcn32_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base) &clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz, &num_entries_per_clk->num_memclk_levels); + /* memclk must have at least one level */ + num_entries_per_clk->num_memclk_levels = num_entries_per_clk->num_memclk_levels ? num_entries_per_clk->num_memclk_levels : 1; + dcn32_init_single_clock(clk_mgr, PPCLK_FCLK, &clk_mgr_base->bw_params->clk_table.entries[0].fclk_mhz, &num_entries_per_clk->num_fclk_levels); -- GitLab From c3d3f35b725bf9c93bec6d3c056f6bb7cfd27403 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 20 Oct 2022 11:46:51 -0400 Subject: [PATCH 0927/1927] drm/amd/display: Enable timing sync on DCN32 Missed enabling timing sync on DCN32 because DCN32 has a different DML param. Tested-by: Mark Broadworth Reviewed-by: Martin Leung Reviewed-by: Jun Lei Acked-by: Rodrigo Siqueira Signed-off-by: Alvin Lee Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c index d680f1c5b69f8..45db40c41882c 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c @@ -1228,6 +1228,7 @@ int dcn20_populate_dml_pipes_from_context( pipes[pipe_cnt].pipe.src.dcc = false; pipes[pipe_cnt].pipe.src.dcc_rate = 1; pipes[pipe_cnt].pipe.dest.synchronized_vblank_all_planes = synchronized_vblank; + pipes[pipe_cnt].pipe.dest.synchronize_timings = synchronized_vblank; pipes[pipe_cnt].pipe.dest.hblank_start = timing->h_total - timing->h_front_porch; pipes[pipe_cnt].pipe.dest.hblank_end = pipes[pipe_cnt].pipe.dest.hblank_start - timing->h_addressable -- GitLab From 9cb0dc6ccb7df9abe1407574ed4ad84895822d11 Mon Sep 17 00:00:00 2001 From: Max Tseng Date: Thu, 20 Oct 2022 11:46:52 -0400 Subject: [PATCH 0928/1927] drm/amd/display: cursor update command incomplete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Missing send cursor_rect width & Height into DMUB. PSR-SU would use these information. But missing these assignment in last refactor commit Reported-by: Timur Kristóf Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2227 Fixes: b73353f7f3d4 ("drm/amd/display: Use the same cursor info across features") Tested-by: Mark Broadworth Reviewed-by: Anthony Koo Acked-by: Rodrigo Siqueira Signed-off-by: Max Tseng Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c index 4996d2810edb8..938dba5249d48 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c @@ -623,6 +623,10 @@ void hubp2_cursor_set_attributes( hubp->att.size.bits.width = attr->width; hubp->att.size.bits.height = attr->height; hubp->att.cur_ctl.bits.mode = attr->color_format; + + hubp->cur_rect.w = attr->width; + hubp->cur_rect.h = attr->height; + hubp->att.cur_ctl.bits.pitch = hw_pitch; hubp->att.cur_ctl.bits.line_per_chunk = lpc; hubp->att.cur_ctl.bits.cur_2x_magnify = attr->attribute_flags.bits.ENABLE_MAGNIFICATION; -- GitLab From 89b3554782e6b65894f0551e9e0a82ad02dac94d Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Mon, 24 Oct 2022 12:47:47 +0800 Subject: [PATCH 0929/1927] drm/amdgpu: set fb_modifiers_not_supported in vkms This patch to fix the gdm3 start failure with virual display: /usr/libexec/gdm-x-session[1711]: (II) AMDGPU(0): Setting screen physical size to 270 x 203 /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): Failed to make import prime FD as pixmap: 22 /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): failed to set mode: Invalid argument /usr/libexec/gdm-x-session[1711]: (WW) AMDGPU(0): Failed to set mode on CRTC 0 /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): Failed to enable any CRTC gnome-shell[1840]: Running GNOME Shell (using mutter 42.2) as a X11 window and compositing manager /usr/libexec/gdm-x-session[1711]: (EE) AMDGPU(0): failed to set mode: Invalid argument vkms doesn't have modifiers support, set fb_modifiers_not_supported to bring the gdm back. Signed-off-by: Yifan Zhang Acked-by: Guchun Chen Reviewed-by: Tim Huang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c index f4b5301ea2a02..500a1dc4fe029 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c @@ -500,6 +500,8 @@ static int amdgpu_vkms_sw_init(void *handle) adev_to_drm(adev)->mode_config.fb_base = adev->gmc.aper_base; + adev_to_drm(adev)->mode_config.fb_modifiers_not_supported = true; + r = amdgpu_display_modeset_create_props(adev); if (r) return r; -- GitLab From e542ca6e3e554bad53b2ea5741873b67f4585ea9 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Tue, 25 Oct 2022 14:42:13 -0400 Subject: [PATCH 0930/1927] drm/amdgpu: correct MES debugfs versions Use mes.sched_version, mes.kiq_version for debugfs as mes.ucode_fw_version does not contain correct versioning information. Signed-off-by: Graham Sider Reviewed-by: Jack Xiao Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index bf1ff8f0e7120..4e42dcb1950f7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -337,12 +337,14 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info, fw_info->feature = adev->psp.cap_feature_version; break; case AMDGPU_INFO_FW_MES_KIQ: - fw_info->ver = adev->mes.ucode_fw_version[0]; - fw_info->feature = 0; + fw_info->ver = adev->mes.kiq_version & AMDGPU_MES_VERSION_MASK; + fw_info->feature = (adev->mes.kiq_version & AMDGPU_MES_FEAT_VERSION_MASK) + >> AMDGPU_MES_FEAT_VERSION_SHIFT; break; case AMDGPU_INFO_FW_MES: - fw_info->ver = adev->mes.ucode_fw_version[1]; - fw_info->feature = 0; + fw_info->ver = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; + fw_info->feature = (adev->mes.sched_version & AMDGPU_MES_FEAT_VERSION_MASK) + >> AMDGPU_MES_FEAT_VERSION_SHIFT; break; case AMDGPU_INFO_FW_IMU: fw_info->ver = adev->gfx.imu_fw_version; -- GitLab From 5b994354af3cab770bf13386469c5725713679af Mon Sep 17 00:00:00 2001 From: Yang Li Date: Wed, 26 Oct 2022 10:00:54 +0800 Subject: [PATCH 0931/1927] drm/amdkfd: Fix NULL pointer dereference in svm_migrate_to_ram() ./drivers/gpu/drm/amd/amdkfd/kfd_migrate.c:985:58-62: ERROR: p is NULL but dereferenced. Link: https://bugzilla.openanolis.cn/show_bug.cgi?id=2549 Reported-by: Abaci Robot Signed-off-by: Yang Li Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index 2797029bd5001..22b077ac9a196 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -973,12 +973,10 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf) out_unlock_svms: mutex_unlock(&p->svms.lock); out_unref_process: + pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr); kfd_unref_process(p); out_mmput: mmput(mm); - - pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr); - return r ? VM_FAULT_SIGBUS : 0; } -- GitLab From 8d4de331f1b24a22d18e3c6116aa25228cf54854 Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Wed, 26 Oct 2022 14:03:55 -0500 Subject: [PATCH 0932/1927] drm/amd: Fail the suspend if resources can't be evicted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a system does not have swap and memory is under 100% usage, amdgpu will fail to evict resources. Currently the suspend carries on proceeding to reset the GPU: ``` [drm] evicting device resources failed [drm:amdgpu_device_ip_suspend_phase2 [amdgpu]] *ERROR* suspend of IP block failed -12 [drm] free PSP TMR buffer [TTM] Failed allocating page table [drm] evicting device resources failed amdgpu 0000:03:00.0: amdgpu: MODE1 reset amdgpu 0000:03:00.0: amdgpu: GPU mode1 reset amdgpu 0000:03:00.0: amdgpu: GPU smu mode1 reset ``` At this point if the suspend actually succeeded I think that amdgpu would have recovered because the GPU would have power cut off and restored. However the kernel fails to continue the suspend from the memory pressure and amdgpu fails to run the "resume" from the aborted suspend. ``` ACPI: PM: Preparing to enter system sleep state S3 SLUB: Unable to allocate memory on node -1, gfp=0xdc0(GFP_KERNEL|__GFP_ZERO) cache: Acpi-State, object size: 80, buffer size: 80, default order: 0, min order: 0 node 0: slabs: 22, objs: 1122, free: 0 ACPI Error: AE_NO_MEMORY, Could not update object reference count (20210730/utdelete-651) [drm:psp_hw_start [amdgpu]] *ERROR* PSP load kdb failed! [drm:psp_resume [amdgpu]] *ERROR* PSP resume failed [drm:amdgpu_device_fw_loading [amdgpu]] *ERROR* resume of IP block failed -62 amdgpu 0000:03:00.0: amdgpu: amdgpu_device_ip_resume failed (-62). PM: dpm_run_callback(): pci_pm_resume+0x0/0x100 returns -62 amdgpu 0000:03:00.0: PM: failed to resume async: error -62 ``` To avoid this series of unfortunate events, fail amdgpu's suspend when the memory eviction fails. This will let the system gracefully recover and the user can try suspend again when the memory pressure is relieved. Reported-by: post@davidak.de Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2223 Signed-off-by: Mario Limonciello Reviewed-by: Alex Deucher Acked-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index ddaecb2610c94..64510898eedd0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4060,15 +4060,18 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) * at suspend time. * */ -static void amdgpu_device_evict_resources(struct amdgpu_device *adev) +static int amdgpu_device_evict_resources(struct amdgpu_device *adev) { + int ret; + /* No need to evict vram on APUs for suspend to ram or s2idle */ if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) - return; + return 0; - if (amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM)) + ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); + if (ret) DRM_WARN("evicting device resources failed\n"); - + return ret; } /* @@ -4118,7 +4121,9 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) if (!adev->in_s0ix) amdgpu_amdkfd_suspend(adev, adev->in_runpm); - amdgpu_device_evict_resources(adev); + r = amdgpu_device_evict_resources(adev); + if (r) + return r; amdgpu_fence_driver_hw_fini(adev); -- GitLab From 3aff8aaca4e36dc8b17eaa011684881a80238966 Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Wed, 5 Oct 2022 00:27:18 +0300 Subject: [PATCH 0933/1927] Bluetooth: L2CAP: Fix use-after-free caused by l2cap_reassemble_sdu Fix the race condition between the following two flows that run in parallel: 1. l2cap_reassemble_sdu -> chan->ops->recv (l2cap_sock_recv_cb) -> __sock_queue_rcv_skb. 2. bt_sock_recvmsg -> skb_recv_datagram, skb_free_datagram. An SKB can be queued by the first flow and immediately dequeued and freed by the second flow, therefore the callers of l2cap_reassemble_sdu can't use the SKB after that function returns. However, some places continue accessing struct l2cap_ctrl that resides in the SKB's CB for a short time after l2cap_reassemble_sdu returns, leading to a use-after-free condition (the stack trace is below, line numbers for kernel 5.19.8). Fix it by keeping a local copy of struct l2cap_ctrl. BUG: KASAN: use-after-free in l2cap_rx_state_recv (net/bluetooth/l2cap_core.c:6906) bluetooth Read of size 1 at addr ffff88812025f2f0 by task kworker/u17:3/43169 Workqueue: hci0 hci_rx_work [bluetooth] Call Trace: dump_stack_lvl (lib/dump_stack.c:107 (discriminator 4)) print_report.cold (mm/kasan/report.c:314 mm/kasan/report.c:429) ? l2cap_rx_state_recv (net/bluetooth/l2cap_core.c:6906) bluetooth kasan_report (mm/kasan/report.c:162 mm/kasan/report.c:493) ? l2cap_rx_state_recv (net/bluetooth/l2cap_core.c:6906) bluetooth l2cap_rx_state_recv (net/bluetooth/l2cap_core.c:6906) bluetooth l2cap_rx (net/bluetooth/l2cap_core.c:7236 net/bluetooth/l2cap_core.c:7271) bluetooth ret_from_fork (arch/x86/entry/entry_64.S:306) Allocated by task 43169: kasan_save_stack (mm/kasan/common.c:39) __kasan_slab_alloc (mm/kasan/common.c:45 mm/kasan/common.c:436 mm/kasan/common.c:469) kmem_cache_alloc_node (mm/slab.h:750 mm/slub.c:3243 mm/slub.c:3293) __alloc_skb (net/core/skbuff.c:414) l2cap_recv_frag (./include/net/bluetooth/bluetooth.h:425 net/bluetooth/l2cap_core.c:8329) bluetooth l2cap_recv_acldata (net/bluetooth/l2cap_core.c:8442) bluetooth hci_rx_work (net/bluetooth/hci_core.c:3642 net/bluetooth/hci_core.c:3832) bluetooth process_one_work (kernel/workqueue.c:2289) worker_thread (./include/linux/list.h:292 kernel/workqueue.c:2437) kthread (kernel/kthread.c:376) ret_from_fork (arch/x86/entry/entry_64.S:306) Freed by task 27920: kasan_save_stack (mm/kasan/common.c:39) kasan_set_track (mm/kasan/common.c:45) kasan_set_free_info (mm/kasan/generic.c:372) ____kasan_slab_free (mm/kasan/common.c:368 mm/kasan/common.c:328) slab_free_freelist_hook (mm/slub.c:1780) kmem_cache_free (mm/slub.c:3536 mm/slub.c:3553) skb_free_datagram (./include/net/sock.h:1578 ./include/net/sock.h:1639 net/core/datagram.c:323) bt_sock_recvmsg (net/bluetooth/af_bluetooth.c:295) bluetooth l2cap_sock_recvmsg (net/bluetooth/l2cap_sock.c:1212) bluetooth sock_read_iter (net/socket.c:1087) new_sync_read (./include/linux/fs.h:2052 fs/read_write.c:401) vfs_read (fs/read_write.c:482) ksys_read (fs/read_write.c:620) do_syscall_64 (arch/x86/entry/common.c:50 arch/x86/entry/common.c:80) entry_SYSCALL_64_after_hwframe (arch/x86/entry/entry_64.S:120) Link: https://lore.kernel.org/linux-bluetooth/CAKErNvoqga1WcmoR3-0875esY6TVWFQDandbVZncSiuGPBQXLA@mail.gmail.com/T/#u Fixes: d2a7ac5d5d3a ("Bluetooth: Add the ERTM receive state machine") Fixes: 4b51dae96731 ("Bluetooth: Add streaming mode receive and incoming packet classifier") Signed-off-by: Maxim Mikityanskiy Signed-off-by: Luiz Augusto von Dentz --- net/bluetooth/l2cap_core.c | 48 ++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 1f34b82ca0ec9..2283871d3f013 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -6885,6 +6885,7 @@ static int l2cap_rx_state_recv(struct l2cap_chan *chan, struct l2cap_ctrl *control, struct sk_buff *skb, u8 event) { + struct l2cap_ctrl local_control; int err = 0; bool skb_in_use = false; @@ -6909,15 +6910,32 @@ static int l2cap_rx_state_recv(struct l2cap_chan *chan, chan->buffer_seq = chan->expected_tx_seq; skb_in_use = true; + /* l2cap_reassemble_sdu may free skb, hence invalidate + * control, so make a copy in advance to use it after + * l2cap_reassemble_sdu returns and to avoid the race + * condition, for example: + * + * The current thread calls: + * l2cap_reassemble_sdu + * chan->ops->recv == l2cap_sock_recv_cb + * __sock_queue_rcv_skb + * Another thread calls: + * bt_sock_recvmsg + * skb_recv_datagram + * skb_free_datagram + * Then the current thread tries to access control, but + * it was freed by skb_free_datagram. + */ + local_control = *control; err = l2cap_reassemble_sdu(chan, skb, control); if (err) break; - if (control->final) { + if (local_control.final) { if (!test_and_clear_bit(CONN_REJ_ACT, &chan->conn_state)) { - control->final = 0; - l2cap_retransmit_all(chan, control); + local_control.final = 0; + l2cap_retransmit_all(chan, &local_control); l2cap_ertm_send(chan); } } @@ -7297,11 +7315,27 @@ static int l2cap_rx(struct l2cap_chan *chan, struct l2cap_ctrl *control, static int l2cap_stream_rx(struct l2cap_chan *chan, struct l2cap_ctrl *control, struct sk_buff *skb) { + /* l2cap_reassemble_sdu may free skb, hence invalidate control, so store + * the txseq field in advance to use it after l2cap_reassemble_sdu + * returns and to avoid the race condition, for example: + * + * The current thread calls: + * l2cap_reassemble_sdu + * chan->ops->recv == l2cap_sock_recv_cb + * __sock_queue_rcv_skb + * Another thread calls: + * bt_sock_recvmsg + * skb_recv_datagram + * skb_free_datagram + * Then the current thread tries to access control, but it was freed by + * skb_free_datagram. + */ + u16 txseq = control->txseq; + BT_DBG("chan %p, control %p, skb %p, state %d", chan, control, skb, chan->rx_state); - if (l2cap_classify_txseq(chan, control->txseq) == - L2CAP_TXSEQ_EXPECTED) { + if (l2cap_classify_txseq(chan, txseq) == L2CAP_TXSEQ_EXPECTED) { l2cap_pass_to_tx(chan, control); BT_DBG("buffer_seq %u->%u", chan->buffer_seq, @@ -7324,8 +7358,8 @@ static int l2cap_stream_rx(struct l2cap_chan *chan, struct l2cap_ctrl *control, } } - chan->last_acked_seq = control->txseq; - chan->expected_tx_seq = __next_seq(chan, control->txseq); + chan->last_acked_seq = txseq; + chan->expected_tx_seq = __next_seq(chan, txseq); return 0; } -- GitLab From b36a234dc438cb6b76fc929a8df9a0e59c8acf23 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Tue, 11 Oct 2022 22:25:33 +0300 Subject: [PATCH 0934/1927] Bluetooth: hci_conn: Fix CIS connection dst_type handling hci_connect_cis and iso_connect_cis call hci_bind_cis inconsistently with dst_type being either ISO socket address type or the HCI type, but these values cannot be mixed like this. Fix this by using only the HCI type. CIS connection dst_type was also not initialized in hci_bind_cis, even though it is used in hci_conn_hash_lookup_cis to find existing connections. Set the value in hci_bind_cis, so that existing CIS connections are found e.g. when doing deferred socket connections, also when dst_type is not 0 (ADDR_LE_DEV_PUBLIC). Fixes: 26afbd826ee3 ("Bluetooth: Add initial implementation of CIS connections") Signed-off-by: Pauli Virtanen Signed-off-by: Luiz Augusto von Dentz --- net/bluetooth/hci_conn.c | 7 +------ net/bluetooth/iso.c | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 7a59c44870503..1176bad5d8333 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1761,6 +1761,7 @@ struct hci_conn *hci_bind_cis(struct hci_dev *hdev, bdaddr_t *dst, if (!cis) return ERR_PTR(-ENOMEM); cis->cleanup = cis_cleanup; + cis->dst_type = dst_type; } if (cis->state == BT_CONNECTED) @@ -2140,12 +2141,6 @@ struct hci_conn *hci_connect_cis(struct hci_dev *hdev, bdaddr_t *dst, struct hci_conn *le; struct hci_conn *cis; - /* Convert from ISO socket address type to HCI address type */ - if (dst_type == BDADDR_LE_PUBLIC) - dst_type = ADDR_LE_DEV_PUBLIC; - else - dst_type = ADDR_LE_DEV_RANDOM; - if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) le = hci_connect_le(hdev, dst, dst_type, false, BT_SECURITY_LOW, diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index 613039ba5dbf5..f825857db6d0b 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -235,6 +235,14 @@ static int iso_chan_add(struct iso_conn *conn, struct sock *sk, return err; } +static inline u8 le_addr_type(u8 bdaddr_type) +{ + if (bdaddr_type == BDADDR_LE_PUBLIC) + return ADDR_LE_DEV_PUBLIC; + else + return ADDR_LE_DEV_RANDOM; +} + static int iso_connect_bis(struct sock *sk) { struct iso_conn *conn; @@ -328,14 +336,16 @@ static int iso_connect_cis(struct sock *sk) /* Just bind if DEFER_SETUP has been set */ if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags)) { hcon = hci_bind_cis(hdev, &iso_pi(sk)->dst, - iso_pi(sk)->dst_type, &iso_pi(sk)->qos); + le_addr_type(iso_pi(sk)->dst_type), + &iso_pi(sk)->qos); if (IS_ERR(hcon)) { err = PTR_ERR(hcon); goto done; } } else { hcon = hci_connect_cis(hdev, &iso_pi(sk)->dst, - iso_pi(sk)->dst_type, &iso_pi(sk)->qos); + le_addr_type(iso_pi(sk)->dst_type), + &iso_pi(sk)->qos); if (IS_ERR(hcon)) { err = PTR_ERR(hcon); goto done; -- GitLab From 160fbcf3bfb93c3c086427f9f4c8bc70f217e9be Mon Sep 17 00:00:00 2001 From: Soenke Huster Date: Wed, 12 Oct 2022 09:45:06 +0200 Subject: [PATCH 0935/1927] Bluetooth: virtio_bt: Use skb_put to set length By using skb_put we ensure that skb->tail is set correctly. Currently, skb->tail is always zero, which leads to errors, such as the following page fault in rfcomm_recv_frame: BUG: unable to handle page fault for address: ffffed1021de29ff #PF: supervisor read access in kernel mode #PF: error_code(0x0000) - not-present page RIP: 0010:rfcomm_run+0x831/0x4040 (net/bluetooth/rfcomm/core.c:1751) Fixes: afd2daa26c7a ("Bluetooth: Add support for virtio transport driver") Signed-off-by: Soenke Huster Signed-off-by: Luiz Augusto von Dentz --- drivers/bluetooth/virtio_bt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 67c21263f9e0f..fd281d4395055 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -219,7 +219,7 @@ static void virtbt_rx_work(struct work_struct *work) if (!skb) return; - skb->len = len; + skb_put(skb, len); virtbt_rx_handle(vbt, skb); if (virtbt_add_inbuf(vbt) < 0) -- GitLab From 0d0e2d032811280b927650ff3c15fe5020e82533 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 17 Oct 2022 15:58:13 +0800 Subject: [PATCH 0936/1927] Bluetooth: L2CAP: fix use-after-free in l2cap_conn_del() When l2cap_recv_frame() is invoked to receive data, and the cid is L2CAP_CID_A2MP, if the channel does not exist, it will create a channel. However, after a channel is created, the hold operation of the channel is not performed. In this case, the value of channel reference counting is 1. As a result, after hci_error_reset() is triggered, l2cap_conn_del() invokes the close hook function of A2MP to release the channel. Then l2cap_chan_unlock(chan) will trigger UAF issue. The process is as follows: Receive data: l2cap_data_channel() a2mp_channel_create() --->channel ref is 2 l2cap_chan_put() --->channel ref is 1 Triger event: hci_error_reset() hci_dev_do_close() ... l2cap_disconn_cfm() l2cap_conn_del() l2cap_chan_hold() --->channel ref is 2 l2cap_chan_del() --->channel ref is 1 a2mp_chan_close_cb() --->channel ref is 0, release channel l2cap_chan_unlock() --->UAF of channel The detailed Call Trace is as follows: BUG: KASAN: use-after-free in __mutex_unlock_slowpath+0xa6/0x5e0 Read of size 8 at addr ffff8880160664b8 by task kworker/u11:1/7593 Workqueue: hci0 hci_error_reset Call Trace: dump_stack_lvl+0xcd/0x134 print_report.cold+0x2ba/0x719 kasan_report+0xb1/0x1e0 kasan_check_range+0x140/0x190 __mutex_unlock_slowpath+0xa6/0x5e0 l2cap_conn_del+0x404/0x7b0 l2cap_disconn_cfm+0x8c/0xc0 hci_conn_hash_flush+0x11f/0x260 hci_dev_close_sync+0x5f5/0x11f0 hci_dev_do_close+0x2d/0x70 hci_error_reset+0x9e/0x140 process_one_work+0x98a/0x1620 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Allocated by task 7593: kasan_save_stack+0x1e/0x40 __kasan_kmalloc+0xa9/0xd0 l2cap_chan_create+0x40/0x930 amp_mgr_create+0x96/0x990 a2mp_channel_create+0x7d/0x150 l2cap_recv_frame+0x51b8/0x9a70 l2cap_recv_acldata+0xaa3/0xc00 hci_rx_work+0x702/0x1220 process_one_work+0x98a/0x1620 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Freed by task 7593: kasan_save_stack+0x1e/0x40 kasan_set_track+0x21/0x30 kasan_set_free_info+0x20/0x30 ____kasan_slab_free+0x167/0x1c0 slab_free_freelist_hook+0x89/0x1c0 kfree+0xe2/0x580 l2cap_chan_put+0x22a/0x2d0 l2cap_conn_del+0x3fc/0x7b0 l2cap_disconn_cfm+0x8c/0xc0 hci_conn_hash_flush+0x11f/0x260 hci_dev_close_sync+0x5f5/0x11f0 hci_dev_do_close+0x2d/0x70 hci_error_reset+0x9e/0x140 process_one_work+0x98a/0x1620 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Last potentially related work creation: kasan_save_stack+0x1e/0x40 __kasan_record_aux_stack+0xbe/0xd0 call_rcu+0x99/0x740 netlink_release+0xe6a/0x1cf0 __sock_release+0xcd/0x280 sock_close+0x18/0x20 __fput+0x27c/0xa90 task_work_run+0xdd/0x1a0 exit_to_user_mode_prepare+0x23c/0x250 syscall_exit_to_user_mode+0x19/0x50 do_syscall_64+0x42/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd Second to last potentially related work creation: kasan_save_stack+0x1e/0x40 __kasan_record_aux_stack+0xbe/0xd0 call_rcu+0x99/0x740 netlink_release+0xe6a/0x1cf0 __sock_release+0xcd/0x280 sock_close+0x18/0x20 __fput+0x27c/0xa90 task_work_run+0xdd/0x1a0 exit_to_user_mode_prepare+0x23c/0x250 syscall_exit_to_user_mode+0x19/0x50 do_syscall_64+0x42/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: d0be8347c623 ("Bluetooth: L2CAP: Fix use-after-free caused by l2cap_chan_put") Signed-off-by: Zhengchao Shao Signed-off-by: Luiz Augusto von Dentz --- net/bluetooth/l2cap_core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 2283871d3f013..9a32ce6349194 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -7615,6 +7615,7 @@ static void l2cap_data_channel(struct l2cap_conn *conn, u16 cid, return; } + l2cap_chan_hold(chan); l2cap_chan_lock(chan); } else { BT_DBG("unknown cid 0x%4.4x", cid); -- GitLab From b43088f30db1a7bff61c8486238c195c77788d6d Mon Sep 17 00:00:00 2001 From: Harald Freudenberger Date: Tue, 25 Oct 2022 10:05:25 +0200 Subject: [PATCH 0937/1927] s390/zcrypt: fix warning about field-spanning write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch fixes the warning memcpy: detected field-spanning write (size 60) of single field "to" at drivers/s390/crypto/zcrypt_api.h:173 (size 2) WARNING: CPU: 1 PID: 2114 at drivers/s390/crypto/zcrypt_api.h:173 prep_ep11_ap_msg+0x2c6/0x2e0 [zcrypt] The code has been rewritten to use a union in combination with a flex array to clearly state which part of the buffer the payload is to be copied in via z_copy_from_user function (which may call memcpy() in case of in-kernel calls). Signed-off-by: Harald Freudenberger Suggested-by: Jürgen Christ Reviewed-by: Jürgen Christ Signed-off-by: Vasily Gorbik --- drivers/s390/crypto/zcrypt_msgtype6.c | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 8fb34b8eeb189..5ad2514775931 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -342,7 +342,10 @@ static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg, }; struct { struct type6_hdr hdr; - struct CPRBX cprbx; + union { + struct CPRBX cprbx; + DECLARE_FLEX_ARRAY(u8, userdata); + }; } __packed * msg = ap_msg->msg; int rcblen = CEIL4(xcrb->request_control_blk_length); @@ -403,7 +406,8 @@ static int xcrb_msg_to_type6cprb_msgx(bool userspace, struct ap_message *ap_msg, msg->hdr.fromcardlen2 = xcrb->reply_data_length; /* prepare CPRB */ - if (z_copy_from_user(userspace, &msg->cprbx, xcrb->request_control_blk_addr, + if (z_copy_from_user(userspace, msg->userdata, + xcrb->request_control_blk_addr, xcrb->request_control_blk_length)) return -EFAULT; if (msg->cprbx.cprb_len + sizeof(msg->hdr.function_code) > @@ -469,9 +473,14 @@ static int xcrb_msg_to_type6_ep11cprb_msgx(bool userspace, struct ap_message *ap struct { struct type6_hdr hdr; - struct ep11_cprb cprbx; - unsigned char pld_tag; /* fixed value 0x30 */ - unsigned char pld_lenfmt; /* payload length format */ + union { + struct { + struct ep11_cprb cprbx; + unsigned char pld_tag; /* fixed value 0x30 */ + unsigned char pld_lenfmt; /* length format */ + } __packed; + DECLARE_FLEX_ARRAY(u8, userdata); + }; } __packed * msg = ap_msg->msg; struct pld_hdr { @@ -500,7 +509,7 @@ static int xcrb_msg_to_type6_ep11cprb_msgx(bool userspace, struct ap_message *ap msg->hdr.fromcardlen1 = xcrb->resp_len; /* Import CPRB data from the ioctl input parameter */ - if (z_copy_from_user(userspace, &msg->cprbx.cprb_len, + if (z_copy_from_user(userspace, msg->userdata, (char __force __user *)xcrb->req, xcrb->req_len)) { return -EFAULT; } -- GitLab From bb8738876b24c3bbfccbd2ccbdc57c27402516ab Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 31 Oct 2022 18:29:04 +0100 Subject: [PATCH 0938/1927] s390: update defconfigs Signed-off-by: Heiko Carstens Signed-off-by: Vasily Gorbik --- arch/s390/configs/debug_defconfig | 64 +++++++++++++++--------------- arch/s390/configs/defconfig | 66 +++++++++++++++---------------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index 2a827002934bc..e33dc2d6fd046 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -723,52 +723,42 @@ CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m -CONFIG_CRYPTO_GCM=y -CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_AEGIS128=m -CONFIG_CRYPTO_SEQIV=y -CONFIG_CRYPTO_CFB=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_KEYWRAP=m -CONFIG_CRYPTO_ADIANTUM=m -CONFIG_CRYPTO_HCTR2=m -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_CRC32_S390=y -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_SHA512_S390=m -CONFIG_CRYPTO_SHA1_S390=m -CONFIG_CRYPTO_SHA256_S390=m -CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SHA3_256_S390=m -CONFIG_CRYPTO_SHA3_512_S390=m -CONFIG_CRYPTO_SM3_GENERIC=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_TI=m -CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_ANUBIS=m -CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m -CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_CHACHA_S390=m CONFIG_CRYPTO_SEED=m -CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_CFB=m +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_KEYWRAP=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_MICHAEL_MIC=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_CRC32=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m @@ -779,6 +769,16 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_STATS=y +CONFIG_CRYPTO_CRC32_S390=y +CONFIG_CRYPTO_SHA512_S390=m +CONFIG_CRYPTO_SHA1_S390=m +CONFIG_CRYPTO_SHA256_S390=m +CONFIG_CRYPTO_SHA3_256_S390=m +CONFIG_CRYPTO_SHA3_512_S390=m +CONFIG_CRYPTO_GHASH_S390=m +CONFIG_CRYPTO_AES_S390=m +CONFIG_CRYPTO_DES_S390=m +CONFIG_CRYPTO_CHACHA_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m CONFIG_CRYPTO_PAES_S390=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index fb780e80e4c8f..9cc8f024f3dfe 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -707,53 +707,43 @@ CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m -CONFIG_CRYPTO_GCM=y -CONFIG_CRYPTO_CHACHA20POLY1305=m -CONFIG_CRYPTO_AEGIS128=m -CONFIG_CRYPTO_SEQIV=y -CONFIG_CRYPTO_CFB=m -CONFIG_CRYPTO_LRW=m -CONFIG_CRYPTO_OFB=m -CONFIG_CRYPTO_PCBC=m -CONFIG_CRYPTO_KEYWRAP=m -CONFIG_CRYPTO_ADIANTUM=m -CONFIG_CRYPTO_HCTR2=m -CONFIG_CRYPTO_XCBC=m -CONFIG_CRYPTO_VMAC=m -CONFIG_CRYPTO_CRC32=m -CONFIG_CRYPTO_CRC32_S390=y -CONFIG_CRYPTO_MD4=m -CONFIG_CRYPTO_MD5=y -CONFIG_CRYPTO_MICHAEL_MIC=m -CONFIG_CRYPTO_RMD160=m -CONFIG_CRYPTO_SHA512_S390=m -CONFIG_CRYPTO_SHA1_S390=m -CONFIG_CRYPTO_SHA256_S390=m -CONFIG_CRYPTO_SHA3=m -CONFIG_CRYPTO_SHA3_256_S390=m -CONFIG_CRYPTO_SHA3_512_S390=m -CONFIG_CRYPTO_SM3_GENERIC=m -CONFIG_CRYPTO_WP512=m -CONFIG_CRYPTO_GHASH_S390=m CONFIG_CRYPTO_AES_TI=m -CONFIG_CRYPTO_AES_S390=m CONFIG_CRYPTO_ANUBIS=m -CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_DES=m -CONFIG_CRYPTO_DES_S390=m CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_KHAZAD=m -CONFIG_CRYPTO_CHACHA_S390=m CONFIG_CRYPTO_SEED=m -CONFIG_CRYPTO_ARIA=m CONFIG_CRYPTO_SERPENT=m CONFIG_CRYPTO_SM4_GENERIC=m CONFIG_CRYPTO_TEA=m CONFIG_CRYPTO_TWOFISH=m +CONFIG_CRYPTO_ADIANTUM=m +CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_CFB=m +CONFIG_CRYPTO_HCTR2=m +CONFIG_CRYPTO_KEYWRAP=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_OFB=m +CONFIG_CRYPTO_PCBC=m +CONFIG_CRYPTO_AEGIS128=m +CONFIG_CRYPTO_CHACHA20POLY1305=m +CONFIG_CRYPTO_GCM=y +CONFIG_CRYPTO_SEQIV=y +CONFIG_CRYPTO_MD4=m +CONFIG_CRYPTO_MD5=y +CONFIG_CRYPTO_MICHAEL_MIC=m +CONFIG_CRYPTO_RMD160=m +CONFIG_CRYPTO_SHA3=m +CONFIG_CRYPTO_SM3_GENERIC=m +CONFIG_CRYPTO_VMAC=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_XCBC=m +CONFIG_CRYPTO_CRC32=m CONFIG_CRYPTO_842=m CONFIG_CRYPTO_LZ4=m CONFIG_CRYPTO_LZ4HC=m @@ -764,6 +754,16 @@ CONFIG_CRYPTO_USER_API_SKCIPHER=m CONFIG_CRYPTO_USER_API_RNG=m CONFIG_CRYPTO_USER_API_AEAD=m CONFIG_CRYPTO_STATS=y +CONFIG_CRYPTO_CRC32_S390=y +CONFIG_CRYPTO_SHA512_S390=m +CONFIG_CRYPTO_SHA1_S390=m +CONFIG_CRYPTO_SHA256_S390=m +CONFIG_CRYPTO_SHA3_256_S390=m +CONFIG_CRYPTO_SHA3_512_S390=m +CONFIG_CRYPTO_GHASH_S390=m +CONFIG_CRYPTO_AES_S390=m +CONFIG_CRYPTO_DES_S390=m +CONFIG_CRYPTO_CHACHA_S390=m CONFIG_ZCRYPT=m CONFIG_PKEY=m CONFIG_CRYPTO_PAES_S390=m -- GitLab From 7c9524d929648935bac2bbb4c20437df8f9c3f42 Mon Sep 17 00:00:00 2001 From: Hawkins Jiawei Date: Tue, 18 Oct 2022 10:18:51 +0800 Subject: [PATCH 0939/1927] Bluetooth: L2CAP: Fix memory leak in vhci_write Syzkaller reports a memory leak as follows: ==================================== BUG: memory leak unreferenced object 0xffff88810d81ac00 (size 240): [...] hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [] __alloc_skb+0x1f9/0x270 net/core/skbuff.c:418 [] alloc_skb include/linux/skbuff.h:1257 [inline] [] bt_skb_alloc include/net/bluetooth/bluetooth.h:469 [inline] [] vhci_get_user drivers/bluetooth/hci_vhci.c:391 [inline] [] vhci_write+0x5f/0x230 drivers/bluetooth/hci_vhci.c:511 [] call_write_iter include/linux/fs.h:2192 [inline] [] new_sync_write fs/read_write.c:491 [inline] [] vfs_write+0x42d/0x540 fs/read_write.c:578 [] ksys_write+0x9d/0x160 fs/read_write.c:631 [] do_syscall_x64 arch/x86/entry/common.c:50 [inline] [] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 [] entry_SYSCALL_64_after_hwframe+0x63/0xcd ==================================== HCI core will uses hci_rx_work() to process frame, which is queued to the hdev->rx_q tail in hci_recv_frame() by HCI driver. Yet the problem is that, HCI core may not free the skb after handling ACL data packets. To be more specific, when start fragment does not contain the L2CAP length, HCI core just copies skb into conn->rx_skb and finishes frame process in l2cap_recv_acldata(), without freeing the skb, which triggers the above memory leak. This patch solves it by releasing the relative skb, after processing the above case in l2cap_recv_acldata(). Fixes: 4d7ea8ee90e4 ("Bluetooth: L2CAP: Fix handling fragmented length") Link: https://lore.kernel.org/all/0000000000000d0b1905e6aaef64@google.com/ Reported-and-tested-by: syzbot+8f819e36e01022991cfa@syzkaller.appspotmail.com Signed-off-by: Hawkins Jiawei Signed-off-by: Luiz Augusto von Dentz --- net/bluetooth/l2cap_core.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 9a32ce6349194..1fbe087d6ae4e 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -8461,9 +8461,8 @@ void l2cap_recv_acldata(struct hci_conn *hcon, struct sk_buff *skb, u16 flags) * expected length. */ if (skb->len < L2CAP_LEN_SIZE) { - if (l2cap_recv_frag(conn, skb, conn->mtu) < 0) - goto drop; - return; + l2cap_recv_frag(conn, skb, conn->mtu); + break; } len = get_unaligned_le16(skb->data) + L2CAP_HDR_SIZE; @@ -8507,7 +8506,7 @@ void l2cap_recv_acldata(struct hci_conn *hcon, struct sk_buff *skb, u16 flags) /* Header still could not be read just continue */ if (conn->rx_skb->len < L2CAP_LEN_SIZE) - return; + break; } if (skb->len > conn->rx_len) { -- GitLab From 5638d9ea9c01c77fc11693d48cf719bc7e88f224 Mon Sep 17 00:00:00 2001 From: Luiz Augusto von Dentz Date: Mon, 17 Oct 2022 15:36:23 -0700 Subject: [PATCH 0940/1927] Bluetooth: hci_conn: Fix not restoring ISO buffer count on disconnect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When disconnecting an ISO link the controller may not generate HCI_EV_NUM_COMP_PKTS for unacked packets which needs to be restored in hci_conn_del otherwise the host would assume they are still in use and would not be able to use all the buffers available. Fixes: 26afbd826ee3 ("Bluetooth: Add initial implementation of CIS connections") Signed-off-by: Luiz Augusto von Dentz Tested-by: Frédéric Danis --- net/bluetooth/hci_conn.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index 1176bad5d8333..a6c12863a2532 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -1067,10 +1067,21 @@ int hci_conn_del(struct hci_conn *conn) hdev->acl_cnt += conn->sent; } else { struct hci_conn *acl = conn->link; + if (acl) { acl->link = NULL; hci_conn_drop(acl); } + + /* Unacked ISO frames */ + if (conn->type == ISO_LINK) { + if (hdev->iso_pkts) + hdev->iso_cnt += conn->sent; + else if (hdev->le_pkts) + hdev->le_cnt += conn->sent; + else + hdev->acl_cnt += conn->sent; + } } if (conn->amp_mgr) -- GitLab From a3e5ce56f3d260f2ec8e5242c33f57e60ae9eba7 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Wed, 26 Oct 2022 15:08:24 -0400 Subject: [PATCH 0941/1927] drm/amdgpu: disable GFXOFF during compute for GFX11 Temporary workaround to fix issues observed in some compute applications when GFXOFF is enabled on GFX11. Signed-off-by: Graham Sider Acked-by: Alex Deucher Reviewed-by: Harish Kasiviswanathan Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 0561812aa0a43..5d9a34601a1ac 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -706,6 +706,13 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev, void amdgpu_amdkfd_set_compute_idle(struct amdgpu_device *adev, bool idle) { + /* Temporary workaround to fix issues observed in some + * compute applications when GFXOFF is enabled on GFX11. + */ + if (IP_VERSION_MAJ(adev->ip_versions[GC_HWIP][0]) == 11) { + pr_debug("GFXOFF is %s\n", idle ? "enabled" : "disabled"); + amdgpu_gfx_off_ctrl(adev, idle); + } amdgpu_dpm_switch_power_profile(adev, PP_SMC_POWER_PROFILE_COMPUTE, !idle); -- GitLab From 8fe8ce896c1cc29d6bfebb3c7b3cc948f72cd32c Mon Sep 17 00:00:00 2001 From: Gavin Wan Date: Wed, 26 Oct 2022 13:45:25 -0400 Subject: [PATCH 0942/1927] drm/amdgpu: Disable GPU reset on SRIOV before remove pci. The recent change brought a bug on SRIOV envrionment. It caused unloading amdgpu failed on Guest VM. The reason is that the VF FLR was requested while unloading amdgpu driver, but the VF FLR of SRIOV sequence is wrong while removing PCI device. For SRIOV, the guest driver should not trigger the whole XGMI hive to do the reset. Host driver control how the device been reset. Fixes: f5c7e7797060 ("drm/amdgpu: Adjust removal control flow for smu v13_0_2") Acked-by: Alex Deucher Reviewed-by: Shaoyun Liu Signed-off-by: Gavin Wan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 3c9fecdd6b2f3..bf2d50c8c92ad 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2201,7 +2201,8 @@ amdgpu_pci_remove(struct pci_dev *pdev) pm_runtime_forbid(dev->dev); } - if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)) { + if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2) && + !amdgpu_sriov_vf(adev)) { bool need_to_reset_gpu = false; if (adev->gmc.xgmi.num_physical_nodes > 1) { -- GitLab From bad610c97c08eef3ed1fa769a8b08b94f95b451e Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 7 Oct 2022 12:43:26 -0400 Subject: [PATCH 0943/1927] drm/amd/display: Fix DCN32 DSC delay calculation [Why] DCN32 DSC delay calculation had an unintentional integer division, resulting in a mismatch against the DML spreadsheet. [How] Cast numerator to double before performing the division. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index ad66e241f9ae2..4a3e7a5d2758e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -1746,7 +1746,7 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, } DSCDelayRequirement_val = DSCDelayRequirement_val + (HTotal - HActive) * - dml_ceil(DSCDelayRequirement_val / HActive, 1); + dml_ceil((double)DSCDelayRequirement_val / HActive, 1); DSCDelayRequirement_val = DSCDelayRequirement_val * PixelClock / PixelClockBackEnd; -- GitLab From ab007e5db5d3b8b8975c7eec69992ff38fe2a46c Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 14 Oct 2022 17:36:32 -0400 Subject: [PATCH 0944/1927] drm/amd/display: Use forced DSC bpp in DML [Why] DSC config is calculated separately from DML calculations. DML should use these separately calculated DSC params. The issue is that the calculated bpp is not properly propagated into DML. [How] Correctly used forced_bpp value in DML. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 2 +- drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 5b91660a6496b..60351b2891d07 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -1627,7 +1627,7 @@ static void mode_support_configuration(struct vba_vars_st *v, && !mode_lib->vba.MSOOrODMSplitWithNonDPLink && !mode_lib->vba.NotEnoughLanesForMSO && mode_lib->vba.LinkCapacitySupport[i] == true && !mode_lib->vba.P2IWith420 - && !mode_lib->vba.DSCOnlyIfNecessaryWithBPP + //&& !mode_lib->vba.DSCOnlyIfNecessaryWithBPP && !mode_lib->vba.DSC422NativeNotSupported && !mode_lib->vba.MPCCombineMethodIncompatible && mode_lib->vba.ODMCombine2To1SupportCheckOK[i] == true diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c index 03924aed8d5c7..8e6585dab20ef 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c @@ -625,7 +625,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib) mode_lib->vba.skip_dio_check[mode_lib->vba.NumberOfActivePlanes] = dout->is_virtual; - if (!dout->dsc_enable) + if (dout->dsc_enable) mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = dout->output_bpp; else mode_lib->vba.ForcedOutputLinkBPP[mode_lib->vba.NumberOfActivePlanes] = 0.0; -- GitLab From 8dc323133d74518e3b5b07242e2b2f088799ea6e Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 14 Oct 2022 17:40:08 -0400 Subject: [PATCH 0945/1927] drm/amd/display: Round up DST_after_scaler to nearest int [Why] The DST_after_scaler value that DML spreadsheet outputs is generally the driver value round up to the nearest int. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c index a1276f6b9581b..395ae8761980f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c @@ -291,8 +291,8 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib, dml_print("DML_DLG: %s: vready_after_vcount0 = %d\n", __func__, dlg_regs->vready_after_vcount0); - dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); - dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); + dst_x_after_scaler = dml_ceil(get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1); + dst_y_after_scaler = dml_ceil(get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1); // do some adjustment on the dst_after scaler to account for odm combine mode dml_print("DML_DLG: %s: input dst_x_after_scaler = %d\n", __func__, dst_x_after_scaler); -- GitLab From d5e0fb0d9dea545defb963ec1073bd9a1a8b5395 Mon Sep 17 00:00:00 2001 From: George Shen Date: Fri, 14 Oct 2022 17:46:03 -0400 Subject: [PATCH 0946/1927] drm/amd/display: Add DSC delay factor workaround [Why] Certain 4K high refresh rate modes requiring DSC are exhibiting top of screen underflow corruption. Increasing the DSC delay by a factor of 6 percent stops the underflow for most use cases. [How] Multiply DSC delay requirement in DML by a factor. Add debug option to make this DSC delay factor configurable. Reviewed-by: Alvin Lee Acked-by: Alex Hung Signed-off-by: George Shen Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 + drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 4 +++- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 5 +++-- .../drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c | 5 +++-- .../drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h | 3 ++- drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c | 5 ++++- drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h | 3 +++ 7 files changed, 19 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index bfc5474c0f4c9..737b221ca6890 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -852,6 +852,7 @@ struct dc_debug_options { bool enable_double_buffered_dsc_pg_support; bool enable_dp_dig_pixel_rate_div_policy; enum lttpr_mode lttpr_mode_override; + unsigned int dsc_delay_factor_wa_x1000; }; struct gpu_info_soc_bounding_box_v1_0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index 819de0f110126..f37c9a6b3b7e0 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -2359,9 +2359,11 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) dcn3_2_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes; - } + /* DML DSC delay factor workaround */ + dcn3_2_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_2_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 60351b2891d07..47ff0a8bd1052 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -367,7 +367,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman mode_lib->vba.OutputBpp[k], mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], mode_lib->vba.NumberOfDSCSlices[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], mode_lib->vba.PixelClock[k], - mode_lib->vba.PixelClockBackEnd[k]); + mode_lib->vba.PixelClockBackEnd[k], mode_lib->vba.ip.dsc_delay_factor_wa); } for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) @@ -2475,7 +2475,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.OutputBppPerState[i][k], mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], mode_lib->vba.NumberOfDSCSlices[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], - mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k]); + mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k], + mode_lib->vba.ip.dsc_delay_factor_wa); } for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index 4a3e7a5d2758e..968924c491c18 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -1726,7 +1726,8 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, enum output_format_class OutputFormat, enum output_encoder_class Output, double PixelClock, - double PixelClockBackEnd) + double PixelClockBackEnd, + double dsc_delay_factor_wa) { unsigned int DSCDelayRequirement_val; @@ -1764,7 +1765,7 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, dml_print("DML::%s: DSCDelayRequirement_val = %d\n", __func__, DSCDelayRequirement_val); #endif - return DSCDelayRequirement_val; + return dml_ceil(DSCDelayRequirement_val * dsc_delay_factor_wa, 1); } void dml32_CalculateSurfaceSizeInMall( diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index 55cead0d42374..2c3827546ac77 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -327,7 +327,8 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled, enum output_format_class OutputFormat, enum output_encoder_class Output, double PixelClock, - double PixelClockBackEnd); + double PixelClockBackEnd, + double dsc_delay_factor_wa); void dml32_CalculateSurfaceSizeInMall( unsigned int NumberOfActiveSurfaces, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index 7352f75144101..ec0486efab147 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -29,6 +29,7 @@ #include "dcn321_fpu.h" #include "dcn32/dcn32_resource.h" #include "dcn321/dcn321_resource.h" +#include "dml/dcn32/display_mode_vba_util_32.h" #define DCN3_2_DEFAULT_DET_SIZE 256 @@ -538,9 +539,11 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) dcn3_21_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes; - } + /* DML DSC delay factor workaround */ + dcn3_21_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_21_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index f33a8879b05ad..d7be01ac07514 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -364,6 +364,9 @@ struct _vcs_dpi_ip_params_st { unsigned int max_num_dp2p0_outputs; unsigned int max_num_dp2p0_streams; unsigned int VBlankNomDefaultUS; + + /* DM workarounds */ + double dsc_delay_factor_wa; // TODO: Remove after implementing root cause fix }; struct _vcs_dpi_display_xfc_params_st { -- GitLab From 7461016c5706eb8c477752bf69e5c9f5a38f502b Mon Sep 17 00:00:00 2001 From: Nevenko Stupar Date: Fri, 6 May 2022 16:32:38 -0400 Subject: [PATCH 0947/1927] drm/amd/display: Investigate tool reported FCLK P-state deviations [Why] Fix for some of the tool reported modes for FCLK P-state deviations and UCLK P-state deviations that are coming from DSC terms and/or Scaling terms causing MinActiveFCLKChangeLatencySupported and MaxActiveDRAMClockChangeLatencySupported incorrectly calculated in DML for these configurations. Reviewed-by: Chaitanya Dhere Acked-by: Jasdeep Dhillon Acked-by: Alex Hung Signed-off-by: Nevenko Stupar Tested-by: Mark Broadworth Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 47ff0a8bd1052..3d184679f129e 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -364,7 +364,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) { v->DSCDelay[k] = dml32_DSCDelayRequirement(mode_lib->vba.DSCEnabled[k], mode_lib->vba.ODMCombineEnabled[k], mode_lib->vba.DSCInputBitPerComponent[k], - mode_lib->vba.OutputBpp[k], mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], + mode_lib->vba.OutputBppPerState[mode_lib->vba.VoltageLevel][k], + mode_lib->vba.HActive[k], mode_lib->vba.HTotal[k], mode_lib->vba.NumberOfDSCSlices[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.Output[k], mode_lib->vba.PixelClock[k], mode_lib->vba.PixelClockBackEnd[k], mode_lib->vba.ip.dsc_delay_factor_wa); -- GitLab From 6640f8e5adb69a0550fe1d224d3ac64c10f00eef Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Thu, 13 Oct 2022 21:41:13 -0500 Subject: [PATCH 0948/1927] drm/amdkfd: update GFX11 CWSR trap handler With corresponding FW change fixes issue where triggering CWSR on a workgroup with waves in s_barrier wouldn't lead to a back-off and therefore cause a hang. Signed-off-by: Jay Cornwall Tested-by: Graham Sider Acked-by: Harish Kasiviswanathan Acked-by: Felix Kuehling Reviewed-by: Graham Sider Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- .../gpu/drm/amd/amdkfd/cwsr_trap_handler.h | 764 +++++++++--------- .../amd/amdkfd/cwsr_trap_handler_gfx10.asm | 6 + 2 files changed, 389 insertions(+), 381 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h index c7118843db051..0c4c5499bb5cc 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h @@ -2495,442 +2495,444 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0xbf9f0000, 0x00000000, }; static const uint32_t cwsr_trap_gfx11_hex[] = { - 0xbfa00001, 0xbfa0021e, + 0xbfa00001, 0xbfa00221, 0xb0804006, 0xb8f8f802, 0x9178ff78, 0x00020006, - 0xb8fbf803, 0xbf0d9f6d, - 0xbfa20006, 0x8b6eff78, - 0x00002000, 0xbfa10009, - 0x8b6eff6d, 0x00ff0000, - 0xbfa2001e, 0x8b6eff7b, - 0x00000400, 0xbfa20041, - 0xbf830010, 0xb8fbf803, - 0xbfa0fffa, 0x8b6eff7b, - 0x00000900, 0xbfa20015, - 0x8b6eff7b, 0x000071ff, - 0xbfa10008, 0x8b6fff7b, - 0x00007080, 0xbfa10001, - 0xbeee1287, 0xb8eff801, - 0x846e8c6e, 0x8b6e6f6e, - 0xbfa2000a, 0x8b6eff6d, - 0x00ff0000, 0xbfa20007, - 0xb8eef801, 0x8b6eff6e, - 0x00000800, 0xbfa20003, + 0xb8fbf803, 0xbf0d9e6d, + 0xbfa10001, 0xbfbd0000, + 0xbf0d9f6d, 0xbfa20006, + 0x8b6eff78, 0x00002000, + 0xbfa10009, 0x8b6eff6d, + 0x00ff0000, 0xbfa2001e, 0x8b6eff7b, 0x00000400, - 0xbfa20026, 0xbefa4d82, - 0xbf89fc07, 0x84fa887a, - 0xf4005bbd, 0xf8000010, - 0xbf89fc07, 0x846e976e, - 0x9177ff77, 0x00800000, - 0x8c776e77, 0xf4045bbd, - 0xf8000000, 0xbf89fc07, - 0xf4045ebd, 0xf8000008, - 0xbf89fc07, 0x8bee6e6e, - 0xbfa10001, 0xbe80486e, - 0x8b6eff6d, 0x01ff0000, - 0xbfa20005, 0x8c78ff78, - 0x00002000, 0x80ec886c, - 0x82ed806d, 0xbfa00005, - 0x8b6eff6d, 0x01000000, - 0xbfa20002, 0x806c846c, - 0x826d806d, 0x8b6dff6d, - 0x0000ffff, 0x8bfe7e7e, - 0x8bea6a6a, 0xb978f802, - 0xbe804a6c, 0x8b6dff6d, - 0x0000ffff, 0xbefa0080, - 0xb97a0283, 0xbeee007e, - 0xbeef007f, 0xbefe0180, - 0xbefe4d84, 0xbf89fc07, - 0x8b7aff7f, 0x04000000, - 0x847a857a, 0x8c6d7a6d, - 0xbefa007e, 0x8b7bff7f, - 0x0000ffff, 0xbefe00c1, - 0xbeff00c1, 0xdca6c000, - 0x007a0000, 0x7e000280, - 0xbefe007a, 0xbeff007b, - 0xb8fb02dc, 0x847b997b, - 0xb8fa3b05, 0x807a817a, - 0xbf0d997b, 0xbfa20002, - 0x847a897a, 0xbfa00001, - 0x847a8a7a, 0xb8fb1e06, - 0x847b8a7b, 0x807a7b7a, + 0xbfa20041, 0xbf830010, + 0xb8fbf803, 0xbfa0fffa, + 0x8b6eff7b, 0x00000900, + 0xbfa20015, 0x8b6eff7b, + 0x000071ff, 0xbfa10008, + 0x8b6fff7b, 0x00007080, + 0xbfa10001, 0xbeee1287, + 0xb8eff801, 0x846e8c6e, + 0x8b6e6f6e, 0xbfa2000a, + 0x8b6eff6d, 0x00ff0000, + 0xbfa20007, 0xb8eef801, + 0x8b6eff6e, 0x00000800, + 0xbfa20003, 0x8b6eff7b, + 0x00000400, 0xbfa20026, + 0xbefa4d82, 0xbf89fc07, + 0x84fa887a, 0xf4005bbd, + 0xf8000010, 0xbf89fc07, + 0x846e976e, 0x9177ff77, + 0x00800000, 0x8c776e77, + 0xf4045bbd, 0xf8000000, + 0xbf89fc07, 0xf4045ebd, + 0xf8000008, 0xbf89fc07, + 0x8bee6e6e, 0xbfa10001, + 0xbe80486e, 0x8b6eff6d, + 0x01ff0000, 0xbfa20005, + 0x8c78ff78, 0x00002000, + 0x80ec886c, 0x82ed806d, + 0xbfa00005, 0x8b6eff6d, + 0x01000000, 0xbfa20002, + 0x806c846c, 0x826d806d, + 0x8b6dff6d, 0x0000ffff, + 0x8bfe7e7e, 0x8bea6a6a, + 0xb978f802, 0xbe804a6c, + 0x8b6dff6d, 0x0000ffff, + 0xbefa0080, 0xb97a0283, + 0xbeee007e, 0xbeef007f, + 0xbefe0180, 0xbefe4d84, + 0xbf89fc07, 0x8b7aff7f, + 0x04000000, 0x847a857a, + 0x8c6d7a6d, 0xbefa007e, 0x8b7bff7f, 0x0000ffff, - 0x807aff7a, 0x00000200, - 0x807a7e7a, 0x827b807b, - 0xd7610000, 0x00010870, - 0xd7610000, 0x00010a71, - 0xd7610000, 0x00010c72, - 0xd7610000, 0x00010e73, - 0xd7610000, 0x00011074, - 0xd7610000, 0x00011275, - 0xd7610000, 0x00011476, - 0xd7610000, 0x00011677, - 0xd7610000, 0x00011a79, - 0xd7610000, 0x00011c7e, - 0xd7610000, 0x00011e7f, - 0xbefe00ff, 0x00003fff, - 0xbeff0080, 0xdca6c040, - 0x007a0000, 0xd760007a, - 0x00011d00, 0xd760007b, - 0x00011f00, 0xbefe007a, - 0xbeff007b, 0xbef4007e, - 0x8b75ff7f, 0x0000ffff, - 0x8c75ff75, 0x00040000, - 0xbef60080, 0xbef700ff, - 0x10807fac, 0xbef1007d, - 0xbef00080, 0xb8f302dc, - 0x84739973, 0xbefe00c1, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00002, - 0xbeff00c1, 0xbfa00009, + 0xbefe00c1, 0xbeff00c1, + 0xdca6c000, 0x007a0000, + 0x7e000280, 0xbefe007a, + 0xbeff007b, 0xb8fb02dc, + 0x847b997b, 0xb8fa3b05, + 0x807a817a, 0xbf0d997b, + 0xbfa20002, 0x847a897a, + 0xbfa00001, 0x847a8a7a, + 0xb8fb1e06, 0x847b8a7b, + 0x807a7b7a, 0x8b7bff7f, + 0x0000ffff, 0x807aff7a, + 0x00000200, 0x807a7e7a, + 0x827b807b, 0xd7610000, + 0x00010870, 0xd7610000, + 0x00010a71, 0xd7610000, + 0x00010c72, 0xd7610000, + 0x00010e73, 0xd7610000, + 0x00011074, 0xd7610000, + 0x00011275, 0xd7610000, + 0x00011476, 0xd7610000, + 0x00011677, 0xd7610000, + 0x00011a79, 0xd7610000, + 0x00011c7e, 0xd7610000, + 0x00011e7f, 0xbefe00ff, + 0x00003fff, 0xbeff0080, + 0xdca6c040, 0x007a0000, + 0xd760007a, 0x00011d00, + 0xd760007b, 0x00011f00, + 0xbefe007a, 0xbeff007b, + 0xbef4007e, 0x8b75ff7f, + 0x0000ffff, 0x8c75ff75, + 0x00040000, 0xbef60080, + 0xbef700ff, 0x10807fac, + 0xbef1007d, 0xbef00080, + 0xb8f302dc, 0x84739973, + 0xbefe00c1, 0x857d9973, + 0x8b7d817d, 0xbf06817d, + 0xbfa20002, 0xbeff0080, + 0xbfa00002, 0xbeff00c1, + 0xbfa00009, 0xbef600ff, + 0x01000000, 0xe0685080, + 0x701d0100, 0xe0685100, + 0x701d0200, 0xe0685180, + 0x701d0300, 0xbfa00008, 0xbef600ff, 0x01000000, - 0xe0685080, 0x701d0100, - 0xe0685100, 0x701d0200, - 0xe0685180, 0x701d0300, - 0xbfa00008, 0xbef600ff, - 0x01000000, 0xe0685100, - 0x701d0100, 0xe0685200, - 0x701d0200, 0xe0685300, - 0x701d0300, 0xb8f03b05, - 0x80708170, 0xbf0d9973, - 0xbfa20002, 0x84708970, - 0xbfa00001, 0x84708a70, - 0xb8fa1e06, 0x847a8a7a, - 0x80707a70, 0x8070ff70, - 0x00000200, 0xbef600ff, - 0x01000000, 0x7e000280, - 0x7e020280, 0x7e040280, - 0xbefd0080, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xd7610002, 0x0000fa6c, - 0x807d817d, 0x917aff6d, - 0x80000000, 0xd7610002, - 0x0000fa7a, 0x807d817d, - 0xd7610002, 0x0000fa6e, - 0x807d817d, 0xd7610002, - 0x0000fa6f, 0x807d817d, - 0xd7610002, 0x0000fa78, - 0x807d817d, 0xb8faf803, - 0xd7610002, 0x0000fa7a, - 0x807d817d, 0xd7610002, - 0x0000fa7b, 0x807d817d, - 0xb8f1f801, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xb8f1f814, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xb8f1f815, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xbefe00ff, 0x0000ffff, - 0xbeff0080, 0xe0685000, - 0x701d0200, 0xbefe00c1, + 0xe0685100, 0x701d0100, + 0xe0685200, 0x701d0200, + 0xe0685300, 0x701d0300, 0xb8f03b05, 0x80708170, 0xbf0d9973, 0xbfa20002, 0x84708970, 0xbfa00001, 0x84708a70, 0xb8fa1e06, 0x847a8a7a, 0x80707a70, + 0x8070ff70, 0x00000200, 0xbef600ff, 0x01000000, - 0xbef90080, 0xbefd0080, - 0xbf800000, 0xbe804100, - 0xbe824102, 0xbe844104, - 0xbe864106, 0xbe884108, - 0xbe8a410a, 0xbe8c410c, - 0xbe8e410e, 0xd7610002, - 0x0000f200, 0x80798179, - 0xd7610002, 0x0000f201, + 0x7e000280, 0x7e020280, + 0x7e040280, 0xbefd0080, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xd7610002, + 0x0000fa6c, 0x807d817d, + 0x917aff6d, 0x80000000, + 0xd7610002, 0x0000fa7a, + 0x807d817d, 0xd7610002, + 0x0000fa6e, 0x807d817d, + 0xd7610002, 0x0000fa6f, + 0x807d817d, 0xd7610002, + 0x0000fa78, 0x807d817d, + 0xb8faf803, 0xd7610002, + 0x0000fa7a, 0x807d817d, + 0xd7610002, 0x0000fa7b, + 0x807d817d, 0xb8f1f801, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f814, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f815, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xbefe00ff, + 0x0000ffff, 0xbeff0080, + 0xe0685000, 0x701d0200, + 0xbefe00c1, 0xb8f03b05, + 0x80708170, 0xbf0d9973, + 0xbfa20002, 0x84708970, + 0xbfa00001, 0x84708a70, + 0xb8fa1e06, 0x847a8a7a, + 0x80707a70, 0xbef600ff, + 0x01000000, 0xbef90080, + 0xbefd0080, 0xbf800000, + 0xbe804100, 0xbe824102, + 0xbe844104, 0xbe864106, + 0xbe884108, 0xbe8a410a, + 0xbe8c410c, 0xbe8e410e, + 0xd7610002, 0x0000f200, 0x80798179, 0xd7610002, - 0x0000f202, 0x80798179, - 0xd7610002, 0x0000f203, + 0x0000f201, 0x80798179, + 0xd7610002, 0x0000f202, 0x80798179, 0xd7610002, - 0x0000f204, 0x80798179, - 0xd7610002, 0x0000f205, + 0x0000f203, 0x80798179, + 0xd7610002, 0x0000f204, 0x80798179, 0xd7610002, - 0x0000f206, 0x80798179, - 0xd7610002, 0x0000f207, + 0x0000f205, 0x80798179, + 0xd7610002, 0x0000f206, 0x80798179, 0xd7610002, - 0x0000f208, 0x80798179, - 0xd7610002, 0x0000f209, + 0x0000f207, 0x80798179, + 0xd7610002, 0x0000f208, 0x80798179, 0xd7610002, - 0x0000f20a, 0x80798179, - 0xd7610002, 0x0000f20b, + 0x0000f209, 0x80798179, + 0xd7610002, 0x0000f20a, 0x80798179, 0xd7610002, - 0x0000f20c, 0x80798179, - 0xd7610002, 0x0000f20d, + 0x0000f20b, 0x80798179, + 0xd7610002, 0x0000f20c, 0x80798179, 0xd7610002, - 0x0000f20e, 0x80798179, - 0xd7610002, 0x0000f20f, - 0x80798179, 0xbf06a079, - 0xbfa10006, 0xe0685000, - 0x701d0200, 0x8070ff70, - 0x00000080, 0xbef90080, - 0x7e040280, 0x807d907d, - 0xbf0aff7d, 0x00000060, - 0xbfa2ffbc, 0xbe804100, - 0xbe824102, 0xbe844104, - 0xbe864106, 0xbe884108, - 0xbe8a410a, 0xd7610002, - 0x0000f200, 0x80798179, - 0xd7610002, 0x0000f201, + 0x0000f20d, 0x80798179, + 0xd7610002, 0x0000f20e, 0x80798179, 0xd7610002, - 0x0000f202, 0x80798179, - 0xd7610002, 0x0000f203, + 0x0000f20f, 0x80798179, + 0xbf06a079, 0xbfa10006, + 0xe0685000, 0x701d0200, + 0x8070ff70, 0x00000080, + 0xbef90080, 0x7e040280, + 0x807d907d, 0xbf0aff7d, + 0x00000060, 0xbfa2ffbc, + 0xbe804100, 0xbe824102, + 0xbe844104, 0xbe864106, + 0xbe884108, 0xbe8a410a, + 0xd7610002, 0x0000f200, 0x80798179, 0xd7610002, - 0x0000f204, 0x80798179, - 0xd7610002, 0x0000f205, + 0x0000f201, 0x80798179, + 0xd7610002, 0x0000f202, 0x80798179, 0xd7610002, - 0x0000f206, 0x80798179, - 0xd7610002, 0x0000f207, + 0x0000f203, 0x80798179, + 0xd7610002, 0x0000f204, 0x80798179, 0xd7610002, - 0x0000f208, 0x80798179, - 0xd7610002, 0x0000f209, + 0x0000f205, 0x80798179, + 0xd7610002, 0x0000f206, 0x80798179, 0xd7610002, - 0x0000f20a, 0x80798179, - 0xd7610002, 0x0000f20b, - 0x80798179, 0xe0685000, - 0x701d0200, 0xbefe00c1, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00001, - 0xbeff00c1, 0xb8fb4306, - 0x8b7bc17b, 0xbfa10044, - 0xbfbd0000, 0x8b7aff6d, - 0x80000000, 0xbfa10040, - 0x847b867b, 0x847b827b, - 0xbef6007b, 0xb8f03b05, - 0x80708170, 0xbf0d9973, - 0xbfa20002, 0x84708970, - 0xbfa00001, 0x84708a70, - 0xb8fa1e06, 0x847a8a7a, - 0x80707a70, 0x8070ff70, - 0x00000200, 0x8070ff70, - 0x00000080, 0xbef600ff, - 0x01000000, 0xd71f0000, - 0x000100c1, 0xd7200000, - 0x000200c1, 0x16000084, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbefd0080, - 0xbfa20012, 0xbe8300ff, - 0x00000080, 0xbf800000, - 0xbf800000, 0xbf800000, - 0xd8d80000, 0x01000000, - 0xbf890000, 0xe0685000, - 0x701d0100, 0x807d037d, - 0x80700370, 0xd5250000, - 0x0001ff00, 0x00000080, - 0xbf0a7b7d, 0xbfa2fff4, - 0xbfa00011, 0xbe8300ff, - 0x00000100, 0xbf800000, - 0xbf800000, 0xbf800000, - 0xd8d80000, 0x01000000, - 0xbf890000, 0xe0685000, - 0x701d0100, 0x807d037d, - 0x80700370, 0xd5250000, - 0x0001ff00, 0x00000100, - 0xbf0a7b7d, 0xbfa2fff4, + 0x0000f207, 0x80798179, + 0xd7610002, 0x0000f208, + 0x80798179, 0xd7610002, + 0x0000f209, 0x80798179, + 0xd7610002, 0x0000f20a, + 0x80798179, 0xd7610002, + 0x0000f20b, 0x80798179, + 0xe0685000, 0x701d0200, 0xbefe00c1, 0x857d9973, 0x8b7d817d, 0xbf06817d, - 0xbfa20004, 0xbef000ff, - 0x00000200, 0xbeff0080, - 0xbfa00003, 0xbef000ff, - 0x00000400, 0xbeff00c1, - 0xb8fb3b05, 0x807b817b, - 0x847b827b, 0x857d9973, + 0xbfa20002, 0xbeff0080, + 0xbfa00001, 0xbeff00c1, + 0xb8fb4306, 0x8b7bc17b, + 0xbfa10044, 0xbfbd0000, + 0x8b7aff6d, 0x80000000, + 0xbfa10040, 0x847b867b, + 0x847b827b, 0xbef6007b, + 0xb8f03b05, 0x80708170, + 0xbf0d9973, 0xbfa20002, + 0x84708970, 0xbfa00001, + 0x84708a70, 0xb8fa1e06, + 0x847a8a7a, 0x80707a70, + 0x8070ff70, 0x00000200, + 0x8070ff70, 0x00000080, + 0xbef600ff, 0x01000000, + 0xd71f0000, 0x000100c1, + 0xd7200000, 0x000200c1, + 0x16000084, 0x857d9973, 0x8b7d817d, 0xbf06817d, - 0xbfa20017, 0xbef600ff, - 0x01000000, 0xbefd0084, - 0xbf0a7b7d, 0xbfa10037, - 0x7e008700, 0x7e028701, - 0x7e048702, 0x7e068703, - 0xe0685000, 0x701d0000, - 0xe0685080, 0x701d0100, - 0xe0685100, 0x701d0200, - 0xe0685180, 0x701d0300, - 0x807d847d, 0x8070ff70, - 0x00000200, 0xbf0a7b7d, - 0xbfa2ffef, 0xbfa00025, + 0xbefd0080, 0xbfa20012, + 0xbe8300ff, 0x00000080, + 0xbf800000, 0xbf800000, + 0xbf800000, 0xd8d80000, + 0x01000000, 0xbf890000, + 0xe0685000, 0x701d0100, + 0x807d037d, 0x80700370, + 0xd5250000, 0x0001ff00, + 0x00000080, 0xbf0a7b7d, + 0xbfa2fff4, 0xbfa00011, + 0xbe8300ff, 0x00000100, + 0xbf800000, 0xbf800000, + 0xbf800000, 0xd8d80000, + 0x01000000, 0xbf890000, + 0xe0685000, 0x701d0100, + 0x807d037d, 0x80700370, + 0xd5250000, 0x0001ff00, + 0x00000100, 0xbf0a7b7d, + 0xbfa2fff4, 0xbefe00c1, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20004, + 0xbef000ff, 0x00000200, + 0xbeff0080, 0xbfa00003, + 0xbef000ff, 0x00000400, + 0xbeff00c1, 0xb8fb3b05, + 0x807b817b, 0x847b827b, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20017, 0xbef600ff, 0x01000000, 0xbefd0084, 0xbf0a7b7d, - 0xbfa10011, 0x7e008700, + 0xbfa10037, 0x7e008700, 0x7e028701, 0x7e048702, 0x7e068703, 0xe0685000, - 0x701d0000, 0xe0685100, - 0x701d0100, 0xe0685200, - 0x701d0200, 0xe0685300, + 0x701d0000, 0xe0685080, + 0x701d0100, 0xe0685100, + 0x701d0200, 0xe0685180, 0x701d0300, 0x807d847d, - 0x8070ff70, 0x00000400, + 0x8070ff70, 0x00000200, 0xbf0a7b7d, 0xbfa2ffef, - 0xb8fb1e06, 0x8b7bc17b, - 0xbfa1000c, 0x847b837b, - 0x807b7d7b, 0xbefe00c1, - 0xbeff0080, 0x7e008700, + 0xbfa00025, 0xbef600ff, + 0x01000000, 0xbefd0084, + 0xbf0a7b7d, 0xbfa10011, + 0x7e008700, 0x7e028701, + 0x7e048702, 0x7e068703, 0xe0685000, 0x701d0000, - 0x807d817d, 0x8070ff70, - 0x00000080, 0xbf0a7b7d, - 0xbfa2fff8, 0xbfa00146, - 0xbef4007e, 0x8b75ff7f, - 0x0000ffff, 0x8c75ff75, - 0x00040000, 0xbef60080, - 0xbef700ff, 0x10807fac, - 0xb8f202dc, 0x84729972, - 0x8b6eff7f, 0x04000000, - 0xbfa1003a, 0xbefe00c1, - 0x857d9972, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00001, - 0xbeff00c1, 0xb8ef4306, - 0x8b6fc16f, 0xbfa1002f, - 0x846f866f, 0x846f826f, - 0xbef6006f, 0xb8f83b05, - 0x80788178, 0xbf0d9972, - 0xbfa20002, 0x84788978, - 0xbfa00001, 0x84788a78, - 0xb8ee1e06, 0x846e8a6e, - 0x80786e78, 0x8078ff78, - 0x00000200, 0x8078ff78, - 0x00000080, 0xbef600ff, - 0x01000000, 0x857d9972, - 0x8b7d817d, 0xbf06817d, - 0xbefd0080, 0xbfa2000c, - 0xe0500000, 0x781d0000, - 0xbf8903f7, 0xdac00000, - 0x00000000, 0x807dff7d, - 0x00000080, 0x8078ff78, - 0x00000080, 0xbf0a6f7d, - 0xbfa2fff5, 0xbfa0000b, - 0xe0500000, 0x781d0000, - 0xbf8903f7, 0xdac00000, - 0x00000000, 0x807dff7d, - 0x00000100, 0x8078ff78, - 0x00000100, 0xbf0a6f7d, - 0xbfa2fff5, 0xbef80080, + 0xe0685100, 0x701d0100, + 0xe0685200, 0x701d0200, + 0xe0685300, 0x701d0300, + 0x807d847d, 0x8070ff70, + 0x00000400, 0xbf0a7b7d, + 0xbfa2ffef, 0xb8fb1e06, + 0x8b7bc17b, 0xbfa1000c, + 0x847b837b, 0x807b7d7b, + 0xbefe00c1, 0xbeff0080, + 0x7e008700, 0xe0685000, + 0x701d0000, 0x807d817d, + 0x8070ff70, 0x00000080, + 0xbf0a7b7d, 0xbfa2fff8, + 0xbfa00146, 0xbef4007e, + 0x8b75ff7f, 0x0000ffff, + 0x8c75ff75, 0x00040000, + 0xbef60080, 0xbef700ff, + 0x10807fac, 0xb8f202dc, + 0x84729972, 0x8b6eff7f, + 0x04000000, 0xbfa1003a, 0xbefe00c1, 0x857d9972, 0x8b7d817d, 0xbf06817d, 0xbfa20002, 0xbeff0080, 0xbfa00001, 0xbeff00c1, - 0xb8ef3b05, 0x806f816f, - 0x846f826f, 0x857d9972, - 0x8b7d817d, 0xbf06817d, - 0xbfa20024, 0xbef600ff, - 0x01000000, 0xbeee0078, + 0xb8ef4306, 0x8b6fc16f, + 0xbfa1002f, 0x846f866f, + 0x846f826f, 0xbef6006f, + 0xb8f83b05, 0x80788178, + 0xbf0d9972, 0xbfa20002, + 0x84788978, 0xbfa00001, + 0x84788a78, 0xb8ee1e06, + 0x846e8a6e, 0x80786e78, 0x8078ff78, 0x00000200, - 0xbefd0084, 0xbf0a6f7d, - 0xbfa10050, 0xe0505000, - 0x781d0000, 0xe0505080, - 0x781d0100, 0xe0505100, - 0x781d0200, 0xe0505180, - 0x781d0300, 0xbf8903f7, - 0x7e008500, 0x7e028501, - 0x7e048502, 0x7e068503, - 0x807d847d, 0x8078ff78, - 0x00000200, 0xbf0a6f7d, - 0xbfa2ffee, 0xe0505000, - 0x6e1d0000, 0xe0505080, - 0x6e1d0100, 0xe0505100, - 0x6e1d0200, 0xe0505180, - 0x6e1d0300, 0xbf8903f7, - 0xbfa00034, 0xbef600ff, - 0x01000000, 0xbeee0078, - 0x8078ff78, 0x00000400, - 0xbefd0084, 0xbf0a6f7d, - 0xbfa10012, 0xe0505000, - 0x781d0000, 0xe0505100, - 0x781d0100, 0xe0505200, - 0x781d0200, 0xe0505300, - 0x781d0300, 0xbf8903f7, - 0x7e008500, 0x7e028501, - 0x7e048502, 0x7e068503, - 0x807d847d, 0x8078ff78, - 0x00000400, 0xbf0a6f7d, - 0xbfa2ffee, 0xb8ef1e06, - 0x8b6fc16f, 0xbfa1000e, - 0x846f836f, 0x806f7d6f, - 0xbefe00c1, 0xbeff0080, + 0x8078ff78, 0x00000080, + 0xbef600ff, 0x01000000, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbefd0080, + 0xbfa2000c, 0xe0500000, + 0x781d0000, 0xbf8903f7, + 0xdac00000, 0x00000000, + 0x807dff7d, 0x00000080, + 0x8078ff78, 0x00000080, + 0xbf0a6f7d, 0xbfa2fff5, + 0xbfa0000b, 0xe0500000, + 0x781d0000, 0xbf8903f7, + 0xdac00000, 0x00000000, + 0x807dff7d, 0x00000100, + 0x8078ff78, 0x00000100, + 0xbf0a6f7d, 0xbfa2fff5, + 0xbef80080, 0xbefe00c1, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbfa20002, + 0xbeff0080, 0xbfa00001, + 0xbeff00c1, 0xb8ef3b05, + 0x806f816f, 0x846f826f, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbfa20024, + 0xbef600ff, 0x01000000, + 0xbeee0078, 0x8078ff78, + 0x00000200, 0xbefd0084, + 0xbf0a6f7d, 0xbfa10050, 0xe0505000, 0x781d0000, + 0xe0505080, 0x781d0100, + 0xe0505100, 0x781d0200, + 0xe0505180, 0x781d0300, 0xbf8903f7, 0x7e008500, - 0x807d817d, 0x8078ff78, - 0x00000080, 0xbf0a6f7d, - 0xbfa2fff7, 0xbeff00c1, + 0x7e028501, 0x7e048502, + 0x7e068503, 0x807d847d, + 0x8078ff78, 0x00000200, + 0xbf0a6f7d, 0xbfa2ffee, 0xe0505000, 0x6e1d0000, - 0xe0505100, 0x6e1d0100, - 0xe0505200, 0x6e1d0200, - 0xe0505300, 0x6e1d0300, - 0xbf8903f7, 0xb8f83b05, - 0x80788178, 0xbf0d9972, - 0xbfa20002, 0x84788978, - 0xbfa00001, 0x84788a78, - 0xb8ee1e06, 0x846e8a6e, - 0x80786e78, 0x8078ff78, - 0x00000200, 0x80f8ff78, - 0x00000050, 0xbef600ff, - 0x01000000, 0xbefd00ff, - 0x0000006c, 0x80f89078, - 0xf428403a, 0xf0000000, - 0xbf89fc07, 0x80fd847d, - 0xbf800000, 0xbe804300, - 0xbe824302, 0x80f8a078, - 0xf42c403a, 0xf0000000, - 0xbf89fc07, 0x80fd887d, - 0xbf800000, 0xbe804300, - 0xbe824302, 0xbe844304, - 0xbe864306, 0x80f8c078, - 0xf430403a, 0xf0000000, - 0xbf89fc07, 0x80fd907d, - 0xbf800000, 0xbe804300, - 0xbe824302, 0xbe844304, - 0xbe864306, 0xbe884308, - 0xbe8a430a, 0xbe8c430c, - 0xbe8e430e, 0xbf06807d, - 0xbfa1fff0, 0xb980f801, - 0x00000000, 0xbfbd0000, + 0xe0505080, 0x6e1d0100, + 0xe0505100, 0x6e1d0200, + 0xe0505180, 0x6e1d0300, + 0xbf8903f7, 0xbfa00034, + 0xbef600ff, 0x01000000, + 0xbeee0078, 0x8078ff78, + 0x00000400, 0xbefd0084, + 0xbf0a6f7d, 0xbfa10012, + 0xe0505000, 0x781d0000, + 0xe0505100, 0x781d0100, + 0xe0505200, 0x781d0200, + 0xe0505300, 0x781d0300, + 0xbf8903f7, 0x7e008500, + 0x7e028501, 0x7e048502, + 0x7e068503, 0x807d847d, + 0x8078ff78, 0x00000400, + 0xbf0a6f7d, 0xbfa2ffee, + 0xb8ef1e06, 0x8b6fc16f, + 0xbfa1000e, 0x846f836f, + 0x806f7d6f, 0xbefe00c1, + 0xbeff0080, 0xe0505000, + 0x781d0000, 0xbf8903f7, + 0x7e008500, 0x807d817d, + 0x8078ff78, 0x00000080, + 0xbf0a6f7d, 0xbfa2fff7, + 0xbeff00c1, 0xe0505000, + 0x6e1d0000, 0xe0505100, + 0x6e1d0100, 0xe0505200, + 0x6e1d0200, 0xe0505300, + 0x6e1d0300, 0xbf8903f7, 0xb8f83b05, 0x80788178, 0xbf0d9972, 0xbfa20002, 0x84788978, 0xbfa00001, 0x84788a78, 0xb8ee1e06, 0x846e8a6e, 0x80786e78, 0x8078ff78, 0x00000200, + 0x80f8ff78, 0x00000050, 0xbef600ff, 0x01000000, - 0xf4205bfa, 0xf0000000, - 0x80788478, 0xf4205b3a, + 0xbefd00ff, 0x0000006c, + 0x80f89078, 0xf428403a, + 0xf0000000, 0xbf89fc07, + 0x80fd847d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0x80f8a078, 0xf42c403a, + 0xf0000000, 0xbf89fc07, + 0x80fd887d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0xbe844304, 0xbe864306, + 0x80f8c078, 0xf430403a, + 0xf0000000, 0xbf89fc07, + 0x80fd907d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0xbe844304, 0xbe864306, + 0xbe884308, 0xbe8a430a, + 0xbe8c430c, 0xbe8e430e, + 0xbf06807d, 0xbfa1fff0, + 0xb980f801, 0x00000000, + 0xbfbd0000, 0xb8f83b05, + 0x80788178, 0xbf0d9972, + 0xbfa20002, 0x84788978, + 0xbfa00001, 0x84788a78, + 0xb8ee1e06, 0x846e8a6e, + 0x80786e78, 0x8078ff78, + 0x00000200, 0xbef600ff, + 0x01000000, 0xf4205bfa, 0xf0000000, 0x80788478, - 0xf4205b7a, 0xf0000000, - 0x80788478, 0xf4205c3a, + 0xf4205b3a, 0xf0000000, + 0x80788478, 0xf4205b7a, 0xf0000000, 0x80788478, - 0xf4205c7a, 0xf0000000, - 0x80788478, 0xf4205eba, + 0xf4205c3a, 0xf0000000, + 0x80788478, 0xf4205c7a, 0xf0000000, 0x80788478, - 0xf4205efa, 0xf0000000, - 0x80788478, 0xf4205e7a, + 0xf4205eba, 0xf0000000, + 0x80788478, 0xf4205efa, 0xf0000000, 0x80788478, - 0xf4205cfa, 0xf0000000, - 0x80788478, 0xf4205bba, + 0xf4205e7a, 0xf0000000, + 0x80788478, 0xf4205cfa, 0xf0000000, 0x80788478, - 0xbf89fc07, 0xb96ef814, 0xf4205bba, 0xf0000000, 0x80788478, 0xbf89fc07, - 0xb96ef815, 0xbefd006f, - 0xbefe0070, 0xbeff0071, - 0x8b6f7bff, 0x000003ff, - 0xb96f4803, 0x8b6f7bff, - 0xfffff800, 0x856f8b6f, - 0xb96fa2c3, 0xb973f801, - 0xb8ee3b05, 0x806e816e, - 0xbf0d9972, 0xbfa20002, - 0x846e896e, 0xbfa00001, - 0x846e8a6e, 0xb8ef1e06, - 0x846f8a6f, 0x806e6f6e, - 0x806eff6e, 0x00000200, - 0x806e746e, 0x826f8075, - 0x8b6fff6f, 0x0000ffff, - 0xf4085c37, 0xf8000050, - 0xf4085d37, 0xf8000060, - 0xf4005e77, 0xf8000074, - 0xbf89fc07, 0x8b6dff6d, - 0x0000ffff, 0x8bfe7e7e, - 0x8bea6a6a, 0xb8eef802, - 0xbf0d866e, 0xbfa20002, - 0xb97af802, 0xbe80486c, - 0xb97af802, 0xbe804a6c, - 0xbfb00000, 0xbf9f0000, + 0xb96ef814, 0xf4205bba, + 0xf0000000, 0x80788478, + 0xbf89fc07, 0xb96ef815, + 0xbefd006f, 0xbefe0070, + 0xbeff0071, 0x8b6f7bff, + 0x000003ff, 0xb96f4803, + 0x8b6f7bff, 0xfffff800, + 0x856f8b6f, 0xb96fa2c3, + 0xb973f801, 0xb8ee3b05, + 0x806e816e, 0xbf0d9972, + 0xbfa20002, 0x846e896e, + 0xbfa00001, 0x846e8a6e, + 0xb8ef1e06, 0x846f8a6f, + 0x806e6f6e, 0x806eff6e, + 0x00000200, 0x806e746e, + 0x826f8075, 0x8b6fff6f, + 0x0000ffff, 0xf4085c37, + 0xf8000050, 0xf4085d37, + 0xf8000060, 0xf4005e77, + 0xf8000074, 0xbf89fc07, + 0x8b6dff6d, 0x0000ffff, + 0x8bfe7e7e, 0x8bea6a6a, + 0xb8eef802, 0xbf0d866e, + 0xbfa20002, 0xb97af802, + 0xbe80486c, 0xb97af802, + 0xbe804a6c, 0xbfb00000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, + 0xbf9f0000, 0x00000000, }; diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm index 0f81670f6f9c6..8b92c33c2a7c5 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm @@ -186,6 +186,12 @@ L_SKIP_RESTORE: s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS) #if SW_SA_TRAP + // If ttmp1[30] is set then issue s_barrier to unblock dependent waves. + s_bitcmp1_b32 s_save_pc_hi, 30 + s_cbranch_scc0 L_TRAP_NO_BARRIER + s_barrier + +L_TRAP_NO_BARRIER: // If ttmp1[31] is set then trap may occur early. // Spin wait until SAVECTX exception is raised. s_bitcmp1_b32 s_save_pc_hi, 31 -- GitLab From 711f8c3fb3db61897080468586b970c87c61d9e4 Mon Sep 17 00:00:00 2001 From: Luiz Augusto von Dentz Date: Mon, 31 Oct 2022 16:10:32 -0700 Subject: [PATCH 0949/1927] Bluetooth: L2CAP: Fix accepting connection request for invalid SPSM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Bluetooth spec states that the valid range for SPSM is from 0x0001-0x00ff so it is invalid to accept values outside of this range: BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 3, Part A page 1059: Table 4.15: L2CAP_LE_CREDIT_BASED_CONNECTION_REQ SPSM ranges CVE: CVE-2022-42896 CC: stable@vger.kernel.org Reported-by: Tamás Koczka Signed-off-by: Luiz Augusto von Dentz Reviewed-by: Tedd Ho-Jeong An --- net/bluetooth/l2cap_core.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 1fbe087d6ae4e..3eee915fb2456 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -5813,6 +5813,19 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, BT_DBG("psm 0x%2.2x scid 0x%4.4x mtu %u mps %u", __le16_to_cpu(psm), scid, mtu, mps); + /* BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 3, Part A + * page 1059: + * + * Valid range: 0x0001-0x00ff + * + * Table 4.15: L2CAP_LE_CREDIT_BASED_CONNECTION_REQ SPSM ranges + */ + if (!psm || __le16_to_cpu(psm) > L2CAP_PSM_LE_DYN_END) { + result = L2CAP_CR_LE_BAD_PSM; + chan = NULL; + goto response; + } + /* Check if we have socket listening on psm */ pchan = l2cap_global_chan_by_psm(BT_LISTEN, psm, &conn->hcon->src, &conn->hcon->dst, LE_LINK); @@ -6001,6 +6014,18 @@ static inline int l2cap_ecred_conn_req(struct l2cap_conn *conn, psm = req->psm; + /* BLUETOOTH CORE SPECIFICATION Version 5.3 | Vol 3, Part A + * page 1059: + * + * Valid range: 0x0001-0x00ff + * + * Table 4.15: L2CAP_LE_CREDIT_BASED_CONNECTION_REQ SPSM ranges + */ + if (!psm || __le16_to_cpu(psm) > L2CAP_PSM_LE_DYN_END) { + result = L2CAP_CR_LE_BAD_PSM; + goto response; + } + BT_DBG("psm 0x%2.2x mtu %u mps %u", __le16_to_cpu(psm), mtu, mps); memset(&pdu, 0, sizeof(pdu)); -- GitLab From f937b758a188d6fd328a81367087eddbb2fce50f Mon Sep 17 00:00:00 2001 From: Luiz Augusto von Dentz Date: Mon, 31 Oct 2022 16:10:33 -0700 Subject: [PATCH 0950/1927] Bluetooth: L2CAP: Fix l2cap_global_chan_by_psm l2cap_global_chan_by_psm shall not return fixed channels as they are not meant to be connected by (S)PSM. Signed-off-by: Luiz Augusto von Dentz Reviewed-by: Tedd Ho-Jeong An --- net/bluetooth/l2cap_core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 3eee915fb2456..4d8a1d862f6bb 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -1990,7 +1990,7 @@ static struct l2cap_chan *l2cap_global_chan_by_psm(int state, __le16 psm, if (link_type == LE_LINK && c->src_type == BDADDR_BREDR) continue; - if (c->psm == psm) { + if (c->chan_type != L2CAP_CHAN_FIXED && c->psm == psm) { int src_match, dst_match; int src_any, dst_any; -- GitLab From b1a2cd50c0357f243b7435a732b4e62ba3157a2e Mon Sep 17 00:00:00 2001 From: Luiz Augusto von Dentz Date: Mon, 31 Oct 2022 16:10:52 -0700 Subject: [PATCH 0951/1927] Bluetooth: L2CAP: Fix attempting to access uninitialized memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On l2cap_parse_conf_req the variable efs is only initialized if remote_efs has been set. CVE: CVE-2022-42895 CC: stable@vger.kernel.org Reported-by: Tamás Koczka Signed-off-by: Luiz Augusto von Dentz Reviewed-by: Tedd Ho-Jeong An --- net/bluetooth/l2cap_core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index 4d8a1d862f6bb..9c24947aa41ef 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -3764,7 +3764,8 @@ static int l2cap_parse_conf_req(struct l2cap_chan *chan, void *data, size_t data l2cap_add_conf_opt(&ptr, L2CAP_CONF_RFC, sizeof(rfc), (unsigned long) &rfc, endptr - ptr); - if (test_bit(FLAG_EFS_ENABLE, &chan->flags)) { + if (remote_efs && + test_bit(FLAG_EFS_ENABLE, &chan->flags)) { chan->remote_id = efs.id; chan->remote_stype = efs.stype; chan->remote_msdu = le16_to_cpu(efs.msdu); -- GitLab From 038ee49fef18710bedd38b531d173ccd746b2d8d Mon Sep 17 00:00:00 2001 From: Lukas Wunner Date: Tue, 27 Sep 2022 13:52:34 +0200 Subject: [PATCH 0952/1927] serial: 8250: 8250_omap: Avoid RS485 RTS glitch on ->set_termios() RS485-enabled UART ports on TI Sitara SoCs with active-low polarity exhibit a Transmit Enable glitch on ->set_termios(): omap8250_restore_regs(), which is called from omap_8250_set_termios(), sets the TCRTLR bit in the MCR register and clears all other bits, including RTS. If RTS uses active-low polarity, it is now asserted for no reason. The TCRTLR bit is subsequently cleared by writing up->mcr to the MCR register. That variable is always zero, so the RTS bit is still cleared (incorrectly so if RTS is active-high). (up->mcr is not, as one might think, a cache of the MCR register's current value. Rather, it only caches a single bit of that register, the AFE bit. And it only does so if the UART supports the AFE bit, which OMAP does not. For details see serial8250_do_set_termios() and serial8250_do_set_mctrl().) Finally at the end of omap8250_restore_regs(), the MCR register is restored (and RTS deasserted) by a call to up->port.ops->set_mctrl() (which equals serial8250_set_mctrl()) and serial8250_em485_stop_tx(). So there's an RTS glitch between setting TCRTLR and calling serial8250_em485_stop_tx(). Avoid by using a read-modify-write when setting TCRTLR. While at it, drop a redundant initialization of up->mcr. As explained above, the variable isn't used by the driver and it is already initialized to zero because it is part of the static struct serial8250_ports[] declared in 8250_core.c. (Static structs are initialized to zero per section 6.7.8 nr. 10 of the C99 standard.) Cc: Jan Kiszka Cc: Su Bao Cheng Tested-by: Matthias Schiffer Signed-off-by: Lukas Wunner Link: https://lore.kernel.org/r/6554b0241a2c7fd50f32576fdbafed96709e11e8.1664278942.git.lukas@wunner.de Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_omap.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 41b8c6b27136a..68f5a167377fd 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -292,6 +292,7 @@ static void omap8250_restore_regs(struct uart_8250_port *up) { struct omap8250_priv *priv = up->port.private_data; struct uart_8250_dma *dma = up->dma; + u8 mcr = serial8250_in_MCR(up); if (dma && dma->tx_running) { /* @@ -308,7 +309,7 @@ static void omap8250_restore_regs(struct uart_8250_port *up) serial_out(up, UART_EFR, UART_EFR_ECB); serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A); - serial8250_out_MCR(up, UART_MCR_TCRTLR); + serial8250_out_MCR(up, mcr | UART_MCR_TCRTLR); serial_out(up, UART_FCR, up->fcr); omap8250_update_scr(up, priv); @@ -324,7 +325,8 @@ static void omap8250_restore_regs(struct uart_8250_port *up) serial_out(up, UART_LCR, 0); /* drop TCR + TLR access, we setup XON/XOFF later */ - serial8250_out_MCR(up, up->mcr); + serial8250_out_MCR(up, mcr); + serial_out(up, UART_IER, up->ier); serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B); @@ -669,7 +671,6 @@ static int omap_8250_startup(struct uart_port *port) pm_runtime_get_sync(port->dev); - up->mcr = 0; serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT); serial_out(up, UART_LCR, UART_LCR_WLEN8); -- GitLab From 93810191f5d23652c0b8a1a9b3a4a89d6fd5063e Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Mon, 24 Oct 2022 09:36:13 +0300 Subject: [PATCH 0953/1927] serial: 8250: omap: Fix missing PM runtime calls for omap8250_set_mctrl() There are cases where omap8250_set_mctrl() may get called after the UART has already autoidled causing an asynchronous external abort. This can happen on ttyport_open(): mem_serial_in from omap8250_set_mctrl+0x38/0xa0 omap8250_set_mctrl from uart_update_mctrl+0x4c/0x58 uart_update_mctrl from uart_dtr_rts+0x60/0xa8 uart_dtr_rts from tty_port_block_til_ready+0xd0/0x2a8 tty_port_block_til_ready from uart_open+0x14/0x1c uart_open from ttyport_open+0x64/0x148 And on ttyport_close(): omap8250_set_mctrl from uart_update_mctrl+0x3c/0x48 uart_update_mctrl from uart_dtr_rts+0x54/0x9c uart_dtr_rts from tty_port_shutdown+0x78/0x9c tty_port_shutdown from tty_port_close+0x3c/0x74 tty_port_close from ttyport_close+0x40/0x58 It can also happen on disassociate_ctty() calling uart_shutdown() that ends up calling omap8250_set_mctrl(). Let's fix the issue by adding missing PM runtime calls to omap8250_set_mctrl(). To do this, we need to add __omap8250_set_mctrl() that can be called from both omap8250_set_mctrl(), and from runtime PM resume path when restoring the registers. Fixes: 61929cf0169d ("tty: serial: Add 8250-core based omap driver") Reported-by: Merlijn Wajer Reported-by: Romain Naour Reported-by: Ivaylo Dimitrov Tested-by: Ivaylo Dimitrov Signed-off-by: Tony Lindgren Depends-on: dd8088d5a896 ("PM: runtime: Add pm_runtime_resume_and_get to deal with usage counter") Link: https://lore.kernel.org/r/20221024063613.25943-1-tony@atomide.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_omap.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 68f5a167377fd..7bfa870941255 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -157,7 +157,11 @@ static u32 uart_read(struct uart_8250_port *up, u32 reg) return readl(up->port.membase + (reg << up->port.regshift)); } -static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl) +/* + * Called on runtime PM resume path from omap8250_restore_regs(), and + * omap8250_set_mctrl(). + */ +static void __omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl) { struct uart_8250_port *up = up_to_u8250p(port); struct omap8250_priv *priv = up->port.private_data; @@ -181,6 +185,20 @@ static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl) } } +static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl) +{ + int err; + + err = pm_runtime_resume_and_get(port->dev); + if (err) + return; + + __omap8250_set_mctrl(port, mctrl); + + pm_runtime_mark_last_busy(port->dev); + pm_runtime_put_autosuspend(port->dev); +} + /* * Work Around for Errata i202 (2430, 3430, 3630, 4430 and 4460) * The access to uart register after MDR1 Access @@ -343,7 +361,7 @@ static void omap8250_restore_regs(struct uart_8250_port *up) omap8250_update_mdr1(up, priv); - up->port.ops->set_mctrl(&up->port, up->port.mctrl); + __omap8250_set_mctrl(&up->port, up->port.mctrl); if (up->port.rs485.flags & SER_RS485_ENABLED) serial8250_em485_stop_tx(up); -- GitLab From e828e56684d61b17317e0cfdef83791fa61cb76b Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Thu, 13 Oct 2022 13:23:39 +0200 Subject: [PATCH 0954/1927] serial: 8250_omap: remove wait loop from Errata i202 workaround MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We were occasionally seeing the "Errata i202: timedout" on an AM335x board when repeatedly opening and closing a UART connected to an active sender. As new input may arrive at any time, it is possible to miss the "RX FIFO empty" condition, forcing the loop to wait until it times out. Nothing in the i202 Advisory states that such a wait is even necessary; other FIFO clear functions like serial8250_clear_fifos() do not wait either. For this reason, it seems safe to remove the wait, fixing the mentioned issue. Fixes: 61929cf0169d ("tty: serial: Add 8250-core based omap driver") Reviewed-by: Ilpo Järvinen Signed-off-by: Matthias Schiffer Link: https://lore.kernel.org/r/20221013112339.2540767-1-matthias.schiffer@ew.tq-group.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_omap.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 7bfa870941255..766c57d50b4c6 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -211,27 +211,10 @@ static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl) static void omap_8250_mdr1_errataset(struct uart_8250_port *up, struct omap8250_priv *priv) { - u8 timeout = 255; - serial_out(up, UART_OMAP_MDR1, priv->mdr1); udelay(2); serial_out(up, UART_FCR, up->fcr | UART_FCR_CLEAR_XMIT | UART_FCR_CLEAR_RCVR); - /* - * Wait for FIFO to empty: when empty, RX_FIFO_E bit is 0 and - * TX_FIFO_E bit is 1. - */ - while (UART_LSR_THRE != (serial_in(up, UART_LSR) & - (UART_LSR_THRE | UART_LSR_DR))) { - timeout--; - if (!timeout) { - /* Should *never* happen. we warn and carry on */ - dev_crit(up->port.dev, "Errata i202: timedout %x\n", - serial_in(up, UART_LSR)); - break; - } - udelay(1); - } } static void omap_8250_get_divisor(struct uart_port *port, unsigned int baud, -- GitLab From e3f0c638f428fd66b5871154b62706772045f91a Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Fri, 28 Oct 2022 13:58:13 +0300 Subject: [PATCH 0955/1927] serial: 8250: omap: Fix unpaired pm_runtime_put_sync() in omap8250_remove() On remove, we get an error for "Runtime PM usage count underflow!". I guess this driver is mostly built-in, and this issue has gone unnoticed for a while. Somehow I did not catch this issue with my earlier fix done with commit 4e0f5cc65098 ("serial: 8250_omap: Fix probe and remove for PM runtime"). Fixes: 4e0f5cc65098 ("serial: 8250_omap: Fix probe and remove for PM runtime") Signed-off-by: Tony Lindgren Depends-on: dd8088d5a896 ("PM: runtime: Add pm_runtime_resume_and_get to deal with usage counter") Link: https://lore.kernel.org/r/20221028105813.54290-1-tony@atomide.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_omap.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index 766c57d50b4c6..c6ec6eb7ee216 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1460,6 +1460,11 @@ static int omap8250_probe(struct platform_device *pdev) static int omap8250_remove(struct platform_device *pdev) { struct omap8250_priv *priv = platform_get_drvdata(pdev); + int err; + + err = pm_runtime_resume_and_get(&pdev->dev); + if (err) + return err; pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); -- GitLab From d0b68629bd2fb61e0171a62f2e8da3db322f5cf6 Mon Sep 17 00:00:00 2001 From: Tony Lindgren Date: Fri, 28 Oct 2022 14:00:44 +0300 Subject: [PATCH 0956/1927] serial: 8250: omap: Flush PM QOS work on remove Rebinding 8250_omap in a loop will at some point produce a warning for kernel/power/qos.c:296 cpu_latency_qos_update_request() with error "cpu_latency_qos_update_request called for unknown object". Let's flush the possibly pending PM QOS work scheduled from omap8250_runtime_suspend() before we disable runtime PM. Fixes: 61929cf0169d ("tty: serial: Add 8250-core based omap driver") Signed-off-by: Tony Lindgren Link: https://lore.kernel.org/r/20221028110044.54719-1-tony@atomide.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_omap.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/tty/serial/8250/8250_omap.c b/drivers/tty/serial/8250/8250_omap.c index c6ec6eb7ee216..3f33014022f0e 100644 --- a/drivers/tty/serial/8250/8250_omap.c +++ b/drivers/tty/serial/8250/8250_omap.c @@ -1468,6 +1468,7 @@ static int omap8250_remove(struct platform_device *pdev) pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_put_sync(&pdev->dev); + flush_work(&priv->qos_work); pm_runtime_disable(&pdev->dev); serial8250_unregister_port(priv->line); cpu_latency_qos_remove_request(&priv->pm_qos_request); -- GitLab From 76bad3f88750f8cc465c489e6846249e0bc3d8f5 Mon Sep 17 00:00:00 2001 From: Sherry Sun Date: Mon, 24 Oct 2022 16:58:44 +0800 Subject: [PATCH 0957/1927] tty: serial: fsl_lpuart: don't break the on-going transfer when global reset lpuart_global_reset() shouldn't break the on-going transmit engine, need to recover the on-going data transfer after reset. This can help earlycon here, since commit 60f361722ad2 ("serial: fsl_lpuart: Reset prior to registration") moved lpuart_global_reset() before uart_add_one_port(), earlycon is writing during global reset, as global reset will disable the TX and clear the baud rate register, which caused the earlycon cannot work any more after reset, needs to restore the baud rate and re-enable the transmitter to recover the earlycon write. Also move the lpuart_global_reset() down, then we can reuse the lpuart32_tx_empty() without declaration. Fixes: bd5305dcabbc ("tty: serial: fsl_lpuart: do software reset for imx7ulp and imx8qxp") Signed-off-by: Sherry Sun Link: https://lore.kernel.org/r/20221024085844.22786-1-sherry.sun@nxp.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/fsl_lpuart.c | 76 +++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c index 67fa113f77d47..888e01fbd9c5f 100644 --- a/drivers/tty/serial/fsl_lpuart.c +++ b/drivers/tty/serial/fsl_lpuart.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -404,33 +405,6 @@ static unsigned int lpuart_get_baud_clk_rate(struct lpuart_port *sport) #define lpuart_enable_clks(x) __lpuart_enable_clks(x, true) #define lpuart_disable_clks(x) __lpuart_enable_clks(x, false) -static int lpuart_global_reset(struct lpuart_port *sport) -{ - struct uart_port *port = &sport->port; - void __iomem *global_addr; - int ret; - - if (uart_console(port)) - return 0; - - ret = clk_prepare_enable(sport->ipg_clk); - if (ret) { - dev_err(sport->port.dev, "failed to enable uart ipg clk: %d\n", ret); - return ret; - } - - if (is_imx7ulp_lpuart(sport) || is_imx8qxp_lpuart(sport)) { - global_addr = port->membase + UART_GLOBAL - IMX_REG_OFF; - writel(UART_GLOBAL_RST, global_addr); - usleep_range(GLOBAL_RST_MIN_US, GLOBAL_RST_MAX_US); - writel(0, global_addr); - usleep_range(GLOBAL_RST_MIN_US, GLOBAL_RST_MAX_US); - } - - clk_disable_unprepare(sport->ipg_clk); - return 0; -} - static void lpuart_stop_tx(struct uart_port *port) { unsigned char temp; @@ -2636,6 +2610,54 @@ static const struct serial_rs485 lpuart_rs485_supported = { /* delay_rts_* and RX_DURING_TX are not supported */ }; +static int lpuart_global_reset(struct lpuart_port *sport) +{ + struct uart_port *port = &sport->port; + void __iomem *global_addr; + unsigned long ctrl, bd; + unsigned int val = 0; + int ret; + + ret = clk_prepare_enable(sport->ipg_clk); + if (ret) { + dev_err(sport->port.dev, "failed to enable uart ipg clk: %d\n", ret); + return ret; + } + + if (is_imx7ulp_lpuart(sport) || is_imx8qxp_lpuart(sport)) { + /* + * If the transmitter is used by earlycon, wait for transmit engine to + * complete and then reset. + */ + ctrl = lpuart32_read(port, UARTCTRL); + if (ctrl & UARTCTRL_TE) { + bd = lpuart32_read(&sport->port, UARTBAUD); + if (read_poll_timeout(lpuart32_tx_empty, val, val, 1, 100000, false, + port)) { + dev_warn(sport->port.dev, + "timeout waiting for transmit engine to complete\n"); + clk_disable_unprepare(sport->ipg_clk); + return 0; + } + } + + global_addr = port->membase + UART_GLOBAL - IMX_REG_OFF; + writel(UART_GLOBAL_RST, global_addr); + usleep_range(GLOBAL_RST_MIN_US, GLOBAL_RST_MAX_US); + writel(0, global_addr); + usleep_range(GLOBAL_RST_MIN_US, GLOBAL_RST_MAX_US); + + /* Recover the transmitter for earlycon. */ + if (ctrl & UARTCTRL_TE) { + lpuart32_write(port, bd, UARTBAUD); + lpuart32_write(port, ctrl, UARTCTRL); + } + } + + clk_disable_unprepare(sport->ipg_clk); + return 0; +} + static int lpuart_probe(struct platform_device *pdev) { const struct lpuart_soc_data *sdata = of_device_get_match_data(&pdev->dev); -- GitLab From 4561d8008a467cb05ac632a215391d6b787f40aa Mon Sep 17 00:00:00 2001 From: Shawn Guo Date: Wed, 12 Oct 2022 20:13:53 +0800 Subject: [PATCH 0958/1927] serial: imx: Add missing .thaw_noirq hook The following warning is seen with non-console UART instance when system hibernates. [ 37.371969] ------------[ cut here ]------------ [ 37.376599] uart3_root_clk already disabled [ 37.380810] WARNING: CPU: 0 PID: 296 at drivers/clk/clk.c:952 clk_core_disable+0xa4/0xb0 ... [ 37.506986] Call trace: [ 37.509432] clk_core_disable+0xa4/0xb0 [ 37.513270] clk_disable+0x34/0x50 [ 37.516672] imx_uart_thaw+0x38/0x5c [ 37.520250] platform_pm_thaw+0x30/0x6c [ 37.524089] dpm_run_callback.constprop.0+0x3c/0xd4 [ 37.528972] device_resume+0x7c/0x160 [ 37.532633] dpm_resume+0xe8/0x230 [ 37.536036] hibernation_snapshot+0x288/0x430 [ 37.540397] hibernate+0x10c/0x2e0 [ 37.543798] state_store+0xc4/0xd0 [ 37.547203] kobj_attr_store+0x1c/0x30 [ 37.550953] sysfs_kf_write+0x48/0x60 [ 37.554619] kernfs_fop_write_iter+0x118/0x1ac [ 37.559063] new_sync_write+0xe8/0x184 [ 37.562812] vfs_write+0x230/0x290 [ 37.566214] ksys_write+0x68/0xf4 [ 37.569529] __arm64_sys_write+0x20/0x2c [ 37.573452] invoke_syscall.constprop.0+0x50/0xf0 [ 37.578156] do_el0_svc+0x11c/0x150 [ 37.581648] el0_svc+0x30/0x140 [ 37.584792] el0t_64_sync_handler+0xe8/0xf0 [ 37.588976] el0t_64_sync+0x1a0/0x1a4 [ 37.592639] ---[ end trace 56e22eec54676d75 ]--- On hibernating, pm core calls into related hooks in sequence like: .freeze .freeze_noirq .thaw_noirq .thaw With .thaw_noirq hook being absent, the clock will be disabled in a unbalanced call which results the warning above. imx_uart_freeze() clk_prepare_enable() imx_uart_suspend_noirq() clk_disable() imx_uart_thaw clk_disable_unprepare() Adding the missing .thaw_noirq hook as imx_uart_resume_noirq() will have the call sequence corrected as below and thus fix the warning. imx_uart_freeze() clk_prepare_enable() imx_uart_suspend_noirq() clk_disable() imx_uart_resume_noirq() clk_enable() imx_uart_thaw clk_disable_unprepare() Fixes: 09df0b3464e5 ("serial: imx: fix endless loop during suspend") Reviewed-by: Martin Kaiser Signed-off-by: Shawn Guo Link: https://lore.kernel.org/r/20221012121353.2346280-1-shawn.guo@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/imx.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 05b432dc7a85c..aadda66405b47 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -2594,6 +2594,7 @@ static const struct dev_pm_ops imx_uart_pm_ops = { .suspend_noirq = imx_uart_suspend_noirq, .resume_noirq = imx_uart_resume_noirq, .freeze_noirq = imx_uart_suspend_noirq, + .thaw_noirq = imx_uart_resume_noirq, .restore_noirq = imx_uart_resume_noirq, .suspend = imx_uart_suspend, .resume = imx_uart_resume, -- GitLab From acdab4cb4ba7e5f94d2b422ebd7bf4bf68178fb2 Mon Sep 17 00:00:00 2001 From: Fedor Pchelkin Date: Sat, 8 Oct 2022 14:02:20 +0300 Subject: [PATCH 0959/1927] Revert "tty: n_gsm: avoid call of sleeping functions from atomic context" This reverts commit 902e02ea9385373ce4b142576eef41c642703955. The above commit is reverted as the usage of tx_mutex seems not to solve the problem described in 902e02ea9385 ("tty: n_gsm: avoid call of sleeping functions from atomic context") and just moves the bug to another place. Signed-off-by: Fedor Pchelkin Signed-off-by: Alexey Khoroshilov Reviewed-by: Daniel Starke Link: https://lore.kernel.org/r/20221008110221.13645-2-pchelkin@ispras.ru Signed-off-by: Greg Kroah-Hartman --- drivers/tty/n_gsm.c | 53 +++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index 5e516f5cac5a3..b66e08f3d006b 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -264,7 +264,7 @@ struct gsm_mux { bool constipated; /* Asked by remote to shut up */ bool has_devices; /* Devices were registered */ - struct mutex tx_mutex; + spinlock_t tx_lock; unsigned int tx_bytes; /* TX data outstanding */ #define TX_THRESH_HI 8192 #define TX_THRESH_LO 2048 @@ -700,6 +700,7 @@ static int gsm_send(struct gsm_mux *gsm, int addr, int cr, int control) struct gsm_msg *msg; u8 *dp; int ocr; + unsigned long flags; msg = gsm_data_alloc(gsm, addr, 0, control); if (!msg) @@ -721,10 +722,10 @@ static int gsm_send(struct gsm_mux *gsm, int addr, int cr, int control) gsm_print_packet("Q->", addr, cr, control, NULL, 0); - mutex_lock(&gsm->tx_mutex); + spin_lock_irqsave(&gsm->tx_lock, flags); list_add_tail(&msg->list, &gsm->tx_ctrl_list); gsm->tx_bytes += msg->len; - mutex_unlock(&gsm->tx_mutex); + spin_unlock_irqrestore(&gsm->tx_lock, flags); gsmld_write_trigger(gsm); return 0; @@ -749,7 +750,7 @@ static void gsm_dlci_clear_queues(struct gsm_mux *gsm, struct gsm_dlci *dlci) spin_unlock_irqrestore(&dlci->lock, flags); /* Clear data packets in MUX write queue */ - mutex_lock(&gsm->tx_mutex); + spin_lock_irqsave(&gsm->tx_lock, flags); list_for_each_entry_safe(msg, nmsg, &gsm->tx_data_list, list) { if (msg->addr != addr) continue; @@ -757,7 +758,7 @@ static void gsm_dlci_clear_queues(struct gsm_mux *gsm, struct gsm_dlci *dlci) list_del(&msg->list); kfree(msg); } - mutex_unlock(&gsm->tx_mutex); + spin_unlock_irqrestore(&gsm->tx_lock, flags); } /** @@ -1043,9 +1044,10 @@ static void __gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) static void gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) { - mutex_lock(&dlci->gsm->tx_mutex); + unsigned long flags; + spin_lock_irqsave(&dlci->gsm->tx_lock, flags); __gsm_data_queue(dlci, msg); - mutex_unlock(&dlci->gsm->tx_mutex); + spin_unlock_irqrestore(&dlci->gsm->tx_lock, flags); } /** @@ -1057,7 +1059,7 @@ static void gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) * is data. Keep to the MRU of the mux. This path handles the usual tty * interface which is a byte stream with optional modem data. * - * Caller must hold the tx_mutex of the mux. + * Caller must hold the tx_lock of the mux. */ static int gsm_dlci_data_output(struct gsm_mux *gsm, struct gsm_dlci *dlci) @@ -1117,7 +1119,7 @@ static int gsm_dlci_data_output(struct gsm_mux *gsm, struct gsm_dlci *dlci) * is data. Keep to the MRU of the mux. This path handles framed data * queued as skbuffs to the DLCI. * - * Caller must hold the tx_mutex of the mux. + * Caller must hold the tx_lock of the mux. */ static int gsm_dlci_data_output_framed(struct gsm_mux *gsm, @@ -1133,7 +1135,7 @@ static int gsm_dlci_data_output_framed(struct gsm_mux *gsm, if (dlci->adaption == 4) overhead = 1; - /* dlci->skb is locked by tx_mutex */ + /* dlci->skb is locked by tx_lock */ if (dlci->skb == NULL) { dlci->skb = skb_dequeue_tail(&dlci->skb_list); if (dlci->skb == NULL) @@ -1187,7 +1189,7 @@ static int gsm_dlci_data_output_framed(struct gsm_mux *gsm, * Push an empty frame in to the transmit queue to update the modem status * bits and to transmit an optional break. * - * Caller must hold the tx_mutex of the mux. + * Caller must hold the tx_lock of the mux. */ static int gsm_dlci_modem_output(struct gsm_mux *gsm, struct gsm_dlci *dlci, @@ -1301,12 +1303,13 @@ static int gsm_dlci_data_sweep(struct gsm_mux *gsm) static void gsm_dlci_data_kick(struct gsm_dlci *dlci) { + unsigned long flags; int sweep; if (dlci->constipated) return; - mutex_lock(&dlci->gsm->tx_mutex); + spin_lock_irqsave(&dlci->gsm->tx_lock, flags); /* If we have nothing running then we need to fire up */ sweep = (dlci->gsm->tx_bytes < TX_THRESH_LO); if (dlci->gsm->tx_bytes == 0) { @@ -1317,7 +1320,7 @@ static void gsm_dlci_data_kick(struct gsm_dlci *dlci) } if (sweep) gsm_dlci_data_sweep(dlci->gsm); - mutex_unlock(&dlci->gsm->tx_mutex); + spin_unlock_irqrestore(&dlci->gsm->tx_lock, flags); } /* @@ -2029,13 +2032,14 @@ static void gsm_dlci_command(struct gsm_dlci *dlci, const u8 *data, int len) static void gsm_kick_timeout(struct work_struct *work) { struct gsm_mux *gsm = container_of(work, struct gsm_mux, kick_timeout.work); + unsigned long flags; int sent = 0; - mutex_lock(&gsm->tx_mutex); + spin_lock_irqsave(&gsm->tx_lock, flags); /* If we have nothing running then we need to fire up */ if (gsm->tx_bytes < TX_THRESH_LO) sent = gsm_dlci_data_sweep(gsm); - mutex_unlock(&gsm->tx_mutex); + spin_unlock_irqrestore(&gsm->tx_lock, flags); if (sent && debug & DBG_DATA) pr_info("%s TX queue stalled\n", __func__); @@ -2565,7 +2569,6 @@ static void gsm_free_mux(struct gsm_mux *gsm) break; } } - mutex_destroy(&gsm->tx_mutex); mutex_destroy(&gsm->mutex); kfree(gsm->txframe); kfree(gsm->buf); @@ -2637,7 +2640,6 @@ static struct gsm_mux *gsm_alloc_mux(void) } spin_lock_init(&gsm->lock); mutex_init(&gsm->mutex); - mutex_init(&gsm->tx_mutex); kref_init(&gsm->ref); INIT_LIST_HEAD(&gsm->tx_ctrl_list); INIT_LIST_HEAD(&gsm->tx_data_list); @@ -2646,6 +2648,7 @@ static struct gsm_mux *gsm_alloc_mux(void) INIT_WORK(&gsm->tx_work, gsmld_write_task); init_waitqueue_head(&gsm->event); spin_lock_init(&gsm->control_lock); + spin_lock_init(&gsm->tx_lock); gsm->t1 = T1; gsm->t2 = T2; @@ -2670,7 +2673,6 @@ static struct gsm_mux *gsm_alloc_mux(void) } spin_unlock(&gsm_mux_lock); if (i == MAX_MUX) { - mutex_destroy(&gsm->tx_mutex); mutex_destroy(&gsm->mutex); kfree(gsm->txframe); kfree(gsm->buf); @@ -2826,16 +2828,17 @@ static void gsmld_write_trigger(struct gsm_mux *gsm) static void gsmld_write_task(struct work_struct *work) { struct gsm_mux *gsm = container_of(work, struct gsm_mux, tx_work); + unsigned long flags; int i, ret; /* All outstanding control channel and control messages and one data * frame is sent. */ ret = -ENODEV; - mutex_lock(&gsm->tx_mutex); + spin_lock_irqsave(&gsm->tx_lock, flags); if (gsm->tty) ret = gsm_data_kick(gsm); - mutex_unlock(&gsm->tx_mutex); + spin_unlock_irqrestore(&gsm->tx_lock, flags); if (ret >= 0) for (i = 0; i < NUM_DLCI; i++) @@ -3042,6 +3045,7 @@ static ssize_t gsmld_write(struct tty_struct *tty, struct file *file, const unsigned char *buf, size_t nr) { struct gsm_mux *gsm = tty->disc_data; + unsigned long flags; int space; int ret; @@ -3049,13 +3053,13 @@ static ssize_t gsmld_write(struct tty_struct *tty, struct file *file, return -ENODEV; ret = -ENOBUFS; - mutex_lock(&gsm->tx_mutex); + spin_lock_irqsave(&gsm->tx_lock, flags); space = tty_write_room(tty); if (space >= nr) ret = tty->ops->write(tty, buf, nr); else set_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); - mutex_unlock(&gsm->tx_mutex); + spin_unlock_irqrestore(&gsm->tx_lock, flags); return ret; } @@ -3352,13 +3356,14 @@ static struct tty_ldisc_ops tty_ldisc_packet = { static void gsm_modem_upd_via_data(struct gsm_dlci *dlci, u8 brk) { struct gsm_mux *gsm = dlci->gsm; + unsigned long flags; if (dlci->state != DLCI_OPEN || dlci->adaption != 2) return; - mutex_lock(&gsm->tx_mutex); + spin_lock_irqsave(&gsm->tx_lock, flags); gsm_dlci_modem_output(gsm, dlci, brk); - mutex_unlock(&gsm->tx_mutex); + spin_unlock_irqrestore(&gsm->tx_lock, flags); } /** -- GitLab From 15743ae50e04aa907131e3ae8d66e9a2964ea232 Mon Sep 17 00:00:00 2001 From: Fedor Pchelkin Date: Sat, 8 Oct 2022 14:02:21 +0300 Subject: [PATCH 0960/1927] Revert "tty: n_gsm: replace kicktimer with delayed_work" This reverts commit c9ab053e56ce13a949977398c8edc12e6c02fc95. The above commit is reverted as it was a prerequisite for tx_mutex introduction and tx_mutex has been removed as it does not correctly work in order to protect tx data. Signed-off-by: Fedor Pchelkin Signed-off-by: Alexey Khoroshilov Reviewed-by: Daniel Starke Link: https://lore.kernel.org/r/20221008110221.13645-3-pchelkin@ispras.ru Signed-off-by: Greg Kroah-Hartman --- drivers/tty/n_gsm.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index b66e08f3d006b..5c9f76073fcd8 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -272,7 +272,7 @@ struct gsm_mux { struct list_head tx_data_list; /* Pending data packets */ /* Control messages */ - struct delayed_work kick_timeout; /* Kick TX queuing on timeout */ + struct timer_list kick_timer; /* Kick TX queuing on timeout */ struct timer_list t2_timer; /* Retransmit timer for commands */ int cretries; /* Command retry counter */ struct gsm_control *pending_cmd;/* Our current pending command */ @@ -1029,7 +1029,7 @@ static void __gsm_data_queue(struct gsm_dlci *dlci, struct gsm_msg *msg) gsm->tx_bytes += msg->len; gsmld_write_trigger(gsm); - schedule_delayed_work(&gsm->kick_timeout, 10 * gsm->t1 * HZ / 100); + mod_timer(&gsm->kick_timer, jiffies + 10 * gsm->t1 * HZ / 100); } /** @@ -2022,16 +2022,16 @@ static void gsm_dlci_command(struct gsm_dlci *dlci, const u8 *data, int len) } /** - * gsm_kick_timeout - transmit if possible - * @work: work contained in our gsm object + * gsm_kick_timer - transmit if possible + * @t: timer contained in our gsm object * * Transmit data from DLCIs if the queue is empty. We can't rely on * a tty wakeup except when we filled the pipe so we need to fire off * new data ourselves in other cases. */ -static void gsm_kick_timeout(struct work_struct *work) +static void gsm_kick_timer(struct timer_list *t) { - struct gsm_mux *gsm = container_of(work, struct gsm_mux, kick_timeout.work); + struct gsm_mux *gsm = from_timer(gsm, t, kick_timer); unsigned long flags; int sent = 0; @@ -2496,7 +2496,7 @@ static void gsm_cleanup_mux(struct gsm_mux *gsm, bool disc) } /* Finish outstanding timers, making sure they are done */ - cancel_delayed_work_sync(&gsm->kick_timeout); + del_timer_sync(&gsm->kick_timer); del_timer_sync(&gsm->t2_timer); /* Finish writing to ldisc */ @@ -2643,7 +2643,7 @@ static struct gsm_mux *gsm_alloc_mux(void) kref_init(&gsm->ref); INIT_LIST_HEAD(&gsm->tx_ctrl_list); INIT_LIST_HEAD(&gsm->tx_data_list); - INIT_DELAYED_WORK(&gsm->kick_timeout, gsm_kick_timeout); + timer_setup(&gsm->kick_timer, gsm_kick_timer, 0); timer_setup(&gsm->t2_timer, gsm_control_retransmit, 0); INIT_WORK(&gsm->tx_work, gsmld_write_task); init_waitqueue_head(&gsm->event); -- GitLab From 7b7dfe4833c70a11cdfa51b38705103bd31eddaa Mon Sep 17 00:00:00 2001 From: Duoming Zhou Date: Sun, 2 Oct 2022 12:07:09 +0800 Subject: [PATCH 0961/1927] tty: n_gsm: fix sleep-in-atomic-context bug in gsm_control_send The function gsm_dlci_t1() is a timer handler that runs in an atomic context, but it calls "kzalloc(..., GFP_KERNEL)" that may sleep. As a result, the sleep-in-atomic-context bug will happen. The process is shown below: gsm_dlci_t1() gsm_dlci_open() gsm_modem_update() gsm_modem_upd_via_msc() gsm_control_send() kzalloc(sizeof(.., GFP_KERNEL) //may sleep This patch changes the gfp_t parameter of kzalloc() from GFP_KERNEL to GFP_ATOMIC in order to mitigate the bug. Fixes: e1eaea46bb40 ("tty: n_gsm line discipline") Signed-off-by: Duoming Zhou Link: https://lore.kernel.org/r/20221002040709.27849-1-duoming@zju.edu.cn Signed-off-by: Greg Kroah-Hartman --- drivers/tty/n_gsm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/tty/n_gsm.c b/drivers/tty/n_gsm.c index 5c9f76073fcd8..b6e0cc4571eac 100644 --- a/drivers/tty/n_gsm.c +++ b/drivers/tty/n_gsm.c @@ -1711,7 +1711,7 @@ static struct gsm_control *gsm_control_send(struct gsm_mux *gsm, unsigned int command, u8 *data, int clen) { struct gsm_control *ctrl = kzalloc(sizeof(struct gsm_control), - GFP_KERNEL); + GFP_ATOMIC); unsigned long flags; if (ctrl == NULL) return NULL; -- GitLab From 40e4eb324c59e11fcb927aa46742d28aba6ecb8a Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Mon, 31 Oct 2022 21:26:45 +0800 Subject: [PATCH 0962/1927] net: mdio: fix undefined behavior in bit shift for __mdiobus_register Shifting signed 32-bit value by 31 bits is undefined, so changing significant bit to unsigned. The UBSAN warning calltrace like below: UBSAN: shift-out-of-bounds in drivers/net/phy/mdio_bus.c:586:27 left shift of 1 by 31 places cannot be represented in type 'int' Call Trace: dump_stack_lvl+0x7d/0xa5 dump_stack+0x15/0x1b ubsan_epilogue+0xe/0x4e __ubsan_handle_shift_out_of_bounds+0x1e7/0x20c __mdiobus_register+0x49d/0x4e0 fixed_mdio_bus_init+0xd8/0x12d do_one_initcall+0x76/0x430 kernel_init_freeable+0x3b3/0x422 kernel_init+0x24/0x1e0 ret_from_fork+0x1f/0x30 Fixes: 4fd5f812c23c ("phylib: allow incremental scanning of an mii bus") Signed-off-by: Gaosheng Cui Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20221031132645.168421-1-cuigaosheng1@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/phy/mdio_bus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index f82090bdf7ab8..1cd604cd1fa1b 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -583,7 +583,7 @@ int __mdiobus_register(struct mii_bus *bus, struct module *owner) } for (i = 0; i < PHY_MAX_ADDR; i++) { - if ((bus->phy_mask & (1 << i)) == 0) { + if ((bus->phy_mask & BIT(i)) == 0) { struct phy_device *phydev; phydev = mdiobus_scan(bus, i); -- GitLab From d6dd2fe71153f0ff748bf188bd4af076fe09a0a6 Mon Sep 17 00:00:00 2001 From: Nick Child Date: Mon, 31 Oct 2022 10:06:42 -0500 Subject: [PATCH 0963/1927] ibmvnic: Free rwi on reset success Free the rwi structure in the event that the last rwi in the list processed successfully. The logic in commit 4f408e1fa6e1 ("ibmvnic: retry reset if there are no other resets") introduces an issue that results in a 32 byte memory leak whenever the last rwi in the list gets processed. Fixes: 4f408e1fa6e1 ("ibmvnic: retry reset if there are no other resets") Signed-off-by: Nick Child Link: https://lore.kernel.org/r/20221031150642.13356-1-nnac123@linux.ibm.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmvnic.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 65dbfbec487a3..9282381a438fe 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -3007,19 +3007,19 @@ static void __ibmvnic_reset(struct work_struct *work) rwi = get_next_rwi(adapter); /* - * If there is another reset queued, free the previous rwi - * and process the new reset even if previous reset failed - * (the previous reset could have failed because of a fail - * over for instance, so process the fail over). - * * If there are no resets queued and the previous reset failed, * the adapter would be in an undefined state. So retry the * previous reset as a hard reset. + * + * Else, free the previous rwi and, if there is another reset + * queued, process the new reset even if previous reset failed + * (the previous reset could have failed because of a fail + * over for instance, so process the fail over). */ - if (rwi) - kfree(tmprwi); - else if (rc) + if (!rwi && rc) rwi = tmprwi; + else + kfree(tmprwi); if (rwi && (rwi->reset_reason == VNIC_RESET_FAILOVER || rwi->reset_reason == VNIC_RESET_MOBILITY || rc)) -- GitLab From 2ae34111fe4eebb69986f6490015b57c88804373 Mon Sep 17 00:00:00 2001 From: Liu Peibao Date: Tue, 1 Nov 2022 14:02:18 +0800 Subject: [PATCH 0964/1927] stmmac: dwmac-loongson: fix invalid mdio_node In current code "plat->mdio_node" is always NULL, the mdio support is lost as there is no "mdio_bus_data". The original driver could work as the "mdio" variable is never set to false, which is described in commit ("stmmac: dwmac-loongson: fix uninitialized variable ......"). And after this commit merged, the "mdio" variable is always false, causing the mdio supoort logic lost. Fixes: 30bba69d7db4 ("stmmac: pci: Add dwmac support for Loongson") Signed-off-by: Liu Peibao Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20221101060218.16453-1-liupeibao@loongson.cn Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 017dbbda0c1c4..79fa7870563b8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -51,7 +51,6 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id struct stmmac_resources res; struct device_node *np; int ret, i, phy_mode; - bool mdio = false; np = dev_of_node(&pdev->dev); @@ -69,12 +68,10 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id if (!plat) return -ENOMEM; + plat->mdio_node = of_get_child_by_name(np, "mdio"); if (plat->mdio_node) { - dev_err(&pdev->dev, "Found MDIO subnode\n"); - mdio = true; - } + dev_info(&pdev->dev, "Found MDIO subnode\n"); - if (mdio) { plat->mdio_bus_data = devm_kzalloc(&pdev->dev, sizeof(*plat->mdio_bus_data), GFP_KERNEL); -- GitLab From 62ff373da2534534c55debe6c724c7fe14adb97f Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Tue, 1 Nov 2022 17:37:22 +0800 Subject: [PATCH 0965/1927] net/smc: Fix possible leaked pernet namespace in smc_init() In smc_init(), register_pernet_subsys(&smc_net_stat_ops) is called without any error handling. If it fails, registering of &smc_net_ops won't be reverted. And if smc_nl_init() fails, &smc_net_stat_ops itself won't be reverted. This leaves wild ops in subsystem linkedlist and when another module tries to call register_pernet_operations() it triggers page fault: BUG: unable to handle page fault for address: fffffbfff81b964c RIP: 0010:register_pernet_operations+0x1b9/0x5f0 Call Trace: register_pernet_subsys+0x29/0x40 ebtables_init+0x58/0x1000 [ebtables] ... Fixes: 194730a9beb5 ("net/smc: Make SMC statistics network namespace aware") Signed-off-by: Chen Zhongjin Reviewed-by: Tony Lu Reviewed-by: Wenjia Zhang Link: https://lore.kernel.org/r/20221101093722.127223-1-chenzhongjin@huawei.com Signed-off-by: Jakub Kicinski --- net/smc/af_smc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 3ccbf3c201cd2..e12d4fa5aece6 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -3380,14 +3380,14 @@ static int __init smc_init(void) rc = register_pernet_subsys(&smc_net_stat_ops); if (rc) - return rc; + goto out_pernet_subsys; smc_ism_init(); smc_clc_init(); rc = smc_nl_init(); if (rc) - goto out_pernet_subsys; + goto out_pernet_subsys_stat; rc = smc_pnet_init(); if (rc) @@ -3480,6 +3480,8 @@ static int __init smc_init(void) smc_pnet_exit(); out_nl: smc_nl_exit(); +out_pernet_subsys_stat: + unregister_pernet_subsys(&smc_net_stat_ops); out_pernet_subsys: unregister_pernet_subsys(&smc_net_ops); -- GitLab From f8017317cb0b279b8ab98b0f3901a2e0ac880dad Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Tue, 1 Nov 2022 20:15:52 +0800 Subject: [PATCH 0966/1927] net, neigh: Fix null-ptr-deref in neigh_table_clear() When IPv6 module gets initialized but hits an error in the middle, kenel panic with: KASAN: null-ptr-deref in range [0x0000000000000598-0x000000000000059f] CPU: 1 PID: 361 Comm: insmod Hardware name: QEMU Standard PC (i440FX + PIIX, 1996) RIP: 0010:__neigh_ifdown.isra.0+0x24b/0x370 RSP: 0018:ffff888012677908 EFLAGS: 00000202 ... Call Trace: neigh_table_clear+0x94/0x2d0 ndisc_cleanup+0x27/0x40 [ipv6] inet6_init+0x21c/0x2cb [ipv6] do_one_initcall+0xd3/0x4d0 do_init_module+0x1ae/0x670 ... Kernel panic - not syncing: Fatal exception When ipv6 initialization fails, it will try to cleanup and calls: neigh_table_clear() neigh_ifdown(tbl, NULL) pneigh_queue_purge(&tbl->proxy_queue, dev_net(dev == NULL)) # dev_net(NULL) triggers null-ptr-deref. Fix it by passing NULL to pneigh_queue_purge() in neigh_ifdown() if dev is NULL, to make kernel not panic immediately. Fixes: 66ba215cb513 ("neigh: fix possible DoS due to net iface start/stop loop") Signed-off-by: Chen Zhongjin Reviewed-by: Eric Dumazet Reviewed-by: Denis V. Lunev Link: https://lore.kernel.org/r/20221101121552.21890-1-chenzhongjin@huawei.com Signed-off-by: Jakub Kicinski --- net/core/neighbour.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/core/neighbour.c b/net/core/neighbour.c index 3c4786b999070..a77a85e357e0a 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -409,7 +409,7 @@ static int __neigh_ifdown(struct neigh_table *tbl, struct net_device *dev, write_lock_bh(&tbl->lock); neigh_flush_dev(tbl, dev, skip_perm); pneigh_ifdown_and_unlock(tbl, dev); - pneigh_queue_purge(&tbl->proxy_queue, dev_net(dev)); + pneigh_queue_purge(&tbl->proxy_queue, dev ? dev_net(dev) : NULL); if (skb_queue_empty_lockless(&tbl->proxy_queue)) del_timer_sync(&tbl->proxy_timer); return 0; -- GitLab From 628ac04a75ed5ff13647e725f40192da22ef2be8 Mon Sep 17 00:00:00 2001 From: Ido Schimmel Date: Tue, 1 Nov 2022 20:57:53 +0200 Subject: [PATCH 0967/1927] bridge: Fix flushing of dynamic FDB entries The following commands should result in all the dynamic FDB entries being flushed, but instead all the non-local (non-permanent) entries are flushed: # bridge fdb add 00:aa:bb:cc:dd:ee dev dummy1 master static # bridge fdb add 00:11:22:33:44:55 dev dummy1 master dynamic # ip link set dev br0 type bridge fdb_flush # bridge fdb show brport dummy1 00:00:00:00:00:01 master br0 permanent 33:33:00:00:00:01 self permanent 01:00:5e:00:00:01 self permanent This is because br_fdb_flush() works with FDB flags and not the corresponding enumerator values. Fix by passing the FDB flag instead. After the fix: # bridge fdb add 00:aa:bb:cc:dd:ee dev dummy1 master static # bridge fdb add 00:11:22:33:44:55 dev dummy1 master dynamic # ip link set dev br0 type bridge fdb_flush # bridge fdb show brport dummy1 00:aa:bb:cc:dd:ee master br0 static 00:00:00:00:00:01 master br0 permanent 33:33:00:00:00:01 self permanent 01:00:5e:00:00:01 self permanent Fixes: 1f78ee14eeac ("net: bridge: fdb: add support for fine-grained flushing") Signed-off-by: Ido Schimmel Acked-by: Nikolay Aleksandrov Link: https://lore.kernel.org/r/20221101185753.2120691-1-idosch@nvidia.com Signed-off-by: Jakub Kicinski --- net/bridge/br_netlink.c | 2 +- net/bridge/br_sysfs_br.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c index 5aeb3646e74c7..d087fd4c784ac 100644 --- a/net/bridge/br_netlink.c +++ b/net/bridge/br_netlink.c @@ -1332,7 +1332,7 @@ static int br_changelink(struct net_device *brdev, struct nlattr *tb[], if (data[IFLA_BR_FDB_FLUSH]) { struct net_bridge_fdb_flush_desc desc = { - .flags_mask = BR_FDB_STATIC + .flags_mask = BIT(BR_FDB_STATIC) }; br_fdb_flush(br, &desc); diff --git a/net/bridge/br_sysfs_br.c b/net/bridge/br_sysfs_br.c index 612e367fff20d..ea733542244c7 100644 --- a/net/bridge/br_sysfs_br.c +++ b/net/bridge/br_sysfs_br.c @@ -345,7 +345,7 @@ static int set_flush(struct net_bridge *br, unsigned long val, struct netlink_ext_ack *extack) { struct net_bridge_fdb_flush_desc desc = { - .flags_mask = BR_FDB_STATIC + .flags_mask = BIT(BR_FDB_STATIC) }; br_fdb_flush(br, &desc); -- GitLab From 768b3c745fe5789f2430bdab02f35a9ad1148d97 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 2 Nov 2022 10:06:10 +0800 Subject: [PATCH 0968/1927] ipv6: fix WARNING in ip6_route_net_exit_late() During the initialization of ip6_route_net_init_late(), if file ipv6_route or rt6_stats fails to be created, the initialization is successful by default. Therefore, the ipv6_route or rt6_stats file doesn't be found during the remove in ip6_route_net_exit_late(). It will cause WRNING. The following is the stack information: name 'rt6_stats' WARNING: CPU: 0 PID: 9 at fs/proc/generic.c:712 remove_proc_entry+0x389/0x460 Modules linked in: Workqueue: netns cleanup_net RIP: 0010:remove_proc_entry+0x389/0x460 PKRU: 55555554 Call Trace: ops_exit_list+0xb0/0x170 cleanup_net+0x4ea/0xb00 process_one_work+0x9bf/0x1710 worker_thread+0x665/0x1080 kthread+0x2e4/0x3a0 ret_from_fork+0x1f/0x30 Fixes: cdb1876192db ("[NETNS][IPV6] route6 - create route6 proc files for the namespace") Signed-off-by: Zhengchao Shao Reviewed-by: Eric Dumazet Link: https://lore.kernel.org/r/20221102020610.351330-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- net/ipv6/route.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 69252eb462b2d..2f355f0ec32ac 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -6555,10 +6555,16 @@ static void __net_exit ip6_route_net_exit(struct net *net) static int __net_init ip6_route_net_init_late(struct net *net) { #ifdef CONFIG_PROC_FS - proc_create_net("ipv6_route", 0, net->proc_net, &ipv6_route_seq_ops, - sizeof(struct ipv6_route_iter)); - proc_create_net_single("rt6_stats", 0444, net->proc_net, - rt6_stats_seq_show, NULL); + if (!proc_create_net("ipv6_route", 0, net->proc_net, + &ipv6_route_seq_ops, + sizeof(struct ipv6_route_iter))) + return -ENOMEM; + + if (!proc_create_net_single("rt6_stats", 0444, net->proc_net, + rt6_stats_seq_show, NULL)) { + remove_proc_entry("ipv6_route", net->proc_net); + return -ENOMEM; + } #endif return 0; } -- GitLab From 7433632c9ff68a991bd0bc38cabf354e9d2de410 Mon Sep 17 00:00:00 2001 From: "Steven Rostedt (Google)" Date: Tue, 1 Nov 2022 19:10:09 -0400 Subject: [PATCH 0969/1927] ring-buffer: Check for NULL cpu_buffer in ring_buffer_wake_waiters() On some machines the number of listed CPUs may be bigger than the actual CPUs that exist. The tracing subsystem allocates a per_cpu directory with access to the per CPU ring buffer via a cpuX file. But to save space, the ring buffer will only allocate buffers for online CPUs, even though the CPU array will be as big as the nr_cpu_ids. With the addition of waking waiters on the ring buffer when closing the file, the ring_buffer_wake_waiters() now needs to make sure that the buffer is allocated (with the irq_work allocated with it) before trying to wake waiters, as it will cause a NULL pointer dereference. While debugging this, I added a NULL check for the buffer itself (which is OK to do), and also NULL pointer checks against buffer->buffers (which is not fine, and will WARN) as well as making sure the CPU number passed in is within the nr_cpu_ids (which is also not fine if it isn't). Link: https://lore.kernel.org/all/87h6zklb6n.wl-tiwai@suse.de/ Link: https://lore.kernel.org/all/CAM6Wdxc0KRJMXVAA0Y=u6Jh2V=uWB-_Fn6M4xRuNppfXzL1mUg@mail.gmail.com/ Link: https://lkml.kernel.org/linux-trace-kernel/20221101191009.1e7378c8@rorschach.local.home Cc: stable@vger.kernel.org Cc: Steven Noonan Bugzilla: https://bugzilla.opensuse.org/show_bug.cgi?id=1204705 Reported-by: Takashi Iwai Reported-by: Roland Ruckerbauer Fixes: f3ddb74ad079 ("tracing: Wake up ring buffer waiters on closing of the file") Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ring_buffer.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 199759c735196..9712083832f41 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -937,6 +937,9 @@ void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu) struct ring_buffer_per_cpu *cpu_buffer; struct rb_irq_work *rbwork; + if (!buffer) + return; + if (cpu == RING_BUFFER_ALL_CPUS) { /* Wake up individual ones too. One level recursion */ @@ -945,7 +948,15 @@ void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu) rbwork = &buffer->irq_work; } else { + if (WARN_ON_ONCE(!buffer->buffers)) + return; + if (WARN_ON_ONCE(cpu >= nr_cpu_ids)) + return; + cpu_buffer = buffer->buffers[cpu]; + /* The CPU buffer may not have been initialized yet */ + if (!cpu_buffer) + return; rbwork = &cpu_buffer->irq_work; } -- GitLab From 0e792b89e6800cd9cb4757a76a96f7ef3e8b6294 Mon Sep 17 00:00:00 2001 From: Li Huafei Date: Thu, 3 Nov 2022 11:10:10 +0800 Subject: [PATCH 0970/1927] ftrace: Fix use-after-free for dynamic ftrace_ops KASAN reported a use-after-free with ftrace ops [1]. It was found from vmcore that perf had registered two ops with the same content successively, both dynamic. After unregistering the second ops, a use-after-free occurred. In ftrace_shutdown(), when the second ops is unregistered, the FTRACE_UPDATE_CALLS command is not set because there is another enabled ops with the same content. Also, both ops are dynamic and the ftrace callback function is ftrace_ops_list_func, so the FTRACE_UPDATE_TRACE_FUNC command will not be set. Eventually the value of 'command' will be 0 and ftrace_shutdown() will skip the rcu synchronization. However, ftrace may be activated. When the ops is released, another CPU may be accessing the ops. Add the missing synchronization to fix this problem. [1] BUG: KASAN: use-after-free in __ftrace_ops_list_func kernel/trace/ftrace.c:7020 [inline] BUG: KASAN: use-after-free in ftrace_ops_list_func+0x2b0/0x31c kernel/trace/ftrace.c:7049 Read of size 8 at addr ffff56551965bbc8 by task syz-executor.2/14468 CPU: 1 PID: 14468 Comm: syz-executor.2 Not tainted 5.10.0 #7 Hardware name: linux,dummy-virt (DT) Call trace: dump_backtrace+0x0/0x40c arch/arm64/kernel/stacktrace.c:132 show_stack+0x30/0x40 arch/arm64/kernel/stacktrace.c:196 __dump_stack lib/dump_stack.c:77 [inline] dump_stack+0x1b4/0x248 lib/dump_stack.c:118 print_address_description.constprop.0+0x28/0x48c mm/kasan/report.c:387 __kasan_report mm/kasan/report.c:547 [inline] kasan_report+0x118/0x210 mm/kasan/report.c:564 check_memory_region_inline mm/kasan/generic.c:187 [inline] __asan_load8+0x98/0xc0 mm/kasan/generic.c:253 __ftrace_ops_list_func kernel/trace/ftrace.c:7020 [inline] ftrace_ops_list_func+0x2b0/0x31c kernel/trace/ftrace.c:7049 ftrace_graph_call+0x0/0x4 __might_sleep+0x8/0x100 include/linux/perf_event.h:1170 __might_fault mm/memory.c:5183 [inline] __might_fault+0x58/0x70 mm/memory.c:5171 do_strncpy_from_user lib/strncpy_from_user.c:41 [inline] strncpy_from_user+0x1f4/0x4b0 lib/strncpy_from_user.c:139 getname_flags+0xb0/0x31c fs/namei.c:149 getname+0x2c/0x40 fs/namei.c:209 [...] Allocated by task 14445: kasan_save_stack+0x24/0x50 mm/kasan/common.c:48 kasan_set_track mm/kasan/common.c:56 [inline] __kasan_kmalloc mm/kasan/common.c:479 [inline] __kasan_kmalloc.constprop.0+0x110/0x13c mm/kasan/common.c:449 kasan_kmalloc+0xc/0x14 mm/kasan/common.c:493 kmem_cache_alloc_trace+0x440/0x924 mm/slub.c:2950 kmalloc include/linux/slab.h:563 [inline] kzalloc include/linux/slab.h:675 [inline] perf_event_alloc.part.0+0xb4/0x1350 kernel/events/core.c:11230 perf_event_alloc kernel/events/core.c:11733 [inline] __do_sys_perf_event_open kernel/events/core.c:11831 [inline] __se_sys_perf_event_open+0x550/0x15f4 kernel/events/core.c:11723 __arm64_sys_perf_event_open+0x6c/0x80 kernel/events/core.c:11723 [...] Freed by task 14445: kasan_save_stack+0x24/0x50 mm/kasan/common.c:48 kasan_set_track+0x24/0x34 mm/kasan/common.c:56 kasan_set_free_info+0x20/0x40 mm/kasan/generic.c:358 __kasan_slab_free.part.0+0x11c/0x1b0 mm/kasan/common.c:437 __kasan_slab_free mm/kasan/common.c:445 [inline] kasan_slab_free+0x2c/0x40 mm/kasan/common.c:446 slab_free_hook mm/slub.c:1569 [inline] slab_free_freelist_hook mm/slub.c:1608 [inline] slab_free mm/slub.c:3179 [inline] kfree+0x12c/0xc10 mm/slub.c:4176 perf_event_alloc.part.0+0xa0c/0x1350 kernel/events/core.c:11434 perf_event_alloc kernel/events/core.c:11733 [inline] __do_sys_perf_event_open kernel/events/core.c:11831 [inline] __se_sys_perf_event_open+0x550/0x15f4 kernel/events/core.c:11723 [...] Link: https://lore.kernel.org/linux-trace-kernel/20221103031010.166498-1-lihuafei1@huawei.com Fixes: edb096e00724f ("ftrace: Fix memleak when unregistering dynamic ops when tracing disabled") Cc: stable@vger.kernel.org Suggested-by: Steven Rostedt Signed-off-by: Li Huafei Signed-off-by: Steven Rostedt (Google) --- kernel/trace/ftrace.c | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index fbf2543111c05..7dc023641bf10 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -3028,18 +3028,8 @@ int ftrace_shutdown(struct ftrace_ops *ops, int command) command |= FTRACE_UPDATE_TRACE_FUNC; } - if (!command || !ftrace_enabled) { - /* - * If these are dynamic or per_cpu ops, they still - * need their data freed. Since, function tracing is - * not currently active, we can just free them - * without synchronizing all CPUs. - */ - if (ops->flags & FTRACE_OPS_FL_DYNAMIC) - goto free_ops; - - return 0; - } + if (!command || !ftrace_enabled) + goto out; /* * If the ops uses a trampoline, then it needs to be @@ -3076,6 +3066,7 @@ int ftrace_shutdown(struct ftrace_ops *ops, int command) removed_ops = NULL; ops->flags &= ~FTRACE_OPS_FL_REMOVING; +out: /* * Dynamic ops may be freed, we must make sure that all * callers are done before leaving this function. @@ -3103,7 +3094,6 @@ int ftrace_shutdown(struct ftrace_ops *ops, int command) if (IS_ENABLED(CONFIG_PREEMPTION)) synchronize_rcu_tasks(); - free_ops: ftrace_trampoline_free(ops); } -- GitLab From 682f99b8ae886c22ba9f16df454aecc8c6d803ba Mon Sep 17 00:00:00 2001 From: "maxime@cerno.tech" Date: Mon, 24 Oct 2022 11:36:33 +0200 Subject: [PATCH 0971/1927] drm/vc4: hdmi: Take our lock to reset the link We access some fields protected by our internal mutex in vc4_hdmi_reset_link() (saved_adjusted_mode, output_bpc, output_format) and are calling functions that need to have that lock taken (vc4_hdmi_supports_scrambling()). However, the current code doesn't lock that mutex. Let's make sure it does. Fixes: 6bed2ea3cb38 ("drm/vc4: hdmi: Reset link on hotplug") Reviewed-by: Javier Martinez Canillas Link: https://lore.kernel.org/r/20221024093634.118190-1-maxime@cerno.tech Signed-off-by: Maxime Ripard --- drivers/gpu/drm/vc4/vc4_hdmi.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index 596e311d6e58e..9e549fa074676 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -349,27 +349,40 @@ static int vc4_hdmi_reset_link(struct drm_connector *connector, if (!crtc_state->active) return 0; - if (!vc4_hdmi_supports_scrambling(encoder)) + mutex_lock(&vc4_hdmi->mutex); + + if (!vc4_hdmi_supports_scrambling(encoder)) { + mutex_unlock(&vc4_hdmi->mutex); return 0; + } scrambling_needed = vc4_hdmi_mode_needs_scrambling(&vc4_hdmi->saved_adjusted_mode, vc4_hdmi->output_bpc, vc4_hdmi->output_format); - if (!scrambling_needed) + if (!scrambling_needed) { + mutex_unlock(&vc4_hdmi->mutex); return 0; + } if (conn_state->commit && - !try_wait_for_completion(&conn_state->commit->hw_done)) + !try_wait_for_completion(&conn_state->commit->hw_done)) { + mutex_unlock(&vc4_hdmi->mutex); return 0; + } ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config); if (ret < 0) { drm_err(drm, "Failed to read TMDS config: %d\n", ret); + mutex_unlock(&vc4_hdmi->mutex); return 0; } - if (!!(config & SCDC_SCRAMBLING_ENABLE) == scrambling_needed) + if (!!(config & SCDC_SCRAMBLING_ENABLE) == scrambling_needed) { + mutex_unlock(&vc4_hdmi->mutex); return 0; + } + + mutex_unlock(&vc4_hdmi->mutex); /* * HDMI 2.0 says that one should not send scrambled data -- GitLab From 76ffa2af16c67bbb806b8224a5289eb03f7a537d Mon Sep 17 00:00:00 2001 From: "maxime@cerno.tech" Date: Mon, 24 Oct 2022 11:36:34 +0200 Subject: [PATCH 0972/1927] drm/vc4: hdmi: Fix outdated function name in comment A comment introduced by commit 6bed2ea3cb38 ("drm/vc4: hdmi: Reset link on hotplug") mentions a drm_atomic_helper_connector_hdmi_reset_link() function that was part of the earlier versions but got moved internally and is now named vc4_hdmi_reset_link(). Let's fix the function name. Fixes: 6bed2ea3cb38 ("drm/vc4: hdmi: Reset link on hotplug") Reviewed-by: Javier Martinez Canillas Link: https://lore.kernel.org/r/20221024093634.118190-2-maxime@cerno.tech Signed-off-by: Maxime Ripard --- drivers/gpu/drm/vc4/vc4_hdmi.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index 9e549fa074676..79eda8f5fea03 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -410,9 +410,8 @@ static void vc4_hdmi_handle_hotplug(struct vc4_hdmi *vc4_hdmi, * .adap_enable, which leads to that funtion being called with * our mutex held. * - * A similar situation occurs with - * drm_atomic_helper_connector_hdmi_reset_link() that will call - * into our KMS hooks if the scrambling was enabled. + * A similar situation occurs with vc4_hdmi_reset_link() that + * will call into our KMS hooks if the scrambling was enabled. * * Concurrency isn't an issue at the moment since we don't share * any state with any of the other frameworks so we can ignore -- GitLab From 3bc6a37f59f21a8bfaf74d0975b2eb0b2d52a065 Mon Sep 17 00:00:00 2001 From: "maxime@cerno.tech" Date: Fri, 21 Oct 2022 15:13:39 +0200 Subject: [PATCH 0973/1927] drm/vc4: hdmi: Fix HSM clock too low on Pi4 Commit ae71ab585c81 ("drm/vc4: hdmi: Enforce the minimum rate at runtime_resume") reintroduced the call to clk_set_min_rate in an attempt to fix the boot without a monitor connected on the RaspberryPi3. However, that introduced a regression breaking the display output entirely (black screen but no vblank timeout) on the Pi4. This is due to the fact that we now have in a typical modeset at boot, in vc4_hdmi_encoder_pre_crtc_configure(), we have a first call to clk_set_min_rate() asking for the minimum rate of the HSM clock for our given resolution, and then a call to pm_runtime_resume_and_get(). We will thus execute vc4_hdmi_runtime_resume() which, since the commit mentioned above, will call clk_set_min_rate() a second time with the absolute minimum rate we want to enforce on the HSM clock. We're thus effectively erasing the minimum mandated by the mode we're trying to set. The fact that only the Pi4 is affected is due to the fact that it uses a different clock driver that tries to minimize the HSM clock at all time. It will thus lower the HSM clock rate to 120MHz on the second clk_set_min_rate() call. The Pi3 doesn't use the same driver and will not change the frequency on the second clk_set_min_rate() call since it's still within the new boundaries and it doesn't have the code to minimize the clock rate as needed. So even though the boundaries are still off, the clock rate is still the right one for our given mode, so everything works. There is a lot of moving parts, so I couldn't find any obvious solution: - Reverting the original is not an option, as that would break the Pi3 again. - We can't move the clk_set_min_rate() call in _pre_crtc_configure() since because, on the Pi3, the HSM clock has the CLK_SET_RATE_GATE flag which prevents the clock rate from being changed after it's been enabled. Our calls to clk_set_min_rate() can change it, so they need to be done before clk_prepare_enable(). - We can't remove the call to clk_prepare_enable() from the runtime_resume hook to put it into _pre_crtc_configure() either, since we need that clock to be enabled to access the registers, and we can't count on the fact that the display will be active in all situations (doing any CEC operation, or listing the modes while inactive are valid for example()). - We can't drop the call to clk_set_min_rate() in _pre_crtc_configure() since we would need to still enforce the minimum rate for a given resolution, and runtime_resume doesn't have access to the current mode, if there's any. - We can't copy the TMDS character rate into vc4_hdmi and reuse it since, because it's part of the KMS atomic state, it needs to be protected by a mutex. Unfortunately, some functions (CEC operations, mostly) can be reentrant (through the CEC framework) and still need a pm_runtime_get. However, we can work around this issue by leveraging the fact that the clk_set_min_rate() calls set boundaries for its given struct clk, and that each different clk_get() call will return a different instance of struct clk. The clock framework will then aggregate the boundaries for each struct clk instances linked to a given clock, plus its hardware boundaries, and will use that. We can thus get an extra HSM clock user for runtime_pm use only, and use our different clock instances depending on the context: runtime_pm will use its own to set the absolute minimum clock setup so that we never lock the CPU waiting for a register access, and the modeset part will set its requirement for the current resolution. And we let the CCF do the coordination. It's not an ideal solution, but it's fairly unintrusive and doesn't really change any part of the logic so it looks like a rather safe fix. Link: https://bugzilla.redhat.com/show_bug.cgi?id=2136234 Fixes: ae71ab585c81 ("drm/vc4: hdmi: Enforce the minimum rate at runtime_resume") Reported-by: Peter Robinson Reviewed-by: Javier Martinez Canillas Tested-by: Peter Robinson Link: https://lore.kernel.org/r/20221021131339.2203291-1-maxime@cerno.tech Signed-off-by: Maxime Ripard --- drivers/gpu/drm/vc4/vc4_hdmi.c | 21 +++++++++++++++++---- drivers/gpu/drm/vc4/vc4_hdmi.h | 1 + 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index 79eda8f5fea03..470432c8fd707 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -3172,9 +3172,16 @@ static int vc4_hdmi_init_resources(struct drm_device *drm, DRM_ERROR("Failed to get HDMI state machine clock\n"); return PTR_ERR(vc4_hdmi->hsm_clock); } + vc4_hdmi->audio_clock = vc4_hdmi->hsm_clock; vc4_hdmi->cec_clock = vc4_hdmi->hsm_clock; + vc4_hdmi->hsm_rpm_clock = devm_clk_get(dev, "hdmi"); + if (IS_ERR(vc4_hdmi->hsm_rpm_clock)) { + DRM_ERROR("Failed to get HDMI state machine clock\n"); + return PTR_ERR(vc4_hdmi->hsm_rpm_clock); + } + return 0; } @@ -3257,6 +3264,12 @@ static int vc5_hdmi_init_resources(struct drm_device *drm, return PTR_ERR(vc4_hdmi->hsm_clock); } + vc4_hdmi->hsm_rpm_clock = devm_clk_get(dev, "hdmi"); + if (IS_ERR(vc4_hdmi->hsm_rpm_clock)) { + DRM_ERROR("Failed to get HDMI state machine clock\n"); + return PTR_ERR(vc4_hdmi->hsm_rpm_clock); + } + vc4_hdmi->pixel_bvb_clock = devm_clk_get(dev, "bvb"); if (IS_ERR(vc4_hdmi->pixel_bvb_clock)) { DRM_ERROR("Failed to get pixel bvb clock\n"); @@ -3320,7 +3333,7 @@ static int vc4_hdmi_runtime_suspend(struct device *dev) { struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev); - clk_disable_unprepare(vc4_hdmi->hsm_clock); + clk_disable_unprepare(vc4_hdmi->hsm_rpm_clock); return 0; } @@ -3338,11 +3351,11 @@ static int vc4_hdmi_runtime_resume(struct device *dev) * its frequency while the power domain is active so that it * keeps its rate. */ - ret = clk_set_min_rate(vc4_hdmi->hsm_clock, HSM_MIN_CLOCK_FREQ); + ret = clk_set_min_rate(vc4_hdmi->hsm_rpm_clock, HSM_MIN_CLOCK_FREQ); if (ret) return ret; - ret = clk_prepare_enable(vc4_hdmi->hsm_clock); + ret = clk_prepare_enable(vc4_hdmi->hsm_rpm_clock); if (ret) return ret; @@ -3355,7 +3368,7 @@ static int vc4_hdmi_runtime_resume(struct device *dev) * case, it will lead to a silent CPU stall. Let's make sure we * prevent such a case. */ - rate = clk_get_rate(vc4_hdmi->hsm_clock); + rate = clk_get_rate(vc4_hdmi->hsm_rpm_clock); if (!rate) { ret = -EINVAL; goto err_disable_clk; diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.h b/drivers/gpu/drm/vc4/vc4_hdmi.h index db823efb25639..1ad8e8c377e27 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.h +++ b/drivers/gpu/drm/vc4/vc4_hdmi.h @@ -172,6 +172,7 @@ struct vc4_hdmi { struct clk *cec_clock; struct clk *pixel_clock; struct clk *hsm_clock; + struct clk *hsm_rpm_clock; struct clk *audio_clock; struct clk *pixel_bvb_clock; -- GitLab From 639325e4269750fbd3ccc2c4beb7e5cc40e2c4fd Mon Sep 17 00:00:00 2001 From: Vinod Govindapillai Date: Tue, 11 Oct 2022 12:30:48 +0300 Subject: [PATCH 0974/1927] drm/i915: update DSC feature flag handling during device init DSC feature information is no longer part of the DFSM register in some display generations. Bspec:50075 Signed-off-by: Vinod Govindapillai Reviewed-by: Stanislav Lisovskiy Signed-off-by: Stanislav Lisovskiy Link: https://patchwork.freedesktop.org/patch/msgid/20221011093048.447177-1-vinod.govindapillai@intel.com --- drivers/gpu/drm/i915/intel_device_info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c index 1dc1fb29a776f..e0cc0227b4d79 100644 --- a/drivers/gpu/drm/i915/intel_device_info.c +++ b/drivers/gpu/drm/i915/intel_device_info.c @@ -488,7 +488,7 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv) if (DISPLAY_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE)) runtime->has_dmc = 0; - if (DISPLAY_VER(dev_priv) >= 10 && + if (IS_DISPLAY_VER(dev_priv, 10, 12) && (dfsm & GLK_DFSM_DISPLAY_DSC_DISABLE)) runtime->has_dsc = 0; } -- GitLab From 354d8a4b165697e6da9585b3f651d87735f30415 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 20 Oct 2022 13:21:43 +0200 Subject: [PATCH 0975/1927] x86/xen: silence smatch warning in pmu_msr_chk_emulated() Commit 8714f7bcd3c2 ("xen/pv: add fault recovery control to pmu msr accesses") introduced code resulting in a warning issued by the smatch static checker, claiming to use an uninitialized variable. This is a false positive, but work around the warning nevertheless. Fixes: 8714f7bcd3c2 ("xen/pv: add fault recovery control to pmu msr accesses") Reported-by: Dan Carpenter Signed-off-by: Juergen Gross --- arch/x86/xen/pmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c index 68aff13828728..246d67dab5109 100644 --- a/arch/x86/xen/pmu.c +++ b/arch/x86/xen/pmu.c @@ -302,7 +302,7 @@ static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read) static bool pmu_msr_chk_emulated(unsigned int msr, uint64_t *val, bool is_read, bool *emul) { - int type, index; + int type, index = 0; if (is_amd_pmu_msr(msr)) *emul = xen_amd_pmu_emulate(msr, val, is_read); -- GitLab From 4bff677b30156435afa2cc4c3601b542b4ddd439 Mon Sep 17 00:00:00 2001 From: Juergen Gross Date: Thu, 20 Oct 2022 13:25:12 +0200 Subject: [PATCH 0976/1927] x86/xen: simplify sysenter and syscall setup xen_enable_sysenter() and xen_enable_syscall() can be simplified a lot. While at it, switch to use cpu_feature_enabled() instead of boot_cpu_has(). Signed-off-by: Juergen Gross --- arch/x86/xen/setup.c | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index cfa99e8f054be..4f4309500559a 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -910,17 +910,9 @@ static int register_callback(unsigned type, const void *func) void xen_enable_sysenter(void) { - int ret; - unsigned sysenter_feature; - - sysenter_feature = X86_FEATURE_SYSENTER32; - - if (!boot_cpu_has(sysenter_feature)) - return; - - ret = register_callback(CALLBACKTYPE_sysenter, xen_entry_SYSENTER_compat); - if(ret != 0) - setup_clear_cpu_cap(sysenter_feature); + if (cpu_feature_enabled(X86_FEATURE_SYSENTER32) && + register_callback(CALLBACKTYPE_sysenter, xen_entry_SYSENTER_compat)) + setup_clear_cpu_cap(X86_FEATURE_SYSENTER32); } void xen_enable_syscall(void) @@ -934,12 +926,9 @@ void xen_enable_syscall(void) mechanism for syscalls. */ } - if (boot_cpu_has(X86_FEATURE_SYSCALL32)) { - ret = register_callback(CALLBACKTYPE_syscall32, - xen_entry_SYSCALL_compat); - if (ret != 0) - setup_clear_cpu_cap(X86_FEATURE_SYSCALL32); - } + if (cpu_feature_enabled(X86_FEATURE_SYSCALL32) && + register_callback(CALLBACKTYPE_syscall32, xen_entry_SYSCALL_compat)) + setup_clear_cpu_cap(X86_FEATURE_SYSCALL32); } static void __init xen_pvmmu_arch_setup(void) -- GitLab From cf6ff0df0fd123493e57278a1bd4414a97511a34 Mon Sep 17 00:00:00 2001 From: Dexuan Cui Date: Mon, 31 Oct 2022 19:17:05 -0700 Subject: [PATCH 0977/1927] vsock: remove the unused 'wait' in vsock_connectible_recvmsg() Remove the unused variable introduced by 19c1b90e1979. Fixes: 19c1b90e1979 ("af_vsock: separate receive data loop") Signed-off-by: Dexuan Cui Reviewed-by: Stefano Garzarella Signed-off-by: Paolo Abeni --- net/vmw_vsock/af_vsock.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index ee418701cdee9..d258fd43092ef 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -2092,8 +2092,6 @@ vsock_connectible_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, const struct vsock_transport *transport; int err; - DEFINE_WAIT(wait); - sk = sock->sk; vsk = vsock_sk(sk); err = 0; -- GitLab From 466a85336fee6e3b35eb97b8405a28302fd25809 Mon Sep 17 00:00:00 2001 From: Dexuan Cui Date: Mon, 31 Oct 2022 19:17:06 -0700 Subject: [PATCH 0978/1927] vsock: fix possible infinite sleep in vsock_connectible_wait_data() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently vsock_connectible_has_data() may miss a wakeup operation between vsock_connectible_has_data() == 0 and the prepare_to_wait(). Fix the race by adding the process to the wait queue before checking vsock_connectible_has_data(). Fixes: b3f7fd54881b ("af_vsock: separate wait data loop") Signed-off-by: Dexuan Cui Reviewed-by: Stefano Garzarella Reported-by: Frédéric Dalleau Tested-by: Frédéric Dalleau Signed-off-by: Paolo Abeni --- net/vmw_vsock/af_vsock.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index d258fd43092ef..884eca7f6743a 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1905,8 +1905,11 @@ static int vsock_connectible_wait_data(struct sock *sk, err = 0; transport = vsk->transport; - while ((data = vsock_connectible_has_data(vsk)) == 0) { + while (1) { prepare_to_wait(sk_sleep(sk), wait, TASK_INTERRUPTIBLE); + data = vsock_connectible_has_data(vsk); + if (data != 0) + break; if (sk->sk_err != 0 || (sk->sk_shutdown & RCV_SHUTDOWN) || -- GitLab From 6f85403ef4d0034fff11c77ed170aa2130329544 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Wed, 2 Nov 2022 16:57:09 +0100 Subject: [PATCH 0979/1927] drm/i915/selftests: Reduce oversaturation of request smoketesting The goal in launching the request smoketest is to have sufficient tasks running across the system such that we are likely to detect concurrency issues. We aim to have 2 tasks using the same engine, gt, device (each level of locking around submission and signaling) running at the same time. While tasks may not be running all the time as they synchronise with the gpu, they will be running most of the time, in which case having many more tasks than cores available is wasteful (and dramatically increases the workload causing excess runtime). Aim to limit the number of tasks such that there is at least 2 running per engine, spreading surplus cores around the engines (rather than running a task per core per engine.) Signed-off-by: Chris Wilson Reviewed-by: Nirmoy Das Tested-by: Nirmoy Das Signed-off-by: Nirmoy Das Reviewed-by: Tvrtko Ursulin Signed-off-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20221102155709.31717-1-nirmoy.das@intel.com --- drivers/gpu/drm/i915/selftests/i915_request.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index a46350c37e9d4..4380473ceb986 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -1710,7 +1710,8 @@ static int live_breadcrumbs_smoketest(void *arg) { struct drm_i915_private *i915 = arg; const unsigned int nengines = num_uabi_engines(i915); - const unsigned int ncpus = num_online_cpus(); + const unsigned int ncpus = /* saturate with nengines * ncpus */ + max_t(int, 2, DIV_ROUND_UP(num_online_cpus(), nengines)); unsigned long num_waits, num_fences; struct intel_engine_cs *engine; struct smoke_thread *threads; @@ -1782,7 +1783,7 @@ static int live_breadcrumbs_smoketest(void *arg) goto out_flush; } /* One ring interleaved between requests from all cpus */ - smoke[idx].max_batch /= num_online_cpus() + 1; + smoke[idx].max_batch /= ncpus + 1; pr_debug("Limiting batches to %d requests on %s\n", smoke[idx].max_batch, engine->name); -- GitLab From aec1dc972d27c837d1406310dab5170189eb01e5 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 2 Nov 2022 11:25:16 -0700 Subject: [PATCH 0980/1927] net/ipv4: Fix linux/in.h header dependencies __DECLARE_FLEX_ARRAY is defined in include/uapi/linux/stddef.h but doesn't seem to be explicitly included from include/uapi/linux/in.h, which breaks BPF selftests builds (once we sync linux/stddef.h into tools/include directory in the next patch). Fix this by explicitly including linux/stddef.h. Given this affects BPF CI and bpf tree, targeting this for bpf tree. Fixes: 5854a09b4957 ("net/ipv4: Use __DECLARE_FLEX_ARRAY() helper") Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Cc: Gustavo A. R. Silva Cc: Jakub Kicinski Link: https://lore.kernel.org/bpf/20221102182517.2675301-1-andrii@kernel.org --- include/uapi/linux/in.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/uapi/linux/in.h b/include/uapi/linux/in.h index f243ce665f74f..07a4cb149305b 100644 --- a/include/uapi/linux/in.h +++ b/include/uapi/linux/in.h @@ -20,6 +20,7 @@ #define _UAPI_LINUX_IN_H #include +#include #include #include -- GitLab From a778f5d46b6287ebe26e24b48f3e8079c2db8ed2 Mon Sep 17 00:00:00 2001 From: Andrii Nakryiko Date: Wed, 2 Nov 2022 11:25:17 -0700 Subject: [PATCH 0981/1927] tools/headers: Pull in stddef.h to uapi to fix BPF selftests build in CI With recent sync of linux/in.h tools/include headers are now relying on __DECLARE_FLEX_ARRAY macro, which isn't itself defined inside tools/include headers anywhere and is instead assumed to be present in system-wide UAPI header. This breaks isolated environments that don't have kernel UAPI headers installed system-wide, like BPF CI ([0]). To fix this, bring in include/uapi/linux/stddef.h into tools/include. We can't just copy/paste it, though, it has to be processed with scripts/headers_install.sh, which has a dependency on scripts/unifdef. So the full command to (re-)generate stddef.h for inclusion into tools/include directory is: $ make scripts_unifdef && \ cp $KBUILD_OUTPUT/scripts/unifdef scripts/ && \ scripts/headers_install.sh include/uapi/linux/stddef.h tools/include/uapi/linux/stddef.h This assumes KBUILD_OUTPUT envvar is set and used for out-of-tree builds. [0] https://github.com/kernel-patches/bpf/actions/runs/3379432493/jobs/5610982609 Fixes: 036b8f5b8970 ("tools headers uapi: Update linux/in.h copy") Signed-off-by: Andrii Nakryiko Signed-off-by: Daniel Borkmann Cc: Jakub Kicinski Cc: Arnaldo Carvalho de Melo Link: https://lore.kernel.org/bpf/20221102182517.2675301-2-andrii@kernel.org --- tools/include/uapi/linux/in.h | 1 + tools/include/uapi/linux/stddef.h | 47 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 tools/include/uapi/linux/stddef.h diff --git a/tools/include/uapi/linux/in.h b/tools/include/uapi/linux/in.h index f243ce665f74f..07a4cb149305b 100644 --- a/tools/include/uapi/linux/in.h +++ b/tools/include/uapi/linux/in.h @@ -20,6 +20,7 @@ #define _UAPI_LINUX_IN_H #include +#include #include #include diff --git a/tools/include/uapi/linux/stddef.h b/tools/include/uapi/linux/stddef.h new file mode 100644 index 0000000000000..bb6ea517efb51 --- /dev/null +++ b/tools/include/uapi/linux/stddef.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _LINUX_STDDEF_H +#define _LINUX_STDDEF_H + + + +#ifndef __always_inline +#define __always_inline __inline__ +#endif + +/** + * __struct_group() - Create a mirrored named and anonyomous struct + * + * @TAG: The tag name for the named sub-struct (usually empty) + * @NAME: The identifier name of the mirrored sub-struct + * @ATTRS: Any struct attributes (usually empty) + * @MEMBERS: The member declarations for the mirrored structs + * + * Used to create an anonymous union of two structs with identical layout + * and size: one anonymous and one named. The former's members can be used + * normally without sub-struct naming, and the latter can be used to + * reason about the start, end, and size of the group of struct members. + * The named struct can also be explicitly tagged for layer reuse, as well + * as both having struct attributes appended. + */ +#define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \ + union { \ + struct { MEMBERS } ATTRS; \ + struct TAG { MEMBERS } ATTRS NAME; \ + } + +/** + * __DECLARE_FLEX_ARRAY() - Declare a flexible array usable in a union + * + * @TYPE: The type of each flexible array element + * @NAME: The name of the flexible array member + * + * In order to have a flexible array member in a union or alone in a + * struct, it needs to be wrapped in an anonymous struct with at least 1 + * named member, but that member can be empty. + */ +#define __DECLARE_FLEX_ARRAY(TYPE, NAME) \ + struct { \ + struct { } __empty_ ## NAME; \ + TYPE NAME[]; \ + } +#endif -- GitLab From 8bbabb3fddcd0f858be69ed5abc9b470a239d6f2 Mon Sep 17 00:00:00 2001 From: Cong Wang Date: Tue, 1 Nov 2022 21:34:17 -0700 Subject: [PATCH 0982/1927] bpf, sock_map: Move cancel_work_sync() out of sock lock Stanislav reported a lockdep warning, which is caused by the cancel_work_sync() called inside sock_map_close(), as analyzed below by Jakub: psock->work.func = sk_psock_backlog() ACQUIRE psock->work_mutex sk_psock_handle_skb() skb_send_sock() __skb_send_sock() sendpage_unlocked() kernel_sendpage() sock->ops->sendpage = inet_sendpage() sk->sk_prot->sendpage = tcp_sendpage() ACQUIRE sk->sk_lock tcp_sendpage_locked() RELEASE sk->sk_lock RELEASE psock->work_mutex sock_map_close() ACQUIRE sk->sk_lock sk_psock_stop() sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED) cancel_work_sync() __cancel_work_timer() __flush_work() // wait for psock->work to finish RELEASE sk->sk_lock We can move the cancel_work_sync() out of the sock lock protection, but still before saved_close() was called. Fixes: 799aa7f98d53 ("skmsg: Avoid lock_sock() in sk_psock_backlog()") Reported-by: Stanislav Fomichev Signed-off-by: Cong Wang Signed-off-by: Daniel Borkmann Tested-by: Jakub Sitnicki Acked-by: John Fastabend Acked-by: Jakub Sitnicki Link: https://lore.kernel.org/bpf/20221102043417.279409-1-xiyou.wangcong@gmail.com --- include/linux/skmsg.h | 2 +- net/core/skmsg.c | 7 ++----- net/core/sock_map.c | 7 ++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 48f4b645193b7..70d6cb94e5802 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -376,7 +376,7 @@ static inline void sk_psock_report_error(struct sk_psock *psock, int err) } struct sk_psock *sk_psock_init(struct sock *sk, int node); -void sk_psock_stop(struct sk_psock *psock, bool wait); +void sk_psock_stop(struct sk_psock *psock); #if IS_ENABLED(CONFIG_BPF_STREAM_PARSER) int sk_psock_init_strp(struct sock *sk, struct sk_psock *psock); diff --git a/net/core/skmsg.c b/net/core/skmsg.c index 1efdc47a999b4..e6b9ced3eda82 100644 --- a/net/core/skmsg.c +++ b/net/core/skmsg.c @@ -803,16 +803,13 @@ static void sk_psock_link_destroy(struct sk_psock *psock) } } -void sk_psock_stop(struct sk_psock *psock, bool wait) +void sk_psock_stop(struct sk_psock *psock) { spin_lock_bh(&psock->ingress_lock); sk_psock_clear_state(psock, SK_PSOCK_TX_ENABLED); sk_psock_cork_free(psock); __sk_psock_zap_ingress(psock); spin_unlock_bh(&psock->ingress_lock); - - if (wait) - cancel_work_sync(&psock->work); } static void sk_psock_done_strp(struct sk_psock *psock); @@ -850,7 +847,7 @@ void sk_psock_drop(struct sock *sk, struct sk_psock *psock) sk_psock_stop_verdict(sk, psock); write_unlock_bh(&sk->sk_callback_lock); - sk_psock_stop(psock, false); + sk_psock_stop(psock); INIT_RCU_WORK(&psock->rwork, sk_psock_destroy); queue_rcu_work(system_wq, &psock->rwork); diff --git a/net/core/sock_map.c b/net/core/sock_map.c index a660baedd9e79..81beb16ab1ebf 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -1596,7 +1596,7 @@ void sock_map_destroy(struct sock *sk) saved_destroy = psock->saved_destroy; sock_map_remove_links(sk, psock); rcu_read_unlock(); - sk_psock_stop(psock, false); + sk_psock_stop(psock); sk_psock_put(sk, psock); saved_destroy(sk); } @@ -1619,9 +1619,10 @@ void sock_map_close(struct sock *sk, long timeout) saved_close = psock->saved_close; sock_map_remove_links(sk, psock); rcu_read_unlock(); - sk_psock_stop(psock, true); - sk_psock_put(sk, psock); + sk_psock_stop(psock); release_sock(sk); + cancel_work_sync(&psock->work); + sk_psock_put(sk, psock); saved_close(sk, timeout); } EXPORT_SYMBOL_GPL(sock_map_close); -- GitLab From b9a2b0944cf77aadbeb82b05c3d1faa472d83778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jouni=20H=C3=B6gander?= Date: Wed, 2 Nov 2022 19:45:43 +0200 Subject: [PATCH 0983/1927] drm/i915/psr: Ensure panel granularity aligns with DSC slice height MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Do not enable psr2 if panel ganularity is not aligned with DSC slice height when DSC is enabled Cc: José Roberto de Souza Cc: Mika Kahola Signed-off-by: Jouni Högander Reviewed-by: José Roberto de Souza Reviewed-by: Manasi Navare Signed-off-by: José Roberto de Souza Link: https://patchwork.freedesktop.org/patch/msgid/20221102174544.2288205-2-jouni.hogander@intel.com --- drivers/gpu/drm/i915/display/intel_psr.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index e11b0592055fb..57575b5c6d48c 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -779,6 +779,7 @@ static bool psr2_granularity_check(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state) { struct drm_i915_private *dev_priv = dp_to_i915(intel_dp); + const struct drm_dsc_config *vdsc_cfg = &crtc_state->dsc.config; const int crtc_hdisplay = crtc_state->hw.adjusted_mode.crtc_hdisplay; const int crtc_vdisplay = crtc_state->hw.adjusted_mode.crtc_vdisplay; u16 y_granularity = 0; @@ -809,6 +810,10 @@ static bool psr2_granularity_check(struct intel_dp *intel_dp, if (y_granularity == 0 || crtc_vdisplay % y_granularity) return false; + if (crtc_state->dsc.compression_enable && + vdsc_cfg->slice_height % y_granularity) + return false; + crtc_state->su_y_granularity = y_granularity; return true; } -- GitLab From f46e3f5ffc0fff6224a27117126008b2f4d94eba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jouni=20H=C3=B6gander?= Date: Wed, 2 Nov 2022 19:45:44 +0200 Subject: [PATCH 0984/1927] drm/i915/psr: Remove inappropriate DSC slice alignment warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selective update area is now aligned with DSC slice height when DSC is enabled. Remove inappropriate warning about missing DSC alignment. Cc: José Roberto de Souza Cc: Mika Kahola Fixes: 47d4ae2192cb ("drm/i915/mtl: Extend PSR support") Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7212 Signed-off-by: Jouni Högander Signed-off-by: Anshuman Gupta Reviewed-by: Mika Kahola Reviewed-by: José Roberto de Souza Signed-off-by: José Roberto de Souza Link: https://patchwork.freedesktop.org/patch/msgid/20221102174544.2288205-3-jouni.hogander@intel.com --- drivers/gpu/drm/i915/display/intel_psr.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 57575b5c6d48c..a75b378515041 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -1684,9 +1684,6 @@ static void intel_psr2_sel_fetch_pipe_alignment(const struct intel_crtc_state *c pipe_clip->y1 -= pipe_clip->y1 % y_alignment; if (pipe_clip->y2 % y_alignment) pipe_clip->y2 = ((pipe_clip->y2 / y_alignment) + 1) * y_alignment; - - if (IS_ALDERLAKE_P(dev_priv) && crtc_state->dsc.compression_enable) - drm_warn(&dev_priv->drm, "Missing PSR2 sel fetch alignment with DSC\n"); } /* -- GitLab From 62aa1a344b0904549f6de7af958e8a1136fd5228 Mon Sep 17 00:00:00 2001 From: Sean Nyekjaer Date: Thu, 3 Nov 2022 09:00:42 +0100 Subject: [PATCH 0985/1927] spi: stm32: fix stm32_spi_prepare_mbr() that halves spi clk for every run When this driver is used with a driver that uses preallocated spi_transfer structs. The speed_hz is halved by every run. This results in: spi_stm32 44004000.spi: SPI transfer setup failed ads7846 spi0.0: SPI transfer failed: -22 Example when running with DIV_ROUND_UP(): - First run; speed_hz = 1000000, spi->clk_rate 125000000 div 125 -> mbrdiv = 7, cur_speed = 976562 - Second run; speed_hz = 976562 div 128,00007 (roundup to 129) -> mbrdiv = 8, cur_speed = 488281 - Third run; speed_hz = 488281 div 256,000131072067109 (roundup to 257) and then -EINVAL is returned. Use DIV_ROUND_CLOSEST to allow to round down and allow us to keep the set speed. Signed-off-by: Sean Nyekjaer Link: https://lore.kernel.org/r/20221103080043.3033414-1-sean@geanix.com Signed-off-by: Mark Brown --- drivers/spi/spi-stm32.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-stm32.c b/drivers/spi/spi-stm32.c index 3c2fa2e2f94a3..def09cf0dc147 100644 --- a/drivers/spi/spi-stm32.c +++ b/drivers/spi/spi-stm32.c @@ -434,7 +434,7 @@ static int stm32_spi_prepare_mbr(struct stm32_spi *spi, u32 speed_hz, u32 div, mbrdiv; /* Ensure spi->clk_rate is even */ - div = DIV_ROUND_UP(spi->clk_rate & ~0x1, speed_hz); + div = DIV_ROUND_CLOSEST(spi->clk_rate & ~0x1, speed_hz); /* * SPI framework set xfer->speed_hz to master->max_speed_hz if -- GitLab From 074c008007197297aaff9dd93627fb89b27d21e5 Mon Sep 17 00:00:00 2001 From: Ben Gardon Date: Wed, 2 Nov 2022 20:53:59 +0000 Subject: [PATCH 0986/1927] KVM: x86: Use SRCU to protect zap in __kvm_set_or_clear_apicv_inhibit() kvm_zap_gfn_range() must be called in an SRCU read-critical section, but there is no SRCU annotation in __kvm_set_or_clear_apicv_inhibit(). This can lead to the following warning via kvm_arch_vcpu_ioctl_set_guest_debug() if a Shadow MMU is in use (TDP MMU disabled or nesting): [ 1416.659809] ============================= [ 1416.659810] WARNING: suspicious RCU usage [ 1416.659839] 6.1.0-dbg-DEV #1 Tainted: G S I [ 1416.659853] ----------------------------- [ 1416.659854] include/linux/kvm_host.h:954 suspicious rcu_dereference_check() usage! [ 1416.659856] ... [ 1416.659904] dump_stack_lvl+0x84/0xaa [ 1416.659910] dump_stack+0x10/0x15 [ 1416.659913] lockdep_rcu_suspicious+0x11e/0x130 [ 1416.659919] kvm_zap_gfn_range+0x226/0x5e0 [ 1416.659926] ? kvm_make_all_cpus_request_except+0x18b/0x1e0 [ 1416.659935] __kvm_set_or_clear_apicv_inhibit+0xcc/0x100 [ 1416.659940] kvm_arch_vcpu_ioctl_set_guest_debug+0x350/0x390 [ 1416.659946] kvm_vcpu_ioctl+0x2fc/0x620 [ 1416.659955] __se_sys_ioctl+0x77/0xc0 [ 1416.659962] __x64_sys_ioctl+0x1d/0x20 [ 1416.659965] do_syscall_64+0x3d/0x80 [ 1416.659969] entry_SYSCALL_64_after_hwframe+0x63/0xcd Always take the KVM SRCU read lock in __kvm_set_or_clear_apicv_inhibit() to protect the GFN to memslot translation. The SRCU read lock is not technically required when no Shadow MMUs are in use, since the TDP MMU walks the paging structures from the roots and does not need to look up GFN translations in the memslots, but make the SRCU locking unconditional for simplicty. In most cases, the SRCU locking is taken care of in the vCPU run loop, but when called through other ioctls (such as KVM_SET_GUEST_DEBUG) there is no srcu_read_lock. Tested: ran tools/testing/selftests/kvm/x86_64/debug_regs on a DBG build. This patch causes the suspicious RCU warning to disappear. Note that the warning is hit in __kvm_zap_rmaps(), so kvm_memslots_have_rmaps() must return true in order for this to repro (i.e. the TDP MMU must be off or nesting in use.) Reported-by: Greg Thelen Fixes: 36222b117e36 ("KVM: x86: don't disable APICv memslot when inhibited") Signed-off-by: Ben Gardon Message-Id: <20221102205359.1260980-1-bgardon@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 521b433f978c7..5f5eb577d583c 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10404,7 +10404,10 @@ void __kvm_set_or_clear_apicv_inhibit(struct kvm *kvm, kvm->arch.apicv_inhibit_reasons = new; if (new) { unsigned long gfn = gpa_to_gfn(APIC_DEFAULT_PHYS_BASE); + int idx = srcu_read_lock(&kvm->srcu); + kvm_zap_gfn_range(kvm, gfn, gfn+1); + srcu_read_unlock(&kvm->srcu, idx); } } else { kvm->arch.apicv_inhibit_reasons = new; -- GitLab From 8670866b236eafbe9d502294561c3ddd298266bc Mon Sep 17 00:00:00 2001 From: Liao Chang Date: Thu, 3 Nov 2022 09:17:49 +0800 Subject: [PATCH 0987/1927] KVM: x86: Fix a typo about the usage of kvcalloc() Swap the 1st and 2nd arguments to be consistent with the usage of kvcalloc(). Fixes: c9b8fecddb5b ("KVM: use kvcalloc for array allocations") Signed-off-by: Liao Chang Message-Id: <20221103011749.139262-1-liaochang1@huawei.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/cpuid.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 0810e93cbedc0..62bc7a01cecca 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1338,7 +1338,7 @@ int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid, if (sanity_check_entries(entries, cpuid->nent, type)) return -EINVAL; - array.entries = kvcalloc(sizeof(struct kvm_cpuid_entry2), cpuid->nent, GFP_KERNEL); + array.entries = kvcalloc(cpuid->nent, sizeof(struct kvm_cpuid_entry2), GFP_KERNEL); if (!array.entries) return -ENOMEM; -- GitLab From 507d7c17cab274016dd43c8661d4586ba7504972 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:09 +0200 Subject: [PATCH 0988/1927] drm/i915/gmbus: move GPIO enum to gmbus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GPIO enum is only used in intel_gmbus.c, hide it there. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/3c386ce08d7d53a45c14c2e7519e4cc78a8161be.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_display.h | 18 ------------------ drivers/gpu/drm/i915/display/intel_gmbus.c | 20 +++++++++++++++++++- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index 918c8f432172b..3cc85ef3b111d 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -61,24 +61,6 @@ struct intel_remapped_info; struct intel_rotation_info; struct pci_dev; -enum i915_gpio { - GPIOA, - GPIOB, - GPIOC, - GPIOD, - GPIOE, - GPIOF, - GPIOG, - GPIOH, - __GPIOI_UNUSED, - GPIOJ, - GPIOK, - GPIOL, - GPIOM, - GPION, - GPIOO, -}; - /* * Keep the pipe enum values fixed: the code assumes that PIPE_A=0, the * rest have consecutive values and match the enum values of transcoders diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index 74443f57f62df..860e0f8b6b19f 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -49,9 +49,27 @@ struct intel_gmbus { struct drm_i915_private *i915; }; +enum gmbus_gpio { + GPIOA, + GPIOB, + GPIOC, + GPIOD, + GPIOE, + GPIOF, + GPIOG, + GPIOH, + __GPIOI_UNUSED, + GPIOJ, + GPIOK, + GPIOL, + GPIOM, + GPION, + GPIOO, +}; + struct gmbus_pin { const char *name; - enum i915_gpio gpio; + enum gmbus_gpio gpio; }; /* Map gmbus pin pairs to names and registers. */ -- GitLab From 10b60b56a05b495a5ba0ced173ba995e8a373f39 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:10 +0200 Subject: [PATCH 0989/1927] drm/i915: reduce includes in intel_connector.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only include what's needed. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/3aa1e27284e0ac308938978ae7da9ea9fbacad9e.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_connector.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/display/intel_connector.h b/drivers/gpu/drm/i915/display/intel_connector.h index 7d7b588d2286f..9d2bc261b2042 100644 --- a/drivers/gpu/drm/i915/display/intel_connector.h +++ b/drivers/gpu/drm/i915/display/intel_connector.h @@ -6,7 +6,7 @@ #ifndef __INTEL_CONNECTOR_H__ #define __INTEL_CONNECTOR_H__ -#include "intel_display.h" +#include struct drm_connector; struct edid; -- GitLab From 19cfeb414ea3d7d1beba0dadc466618e5671805e Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:11 +0200 Subject: [PATCH 0990/1927] drm/i915: reduce includes in intel_fifo_underrun.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only include what's needed. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/29b57e168e8af842baad2626959cea258402a2c1.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_fifo_underrun.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.h b/drivers/gpu/drm/i915/display/intel_fifo_underrun.h index e04f22ac1f49a..2e47d7d3c101e 100644 --- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.h +++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.h @@ -8,9 +8,8 @@ #include -#include "intel_display.h" - struct drm_i915_private; +enum pipe; bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv, enum pipe pipe, bool enable); -- GitLab From d29c410f77fd3f7dc632f5f153f455e6163d5449 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:12 +0200 Subject: [PATCH 0991/1927] drm/i915: un-inline icl_hdr_plane_mask() to simplify includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This lets us drop the heavy intel_display.h include from intel_sprite.h. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/16e13b6f207f52f8810a06f71a08e637f6397dc8.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_sprite.h | 9 +-------- drivers/gpu/drm/i915/display/skl_universal_plane.c | 5 +++++ drivers/gpu/drm/i915/display/skl_universal_plane.h | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_sprite.h b/drivers/gpu/drm/i915/display/intel_sprite.h index 4f63e49677314..4635c7ad23f95 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.h +++ b/drivers/gpu/drm/i915/display/intel_sprite.h @@ -8,14 +8,13 @@ #include -#include "intel_display.h" - struct drm_device; struct drm_display_mode; struct drm_file; struct drm_i915_private; struct intel_crtc_state; struct intel_plane_state; +enum pipe; /* * FIXME: We should instead only take spinlocks once for the entire update @@ -34,12 +33,6 @@ int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data, int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state); int chv_plane_check_rotation(const struct intel_plane_state *plane_state); -static inline u8 icl_hdr_plane_mask(void) -{ - return BIT(PLANE_PRIMARY) | - BIT(PLANE_SPRITE0) | BIT(PLANE_SPRITE1); -} - int ivb_plane_min_cdclk(const struct intel_crtc_state *crtc_state, const struct intel_plane_state *plane_state); int hsw_plane_min_cdclk(const struct intel_crtc_state *crtc_state, diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 7cb7130434089..46e10b369a3dd 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -246,6 +246,11 @@ bool icl_is_nv12_y_plane(struct drm_i915_private *dev_priv, icl_nv12_y_plane_mask(dev_priv) & BIT(plane_id); } +u8 icl_hdr_plane_mask(void) +{ + return BIT(PLANE_PRIMARY) | BIT(PLANE_SPRITE0) | BIT(PLANE_SPRITE1); +} + bool icl_is_hdr_plane(struct drm_i915_private *dev_priv, enum plane_id plane_id) { return DISPLAY_VER(dev_priv) >= 11 && diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.h b/drivers/gpu/drm/i915/display/skl_universal_plane.h index 351040b64dc74..be64c201f9b3e 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.h +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.h @@ -30,6 +30,7 @@ int skl_calc_main_surface_offset(const struct intel_plane_state *plane_state, bool icl_is_nv12_y_plane(struct drm_i915_private *dev_priv, enum plane_id plane_id); +u8 icl_hdr_plane_mask(void); bool icl_is_hdr_plane(struct drm_i915_private *dev_priv, enum plane_id plane_id); #endif -- GitLab From 2461bdb35e8ac4c7943d8277c118d5bba719d99c Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:13 +0200 Subject: [PATCH 0992/1927] drm/i915/dpio: un-inline the vlv phy/channel mapping functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify the heavy intel_display_types.h header. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/943dd3e9812138b1cf3ddcfde896cfec006f3847.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_display.c | 1 + .../drm/i915/display/intel_display_types.h | 45 ------------------- drivers/gpu/drm/i915/display/intel_dpio_phy.c | 42 +++++++++++++++++ drivers/gpu/drm/i915/display/intel_dpio_phy.h | 6 +++ drivers/gpu/drm/i915/display/intel_dpll.c | 1 + drivers/gpu/drm/i915/display/intel_pps.c | 1 + 6 files changed, 51 insertions(+), 45 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 0691b49f38f1f..31e91dbaa368a 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -90,6 +90,7 @@ #include "intel_display_types.h" #include "intel_dmc.h" #include "intel_dp_link_training.h" +#include "intel_dpio_phy.h" #include "intel_dpt.h" #include "intel_dsb.h" #include "intel_fbc.h" diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 5b38937c6bbdc..64314273995dc 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -1803,51 +1803,6 @@ struct intel_dp_mst_encoder { struct intel_connector *connector; }; -static inline enum dpio_channel -vlv_dig_port_to_channel(struct intel_digital_port *dig_port) -{ - switch (dig_port->base.port) { - default: - MISSING_CASE(dig_port->base.port); - fallthrough; - case PORT_B: - case PORT_D: - return DPIO_CH0; - case PORT_C: - return DPIO_CH1; - } -} - -static inline enum dpio_phy -vlv_dig_port_to_phy(struct intel_digital_port *dig_port) -{ - switch (dig_port->base.port) { - default: - MISSING_CASE(dig_port->base.port); - fallthrough; - case PORT_B: - case PORT_C: - return DPIO_PHY0; - case PORT_D: - return DPIO_PHY1; - } -} - -static inline enum dpio_channel -vlv_pipe_to_channel(enum pipe pipe) -{ - switch (pipe) { - default: - MISSING_CASE(pipe); - fallthrough; - case PIPE_A: - case PIPE_C: - return DPIO_CH0; - case PIPE_B: - return DPIO_CH1; - } -} - struct intel_load_detect_pipe { struct drm_atomic_state *restore_state; }; diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c index 8732b8722ed7a..6fc5b9e581528 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c @@ -655,6 +655,48 @@ bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder) return mask; } +enum dpio_channel vlv_dig_port_to_channel(struct intel_digital_port *dig_port) +{ + switch (dig_port->base.port) { + default: + MISSING_CASE(dig_port->base.port); + fallthrough; + case PORT_B: + case PORT_D: + return DPIO_CH0; + case PORT_C: + return DPIO_CH1; + } +} + +enum dpio_phy vlv_dig_port_to_phy(struct intel_digital_port *dig_port) +{ + switch (dig_port->base.port) { + default: + MISSING_CASE(dig_port->base.port); + fallthrough; + case PORT_B: + case PORT_C: + return DPIO_PHY0; + case PORT_D: + return DPIO_PHY1; + } +} + +enum dpio_channel vlv_pipe_to_channel(enum pipe pipe) +{ + switch (pipe) { + default: + MISSING_CASE(pipe); + fallthrough; + case PIPE_A: + case PIPE_C: + return DPIO_CH0; + case PIPE_B: + return DPIO_CH1; + } +} + void chv_set_phy_signal_level(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, u32 deemph_reg_value, u32 margin_reg_value, diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.h b/drivers/gpu/drm/i915/display/intel_dpio_phy.h index 9c3d008e8e1a6..828ad58624d85 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.h +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.h @@ -10,9 +10,11 @@ enum dpio_channel; enum dpio_phy; +enum pipe; enum port; struct drm_i915_private; struct intel_crtc_state; +struct intel_digital_port; struct intel_encoder; void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, @@ -30,6 +32,10 @@ void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, u8 lane_lat_optim_mask); u8 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); +enum dpio_channel vlv_dig_port_to_channel(struct intel_digital_port *dig_port); +enum dpio_phy vlv_dig_port_to_phy(struct intel_digital_port *dig_port); +enum dpio_channel vlv_pipe_to_channel(enum pipe pipe); + void chv_set_phy_signal_level(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, u32 deemph_reg_value, u32 margin_reg_value, diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c index b15ba78d64d62..fdc6fa4f2ed9e 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.c +++ b/drivers/gpu/drm/i915/display/intel_dpll.c @@ -10,6 +10,7 @@ #include "intel_de.h" #include "intel_display.h" #include "intel_display_types.h" +#include "intel_dpio_phy.h" #include "intel_dpll.h" #include "intel_lvds.h" #include "intel_panel.h" diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c index 21944f5bf3a83..22f5e08d396b1 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.c +++ b/drivers/gpu/drm/i915/display/intel_pps.c @@ -9,6 +9,7 @@ #include "intel_display_power_well.h" #include "intel_display_types.h" #include "intel_dp.h" +#include "intel_dpio_phy.h" #include "intel_dpll.h" #include "intel_lvds.h" #include "intel_pps.h" -- GitLab From 99417adb60e0d2fada04f57074358fba98e13c4a Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:14 +0200 Subject: [PATCH 0993/1927] drm/i915/dpio: move dpio_channel and dpio_phy enums to intel_dpio_phy.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reduce the size of intel_display.h by moving out the dpio_channel and dpio_phy enums. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/7c3ee7a6482540a0267f7b2974d22cab8188707a.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_display.h | 11 ----------- .../gpu/drm/i915/display/intel_display_power_well.h | 2 +- drivers/gpu/drm/i915/display/intel_dpio_phy.h | 13 +++++++++++-- drivers/gpu/drm/i915/gvt/display.c | 2 ++ drivers/gpu/drm/i915/gvt/handlers.c | 1 + drivers/gpu/drm/i915/gvt/mmio.c | 1 + drivers/gpu/drm/i915/intel_gvt_mmio_table.c | 1 + drivers/gpu/drm/i915/vlv_sideband.c | 2 ++ 8 files changed, 19 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index 3cc85ef3b111d..d9c8b8447f9e8 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -261,17 +261,6 @@ enum tc_port_mode { TC_PORT_LEGACY, }; -enum dpio_channel { - DPIO_CH0, - DPIO_CH1 -}; - -enum dpio_phy { - DPIO_PHY0, - DPIO_PHY1, - DPIO_PHY2, -}; - enum aux_ch { AUX_CH_A, AUX_CH_B, diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.h b/drivers/gpu/drm/i915/display/intel_display_power_well.h index e13b521e322a9..ba7cb977e7c7f 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power_well.h +++ b/drivers/gpu/drm/i915/display/intel_display_power_well.h @@ -7,8 +7,8 @@ #include -#include "intel_display.h" #include "intel_display_power.h" +#include "intel_dpio_phy.h" struct drm_i915_private; struct i915_power_well; diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.h b/drivers/gpu/drm/i915/display/intel_dpio_phy.h index 828ad58624d85..9c7725dacb470 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.h +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.h @@ -8,8 +8,6 @@ #include -enum dpio_channel; -enum dpio_phy; enum pipe; enum port; struct drm_i915_private; @@ -17,6 +15,17 @@ struct intel_crtc_state; struct intel_digital_port; struct intel_encoder; +enum dpio_channel { + DPIO_CH0, + DPIO_CH1, +}; + +enum dpio_phy { + DPIO_PHY0, + DPIO_PHY1, + DPIO_PHY2, +}; + void bxt_port_to_phy_channel(struct drm_i915_private *dev_priv, enum port port, enum dpio_phy *phy, enum dpio_channel *ch); void bxt_ddi_phy_set_signal_levels(struct intel_encoder *encoder, diff --git a/drivers/gpu/drm/i915/gvt/display.c b/drivers/gpu/drm/i915/gvt/display.c index c7722c818b4d5..c033249e73f4e 100644 --- a/drivers/gpu/drm/i915/gvt/display.c +++ b/drivers/gpu/drm/i915/gvt/display.c @@ -36,6 +36,8 @@ #include "i915_reg.h" #include "gvt.h" +#include "display/intel_dpio_phy.h" + static int get_edp_pipe(struct intel_vgpu *vgpu) { u32 data = vgpu_vreg(vgpu, _TRANS_DDI_FUNC_CTL_EDP); diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index e7e33f95cc515..a2439a009cad9 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -43,6 +43,7 @@ #include "intel_mchbar_regs.h" #include "display/intel_display_types.h" #include "display/intel_dmc_regs.h" +#include "display/intel_dpio_phy.h" #include "display/intel_fbc.h" #include "display/vlv_dsi_pll_regs.h" #include "gt/intel_gt_regs.h" diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c index 9acc00505fde1..5b5def6ddef7a 100644 --- a/drivers/gpu/drm/i915/gvt/mmio.c +++ b/drivers/gpu/drm/i915/gvt/mmio.c @@ -37,6 +37,7 @@ #include "i915_reg.h" #include "gvt.h" +#include "display/intel_dpio_phy.h" #include "gt/intel_gt_regs.h" /** diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c index e015bc91a26f6..26f6418af2f7f 100644 --- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c +++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c @@ -6,6 +6,7 @@ #include "display/intel_audio_regs.h" #include "display/intel_backlight_regs.h" #include "display/intel_dmc_regs.h" +#include "display/intel_dpio_phy.h" #include "display/vlv_dsi_pll_regs.h" #include "gt/intel_gt_regs.h" #include "gvt/gvt.h" diff --git a/drivers/gpu/drm/i915/vlv_sideband.c b/drivers/gpu/drm/i915/vlv_sideband.c index c26001300ebd8..6eea6e1a99c08 100644 --- a/drivers/gpu/drm/i915/vlv_sideband.c +++ b/drivers/gpu/drm/i915/vlv_sideband.c @@ -8,6 +8,8 @@ #include "i915_reg.h" #include "vlv_sideband.h" +#include "display/intel_dpio_phy.h" + /* * IOSF sideband, see VLV2_SidebandMsg_HAS.docx and * VLV_VLV2_PUNIT_HAS_0.8.docx -- GitLab From 50ae1a1c5edee3c6dfa3b7db7379ac0c4466051b Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:15 +0200 Subject: [PATCH 0994/1927] drm/i915: reduce includes in intel_display_power.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only include what's needed. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/a7f41cf6d13ee78c0b3a7c5927680bb94edfc5fb.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_display_power.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/display/intel_display_power.h b/drivers/gpu/drm/i915/display/intel_display_power.h index 7136ea3f233e9..1e77e52c87fec 100644 --- a/drivers/gpu/drm/i915/display/intel_display_power.h +++ b/drivers/gpu/drm/i915/display/intel_display_power.h @@ -6,11 +6,12 @@ #ifndef __INTEL_DISPLAY_POWER_H__ #define __INTEL_DISPLAY_POWER_H__ -#include "intel_runtime_pm.h" +#include "intel_wakeref.h" enum aux_ch; enum dpio_channel; enum dpio_phy; +enum i915_drm_suspend_mode; enum port; struct drm_i915_private; struct i915_power_well; -- GitLab From 356d725d62fe9d60157fcfd482ccc3fc70ee1c6c Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:16 +0200 Subject: [PATCH 0995/1927] drm/i915/display: reduce the includes in intel_dvo_dev.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only include what's needed. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/8523aa88daaf2dbf0ce138980142dbcd6d1ae0bc.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_dvo_dev.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_dvo_dev.h b/drivers/gpu/drm/i915/display/intel_dvo_dev.h index 50205f064d932..ecff7b190856c 100644 --- a/drivers/gpu/drm/i915/display/intel_dvo_dev.h +++ b/drivers/gpu/drm/i915/display/intel_dvo_dev.h @@ -23,12 +23,12 @@ #ifndef __INTEL_DVO_DEV_H__ #define __INTEL_DVO_DEV_H__ -#include - -#include - #include "i915_reg_defs.h" +enum drm_connector_status; +struct drm_display_mode; +struct i2c_adapter; + struct intel_dvo_device { const char *name; int type; -- GitLab From 22b2c7691cace5928fffef878a0d2d7435abbf51 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:17 +0200 Subject: [PATCH 0996/1927] drm/i915/display: reduce includes in intel_hdmi.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only include what's needed. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/5949a57979dae615731b6ff54d5d150b91e34d27.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_hdmi.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h index 93f65a917c368..774dda2376ed2 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.h +++ b/drivers/gpu/drm/i915/display/intel_hdmi.h @@ -6,20 +6,20 @@ #ifndef __INTEL_HDMI_H__ #define __INTEL_HDMI_H__ -#include #include +enum hdmi_infoframe_type; +enum port; struct drm_connector; +struct drm_connector_state; struct drm_encoder; struct drm_i915_private; struct intel_connector; +struct intel_crtc_state; struct intel_digital_port; struct intel_encoder; -struct intel_crtc_state; struct intel_hdmi; -struct drm_connector_state; union hdmi_infoframe; -enum port; void intel_hdmi_init_connector(struct intel_digital_port *dig_port, struct intel_connector *intel_connector); -- GitLab From 831209ec305ee9984a5036695cf73a9eb3f755bd Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:18 +0200 Subject: [PATCH 0997/1927] drm/i915/display: reduce includes in g4x_dp.h includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only include what's needed. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/e68c71ad1c1c1de7f5bc659edeb208818cddde72.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/g4x_dp.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/display/g4x_dp.h b/drivers/gpu/drm/i915/display/g4x_dp.h index e1f50263a7256..a38b3e1e01d31 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.h +++ b/drivers/gpu/drm/i915/display/g4x_dp.h @@ -8,7 +8,7 @@ #include -#include "i915_reg.h" +#include "i915_reg_defs.h" enum pipe; enum port; -- GitLab From ad7632ff0ebfacab5936e70a4187c2f0fa789627 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:19 +0200 Subject: [PATCH 0998/1927] drm/i915/irq: make gen2_irq_init()/gen2_irq_reset() static MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gen2 irq functions aren't used outside of i915_irq.h. Make them static, and remove the useless macro wrappers. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/b28f45ef4ef69ab7a6f96ffa3fa3118994667332.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/i915_irq.c | 10 +++++----- drivers/gpu/drm/i915/i915_irq.h | 9 --------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index b141001a090a2..b232a96478f66 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -248,7 +248,7 @@ void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr, intel_uncore_posting_read(uncore, iir); } -void gen2_irq_reset(struct intel_uncore *uncore) +static void gen2_irq_reset(struct intel_uncore *uncore) { intel_uncore_write16(uncore, GEN2_IMR, 0xffff); intel_uncore_posting_read16(uncore, GEN2_IMR); @@ -309,8 +309,8 @@ void gen3_irq_init(struct intel_uncore *uncore, intel_uncore_posting_read(uncore, imr); } -void gen2_irq_init(struct intel_uncore *uncore, - u32 imr_val, u32 ier_val) +static void gen2_irq_init(struct intel_uncore *uncore, + u32 imr_val, u32 ier_val) { gen2_assert_iir_is_zero(uncore); @@ -3871,7 +3871,7 @@ static void i8xx_irq_reset(struct drm_i915_private *dev_priv) i9xx_pipestat_irq_reset(dev_priv); - GEN2_IRQ_RESET(uncore); + gen2_irq_reset(uncore); dev_priv->irq_mask = ~0u; } @@ -3897,7 +3897,7 @@ static void i8xx_irq_postinstall(struct drm_i915_private *dev_priv) I915_MASTER_ERROR_INTERRUPT | I915_USER_INTERRUPT; - GEN2_IRQ_INIT(uncore, dev_priv->irq_mask, enable_mask); + gen2_irq_init(uncore, dev_priv->irq_mask, enable_mask); /* Interrupt setup is already guaranteed to be single-threaded, this is * just to make the assert_spin_locked check happy. */ diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h index 82639d9d7e820..9b004fc3444eb 100644 --- a/drivers/gpu/drm/i915/i915_irq.h +++ b/drivers/gpu/drm/i915/i915_irq.h @@ -90,12 +90,9 @@ void i965_disable_vblank(struct drm_crtc *crtc); void ilk_disable_vblank(struct drm_crtc *crtc); void bdw_disable_vblank(struct drm_crtc *crtc); -void gen2_irq_reset(struct intel_uncore *uncore); void gen3_irq_reset(struct intel_uncore *uncore, i915_reg_t imr, i915_reg_t iir, i915_reg_t ier); -void gen2_irq_init(struct intel_uncore *uncore, - u32 imr_val, u32 ier_val); void gen3_irq_init(struct intel_uncore *uncore, i915_reg_t imr, u32 imr_val, i915_reg_t ier, u32 ier_val, @@ -111,9 +108,6 @@ void gen3_irq_init(struct intel_uncore *uncore, #define GEN3_IRQ_RESET(uncore, type) \ gen3_irq_reset((uncore), type##IMR, type##IIR, type##IER) -#define GEN2_IRQ_RESET(uncore) \ - gen2_irq_reset(uncore) - #define GEN8_IRQ_INIT_NDX(uncore, type, which, imr_val, ier_val) \ ({ \ unsigned int which_ = which; \ @@ -129,7 +123,4 @@ void gen3_irq_init(struct intel_uncore *uncore, type##IER, ier_val, \ type##IIR) -#define GEN2_IRQ_INIT(uncore, imr_val, ier_val) \ - gen2_irq_init((uncore), imr_val, ier_val) - #endif /* __I915_IRQ_H__ */ -- GitLab From a4df7ac78513131abcaeafd9dc12467e2463f573 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:38:56 +0300 Subject: [PATCH 0999/1927] drm/i915: Use sizeof(variable) instead sizeof(type) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use sizeof(variable) instead of sizeof(type) in the hopes of less chance of screwing things up. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-2-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 4bb113c39f4b1..92cc43d5bad6e 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -565,7 +565,7 @@ create_linear_lut(struct drm_i915_private *i915, int lut_size) int i; blob = drm_property_create_blob(&i915->drm, - sizeof(struct drm_color_lut) * lut_size, + sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) return blob; @@ -1895,7 +1895,7 @@ static struct drm_property_blob *i9xx_read_lut_8(struct intel_crtc *crtc) int i; blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * LEGACY_LUT_LENGTH, + sizeof(lut[0]) * LEGACY_LUT_LENGTH, NULL); if (IS_ERR(blob)) return NULL; @@ -1930,7 +1930,7 @@ static struct drm_property_blob *i965_read_lut_10p6(struct intel_crtc *crtc) struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * lut_size, + sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) return NULL; @@ -1973,7 +1973,7 @@ static struct drm_property_blob *chv_read_cgm_gamma(struct intel_crtc *crtc) struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * lut_size, + sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) return NULL; @@ -2009,7 +2009,7 @@ static struct drm_property_blob *ilk_read_lut_8(struct intel_crtc *crtc) int i; blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * LEGACY_LUT_LENGTH, + sizeof(lut[0]) * LEGACY_LUT_LENGTH, NULL); if (IS_ERR(blob)) return NULL; @@ -2034,7 +2034,7 @@ static struct drm_property_blob *ilk_read_lut_10(struct intel_crtc *crtc) struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * lut_size, + sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) return NULL; @@ -2087,7 +2087,7 @@ static struct drm_property_blob *bdw_read_lut_10(struct intel_crtc *crtc, drm_WARN_ON(&dev_priv->drm, lut_size != hw_lut_size); blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * lut_size, + sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) return NULL; @@ -2138,7 +2138,7 @@ icl_read_lut_multi_segment(struct intel_crtc *crtc) struct drm_color_lut *lut; blob = drm_property_create_blob(&dev_priv->drm, - sizeof(struct drm_color_lut) * lut_size, + sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) return NULL; -- GitLab From 05ca98523481aa687c5a8dce8939fec539632153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:38:57 +0300 Subject: [PATCH 1000/1927] drm/i915: Use _MMIO_PIPE() for SKL_BOTTOM_COLOR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need to use _MMIO_PIPE2() for SKL_BOTTOM_COLOR since all pipe registers are evenly spread on skl+. Switch to _MMIO_PIPE() and thus avoid the hidden dev_priv. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-3-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/i915_reg.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index c5e180aec3447..5a062ee46ed1d 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -3747,9 +3747,10 @@ /* Skylake+ pipe bottom (background) color */ #define _SKL_BOTTOM_COLOR_A 0x70034 +#define _SKL_BOTTOM_COLOR_B 0x71034 #define SKL_BOTTOM_COLOR_GAMMA_ENABLE REG_BIT(31) #define SKL_BOTTOM_COLOR_CSC_ENABLE REG_BIT(30) -#define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE2(pipe, _SKL_BOTTOM_COLOR_A) +#define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE(pipe, _SKL_BOTTOM_COLOR_A, _SKL_BOTTOM_COLOR_B) #define _ICL_PIPE_A_STATUS 0x70058 #define ICL_PIPESTATUS(pipe) _MMIO_PIPE2(pipe, _ICL_PIPE_A_STATUS) -- GitLab From c9d4911cab39a7d91c5115c1f8c0414a69d27974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:38:58 +0300 Subject: [PATCH 1001/1927] drm/i915: s/dev_priv/i915/ in intel_color.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch intel_color.c over to the modern 'i915' variable naming scehme. The only exceptions are the i9xx LUT access functions which still need the magic 'dev_priv' for the register macros. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-4-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 278 ++++++++++----------- 1 file changed, 139 insertions(+), 139 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 92cc43d5bad6e..415e0a6839a4d 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -184,31 +184,31 @@ static void ilk_update_pipe_csc(struct intel_crtc *crtc, const u16 coeff[9], const u16 postoff[3]) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - intel_de_write_fw(dev_priv, PIPE_CSC_PREOFF_HI(pipe), preoff[0]); - intel_de_write_fw(dev_priv, PIPE_CSC_PREOFF_ME(pipe), preoff[1]); - intel_de_write_fw(dev_priv, PIPE_CSC_PREOFF_LO(pipe), preoff[2]); + intel_de_write_fw(i915, PIPE_CSC_PREOFF_HI(pipe), preoff[0]); + intel_de_write_fw(i915, PIPE_CSC_PREOFF_ME(pipe), preoff[1]); + intel_de_write_fw(i915, PIPE_CSC_PREOFF_LO(pipe), preoff[2]); - intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_RY_GY(pipe), + intel_de_write_fw(i915, PIPE_CSC_COEFF_RY_GY(pipe), coeff[0] << 16 | coeff[1]); - intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_BY(pipe), coeff[2] << 16); + intel_de_write_fw(i915, PIPE_CSC_COEFF_BY(pipe), coeff[2] << 16); - intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_RU_GU(pipe), + intel_de_write_fw(i915, PIPE_CSC_COEFF_RU_GU(pipe), coeff[3] << 16 | coeff[4]); - intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_BU(pipe), coeff[5] << 16); + intel_de_write_fw(i915, PIPE_CSC_COEFF_BU(pipe), coeff[5] << 16); - intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_RV_GV(pipe), + intel_de_write_fw(i915, PIPE_CSC_COEFF_RV_GV(pipe), coeff[6] << 16 | coeff[7]); - intel_de_write_fw(dev_priv, PIPE_CSC_COEFF_BV(pipe), coeff[8] << 16); + intel_de_write_fw(i915, PIPE_CSC_COEFF_BV(pipe), coeff[8] << 16); - if (DISPLAY_VER(dev_priv) >= 7) { - intel_de_write_fw(dev_priv, PIPE_CSC_POSTOFF_HI(pipe), + if (DISPLAY_VER(i915) >= 7) { + intel_de_write_fw(i915, PIPE_CSC_POSTOFF_HI(pipe), postoff[0]); - intel_de_write_fw(dev_priv, PIPE_CSC_POSTOFF_ME(pipe), + intel_de_write_fw(i915, PIPE_CSC_POSTOFF_ME(pipe), postoff[1]); - intel_de_write_fw(dev_priv, PIPE_CSC_POSTOFF_LO(pipe), + intel_de_write_fw(i915, PIPE_CSC_POSTOFF_LO(pipe), postoff[2]); } } @@ -218,44 +218,44 @@ static void icl_update_output_csc(struct intel_crtc *crtc, const u16 coeff[9], const u16 postoff[3]) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_PREOFF_HI(pipe), preoff[0]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_PREOFF_ME(pipe), preoff[1]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_PREOFF_LO(pipe), preoff[2]); + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_PREOFF_HI(pipe), preoff[0]); + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_PREOFF_ME(pipe), preoff[1]); + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_PREOFF_LO(pipe), preoff[2]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe), + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_COEFF_RY_GY(pipe), coeff[0] << 16 | coeff[1]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_BY(pipe), + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_COEFF_BY(pipe), coeff[2] << 16); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe), + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_COEFF_RU_GU(pipe), coeff[3] << 16 | coeff[4]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_BU(pipe), + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_COEFF_BU(pipe), coeff[5] << 16); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe), + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_COEFF_RV_GV(pipe), coeff[6] << 16 | coeff[7]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_COEFF_BV(pipe), + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_COEFF_BV(pipe), coeff[8] << 16); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_HI(pipe), postoff[0]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_ME(pipe), postoff[1]); - intel_de_write_fw(dev_priv, PIPE_CSC_OUTPUT_POSTOFF_LO(pipe), postoff[2]); + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_POSTOFF_HI(pipe), postoff[0]); + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_POSTOFF_ME(pipe), postoff[1]); + intel_de_write_fw(i915, PIPE_CSC_OUTPUT_POSTOFF_LO(pipe), postoff[2]); } static bool ilk_csc_limited_range(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); /* * FIXME if there's a gamma LUT after the CSC, we should * do the range compression using the gamma LUT instead. */ return crtc_state->limited_color_range && - (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) || - IS_DISPLAY_VER(dev_priv, 9, 10)); + (IS_HASWELL(i915) || IS_BROADWELL(i915) || + IS_DISPLAY_VER(i915, 9, 10)); } static void ilk_csc_convert_ctm(const struct intel_crtc_state *crtc_state, @@ -313,7 +313,7 @@ static void ilk_csc_convert_ctm(const struct intel_crtc_state *crtc_state, static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); bool limited_color_range = ilk_csc_limited_range(crtc_state); if (crtc_state->hw.ctm) { @@ -339,7 +339,7 @@ static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) * LUT is needed but CSC is not we need to load an * identity matrix. */ - drm_WARN_ON(&dev_priv->drm, !IS_GEMINILAKE(dev_priv)); + drm_WARN_ON(&i915->drm, !IS_GEMINILAKE(i915)); ilk_update_pipe_csc(crtc, ilk_csc_off_zero, ilk_csc_coeff_identity, @@ -373,7 +373,7 @@ static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state) static void chv_load_cgm_csc(struct intel_crtc *crtc, const struct drm_property_blob *blob) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_color_ctm *ctm = blob->data; enum pipe pipe = crtc->pipe; u16 coeffs[9]; @@ -397,15 +397,15 @@ static void chv_load_cgm_csc(struct intel_crtc *crtc, coeffs[i] |= (abs_coeff >> 20) & 0xfff; } - intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF01(pipe), + intel_de_write_fw(i915, CGM_PIPE_CSC_COEFF01(pipe), coeffs[1] << 16 | coeffs[0]); - intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF23(pipe), + intel_de_write_fw(i915, CGM_PIPE_CSC_COEFF23(pipe), coeffs[3] << 16 | coeffs[2]); - intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF45(pipe), + intel_de_write_fw(i915, CGM_PIPE_CSC_COEFF45(pipe), coeffs[5] << 16 | coeffs[4]); - intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF67(pipe), + intel_de_write_fw(i915, CGM_PIPE_CSC_COEFF67(pipe), coeffs[7] << 16 | coeffs[6]); - intel_de_write_fw(dev_priv, CGM_PIPE_CSC_COEFF8(pipe), + intel_de_write_fw(i915, CGM_PIPE_CSC_COEFF8(pipe), coeffs[8]); } @@ -511,31 +511,31 @@ static void i9xx_color_commit_arm(const struct intel_crtc_state *crtc_state) static void ilk_color_commit_arm(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); /* update PIPECONF GAMMA_MODE */ ilk_set_pipeconf(crtc_state); - intel_de_write_fw(dev_priv, PIPE_CSC_MODE(crtc->pipe), + intel_de_write_fw(i915, PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode); } static void hsw_color_commit_arm(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); - intel_de_write(dev_priv, GAMMA_MODE(crtc->pipe), + intel_de_write(i915, GAMMA_MODE(crtc->pipe), crtc_state->gamma_mode); - intel_de_write_fw(dev_priv, PIPE_CSC_MODE(crtc->pipe), + intel_de_write_fw(i915, PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode); } static void skl_color_commit_arm(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; u32 val = 0; @@ -548,12 +548,12 @@ static void skl_color_commit_arm(const struct intel_crtc_state *crtc_state) val |= SKL_BOTTOM_COLOR_GAMMA_ENABLE; if (crtc_state->csc_enable) val |= SKL_BOTTOM_COLOR_CSC_ENABLE; - intel_de_write(dev_priv, SKL_BOTTOM_COLOR(pipe), val); + intel_de_write(i915, SKL_BOTTOM_COLOR(pipe), val); - intel_de_write(dev_priv, GAMMA_MODE(crtc->pipe), + intel_de_write(i915, GAMMA_MODE(crtc->pipe), crtc_state->gamma_mode); - intel_de_write_fw(dev_priv, PIPE_CSC_MODE(crtc->pipe), + intel_de_write_fw(i915, PIPE_CSC_MODE(crtc->pipe), crtc_state->csc_mode); } @@ -643,7 +643,7 @@ static void i965_load_luts(const struct intel_crtc_state *crtc_state) static void ilk_load_lut_8(struct intel_crtc *crtc, const struct drm_property_blob *blob) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_color_lut *lut; enum pipe pipe = crtc->pipe; int i; @@ -654,20 +654,20 @@ static void ilk_load_lut_8(struct intel_crtc *crtc, lut = blob->data; for (i = 0; i < 256; i++) - intel_de_write_fw(dev_priv, LGC_PALETTE(pipe, i), + intel_de_write_fw(i915, LGC_PALETTE(pipe, i), i9xx_lut_8(&lut[i])); } static void ilk_load_lut_10(struct intel_crtc *crtc, const struct drm_property_blob *blob) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; for (i = 0; i < lut_size; i++) - intel_de_write_fw(dev_priv, PREC_PALETTE(pipe, i), + intel_de_write_fw(i915, PREC_PALETTE(pipe, i), ilk_lut_10(&lut[i])); } @@ -708,7 +708,7 @@ static void ivb_load_lut_10(struct intel_crtc *crtc, const struct drm_property_blob *blob, u32 prec_index) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); int hw_lut_size = ivb_lut_10_size(prec_index); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); @@ -719,8 +719,8 @@ static void ivb_load_lut_10(struct intel_crtc *crtc, const struct drm_color_lut *entry = &lut[i * (lut_size - 1) / (hw_lut_size - 1)]; - intel_de_write_fw(dev_priv, PREC_PAL_INDEX(pipe), prec_index++); - intel_de_write_fw(dev_priv, PREC_PAL_DATA(pipe), + intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), prec_index++); + intel_de_write_fw(i915, PREC_PAL_DATA(pipe), ilk_lut_10(entry)); } @@ -728,7 +728,7 @@ static void ivb_load_lut_10(struct intel_crtc *crtc, * Reset the index, otherwise it prevents the legacy palette to be * written properly. */ - intel_de_write_fw(dev_priv, PREC_PAL_INDEX(pipe), 0); + intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), 0); } /* On BDW+ the index auto increment mode actually works */ @@ -736,13 +736,13 @@ static void bdw_load_lut_10(struct intel_crtc *crtc, const struct drm_property_blob *blob, u32 prec_index) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); int hw_lut_size = ivb_lut_10_size(prec_index); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; - intel_de_write_fw(dev_priv, PREC_PAL_INDEX(pipe), + intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), prec_index | PAL_PREC_AUTO_INCREMENT); for (i = 0; i < hw_lut_size; i++) { @@ -750,7 +750,7 @@ static void bdw_load_lut_10(struct intel_crtc *crtc, const struct drm_color_lut *entry = &lut[i * (lut_size - 1) / (hw_lut_size - 1)]; - intel_de_write_fw(dev_priv, PREC_PAL_DATA(pipe), + intel_de_write_fw(i915, PREC_PAL_DATA(pipe), ilk_lut_10(entry)); } @@ -758,13 +758,13 @@ static void bdw_load_lut_10(struct intel_crtc *crtc, * Reset the index, otherwise it prevents the legacy palette to be * written properly. */ - intel_de_write_fw(dev_priv, PREC_PAL_INDEX(pipe), 0); + intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), 0); } static void ivb_load_lut_ext_max(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; /* Program the max register to clamp values > 1.0. */ @@ -777,7 +777,7 @@ static void ivb_load_lut_ext_max(const struct intel_crtc_state *crtc_state) * ToDo: Extend the ABI to be able to program values * from 3.0 to 7.0 */ - if (DISPLAY_VER(dev_priv) >= 10) { + if (DISPLAY_VER(i915) >= 10) { intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 0), 1 << 16); intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 1), @@ -858,7 +858,7 @@ static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state, const struct drm_property_blob *blob) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; @@ -868,8 +868,8 @@ static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state, * ignore the index bits, so we need to reset it to index 0 * separately. */ - intel_de_write_fw(dev_priv, PRE_CSC_GAMC_INDEX(pipe), 0); - intel_de_write_fw(dev_priv, PRE_CSC_GAMC_INDEX(pipe), + intel_de_write_fw(i915, PRE_CSC_GAMC_INDEX(pipe), 0); + intel_de_write_fw(i915, PRE_CSC_GAMC_INDEX(pipe), PRE_CSC_GAMC_AUTO_INCREMENT); for (i = 0; i < lut_size; i++) { @@ -886,15 +886,15 @@ static void glk_load_degamma_lut(const struct intel_crtc_state *crtc_state, * ToDo: Extend to max 7.0. Enable 32 bit input value * as compared to just 16 to achieve this. */ - intel_de_write_fw(dev_priv, PRE_CSC_GAMC_DATA(pipe), + intel_de_write_fw(i915, PRE_CSC_GAMC_DATA(pipe), lut[i].green); } /* Clamp values > 1.0. */ - while (i++ < glk_degamma_lut_size(dev_priv)) - intel_de_write_fw(dev_priv, PRE_CSC_GAMC_DATA(pipe), 1 << 16); + while (i++ < glk_degamma_lut_size(i915)) + intel_de_write_fw(i915, PRE_CSC_GAMC_DATA(pipe), 1 << 16); - intel_de_write_fw(dev_priv, PRE_CSC_GAMC_INDEX(pipe), 0); + intel_de_write_fw(i915, PRE_CSC_GAMC_INDEX(pipe), 0); } static void glk_load_luts(const struct intel_crtc_state *crtc_state) @@ -1075,15 +1075,15 @@ static u32 chv_cgm_degamma_udw(const struct drm_color_lut *color) static void chv_load_cgm_degamma(struct intel_crtc *crtc, const struct drm_property_blob *blob) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; for (i = 0; i < lut_size; i++) { - intel_de_write_fw(dev_priv, CGM_PIPE_DEGAMMA(pipe, i, 0), + intel_de_write_fw(i915, CGM_PIPE_DEGAMMA(pipe, i, 0), chv_cgm_degamma_ldw(&lut[i])); - intel_de_write_fw(dev_priv, CGM_PIPE_DEGAMMA(pipe, i, 1), + intel_de_write_fw(i915, CGM_PIPE_DEGAMMA(pipe, i, 1), chv_cgm_degamma_udw(&lut[i])); } } @@ -1109,15 +1109,15 @@ static void chv_cgm_gamma_pack(struct drm_color_lut *entry, u32 ldw, u32 udw) static void chv_load_cgm_gamma(struct intel_crtc *crtc, const struct drm_property_blob *blob) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; for (i = 0; i < lut_size; i++) { - intel_de_write_fw(dev_priv, CGM_PIPE_GAMMA(pipe, i, 0), + intel_de_write_fw(i915, CGM_PIPE_GAMMA(pipe, i, 0), chv_cgm_gamma_ldw(&lut[i])); - intel_de_write_fw(dev_priv, CGM_PIPE_GAMMA(pipe, i, 1), + intel_de_write_fw(i915, CGM_PIPE_GAMMA(pipe, i, 1), chv_cgm_gamma_udw(&lut[i])); } } @@ -1125,7 +1125,7 @@ static void chv_load_cgm_gamma(struct intel_crtc *crtc, static void chv_load_luts(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); const struct drm_property_blob *pre_csc_lut = crtc_state->pre_csc_lut; const struct drm_property_blob *post_csc_lut = crtc_state->post_csc_lut; const struct drm_property_blob *ctm = crtc_state->hw.ctm; @@ -1141,30 +1141,30 @@ static void chv_load_luts(const struct intel_crtc_state *crtc_state) else i965_load_luts(crtc_state); - intel_de_write_fw(dev_priv, CGM_PIPE_MODE(crtc->pipe), + intel_de_write_fw(i915, CGM_PIPE_MODE(crtc->pipe), crtc_state->cgm_mode); } void intel_color_load_luts(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - dev_priv->display.funcs.color->load_luts(crtc_state); + i915->display.funcs.color->load_luts(crtc_state); } void intel_color_commit_noarm(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - if (dev_priv->display.funcs.color->color_commit_noarm) - dev_priv->display.funcs.color->color_commit_noarm(crtc_state); + if (i915->display.funcs.color->color_commit_noarm) + i915->display.funcs.color->color_commit_noarm(crtc_state); } void intel_color_commit_arm(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - dev_priv->display.funcs.color->color_commit_arm(crtc_state); + i915->display.funcs.color->color_commit_arm(crtc_state); } static bool intel_can_preload_luts(const struct intel_crtc_state *new_crtc_state) @@ -1200,23 +1200,23 @@ static bool chv_can_preload_luts(const struct intel_crtc_state *new_crtc_state) int intel_color_check(struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - return dev_priv->display.funcs.color->color_check(crtc_state); + return i915->display.funcs.color->color_check(crtc_state); } void intel_color_get_config(struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - if (dev_priv->display.funcs.color->read_luts) - dev_priv->display.funcs.color->read_luts(crtc_state); + if (i915->display.funcs.color->read_luts) + i915->display.funcs.color->read_luts(crtc_state); } static bool need_plane_update(struct intel_plane *plane, const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(plane->base.dev); + struct drm_i915_private *i915 = to_i915(plane->base.dev); /* * On pre-SKL the pipe gamma enable and pipe csc enable for @@ -1224,7 +1224,7 @@ static bool need_plane_update(struct intel_plane *plane, * We have to reconfigure that even if the plane is inactive. */ return crtc_state->active_planes & BIT(plane->id) || - (DISPLAY_VER(dev_priv) < 9 && + (DISPLAY_VER(i915) < 9 && plane->id == PLANE_PRIMARY); } @@ -1232,7 +1232,7 @@ static int intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state) { struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state); const struct intel_crtc_state *old_crtc_state = @@ -1247,7 +1247,7 @@ intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state) new_crtc_state->csc_enable == old_crtc_state->csc_enable) return 0; - for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) { + for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) { struct intel_plane_state *plane_state; if (!need_plane_update(plane, new_crtc_state)) @@ -1260,7 +1260,7 @@ intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state) new_crtc_state->update_planes |= BIT(plane->id); /* plane control register changes blocked by CxSR */ - if (HAS_GMCH(dev_priv)) + if (HAS_GMCH(i915)) new_crtc_state->disable_cxsr = true; } @@ -1286,7 +1286,7 @@ static int check_lut_size(const struct drm_property_blob *lut, int expected) static int check_luts(const struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); const struct drm_property_blob *gamma_lut = crtc_state->hw.gamma_lut; const struct drm_property_blob *degamma_lut = crtc_state->hw.degamma_lut; int gamma_length, degamma_length; @@ -1298,15 +1298,15 @@ static int check_luts(const struct intel_crtc_state *crtc_state) /* C8 relies on its palette being stored in the legacy LUT */ if (crtc_state->c8_planes) { - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(&i915->drm, "C8 pixelformat requires the legacy LUT\n"); return -EINVAL; } - degamma_length = INTEL_INFO(dev_priv)->display.color.degamma_lut_size; - gamma_length = INTEL_INFO(dev_priv)->display.color.gamma_lut_size; - degamma_tests = INTEL_INFO(dev_priv)->display.color.degamma_lut_tests; - gamma_tests = INTEL_INFO(dev_priv)->display.color.gamma_lut_tests; + degamma_length = INTEL_INFO(i915)->display.color.degamma_lut_size; + gamma_length = INTEL_INFO(i915)->display.color.gamma_lut_size; + degamma_tests = INTEL_INFO(i915)->display.color.degamma_lut_tests; + gamma_tests = INTEL_INFO(i915)->display.color.gamma_lut_tests; if (check_lut_size(degamma_lut, degamma_length) || check_lut_size(gamma_lut, gamma_length)) @@ -1550,7 +1550,7 @@ static u32 ivb_csc_mode(const struct intel_crtc_state *crtc_state) static int ivb_color_check(struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); bool limited_color_range = ilk_csc_limited_range(crtc_state); int ret; @@ -1560,7 +1560,7 @@ static int ivb_color_check(struct intel_crtc_state *crtc_state) if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && crtc_state->hw.ctm) { - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(&i915->drm, "YCBCR and CTM together are not possible\n"); return -EINVAL; } @@ -1617,7 +1617,7 @@ static void glk_assign_luts(struct intel_crtc_state *crtc_state) static int glk_color_check(struct intel_crtc_state *crtc_state) { - struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev); + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); int ret; ret = check_luts(crtc_state); @@ -1626,7 +1626,7 @@ static int glk_color_check(struct intel_crtc_state *crtc_state) if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && crtc_state->hw.ctm) { - drm_dbg_kms(&dev_priv->drm, + drm_dbg_kms(&i915->drm, "YCBCR and CTM together are not possible\n"); return -EINVAL; } @@ -1798,19 +1798,19 @@ static int icl_gamma_precision(const struct intel_crtc_state *crtc_state) int intel_color_get_gamma_bit_precision(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); - if (HAS_GMCH(dev_priv)) { - if (IS_CHERRYVIEW(dev_priv)) + if (HAS_GMCH(i915)) { + if (IS_CHERRYVIEW(i915)) return chv_gamma_precision(crtc_state); else return i9xx_gamma_precision(crtc_state); } else { - if (DISPLAY_VER(dev_priv) >= 11) + if (DISPLAY_VER(i915) >= 11) return icl_gamma_precision(crtc_state); - else if (DISPLAY_VER(dev_priv) == 10) + else if (DISPLAY_VER(i915) == 10) return glk_gamma_precision(crtc_state); - else if (IS_IRONLAKE(dev_priv)) + else if (IS_IRONLAKE(i915)) return ilk_gamma_precision(crtc_state); } @@ -1966,13 +1966,13 @@ static void i965_read_luts(struct intel_crtc_state *crtc_state) static struct drm_property_blob *chv_read_cgm_gamma(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - int i, lut_size = INTEL_INFO(dev_priv)->display.color.gamma_lut_size; + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + int i, lut_size = INTEL_INFO(i915)->display.color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; struct drm_color_lut *lut; - blob = drm_property_create_blob(&dev_priv->drm, + blob = drm_property_create_blob(&i915->drm, sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) @@ -1981,8 +1981,8 @@ static struct drm_property_blob *chv_read_cgm_gamma(struct intel_crtc *crtc) lut = blob->data; for (i = 0; i < lut_size; i++) { - u32 ldw = intel_de_read_fw(dev_priv, CGM_PIPE_GAMMA(pipe, i, 0)); - u32 udw = intel_de_read_fw(dev_priv, CGM_PIPE_GAMMA(pipe, i, 1)); + u32 ldw = intel_de_read_fw(i915, CGM_PIPE_GAMMA(pipe, i, 0)); + u32 udw = intel_de_read_fw(i915, CGM_PIPE_GAMMA(pipe, i, 1)); chv_cgm_gamma_pack(&lut[i], ldw, udw); } @@ -2002,13 +2002,13 @@ static void chv_read_luts(struct intel_crtc_state *crtc_state) static struct drm_property_blob *ilk_read_lut_8(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; struct drm_color_lut *lut; int i; - blob = drm_property_create_blob(&dev_priv->drm, + blob = drm_property_create_blob(&i915->drm, sizeof(lut[0]) * LEGACY_LUT_LENGTH, NULL); if (IS_ERR(blob)) @@ -2017,7 +2017,7 @@ static struct drm_property_blob *ilk_read_lut_8(struct intel_crtc *crtc) lut = blob->data; for (i = 0; i < LEGACY_LUT_LENGTH; i++) { - u32 val = intel_de_read_fw(dev_priv, LGC_PALETTE(pipe, i)); + u32 val = intel_de_read_fw(i915, LGC_PALETTE(pipe, i)); i9xx_lut_8_pack(&lut[i], val); } @@ -2027,13 +2027,13 @@ static struct drm_property_blob *ilk_read_lut_8(struct intel_crtc *crtc) static struct drm_property_blob *ilk_read_lut_10(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - int i, lut_size = INTEL_INFO(dev_priv)->display.color.gamma_lut_size; + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + int i, lut_size = INTEL_INFO(i915)->display.color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; struct drm_color_lut *lut; - blob = drm_property_create_blob(&dev_priv->drm, + blob = drm_property_create_blob(&i915->drm, sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) @@ -2042,7 +2042,7 @@ static struct drm_property_blob *ilk_read_lut_10(struct intel_crtc *crtc) lut = blob->data; for (i = 0; i < lut_size; i++) { - u32 val = intel_de_read_fw(dev_priv, PREC_PALETTE(pipe, i)); + u32 val = intel_de_read_fw(i915, PREC_PALETTE(pipe, i)); ilk_lut_10_pack(&lut[i], val); } @@ -2077,16 +2077,16 @@ static void ilk_read_luts(struct intel_crtc_state *crtc_state) static struct drm_property_blob *bdw_read_lut_10(struct intel_crtc *crtc, u32 prec_index) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); + struct drm_i915_private *i915 = to_i915(crtc->base.dev); int i, hw_lut_size = ivb_lut_10_size(prec_index); - int lut_size = INTEL_INFO(dev_priv)->display.color.gamma_lut_size; + int lut_size = INTEL_INFO(i915)->display.color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; struct drm_color_lut *lut; - drm_WARN_ON(&dev_priv->drm, lut_size != hw_lut_size); + drm_WARN_ON(&i915->drm, lut_size != hw_lut_size); - blob = drm_property_create_blob(&dev_priv->drm, + blob = drm_property_create_blob(&i915->drm, sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) @@ -2094,16 +2094,16 @@ static struct drm_property_blob *bdw_read_lut_10(struct intel_crtc *crtc, lut = blob->data; - intel_de_write_fw(dev_priv, PREC_PAL_INDEX(pipe), + intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), prec_index | PAL_PREC_AUTO_INCREMENT); for (i = 0; i < lut_size; i++) { - u32 val = intel_de_read_fw(dev_priv, PREC_PAL_DATA(pipe)); + u32 val = intel_de_read_fw(i915, PREC_PAL_DATA(pipe)); ilk_lut_10_pack(&lut[i], val); } - intel_de_write_fw(dev_priv, PREC_PAL_INDEX(pipe), 0); + intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), 0); return blob; } @@ -2131,13 +2131,13 @@ static void glk_read_luts(struct intel_crtc_state *crtc_state) static struct drm_property_blob * icl_read_lut_multi_segment(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - int i, lut_size = INTEL_INFO(dev_priv)->display.color.gamma_lut_size; + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + int i, lut_size = INTEL_INFO(i915)->display.color.gamma_lut_size; enum pipe pipe = crtc->pipe; struct drm_property_blob *blob; struct drm_color_lut *lut; - blob = drm_property_create_blob(&dev_priv->drm, + blob = drm_property_create_blob(&i915->drm, sizeof(lut[0]) * lut_size, NULL); if (IS_ERR(blob)) @@ -2145,17 +2145,17 @@ icl_read_lut_multi_segment(struct intel_crtc *crtc) lut = blob->data; - intel_de_write_fw(dev_priv, PREC_PAL_MULTI_SEG_INDEX(pipe), + intel_de_write_fw(i915, PREC_PAL_MULTI_SEG_INDEX(pipe), PAL_PREC_AUTO_INCREMENT); for (i = 0; i < 9; i++) { - u32 ldw = intel_de_read_fw(dev_priv, PREC_PAL_MULTI_SEG_DATA(pipe)); - u32 udw = intel_de_read_fw(dev_priv, PREC_PAL_MULTI_SEG_DATA(pipe)); + u32 ldw = intel_de_read_fw(i915, PREC_PAL_MULTI_SEG_DATA(pipe)); + u32 udw = intel_de_read_fw(i915, PREC_PAL_MULTI_SEG_DATA(pipe)); icl_lut_multi_seg_pack(&lut[i], ldw, udw); } - intel_de_write_fw(dev_priv, PREC_PAL_MULTI_SEG_INDEX(pipe), 0); + intel_de_write_fw(i915, PREC_PAL_MULTI_SEG_INDEX(pipe), 0); /* * FIXME readouts from PAL_PREC_DATA register aren't giving @@ -2268,15 +2268,15 @@ static const struct intel_color_funcs ilk_color_funcs = { void intel_color_crtc_init(struct intel_crtc *crtc) { - struct drm_i915_private *dev_priv = to_i915(crtc->base.dev); - bool has_ctm = INTEL_INFO(dev_priv)->display.color.degamma_lut_size != 0; + struct drm_i915_private *i915 = to_i915(crtc->base.dev); + bool has_ctm = INTEL_INFO(i915)->display.color.degamma_lut_size != 0; drm_mode_crtc_set_gamma_size(&crtc->base, 256); drm_crtc_enable_color_mgmt(&crtc->base, - INTEL_INFO(dev_priv)->display.color.degamma_lut_size, + INTEL_INFO(i915)->display.color.degamma_lut_size, has_ctm, - INTEL_INFO(dev_priv)->display.color.gamma_lut_size); + INTEL_INFO(i915)->display.color.gamma_lut_size); } int intel_color_init(struct drm_i915_private *i915) -- GitLab From fdaa243adf43d1d305d58f70cbc19e54c966c8e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:38:59 +0300 Subject: [PATCH 1002/1927] drm/i915: s/icl_load_gcmax/ivb_load_lut_max/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unify icl_load_gcmax() with the rest of the function naming scheme by calling it ivb_load_lut_max() instead. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-5-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 415e0a6839a4d..e73e6ea6f82f9 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -935,8 +935,8 @@ static u32 ilk_lut_12p4_ldw(const struct drm_color_lut *color) } static void -icl_load_gcmax(const struct intel_crtc_state *crtc_state, - const struct drm_color_lut *color) +ivb_load_lut_max(const struct intel_crtc_state *crtc_state, + const struct drm_color_lut *color) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); enum pipe pipe = crtc->pipe; @@ -1028,7 +1028,7 @@ icl_program_gamma_multi_segment(const struct intel_crtc_state *crtc_state) /* The last entry in the LUT is to be programmed in GCMAX */ entry = &lut[256 * 8 * 128]; - icl_load_gcmax(crtc_state, entry); + ivb_load_lut_max(crtc_state, entry); ivb_load_lut_ext_max(crtc_state); } -- GitLab From 4ad1aa571214e8d6468a1806794d987b374b5a08 Mon Sep 17 00:00:00 2001 From: Anirudh Rayabharam Date: Thu, 27 Oct 2022 15:27:28 +0530 Subject: [PATCH 1003/1927] clocksource/drivers/hyperv: add data structure for reference TSC MSR Add a data structure to represent the reference TSC MSR similar to other MSRs. This simplifies the code for updating the MSR. Signed-off-by: Anirudh Rayabharam Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20221027095729.1676394-2-anrayabh@linux.microsoft.com Signed-off-by: Wei Liu --- drivers/clocksource/hyperv_timer.c | 29 +++++++++++++++-------------- include/asm-generic/hyperv-tlfs.h | 9 +++++++++ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/drivers/clocksource/hyperv_timer.c b/drivers/clocksource/hyperv_timer.c index bb47610bbd1c4..18de1f439ffd5 100644 --- a/drivers/clocksource/hyperv_timer.c +++ b/drivers/clocksource/hyperv_timer.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -395,25 +396,25 @@ static u64 notrace read_hv_sched_clock_tsc(void) static void suspend_hv_clock_tsc(struct clocksource *arg) { - u64 tsc_msr; + union hv_reference_tsc_msr tsc_msr; /* Disable the TSC page */ - tsc_msr = hv_get_register(HV_REGISTER_REFERENCE_TSC); - tsc_msr &= ~BIT_ULL(0); - hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr); + tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); + tsc_msr.enable = 0; + hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); } static void resume_hv_clock_tsc(struct clocksource *arg) { phys_addr_t phys_addr = virt_to_phys(&tsc_pg); - u64 tsc_msr; + union hv_reference_tsc_msr tsc_msr; /* Re-enable the TSC page */ - tsc_msr = hv_get_register(HV_REGISTER_REFERENCE_TSC); - tsc_msr &= GENMASK_ULL(11, 0); - tsc_msr |= BIT_ULL(0) | (u64)phys_addr; - hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr); + tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); + tsc_msr.enable = 1; + tsc_msr.pfn = HVPFN_DOWN(phys_addr); + hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); } #ifdef HAVE_VDSO_CLOCKMODE_HVCLOCK @@ -495,7 +496,7 @@ static __always_inline void hv_setup_sched_clock(void *sched_clock) {} static bool __init hv_init_tsc_clocksource(void) { - u64 tsc_msr; + union hv_reference_tsc_msr tsc_msr; phys_addr_t phys_addr; if (!(ms_hyperv.features & HV_MSR_REFERENCE_TSC_AVAILABLE)) @@ -530,10 +531,10 @@ static bool __init hv_init_tsc_clocksource(void) * (which already has at least the low 12 bits set to zero since * it is page aligned). Also set the "enable" bit, which is bit 0. */ - tsc_msr = hv_get_register(HV_REGISTER_REFERENCE_TSC); - tsc_msr &= GENMASK_ULL(11, 0); - tsc_msr = tsc_msr | 0x1 | (u64)phys_addr; - hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr); + tsc_msr.as_uint64 = hv_get_register(HV_REGISTER_REFERENCE_TSC); + tsc_msr.enable = 1; + tsc_msr.pfn = HVPFN_DOWN(phys_addr); + hv_set_register(HV_REGISTER_REFERENCE_TSC, tsc_msr.as_uint64); clocksource_register_hz(&hyperv_cs_tsc, NSEC_PER_SEC/100); diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h index fdce7a4cfc6ff..b17c6eeb9afa2 100644 --- a/include/asm-generic/hyperv-tlfs.h +++ b/include/asm-generic/hyperv-tlfs.h @@ -102,6 +102,15 @@ struct ms_hyperv_tsc_page { volatile s64 tsc_offset; } __packed; +union hv_reference_tsc_msr { + u64 as_uint64; + struct { + u64 enable:1; + u64 reserved:11; + u64 pfn:52; + } __packed; +}; + /* * The guest OS needs to register the guest ID with the hypervisor. * The guest ID is a 64 bit entity and the structure of this ID is -- GitLab From 2982635a0b3d08d6fee2ff05632206286df0e703 Mon Sep 17 00:00:00 2001 From: Anirudh Rayabharam Date: Thu, 27 Oct 2022 15:27:29 +0530 Subject: [PATCH 1004/1927] x86/hyperv: fix invalid writes to MSRs during root partition kexec hyperv_cleanup resets the hypercall page by setting the MSR to 0. However, the root partition is not allowed to write to the GPA bits of the MSR. Instead, it uses the hypercall page provided by the MSR. Similar is the case with the reference TSC MSR. Clear only the enable bit instead of zeroing the entire MSR to make the code valid for root partition too. Signed-off-by: Anirudh Rayabharam Reviewed-by: Michael Kelley Link: https://lore.kernel.org/r/20221027095729.1676394-3-anrayabh@linux.microsoft.com Signed-off-by: Wei Liu --- arch/x86/hyperv/hv_init.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index d46f70522ddf2..f49bc3ec76e6a 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -535,6 +535,7 @@ void __init hyperv_init(void) void hyperv_cleanup(void) { union hv_x64_msr_hypercall_contents hypercall_msr; + union hv_reference_tsc_msr tsc_msr; unregister_syscore_ops(&hv_syscore_ops); @@ -550,12 +551,14 @@ void hyperv_cleanup(void) hv_hypercall_pg = NULL; /* Reset the hypercall page */ - hypercall_msr.as_uint64 = 0; - wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); + hypercall_msr.as_uint64 = hv_get_register(HV_X64_MSR_HYPERCALL); + hypercall_msr.enable = 0; + hv_set_register(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64); /* Reset the TSC page */ - hypercall_msr.as_uint64 = 0; - wrmsrl(HV_X64_MSR_REFERENCE_TSC, hypercall_msr.as_uint64); + tsc_msr.as_uint64 = hv_get_register(HV_X64_MSR_REFERENCE_TSC); + tsc_msr.enable = 0; + hv_set_register(HV_X64_MSR_REFERENCE_TSC, tsc_msr.as_uint64); } void hyperv_report_panic(struct pt_regs *regs, long err, bool in_die) -- GitLab From 9539c7fa38d35f869e83d56eb65a0b681644031b Mon Sep 17 00:00:00 2001 From: Stephen Hemminger Date: Fri, 28 Oct 2022 08:37:42 -0700 Subject: [PATCH 1005/1927] MAINTAINERS: remove sthemmin Leaving Microsoft, the Hyper-V drivers have lots of other maintainers. Signed-off-by: Stephen Hemminger Link: https://lore.kernel.org/r/20221028153741.25470-1-stephen@networkplumber.org Signed-off-by: Wei Liu --- MAINTAINERS | 1 - 1 file changed, 1 deletion(-) diff --git a/MAINTAINERS b/MAINTAINERS index e04d944005ba8..8417029432faf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -9504,7 +9504,6 @@ F: drivers/media/i2c/hi847.c Hyper-V/Azure CORE AND DRIVERS M: "K. Y. Srinivasan" M: Haiyang Zhang -M: Stephen Hemminger M: Wei Liu M: Dexuan Cui L: linux-hyperv@vger.kernel.org -- GitLab From e70af8d040d2b7904dca93d942ba23fb722e21b1 Mon Sep 17 00:00:00 2001 From: Dexuan Cui Date: Thu, 27 Oct 2022 13:52:56 -0700 Subject: [PATCH 1006/1927] PCI: hv: Fix the definition of vector in hv_compose_msi_msg() The local variable 'vector' must be u32 rather than u8: see the struct hv_msi_desc3. 'vector_count' should be u16 rather than u8: see struct hv_msi_desc, hv_msi_desc2 and hv_msi_desc3. Fixes: a2bad844a67b ("PCI: hv: Fix interrupt mapping for multi-MSI") Signed-off-by: Dexuan Cui Cc: Jeffrey Hugo Cc: Carl Vanderlip Reviewed-by: Jeffrey Hugo Link: https://lore.kernel.org/r/20221027205256.17678-1-decui@microsoft.com Signed-off-by: Wei Liu --- drivers/pci/controller/pci-hyperv.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index e7c6f6629e7c5..ba64284eaf9fa 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -1614,7 +1614,7 @@ static void hv_pci_compose_compl(void *context, struct pci_response *resp, static u32 hv_compose_msi_req_v1( struct pci_create_interrupt *int_pkt, const struct cpumask *affinity, - u32 slot, u8 vector, u8 vector_count) + u32 slot, u8 vector, u16 vector_count) { int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE; int_pkt->wslot.slot = slot; @@ -1642,7 +1642,7 @@ static int hv_compose_msi_req_get_cpu(const struct cpumask *affinity) static u32 hv_compose_msi_req_v2( struct pci_create_interrupt2 *int_pkt, const struct cpumask *affinity, - u32 slot, u8 vector, u8 vector_count) + u32 slot, u8 vector, u16 vector_count) { int cpu; @@ -1661,7 +1661,7 @@ static u32 hv_compose_msi_req_v2( static u32 hv_compose_msi_req_v3( struct pci_create_interrupt3 *int_pkt, const struct cpumask *affinity, - u32 slot, u32 vector, u8 vector_count) + u32 slot, u32 vector, u16 vector_count) { int cpu; @@ -1701,7 +1701,12 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) struct compose_comp_ctxt comp; struct tran_int_desc *int_desc; struct msi_desc *msi_desc; - u8 vector, vector_count; + /* + * vector_count should be u16: see hv_msi_desc, hv_msi_desc2 + * and hv_msi_desc3. vector must be u32: see hv_msi_desc3. + */ + u16 vector_count; + u32 vector; struct { struct pci_packet pci_pkt; union { @@ -1767,6 +1772,11 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) vector_count = 1; } + /* + * hv_compose_msi_req_v1 and v2 are for x86 only, meaning 'vector' + * can't exceed u8. Cast 'vector' down to u8 for v1/v2 explicitly + * for better readability. + */ memset(&ctxt, 0, sizeof(ctxt)); init_completion(&comp.comp_pkt.host_event); ctxt.pci_pkt.completion_func = hv_pci_compose_compl; @@ -1777,7 +1787,7 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) size = hv_compose_msi_req_v1(&ctxt.int_pkts.v1, dest, hpdev->desc.win_slot.slot, - vector, + (u8)vector, vector_count); break; @@ -1786,7 +1796,7 @@ static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) size = hv_compose_msi_req_v2(&ctxt.int_pkts.v2, dest, hpdev->desc.win_slot.slot, - vector, + (u8)vector, vector_count); break; -- GitLab From 8f079f08833d36d8b756a08f613bccba10e23784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:00 +0300 Subject: [PATCH 1007/1927] drm/i915: Split ivb_load_lut_ext_max() into two parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the EXT2_MAX register programming into its own function. More in line with the whole "cobble together stuff from small pieces" approach used in this code. The EXT(2)_MAX registers are also not really part of the multi-segment section of the LUT, so hoist the calls to a higher level, just like we do in other gamma modes as well. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-6-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 29 +++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index e73e6ea6f82f9..3b78b882e0c04 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -764,27 +764,23 @@ static void bdw_load_lut_10(struct intel_crtc *crtc, static void ivb_load_lut_ext_max(const struct intel_crtc_state *crtc_state) { struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); - struct drm_i915_private *i915 = to_i915(crtc->base.dev); enum pipe pipe = crtc->pipe; /* Program the max register to clamp values > 1.0. */ intel_dsb_reg_write(crtc_state, PREC_PAL_EXT_GC_MAX(pipe, 0), 1 << 16); intel_dsb_reg_write(crtc_state, PREC_PAL_EXT_GC_MAX(pipe, 1), 1 << 16); intel_dsb_reg_write(crtc_state, PREC_PAL_EXT_GC_MAX(pipe, 2), 1 << 16); +} - /* - * Program the gc max 2 register to clamp values > 1.0. - * ToDo: Extend the ABI to be able to program values - * from 3.0 to 7.0 - */ - if (DISPLAY_VER(i915) >= 10) { - intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 0), - 1 << 16); - intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 1), - 1 << 16); - intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 2), - 1 << 16); - } +static void glk_load_lut_ext2_max(const struct intel_crtc_state *crtc_state) +{ + struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc); + enum pipe pipe = crtc->pipe; + + /* Program the max register to clamp values > 1.0. */ + intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 0), 1 << 16); + intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 1), 1 << 16); + intel_dsb_reg_write(crtc_state, PREC_PAL_EXT2_GC_MAX(pipe, 2), 1 << 16); } static void ivb_load_luts(const struct intel_crtc_state *crtc_state) @@ -913,6 +909,7 @@ static void glk_load_luts(const struct intel_crtc_state *crtc_state) case GAMMA_MODE_MODE_10BIT: bdw_load_lut_10(crtc, post_csc_lut, PAL_PREC_INDEX_VALUE(0)); ivb_load_lut_ext_max(crtc_state); + glk_load_lut_ext2_max(crtc_state); break; default: MISSING_CASE(crtc_state->gamma_mode); @@ -1029,7 +1026,6 @@ icl_program_gamma_multi_segment(const struct intel_crtc_state *crtc_state) /* The last entry in the LUT is to be programmed in GCMAX */ entry = &lut[256 * 8 * 128]; ivb_load_lut_max(crtc_state, entry); - ivb_load_lut_ext_max(crtc_state); } static void icl_load_luts(const struct intel_crtc_state *crtc_state) @@ -1048,10 +1044,13 @@ static void icl_load_luts(const struct intel_crtc_state *crtc_state) case GAMMA_MODE_MODE_12BIT_MULTI_SEGMENTED: icl_program_gamma_superfine_segment(crtc_state); icl_program_gamma_multi_segment(crtc_state); + ivb_load_lut_ext_max(crtc_state); + glk_load_lut_ext2_max(crtc_state); break; case GAMMA_MODE_MODE_10BIT: bdw_load_lut_10(crtc, post_csc_lut, PAL_PREC_INDEX_VALUE(0)); ivb_load_lut_ext_max(crtc_state); + glk_load_lut_ext2_max(crtc_state); break; default: MISSING_CASE(crtc_state->gamma_mode); -- GitLab From 28c9fa7770ea9c818b5a9fdf64e242fd9e4fdc2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:01 +0300 Subject: [PATCH 1008/1927] drm/i915: Deconfuse the ilk+ 12.4 LUT entry functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit s/icl_lut_multi_seg_pack/ilk_lut_12p4_pack/ since that's what it is and group the corresponding "unpack" functions next to it. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-7-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 38 +++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 3b78b882e0c04..e881c95ee4510 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -482,14 +482,28 @@ static void ilk_lut_10_pack(struct drm_color_lut *entry, u32 val) entry->blue = intel_color_lut_pack(REG_FIELD_GET(PREC_PALETTE_BLUE_MASK, val), 10); } -static void icl_lut_multi_seg_pack(struct drm_color_lut *entry, u32 ldw, u32 udw) +/* ilk+ "12.4" interpolated format (high 10 bits) */ +static u32 ilk_lut_12p4_udw(const struct drm_color_lut *color) +{ + return (color->red >> 6) << 20 | (color->green >> 6) << 10 | + (color->blue >> 6); +} + +/* ilk+ "12.4" interpolated format (low 6 bits) */ +static u32 ilk_lut_12p4_ldw(const struct drm_color_lut *color) +{ + return (color->red & 0x3f) << 24 | (color->green & 0x3f) << 14 | + (color->blue & 0x3f) << 4; +} + +static void ilk_lut_12p4_pack(struct drm_color_lut *entry, u32 ldw, u32 udw) { entry->red = REG_FIELD_GET(PAL_PREC_MULTI_SEG_RED_UDW_MASK, udw) << 6 | - REG_FIELD_GET(PAL_PREC_MULTI_SEG_RED_LDW_MASK, ldw); + REG_FIELD_GET(PAL_PREC_MULTI_SEG_RED_LDW_MASK, ldw); entry->green = REG_FIELD_GET(PAL_PREC_MULTI_SEG_GREEN_UDW_MASK, udw) << 6 | - REG_FIELD_GET(PAL_PREC_MULTI_SEG_GREEN_LDW_MASK, ldw); + REG_FIELD_GET(PAL_PREC_MULTI_SEG_GREEN_LDW_MASK, ldw); entry->blue = REG_FIELD_GET(PAL_PREC_MULTI_SEG_BLUE_UDW_MASK, udw) << 6 | - REG_FIELD_GET(PAL_PREC_MULTI_SEG_BLUE_LDW_MASK, ldw); + REG_FIELD_GET(PAL_PREC_MULTI_SEG_BLUE_LDW_MASK, ldw); } static void icl_color_commit_noarm(const struct intel_crtc_state *crtc_state) @@ -917,20 +931,6 @@ static void glk_load_luts(const struct intel_crtc_state *crtc_state) } } -/* ilk+ "12.4" interpolated format (high 10 bits) */ -static u32 ilk_lut_12p4_udw(const struct drm_color_lut *color) -{ - return (color->red >> 6) << 20 | (color->green >> 6) << 10 | - (color->blue >> 6); -} - -/* ilk+ "12.4" interpolated format (low 6 bits) */ -static u32 ilk_lut_12p4_ldw(const struct drm_color_lut *color) -{ - return (color->red & 0x3f) << 24 | (color->green & 0x3f) << 14 | - (color->blue & 0x3f) << 4; -} - static void ivb_load_lut_max(const struct intel_crtc_state *crtc_state, const struct drm_color_lut *color) @@ -2151,7 +2151,7 @@ icl_read_lut_multi_segment(struct intel_crtc *crtc) u32 ldw = intel_de_read_fw(i915, PREC_PAL_MULTI_SEG_DATA(pipe)); u32 udw = intel_de_read_fw(i915, PREC_PAL_MULTI_SEG_DATA(pipe)); - icl_lut_multi_seg_pack(&lut[i], ldw, udw); + ilk_lut_12p4_pack(&lut[i], ldw, udw); } intel_de_write_fw(i915, PREC_PAL_MULTI_SEG_INDEX(pipe), 0); -- GitLab From ad105b715e0092b0c4c8534d07d70692adca8516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:02 +0300 Subject: [PATCH 1009/1927] drm/i915: Pass limited_range explicitly to ilk_csc_convert_ctm() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since pre-icl vs. icl+ handle the limited range output stuff a bit differently it's probably less confusing if we just pass that information explicitly into ilk_csc_convert_ctm(). Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-8-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index e881c95ee4510..946fb767f3e07 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -259,14 +259,14 @@ static bool ilk_csc_limited_range(const struct intel_crtc_state *crtc_state) } static void ilk_csc_convert_ctm(const struct intel_crtc_state *crtc_state, - u16 coeffs[9]) + u16 coeffs[9], bool limited_color_range) { const struct drm_color_ctm *ctm = crtc_state->hw.ctm->data; const u64 *input; u64 temp[9]; int i; - if (ilk_csc_limited_range(crtc_state)) + if (limited_color_range) input = ctm_mult_by_limited(temp, ctm->matrix); else input = ctm->matrix; @@ -319,7 +319,7 @@ static void ilk_load_csc_matrix(const struct intel_crtc_state *crtc_state) if (crtc_state->hw.ctm) { u16 coeff[9]; - ilk_csc_convert_ctm(crtc_state, coeff); + ilk_csc_convert_ctm(crtc_state, coeff, limited_color_range); ilk_update_pipe_csc(crtc, ilk_csc_off_zero, coeff, limited_color_range ? ilk_csc_postoff_limited_range : @@ -354,7 +354,7 @@ static void icl_load_csc_matrix(const struct intel_crtc_state *crtc_state) if (crtc_state->hw.ctm) { u16 coeff[9]; - ilk_csc_convert_ctm(crtc_state, coeff); + ilk_csc_convert_ctm(crtc_state, coeff, false); ilk_update_pipe_csc(crtc, ilk_csc_off_zero, coeff, ilk_csc_off_zero); } -- GitLab From 6d5e733f3ac1ad68a6230f44069b14bea2a9bdb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:03 +0300 Subject: [PATCH 1010/1927] drm/i915: Reuse ilk_gamma_mode() on ivb+ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apart from the split gamma mode ivb+ LUTs work just like ilk+ LUTs. So let's handle the special case, and then just fall back to ilk_gamma_mode() to avoid having to duplicate the same logic. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-9-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 946fb767f3e07..435394cad359e 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -1521,14 +1521,10 @@ static int ilk_color_check(struct intel_crtc_state *crtc_state) static u32 ivb_gamma_mode(const struct intel_crtc_state *crtc_state) { - if (!crtc_state->gamma_enable || - crtc_state_is_legacy_gamma(crtc_state)) - return GAMMA_MODE_MODE_8BIT; - else if (crtc_state->hw.gamma_lut && - crtc_state->hw.degamma_lut) + if (crtc_state->hw.degamma_lut && crtc_state->hw.gamma_lut) return GAMMA_MODE_MODE_SPLIT; - else - return GAMMA_MODE_MODE_10BIT; + + return ilk_gamma_mode(crtc_state); } static u32 ivb_csc_mode(const struct intel_crtc_state *crtc_state) -- GitLab From 4c0119dd0a7728456a2394c1bc0aa612e1f2c3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:04 +0300 Subject: [PATCH 1011/1927] drm/i915: Reject YCbCr output with degamma+gamma on pre-icl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the pipe CSC sits between the degamma and gamma LUTs there is no way to make us it for RGB->YCbCr conversion when both LUTs are also active. Simply reject such combos. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-10-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 435394cad359e..926784f266f23 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -1556,7 +1556,14 @@ static int ivb_color_check(struct intel_crtc_state *crtc_state) if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && crtc_state->hw.ctm) { drm_dbg_kms(&i915->drm, - "YCBCR and CTM together are not possible\n"); + "YCbCr and CTM together are not possible\n"); + return -EINVAL; + } + + if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && + crtc_state->hw.degamma_lut && crtc_state->hw.gamma_lut) { + drm_dbg_kms(&i915->drm, + "YCbCr and degamma+gamma together are not possible\n"); return -EINVAL; } @@ -1622,7 +1629,14 @@ static int glk_color_check(struct intel_crtc_state *crtc_state) if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && crtc_state->hw.ctm) { drm_dbg_kms(&i915->drm, - "YCBCR and CTM together are not possible\n"); + "YCbCr and CTM together are not possible\n"); + return -EINVAL; + } + + if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && + crtc_state->hw.degamma_lut && crtc_state->hw.gamma_lut) { + drm_dbg_kms(&i915->drm, + "YCbCr and degamma+gamma together are not possible\n"); return -EINVAL; } -- GitLab From 9034f9c4e284138d5e5646b89285d7a89b840f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:05 +0300 Subject: [PATCH 1012/1927] drm/i915: Share {csc,gamma}_enable calculation for ilk/snb vs. ivb+ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ilk/snb vs. ivb+ hardware is mostly identical except for the addition of the split gamma mode on ivb. Thus we can share the csc_enable and gamma_enable calculation for both variants. Pull that stuff into a few helpers. Note that this also fills in the missing ctm/degamma stuff into ilk_color_check() pretty much, so for good measure let's also add a few extra checks relating to that, although we still don't expose ctm/degamma to userspace. But now it'll be trivial to do so if we wish. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-11-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 49 ++++++++++++++-------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 926784f266f23..33871bfacee7e 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -1442,6 +1442,20 @@ static int chv_color_check(struct intel_crtc_state *crtc_state) return 0; } +static bool ilk_gamma_enable(const struct intel_crtc_state *crtc_state) +{ + return (crtc_state->hw.gamma_lut || + crtc_state->hw.degamma_lut) && + !crtc_state->c8_planes; +} + +static bool ilk_csc_enable(const struct intel_crtc_state *crtc_state) +{ + return crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB || + ilk_csc_limited_range(crtc_state) || + crtc_state->hw.ctm; +} + static u32 ilk_gamma_mode(const struct intel_crtc_state *crtc_state) { if (!crtc_state->gamma_enable || @@ -1487,22 +1501,29 @@ static void ilk_assign_luts(struct intel_crtc_state *crtc_state) static int ilk_color_check(struct intel_crtc_state *crtc_state) { + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); int ret; ret = check_luts(crtc_state); if (ret) return ret; - crtc_state->gamma_enable = - crtc_state->hw.gamma_lut && - !crtc_state->c8_planes; + if (crtc_state->hw.degamma_lut && crtc_state->hw.gamma_lut) { + drm_dbg_kms(&i915->drm, + "Degamma and gamma together are not possible\n"); + return -EINVAL; + } - /* - * We don't expose the ctm on ilk/snb currently, also RGB - * limited range output is handled by the hw automagically. - */ - crtc_state->csc_enable = - crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB; + if (crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB && + crtc_state->hw.ctm) { + drm_dbg_kms(&i915->drm, + "YCbCr and CTM together are not possible\n"); + return -EINVAL; + } + + crtc_state->gamma_enable = ilk_gamma_enable(crtc_state); + + crtc_state->csc_enable = ilk_csc_enable(crtc_state); crtc_state->gamma_mode = ilk_gamma_mode(crtc_state); @@ -1546,7 +1567,6 @@ static u32 ivb_csc_mode(const struct intel_crtc_state *crtc_state) static int ivb_color_check(struct intel_crtc_state *crtc_state) { struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); - bool limited_color_range = ilk_csc_limited_range(crtc_state); int ret; ret = check_luts(crtc_state); @@ -1567,14 +1587,9 @@ static int ivb_color_check(struct intel_crtc_state *crtc_state) return -EINVAL; } - crtc_state->gamma_enable = - (crtc_state->hw.gamma_lut || - crtc_state->hw.degamma_lut) && - !crtc_state->c8_planes; + crtc_state->gamma_enable = ilk_gamma_enable(crtc_state); - crtc_state->csc_enable = - crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB || - crtc_state->hw.ctm || limited_color_range; + crtc_state->csc_enable = ilk_csc_enable(crtc_state); crtc_state->gamma_mode = ivb_gamma_mode(crtc_state); -- GitLab From 882ecff709b50b36128e07a6b0035f476c769f50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Mon, 31 Oct 2022 23:40:36 +0200 Subject: [PATCH 1013/1927] drm/i915: Use intel_crtc_needs_modeset() more MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prefer our own intel_crtc_needs_modeset() wrapper to drm_atomic_crtc_needs_modeset() whenever we are dealing with the intel_ types instead of drm_ types. Makes things a bit neater in general. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221031214037.1636-1-ville.syrjala@linux.intel.com Reviewed-by: Jani Nikula --- drivers/gpu/drm/i915/display/intel_cdclk.c | 2 +- drivers/gpu/drm/i915/display/intel_color.c | 2 +- drivers/gpu/drm/i915/display/intel_display.c | 2 +- drivers/gpu/drm/i915/display/intel_fbc.c | 2 +- drivers/gpu/drm/i915/display/skl_watermark.c | 2 +- drivers/gpu/drm/i915/intel_pm.c | 11 +++++------ 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index eada931cb1c85..8a9031012d744 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -2755,7 +2755,7 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state) if (IS_ERR(crtc_state)) return PTR_ERR(crtc_state); - if (drm_atomic_crtc_needs_modeset(&crtc_state->uapi)) + if (intel_crtc_needs_modeset(crtc_state)) pipe = INVALID_PIPE; } diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 33871bfacee7e..458e69578da6e 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -1239,7 +1239,7 @@ intel_color_add_affected_planes(struct intel_crtc_state *new_crtc_state) struct intel_plane *plane; if (!new_crtc_state->hw.active || - drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) + intel_crtc_needs_modeset(new_crtc_state)) return 0; if (new_crtc_state->gamma_enable == old_crtc_state->gamma_enable && diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 31e91dbaa368a..c4d31db935fe1 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -5931,7 +5931,7 @@ int intel_modeset_all_pipes(struct intel_atomic_state *state, return PTR_ERR(crtc_state); if (!crtc_state->hw.active || - drm_atomic_crtc_needs_modeset(&crtc_state->uapi)) + intel_crtc_needs_modeset(crtc_state)) continue; drm_dbg_kms(&dev_priv->drm, "[CRTC:%d:%s] Full modeset due to %s\n", diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 3f24f326b9890..b5ee5ea0d010e 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -1183,7 +1183,7 @@ static bool intel_fbc_can_flip_nuke(struct intel_atomic_state *state, const struct drm_framebuffer *old_fb = old_plane_state->hw.fb; const struct drm_framebuffer *new_fb = new_plane_state->hw.fb; - if (drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) + if (intel_crtc_needs_modeset(new_crtc_state)) return false; if (!intel_fbc_is_ok(old_plane_state) || diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c index d58e667016e46..e0766d1be966f 100644 --- a/drivers/gpu/drm/i915/display/skl_watermark.c +++ b/drivers/gpu/drm/i915/display/skl_watermark.c @@ -2744,7 +2744,7 @@ static int skl_wm_add_affected_planes(struct intel_atomic_state *state, * power well the hardware state will go out of sync * with the software state. */ - if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) && + if (!intel_crtc_needs_modeset(new_crtc_state) && skl_plane_selected_wm_equals(plane, &old_crtc_state->wm.skl.optimal, &new_crtc_state->wm.skl.optimal)) diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c index a204a32bce44a..016ac568dc874 100644 --- a/drivers/gpu/drm/i915/intel_pm.c +++ b/drivers/gpu/drm/i915/intel_pm.c @@ -1434,7 +1434,7 @@ static int g4x_compute_intermediate_wm(struct intel_atomic_state *state, enum plane_id plane_id; if (!new_crtc_state->hw.active || - drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) { + intel_crtc_needs_modeset(new_crtc_state)) { *intermediate = *optimal; intermediate->cxsr = false; @@ -1922,7 +1922,6 @@ static int vlv_compute_pipe_wm(struct intel_atomic_state *state, { struct intel_crtc_state *crtc_state = intel_atomic_get_new_crtc_state(state, crtc); - bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi); const struct intel_plane_state *old_plane_state; const struct intel_plane_state *new_plane_state; struct intel_plane *plane; @@ -1949,7 +1948,7 @@ static int vlv_compute_pipe_wm(struct intel_atomic_state *state, * FIFO setting we took over from the BIOS even if there * are no active planes on the crtc. */ - if (needs_modeset) + if (intel_crtc_needs_modeset(crtc_state)) dirty = ~0; if (!dirty) @@ -1969,7 +1968,7 @@ static int vlv_compute_pipe_wm(struct intel_atomic_state *state, if (ret) return ret; - if (needs_modeset || + if (intel_crtc_needs_modeset(crtc_state) || memcmp(old_fifo_state, new_fifo_state, sizeof(*new_fifo_state)) != 0) crtc_state->fifo_changed = true; @@ -2092,7 +2091,7 @@ static int vlv_compute_intermediate_wm(struct intel_atomic_state *state, int level; if (!new_crtc_state->hw.active || - drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) { + intel_crtc_needs_modeset(new_crtc_state)) { *intermediate = *optimal; intermediate->cxsr = false; @@ -3150,7 +3149,7 @@ static int ilk_compute_intermediate_wm(struct intel_atomic_state *state, */ *a = new_crtc_state->wm.ilk.optimal; if (!new_crtc_state->hw.active || - drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) || + intel_crtc_needs_modeset(new_crtc_state) || state->skip_intermediate_wm) return 0; -- GitLab From 23715a26c8d812912a70c6ac1ce67af649b95914 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Fri, 28 Oct 2022 16:39:14 +0200 Subject: [PATCH 1014/1927] arm64: efi: Recover from synchronous exceptions occurring in firmware Unlike x86, which has machinery to deal with page faults that occur during the execution of EFI runtime services, arm64 has nothing like that, and a synchronous exception raised by firmware code brings down the whole system. With more EFI based systems appearing that were not built to run Linux (such as the Windows-on-ARM laptops based on Qualcomm SOCs), as well as the introduction of PRM (platform specific firmware routines that are callable just like EFI runtime services), we are more likely to run into issues of this sort, and it is much more likely that we can identify and work around such issues if they don't bring down the system entirely. Since we already use a EFI runtime services call wrapper in assembler, we can quite easily add some code that captures the execution state at the point where the call is made, allowing us to revert to this state and proceed execution if the call triggered a synchronous exception. Given that the kernel and the firmware don't share any data structures that could end up in an indeterminate state, we can happily continue running, as long as we mark the EFI runtime services as unavailable from that point on. Signed-off-by: Ard Biesheuvel Acked-by: Catalin Marinas --- arch/arm64/include/asm/efi.h | 8 ++++++++ arch/arm64/kernel/efi-rt-wrapper.S | 33 ++++++++++++++++++++++++++++-- arch/arm64/kernel/efi.c | 26 +++++++++++++++++++++++ arch/arm64/mm/fault.c | 4 ++++ 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 439e2bc5d5d8b..d6cf535d8352b 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -14,8 +14,16 @@ #ifdef CONFIG_EFI extern void efi_init(void); + +bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg); #else #define efi_init() + +static inline +bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg) +{ + return false; +} #endif int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md); diff --git a/arch/arm64/kernel/efi-rt-wrapper.S b/arch/arm64/kernel/efi-rt-wrapper.S index 75691a2641c1c..67babd5f04c27 100644 --- a/arch/arm64/kernel/efi-rt-wrapper.S +++ b/arch/arm64/kernel/efi-rt-wrapper.S @@ -6,7 +6,7 @@ #include SYM_FUNC_START(__efi_rt_asm_wrapper) - stp x29, x30, [sp, #-32]! + stp x29, x30, [sp, #-112]! mov x29, sp /* @@ -16,6 +16,20 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) */ stp x1, x18, [sp, #16] + /* + * Preserve all callee saved registers and record the stack pointer + * value in a per-CPU variable so we can recover from synchronous + * exceptions occurring while running the firmware routines. + */ + stp x19, x20, [sp, #32] + stp x21, x22, [sp, #48] + stp x23, x24, [sp, #64] + stp x25, x26, [sp, #80] + stp x27, x28, [sp, #96] + + adr_this_cpu x8, __efi_rt_asm_recover_sp, x9 + str x29, [x8] + /* * We are lucky enough that no EFI runtime services take more than * 5 arguments, so all are passed in registers rather than via the @@ -31,7 +45,7 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) ldp x1, x2, [sp, #16] cmp x2, x18 - ldp x29, x30, [sp], #32 + ldp x29, x30, [sp], #112 b.ne 0f ret 0: @@ -45,3 +59,18 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) mov x18, x2 b efi_handle_corrupted_x18 // tail call SYM_FUNC_END(__efi_rt_asm_wrapper) + +SYM_FUNC_START(__efi_rt_asm_recover) + ldr_this_cpu x8, __efi_rt_asm_recover_sp, x9 + mov sp, x8 + + ldp x0, x18, [sp, #16] + ldp x19, x20, [sp, #32] + ldp x21, x22, [sp, #48] + ldp x23, x24, [sp, #64] + ldp x25, x26, [sp, #80] + ldp x27, x28, [sp, #96] + ldp x29, x30, [sp], #112 + + b efi_handle_runtime_exception +SYM_FUNC_END(__efi_rt_asm_recover) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index e1be6c429810d..8d36e66a6e64c 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -9,6 +9,7 @@ #include #include +#include #include @@ -128,3 +129,28 @@ asmlinkage efi_status_t efi_handle_corrupted_x18(efi_status_t s, const char *f) pr_err_ratelimited(FW_BUG "register x18 corrupted by EFI %s\n", f); return s; } + +asmlinkage DEFINE_PER_CPU(u64, __efi_rt_asm_recover_sp); + +asmlinkage efi_status_t __efi_rt_asm_recover(void); + +asmlinkage efi_status_t efi_handle_runtime_exception(const char *f) +{ + pr_err(FW_BUG "Synchronous exception occurred in EFI runtime service %s()\n", f); + clear_bit(EFI_RUNTIME_SERVICES, &efi.flags); + return EFI_ABORTED; +} + +bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg) +{ + /* Check whether the exception occurred while running the firmware */ + if (current_work() != &efi_rts_work.work || regs->pc >= TASK_SIZE_64) + return false; + + pr_err(FW_BUG "Unable to handle %s in EFI runtime service\n", msg); + add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK); + dump_stack(); + + regs->pc = (u64)__efi_rt_asm_recover; + return true; +} diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 5b391490e045b..3e9cf9826417a 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -391,6 +392,9 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr, msg = "paging request"; } + if (efi_runtime_fixup_exception(regs, msg)) + return; + die_kernel_fault(msg, addr, esr, regs); } -- GitLab From a207620123f27e6f63bab13b46b20e03a3deec8d Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Mon, 31 Oct 2022 10:29:20 +0100 Subject: [PATCH 1015/1927] mm/slab_common: repair kernel-doc for __ksize() Commit 445d41d7a7c1 ("Merge branch 'slab/for-6.1/kmalloc_size_roundup' into slab/for-next") resolved a conflict of two concurrent changes to __ksize(). However, it did not adjust the kernel-doc comment of __ksize(), while the name of the argument to __ksize() was renamed. Hence, ./scripts/ kernel-doc -none mm/slab_common.c warns about it. Adjust the kernel-doc comment for __ksize() for make W=1 happiness. Signed-off-by: Lukas Bulwahn Acked-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Signed-off-by: Vlastimil Babka --- mm/slab_common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 33b1886b06ebf..74a991fd9d31d 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1010,7 +1010,7 @@ EXPORT_SYMBOL(kfree); /** * __ksize -- Report full size of underlying allocation - * @objp: pointer to the object + * @object: pointer to the object * * This should only be used internally to query the true size of allocations. * It is not meant to be a way to discover the usable size of an allocation @@ -1018,7 +1018,7 @@ EXPORT_SYMBOL(kfree); * the originally requested allocation size may trigger KASAN, UBSAN_BOUNDS, * and/or FORTIFY_SOURCE. * - * Return: size of the actual memory used by @objp in bytes + * Return: size of the actual memory used by @object in bytes */ size_t __ksize(const void *object) { -- GitLab From 85f1506337f0c79a4955edfeee86a18628e3735f Mon Sep 17 00:00:00 2001 From: Amit Daniel Kachhap Date: Thu, 3 Nov 2022 13:52:32 +0530 Subject: [PATCH 1016/1927] arm64: cpufeature: Fix the visibility of compat hwcaps Commit 237405ebef58 ("arm64: cpufeature: Force HWCAP to be based on the sysreg visible to user-space") forced the hwcaps to use sanitised user-space view of the id registers. However, the ID register structures used to select few compat cpufeatures (vfp, crc32, ...) are masked and hence such hwcaps do not appear in /proc/cpuinfo anymore for PER_LINUX32 personality. Add the ID register structures explicitly and set the relevant entry as visible. As these ID registers are now of type visible so make them available in 64-bit userspace by making necessary changes in register emulation logic and documentation. While at it, update the comment for structure ftr_generic_32bits[] which lists the ID register that use it. Fixes: 237405ebef58 ("arm64: cpufeature: Force HWCAP to be based on the sysreg visible to user-space") Cc: Suzuki K Poulose Reviewed-by: James Morse Signed-off-by: Amit Daniel Kachhap Link: https://lore.kernel.org/r/20221103082232.19189-1-amit.kachhap@arm.com Signed-off-by: Catalin Marinas --- Documentation/arm64/cpu-feature-registers.rst | 38 ++++++++++++++++- arch/arm64/kernel/cpufeature.c | 42 +++++++++++++++---- 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst index 04ba83e1965fb..c7adc7897df60 100644 --- a/Documentation/arm64/cpu-feature-registers.rst +++ b/Documentation/arm64/cpu-feature-registers.rst @@ -92,7 +92,7 @@ operation if the source belongs to the supported system register space. The infrastructure emulates only the following system register space:: - Op0=3, Op1=0, CRn=0, CRm=0,4,5,6,7 + Op0=3, Op1=0, CRn=0, CRm=0,2,3,4,5,6,7 (See Table C5-6 'System instruction encodings for non-Debug System register accesses' in ARMv8 ARM DDI 0487A.h, for the list of @@ -293,6 +293,42 @@ infrastructure: | WFXT | [3-0] | y | +------------------------------+---------+---------+ + 10) MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | FPDP | [11-8] | y | + +------------------------------+---------+---------+ + + 11) MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | SIMDFMAC | [31-28] | y | + +------------------------------+---------+---------+ + | SIMDSP | [19-16] | y | + +------------------------------+---------+---------+ + | SIMDInt | [15-12] | y | + +------------------------------+---------+---------+ + | SIMDLS | [11-8] | y | + +------------------------------+---------+---------+ + + 12) ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | CRC32 | [19-16] | y | + +------------------------------+---------+---------+ + | SHA2 | [15-12] | y | + +------------------------------+---------+---------+ + | SHA1 | [11-8] | y | + +------------------------------+---------+---------+ + | AES | [7-4] | y | + +------------------------------+---------+---------+ + Appendix I: Example ------------------- diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 6062454a90674..b3f37e2209ad3 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -428,6 +428,30 @@ static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_mvfr0[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPROUND_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSHVEC_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSQRT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDIVIDE_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPTRAP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPDP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_FPSP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_SIMD_SHIFT, 4, 0), + ARM64_FTR_END, +}; + +static const struct arm64_ftr_bits ftr_mvfr1[] = { + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDFMAC_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPHP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDHP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDSP_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDINT_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_SIMDLS_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPDNAN_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_FPFTZ_SHIFT, 4, 0), + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_mvfr2[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_FPMISC_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_SIMDMISC_SHIFT, 4, 0), @@ -458,10 +482,10 @@ static const struct arm64_ftr_bits ftr_id_isar0[] = { static const struct arm64_ftr_bits ftr_id_isar5[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0), - ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0), + ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0), ARM64_FTR_END, }; @@ -574,7 +598,7 @@ static const struct arm64_ftr_bits ftr_smcr[] = { * Common ftr bits for a 32bit register with all hidden, strict * attributes, with 4bit feature fields and a default safe value of * 0. Covers the following 32bit registers: - * id_isar[1-4], id_mmfr[1-3], id_pfr1, mvfr[0-1] + * id_isar[1-3], id_mmfr[1-3] */ static const struct arm64_ftr_bits ftr_generic_32bits[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0), @@ -645,8 +669,8 @@ static const struct __ftr_reg_entry { ARM64_FTR_REG(SYS_ID_ISAR6_EL1, ftr_id_isar6), /* Op1 = 0, CRn = 0, CRm = 3 */ - ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits), - ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits), + ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_mvfr0), + ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_mvfr1), ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2), ARM64_FTR_REG(SYS_ID_PFR2_EL1, ftr_id_pfr2), ARM64_FTR_REG(SYS_ID_DFR1_EL1, ftr_id_dfr1), @@ -3339,7 +3363,7 @@ static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *c /* * We emulate only the following system register space. - * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7] + * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 2 - 7] * See Table C5-6 System instruction encodings for System register accesses, * ARMv8 ARM(ARM DDI 0487A.f) for more details. */ @@ -3349,7 +3373,7 @@ static inline bool __attribute_const__ is_emulated(u32 id) sys_reg_CRn(id) == 0x0 && sys_reg_Op1(id) == 0x0 && (sys_reg_CRm(id) == 0 || - ((sys_reg_CRm(id) >= 4) && (sys_reg_CRm(id) <= 7)))); + ((sys_reg_CRm(id) >= 2) && (sys_reg_CRm(id) <= 7)))); } /* -- GitLab From 4f1aa35f1fb7d51b125487c835982af792697ecb Mon Sep 17 00:00:00 2001 From: Yu Zhe Date: Tue, 27 Sep 2022 15:02:47 +0800 Subject: [PATCH 1017/1927] cxl/pmem: Use size_add() against integer overflow "struct_size() + n" may cause a integer overflow, use size_add() to handle it. Signed-off-by: Yu Zhe Link: https://lore.kernel.org/r/20220927070247.23148-1-yuzhe@nfschina.com Signed-off-by: Dan Williams --- drivers/cxl/pmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 115a7b79f3439..0bac05d804bc5 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -148,7 +148,7 @@ static int cxl_pmem_set_config_data(struct cxl_dev_state *cxlds, return -EINVAL; /* 4-byte status follows the input data in the payload */ - if (struct_size(cmd, in_buf, cmd->in_length) + 4 > buf_len) + if (size_add(struct_size(cmd, in_buf, cmd->in_length), 4) > buf_len) return -EINVAL; set_lsa = -- GitLab From c7e37cc6240767f794678d11704935d49cc81d59 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Thu, 3 Nov 2022 11:31:43 -0700 Subject: [PATCH 1018/1927] Input: goodix - try resetting the controller when no config is set On ACPI systems (irq_pin_access_method == IRQ_PIN_ACCESS_ACPI_*) the driver does not reset the controller at probe time, because sometimes the system firmware loads a config and resetting might loose this config. On the Nanote UMPC-01 device OTOH the config is in flash of the controller, the controller needs a reset to load this; and the system firmware does not reset the controller on a cold boot. To fix the Nanote UMPC-01 touchscreen not working on a cold boot, try resetting the controller and then re-reading the config when encountering a config with 0 width/height/max_touch_num value and the controller has not already been reset by goodix_ts_probe(). This should be safe to do in general because normally we should never encounter a config with 0 width/height/max_touch_num. Doing this in general not only avoids the need for a DMI quirk, but also might help other systems. Signed-off-by: Hans de Goede Reviewed-by: Bastien Nocera Link: https://lore.kernel.org/r/20221025122930.421377-2-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/touchscreen/goodix.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c index a33cc7950cf5b..c281e49826c23 100644 --- a/drivers/input/touchscreen/goodix.c +++ b/drivers/input/touchscreen/goodix.c @@ -1158,6 +1158,7 @@ static int goodix_configure_dev(struct goodix_ts_data *ts) input_set_abs_params(ts->input_dev, ABS_MT_WIDTH_MAJOR, 0, 255, 0, 0); input_set_abs_params(ts->input_dev, ABS_MT_TOUCH_MAJOR, 0, 255, 0, 0); +retry_read_config: /* Read configuration and apply touchscreen parameters */ goodix_read_config(ts); @@ -1165,6 +1166,16 @@ static int goodix_configure_dev(struct goodix_ts_data *ts) touchscreen_parse_properties(ts->input_dev, true, &ts->prop); if (!ts->prop.max_x || !ts->prop.max_y || !ts->max_touch_num) { + if (!ts->reset_controller_at_probe && + ts->irq_pin_access_method != IRQ_PIN_ACCESS_NONE) { + dev_info(&ts->client->dev, "Config not set, resetting controller\n"); + /* Retry after a controller reset */ + ts->reset_controller_at_probe = true; + error = goodix_reset(ts); + if (error) + return error; + goto retry_read_config; + } dev_err(&ts->client->dev, "Invalid config (%d, %d, %d), using defaults\n", ts->prop.max_x, ts->prop.max_y, ts->max_touch_num); -- GitLab From f1db20814af532f85e091231223e5e4818e8464b Mon Sep 17 00:00:00 2001 From: Youlin Li Date: Thu, 3 Nov 2022 17:34:39 +0800 Subject: [PATCH 1019/1927] bpf: Fix wrong reg type conversion in release_reference() Some helper functions will allocate memory. To avoid memory leaks, the verifier requires the eBPF program to release these memories by calling the corresponding helper functions. When a resource is released, all pointer registers corresponding to the resource should be invalidated. The verifier use release_references() to do this job, by apply __mark_reg_unknown() to each relevant register. It will give these registers the type of SCALAR_VALUE. A register that will contain a pointer value at runtime, but of type SCALAR_VALUE, which may allow the unprivileged user to get a kernel pointer by storing this register into a map. Using __mark_reg_not_init() while NOT allow_ptr_leaks can mitigate this problem. Fixes: fd978bf7fd31 ("bpf: Add reference tracking to verifier") Signed-off-by: Youlin Li Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20221103093440.3161-1-liulin063@gmail.com --- kernel/bpf/verifier.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index dd9019c8b0db0..225666307bba3 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6623,8 +6623,12 @@ static int release_reference(struct bpf_verifier_env *env, return err; bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({ - if (reg->ref_obj_id == ref_obj_id) - __mark_reg_unknown(env, reg); + if (reg->ref_obj_id == ref_obj_id) { + if (!env->allow_ptr_leaks) + __mark_reg_not_init(env, reg); + else + __mark_reg_unknown(env, reg); + } })); return 0; -- GitLab From 475244f5e06beeda7b557d9dde46a5f439bf3379 Mon Sep 17 00:00:00 2001 From: Youlin Li Date: Thu, 3 Nov 2022 17:34:40 +0800 Subject: [PATCH 1020/1927] selftests/bpf: Add verifier test for release_reference() Add a test case to ensure that released pointer registers will not be leaked into the map. Before fix: ./test_verifier 984 984/u reference tracking: try to leak released ptr reg FAIL Unexpected success to load! verification time 67 usec stack depth 4 processed 23 insns (limit 1000000) max_states_per_insn 0 total_states 2 peak_states 2 mark_read 1 984/p reference tracking: try to leak released ptr reg OK Summary: 1 PASSED, 0 SKIPPED, 1 FAILED After fix: ./test_verifier 984 984/u reference tracking: try to leak released ptr reg OK 984/p reference tracking: try to leak released ptr reg OK Summary: 2 PASSED, 0 SKIPPED, 0 FAILED Signed-off-by: Youlin Li Signed-off-by: Daniel Borkmann Link: https://lore.kernel.org/bpf/20221103093440.3161-2-liulin063@gmail.com --- .../selftests/bpf/verifier/ref_tracking.c | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tools/testing/selftests/bpf/verifier/ref_tracking.c b/tools/testing/selftests/bpf/verifier/ref_tracking.c index f18ce867271fc..fd683a32a2766 100644 --- a/tools/testing/selftests/bpf/verifier/ref_tracking.c +++ b/tools/testing/selftests/bpf/verifier/ref_tracking.c @@ -1044,3 +1044,39 @@ .result_unpriv = REJECT, .errstr_unpriv = "unknown func", }, +{ + "reference tracking: try to leak released ptr reg", + .insns = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_0, -4), + BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), + BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_9, BPF_REG_0), + + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_LD_MAP_FD(BPF_REG_1, 0), + BPF_MOV64_IMM(BPF_REG_2, 8), + BPF_MOV64_IMM(BPF_REG_3, 0), + BPF_EMIT_CALL(BPF_FUNC_ringbuf_reserve), + BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 1), + BPF_EXIT_INSN(), + BPF_MOV64_REG(BPF_REG_8, BPF_REG_0), + + BPF_MOV64_REG(BPF_REG_1, BPF_REG_8), + BPF_MOV64_IMM(BPF_REG_2, 0), + BPF_EMIT_CALL(BPF_FUNC_ringbuf_discard), + BPF_MOV64_IMM(BPF_REG_0, 0), + + BPF_STX_MEM(BPF_DW, BPF_REG_9, BPF_REG_8, 0), + BPF_EXIT_INSN() + }, + .fixup_map_array_48b = { 4 }, + .fixup_map_ringbuf = { 11 }, + .result = ACCEPT, + .result_unpriv = REJECT, + .errstr_unpriv = "R8 !read_ok" +}, -- GitLab From 4a6f316d6855a434f56dbbeba05e14c01acde8f8 Mon Sep 17 00:00:00 2001 From: Li Qiang Date: Fri, 4 Nov 2022 08:49:31 +0900 Subject: [PATCH 1021/1927] kprobe: reverse kp->flags when arm_kprobe failed In aggregate kprobe case, when arm_kprobe failed, we need set the kp->flags with KPROBE_FLAG_DISABLED again. If not, the 'kp' kprobe will been considered as enabled but it actually not enabled. Link: https://lore.kernel.org/all/20220902155820.34755-1-liq3ea@163.com/ Fixes: 12310e343755 ("kprobes: Propagate error from arm_kprobe_ftrace()") Cc: stable@vger.kernel.org Signed-off-by: Li Qiang Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/kprobes.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 3220b0a2fb4a3..cd9f5a66a6909 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -2429,8 +2429,11 @@ int enable_kprobe(struct kprobe *kp) if (!kprobes_all_disarmed && kprobe_disabled(p)) { p->flags &= ~KPROBE_FLAG_DISABLED; ret = arm_kprobe(p); - if (ret) + if (ret) { p->flags |= KPROBE_FLAG_DISABLED; + if (p != kp) + kp->flags |= KPROBE_FLAG_DISABLED; + } } out: mutex_unlock(&kprobe_mutex); -- GitLab From d05ea35e7eea14d32f29fd688d3daeb9089de1a5 Mon Sep 17 00:00:00 2001 From: Rafael Mendonca Date: Tue, 25 Oct 2022 00:12:08 -0300 Subject: [PATCH 1022/1927] fprobe: Check rethook_alloc() return in rethook initialization Check if fp->rethook succeeded to be allocated. Otherwise, if rethook_alloc() fails, then we end up dereferencing a NULL pointer in rethook_add_node(). Link: https://lore.kernel.org/all/20221025031209.954836-1-rafaelmendsr@gmail.com/ Fixes: 5b0ab78998e3 ("fprobe: Add exit_handler support") Cc: stable@vger.kernel.org Signed-off-by: Rafael Mendonca Acked-by: Steven Rostedt (Google) Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/fprobe.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index aac63ca9c3d1e..71614b2a67ff3 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -141,6 +141,8 @@ static int fprobe_init_rethook(struct fprobe *fp, int num) return -E2BIG; fp->rethook = rethook_alloc((void *)fp, fprobe_exit_handler); + if (!fp->rethook) + return -ENOMEM; for (i = 0; i < size; i++) { struct fprobe_rethook_node *node; -- GitLab From 61b304b73ab4b48b1cd7796efe42a570e2a0e0fc Mon Sep 17 00:00:00 2001 From: "Masami Hiramatsu (Google)" Date: Sun, 23 Oct 2022 11:11:43 +0900 Subject: [PATCH 1023/1927] tracing/fprobe: Fix to check whether fprobe is registered correctly Since commit ab51e15d535e ("fprobe: Introduce FPROBE_FL_KPROBE_SHARED flag for fprobe") introduced fprobe_kprobe_handler() for fprobe::ops::func, unregister_fprobe() fails to unregister the registered if user specifies FPROBE_FL_KPROBE_SHARED flag. Moreover, __register_ftrace_function() is possible to change the ftrace_ops::func, thus we have to check fprobe::ops::saved_func instead. To check it correctly, it should confirm the fprobe::ops::saved_func is either fprobe_handler() or fprobe_kprobe_handler(). Link: https://lore.kernel.org/all/166677683946.1459107.15997653945538644683.stgit@devnote3/ Fixes: cad9931f64dc ("fprobe: Add ftrace based probe APIs") Cc: stable@vger.kernel.org Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/fprobe.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 71614b2a67ff3..e8143e3680744 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -303,7 +303,8 @@ int unregister_fprobe(struct fprobe *fp) { int ret; - if (!fp || fp->ops.func != fprobe_handler) + if (!fp || (fp->ops.saved_func != fprobe_handler && + fp->ops.saved_func != fprobe_kprobe_handler)) return -EINVAL; /* -- GitLab From 66f0919c953ef7b55e5ab94389a013da2ce80a2c Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Wed, 2 Nov 2022 15:29:54 +0800 Subject: [PATCH 1024/1927] tracing: kprobe: Fix memory leak in test_gen_kprobe/kretprobe_cmd() test_gen_kprobe_cmd() only free buf in fail path, hence buf will leak when there is no failure. Move kfree(buf) from fail path to common path to prevent the memleak. The same reason and solution in test_gen_kretprobe_cmd(). unreferenced object 0xffff888143b14000 (size 2048): comm "insmod", pid 52490, jiffies 4301890980 (age 40.553s) hex dump (first 32 bytes): 70 3a 6b 70 72 6f 62 65 73 2f 67 65 6e 5f 6b 70 p:kprobes/gen_kp 72 6f 62 65 5f 74 65 73 74 20 64 6f 5f 73 79 73 robe_test do_sys backtrace: [<000000006d7b836b>] kmalloc_trace+0x27/0xa0 [<0000000009528b5b>] 0xffffffffa059006f [<000000008408b580>] do_one_initcall+0x87/0x2a0 [<00000000c4980a7e>] do_init_module+0xdf/0x320 [<00000000d775aad0>] load_module+0x3006/0x3390 [<00000000e9a74b80>] __do_sys_finit_module+0x113/0x1b0 [<000000003726480d>] do_syscall_64+0x35/0x80 [<000000003441e93b>] entry_SYSCALL_64_after_hwframe+0x46/0xb0 Link: https://lore.kernel.org/all/20221102072954.26555-1-shangxiaojing@huawei.com/ Fixes: 64836248dda2 ("tracing: Add kprobe event command generation test module") Cc: stable@vger.kernel.org Signed-off-by: Shang XiaoJing Acked-by: Masami Hiramatsu (Google) Signed-off-by: Masami Hiramatsu (Google) --- kernel/trace/kprobe_event_gen_test.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/kernel/trace/kprobe_event_gen_test.c b/kernel/trace/kprobe_event_gen_test.c index 80e04a1e19772..d81f7c51025c7 100644 --- a/kernel/trace/kprobe_event_gen_test.c +++ b/kernel/trace/kprobe_event_gen_test.c @@ -100,20 +100,20 @@ static int __init test_gen_kprobe_cmd(void) KPROBE_GEN_TEST_FUNC, KPROBE_GEN_TEST_ARG0, KPROBE_GEN_TEST_ARG1); if (ret) - goto free; + goto out; /* Use kprobe_event_add_fields to add the rest of the fields */ ret = kprobe_event_add_fields(&cmd, KPROBE_GEN_TEST_ARG2, KPROBE_GEN_TEST_ARG3); if (ret) - goto free; + goto out; /* * This actually creates the event. */ ret = kprobe_event_gen_cmd_end(&cmd); if (ret) - goto free; + goto out; /* * Now get the gen_kprobe_test event file. We need to prevent @@ -136,13 +136,11 @@ static int __init test_gen_kprobe_cmd(void) goto delete; } out: + kfree(buf); return ret; delete: /* We got an error after creating the event, delete it */ ret = kprobe_event_delete("gen_kprobe_test"); - free: - kfree(buf); - goto out; } @@ -170,14 +168,14 @@ static int __init test_gen_kretprobe_cmd(void) KPROBE_GEN_TEST_FUNC, "$retval"); if (ret) - goto free; + goto out; /* * This actually creates the event. */ ret = kretprobe_event_gen_cmd_end(&cmd); if (ret) - goto free; + goto out; /* * Now get the gen_kretprobe_test event file. We need to @@ -201,13 +199,11 @@ static int __init test_gen_kretprobe_cmd(void) goto delete; } out: + kfree(buf); return ret; delete: /* We got an error after creating the event, delete it */ ret = kprobe_event_delete("gen_kretprobe_test"); - free: - kfree(buf); - goto out; } -- GitLab From 9e4b7a99a03aefd37ba7bb1f022c8efab5019165 Mon Sep 17 00:00:00 2001 From: Jiri Benc Date: Wed, 2 Nov 2022 17:53:25 +0100 Subject: [PATCH 1025/1927] net: gso: fix panic on frag_list with mixed head alloc types Since commit 3dcbdb134f32 ("net: gso: Fix skb_segment splat when splitting gso_size mangled skb having linear-headed frag_list"), it is allowed to change gso_size of a GRO packet. However, that commit assumes that "checking the first list_skb member suffices; i.e if either of the list_skb members have non head_frag head, then the first one has too". It turns out this assumption does not hold. We've seen BUG_ON being hit in skb_segment when skbs on the frag_list had differing head_frag with the vmxnet3 driver. This happens because __netdev_alloc_skb and __napi_alloc_skb can return a skb that is page backed or kmalloced depending on the requested size. As the result, the last small skb in the GRO packet can be kmalloced. There are three different locations where this can be fixed: (1) We could check head_frag in GRO and not allow GROing skbs with different head_frag. However, that would lead to performance regression on normal forward paths with unmodified gso_size, where !head_frag in the last packet is not a problem. (2) Set a flag in bpf_skb_net_grow and bpf_skb_net_shrink indicating that NETIF_F_SG is undesirable. That would need to eat a bit in sk_buff. Furthermore, that flag can be unset when all skbs on the frag_list are page backed. To retain good performance, bpf_skb_net_grow/shrink would have to walk the frag_list. (3) Walk the frag_list in skb_segment when determining whether NETIF_F_SG should be cleared. This of course slows things down. This patch implements (3). To limit the performance impact in skb_segment, the list is walked only for skbs with SKB_GSO_DODGY set that have gso_size changed. Normal paths thus will not hit it. We could check only the last skb but since we need to walk the whole list anyway, let's stay on the safe side. Fixes: 3dcbdb134f32 ("net: gso: Fix skb_segment splat when splitting gso_size mangled skb having linear-headed frag_list") Signed-off-by: Jiri Benc Reviewed-by: Willem de Bruijn Link: https://lore.kernel.org/r/e04426a6a91baf4d1081e1b478c82b5de25fdf21.1667407944.git.jbenc@redhat.com Signed-off-by: Jakub Kicinski --- net/core/skbuff.c | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index d1a3fa6f3f126..88fa40571d0c7 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -4134,23 +4134,25 @@ struct sk_buff *skb_segment(struct sk_buff *head_skb, int i = 0; int pos; - if (list_skb && !list_skb->head_frag && skb_headlen(list_skb) && - (skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY)) { - /* gso_size is untrusted, and we have a frag_list with a linear - * non head_frag head. - * - * (we assume checking the first list_skb member suffices; - * i.e if either of the list_skb members have non head_frag - * head, then the first one has too). - * - * If head_skb's headlen does not fit requested gso_size, it - * means that the frag_list members do NOT terminate on exact - * gso_size boundaries. Hence we cannot perform skb_frag_t page - * sharing. Therefore we must fallback to copying the frag_list - * skbs; we do so by disabling SG. - */ - if (mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) - features &= ~NETIF_F_SG; + if ((skb_shinfo(head_skb)->gso_type & SKB_GSO_DODGY) && + mss != GSO_BY_FRAGS && mss != skb_headlen(head_skb)) { + struct sk_buff *check_skb; + + for (check_skb = list_skb; check_skb; check_skb = check_skb->next) { + if (skb_headlen(check_skb) && !check_skb->head_frag) { + /* gso_size is untrusted, and we have a frag_list with + * a linear non head_frag item. + * + * If head_skb's headlen does not fit requested gso_size, + * it means that the frag_list members do NOT terminate + * on exact gso_size boundaries. Hence we cannot perform + * skb_frag_t page sharing. Therefore we must fallback to + * copying the frag_list skbs; we do so by disabling SG. + */ + features &= ~NETIF_F_SG; + break; + } + } } __skb_push(head_skb, doffset); -- GitLab From f77810f744139572a63e5a85ab6a8c10c2d44fb1 Mon Sep 17 00:00:00 2001 From: Jason Gerecke Date: Thu, 3 Nov 2022 10:33:04 -0700 Subject: [PATCH 1026/1927] HID: wacom: Fix logic used for 3rd barrel switch emulation When support was added for devices using an explicit 3rd barrel switch, the logic used by devices emulating this feature was broken. The 'if' statement / block that was introduced only handles the case where the button is pressed (i.e. 'barrelswitch' and 'barrelswitch2' are both set) but not the case where it is released (i.e. one or both being cleared). This results in a BTN_STYLUS3 "down" event being sent when the button is pressed, but no "up" event ever being sent afterwards. This patch restores the previously-used logic for determining button states in the emulated case so that switches are reported correctly again. Link: https://github.com/linuxwacom/xf86-input-wacom/issues/292 Fixes: 6d09085b38e5 ("HID: wacom: Adding Support for new usages") CC: stable@vger.kernel.org #v5.19+ Signed-off-by: Jason Gerecke Tested-by: Joshua Dickens Reviewed-by: Ping Cheng Signed-off-by: Jiri Kosina --- drivers/hid/wacom_wac.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index 77486962a773f..0f3d57b426846 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -2520,11 +2520,12 @@ static void wacom_wac_pen_report(struct hid_device *hdev, if (!delay_pen_events(wacom_wac) && wacom_wac->tool[0]) { int id = wacom_wac->id[0]; - if (wacom_wac->features.quirks & WACOM_QUIRK_PEN_BUTTON3 && - wacom_wac->hid_data.barrelswitch & wacom_wac->hid_data.barrelswitch2) { - wacom_wac->hid_data.barrelswitch = 0; - wacom_wac->hid_data.barrelswitch2 = 0; - wacom_wac->hid_data.barrelswitch3 = 1; + if (wacom_wac->features.quirks & WACOM_QUIRK_PEN_BUTTON3) { + int sw_state = wacom_wac->hid_data.barrelswitch | + (wacom_wac->hid_data.barrelswitch2 << 1); + wacom_wac->hid_data.barrelswitch = sw_state == 1; + wacom_wac->hid_data.barrelswitch2 = sw_state == 2; + wacom_wac->hid_data.barrelswitch3 = sw_state == 3; } input_report_key(input, BTN_STYLUS, wacom_wac->hid_data.barrelswitch); input_report_key(input, BTN_STYLUS2, wacom_wac->hid_data.barrelswitch2); -- GitLab From 4581dd480c9e42a1ad21dd8b9c110abe41878ce5 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Mon, 31 Oct 2022 22:08:13 -0700 Subject: [PATCH 1027/1927] net: octeontx2-pf: mcs: consider MACSEC setting Fix build errors when MACSEC=m and OCTEONTX2_PF=y by having OCTEONTX2_PF depend on MACSEC if it is enabled. By adding "|| !MACSEC", this means that MACSEC is not required -- it can be disabled for this driver. drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.o: in function `otx2_remove': ../drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c:(.text+0x2fd0): undefined reference to `cn10k_mcs_free' mips64-linux-ld: drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.o: in function `otx2_mbox_up_handler_mcs_intr_notify': ../drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c:(.text+0x4610): undefined reference to `cn10k_handle_mcs_event' Reported-by: kernel test robot Fixes: c54ffc73601c ("octeontx2-pf: mcs: Introduce MACSEC hardware offloading") Signed-off-by: Randy Dunlap Cc: Subbaraya Sundeep Cc: Sunil Goutham Cc: Geetha sowjanya Cc: hariprasad Cc: "David S. Miller" Cc: Eric Dumazet Cc: Jakub Kicinski Cc: Paolo Abeni Signed-off-by: David S. Miller --- drivers/net/ethernet/marvell/octeontx2/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig index e1036b0eb6b15..993ac180a5db8 100644 --- a/drivers/net/ethernet/marvell/octeontx2/Kconfig +++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig @@ -36,6 +36,7 @@ config OCTEONTX2_PF select DIMLIB depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + depends on MACSEC || !MACSEC help This driver supports Marvell's OcteonTX2 NIC physical function. -- GitLab From cfdcb075048c1e886c45a9c9e681ed222f74ecb9 Mon Sep 17 00:00:00 2001 From: Guangbin Huang Date: Tue, 1 Nov 2022 15:48:38 +0800 Subject: [PATCH 1028/1927] net: hns3: fix get wrong value of function hclge_get_dscp_prio() As the argument struct hnae3_handle *h of function hclge_get_dscp_prio() can be other client registered in hnae3 layer, we need to transform it into hnae3_handle of local nic client to get right dscp settings for other clients. Fixes: dfea275e06c2 ("net: hns3: optimize converting dscp to priority process of hns3_nic_select_queue()") Signed-off-by: Guangbin Huang Signed-off-by: David S. Miller --- drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 6962a9d69cf8d..987271da6e9bc 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -12984,14 +12984,16 @@ static void hclge_clean_vport_config(struct hnae3_ae_dev *ae_dev, int num_vfs) static int hclge_get_dscp_prio(struct hnae3_handle *h, u8 dscp, u8 *tc_mode, u8 *priority) { + struct hclge_vport *vport = hclge_get_vport(h); + if (dscp >= HNAE3_MAX_DSCP) return -EINVAL; if (tc_mode) - *tc_mode = h->kinfo.tc_map_mode; + *tc_mode = vport->nic.kinfo.tc_map_mode; if (priority) - *priority = h->kinfo.dscp_prio[dscp] == HNAE3_PRIO_ID_INVALID ? 0 : - h->kinfo.dscp_prio[dscp]; + *priority = vport->nic.kinfo.dscp_prio[dscp] == HNAE3_PRIO_ID_INVALID ? 0 : + vport->nic.kinfo.dscp_prio[dscp]; return 0; } -- GitLab From cdb525ca92b196f8916102b62431aa0d9a644ff2 Mon Sep 17 00:00:00 2001 From: Adrien Thierry Date: Tue, 1 Nov 2022 14:48:08 -0400 Subject: [PATCH 1029/1927] selftests/net: give more time to udpgro bg processes to complete startup In some conditions, background processes in udpgro don't have enough time to set up the sockets. When foreground processes start, this results in the test failing with "./udpgso_bench_tx: sendmsg: Connection refused". For instance, this happens from time to time on a Qualcomm SA8540P SoC running CentOS Stream 9. To fix this, increase the time given to background processes to complete the startup before foreground processes start. Signed-off-by: Adrien Thierry Signed-off-by: David S. Miller --- tools/testing/selftests/net/udpgro.sh | 4 ++-- tools/testing/selftests/net/udpgro_bench.sh | 2 +- tools/testing/selftests/net/udpgro_frglist.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/net/udpgro.sh b/tools/testing/selftests/net/udpgro.sh index ebbd0b2824327..6a443ca3cd3a4 100755 --- a/tools/testing/selftests/net/udpgro.sh +++ b/tools/testing/selftests/net/udpgro.sh @@ -50,7 +50,7 @@ run_one() { echo "failed" & # Hack: let bg programs complete the startup - sleep 0.1 + sleep 0.2 ./udpgso_bench_tx ${tx_args} ret=$? wait $(jobs -p) @@ -117,7 +117,7 @@ run_one_2sock() { echo "failed" & # Hack: let bg programs complete the startup - sleep 0.1 + sleep 0.2 ./udpgso_bench_tx ${tx_args} -p 12345 sleep 0.1 # first UDP GSO socket should be closed at this point diff --git a/tools/testing/selftests/net/udpgro_bench.sh b/tools/testing/selftests/net/udpgro_bench.sh index fad2d1a71cac3..8a1109a545dba 100755 --- a/tools/testing/selftests/net/udpgro_bench.sh +++ b/tools/testing/selftests/net/udpgro_bench.sh @@ -39,7 +39,7 @@ run_one() { ip netns exec "${PEER_NS}" ./udpgso_bench_rx -t ${rx_args} -r & # Hack: let bg programs complete the startup - sleep 0.1 + sleep 0.2 ./udpgso_bench_tx ${tx_args} } diff --git a/tools/testing/selftests/net/udpgro_frglist.sh b/tools/testing/selftests/net/udpgro_frglist.sh index 832c738cc3c29..7fe85ba51075d 100755 --- a/tools/testing/selftests/net/udpgro_frglist.sh +++ b/tools/testing/selftests/net/udpgro_frglist.sh @@ -44,7 +44,7 @@ run_one() { ip netns exec "${PEER_NS}" ./udpgso_bench_rx ${rx_args} -r & # Hack: let bg programs complete the startup - sleep 0.1 + sleep 0.2 ./udpgso_bench_tx ${tx_args} } -- GitLab From 8bcd560ae8784da57c610d857118c5d6576b1a8f Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 2 Nov 2022 22:33:12 +0100 Subject: [PATCH 1030/1927] Revert "net: macsec: report real_dev features when HW offloading is enabled" This reverts commit c850240b6c4132574a00f2da439277ab94265b66. That commit tried to improve the performance of macsec offload by taking advantage of some of the NIC's features, but in doing so, broke macsec offload when the lower device supports both macsec and ipsec offload, as the ipsec offload feature flags (mainly NETIF_F_HW_ESP) were copied from the real device. Since the macsec device doesn't provide xdo_* ops, the XFRM core rejects the registration of the new macsec device in xfrm_api_check. Example perf trace when running ip link add link eni1np1 type macsec port 4 offload mac ip 737 [003] 795.477676: probe:xfrm_dev_event__REGISTER name="macsec0" features=0x1c000080014869 xfrm_dev_event+0x3a notifier_call_chain+0x47 register_netdevice+0x846 macsec_newlink+0x25a ip 737 [003] 795.477687: probe:xfrm_dev_event__return ret=0x8002 (NOTIFY_BAD) notifier_call_chain+0x47 register_netdevice+0x846 macsec_newlink+0x25a dev->features includes NETIF_F_HW_ESP (0x04000000000000), so xfrm_api_check returns NOTIFY_BAD because we don't have dev->xfrmdev_ops on the macsec device. We could probably propagate GSO and a few other features from the lower device, similar to macvlan. This will be done in a future patch. Signed-off-by: Sabrina Dubroca Reviewed-by: Antoine Tenart Reviewed-by: Leon Romanovsky Signed-off-by: David S. Miller --- drivers/net/macsec.c | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index c891b60937a7f..b3f76e8071f24 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -2654,11 +2654,6 @@ static int macsec_upd_offload(struct sk_buff *skb, struct genl_info *info) if (ret) goto rollback; - /* Force features update, since they are different for SW MACSec and - * HW offloading cases. - */ - netdev_update_features(dev); - rtnl_unlock(); return 0; @@ -3432,16 +3427,9 @@ static netdev_tx_t macsec_start_xmit(struct sk_buff *skb, return ret; } -#define SW_MACSEC_FEATURES \ +#define MACSEC_FEATURES \ (NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_FRAGLIST) -/* If h/w offloading is enabled, use real device features save for - * VLAN_FEATURES - they require additional ops - * HW_MACSEC - no reason to report it - */ -#define REAL_DEV_FEATURES(dev) \ - ((dev)->features & ~(NETIF_F_VLAN_FEATURES | NETIF_F_HW_MACSEC)) - static int macsec_dev_init(struct net_device *dev) { struct macsec_dev *macsec = macsec_priv(dev); @@ -3458,12 +3446,8 @@ static int macsec_dev_init(struct net_device *dev) return err; } - if (macsec_is_offloaded(macsec)) { - dev->features = REAL_DEV_FEATURES(real_dev); - } else { - dev->features = real_dev->features & SW_MACSEC_FEATURES; - dev->features |= NETIF_F_LLTX | NETIF_F_GSO_SOFTWARE; - } + dev->features = real_dev->features & MACSEC_FEATURES; + dev->features |= NETIF_F_LLTX | NETIF_F_GSO_SOFTWARE; dev->needed_headroom = real_dev->needed_headroom + MACSEC_NEEDED_HEADROOM; @@ -3495,10 +3479,7 @@ static netdev_features_t macsec_fix_features(struct net_device *dev, struct macsec_dev *macsec = macsec_priv(dev); struct net_device *real_dev = macsec->real_dev; - if (macsec_is_offloaded(macsec)) - return REAL_DEV_FEATURES(real_dev); - - features &= (real_dev->features & SW_MACSEC_FEATURES) | + features &= (real_dev->features & MACSEC_FEATURES) | NETIF_F_GSO_SOFTWARE | NETIF_F_SOFT_FEATURES; features |= NETIF_F_LLTX; -- GitLab From 93a30947821c203d08865c4e17ea181c9668ce52 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 2 Nov 2022 22:33:13 +0100 Subject: [PATCH 1031/1927] macsec: delete new rxsc when offload fails Currently we get an inconsistent state: - netlink returns the error to userspace - the RXSC is installed but not offloaded Then the device could get confused when we try to add an RXSA, because the RXSC isn't supposed to exist. Fixes: 3cf3227a21d1 ("net: macsec: hardware offloading infrastructure") Signed-off-by: Sabrina Dubroca Reviewed-by: Antoine Tenart Reviewed-by: Leon Romanovsky Signed-off-by: David S. Miller --- drivers/net/macsec.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index b3f76e8071f24..0d6fe34b91aeb 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -1876,7 +1876,6 @@ static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info) struct macsec_rx_sc *rx_sc; struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1]; struct macsec_secy *secy; - bool was_active; int ret; if (!attrs[MACSEC_ATTR_IFINDEX]) @@ -1904,7 +1903,6 @@ static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info) return PTR_ERR(rx_sc); } - was_active = rx_sc->active; if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) rx_sc->active = !!nla_get_u8(tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]); @@ -1931,7 +1929,8 @@ static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info) return 0; cleanup: - rx_sc->active = was_active; + del_rx_sc(secy, sci); + free_rx_sc(rx_sc); rtnl_unlock(); return ret; } -- GitLab From 73a4b31c9d11f98ae3bc5286d5382930adb0e9c7 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 2 Nov 2022 22:33:14 +0100 Subject: [PATCH 1032/1927] macsec: fix secy->n_rx_sc accounting secy->n_rx_sc is supposed to be the number of _active_ rxsc's within a secy. This is then used by macsec_send_sci to help decide if we should add the SCI to the header or not. This logic is currently broken when we create a new RXSC and turn it off at creation, as create_rx_sc always sets ->active to true (and immediately uses that to increment n_rx_sc), and only later macsec_add_rxsc sets rx_sc->active. Fixes: c09440f7dcb3 ("macsec: introduce IEEE 802.1AE driver") Signed-off-by: Sabrina Dubroca Reviewed-by: Antoine Tenart Reviewed-by: Leon Romanovsky Signed-off-by: David S. Miller --- drivers/net/macsec.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 0d6fe34b91aeb..1b4d856f4bd70 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -1413,7 +1413,8 @@ static struct macsec_rx_sc *del_rx_sc(struct macsec_secy *secy, sci_t sci) return NULL; } -static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci) +static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci, + bool active) { struct macsec_rx_sc *rx_sc; struct macsec_dev *macsec; @@ -1437,7 +1438,7 @@ static struct macsec_rx_sc *create_rx_sc(struct net_device *dev, sci_t sci) } rx_sc->sci = sci; - rx_sc->active = true; + rx_sc->active = active; refcount_set(&rx_sc->refcnt, 1); secy = &macsec_priv(dev)->secy; @@ -1876,6 +1877,7 @@ static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info) struct macsec_rx_sc *rx_sc; struct nlattr *tb_rxsc[MACSEC_RXSC_ATTR_MAX + 1]; struct macsec_secy *secy; + bool active = true; int ret; if (!attrs[MACSEC_ATTR_IFINDEX]) @@ -1897,15 +1899,15 @@ static int macsec_add_rxsc(struct sk_buff *skb, struct genl_info *info) secy = &macsec_priv(dev)->secy; sci = nla_get_sci(tb_rxsc[MACSEC_RXSC_ATTR_SCI]); - rx_sc = create_rx_sc(dev, sci); + if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) + active = nla_get_u8(tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]); + + rx_sc = create_rx_sc(dev, sci, active); if (IS_ERR(rx_sc)) { rtnl_unlock(); return PTR_ERR(rx_sc); } - if (tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]) - rx_sc->active = !!nla_get_u8(tb_rxsc[MACSEC_RXSC_ATTR_ACTIVE]); - if (macsec_is_offloaded(netdev_priv(dev))) { const struct macsec_ops *ops; struct macsec_context ctx; -- GitLab From 80df4706357a5a06bbbc70273bf2611df1ceee04 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 2 Nov 2022 22:33:15 +0100 Subject: [PATCH 1033/1927] macsec: fix detection of RXSCs when toggling offloading macsec_is_configured incorrectly uses secy->n_rx_sc to check if some RXSCs exist. secy->n_rx_sc only counts the number of active RXSCs, but there can also be inactive SCs as well, which may be stored in the driver (in case we're disabling offloading), or would have to be pushed to the device (in case we're trying to enable offloading). As long as RXSCs active on creation and never turned off, the issue is not visible. Fixes: dcb780fb2795 ("net: macsec: add nla support for changing the offloading selection") Signed-off-by: Sabrina Dubroca Reviewed-by: Antoine Tenart Reviewed-by: Leon Romanovsky Signed-off-by: David S. Miller --- drivers/net/macsec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 1b4d856f4bd70..700a8f96c6c2f 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -2571,7 +2571,7 @@ static bool macsec_is_configured(struct macsec_dev *macsec) struct macsec_tx_sc *tx_sc = &secy->tx_sc; int i; - if (secy->n_rx_sc > 0) + if (secy->rx_sc) return true; for (i = 0; i < MACSEC_NUM_AN; i++) -- GitLab From aaab73f8fba4fd38f4d2617440d541a1c334e819 Mon Sep 17 00:00:00 2001 From: Sabrina Dubroca Date: Wed, 2 Nov 2022 22:33:16 +0100 Subject: [PATCH 1034/1927] macsec: clear encryption keys from the stack after setting up offload macsec_add_rxsa and macsec_add_txsa copy the key to an on-stack offloading context to pass it to the drivers, but leaves it there when it's done. Clear it with memzero_explicit as soon as it's not needed anymore. Fixes: 3cf3227a21d1 ("net: macsec: hardware offloading infrastructure") Signed-off-by: Sabrina Dubroca Reviewed-by: Antoine Tenart Reviewed-by: Leon Romanovsky Signed-off-by: David S. Miller --- drivers/net/macsec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c index 700a8f96c6c2f..85376d2f24ca8 100644 --- a/drivers/net/macsec.c +++ b/drivers/net/macsec.c @@ -1839,6 +1839,7 @@ static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info) secy->key_len); err = macsec_offload(ops->mdo_add_rxsa, &ctx); + memzero_explicit(ctx.sa.key, secy->key_len); if (err) goto cleanup; } @@ -2081,6 +2082,7 @@ static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info) secy->key_len); err = macsec_offload(ops->mdo_add_txsa, &ctx); + memzero_explicit(ctx.sa.key, secy->key_len); if (err) goto cleanup; } -- GitLab From 1a0c016a4831ea29be09bbc8162d4a2a0690b4b8 Mon Sep 17 00:00:00 2001 From: Roger Quadros Date: Wed, 2 Nov 2022 12:31:44 +0200 Subject: [PATCH 1035/1927] net: ethernet: ti: am65-cpsw: Fix segmentation fault at module unload Move am65_cpsw_nuss_phylink_cleanup() call to after am65_cpsw_nuss_cleanup_ndev() so phylink is still valid to prevent the below Segmentation fault on module remove when first slave link is up. [ 31.652944] Unable to handle kernel paging request at virtual address 00040008000005f4 [ 31.684627] Mem abort info: [ 31.687446] ESR = 0x0000000096000004 [ 31.704614] EC = 0x25: DABT (current EL), IL = 32 bits [ 31.720663] SET = 0, FnV = 0 [ 31.723729] EA = 0, S1PTW = 0 [ 31.740617] FSC = 0x04: level 0 translation fault [ 31.756624] Data abort info: [ 31.759508] ISV = 0, ISS = 0x00000004 [ 31.776705] CM = 0, WnR = 0 [ 31.779695] [00040008000005f4] address between user and kernel address ranges [ 31.808644] Internal error: Oops: 0000000096000004 [#1] PREEMPT SMP [ 31.814928] Modules linked in: wlcore_sdio wl18xx wlcore mac80211 libarc4 cfg80211 rfkill crct10dif_ce phy_gmii_sel ti_am65_cpsw_nuss(-) sch_fq_codel ipv6 [ 31.828776] CPU: 0 PID: 1026 Comm: modprobe Not tainted 6.1.0-rc2-00012-gfabfcf7dafdb-dirty #160 [ 31.837547] Hardware name: Texas Instruments AM625 (DT) [ 31.842760] pstate: 40000005 (nZcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) [ 31.849709] pc : phy_stop+0x18/0xf8 [ 31.853202] lr : phylink_stop+0x38/0xf8 [ 31.857031] sp : ffff80000a0839f0 [ 31.860335] x29: ffff80000a0839f0 x28: ffff000000de1c80 x27: 0000000000000000 [ 31.867462] x26: 0000000000000000 x25: 0000000000000000 x24: ffff80000a083b98 [ 31.874589] x23: 0000000000000800 x22: 0000000000000001 x21: ffff000001bfba90 [ 31.881715] x20: ffff0000015ee000 x19: 0004000800000200 x18: 0000000000000000 [ 31.888842] x17: ffff800076c45000 x16: ffff800008004000 x15: 000058e39660b106 [ 31.895969] x14: 0000000000000144 x13: 0000000000000144 x12: 0000000000000000 [ 31.903095] x11: 000000000000275f x10: 00000000000009e0 x9 : ffff80000a0837d0 [ 31.910222] x8 : ffff000000de26c0 x7 : ffff00007fbd6540 x6 : ffff00007fbd64c0 [ 31.917349] x5 : ffff00007fbd0b10 x4 : ffff00007fbd0b10 x3 : ffff00007fbd3920 [ 31.924476] x2 : d0a07fcff8b8d500 x1 : 0000000000000000 x0 : 0004000800000200 [ 31.931603] Call trace: [ 31.934042] phy_stop+0x18/0xf8 [ 31.937177] phylink_stop+0x38/0xf8 [ 31.940657] am65_cpsw_nuss_ndo_slave_stop+0x28/0x1e0 [ti_am65_cpsw_nuss] [ 31.947452] __dev_close_many+0xa4/0x140 [ 31.951371] dev_close_many+0x84/0x128 [ 31.955115] unregister_netdevice_many+0x130/0x6d0 [ 31.959897] unregister_netdevice_queue+0x94/0xd8 [ 31.964591] unregister_netdev+0x24/0x38 [ 31.968504] am65_cpsw_nuss_cleanup_ndev.isra.0+0x48/0x70 [ti_am65_cpsw_nuss] [ 31.975637] am65_cpsw_nuss_remove+0x58/0xf8 [ti_am65_cpsw_nuss] Cc: # v5.18+ Fixes: e8609e69470f ("net: ethernet: ti: am65-cpsw: Convert to PHYLINK") Signed-off-by: Roger Quadros Signed-off-by: David S. Miller --- drivers/net/ethernet/ti/am65-cpsw-nuss.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 7f86068f3ff63..c50b137f92d7b 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -2823,7 +2823,6 @@ static int am65_cpsw_nuss_remove(struct platform_device *pdev) if (ret < 0) return ret; - am65_cpsw_nuss_phylink_cleanup(common); am65_cpsw_unregister_devlink(common); am65_cpsw_unregister_notifiers(common); @@ -2831,6 +2830,7 @@ static int am65_cpsw_nuss_remove(struct platform_device *pdev) * dma_deconfigure(dev) before devres_release_all(dev) */ am65_cpsw_nuss_cleanup_ndev(common); + am65_cpsw_nuss_phylink_cleanup(common); of_platform_device_destroy(common->mdio_dev, NULL); -- GitLab From 51afe9026d0c63263abe9840e629f118d7405b36 Mon Sep 17 00:00:00 2001 From: Ratheesh Kannoth Date: Wed, 2 Nov 2022 08:41:13 +0530 Subject: [PATCH 1036/1927] octeontx2-pf: NIX TX overwrites SQ_CTX_HW_S[SQ_INT] In scenarios where multiple errors have occurred for a SQ before SW starts handling error interrupt, SQ_CTX[OP_INT] may get overwritten leading to NIX_LF_SQ_OP_INT returning incorrect value. To workaround this read LMT, MNQ and SQ individual error status registers to determine the cause of error. Fixes: 4ff7d1488a84 ("octeontx2-pf: Error handling support") Signed-off-by: Ratheesh Kannoth Reviewed-by: Sunil Kovvuri Goutham Signed-off-by: David S. Miller --- .../ethernet/marvell/octeontx2/nic/otx2_pf.c | 135 ++++++++++++++---- .../marvell/octeontx2/nic/otx2_struct.h | 57 ++++++++ 2 files changed, 162 insertions(+), 30 deletions(-) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 892ca88e0cf43..303930499a4c0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "otx2_reg.h" #include "otx2_common.h" @@ -1171,6 +1172,59 @@ int otx2_set_real_num_queues(struct net_device *netdev, } EXPORT_SYMBOL(otx2_set_real_num_queues); +static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = { + "NIX_SQOPERR_OOR", + "NIX_SQOPERR_CTX_FAULT", + "NIX_SQOPERR_CTX_POISON", + "NIX_SQOPERR_DISABLED", + "NIX_SQOPERR_SIZE_ERR", + "NIX_SQOPERR_OFLOW", + "NIX_SQOPERR_SQB_NULL", + "NIX_SQOPERR_SQB_FAULT", + "NIX_SQOPERR_SQE_SZ_ZERO", +}; + +static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = { + "NIX_MNQERR_SQ_CTX_FAULT", + "NIX_MNQERR_SQ_CTX_POISON", + "NIX_MNQERR_SQB_FAULT", + "NIX_MNQERR_SQB_POISON", + "NIX_MNQERR_TOTAL_ERR", + "NIX_MNQERR_LSO_ERR", + "NIX_MNQERR_CQ_QUERY_ERR", + "NIX_MNQERR_MAX_SQE_SIZE_ERR", + "NIX_MNQERR_MAXLEN_ERR", + "NIX_MNQERR_SQE_SIZEM1_ZERO", +}; + +static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = { + "NIX_SND_STATUS_GOOD", + "NIX_SND_STATUS_SQ_CTX_FAULT", + "NIX_SND_STATUS_SQ_CTX_POISON", + "NIX_SND_STATUS_SQB_FAULT", + "NIX_SND_STATUS_SQB_POISON", + "NIX_SND_STATUS_HDR_ERR", + "NIX_SND_STATUS_EXT_ERR", + "NIX_SND_STATUS_JUMP_FAULT", + "NIX_SND_STATUS_JUMP_POISON", + "NIX_SND_STATUS_CRC_ERR", + "NIX_SND_STATUS_IMM_ERR", + "NIX_SND_STATUS_SG_ERR", + "NIX_SND_STATUS_MEM_ERR", + "NIX_SND_STATUS_INVALID_SUBDC", + "NIX_SND_STATUS_SUBDC_ORDER_ERR", + "NIX_SND_STATUS_DATA_FAULT", + "NIX_SND_STATUS_DATA_POISON", + "NIX_SND_STATUS_NPC_DROP_ACTION", + "NIX_SND_STATUS_LOCK_VIOL", + "NIX_SND_STATUS_NPC_UCAST_CHAN_ERR", + "NIX_SND_STATUS_NPC_MCAST_CHAN_ERR", + "NIX_SND_STATUS_NPC_MCAST_ABORT", + "NIX_SND_STATUS_NPC_VTAG_PTR_ERR", + "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR", + "NIX_SND_STATUS_SEND_STATS_ERR", +}; + static irqreturn_t otx2_q_intr_handler(int irq, void *data) { struct otx2_nic *pf = data; @@ -1204,46 +1258,67 @@ static irqreturn_t otx2_q_intr_handler(int irq, void *data) /* SQ */ for (qidx = 0; qidx < pf->hw.tot_tx_queues; qidx++) { + u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg; + u8 sq_op_err_code, mnq_err_code, snd_err_code; + + /* Below debug registers captures first errors corresponding to + * those registers. We don't have to check against SQ qid as + * these are fatal errors. + */ + ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT); val = otx2_atomic64_add((qidx << 44), ptr); otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) | (val & NIX_SQINT_BITS)); - if (!(val & (NIX_SQINT_BITS | BIT_ULL(42)))) - continue; - if (val & BIT_ULL(42)) { netdev_err(pf->netdev, "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n", qidx, otx2_read64(pf, NIX_LF_ERR_INT)); - } else { - if (val & BIT_ULL(NIX_SQINT_LMT_ERR)) { - netdev_err(pf->netdev, "SQ%lld: LMT store error NIX_LF_SQ_OP_ERR_DBG:0x%llx", - qidx, - otx2_read64(pf, - NIX_LF_SQ_OP_ERR_DBG)); - otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_MNQ_ERR)) { - netdev_err(pf->netdev, "SQ%lld: Meta-descriptor enqueue error NIX_LF_MNQ_ERR_DGB:0x%llx\n", - qidx, - otx2_read64(pf, NIX_LF_MNQ_ERR_DBG)); - otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_SEND_ERR)) { - netdev_err(pf->netdev, "SQ%lld: Send error, NIX_LF_SEND_ERR_DBG 0x%llx", - qidx, - otx2_read64(pf, - NIX_LF_SEND_ERR_DBG)); - otx2_write64(pf, NIX_LF_SEND_ERR_DBG, - BIT_ULL(44)); - } - if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) - netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", - qidx); + goto done; } + sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG); + if (!(sq_op_err_dbg & BIT(44))) + goto chk_mnq_err_dbg; + + sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(%llx) err=%s\n", + qidx, sq_op_err_dbg, nix_sqoperr_e_str[sq_op_err_code]); + + otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44)); + + if (sq_op_err_code == NIX_SQOPERR_SQB_NULL) + goto chk_mnq_err_dbg; + + /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure. + * TODO: But we are in irq context. How to call mbox functions which does sleep + */ + +chk_mnq_err_dbg: + mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG); + if (!(mnq_err_dbg & BIT(44))) + goto chk_snd_err_dbg; + + mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_MNQ_ERR_DBG(%llx) err=%s\n", + qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code]); + otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44)); + +chk_snd_err_dbg: + snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG); + if (snd_err_dbg & BIT(44)) { + snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg); + netdev_err(pf->netdev, "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s\n", + qidx, snd_err_dbg, nix_snd_status_e_str[snd_err_code]); + otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44)); + } + +done: + /* Print values and reset */ + if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) + netdev_err(pf->netdev, "SQ%lld: SQB allocation failed", + qidx); + schedule_work(&pf->reset_task); } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h index aa205a0d158f6..fa37b9f312cae 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h @@ -281,4 +281,61 @@ enum nix_sqint_e { BIT_ULL(NIX_SQINT_SEND_ERR) | \ BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL)) +enum nix_sqoperr_e { + NIX_SQOPERR_OOR = 0, + NIX_SQOPERR_CTX_FAULT = 1, + NIX_SQOPERR_CTX_POISON = 2, + NIX_SQOPERR_DISABLED = 3, + NIX_SQOPERR_SIZE_ERR = 4, + NIX_SQOPERR_OFLOW = 5, + NIX_SQOPERR_SQB_NULL = 6, + NIX_SQOPERR_SQB_FAULT = 7, + NIX_SQOPERR_SQE_SZ_ZERO = 8, + NIX_SQOPERR_MAX, +}; + +enum nix_mnqerr_e { + NIX_MNQERR_SQ_CTX_FAULT = 0, + NIX_MNQERR_SQ_CTX_POISON = 1, + NIX_MNQERR_SQB_FAULT = 2, + NIX_MNQERR_SQB_POISON = 3, + NIX_MNQERR_TOTAL_ERR = 4, + NIX_MNQERR_LSO_ERR = 5, + NIX_MNQERR_CQ_QUERY_ERR = 6, + NIX_MNQERR_MAX_SQE_SIZE_ERR = 7, + NIX_MNQERR_MAXLEN_ERR = 8, + NIX_MNQERR_SQE_SIZEM1_ZERO = 9, + NIX_MNQERR_MAX, +}; + +enum nix_snd_status_e { + NIX_SND_STATUS_GOOD = 0x0, + NIX_SND_STATUS_SQ_CTX_FAULT = 0x1, + NIX_SND_STATUS_SQ_CTX_POISON = 0x2, + NIX_SND_STATUS_SQB_FAULT = 0x3, + NIX_SND_STATUS_SQB_POISON = 0x4, + NIX_SND_STATUS_HDR_ERR = 0x5, + NIX_SND_STATUS_EXT_ERR = 0x6, + NIX_SND_STATUS_JUMP_FAULT = 0x7, + NIX_SND_STATUS_JUMP_POISON = 0x8, + NIX_SND_STATUS_CRC_ERR = 0x9, + NIX_SND_STATUS_IMM_ERR = 0x10, + NIX_SND_STATUS_SG_ERR = 0x11, + NIX_SND_STATUS_MEM_ERR = 0x12, + NIX_SND_STATUS_INVALID_SUBDC = 0x13, + NIX_SND_STATUS_SUBDC_ORDER_ERR = 0x14, + NIX_SND_STATUS_DATA_FAULT = 0x15, + NIX_SND_STATUS_DATA_POISON = 0x16, + NIX_SND_STATUS_NPC_DROP_ACTION = 0x17, + NIX_SND_STATUS_LOCK_VIOL = 0x18, + NIX_SND_STATUS_NPC_UCAST_CHAN_ERR = 0x19, + NIX_SND_STATUS_NPC_MCAST_CHAN_ERR = 0x20, + NIX_SND_STATUS_NPC_MCAST_ABORT = 0x21, + NIX_SND_STATUS_NPC_VTAG_PTR_ERR = 0x22, + NIX_SND_STATUS_NPC_VTAG_SIZE_ERR = 0x23, + NIX_SND_STATUS_SEND_MEM_FAULT = 0x24, + NIX_SND_STATUS_SEND_STATS_ERR = 0x25, + NIX_SND_STATUS_MAX, +}; + #endif /* OTX2_STRUCT_H */ -- GitLab From 1118b2049d77ca0b505775fc1a8d1909cf19a7ec Mon Sep 17 00:00:00 2001 From: Wang Yufen Date: Wed, 2 Nov 2022 17:41:19 +0800 Subject: [PATCH 1037/1927] net: tun: Fix memory leaks of napi_get_frags kmemleak reports after running test_progs: unreferenced object 0xffff8881b1672dc0 (size 232): comm "test_progs", pid 394388, jiffies 4354712116 (age 841.975s) hex dump (first 32 bytes): e0 84 d7 a8 81 88 ff ff 80 2c 67 b1 81 88 ff ff .........,g..... 00 40 c5 9b 81 88 ff ff 00 00 00 00 00 00 00 00 .@.............. backtrace: [<00000000c8f01748>] napi_skb_cache_get+0xd4/0x150 [<0000000041c7fc09>] __napi_build_skb+0x15/0x50 [<00000000431c7079>] __napi_alloc_skb+0x26e/0x540 [<000000003ecfa30e>] napi_get_frags+0x59/0x140 [<0000000099b2199e>] tun_get_user+0x183d/0x3bb0 [tun] [<000000008a5adef0>] tun_chr_write_iter+0xc0/0x1b1 [tun] [<0000000049993ff4>] do_iter_readv_writev+0x19f/0x320 [<000000008f338ea2>] do_iter_write+0x135/0x630 [<000000008a3377a4>] vfs_writev+0x12e/0x440 [<00000000a6b5639a>] do_writev+0x104/0x280 [<00000000ccf065d8>] do_syscall_64+0x3b/0x90 [<00000000d776e329>] entry_SYSCALL_64_after_hwframe+0x63/0xcd The issue occurs in the following scenarios: tun_get_user() napi_gro_frags() napi_frags_finish() case GRO_NORMAL: gro_normal_one() list_add_tail(&skb->list, &napi->rx_list); <-- While napi->rx_count < READ_ONCE(gro_normal_batch), <-- gro_normal_list() is not called, napi->rx_list is not empty <-- not ask to complete the gro work, will cause memory leaks in <-- following tun_napi_del() ... tun_napi_del() netif_napi_del() __netif_napi_del() <-- &napi->rx_list is not empty, which caused memory leaks To fix, add napi_complete() after napi_gro_frags(). Fixes: 90e33d459407 ("tun: enable napi_gro_frags() for TUN/TAP driver") Signed-off-by: Wang Yufen Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- drivers/net/tun.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 946628050f282..eb12f3136a549 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1977,6 +1977,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, local_bh_disable(); napi_gro_frags(&tfile->napi); + napi_complete(&tfile->napi); local_bh_enable(); mutex_unlock(&tfile->napi_mutex); } else if (tfile->napi_enabled) { -- GitLab From 392cc13c5ec72ccd6bbfb1bc2339502cc59dd285 Mon Sep 17 00:00:00 2001 From: Jason Montleon Date: Thu, 3 Nov 2022 10:46:11 -0400 Subject: [PATCH 1038/1927] ASoC: rt5514: fix legacy dai naming Starting with 6.0-rc1 these messages are logged and the sound card is unavailable. Adding legacy_dai_naming to the rt5514-spi causes it to function properly again. [ 16.928454] kbl_r5514_5663_max kbl_r5514_5663_max: ASoC: CPU DAI spi-PRP0001:00 not registered [ 16.928561] platform kbl_r5514_5663_max: deferred probe pending Fixes: fc34ece41f71 ("ASoC: Refactor non_legacy_dai_naming flag") BugLink: https://bugzilla.kernel.org/show_bug.cgi?id=216641 Signed-off-by: Jason Montleon Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20221103144612.4431-1-jmontleo@redhat.com Signed-off-by: Mark Brown --- sound/soc/codecs/rt5514-spi.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sound/soc/codecs/rt5514-spi.c b/sound/soc/codecs/rt5514-spi.c index 1a25a37879356..362663abcb89e 100644 --- a/sound/soc/codecs/rt5514-spi.c +++ b/sound/soc/codecs/rt5514-spi.c @@ -298,13 +298,14 @@ static int rt5514_spi_pcm_new(struct snd_soc_component *component, } static const struct snd_soc_component_driver rt5514_spi_component = { - .name = DRV_NAME, - .probe = rt5514_spi_pcm_probe, - .open = rt5514_spi_pcm_open, - .hw_params = rt5514_spi_hw_params, - .hw_free = rt5514_spi_hw_free, - .pointer = rt5514_spi_pcm_pointer, - .pcm_construct = rt5514_spi_pcm_new, + .name = DRV_NAME, + .probe = rt5514_spi_pcm_probe, + .open = rt5514_spi_pcm_open, + .hw_params = rt5514_spi_hw_params, + .hw_free = rt5514_spi_hw_free, + .pointer = rt5514_spi_pcm_pointer, + .pcm_construct = rt5514_spi_pcm_new, + .legacy_dai_naming = 1, }; /** -- GitLab From a1dca8774faf3f77eb34fa0ac6f3e2b82290b1e4 Mon Sep 17 00:00:00 2001 From: Jason Montleon Date: Thu, 3 Nov 2022 10:46:12 -0400 Subject: [PATCH 1039/1927] ASoC: rt5677: fix legacy dai naming Starting with 6.0-rc1 the CPU DAI is not registered and the sound card is unavailable. Adding legacy_dai_naming causes it to function properly again. Fixes: fc34ece41f71 ("ASoC: Refactor non_legacy_dai_naming flag") Signed-off-by: Jason Montleon Acked-by: Charles Keepax Link: https://lore.kernel.org/r/20221103144612.4431-2-jmontleo@redhat.com Signed-off-by: Mark Brown --- sound/soc/codecs/rt5677-spi.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sound/soc/codecs/rt5677-spi.c b/sound/soc/codecs/rt5677-spi.c index 8f3993a4c1cc7..d25703dd74996 100644 --- a/sound/soc/codecs/rt5677-spi.c +++ b/sound/soc/codecs/rt5677-spi.c @@ -396,15 +396,16 @@ static int rt5677_spi_pcm_probe(struct snd_soc_component *component) } static const struct snd_soc_component_driver rt5677_spi_dai_component = { - .name = DRV_NAME, - .probe = rt5677_spi_pcm_probe, - .open = rt5677_spi_pcm_open, - .close = rt5677_spi_pcm_close, - .hw_params = rt5677_spi_hw_params, - .hw_free = rt5677_spi_hw_free, - .prepare = rt5677_spi_prepare, - .pointer = rt5677_spi_pcm_pointer, - .pcm_construct = rt5677_spi_pcm_new, + .name = DRV_NAME, + .probe = rt5677_spi_pcm_probe, + .open = rt5677_spi_pcm_open, + .close = rt5677_spi_pcm_close, + .hw_params = rt5677_spi_hw_params, + .hw_free = rt5677_spi_hw_free, + .prepare = rt5677_spi_prepare, + .pointer = rt5677_spi_pcm_pointer, + .pcm_construct = rt5677_spi_pcm_new, + .legacy_dai_naming = 1, }; /* Select a suitable transfer command for the next transfer to ensure -- GitLab From bff6bef701db784bb159a659e99c785b4594fc96 Mon Sep 17 00:00:00 2001 From: Vitaly Rodionov Date: Fri, 4 Nov 2022 10:06:37 +0000 Subject: [PATCH 1040/1927] spi: amd: Fix SPI_SPD7 value According to data sheet SPI_SPD7 should be set to 7. Signed-off-by: Vitaly Rodionov Link: https://lore.kernel.org/r/20221104100637.13376-1-vitalyr@opensource.cirrus.com Signed-off-by: Mark Brown --- drivers/spi/spi-amd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c index e23121456c706..bfc3ab5f39eaa 100644 --- a/drivers/spi/spi-amd.c +++ b/drivers/spi/spi-amd.c @@ -65,7 +65,7 @@ enum amd_spi_speed { F_16_66MHz, F_100MHz, F_800KHz, - SPI_SPD7, + SPI_SPD7 = 0x7, F_50MHz = 0x4, F_4MHz = 0x32, F_3_17MHz = 0x3F -- GitLab From 6f7de35b50860c345babf8ed0aa0d75f9315eee4 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 28 Oct 2022 14:06:35 +0100 Subject: [PATCH 1041/1927] drm/i915/userptr: restore probe_range behaviour The conversion looks harmless, however the addr value is updated inside the loop with the previous vm_end, which then incorrectly leads to for_each_vma_range() iterating over stuff outside the range we care about. Fix this by storing the end value separately. Also fix the case where the range doesn't intersect with any vma, or if the vma itself doesn't extend the entire range, which must mean we have hole at the end. Both should result in an error, as per the previous behaviour. v2: Fix the cases where the range is empty, or if there's a hole at the end of the range Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7247 Testcase: igt@gem_userptr_blits@probe Fixes: f683b9d61319 ("i915: use the VMA iterator") Reported-by: kernel test robot Signed-off-by: Matthew Auld Cc: Tvrtko Ursulin Cc: Matthew Wilcox (Oracle) Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Yu Zhao Reviewed-by: Liam R. Howlett Reviewed-by: Andrzej Hajda Link: https://patchwork.freedesktop.org/patch/msgid/20221028130635.465839-1-matthew.auld@intel.com --- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index 1b1a22716722e..ca7a388ba2bf5 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -427,9 +427,10 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) { VMA_ITERATOR(vmi, mm, addr); struct vm_area_struct *vma; + unsigned long end = addr + len; mmap_read_lock(mm); - for_each_vma_range(vmi, vma, addr + len) { + for_each_vma_range(vmi, vma, end) { /* Check for holes, note that we also update the addr below */ if (vma->vm_start > addr) break; @@ -441,7 +442,7 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) } mmap_read_unlock(mm); - if (vma) + if (vma || addr < end) return -EFAULT; return 0; } -- GitLab From eb4940d4adf590590a9d0c47e38d2799c2ff9670 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Fri, 4 Nov 2022 13:57:11 +0100 Subject: [PATCH 1042/1927] mm/slab: remove !CONFIG_TRACING variants of kmalloc_[node_]trace() For !CONFIG_TRACING kernels, the kmalloc() implementation tries (in cases where the allocation size is build-time constant) to save a function call, by inlining kmalloc_trace() to a kmem_cache_alloc() call. However since commit 6edf2576a6cc ("mm/slub: enable debugging memory wasting of kmalloc") this path now fails to pass the original request size to be eventually recorded (for kmalloc caches with debugging enabled). We could adjust the code to call __kmem_cache_alloc_node() as the CONFIG_TRACING variant, but that would as a result inline a call with 5 parameters, bloating the kmalloc() call sites. The cost of extra function call (to kmalloc_trace()) seems like a lesser evil. It also appears that the !CONFIG_TRACING variant is incompatible with upcoming hardening efforts [1] so it's easier if we just remove it now. Kernels with no tracing are rare these days and the benefit is dubious anyway. [1] https://lore.kernel.org/linux-mm/20221101222520.never.109-kees@kernel.org/T/#m20ecf14390e406247bde0ea9cce368f469c539ed Link: https://lore.kernel.org/all/097d8fba-bd10-a312-24a3-a4068c4f424c@suse.cz/ Suggested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Signed-off-by: Vlastimil Babka --- include/linux/slab.h | 23 ----------------------- mm/slab_common.c | 2 -- 2 files changed, 25 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index 90877fcde70bd..45efc6c553b82 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -470,35 +470,12 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignm void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment __malloc; -#ifdef CONFIG_TRACING void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) __assume_kmalloc_alignment __alloc_size(3); void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) __assume_kmalloc_alignment __alloc_size(4); -#else /* CONFIG_TRACING */ -/* Save a function call when CONFIG_TRACING=n */ -static __always_inline __alloc_size(3) -void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) -{ - void *ret = kmem_cache_alloc(s, flags); - - ret = kasan_kmalloc(s, ret, size, flags); - return ret; -} - -static __always_inline __alloc_size(4) -void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, - int node, size_t size) -{ - void *ret = kmem_cache_alloc_node(s, gfpflags, node); - - ret = kasan_kmalloc(s, ret, size, gfpflags); - return ret; -} -#endif /* CONFIG_TRACING */ - void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment __alloc_size(1); diff --git a/mm/slab_common.c b/mm/slab_common.c index 74a991fd9d31d..206e59051c1d6 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1040,7 +1040,6 @@ size_t __ksize(const void *object) return slab_ksize(folio_slab(folio)->slab_cache); } -#ifdef CONFIG_TRACING void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { void *ret = __kmem_cache_alloc_node(s, gfpflags, NUMA_NO_NODE, @@ -1064,7 +1063,6 @@ void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, return ret; } EXPORT_SYMBOL(kmalloc_node_trace); -#endif /* !CONFIG_TRACING */ #endif /* !CONFIG_SLOB */ gfp_t kmalloc_fix_flags(gfp_t flags) -- GitLab From d3fd203f36d46aa29600a72d57a1b61af80e4a25 Mon Sep 17 00:00:00 2001 From: Baisong Zhong Date: Wed, 2 Nov 2022 16:16:20 +0800 Subject: [PATCH 1043/1927] bpf, test_run: Fix alignment problem in bpf_prog_test_run_skb() We got a syzkaller problem because of aarch64 alignment fault if KFENCE enabled. When the size from user bpf program is an odd number, like 399, 407, etc, it will cause the struct skb_shared_info's unaligned access. As seen below: BUG: KFENCE: use-after-free read in __skb_clone+0x23c/0x2a0 net/core/skbuff.c:1032 Use-after-free read at 0xffff6254fffac077 (in kfence-#213): __lse_atomic_add arch/arm64/include/asm/atomic_lse.h:26 [inline] arch_atomic_add arch/arm64/include/asm/atomic.h:28 [inline] arch_atomic_inc include/linux/atomic-arch-fallback.h:270 [inline] atomic_inc include/asm-generic/atomic-instrumented.h:241 [inline] __skb_clone+0x23c/0x2a0 net/core/skbuff.c:1032 skb_clone+0xf4/0x214 net/core/skbuff.c:1481 ____bpf_clone_redirect net/core/filter.c:2433 [inline] bpf_clone_redirect+0x78/0x1c0 net/core/filter.c:2420 bpf_prog_d3839dd9068ceb51+0x80/0x330 bpf_dispatcher_nop_func include/linux/bpf.h:728 [inline] bpf_test_run+0x3c0/0x6c0 net/bpf/test_run.c:53 bpf_prog_test_run_skb+0x638/0xa7c net/bpf/test_run.c:594 bpf_prog_test_run kernel/bpf/syscall.c:3148 [inline] __do_sys_bpf kernel/bpf/syscall.c:4441 [inline] __se_sys_bpf+0xad0/0x1634 kernel/bpf/syscall.c:4381 kfence-#213: 0xffff6254fffac000-0xffff6254fffac196, size=407, cache=kmalloc-512 allocated by task 15074 on cpu 0 at 1342.585390s: kmalloc include/linux/slab.h:568 [inline] kzalloc include/linux/slab.h:675 [inline] bpf_test_init.isra.0+0xac/0x290 net/bpf/test_run.c:191 bpf_prog_test_run_skb+0x11c/0xa7c net/bpf/test_run.c:512 bpf_prog_test_run kernel/bpf/syscall.c:3148 [inline] __do_sys_bpf kernel/bpf/syscall.c:4441 [inline] __se_sys_bpf+0xad0/0x1634 kernel/bpf/syscall.c:4381 __arm64_sys_bpf+0x50/0x60 kernel/bpf/syscall.c:4381 To fix the problem, we adjust @size so that (@size + @hearoom) is a multiple of SMP_CACHE_BYTES. So we make sure the struct skb_shared_info is aligned to a cache line. Fixes: 1cf1cae963c2 ("bpf: introduce BPF_PROG_TEST_RUN command") Signed-off-by: Baisong Zhong Signed-off-by: Daniel Borkmann Cc: Eric Dumazet Link: https://lore.kernel.org/bpf/20221102081620.1465154-1-zhongbaisong@huawei.com --- net/bpf/test_run.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c index 13d578ce2a096..fcb3e6c5e03c0 100644 --- a/net/bpf/test_run.c +++ b/net/bpf/test_run.c @@ -774,6 +774,7 @@ static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size, if (user_size > size) return ERR_PTR(-EMSGSIZE); + size = SKB_DATA_ALIGN(size); data = kzalloc(size + headroom + tailroom, GFP_USER); if (!data) return ERR_PTR(-ENOMEM); -- GitLab From 6e59419fd0a244dd55e53e798797f0697dec8b1c Mon Sep 17 00:00:00 2001 From: Peng Wu Date: Sun, 11 Sep 2022 06:00:53 +0000 Subject: [PATCH 1044/1927] phy: sunplus: Fix an IS_ERR() vs NULL bug in sp_usb_phy_probe The devm_ioremap() function returns NULL on error, it doesn't return error pointers. Fixes: 99d9ccd973852 ("phy: usb: Add USB2.0 phy driver for Sunplus SP7021") Signed-off-by: Peng Wu Link: https://lore.kernel.org/r/20220911060053.123594-1-wupeng58@huawei.com Signed-off-by: Vinod Koul --- drivers/phy/sunplus/phy-sunplus-usb2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/phy/sunplus/phy-sunplus-usb2.c b/drivers/phy/sunplus/phy-sunplus-usb2.c index b932087c55b29..e827b79f6d493 100644 --- a/drivers/phy/sunplus/phy-sunplus-usb2.c +++ b/drivers/phy/sunplus/phy-sunplus-usb2.c @@ -256,8 +256,8 @@ static int sp_usb_phy_probe(struct platform_device *pdev) usbphy->moon4_res_mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "moon4"); usbphy->moon4_regs = devm_ioremap(&pdev->dev, usbphy->moon4_res_mem->start, resource_size(usbphy->moon4_res_mem)); - if (IS_ERR(usbphy->moon4_regs)) - return PTR_ERR(usbphy->moon4_regs); + if (!usbphy->moon4_regs) + return -ENOMEM; usbphy->phy_clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(usbphy->phy_clk)) -- GitLab From cc2e0cf0ad47db7f51283e5c9bd2212a994ad527 Mon Sep 17 00:00:00 2001 From: John Harrison Date: Mon, 31 Oct 2022 15:00:07 -0700 Subject: [PATCH 1045/1927] drm/i915/guc: Remove excessive line feeds in state dumps Some of the GuC state dump messages were adding extra line feeds. When printing via a DRM printer to dmesg, for example, that messes up the log formatting as it loses any prefixing from the printer. Given that the extra line feeds are just in the middle of random bits of GuC state, there isn't any real need for them. So just remove them completely. Signed-off-by: John Harrison Reviewed-by: Umesh Nerlige Ramappa Link: https://patchwork.freedesktop.org/patch/msgid/20221031220007.4176835-1-John.C.Harrison@Intel.com --- drivers/gpu/drm/i915/gt/uc/intel_guc.c | 4 ++-- drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c index 27b09ba1d295f..1bcd61bb50f89 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c @@ -871,14 +871,14 @@ void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p) u32 status = intel_uncore_read(uncore, GUC_STATUS); u32 i; - drm_printf(p, "\nGuC status 0x%08x:\n", status); + drm_printf(p, "GuC status 0x%08x:\n", status); drm_printf(p, "\tBootrom status = 0x%x\n", (status & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); drm_printf(p, "\tuKernel status = 0x%x\n", (status & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); drm_printf(p, "\tMIA Core status = 0x%x\n", (status & GS_MIA_MASK) >> GS_MIA_SHIFT); - drm_puts(p, "\nScratch registers:\n"); + drm_puts(p, "Scratch registers:\n"); for (i = 0; i < 16; i++) { drm_printf(p, "\t%2d: \t0x%x\n", i, intel_uncore_read(uncore, SOFT_SCRATCH(i))); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 4ccb29f9ac55c..4dbdac8002e32 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -4901,7 +4901,7 @@ void intel_guc_submission_print_info(struct intel_guc *guc, drm_printf(p, "GuC Number Outstanding Submission G2H: %u\n", atomic_read(&guc->outstanding_submission_g2h)); - drm_printf(p, "GuC tasklet count: %u\n\n", + drm_printf(p, "GuC tasklet count: %u\n", atomic_read(&sched_engine->tasklet.count)); spin_lock_irqsave(&sched_engine->lock, flags); @@ -4949,7 +4949,7 @@ static inline void guc_log_context(struct drm_printer *p, atomic_read(&ce->pin_count)); drm_printf(p, "\t\tGuC ID Ref Count: %u\n", atomic_read(&ce->guc_id.ref)); - drm_printf(p, "\t\tSchedule State: 0x%x\n\n", + drm_printf(p, "\t\tSchedule State: 0x%x\n", ce->guc_state.sched_state); } @@ -4978,7 +4978,7 @@ void intel_guc_submission_print_context_info(struct intel_guc *guc, READ_ONCE(*ce->parallel.guc.wq_head)); drm_printf(p, "\t\tWQI Tail: %u\n", READ_ONCE(*ce->parallel.guc.wq_tail)); - drm_printf(p, "\t\tWQI Status: %u\n\n", + drm_printf(p, "\t\tWQI Status: %u\n", READ_ONCE(*ce->parallel.guc.wq_status)); } @@ -4986,7 +4986,7 @@ void intel_guc_submission_print_context_info(struct intel_guc *guc, emit_bb_start_parent_no_preempt_mid_batch) { u8 i; - drm_printf(p, "\t\tChildren Go: %u\n\n", + drm_printf(p, "\t\tChildren Go: %u\n", get_children_go_value(ce)); for (i = 0; i < ce->parallel.number_children; ++i) drm_printf(p, "\t\tChildren Join: %u\n", -- GitLab From 403cea4019e206130b5a90f6d34383005d17b54a Mon Sep 17 00:00:00 2001 From: Jay Cornwall Date: Thu, 13 Oct 2022 21:41:13 -0500 Subject: [PATCH 1046/1927] drm/amdkfd: update GFX11 CWSR trap handler With corresponding FW change fixes issue where triggering CWSR on a workgroup with waves in s_barrier wouldn't lead to a back-off and therefore cause a hang. Signed-off-by: Jay Cornwall Tested-by: Graham Sider Acked-by: Harish Kasiviswanathan Acked-by: Felix Kuehling Reviewed-by: Graham Sider Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdkfd/cwsr_trap_handler.h | 764 +++++++++--------- .../amd/amdkfd/cwsr_trap_handler_gfx10.asm | 6 + 2 files changed, 389 insertions(+), 381 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h index c7118843db051..0c4c5499bb5cc 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h @@ -2495,442 +2495,444 @@ static const uint32_t cwsr_trap_gfx10_hex[] = { 0xbf9f0000, 0x00000000, }; static const uint32_t cwsr_trap_gfx11_hex[] = { - 0xbfa00001, 0xbfa0021e, + 0xbfa00001, 0xbfa00221, 0xb0804006, 0xb8f8f802, 0x9178ff78, 0x00020006, - 0xb8fbf803, 0xbf0d9f6d, - 0xbfa20006, 0x8b6eff78, - 0x00002000, 0xbfa10009, - 0x8b6eff6d, 0x00ff0000, - 0xbfa2001e, 0x8b6eff7b, - 0x00000400, 0xbfa20041, - 0xbf830010, 0xb8fbf803, - 0xbfa0fffa, 0x8b6eff7b, - 0x00000900, 0xbfa20015, - 0x8b6eff7b, 0x000071ff, - 0xbfa10008, 0x8b6fff7b, - 0x00007080, 0xbfa10001, - 0xbeee1287, 0xb8eff801, - 0x846e8c6e, 0x8b6e6f6e, - 0xbfa2000a, 0x8b6eff6d, - 0x00ff0000, 0xbfa20007, - 0xb8eef801, 0x8b6eff6e, - 0x00000800, 0xbfa20003, + 0xb8fbf803, 0xbf0d9e6d, + 0xbfa10001, 0xbfbd0000, + 0xbf0d9f6d, 0xbfa20006, + 0x8b6eff78, 0x00002000, + 0xbfa10009, 0x8b6eff6d, + 0x00ff0000, 0xbfa2001e, 0x8b6eff7b, 0x00000400, - 0xbfa20026, 0xbefa4d82, - 0xbf89fc07, 0x84fa887a, - 0xf4005bbd, 0xf8000010, - 0xbf89fc07, 0x846e976e, - 0x9177ff77, 0x00800000, - 0x8c776e77, 0xf4045bbd, - 0xf8000000, 0xbf89fc07, - 0xf4045ebd, 0xf8000008, - 0xbf89fc07, 0x8bee6e6e, - 0xbfa10001, 0xbe80486e, - 0x8b6eff6d, 0x01ff0000, - 0xbfa20005, 0x8c78ff78, - 0x00002000, 0x80ec886c, - 0x82ed806d, 0xbfa00005, - 0x8b6eff6d, 0x01000000, - 0xbfa20002, 0x806c846c, - 0x826d806d, 0x8b6dff6d, - 0x0000ffff, 0x8bfe7e7e, - 0x8bea6a6a, 0xb978f802, - 0xbe804a6c, 0x8b6dff6d, - 0x0000ffff, 0xbefa0080, - 0xb97a0283, 0xbeee007e, - 0xbeef007f, 0xbefe0180, - 0xbefe4d84, 0xbf89fc07, - 0x8b7aff7f, 0x04000000, - 0x847a857a, 0x8c6d7a6d, - 0xbefa007e, 0x8b7bff7f, - 0x0000ffff, 0xbefe00c1, - 0xbeff00c1, 0xdca6c000, - 0x007a0000, 0x7e000280, - 0xbefe007a, 0xbeff007b, - 0xb8fb02dc, 0x847b997b, - 0xb8fa3b05, 0x807a817a, - 0xbf0d997b, 0xbfa20002, - 0x847a897a, 0xbfa00001, - 0x847a8a7a, 0xb8fb1e06, - 0x847b8a7b, 0x807a7b7a, + 0xbfa20041, 0xbf830010, + 0xb8fbf803, 0xbfa0fffa, + 0x8b6eff7b, 0x00000900, + 0xbfa20015, 0x8b6eff7b, + 0x000071ff, 0xbfa10008, + 0x8b6fff7b, 0x00007080, + 0xbfa10001, 0xbeee1287, + 0xb8eff801, 0x846e8c6e, + 0x8b6e6f6e, 0xbfa2000a, + 0x8b6eff6d, 0x00ff0000, + 0xbfa20007, 0xb8eef801, + 0x8b6eff6e, 0x00000800, + 0xbfa20003, 0x8b6eff7b, + 0x00000400, 0xbfa20026, + 0xbefa4d82, 0xbf89fc07, + 0x84fa887a, 0xf4005bbd, + 0xf8000010, 0xbf89fc07, + 0x846e976e, 0x9177ff77, + 0x00800000, 0x8c776e77, + 0xf4045bbd, 0xf8000000, + 0xbf89fc07, 0xf4045ebd, + 0xf8000008, 0xbf89fc07, + 0x8bee6e6e, 0xbfa10001, + 0xbe80486e, 0x8b6eff6d, + 0x01ff0000, 0xbfa20005, + 0x8c78ff78, 0x00002000, + 0x80ec886c, 0x82ed806d, + 0xbfa00005, 0x8b6eff6d, + 0x01000000, 0xbfa20002, + 0x806c846c, 0x826d806d, + 0x8b6dff6d, 0x0000ffff, + 0x8bfe7e7e, 0x8bea6a6a, + 0xb978f802, 0xbe804a6c, + 0x8b6dff6d, 0x0000ffff, + 0xbefa0080, 0xb97a0283, + 0xbeee007e, 0xbeef007f, + 0xbefe0180, 0xbefe4d84, + 0xbf89fc07, 0x8b7aff7f, + 0x04000000, 0x847a857a, + 0x8c6d7a6d, 0xbefa007e, 0x8b7bff7f, 0x0000ffff, - 0x807aff7a, 0x00000200, - 0x807a7e7a, 0x827b807b, - 0xd7610000, 0x00010870, - 0xd7610000, 0x00010a71, - 0xd7610000, 0x00010c72, - 0xd7610000, 0x00010e73, - 0xd7610000, 0x00011074, - 0xd7610000, 0x00011275, - 0xd7610000, 0x00011476, - 0xd7610000, 0x00011677, - 0xd7610000, 0x00011a79, - 0xd7610000, 0x00011c7e, - 0xd7610000, 0x00011e7f, - 0xbefe00ff, 0x00003fff, - 0xbeff0080, 0xdca6c040, - 0x007a0000, 0xd760007a, - 0x00011d00, 0xd760007b, - 0x00011f00, 0xbefe007a, - 0xbeff007b, 0xbef4007e, - 0x8b75ff7f, 0x0000ffff, - 0x8c75ff75, 0x00040000, - 0xbef60080, 0xbef700ff, - 0x10807fac, 0xbef1007d, - 0xbef00080, 0xb8f302dc, - 0x84739973, 0xbefe00c1, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00002, - 0xbeff00c1, 0xbfa00009, + 0xbefe00c1, 0xbeff00c1, + 0xdca6c000, 0x007a0000, + 0x7e000280, 0xbefe007a, + 0xbeff007b, 0xb8fb02dc, + 0x847b997b, 0xb8fa3b05, + 0x807a817a, 0xbf0d997b, + 0xbfa20002, 0x847a897a, + 0xbfa00001, 0x847a8a7a, + 0xb8fb1e06, 0x847b8a7b, + 0x807a7b7a, 0x8b7bff7f, + 0x0000ffff, 0x807aff7a, + 0x00000200, 0x807a7e7a, + 0x827b807b, 0xd7610000, + 0x00010870, 0xd7610000, + 0x00010a71, 0xd7610000, + 0x00010c72, 0xd7610000, + 0x00010e73, 0xd7610000, + 0x00011074, 0xd7610000, + 0x00011275, 0xd7610000, + 0x00011476, 0xd7610000, + 0x00011677, 0xd7610000, + 0x00011a79, 0xd7610000, + 0x00011c7e, 0xd7610000, + 0x00011e7f, 0xbefe00ff, + 0x00003fff, 0xbeff0080, + 0xdca6c040, 0x007a0000, + 0xd760007a, 0x00011d00, + 0xd760007b, 0x00011f00, + 0xbefe007a, 0xbeff007b, + 0xbef4007e, 0x8b75ff7f, + 0x0000ffff, 0x8c75ff75, + 0x00040000, 0xbef60080, + 0xbef700ff, 0x10807fac, + 0xbef1007d, 0xbef00080, + 0xb8f302dc, 0x84739973, + 0xbefe00c1, 0x857d9973, + 0x8b7d817d, 0xbf06817d, + 0xbfa20002, 0xbeff0080, + 0xbfa00002, 0xbeff00c1, + 0xbfa00009, 0xbef600ff, + 0x01000000, 0xe0685080, + 0x701d0100, 0xe0685100, + 0x701d0200, 0xe0685180, + 0x701d0300, 0xbfa00008, 0xbef600ff, 0x01000000, - 0xe0685080, 0x701d0100, - 0xe0685100, 0x701d0200, - 0xe0685180, 0x701d0300, - 0xbfa00008, 0xbef600ff, - 0x01000000, 0xe0685100, - 0x701d0100, 0xe0685200, - 0x701d0200, 0xe0685300, - 0x701d0300, 0xb8f03b05, - 0x80708170, 0xbf0d9973, - 0xbfa20002, 0x84708970, - 0xbfa00001, 0x84708a70, - 0xb8fa1e06, 0x847a8a7a, - 0x80707a70, 0x8070ff70, - 0x00000200, 0xbef600ff, - 0x01000000, 0x7e000280, - 0x7e020280, 0x7e040280, - 0xbefd0080, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xd7610002, 0x0000fa6c, - 0x807d817d, 0x917aff6d, - 0x80000000, 0xd7610002, - 0x0000fa7a, 0x807d817d, - 0xd7610002, 0x0000fa6e, - 0x807d817d, 0xd7610002, - 0x0000fa6f, 0x807d817d, - 0xd7610002, 0x0000fa78, - 0x807d817d, 0xb8faf803, - 0xd7610002, 0x0000fa7a, - 0x807d817d, 0xd7610002, - 0x0000fa7b, 0x807d817d, - 0xb8f1f801, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xb8f1f814, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xb8f1f815, 0xd7610002, - 0x0000fa71, 0x807d817d, - 0xbefe00ff, 0x0000ffff, - 0xbeff0080, 0xe0685000, - 0x701d0200, 0xbefe00c1, + 0xe0685100, 0x701d0100, + 0xe0685200, 0x701d0200, + 0xe0685300, 0x701d0300, 0xb8f03b05, 0x80708170, 0xbf0d9973, 0xbfa20002, 0x84708970, 0xbfa00001, 0x84708a70, 0xb8fa1e06, 0x847a8a7a, 0x80707a70, + 0x8070ff70, 0x00000200, 0xbef600ff, 0x01000000, - 0xbef90080, 0xbefd0080, - 0xbf800000, 0xbe804100, - 0xbe824102, 0xbe844104, - 0xbe864106, 0xbe884108, - 0xbe8a410a, 0xbe8c410c, - 0xbe8e410e, 0xd7610002, - 0x0000f200, 0x80798179, - 0xd7610002, 0x0000f201, + 0x7e000280, 0x7e020280, + 0x7e040280, 0xbefd0080, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xd7610002, + 0x0000fa6c, 0x807d817d, + 0x917aff6d, 0x80000000, + 0xd7610002, 0x0000fa7a, + 0x807d817d, 0xd7610002, + 0x0000fa6e, 0x807d817d, + 0xd7610002, 0x0000fa6f, + 0x807d817d, 0xd7610002, + 0x0000fa78, 0x807d817d, + 0xb8faf803, 0xd7610002, + 0x0000fa7a, 0x807d817d, + 0xd7610002, 0x0000fa7b, + 0x807d817d, 0xb8f1f801, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f814, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xb8f1f815, + 0xd7610002, 0x0000fa71, + 0x807d817d, 0xbefe00ff, + 0x0000ffff, 0xbeff0080, + 0xe0685000, 0x701d0200, + 0xbefe00c1, 0xb8f03b05, + 0x80708170, 0xbf0d9973, + 0xbfa20002, 0x84708970, + 0xbfa00001, 0x84708a70, + 0xb8fa1e06, 0x847a8a7a, + 0x80707a70, 0xbef600ff, + 0x01000000, 0xbef90080, + 0xbefd0080, 0xbf800000, + 0xbe804100, 0xbe824102, + 0xbe844104, 0xbe864106, + 0xbe884108, 0xbe8a410a, + 0xbe8c410c, 0xbe8e410e, + 0xd7610002, 0x0000f200, 0x80798179, 0xd7610002, - 0x0000f202, 0x80798179, - 0xd7610002, 0x0000f203, + 0x0000f201, 0x80798179, + 0xd7610002, 0x0000f202, 0x80798179, 0xd7610002, - 0x0000f204, 0x80798179, - 0xd7610002, 0x0000f205, + 0x0000f203, 0x80798179, + 0xd7610002, 0x0000f204, 0x80798179, 0xd7610002, - 0x0000f206, 0x80798179, - 0xd7610002, 0x0000f207, + 0x0000f205, 0x80798179, + 0xd7610002, 0x0000f206, 0x80798179, 0xd7610002, - 0x0000f208, 0x80798179, - 0xd7610002, 0x0000f209, + 0x0000f207, 0x80798179, + 0xd7610002, 0x0000f208, 0x80798179, 0xd7610002, - 0x0000f20a, 0x80798179, - 0xd7610002, 0x0000f20b, + 0x0000f209, 0x80798179, + 0xd7610002, 0x0000f20a, 0x80798179, 0xd7610002, - 0x0000f20c, 0x80798179, - 0xd7610002, 0x0000f20d, + 0x0000f20b, 0x80798179, + 0xd7610002, 0x0000f20c, 0x80798179, 0xd7610002, - 0x0000f20e, 0x80798179, - 0xd7610002, 0x0000f20f, - 0x80798179, 0xbf06a079, - 0xbfa10006, 0xe0685000, - 0x701d0200, 0x8070ff70, - 0x00000080, 0xbef90080, - 0x7e040280, 0x807d907d, - 0xbf0aff7d, 0x00000060, - 0xbfa2ffbc, 0xbe804100, - 0xbe824102, 0xbe844104, - 0xbe864106, 0xbe884108, - 0xbe8a410a, 0xd7610002, - 0x0000f200, 0x80798179, - 0xd7610002, 0x0000f201, + 0x0000f20d, 0x80798179, + 0xd7610002, 0x0000f20e, 0x80798179, 0xd7610002, - 0x0000f202, 0x80798179, - 0xd7610002, 0x0000f203, + 0x0000f20f, 0x80798179, + 0xbf06a079, 0xbfa10006, + 0xe0685000, 0x701d0200, + 0x8070ff70, 0x00000080, + 0xbef90080, 0x7e040280, + 0x807d907d, 0xbf0aff7d, + 0x00000060, 0xbfa2ffbc, + 0xbe804100, 0xbe824102, + 0xbe844104, 0xbe864106, + 0xbe884108, 0xbe8a410a, + 0xd7610002, 0x0000f200, 0x80798179, 0xd7610002, - 0x0000f204, 0x80798179, - 0xd7610002, 0x0000f205, + 0x0000f201, 0x80798179, + 0xd7610002, 0x0000f202, 0x80798179, 0xd7610002, - 0x0000f206, 0x80798179, - 0xd7610002, 0x0000f207, + 0x0000f203, 0x80798179, + 0xd7610002, 0x0000f204, 0x80798179, 0xd7610002, - 0x0000f208, 0x80798179, - 0xd7610002, 0x0000f209, + 0x0000f205, 0x80798179, + 0xd7610002, 0x0000f206, 0x80798179, 0xd7610002, - 0x0000f20a, 0x80798179, - 0xd7610002, 0x0000f20b, - 0x80798179, 0xe0685000, - 0x701d0200, 0xbefe00c1, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00001, - 0xbeff00c1, 0xb8fb4306, - 0x8b7bc17b, 0xbfa10044, - 0xbfbd0000, 0x8b7aff6d, - 0x80000000, 0xbfa10040, - 0x847b867b, 0x847b827b, - 0xbef6007b, 0xb8f03b05, - 0x80708170, 0xbf0d9973, - 0xbfa20002, 0x84708970, - 0xbfa00001, 0x84708a70, - 0xb8fa1e06, 0x847a8a7a, - 0x80707a70, 0x8070ff70, - 0x00000200, 0x8070ff70, - 0x00000080, 0xbef600ff, - 0x01000000, 0xd71f0000, - 0x000100c1, 0xd7200000, - 0x000200c1, 0x16000084, - 0x857d9973, 0x8b7d817d, - 0xbf06817d, 0xbefd0080, - 0xbfa20012, 0xbe8300ff, - 0x00000080, 0xbf800000, - 0xbf800000, 0xbf800000, - 0xd8d80000, 0x01000000, - 0xbf890000, 0xe0685000, - 0x701d0100, 0x807d037d, - 0x80700370, 0xd5250000, - 0x0001ff00, 0x00000080, - 0xbf0a7b7d, 0xbfa2fff4, - 0xbfa00011, 0xbe8300ff, - 0x00000100, 0xbf800000, - 0xbf800000, 0xbf800000, - 0xd8d80000, 0x01000000, - 0xbf890000, 0xe0685000, - 0x701d0100, 0x807d037d, - 0x80700370, 0xd5250000, - 0x0001ff00, 0x00000100, - 0xbf0a7b7d, 0xbfa2fff4, + 0x0000f207, 0x80798179, + 0xd7610002, 0x0000f208, + 0x80798179, 0xd7610002, + 0x0000f209, 0x80798179, + 0xd7610002, 0x0000f20a, + 0x80798179, 0xd7610002, + 0x0000f20b, 0x80798179, + 0xe0685000, 0x701d0200, 0xbefe00c1, 0x857d9973, 0x8b7d817d, 0xbf06817d, - 0xbfa20004, 0xbef000ff, - 0x00000200, 0xbeff0080, - 0xbfa00003, 0xbef000ff, - 0x00000400, 0xbeff00c1, - 0xb8fb3b05, 0x807b817b, - 0x847b827b, 0x857d9973, + 0xbfa20002, 0xbeff0080, + 0xbfa00001, 0xbeff00c1, + 0xb8fb4306, 0x8b7bc17b, + 0xbfa10044, 0xbfbd0000, + 0x8b7aff6d, 0x80000000, + 0xbfa10040, 0x847b867b, + 0x847b827b, 0xbef6007b, + 0xb8f03b05, 0x80708170, + 0xbf0d9973, 0xbfa20002, + 0x84708970, 0xbfa00001, + 0x84708a70, 0xb8fa1e06, + 0x847a8a7a, 0x80707a70, + 0x8070ff70, 0x00000200, + 0x8070ff70, 0x00000080, + 0xbef600ff, 0x01000000, + 0xd71f0000, 0x000100c1, + 0xd7200000, 0x000200c1, + 0x16000084, 0x857d9973, 0x8b7d817d, 0xbf06817d, - 0xbfa20017, 0xbef600ff, - 0x01000000, 0xbefd0084, - 0xbf0a7b7d, 0xbfa10037, - 0x7e008700, 0x7e028701, - 0x7e048702, 0x7e068703, - 0xe0685000, 0x701d0000, - 0xe0685080, 0x701d0100, - 0xe0685100, 0x701d0200, - 0xe0685180, 0x701d0300, - 0x807d847d, 0x8070ff70, - 0x00000200, 0xbf0a7b7d, - 0xbfa2ffef, 0xbfa00025, + 0xbefd0080, 0xbfa20012, + 0xbe8300ff, 0x00000080, + 0xbf800000, 0xbf800000, + 0xbf800000, 0xd8d80000, + 0x01000000, 0xbf890000, + 0xe0685000, 0x701d0100, + 0x807d037d, 0x80700370, + 0xd5250000, 0x0001ff00, + 0x00000080, 0xbf0a7b7d, + 0xbfa2fff4, 0xbfa00011, + 0xbe8300ff, 0x00000100, + 0xbf800000, 0xbf800000, + 0xbf800000, 0xd8d80000, + 0x01000000, 0xbf890000, + 0xe0685000, 0x701d0100, + 0x807d037d, 0x80700370, + 0xd5250000, 0x0001ff00, + 0x00000100, 0xbf0a7b7d, + 0xbfa2fff4, 0xbefe00c1, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20004, + 0xbef000ff, 0x00000200, + 0xbeff0080, 0xbfa00003, + 0xbef000ff, 0x00000400, + 0xbeff00c1, 0xb8fb3b05, + 0x807b817b, 0x847b827b, + 0x857d9973, 0x8b7d817d, + 0xbf06817d, 0xbfa20017, 0xbef600ff, 0x01000000, 0xbefd0084, 0xbf0a7b7d, - 0xbfa10011, 0x7e008700, + 0xbfa10037, 0x7e008700, 0x7e028701, 0x7e048702, 0x7e068703, 0xe0685000, - 0x701d0000, 0xe0685100, - 0x701d0100, 0xe0685200, - 0x701d0200, 0xe0685300, + 0x701d0000, 0xe0685080, + 0x701d0100, 0xe0685100, + 0x701d0200, 0xe0685180, 0x701d0300, 0x807d847d, - 0x8070ff70, 0x00000400, + 0x8070ff70, 0x00000200, 0xbf0a7b7d, 0xbfa2ffef, - 0xb8fb1e06, 0x8b7bc17b, - 0xbfa1000c, 0x847b837b, - 0x807b7d7b, 0xbefe00c1, - 0xbeff0080, 0x7e008700, + 0xbfa00025, 0xbef600ff, + 0x01000000, 0xbefd0084, + 0xbf0a7b7d, 0xbfa10011, + 0x7e008700, 0x7e028701, + 0x7e048702, 0x7e068703, 0xe0685000, 0x701d0000, - 0x807d817d, 0x8070ff70, - 0x00000080, 0xbf0a7b7d, - 0xbfa2fff8, 0xbfa00146, - 0xbef4007e, 0x8b75ff7f, - 0x0000ffff, 0x8c75ff75, - 0x00040000, 0xbef60080, - 0xbef700ff, 0x10807fac, - 0xb8f202dc, 0x84729972, - 0x8b6eff7f, 0x04000000, - 0xbfa1003a, 0xbefe00c1, - 0x857d9972, 0x8b7d817d, - 0xbf06817d, 0xbfa20002, - 0xbeff0080, 0xbfa00001, - 0xbeff00c1, 0xb8ef4306, - 0x8b6fc16f, 0xbfa1002f, - 0x846f866f, 0x846f826f, - 0xbef6006f, 0xb8f83b05, - 0x80788178, 0xbf0d9972, - 0xbfa20002, 0x84788978, - 0xbfa00001, 0x84788a78, - 0xb8ee1e06, 0x846e8a6e, - 0x80786e78, 0x8078ff78, - 0x00000200, 0x8078ff78, - 0x00000080, 0xbef600ff, - 0x01000000, 0x857d9972, - 0x8b7d817d, 0xbf06817d, - 0xbefd0080, 0xbfa2000c, - 0xe0500000, 0x781d0000, - 0xbf8903f7, 0xdac00000, - 0x00000000, 0x807dff7d, - 0x00000080, 0x8078ff78, - 0x00000080, 0xbf0a6f7d, - 0xbfa2fff5, 0xbfa0000b, - 0xe0500000, 0x781d0000, - 0xbf8903f7, 0xdac00000, - 0x00000000, 0x807dff7d, - 0x00000100, 0x8078ff78, - 0x00000100, 0xbf0a6f7d, - 0xbfa2fff5, 0xbef80080, + 0xe0685100, 0x701d0100, + 0xe0685200, 0x701d0200, + 0xe0685300, 0x701d0300, + 0x807d847d, 0x8070ff70, + 0x00000400, 0xbf0a7b7d, + 0xbfa2ffef, 0xb8fb1e06, + 0x8b7bc17b, 0xbfa1000c, + 0x847b837b, 0x807b7d7b, + 0xbefe00c1, 0xbeff0080, + 0x7e008700, 0xe0685000, + 0x701d0000, 0x807d817d, + 0x8070ff70, 0x00000080, + 0xbf0a7b7d, 0xbfa2fff8, + 0xbfa00146, 0xbef4007e, + 0x8b75ff7f, 0x0000ffff, + 0x8c75ff75, 0x00040000, + 0xbef60080, 0xbef700ff, + 0x10807fac, 0xb8f202dc, + 0x84729972, 0x8b6eff7f, + 0x04000000, 0xbfa1003a, 0xbefe00c1, 0x857d9972, 0x8b7d817d, 0xbf06817d, 0xbfa20002, 0xbeff0080, 0xbfa00001, 0xbeff00c1, - 0xb8ef3b05, 0x806f816f, - 0x846f826f, 0x857d9972, - 0x8b7d817d, 0xbf06817d, - 0xbfa20024, 0xbef600ff, - 0x01000000, 0xbeee0078, + 0xb8ef4306, 0x8b6fc16f, + 0xbfa1002f, 0x846f866f, + 0x846f826f, 0xbef6006f, + 0xb8f83b05, 0x80788178, + 0xbf0d9972, 0xbfa20002, + 0x84788978, 0xbfa00001, + 0x84788a78, 0xb8ee1e06, + 0x846e8a6e, 0x80786e78, 0x8078ff78, 0x00000200, - 0xbefd0084, 0xbf0a6f7d, - 0xbfa10050, 0xe0505000, - 0x781d0000, 0xe0505080, - 0x781d0100, 0xe0505100, - 0x781d0200, 0xe0505180, - 0x781d0300, 0xbf8903f7, - 0x7e008500, 0x7e028501, - 0x7e048502, 0x7e068503, - 0x807d847d, 0x8078ff78, - 0x00000200, 0xbf0a6f7d, - 0xbfa2ffee, 0xe0505000, - 0x6e1d0000, 0xe0505080, - 0x6e1d0100, 0xe0505100, - 0x6e1d0200, 0xe0505180, - 0x6e1d0300, 0xbf8903f7, - 0xbfa00034, 0xbef600ff, - 0x01000000, 0xbeee0078, - 0x8078ff78, 0x00000400, - 0xbefd0084, 0xbf0a6f7d, - 0xbfa10012, 0xe0505000, - 0x781d0000, 0xe0505100, - 0x781d0100, 0xe0505200, - 0x781d0200, 0xe0505300, - 0x781d0300, 0xbf8903f7, - 0x7e008500, 0x7e028501, - 0x7e048502, 0x7e068503, - 0x807d847d, 0x8078ff78, - 0x00000400, 0xbf0a6f7d, - 0xbfa2ffee, 0xb8ef1e06, - 0x8b6fc16f, 0xbfa1000e, - 0x846f836f, 0x806f7d6f, - 0xbefe00c1, 0xbeff0080, + 0x8078ff78, 0x00000080, + 0xbef600ff, 0x01000000, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbefd0080, + 0xbfa2000c, 0xe0500000, + 0x781d0000, 0xbf8903f7, + 0xdac00000, 0x00000000, + 0x807dff7d, 0x00000080, + 0x8078ff78, 0x00000080, + 0xbf0a6f7d, 0xbfa2fff5, + 0xbfa0000b, 0xe0500000, + 0x781d0000, 0xbf8903f7, + 0xdac00000, 0x00000000, + 0x807dff7d, 0x00000100, + 0x8078ff78, 0x00000100, + 0xbf0a6f7d, 0xbfa2fff5, + 0xbef80080, 0xbefe00c1, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbfa20002, + 0xbeff0080, 0xbfa00001, + 0xbeff00c1, 0xb8ef3b05, + 0x806f816f, 0x846f826f, + 0x857d9972, 0x8b7d817d, + 0xbf06817d, 0xbfa20024, + 0xbef600ff, 0x01000000, + 0xbeee0078, 0x8078ff78, + 0x00000200, 0xbefd0084, + 0xbf0a6f7d, 0xbfa10050, 0xe0505000, 0x781d0000, + 0xe0505080, 0x781d0100, + 0xe0505100, 0x781d0200, + 0xe0505180, 0x781d0300, 0xbf8903f7, 0x7e008500, - 0x807d817d, 0x8078ff78, - 0x00000080, 0xbf0a6f7d, - 0xbfa2fff7, 0xbeff00c1, + 0x7e028501, 0x7e048502, + 0x7e068503, 0x807d847d, + 0x8078ff78, 0x00000200, + 0xbf0a6f7d, 0xbfa2ffee, 0xe0505000, 0x6e1d0000, - 0xe0505100, 0x6e1d0100, - 0xe0505200, 0x6e1d0200, - 0xe0505300, 0x6e1d0300, - 0xbf8903f7, 0xb8f83b05, - 0x80788178, 0xbf0d9972, - 0xbfa20002, 0x84788978, - 0xbfa00001, 0x84788a78, - 0xb8ee1e06, 0x846e8a6e, - 0x80786e78, 0x8078ff78, - 0x00000200, 0x80f8ff78, - 0x00000050, 0xbef600ff, - 0x01000000, 0xbefd00ff, - 0x0000006c, 0x80f89078, - 0xf428403a, 0xf0000000, - 0xbf89fc07, 0x80fd847d, - 0xbf800000, 0xbe804300, - 0xbe824302, 0x80f8a078, - 0xf42c403a, 0xf0000000, - 0xbf89fc07, 0x80fd887d, - 0xbf800000, 0xbe804300, - 0xbe824302, 0xbe844304, - 0xbe864306, 0x80f8c078, - 0xf430403a, 0xf0000000, - 0xbf89fc07, 0x80fd907d, - 0xbf800000, 0xbe804300, - 0xbe824302, 0xbe844304, - 0xbe864306, 0xbe884308, - 0xbe8a430a, 0xbe8c430c, - 0xbe8e430e, 0xbf06807d, - 0xbfa1fff0, 0xb980f801, - 0x00000000, 0xbfbd0000, + 0xe0505080, 0x6e1d0100, + 0xe0505100, 0x6e1d0200, + 0xe0505180, 0x6e1d0300, + 0xbf8903f7, 0xbfa00034, + 0xbef600ff, 0x01000000, + 0xbeee0078, 0x8078ff78, + 0x00000400, 0xbefd0084, + 0xbf0a6f7d, 0xbfa10012, + 0xe0505000, 0x781d0000, + 0xe0505100, 0x781d0100, + 0xe0505200, 0x781d0200, + 0xe0505300, 0x781d0300, + 0xbf8903f7, 0x7e008500, + 0x7e028501, 0x7e048502, + 0x7e068503, 0x807d847d, + 0x8078ff78, 0x00000400, + 0xbf0a6f7d, 0xbfa2ffee, + 0xb8ef1e06, 0x8b6fc16f, + 0xbfa1000e, 0x846f836f, + 0x806f7d6f, 0xbefe00c1, + 0xbeff0080, 0xe0505000, + 0x781d0000, 0xbf8903f7, + 0x7e008500, 0x807d817d, + 0x8078ff78, 0x00000080, + 0xbf0a6f7d, 0xbfa2fff7, + 0xbeff00c1, 0xe0505000, + 0x6e1d0000, 0xe0505100, + 0x6e1d0100, 0xe0505200, + 0x6e1d0200, 0xe0505300, + 0x6e1d0300, 0xbf8903f7, 0xb8f83b05, 0x80788178, 0xbf0d9972, 0xbfa20002, 0x84788978, 0xbfa00001, 0x84788a78, 0xb8ee1e06, 0x846e8a6e, 0x80786e78, 0x8078ff78, 0x00000200, + 0x80f8ff78, 0x00000050, 0xbef600ff, 0x01000000, - 0xf4205bfa, 0xf0000000, - 0x80788478, 0xf4205b3a, + 0xbefd00ff, 0x0000006c, + 0x80f89078, 0xf428403a, + 0xf0000000, 0xbf89fc07, + 0x80fd847d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0x80f8a078, 0xf42c403a, + 0xf0000000, 0xbf89fc07, + 0x80fd887d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0xbe844304, 0xbe864306, + 0x80f8c078, 0xf430403a, + 0xf0000000, 0xbf89fc07, + 0x80fd907d, 0xbf800000, + 0xbe804300, 0xbe824302, + 0xbe844304, 0xbe864306, + 0xbe884308, 0xbe8a430a, + 0xbe8c430c, 0xbe8e430e, + 0xbf06807d, 0xbfa1fff0, + 0xb980f801, 0x00000000, + 0xbfbd0000, 0xb8f83b05, + 0x80788178, 0xbf0d9972, + 0xbfa20002, 0x84788978, + 0xbfa00001, 0x84788a78, + 0xb8ee1e06, 0x846e8a6e, + 0x80786e78, 0x8078ff78, + 0x00000200, 0xbef600ff, + 0x01000000, 0xf4205bfa, 0xf0000000, 0x80788478, - 0xf4205b7a, 0xf0000000, - 0x80788478, 0xf4205c3a, + 0xf4205b3a, 0xf0000000, + 0x80788478, 0xf4205b7a, 0xf0000000, 0x80788478, - 0xf4205c7a, 0xf0000000, - 0x80788478, 0xf4205eba, + 0xf4205c3a, 0xf0000000, + 0x80788478, 0xf4205c7a, 0xf0000000, 0x80788478, - 0xf4205efa, 0xf0000000, - 0x80788478, 0xf4205e7a, + 0xf4205eba, 0xf0000000, + 0x80788478, 0xf4205efa, 0xf0000000, 0x80788478, - 0xf4205cfa, 0xf0000000, - 0x80788478, 0xf4205bba, + 0xf4205e7a, 0xf0000000, + 0x80788478, 0xf4205cfa, 0xf0000000, 0x80788478, - 0xbf89fc07, 0xb96ef814, 0xf4205bba, 0xf0000000, 0x80788478, 0xbf89fc07, - 0xb96ef815, 0xbefd006f, - 0xbefe0070, 0xbeff0071, - 0x8b6f7bff, 0x000003ff, - 0xb96f4803, 0x8b6f7bff, - 0xfffff800, 0x856f8b6f, - 0xb96fa2c3, 0xb973f801, - 0xb8ee3b05, 0x806e816e, - 0xbf0d9972, 0xbfa20002, - 0x846e896e, 0xbfa00001, - 0x846e8a6e, 0xb8ef1e06, - 0x846f8a6f, 0x806e6f6e, - 0x806eff6e, 0x00000200, - 0x806e746e, 0x826f8075, - 0x8b6fff6f, 0x0000ffff, - 0xf4085c37, 0xf8000050, - 0xf4085d37, 0xf8000060, - 0xf4005e77, 0xf8000074, - 0xbf89fc07, 0x8b6dff6d, - 0x0000ffff, 0x8bfe7e7e, - 0x8bea6a6a, 0xb8eef802, - 0xbf0d866e, 0xbfa20002, - 0xb97af802, 0xbe80486c, - 0xb97af802, 0xbe804a6c, - 0xbfb00000, 0xbf9f0000, + 0xb96ef814, 0xf4205bba, + 0xf0000000, 0x80788478, + 0xbf89fc07, 0xb96ef815, + 0xbefd006f, 0xbefe0070, + 0xbeff0071, 0x8b6f7bff, + 0x000003ff, 0xb96f4803, + 0x8b6f7bff, 0xfffff800, + 0x856f8b6f, 0xb96fa2c3, + 0xb973f801, 0xb8ee3b05, + 0x806e816e, 0xbf0d9972, + 0xbfa20002, 0x846e896e, + 0xbfa00001, 0x846e8a6e, + 0xb8ef1e06, 0x846f8a6f, + 0x806e6f6e, 0x806eff6e, + 0x00000200, 0x806e746e, + 0x826f8075, 0x8b6fff6f, + 0x0000ffff, 0xf4085c37, + 0xf8000050, 0xf4085d37, + 0xf8000060, 0xf4005e77, + 0xf8000074, 0xbf89fc07, + 0x8b6dff6d, 0x0000ffff, + 0x8bfe7e7e, 0x8bea6a6a, + 0xb8eef802, 0xbf0d866e, + 0xbfa20002, 0xb97af802, + 0xbe80486c, 0xb97af802, + 0xbe804a6c, 0xbfb00000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, 0xbf9f0000, + 0xbf9f0000, 0x00000000, }; diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm index 0f81670f6f9c6..8b92c33c2a7c5 100644 --- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm +++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm @@ -186,6 +186,12 @@ L_SKIP_RESTORE: s_getreg_b32 s_save_trapsts, hwreg(HW_REG_TRAPSTS) #if SW_SA_TRAP + // If ttmp1[30] is set then issue s_barrier to unblock dependent waves. + s_bitcmp1_b32 s_save_pc_hi, 30 + s_cbranch_scc0 L_TRAP_NO_BARRIER + s_barrier + +L_TRAP_NO_BARRIER: // If ttmp1[31] is set then trap may occur early. // Spin wait until SAVECTX exception is raised. s_bitcmp1_b32 s_save_pc_hi, 31 -- GitLab From 9a1662f549343d3ec882c5ffcfd232c6104d3c85 Mon Sep 17 00:00:00 2001 From: Graham Sider Date: Thu, 29 Sep 2022 11:57:34 -0400 Subject: [PATCH 1047/1927] drm/amdgpu: extend halt_if_hws_hang to MES Hang on MES timeout if halt_if_hws_hang is set to 1. Signed-off-by: Graham Sider Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 2 ++ drivers/gpu/drm/amd/amdgpu/mes_v10_1.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/mes_v11_0.c | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 0e6ddf05c23c9..9999c18e7d8e0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -219,10 +219,12 @@ extern int amdgpu_use_xgmi_p2p; extern int sched_policy; extern bool debug_evictions; extern bool no_system_mem_limit; +extern int halt_if_hws_hang; #else static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS; static const bool __maybe_unused debug_evictions; /* = false */ static const bool __maybe_unused no_system_mem_limit; +static const int __maybe_unused halt_if_hws_hang; #endif #ifdef CONFIG_HSA_AMD_P2P extern bool pcie_p2p; diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c index 1abdf8b7ab50c..614394118a53e 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v10_1.c @@ -121,6 +121,10 @@ static int mes_v10_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, if (r < 1) { DRM_ERROR("MES failed to response msg=%d\n", x_pkt->header.opcode); + + while (halt_if_hws_hang) + schedule(); + return -ETIMEDOUT; } diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c index 8ca3417fcfe53..1395453a0662d 100644 --- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c @@ -129,6 +129,10 @@ static int mes_v11_0_submit_pkt_and_poll_completion(struct amdgpu_mes *mes, if (r < 1) { DRM_ERROR("MES failed to response msg=%d\n", x_pkt->header.opcode); + + while (halt_if_hws_hang) + schedule(); + return -ETIMEDOUT; } -- GitLab From 0cfce2401e1553f3bf0ee3b6ef56c9dda0303815 Mon Sep 17 00:00:00 2001 From: Yiqing Yao Date: Fri, 28 Oct 2022 17:03:17 +0800 Subject: [PATCH 1048/1927] drm/amdgpu: Clean up soc21 early init for SRIOV Use virt_init_setting instead of per ip version setting. Signed-off-by: Yiqing Yao Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/soc21.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index e08044008186e..1d4013ed0d100 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -584,10 +584,6 @@ static int soc21_common_early_init(void *handle) AMD_PG_SUPPORT_JPEG | AMD_PG_SUPPORT_ATHUB | AMD_PG_SUPPORT_MMHUB; - if (amdgpu_sriov_vf(adev)) { - adev->cg_flags = 0; - adev->pg_flags = 0; - } adev->external_rev_id = adev->rev_id + 0x1; // TODO: need update break; case IP_VERSION(11, 0, 2): @@ -645,11 +641,6 @@ static int soc21_common_early_init(void *handle) adev->pg_flags = AMD_PG_SUPPORT_VCN | AMD_PG_SUPPORT_VCN_DPG | AMD_PG_SUPPORT_JPEG; - if (amdgpu_sriov_vf(adev)) { - /* hypervisor control CG and PG enablement */ - adev->cg_flags = 0; - adev->pg_flags = 0; - } adev->external_rev_id = adev->rev_id + 0x20; break; default: @@ -657,6 +648,9 @@ static int soc21_common_early_init(void *handle) return -EINVAL; } + if (amdgpu_sriov_vf(adev)) + amdgpu_virt_init_setting(adev); + return 0; } -- GitLab From 8a1fbb4a5ec29d9ae5f792a1d3c049c5c709851c Mon Sep 17 00:00:00 2001 From: Yiqing Yao Date: Fri, 28 Oct 2022 17:30:17 +0800 Subject: [PATCH 1049/1927] drm/amdgpu: Disable MCBP from soc21 for SRIOV [why] Start from soc21, CP does not support MCBP, so disable it. [how] Used amgpu_mcbp flag alone instead of checking if is in SRIOV to enable/disable MCBP. Only set flag to enable on asic_type prior to soc21 in SRIOV. Signed-off-by: Yiqing Yao Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++--- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 4 ++++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 4 ++-- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index ae10acede495e..522820eeaa596 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2397,7 +2397,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) adev->ip_blocks[i].status.hw = true; /* right after GMC hw init, we create CSA */ - if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + if (amdgpu_mcbp) { r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, AMDGPU_GEM_DOMAIN_VRAM, AMDGPU_CSA_SIZE); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 4e42dcb1950f7..d75e0370a0743 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -796,7 +796,7 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) dev_info->ids_flags = 0; if (adev->flags & AMD_IS_APU) dev_info->ids_flags |= AMDGPU_IDS_FLAGS_FUSION; - if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) + if (amdgpu_mcbp) dev_info->ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION; if (amdgpu_is_tmz(adev)) dev_info->ids_flags |= AMDGPU_IDS_FLAGS_TMZ; @@ -1172,7 +1172,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) goto error_vm; } - if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + if (amdgpu_mcbp) { uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK; r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj, @@ -1236,7 +1236,7 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_VCE) != NULL) amdgpu_vce_free_handles(adev, file_priv); - if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { + if (amdgpu_mcbp) { /* TODO: how to handle reserve failure */ BUG_ON(amdgpu_bo_reserve(adev->virt.csa_obj, true)); amdgpu_vm_bo_del(adev, fpriv->csa_va); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c index c73abe54d9747..a226a6c48fb76 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c @@ -64,6 +64,10 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev) ddev->driver_features &= ~DRIVER_ATOMIC; adev->cg_flags = 0; adev->pg_flags = 0; + + /* enable mcbp for sriov asic_type before soc21 */ + amdgpu_mcbp = (adev->asic_type < CHIP_IP_DISCOVERY) ? 1 : 0; + } void amdgpu_virt_kiq_reg_write_reg_wait(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index af94ac580d3e1..ff3c127cf70d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -8489,7 +8489,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, control |= ib->length_dw | (vmid << 24); - if ((amdgpu_sriov_vf(ring->adev) || amdgpu_mcbp) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { + if (amdgpu_mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { control |= INDIRECT_BUFFER_PRE_ENB(1); if (flags & AMDGPU_IB_PREEMPTED) @@ -8664,7 +8664,7 @@ static void gfx_v10_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, { uint32_t dw2 = 0; - if (amdgpu_mcbp || amdgpu_sriov_vf(ring->adev)) + if (amdgpu_mcbp) gfx_v10_0_ring_emit_ce_meta(ring, (!amdgpu_sriov_vf(ring->adev) && flags & AMDGPU_IB_PREEMPTED) ? true : false); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index f68e13b6282ca..84a76c36d9a76 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -5326,7 +5326,7 @@ static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, control |= ib->length_dw | (vmid << 24); - if ((amdgpu_sriov_vf(ring->adev) || amdgpu_mcbp) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { + if (amdgpu_mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { control |= INDIRECT_BUFFER_PRE_ENB(1); if (flags & AMDGPU_IB_PREEMPTED) -- GitLab From f8638ad7fcac50d2ba8e76625e7d86cabd5d5bd2 Mon Sep 17 00:00:00 2001 From: Peng Ju Zhou Date: Fri, 28 Oct 2022 18:24:45 +0800 Subject: [PATCH 1050/1927] drm/amdgpu: Remove unnecessary register program in SRIOV Remove unnecessary register program in SRIOV Signed-off-by: Peng Ju Zhou Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c index 8cf53e039c115..3f8676d23a5ed 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c @@ -397,6 +397,9 @@ static void gfxhub_v2_1_gart_disable(struct amdgpu_device *adev) ENABLE_ADVANCED_DRIVER_MODEL, 0); WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp); + if (amdgpu_sriov_vf(adev)) + return; + /* Setup L2 cache */ WREG32_FIELD15(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0); WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, 0); -- GitLab From 33034c5c2efaf56c7b89aed5a3cf1bcb3f12b853 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 2 Nov 2022 11:17:23 -0400 Subject: [PATCH 1051/1927] drm/amdgpu/gfx9: set gfx.funcs in early init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So the callbacks are set before we use them. Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index 0320be4a5fc6c..c80623c35b52a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1919,8 +1919,6 @@ static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) u32 gb_addr_config; int err; - adev->gfx.funcs = &gfx_v9_0_gfx_funcs; - switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(9, 0, 1): adev->gfx.config.max_hw_contexts = 8; @@ -4539,6 +4537,8 @@ static int gfx_v9_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + adev->gfx.funcs = &gfx_v9_0_gfx_funcs; + if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) adev->gfx.num_gfx_rings = 0; -- GitLab From 105195af02d4e8665e4583cf7ef4e5f69d858d7d Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 2 Nov 2022 11:20:49 -0400 Subject: [PATCH 1052/1927] drm/amdgpu/gfx10: set gfx.funcs in early init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So the callbacks are set early in case we need them. Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index ff3c127cf70d7..7853d3ca58cf7 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4453,8 +4453,6 @@ static void gfx_v10_0_gpu_early_init(struct amdgpu_device *adev) { u32 gb_addr_config; - adev->gfx.funcs = &gfx_v10_0_gfx_funcs; - switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(10, 1, 10): case IP_VERSION(10, 1, 1): @@ -7593,6 +7591,8 @@ static int gfx_v10_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + adev->gfx.funcs = &gfx_v10_0_gfx_funcs; + switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(10, 1, 10): case IP_VERSION(10, 1, 1): -- GitLab From e053d71f8c04fd7ef53241d2ab39f02acee6aaa0 Mon Sep 17 00:00:00 2001 From: Alex Deucher Date: Wed, 2 Nov 2022 11:21:35 -0400 Subject: [PATCH 1053/1927] drm/amdgpu/gfx11: set gfx.funcs in early init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So the callbacks are set early in case we need them. Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 84a76c36d9a76..d799359565b85 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -843,7 +843,6 @@ static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = { static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) { - adev->gfx.funcs = &gfx_v11_0_gfx_funcs; switch (adev->ip_versions[GC_HWIP][0]) { case IP_VERSION(11, 0, 0): @@ -4657,6 +4656,8 @@ static int gfx_v11_0_early_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; + adev->gfx.funcs = &gfx_v11_0_gfx_funcs; + adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS; adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), AMDGPU_MAX_COMPUTE_RINGS); -- GitLab From c81c5bd5cf2f428867e0bcfcccd4e4d2f8c68f51 Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Sat, 29 Oct 2022 16:32:05 +1300 Subject: [PATCH 1054/1927] drm/radeon: Replace one-element array with flexible-array member One-element arrays are deprecated, and we are replacing them with flexible array members instead. So, replace one-element array with flexible-array member in struct _ATOM_FAKE_EDID_PATCH_RECORD and refactor the rest of the code accordingly. It's worth mentioning that doing a build before/after this patch results in no binary output differences. This helps with the ongoing efforts to tighten the FORTIFY_SOURCE routines on memcpy() and help us make progress towards globally enabling -fstrict-flex-arrays=3 [1]. Link: https://github.com/KSPP/linux/issues/79 Link: https://github.com/KSPP/linux/issues/239 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836 [1] Reviewed-by: Kees Cook Signed-off-by: Paulo Miguel Almeida Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/atombios.h | 2 +- drivers/gpu/drm/radeon/radeon_atombios.c | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h index da35a970fcc0d..235e59b547a1e 100644 --- a/drivers/gpu/drm/radeon/atombios.h +++ b/drivers/gpu/drm/radeon/atombios.h @@ -3615,7 +3615,7 @@ typedef struct _ATOM_FAKE_EDID_PATCH_RECORD { UCHAR ucRecordType; UCHAR ucFakeEDIDLength; - UCHAR ucFakeEDIDString[1]; // This actually has ucFakeEdidLength elements. + UCHAR ucFakeEDIDString[]; // This actually has ucFakeEdidLength elements. } ATOM_FAKE_EDID_PATCH_RECORD; typedef struct _ATOM_PANEL_RESOLUTION_PATCH_RECORD diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c index 204127bad89ca..4ad5a328d9202 100644 --- a/drivers/gpu/drm/radeon/radeon_atombios.c +++ b/drivers/gpu/drm/radeon/radeon_atombios.c @@ -1727,8 +1727,11 @@ struct radeon_encoder_atom_dig *radeon_atombios_get_lvds_info(struct } } record += fake_edid_record->ucFakeEDIDLength ? - fake_edid_record->ucFakeEDIDLength + 2 : - sizeof(ATOM_FAKE_EDID_PATCH_RECORD); + struct_size(fake_edid_record, + ucFakeEDIDString, + fake_edid_record->ucFakeEDIDLength) : + /* empty fake edid record must be 3 bytes long */ + sizeof(ATOM_FAKE_EDID_PATCH_RECORD) + 1; break; case LCD_PANEL_RESOLUTION_RECORD_TYPE: panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record; -- GitLab From 320e2590e281d0a7865e861f50155b5b435e9813 Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Sat, 29 Oct 2022 14:30:44 +1300 Subject: [PATCH 1055/1927] drm/amdgpu: Replace one-element array with flexible-array member One-element arrays are deprecated, and we are replacing them with flexible array members instead. So, replace one-element array with flexible-array member in struct _ATOM_FAKE_EDID_PATCH_RECORD and refactor the rest of the code accordingly. Important to mention is that doing a build before/after this patch results in no binary output differences. This helps with the ongoing efforts to tighten the FORTIFY_SOURCE routines on memcpy() and help us make progress towards globally enabling -fstrict-flex-arrays=3 [1]. Link: https://github.com/KSPP/linux/issues/79 Link: https://github.com/KSPP/linux/issues/238 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836 [1] Signed-off-by: Paulo Miguel Almeida Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/atombios_encoders.c | 7 +++++-- drivers/gpu/drm/amd/include/atombios.h | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c index 6be9ac2b9c5bc..18ae9433e463d 100644 --- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c +++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c @@ -2081,8 +2081,11 @@ amdgpu_atombios_encoder_get_lcd_info(struct amdgpu_encoder *encoder) } } record += fake_edid_record->ucFakeEDIDLength ? - fake_edid_record->ucFakeEDIDLength + 2 : - sizeof(ATOM_FAKE_EDID_PATCH_RECORD); + struct_size(fake_edid_record, + ucFakeEDIDString, + fake_edid_record->ucFakeEDIDLength) : + /* empty fake edid record must be 3 bytes long */ + sizeof(ATOM_FAKE_EDID_PATCH_RECORD) + 1; break; case LCD_PANEL_RESOLUTION_RECORD_TYPE: panel_res_record = (ATOM_PANEL_RESOLUTION_PATCH_RECORD *)record; diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h index 15943bc21bc54..b5b1d073f8e24 100644 --- a/drivers/gpu/drm/amd/include/atombios.h +++ b/drivers/gpu/drm/amd/include/atombios.h @@ -4107,7 +4107,7 @@ typedef struct _ATOM_FAKE_EDID_PATCH_RECORD { UCHAR ucRecordType; UCHAR ucFakeEDIDLength; // = 128 means EDID length is 128 bytes, otherwise the EDID length = ucFakeEDIDLength*128 - UCHAR ucFakeEDIDString[1]; // This actually has ucFakeEdidLength elements. + UCHAR ucFakeEDIDString[]; // This actually has ucFakeEdidLength elements. } ATOM_FAKE_EDID_PATCH_RECORD; typedef struct _ATOM_PANEL_RESOLUTION_PATCH_RECORD -- GitLab From f0d0f1087333714ee683cc134a95afe331d7ddd9 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 2 Nov 2022 08:25:39 -0700 Subject: [PATCH 1056/1927] drm/amdgpu: Fix type of second parameter in trans_msg() callback With clang's kernel control flow integrity (kCFI, CONFIG_CFI_CLANG), indirect call targets are validated against the expected function pointer prototype to make sure the call target is valid to help mitigate ROP attacks. If they are not identical, there is a failure at run time, which manifests as either a kernel panic or thread getting killed. A proposed warning in clang aims to catch these at compile time, which reveals: drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c:412:15: error: incompatible function pointer types initializing 'void (*)(struct amdgpu_device *, u32, u32, u32, u32)' (aka 'void (*)(struct amdgpu_device *, unsigned int, unsigned int, unsigned int, unsigned int)') with an expression of type 'void (struct amdgpu_device *, enum idh_request, u32, u32, u32)' (aka 'void (struct amdgpu_device *, enum idh_request, unsigned int, unsigned int, unsigned int)') [-Werror,-Wincompatible-function-pointer-types-strict] .trans_msg = xgpu_ai_mailbox_trans_msg, ^~~~~~~~~~~~~~~~~~~~~~~~~ 1 error generated. drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c:435:15: error: incompatible function pointer types initializing 'void (*)(struct amdgpu_device *, u32, u32, u32, u32)' (aka 'void (*)(struct amdgpu_device *, unsigned int, unsigned int, unsigned int, unsigned int)') with an expression of type 'void (struct amdgpu_device *, enum idh_request, u32, u32, u32)' (aka 'void (struct amdgpu_device *, enum idh_request, unsigned int, unsigned int, unsigned int)') [-Werror,-Wincompatible-function-pointer-types-strict] .trans_msg = xgpu_nv_mailbox_trans_msg, ^~~~~~~~~~~~~~~~~~~~~~~~~ 1 error generated. The type of the second parameter in the prototype should be 'enum idh_request' instead of 'u32'. Update it to clear up the warnings. Link: https://github.com/ClangBuiltLinux/linux/issues/1750 Reported-by: Sami Tolvanen Reviewed-by: Kees Cook Signed-off-by: Nathan Chancellor Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h index 49c4347d154ce..2b9d806e23afb 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h @@ -75,6 +75,8 @@ struct amdgpu_vf_error_buffer { uint64_t data[AMDGPU_VF_ERROR_ENTRY_SIZE]; }; +enum idh_request; + /** * struct amdgpu_virt_ops - amdgpu device virt operations */ @@ -84,7 +86,8 @@ struct amdgpu_virt_ops { int (*req_init_data)(struct amdgpu_device *adev); int (*reset_gpu)(struct amdgpu_device *adev); int (*wait_reset)(struct amdgpu_device *adev); - void (*trans_msg)(struct amdgpu_device *adev, u32 req, u32 data1, u32 data2, u32 data3); + void (*trans_msg)(struct amdgpu_device *adev, enum idh_request req, + u32 data1, u32 data2, u32 data3); }; /* -- GitLab From e4d0ef752081e7aa6ffb7ccac11c499c732a2e05 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Wed, 2 Nov 2022 08:25:40 -0700 Subject: [PATCH 1057/1927] drm/amdgpu: Fix type of second parameter in odn_edit_dpm_table() callback With clang's kernel control flow integrity (kCFI, CONFIG_CFI_CLANG), indirect call targets are validated against the expected function pointer prototype to make sure the call target is valid to help mitigate ROP attacks. If they are not identical, there is a failure at run time, which manifests as either a kernel panic or thread getting killed. A proposed warning in clang aims to catch these at compile time, which reveals: drivers/gpu/drm/amd/amdgpu/../pm/swsmu/amdgpu_smu.c:3008:29: error: incompatible function pointer types initializing 'int (*)(void *, uint32_t, long *, uint32_t)' (aka 'int (*)(void *, unsigned int, long *, unsigned int)') with an expression of type 'int (void *, enum PP_OD_DPM_TABLE_COMMAND, long *, uint32_t)' (aka 'int (void *, enum PP_OD_DPM_TABLE_COMMAND, long *, unsigned int)') [-Werror,-Wincompatible-function-pointer-types-strict] .odn_edit_dpm_table = smu_od_edit_dpm_table, ^~~~~~~~~~~~~~~~~~~~~ 1 error generated. There are only two implementations of ->odn_edit_dpm_table() in 'struct amd_pm_funcs': smu_od_edit_dpm_table() and pp_odn_edit_dpm_table(). One has a second parameter type of 'enum PP_OD_DPM_TABLE_COMMAND' and the other uses 'u32'. Ultimately, smu_od_edit_dpm_table() calls ->od_edit_dpm_table() from 'struct pptable_funcs' and pp_odn_edit_dpm_table() calls ->odn_edit_dpm_table() from 'struct pp_hwmgr_func', which both have a second parameter type of 'enum PP_OD_DPM_TABLE_COMMAND'. Update the type parameter in both the prototype in 'struct amd_pm_funcs' and pp_odn_edit_dpm_table() to 'enum PP_OD_DPM_TABLE_COMMAND', which cleans up the warning. Link: https://github.com/ClangBuiltLinux/linux/issues/1750 Reported-by: Sami Tolvanen Reviewed-by: Kees Cook Signed-off-by: Nathan Chancellor Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/include/kgd_pp_interface.h | 3 ++- drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h index a40ead44778af..d18162e9ed1da 100644 --- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h @@ -354,7 +354,8 @@ struct amd_pm_funcs { int (*get_power_profile_mode)(void *handle, char *buf); int (*set_power_profile_mode)(void *handle, long *input, uint32_t size); int (*set_fine_grain_clk_vol)(void *handle, uint32_t type, long *input, uint32_t size); - int (*odn_edit_dpm_table)(void *handle, uint32_t type, long *input, uint32_t size); + int (*odn_edit_dpm_table)(void *handle, enum PP_OD_DPM_TABLE_COMMAND type, + long *input, uint32_t size); int (*set_mp1_state)(void *handle, enum pp_mp1_state mp1_state); int (*smu_i2c_bus_access)(void *handle, bool acquire); int (*gfx_state_change_set)(void *handle, uint32_t state); diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c index ec055858eb95a..1159ae114dd02 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c @@ -838,7 +838,8 @@ static int pp_set_fine_grain_clk_vol(void *handle, uint32_t type, long *input, u return hwmgr->hwmgr_func->set_fine_grain_clk_vol(hwmgr, type, input, size); } -static int pp_odn_edit_dpm_table(void *handle, uint32_t type, long *input, uint32_t size) +static int pp_odn_edit_dpm_table(void *handle, enum PP_OD_DPM_TABLE_COMMAND type, + long *input, uint32_t size) { struct pp_hwmgr *hwmgr = handle; -- GitLab From cfa61b8f9e9e5e14e7a79f3e9454feb8e4690f99 Mon Sep 17 00:00:00 2001 From: Hawking Zhang Date: Mon, 25 Oct 2021 17:23:38 +0800 Subject: [PATCH 1058/1927] drm/amdgpu: switch to select_se_sh wrapper for gfx v9_0 To allow invoking ip specific callbacks Acked-by: Felix Kuehling Signed-off-by: Hawking Zhang Reviewed-by: Le Ma Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 4 +-- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 28 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index 81e3b528bbc9b..e92b93557c13f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -787,7 +787,7 @@ void kgd_gfx_v9_get_cu_occupancy(struct amdgpu_device *adev, int pasid, for (se_idx = 0; se_idx < se_cnt; se_idx++) { for (sh_idx = 0; sh_idx < sh_cnt; sh_idx++) { - gfx_v9_0_select_se_sh(adev, se_idx, sh_idx, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, se_idx, sh_idx, 0xffffffff); queue_map = RREG32_SOC15(GC, 0, mmSPI_CSQ_WF_ACTIVE_STATUS); /* @@ -820,7 +820,7 @@ void kgd_gfx_v9_get_cu_occupancy(struct amdgpu_device *adev, int pasid, } } - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); soc15_grbm_select(adev, 0, 0, 0, 0); unlock_spi_csq_mutexes(adev); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index c80623c35b52a..ab30c18decfbe 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1564,7 +1564,7 @@ static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) mask = 1; cu_bitmap = 0; counter = 0; - gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff); for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { if (cu_info->bitmap[i][j] & mask) { @@ -1583,7 +1583,7 @@ static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) cu_info->ao_cu_bitmap[i][j] = cu_bitmap; } } - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); } @@ -1605,7 +1605,7 @@ static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); /* set mmRLC_LB_PARAMS = 0x003F_1006 */ @@ -1654,7 +1654,7 @@ static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); /* set mmRLC_LB_PARAMS = 0x003F_1006 */ @@ -2322,13 +2322,13 @@ static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { - gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff); data = gfx_v9_0_get_rb_active_bitmap(adev); active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * rb_bitmap_width_per_sh); } } - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); adev->gfx.config.backend_enable_mask = active_rbs; @@ -2465,14 +2465,14 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) mutex_lock(&adev->grbm_idx_mutex); for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { - gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff); for (k = 0; k < adev->usec_timeout; k++) { if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) break; udelay(1); } if (k == adev->usec_timeout) { - gfx_v9_0_select_se_sh(adev, 0xffffffff, + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); DRM_INFO("Timeout wait for RLC serdes %u,%u\n", @@ -2481,7 +2481,7 @@ static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) } } } - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | @@ -6482,7 +6482,7 @@ static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { - gfx_v9_0_select_se_sh(adev, j, 0x0, k); + amdgpu_gfx_select_se_sh(adev, j, 0x0, k); RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); } } @@ -6544,7 +6544,7 @@ static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { - gfx_v9_0_select_se_sh(adev, j, 0, k); + amdgpu_gfx_select_se_sh(adev, j, 0, k); reg_value = RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); if (reg_value) @@ -6559,7 +6559,7 @@ static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, err_data->ce_count += sec_count; err_data->ue_count += ded_count; - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); gfx_v9_0_query_utc_edc_status(adev, err_data); @@ -6963,7 +6963,7 @@ static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, mask = 1; ao_bitmap = 0; counter = 0; - gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff); gfx_v9_0_set_user_cu_inactive_bitmap( adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); bitmap = gfx_v9_0_get_cu_active_bitmap(adev); @@ -6996,7 +6996,7 @@ static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; } } - gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); + amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); mutex_unlock(&adev->grbm_idx_mutex); cu_info->number = active_cu_number; -- GitLab From c0cc999f3c32e65a7c88fb323893ddf897b24488 Mon Sep 17 00:00:00 2001 From: Ma Jun Date: Wed, 2 Nov 2022 15:53:26 +0800 Subject: [PATCH 1059/1927] drm/amdkfd: Fix the warning of array-index-out-of-bounds For some GPUs with more CUs, the original sibling_map[32] in struct crat_subtype_cache is not enough to save the cache information when create the VCRAT table, so skip filling the struct crat_subtype_cache info instead fill struct kfd_cache_properties directly to fix this problem. Signed-off-by: Ma Jun Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 312 +++------------------- drivers/gpu/drm/amd/amdkfd/kfd_crat.h | 12 + drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 245 ++++++++++++++++- drivers/gpu/drm/amd/amdkfd/kfd_topology.h | 5 +- 4 files changed, 282 insertions(+), 292 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 568cbf1d8259c..267dd69737fae 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -50,16 +50,6 @@ static inline unsigned int get_and_inc_gpu_processor_id( return current_id; } -/* Static table to describe GPU Cache information */ -struct kfd_gpu_cache_info { - uint32_t cache_size; - uint32_t cache_level; - uint32_t flags; - /* Indicates how many Compute Units share this cache - * within a SA. Value = 1 indicates the cache is not shared - */ - uint32_t num_cu_shared; -}; static struct kfd_gpu_cache_info kaveri_cache_info[] = { { @@ -1119,9 +1109,13 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache, props->cachelines_per_tag = cache->lines_per_tag; props->cache_assoc = cache->associativity; props->cache_latency = cache->cache_latency; + memcpy(props->sibling_map, cache->sibling_map, sizeof(props->sibling_map)); + /* set the sibling_map_size as 32 for CRAT from ACPI */ + props->sibling_map_size = CRAT_SIBLINGMAP_SIZE; + if (cache->flags & CRAT_CACHE_FLAGS_DATA_CACHE) props->cache_type |= HSA_CACHE_TYPE_DATA; if (cache->flags & CRAT_CACHE_FLAGS_INST_CACHE) @@ -1338,125 +1332,6 @@ int kfd_parse_crat_table(void *crat_image, struct list_head *device_list, return ret; } -/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ -static int fill_in_l1_pcache(struct crat_subtype_cache *pcache, - struct kfd_gpu_cache_info *pcache_info, - struct kfd_cu_info *cu_info, - int mem_available, - int cu_bitmask, - int cache_type, unsigned int cu_processor_id, - int cu_block) -{ - unsigned int cu_sibling_map_mask; - int first_active_cu; - - /* First check if enough memory is available */ - if (sizeof(struct crat_subtype_cache) > mem_available) - return -ENOMEM; - - cu_sibling_map_mask = cu_bitmask; - cu_sibling_map_mask >>= cu_block; - cu_sibling_map_mask &= - ((1 << pcache_info[cache_type].num_cu_shared) - 1); - first_active_cu = ffs(cu_sibling_map_mask); - - /* CU could be inactive. In case of shared cache find the first active - * CU. and incase of non-shared cache check if the CU is inactive. If - * inactive active skip it - */ - if (first_active_cu) { - memset(pcache, 0, sizeof(struct crat_subtype_cache)); - pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY; - pcache->length = sizeof(struct crat_subtype_cache); - pcache->flags = pcache_info[cache_type].flags; - pcache->processor_id_low = cu_processor_id - + (first_active_cu - 1); - pcache->cache_level = pcache_info[cache_type].cache_level; - pcache->cache_size = pcache_info[cache_type].cache_size; - - /* Sibling map is w.r.t processor_id_low, so shift out - * inactive CU - */ - cu_sibling_map_mask = - cu_sibling_map_mask >> (first_active_cu - 1); - - pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF); - pcache->sibling_map[1] = - (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); - pcache->sibling_map[2] = - (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); - pcache->sibling_map[3] = - (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); - return 0; - } - return 1; -} - -/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ -static int fill_in_l2_l3_pcache(struct crat_subtype_cache *pcache, - struct kfd_gpu_cache_info *pcache_info, - struct kfd_cu_info *cu_info, - int mem_available, - int cache_type, unsigned int cu_processor_id) -{ - unsigned int cu_sibling_map_mask; - int first_active_cu; - int i, j, k; - - /* First check if enough memory is available */ - if (sizeof(struct crat_subtype_cache) > mem_available) - return -ENOMEM; - - cu_sibling_map_mask = cu_info->cu_bitmap[0][0]; - cu_sibling_map_mask &= - ((1 << pcache_info[cache_type].num_cu_shared) - 1); - first_active_cu = ffs(cu_sibling_map_mask); - - /* CU could be inactive. In case of shared cache find the first active - * CU. and incase of non-shared cache check if the CU is inactive. If - * inactive active skip it - */ - if (first_active_cu) { - memset(pcache, 0, sizeof(struct crat_subtype_cache)); - pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY; - pcache->length = sizeof(struct crat_subtype_cache); - pcache->flags = pcache_info[cache_type].flags; - pcache->processor_id_low = cu_processor_id - + (first_active_cu - 1); - pcache->cache_level = pcache_info[cache_type].cache_level; - pcache->cache_size = pcache_info[cache_type].cache_size; - - /* Sibling map is w.r.t processor_id_low, so shift out - * inactive CU - */ - cu_sibling_map_mask = - cu_sibling_map_mask >> (first_active_cu - 1); - k = 0; - for (i = 0; i < cu_info->num_shader_engines; i++) { - for (j = 0; j < cu_info->num_shader_arrays_per_engine; - j++) { - pcache->sibling_map[k] = - (uint8_t)(cu_sibling_map_mask & 0xFF); - pcache->sibling_map[k+1] = - (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); - pcache->sibling_map[k+2] = - (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); - pcache->sibling_map[k+3] = - (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); - k += 4; - cu_sibling_map_mask = - cu_info->cu_bitmap[i % 4][j + i / 4]; - cu_sibling_map_mask &= ( - (1 << pcache_info[cache_type].num_cu_shared) - - 1); - } - } - return 0; - } - return 1; -} - -#define KFD_MAX_CACHE_TYPES 6 static int kfd_fill_gpu_cache_info_from_gfx_config(struct kfd_dev *kdev, struct kfd_gpu_cache_info *pcache_info) @@ -1530,231 +1405,133 @@ static int kfd_fill_gpu_cache_info_from_gfx_config(struct kfd_dev *kdev, return i; } -/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info - * tables - * - * @kdev - [IN] GPU device - * @gpu_processor_id - [IN] GPU processor ID to which these caches - * associate - * @available_size - [IN] Amount of memory available in pcache - * @cu_info - [IN] Compute Unit info obtained from KGD - * @pcache - [OUT] memory into which cache data is to be filled in. - * @size_filled - [OUT] amount of data used up in pcache. - * @num_of_entries - [OUT] number of caches added - */ -static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, - int gpu_processor_id, - int available_size, - struct kfd_cu_info *cu_info, - struct crat_subtype_cache *pcache, - int *size_filled, - int *num_of_entries) +int kfd_get_gpu_cache_info(struct kfd_dev *kdev, struct kfd_gpu_cache_info **pcache_info) { - struct kfd_gpu_cache_info *pcache_info; - struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES]; int num_of_cache_types = 0; - int i, j, k; - int ct = 0; - int mem_available = available_size; - unsigned int cu_processor_id; - int ret; - unsigned int num_cu_shared; switch (kdev->adev->asic_type) { case CHIP_KAVERI: - pcache_info = kaveri_cache_info; + *pcache_info = kaveri_cache_info; num_of_cache_types = ARRAY_SIZE(kaveri_cache_info); break; case CHIP_HAWAII: - pcache_info = hawaii_cache_info; + *pcache_info = hawaii_cache_info; num_of_cache_types = ARRAY_SIZE(hawaii_cache_info); break; case CHIP_CARRIZO: - pcache_info = carrizo_cache_info; + *pcache_info = carrizo_cache_info; num_of_cache_types = ARRAY_SIZE(carrizo_cache_info); break; case CHIP_TONGA: - pcache_info = tonga_cache_info; + *pcache_info = tonga_cache_info; num_of_cache_types = ARRAY_SIZE(tonga_cache_info); break; case CHIP_FIJI: - pcache_info = fiji_cache_info; + *pcache_info = fiji_cache_info; num_of_cache_types = ARRAY_SIZE(fiji_cache_info); break; case CHIP_POLARIS10: - pcache_info = polaris10_cache_info; + *pcache_info = polaris10_cache_info; num_of_cache_types = ARRAY_SIZE(polaris10_cache_info); break; case CHIP_POLARIS11: - pcache_info = polaris11_cache_info; + *pcache_info = polaris11_cache_info; num_of_cache_types = ARRAY_SIZE(polaris11_cache_info); break; case CHIP_POLARIS12: - pcache_info = polaris12_cache_info; + *pcache_info = polaris12_cache_info; num_of_cache_types = ARRAY_SIZE(polaris12_cache_info); break; case CHIP_VEGAM: - pcache_info = vegam_cache_info; + *pcache_info = vegam_cache_info; num_of_cache_types = ARRAY_SIZE(vegam_cache_info); break; default: switch (KFD_GC_VERSION(kdev)) { case IP_VERSION(9, 0, 1): - pcache_info = vega10_cache_info; + *pcache_info = vega10_cache_info; num_of_cache_types = ARRAY_SIZE(vega10_cache_info); break; case IP_VERSION(9, 2, 1): - pcache_info = vega12_cache_info; + *pcache_info = vega12_cache_info; num_of_cache_types = ARRAY_SIZE(vega12_cache_info); break; case IP_VERSION(9, 4, 0): case IP_VERSION(9, 4, 1): - pcache_info = vega20_cache_info; + *pcache_info = vega20_cache_info; num_of_cache_types = ARRAY_SIZE(vega20_cache_info); break; case IP_VERSION(9, 4, 2): - pcache_info = aldebaran_cache_info; + *pcache_info = aldebaran_cache_info; num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info); break; case IP_VERSION(9, 1, 0): case IP_VERSION(9, 2, 2): - pcache_info = raven_cache_info; + *pcache_info = raven_cache_info; num_of_cache_types = ARRAY_SIZE(raven_cache_info); break; case IP_VERSION(9, 3, 0): - pcache_info = renoir_cache_info; + *pcache_info = renoir_cache_info; num_of_cache_types = ARRAY_SIZE(renoir_cache_info); break; case IP_VERSION(10, 1, 10): case IP_VERSION(10, 1, 2): case IP_VERSION(10, 1, 3): case IP_VERSION(10, 1, 4): - pcache_info = navi10_cache_info; + *pcache_info = navi10_cache_info; num_of_cache_types = ARRAY_SIZE(navi10_cache_info); break; case IP_VERSION(10, 1, 1): - pcache_info = navi14_cache_info; + *pcache_info = navi14_cache_info; num_of_cache_types = ARRAY_SIZE(navi14_cache_info); break; case IP_VERSION(10, 3, 0): - pcache_info = sienna_cichlid_cache_info; + *pcache_info = sienna_cichlid_cache_info; num_of_cache_types = ARRAY_SIZE(sienna_cichlid_cache_info); break; case IP_VERSION(10, 3, 2): - pcache_info = navy_flounder_cache_info; + *pcache_info = navy_flounder_cache_info; num_of_cache_types = ARRAY_SIZE(navy_flounder_cache_info); break; case IP_VERSION(10, 3, 4): - pcache_info = dimgrey_cavefish_cache_info; + *pcache_info = dimgrey_cavefish_cache_info; num_of_cache_types = ARRAY_SIZE(dimgrey_cavefish_cache_info); break; case IP_VERSION(10, 3, 1): - pcache_info = vangogh_cache_info; + *pcache_info = vangogh_cache_info; num_of_cache_types = ARRAY_SIZE(vangogh_cache_info); break; case IP_VERSION(10, 3, 5): - pcache_info = beige_goby_cache_info; + *pcache_info = beige_goby_cache_info; num_of_cache_types = ARRAY_SIZE(beige_goby_cache_info); break; case IP_VERSION(10, 3, 3): - pcache_info = yellow_carp_cache_info; + *pcache_info = yellow_carp_cache_info; num_of_cache_types = ARRAY_SIZE(yellow_carp_cache_info); break; case IP_VERSION(10, 3, 6): - pcache_info = gc_10_3_6_cache_info; + *pcache_info = gc_10_3_6_cache_info; num_of_cache_types = ARRAY_SIZE(gc_10_3_6_cache_info); break; case IP_VERSION(10, 3, 7): - pcache_info = gfx1037_cache_info; + *pcache_info = gfx1037_cache_info; num_of_cache_types = ARRAY_SIZE(gfx1037_cache_info); break; case IP_VERSION(11, 0, 0): case IP_VERSION(11, 0, 1): case IP_VERSION(11, 0, 2): case IP_VERSION(11, 0, 3): - pcache_info = cache_info; num_of_cache_types = - kfd_fill_gpu_cache_info_from_gfx_config(kdev, pcache_info); + kfd_fill_gpu_cache_info_from_gfx_config(kdev, *pcache_info); break; default: - pcache_info = dummy_cache_info; + *pcache_info = dummy_cache_info; num_of_cache_types = ARRAY_SIZE(dummy_cache_info); pr_warn("dummy cache info is used temporarily and real cache info need update later.\n"); break; } } - - *size_filled = 0; - *num_of_entries = 0; - - /* For each type of cache listed in the kfd_gpu_cache_info table, - * go through all available Compute Units. - * The [i,j,k] loop will - * if kfd_gpu_cache_info.num_cu_shared = 1 - * will parse through all available CU - * If (kfd_gpu_cache_info.num_cu_shared != 1) - * then it will consider only one CU from - * the shared unit - */ - - for (ct = 0; ct < num_of_cache_types; ct++) { - cu_processor_id = gpu_processor_id; - if (pcache_info[ct].cache_level == 1) { - for (i = 0; i < cu_info->num_shader_engines; i++) { - for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) { - for (k = 0; k < cu_info->num_cu_per_sh; - k += pcache_info[ct].num_cu_shared) { - ret = fill_in_l1_pcache(pcache, - pcache_info, - cu_info, - mem_available, - cu_info->cu_bitmap[i % 4][j + i / 4], - ct, - cu_processor_id, - k); - - if (ret < 0) - break; - - if (!ret) { - pcache++; - (*num_of_entries)++; - mem_available -= sizeof(*pcache); - (*size_filled) += sizeof(*pcache); - } - - /* Move to next CU block */ - num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <= - cu_info->num_cu_per_sh) ? - pcache_info[ct].num_cu_shared : - (cu_info->num_cu_per_sh - k); - cu_processor_id += num_cu_shared; - } - } - } - } else { - ret = fill_in_l2_l3_pcache(pcache, - pcache_info, - cu_info, - mem_available, - ct, - cu_processor_id); - - if (ret < 0) - break; - - if (!ret) { - pcache++; - (*num_of_entries)++; - mem_available -= sizeof(*pcache); - (*size_filled) += sizeof(*pcache); - } - } - } - - pr_debug("Added [%d] GPU cache entries\n", *num_of_entries); - - return 0; + return num_of_cache_types; } static bool kfd_ignore_crat(void) @@ -2313,8 +2090,6 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image, struct kfd_cu_info cu_info; int avail_size = *size; uint32_t total_num_of_cu; - int num_of_cache_entries = 0; - int cache_mem_filled = 0; uint32_t nid = 0; int ret = 0; @@ -2415,31 +2190,12 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image, crat_table->length += sizeof(struct crat_subtype_memory); crat_table->total_entries++; - /* TODO: Fill in cache information. This information is NOT readily - * available in KGD - */ - sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + - sub_type_hdr->length); - ret = kfd_fill_gpu_cache_info(kdev, cu->processor_id_low, - avail_size, - &cu_info, - (struct crat_subtype_cache *)sub_type_hdr, - &cache_mem_filled, - &num_of_cache_entries); - - if (ret < 0) - return ret; - - crat_table->length += cache_mem_filled; - crat_table->total_entries += num_of_cache_entries; - avail_size -= cache_mem_filled; - /* Fill in Subtype: IO_LINKS * Only direct links are added here which is Link from GPU to * its NUMA node. Indirect links are added by userspace. */ sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr + - cache_mem_filled); + sub_type_hdr->length); ret = kfd_fill_gpu_direct_io_link_to_cpu(&avail_size, kdev, (struct crat_subtype_iolink *)sub_type_hdr, proximity_domain); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h index 22893ff7b9a1a..8d1e8ba58dee8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h @@ -295,6 +295,18 @@ struct crat_subtype_generic { struct kfd_dev; +/* Static table to describe GPU Cache information */ +struct kfd_gpu_cache_info { + uint32_t cache_size; + uint32_t cache_level; + uint32_t flags; + /* Indicates how many Compute Units share this cache + * within a SA. Value = 1 indicates the cache is not shared + */ + uint32_t num_cu_shared; +}; +int kfd_get_gpu_cache_info(struct kfd_dev *kdev, struct kfd_gpu_cache_info **pcache_info); + int kfd_create_crat_image_acpi(void **crat_image, size_t *size); void kfd_destroy_crat_image(void *crat_image); int kfd_parse_crat_table(void *crat_image, struct list_head *device_list, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index ec8c9c10d3488..1d9b90d979c42 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -364,7 +364,6 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, /* Making sure that the buffer is an empty string */ buffer[0] = 0; - cache = container_of(attr, struct kfd_cache_properties, attr); if (cache->gpu && kfd_devcgroup_check_permission(cache->gpu)) return -EPERM; @@ -379,12 +378,13 @@ static ssize_t kfd_cache_show(struct kobject *kobj, struct attribute *attr, sysfs_show_32bit_prop(buffer, offs, "association", cache->cache_assoc); sysfs_show_32bit_prop(buffer, offs, "latency", cache->cache_latency); sysfs_show_32bit_prop(buffer, offs, "type", cache->cache_type); + offs += snprintf(buffer+offs, PAGE_SIZE-offs, "sibling_map "); - for (i = 0; i < CRAT_SIBLINGMAP_SIZE; i++) + for (i = 0; i < cache->sibling_map_size; i++) for (j = 0; j < sizeof(cache->sibling_map[0])*8; j++) /* Check each bit */ offs += snprintf(buffer+offs, PAGE_SIZE-offs, "%d,", - (cache->sibling_map[i] >> j) & 1); + (cache->sibling_map[i] >> j) & 1); /* Replace the last "," with end of line */ buffer[offs-1] = '\n'; @@ -1197,7 +1197,6 @@ static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) struct kfd_iolink_properties *iolink; struct kfd_iolink_properties *p2plink; - down_write(&topology_lock); list_for_each_entry(dev, &topology_device_list, list) { /* Discrete GPUs need their own topology device list * entries. Don't assign them to CPU/APU nodes. @@ -1221,7 +1220,6 @@ static struct kfd_topology_device *kfd_assign_gpu(struct kfd_dev *gpu) break; } } - up_write(&topology_lock); return out_dev; } @@ -1592,6 +1590,221 @@ static int kfd_dev_create_p2p_links(void) return ret; } + +/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ +static int fill_in_l1_pcache(struct kfd_cache_properties **props_ext, + struct kfd_gpu_cache_info *pcache_info, + struct kfd_cu_info *cu_info, + int cu_bitmask, + int cache_type, unsigned int cu_processor_id, + int cu_block) +{ + unsigned int cu_sibling_map_mask; + int first_active_cu; + struct kfd_cache_properties *pcache = NULL; + + cu_sibling_map_mask = cu_bitmask; + cu_sibling_map_mask >>= cu_block; + cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); + first_active_cu = ffs(cu_sibling_map_mask); + + /* CU could be inactive. In case of shared cache find the first active + * CU. and incase of non-shared cache check if the CU is inactive. If + * inactive active skip it + */ + if (first_active_cu) { + pcache = kfd_alloc_struct(pcache); + if (!pcache) + return -ENOMEM; + + memset(pcache, 0, sizeof(struct kfd_cache_properties)); + pcache->processor_id_low = cu_processor_id + (first_active_cu - 1); + pcache->cache_level = pcache_info[cache_type].cache_level; + pcache->cache_size = pcache_info[cache_type].cache_size; + + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_DATA; + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_CPU; + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_HSACU; + + /* Sibling map is w.r.t processor_id_low, so shift out + * inactive CU + */ + cu_sibling_map_mask = + cu_sibling_map_mask >> (first_active_cu - 1); + + pcache->sibling_map[0] = (uint8_t)(cu_sibling_map_mask & 0xFF); + pcache->sibling_map[1] = + (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); + pcache->sibling_map[2] = + (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); + pcache->sibling_map[3] = + (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); + + pcache->sibling_map_size = 4; + *props_ext = pcache; + + return 0; + } + return 1; +} + +/* Helper function. See kfd_fill_gpu_cache_info for parameter description */ +static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext, + struct kfd_gpu_cache_info *pcache_info, + struct kfd_cu_info *cu_info, + int cache_type, unsigned int cu_processor_id) +{ + unsigned int cu_sibling_map_mask; + int first_active_cu; + int i, j, k; + struct kfd_cache_properties *pcache = NULL; + + cu_sibling_map_mask = cu_info->cu_bitmap[0][0]; + cu_sibling_map_mask &= + ((1 << pcache_info[cache_type].num_cu_shared) - 1); + first_active_cu = ffs(cu_sibling_map_mask); + + /* CU could be inactive. In case of shared cache find the first active + * CU. and incase of non-shared cache check if the CU is inactive. If + * inactive active skip it + */ + if (first_active_cu) { + pcache = kfd_alloc_struct(pcache); + if (!pcache) + return -ENOMEM; + + memset(pcache, 0, sizeof(struct kfd_cache_properties)); + pcache->processor_id_low = cu_processor_id + + (first_active_cu - 1); + pcache->cache_level = pcache_info[cache_type].cache_level; + pcache->cache_size = pcache_info[cache_type].cache_size; + + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_DATA_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_DATA; + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_INST_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_INSTRUCTION; + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_CPU_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_CPU; + if (pcache_info[cache_type].flags & CRAT_CACHE_FLAGS_SIMD_CACHE) + pcache->cache_type |= HSA_CACHE_TYPE_HSACU; + + /* Sibling map is w.r.t processor_id_low, so shift out + * inactive CU + */ + cu_sibling_map_mask = cu_sibling_map_mask >> (first_active_cu - 1); + k = 0; + + for (i = 0; i < cu_info->num_shader_engines; i++) { + for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) { + pcache->sibling_map[k] = (uint8_t)(cu_sibling_map_mask & 0xFF); + pcache->sibling_map[k+1] = (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF); + pcache->sibling_map[k+2] = (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF); + pcache->sibling_map[k+3] = (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF); + k += 4; + + cu_sibling_map_mask = cu_info->cu_bitmap[i % 4][j + i / 4]; + cu_sibling_map_mask &= ((1 << pcache_info[cache_type].num_cu_shared) - 1); + } + } + pcache->sibling_map_size = k; + *props_ext = pcache; + return 0; + } + return 1; +} + +#define KFD_MAX_CACHE_TYPES 6 + +/* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info + * tables + */ +void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_dev *kdev) +{ + struct kfd_gpu_cache_info *pcache_info = NULL; + int i, j, k; + int ct = 0; + unsigned int cu_processor_id; + int ret; + unsigned int num_cu_shared; + struct kfd_cu_info cu_info; + struct kfd_cu_info *pcu_info; + int gpu_processor_id; + struct kfd_cache_properties *props_ext; + int num_of_entries = 0; + int num_of_cache_types = 0; + struct kfd_gpu_cache_info cache_info[KFD_MAX_CACHE_TYPES]; + + amdgpu_amdkfd_get_cu_info(kdev->adev, &cu_info); + pcu_info = &cu_info; + + gpu_processor_id = dev->node_props.simd_id_base; + + pcache_info = cache_info; + num_of_cache_types = kfd_get_gpu_cache_info(kdev, &pcache_info); + if (!num_of_cache_types) { + pr_warn("no cache info found\n"); + return; + } + + /* For each type of cache listed in the kfd_gpu_cache_info table, + * go through all available Compute Units. + * The [i,j,k] loop will + * if kfd_gpu_cache_info.num_cu_shared = 1 + * will parse through all available CU + * If (kfd_gpu_cache_info.num_cu_shared != 1) + * then it will consider only one CU from + * the shared unit + */ + for (ct = 0; ct < num_of_cache_types; ct++) { + cu_processor_id = gpu_processor_id; + if (pcache_info[ct].cache_level == 1) { + for (i = 0; i < pcu_info->num_shader_engines; i++) { + for (j = 0; j < pcu_info->num_shader_arrays_per_engine; j++) { + for (k = 0; k < pcu_info->num_cu_per_sh; k += pcache_info[ct].num_cu_shared) { + + ret = fill_in_l1_pcache(&props_ext, pcache_info, pcu_info, + pcu_info->cu_bitmap[i % 4][j + i / 4], ct, + cu_processor_id, k); + + if (ret < 0) + break; + + if (!ret) { + num_of_entries++; + list_add_tail(&props_ext->list, &dev->cache_props); + } + + /* Move to next CU block */ + num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <= + pcu_info->num_cu_per_sh) ? + pcache_info[ct].num_cu_shared : + (pcu_info->num_cu_per_sh - k); + cu_processor_id += num_cu_shared; + } + } + } + } else { + ret = fill_in_l2_l3_pcache(&props_ext, pcache_info, + pcu_info, ct, cu_processor_id); + + if (ret < 0) + break; + + if (!ret) { + num_of_entries++; + list_add_tail(&props_ext->list, &dev->cache_props); + } + } + } + dev->node_props.caches_count += num_of_entries; + pr_debug("Added [%d] GPU cache entries\n", num_of_entries); +} + int kfd_topology_add_device(struct kfd_dev *gpu) { uint32_t gpu_id; @@ -1616,9 +1829,9 @@ int kfd_topology_add_device(struct kfd_dev *gpu) * CRAT to create a new topology device. Once created assign the gpu to * that topology device */ + down_write(&topology_lock); dev = kfd_assign_gpu(gpu); if (!dev) { - down_write(&topology_lock); proximity_domain = ++topology_crat_proximity_domain; res = kfd_create_crat_image_virtual(&crat_image, &image_size, @@ -1630,6 +1843,7 @@ int kfd_topology_add_device(struct kfd_dev *gpu) topology_crat_proximity_domain--; return res; } + res = kfd_parse_crat_table(crat_image, &temp_topology_device_list, proximity_domain); @@ -1643,23 +1857,28 @@ int kfd_topology_add_device(struct kfd_dev *gpu) kfd_topology_update_device_list(&temp_topology_device_list, &topology_device_list); + dev = kfd_assign_gpu(gpu); + if (WARN_ON(!dev)) { + res = -ENODEV; + goto err; + } + + /* Fill the cache affinity information here for the GPUs + * using VCRAT + */ + kfd_fill_cache_non_crat_info(dev, gpu); + /* Update the SYSFS tree, since we added another topology * device */ res = kfd_topology_update_sysfs(); - up_write(&topology_lock); - if (!res) sys_props.generation_count++; else pr_err("Failed to update GPU (ID: 0x%x) to sysfs topology. res=%d\n", gpu_id, res); - dev = kfd_assign_gpu(gpu); - if (WARN_ON(!dev)) { - res = -ENODEV; - goto err; - } } + up_write(&topology_lock); dev->gpu_id = gpu_id; gpu->id = gpu_id; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h index 90c069fbd581e..fca30d00a9bb3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.h @@ -80,6 +80,8 @@ struct kfd_mem_properties { struct attribute attr; }; +#define CACHE_SIBLINGMAP_SIZE 64 + struct kfd_cache_properties { struct list_head list; uint32_t processor_id_low; @@ -90,10 +92,11 @@ struct kfd_cache_properties { uint32_t cache_assoc; uint32_t cache_latency; uint32_t cache_type; - uint8_t sibling_map[CRAT_SIBLINGMAP_SIZE]; + uint8_t sibling_map[CACHE_SIBLINGMAP_SIZE]; struct kfd_dev *gpu; struct kobject *kobj; struct attribute attr; + uint32_t sibling_map_size; }; struct kfd_iolink_properties { -- GitLab From 5cefe31b2a34005c83ff45834e21a0578defcbe4 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Thu, 3 Nov 2022 11:36:27 +0800 Subject: [PATCH 1060/1927] drm/amd/amdgpu: temporary workaround to skip ras error for gc_v11_0_3 temporary workaround to skip ras error for gc_v11_0_3 until IFWI release later Signed-off-by: Kenneth Feng Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index d799359565b85..96b9054c790c1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -4689,10 +4689,10 @@ static int gfx_v11_0_ras_late_init(void *handle) ret = amdgpu_ras_feature_enable(adev, gfx_common_if, true); if (ret) - dev_err(adev->dev, "Failed to enable gfx11 ras feature\n"); + dev_warn(adev->dev, "Failed to enable gfx11 ras feature\n"); kfree(gfx_common_if); - return ret; + return 0; } static int gfx_v11_0_late_init(void *handle) -- GitLab From ec4927d463cbf385be212af1ade9f5a6f1474d4f Mon Sep 17 00:00:00 2001 From: Victor Zhao Date: Wed, 26 Oct 2022 11:48:50 +0800 Subject: [PATCH 1061/1927] drm/amdgpu: fix for suspend/resume sequence under sriov - clear kiq ring after suspend/resume under sriov to aviod kiq ring test failure - update irq after resume to fix kiq interrput loss Signed-off-by: Victor Zhao Acked-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 ++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 2 ++ drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c | 2 ++ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 2 ++ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 2 ++ 5 files changed, 10 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 522820eeaa596..5b9f992e46074 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -4197,6 +4197,8 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon) } /* Make sure IB tests flushed */ + if (amdgpu_sriov_vf(adev)) + amdgpu_irq_gpu_reset_resume_helper(adev); flush_delayed_work(&adev->delayed_init_work); if (adev->in_s0ix) { diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 7853d3ca58cf7..49d34c7bbf20f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -6909,6 +6909,8 @@ static int gfx_v10_0_kiq_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); } else { memset((void *)mqd, 0, sizeof(*mqd)); + if (amdgpu_sriov_vf(adev) && adev->in_suspend) + amdgpu_ring_clear_ring(ring); mutex_lock(&adev->srbm_mutex); nv_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index 96b9054c790c1..9447999a3a487 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -4004,6 +4004,8 @@ static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) mutex_unlock(&adev->srbm_mutex); } else { memset((void *)mqd, 0, sizeof(*mqd)); + if (amdgpu_sriov_vf(adev) && adev->in_suspend) + amdgpu_ring_clear_ring(ring); mutex_lock(&adev->srbm_mutex); soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); amdgpu_ring_init_mqd(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 7f0b18b0d4c48..d47135606e3ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -4643,6 +4643,8 @@ static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; + if (amdgpu_sriov_vf(adev) && adev->in_suspend) + amdgpu_ring_clear_ring(ring); mutex_lock(&adev->srbm_mutex); vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); gfx_v8_0_mqd_init(ring); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index ab30c18decfbe..676832da75eb1 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -3581,6 +3581,8 @@ static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; + if (amdgpu_sriov_vf(adev) && adev->in_suspend) + amdgpu_ring_clear_ring(ring); mutex_lock(&adev->srbm_mutex); soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); gfx_v9_0_mqd_init(ring); -- GitLab From 79b72db63392d35a4dbcd4743b95cb668691f790 Mon Sep 17 00:00:00 2001 From: Ao Zhong Date: Mon, 31 Oct 2022 20:37:40 +0100 Subject: [PATCH 1062/1927] drm/amd/display: add DCN support for ARM64 After moving all FPU code to the DML folder, we can enable DCN support for the ARM64 platform. Remove the -mgeneral-regs-only CFLAG from the code in the DML folder that needs to use hardware FPU, and add a control mechanism for ARM Neon. Reviewed-by: Rodrigo Siqueira Signed-off-by: Ao Zhong Signed-off-by: Rodrigo Siqueira Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/Kconfig | 3 ++- .../gpu/drm/amd/display/amdgpu_dm/dc_fpu.c | 6 ++++++ drivers/gpu/drm/amd/display/dc/dml/Makefile | 20 +++++++++++++++---- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig index 6925e0280dbe6..1cdb379a90d73 100644 --- a/drivers/gpu/drm/amd/display/Kconfig +++ b/drivers/gpu/drm/amd/display/Kconfig @@ -6,7 +6,8 @@ config DRM_AMD_DC bool "AMD DC - Enable new display engine" default y select SND_HDA_COMPONENT if SND_HDA_CORE - select DRM_AMD_DC_DCN if (X86 || PPC_LONG_DOUBLE_128) + # !CC_IS_CLANG: https://github.com/ClangBuiltLinux/linux/issues/1752 + select DRM_AMD_DC_DCN if (X86 || PPC_LONG_DOUBLE_128 || (ARM64 && KERNEL_MODE_NEON && !CC_IS_CLANG)) help Choose this option if you want to use the new display engine support for AMDGPU. This adds required support for Vega and diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c index ab0c6d1910382..1743ca0a36419 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/dc_fpu.c @@ -31,6 +31,8 @@ #elif defined(CONFIG_PPC64) #include #include +#elif defined(CONFIG_ARM64) +#include #endif /** @@ -99,6 +101,8 @@ void dc_fpu_begin(const char *function_name, const int line) preempt_disable(); enable_kernel_fp(); } +#elif defined(CONFIG_ARM64) + kernel_neon_begin(); #endif } @@ -136,6 +140,8 @@ void dc_fpu_end(const char *function_name, const int line) disable_kernel_fp(); preempt_enable(); } +#elif defined(CONFIG_ARM64) + kernel_neon_end(); #endif } diff --git a/drivers/gpu/drm/amd/display/dc/dml/Makefile b/drivers/gpu/drm/amd/display/dc/dml/Makefile index ca7d240006213..0ecea87cf48f2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/Makefile +++ b/drivers/gpu/drm/amd/display/dc/dml/Makefile @@ -33,6 +33,10 @@ ifdef CONFIG_PPC64 dml_ccflags := -mhard-float -maltivec endif +ifdef CONFIG_ARM64 +dml_rcflags := -mgeneral-regs-only +endif + ifdef CONFIG_CC_IS_GCC ifneq ($(call gcc-min-version, 70100),y) IS_OLD_GCC = 1 @@ -55,8 +59,6 @@ frame_warn_flag := -Wframe-larger-than=2048 endif CFLAGS_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_ccflags) - -ifdef CONFIG_DRM_AMD_DC_DCN CFLAGS_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn10/dcn10_fpu.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/dcn20/dcn20_fpu.o := $(dml_ccflags) @@ -88,7 +90,6 @@ CFLAGS_$(AMDDALPATH)/dc/dml/calcs/dcn_calcs.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/calcs/dcn_calc_auto.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/calcs/dcn_calc_math.o := $(dml_ccflags) -Wno-tautological-compare CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_vba.o := $(dml_rcflags) -CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn2x/dcn2x.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_rq_dlg_calc_20.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/display_mode_vba_20v2.o := $(dml_rcflags) @@ -105,7 +106,18 @@ CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn32/display_mode_vba_util_32.o := $(dml_rcf CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn301/dcn301_fpu.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/display_mode_lib.o := $(dml_rcflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dsc/rc_calc_fpu.o := $(dml_rcflags) -endif +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn10/dcn10_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn20/dcn20_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn314/display_mode_vba_314.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn314/display_rq_dlg_calc_314.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn314/dcn314_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn30/dcn30_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn32/dcn32_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn321/dcn321_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn31/dcn31_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn302/dcn302_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dcn303/dcn303_fpu.o := $(dml_rcflags) +CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/calcs/dcn_calc_math.o := $(dml_rcflags) CFLAGS_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_ccflags) CFLAGS_$(AMDDALPATH)/dc/dml/display_rq_dlg_helpers.o := $(dml_ccflags) CFLAGS_REMOVE_$(AMDDALPATH)/dc/dml/dml1_display_rq_dlg_calc.o := $(dml_rcflags) -- GitLab From fcf00f8d29f2fc6bf00531a1447be28b99073cc3 Mon Sep 17 00:00:00 2001 From: Xiaogang Chen Date: Fri, 28 Oct 2022 19:13:31 -0500 Subject: [PATCH 1063/1927] drm/amdkfd: Remove skiping userptr buffer mapping when mmu notifier marks it as invalid mmu notifier does not always hold mm->sem during call back. That causes a race condition between kfd userprt buffer mapping and mmu notifier which leds to gpu shadder or SDMA access userptr buffer before it has been mapped to gpu VM. Always map userptr buffer to avoid that though it may make some userprt buffers mapped two times. Suggested-by: Felix Kuehling Signed-off-by: Xiaogang Chen Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index da9d475d7ef2d..ba72a910d0d59 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1907,16 +1907,6 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( */ mutex_lock(&mem->process_info->lock); - /* Lock mmap-sem. If we find an invalid userptr BO, we can be - * sure that the MMU notifier is no longer running - * concurrently and the queues are actually stopped - */ - if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { - mmap_write_lock(current->mm); - is_invalid_userptr = atomic_read(&mem->invalid); - mmap_write_unlock(current->mm); - } - mutex_lock(&mem->lock); domain = mem->domain; -- GitLab From 7beade0dd41d42d797ccb7791b134a77fcebf35b Mon Sep 17 00:00:00 2001 From: Tony Luck Date: Thu, 3 Nov 2022 13:33:10 -0700 Subject: [PATCH 1064/1927] x86/cpu: Add several Intel server CPU model numbers These servers are all on the public versions of the roadmap. The model numbers for Grand Ridge, Granite Rapids, and Sierra Forest were included in the September 2022 edition of the Instruction Set Extensions document. Signed-off-by: Tony Luck Signed-off-by: Borislav Petkov Acked-by: Dave Hansen Link: https://lore.kernel.org/r/20221103203310.5058-1-tony.luck@intel.com --- arch/x86/include/asm/intel-family.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index 5d75fe2293421..347707d459c67 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -107,6 +107,11 @@ #define INTEL_FAM6_SAPPHIRERAPIDS_X 0x8F /* Golden Cove */ +#define INTEL_FAM6_EMERALDRAPIDS_X 0xCF + +#define INTEL_FAM6_GRANITERAPIDS_X 0xAD +#define INTEL_FAM6_GRANITERAPIDS_D 0xAE + #define INTEL_FAM6_ALDERLAKE 0x97 /* Golden Cove / Gracemont */ #define INTEL_FAM6_ALDERLAKE_L 0x9A /* Golden Cove / Gracemont */ #define INTEL_FAM6_ALDERLAKE_N 0xBE @@ -118,7 +123,7 @@ #define INTEL_FAM6_METEORLAKE 0xAC #define INTEL_FAM6_METEORLAKE_L 0xAA -/* "Small Core" Processors (Atom) */ +/* "Small Core" Processors (Atom/E-Core) */ #define INTEL_FAM6_ATOM_BONNELL 0x1C /* Diamondville, Pineview */ #define INTEL_FAM6_ATOM_BONNELL_MID 0x26 /* Silverthorne, Lincroft */ @@ -145,6 +150,10 @@ #define INTEL_FAM6_ATOM_TREMONT 0x96 /* Elkhart Lake */ #define INTEL_FAM6_ATOM_TREMONT_L 0x9C /* Jasper Lake */ +#define INTEL_FAM6_SIERRAFOREST_X 0xAF + +#define INTEL_FAM6_GRANDRIDGE 0xB6 + /* Xeon Phi */ #define INTEL_FAM6_XEON_PHI_KNL 0x57 /* Knights Landing */ -- GitLab From de51de9672a17e242ebe3727b5e6ec0f2b4c1ab4 Mon Sep 17 00:00:00 2001 From: John Harrison Date: Wed, 2 Nov 2022 12:21:08 -0700 Subject: [PATCH 1065/1927] drm/i915/guc: Properly initialise kernel contexts If a context has already been registered prior to first submission then context init code was not being called. The noticeable effect of that was the scheduling priority was left at zero (meaning super high priority) instead of being set to normal. This would occur with kernel contexts at start of day as they are manually pinned up front rather than on first submission. So add a call to initialise those when they are pinned. Signed-off-by: John Harrison Reviewed-by: Daniele Ceraolo Spurio Link: https://patchwork.freedesktop.org/patch/msgid/20221102192109.2492625-2-John.C.Harrison@Intel.com --- drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 4dbdac8002e32..8fbc363564607 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -4111,6 +4111,9 @@ static inline void guc_kernel_context_pin(struct intel_guc *guc, if (context_guc_id_invalid(ce)) pin_guc_id(guc, ce); + if (!test_bit(CONTEXT_GUC_INIT, &ce->flags)) + guc_context_init(ce); + try_context_registration(ce, true); } -- GitLab From 178b8a3668bd63b40303d9dcb17ad58cf4b44007 Mon Sep 17 00:00:00 2001 From: John Harrison Date: Wed, 2 Nov 2022 12:21:09 -0700 Subject: [PATCH 1066/1927] drm/i915/guc: Don't deadlock busyness stats vs reset The engine busyness stats has a worker function to do things like 64bit extend the 32bit hardware counters. The GuC's reset prepare function flushes out this worker function to ensure no corruption happens during the reset. Unforunately, the worker function has an infinite wait for active resets to finish before doing its work. Thus a deadlock would occur if the worker function had actually started just as the reset starts. The function being used to lock the reset-in-progress mutex is called intel_gt_reset_trylock(). However, as noted it does not follow standard 'trylock' conventions and exit if already locked. So rename the current _trylock function to intel_gt_reset_lock_interruptible(), which is the behaviour it actually provides. In addition, add a new implementation of _trylock and call that from the busyness stats worker instead. v2: Rename existing trylock to interruptible rather than trying to preserve the existing (confusing) naming scheme (review comments from Tvrtko). Signed-off-by: John Harrison Reviewed-by: Umesh Nerlige Ramappa Link: https://patchwork.freedesktop.org/patch/msgid/20221102192109.2492625-3-John.C.Harrison@Intel.com --- drivers/gpu/drm/i915/gem/i915_gem_mman.c | 2 +- drivers/gpu/drm/i915/gt/intel_reset.c | 18 ++++++++++++++++-- drivers/gpu/drm/i915/gt/intel_reset.h | 1 + .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 4 +++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index e63329bc80659..c29efdef8313a 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -330,7 +330,7 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) if (ret) goto err_rpm; - ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu); + ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu); if (ret) goto err_pages; diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c index 3159df6cdd492..24736ebee17c2 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.c +++ b/drivers/gpu/drm/i915/gt/intel_reset.c @@ -1407,15 +1407,19 @@ void intel_gt_handle_error(struct intel_gt *gt, intel_runtime_pm_put(gt->uncore->rpm, wakeref); } -int intel_gt_reset_trylock(struct intel_gt *gt, int *srcu) +static int _intel_gt_reset_lock(struct intel_gt *gt, int *srcu, bool retry) { might_lock(>->reset.backoff_srcu); - might_sleep(); + if (retry) + might_sleep(); rcu_read_lock(); while (test_bit(I915_RESET_BACKOFF, >->reset.flags)) { rcu_read_unlock(); + if (!retry) + return -EBUSY; + if (wait_event_interruptible(gt->reset.queue, !test_bit(I915_RESET_BACKOFF, >->reset.flags))) @@ -1429,6 +1433,16 @@ int intel_gt_reset_trylock(struct intel_gt *gt, int *srcu) return 0; } +int intel_gt_reset_trylock(struct intel_gt *gt, int *srcu) +{ + return _intel_gt_reset_lock(gt, srcu, false); +} + +int intel_gt_reset_lock_interruptible(struct intel_gt *gt, int *srcu) +{ + return _intel_gt_reset_lock(gt, srcu, true); +} + void intel_gt_reset_unlock(struct intel_gt *gt, int tag) __releases(>->reset.backoff_srcu) { diff --git a/drivers/gpu/drm/i915/gt/intel_reset.h b/drivers/gpu/drm/i915/gt/intel_reset.h index adc734e673870..25c975b6e8fc0 100644 --- a/drivers/gpu/drm/i915/gt/intel_reset.h +++ b/drivers/gpu/drm/i915/gt/intel_reset.h @@ -39,6 +39,7 @@ int __intel_engine_reset_bh(struct intel_engine_cs *engine, void __i915_request_reset(struct i915_request *rq, bool guilty); int __must_check intel_gt_reset_trylock(struct intel_gt *gt, int *srcu); +int __must_check intel_gt_reset_lock_interruptible(struct intel_gt *gt, int *srcu); void intel_gt_reset_unlock(struct intel_gt *gt, int tag); void intel_gt_set_wedged(struct intel_gt *gt); diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index 8fbc363564607..412c2624e1190 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1401,7 +1401,9 @@ static void guc_timestamp_ping(struct work_struct *wrk) /* * Synchronize with gt reset to make sure the worker does not - * corrupt the engine/guc stats. + * corrupt the engine/guc stats. NB: can't actually block waiting + * for a reset to complete as the reset requires flushing out + * this worker thread if started. So waiting would deadlock. */ ret = intel_gt_reset_trylock(gt, &srcu); if (ret) -- GitLab From 18acb7fac22ff7b36c7ea5a76b12996e7b7dbaba Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 3 Nov 2022 13:00:13 +0100 Subject: [PATCH 1067/1927] bpf: Revert ("Fix dispatcher patchable function entry to 5 bytes nop") MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because __attribute__((patchable_function_entry)) is only available since GCC-8 this solution fails to build on the minimum required GCC version. Undo these changes so we might try again -- without cluttering up the patches with too many changes. This is an almost complete revert of: dbe69b299884 ("bpf: Fix dispatcher patchable function entry to 5 bytes nop") ceea991a019c ("bpf: Move bpf_dispatcher function out of ftrace locations") (notably the arch/x86/Kconfig hunk is kept). Reported-by: David Laight Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Daniel Borkmann Tested-by: Björn Töpel Tested-by: Jiri Olsa Acked-by: Björn Töpel Acked-by: Jiri Olsa Link: https://lkml.kernel.org/r/439d8dc735bb4858875377df67f1b29a@AcuMS.aculab.com Link: https://lore.kernel.org/bpf/20221103120647.728830733@infradead.org --- arch/x86/net/bpf_jit_comp.c | 13 ------------- include/linux/bpf.h | 21 +-------------------- kernel/bpf/dispatcher.c | 6 ------ 3 files changed, 1 insertion(+), 39 deletions(-) diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 00127abd89ee5..99620428ad785 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -389,18 +388,6 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, return ret; } -int __init bpf_arch_init_dispatcher_early(void *ip) -{ - const u8 *nop_insn = x86_nops[5]; - - if (is_endbr(*(u32 *)ip)) - ip += ENDBR_INSN_SIZE; - - if (memcmp(ip, nop_insn, X86_PATCH_SIZE)) - text_poke_early(ip, nop_insn, X86_PATCH_SIZE); - return 0; -} - int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, void *old_addr, void *new_addr) { diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0566705c1d4ea..5cd95716b441d 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -27,7 +27,6 @@ #include #include #include -#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -971,8 +970,6 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); -int __init bpf_arch_init_dispatcher_early(void *ip); - #define BPF_DISPATCHER_INIT(_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ .func = &_name##_func, \ @@ -986,21 +983,7 @@ int __init bpf_arch_init_dispatcher_early(void *ip); }, \ } -#define BPF_DISPATCHER_INIT_CALL(_name) \ - static int __init _name##_init(void) \ - { \ - return bpf_arch_init_dispatcher_early(_name##_func); \ - } \ - early_initcall(_name##_init) - -#ifdef CONFIG_X86_64 -#define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5))) -#else -#define BPF_DISPATCHER_ATTRIBUTES -#endif - #define DEFINE_BPF_DISPATCHER(name) \ - notrace BPF_DISPATCHER_ATTRIBUTES \ noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ @@ -1010,9 +993,7 @@ int __init bpf_arch_init_dispatcher_early(void *ip); } \ EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ struct bpf_dispatcher bpf_dispatcher_##name = \ - BPF_DISPATCHER_INIT(bpf_dispatcher_##name); \ - BPF_DISPATCHER_INIT_CALL(bpf_dispatcher_##name); - + BPF_DISPATCHER_INIT(bpf_dispatcher_##name); #define DECLARE_BPF_DISPATCHER(name) \ unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index 04f0a045dcaa6..fa64b80b8bcab 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -4,7 +4,6 @@ #include #include #include -#include /* The BPF dispatcher is a multiway branch code generator. The * dispatcher is a mechanism to avoid the performance penalty of an @@ -91,11 +90,6 @@ int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int n return -ENOTSUPP; } -int __weak __init bpf_arch_init_dispatcher_early(void *ip) -{ - return -ENOTSUPP; -} - static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf) { s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; -- GitLab From c86df29d11dfba27c0a1f5039cd6fe387fbf4239 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 3 Nov 2022 13:00:14 +0100 Subject: [PATCH 1068/1927] bpf: Convert BPF_DISPATCHER to use static_call() (not ftrace) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dispatcher function is currently abusing the ftrace __fentry__ call location for its own purposes -- this obviously gives trouble when the dispatcher and ftrace are both in use. A previous solution tried using __attribute__((patchable_function_entry())) which works, except it is GCC-8+ only, breaking the build on the earlier still supported compilers. Instead use static_call() -- which has its own annotations and does not conflict with ftrace -- to rewrite the dispatch function. By using: return static_call()(ctx, insni, bpf_func) you get a perfect forwarding tail call as function body (iow a single jmp instruction). By having the default static_call() target be bpf_dispatcher_nop_func() it retains the default behaviour (an indirect call to the argument function). Only once a dispatcher program is attached is the target rewritten to directly call the JIT'ed image. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Daniel Borkmann Tested-by: Björn Töpel Tested-by: Jiri Olsa Acked-by: Björn Töpel Acked-by: Jiri Olsa Link: https://lkml.kernel.org/r/Y1/oBlK0yFk5c/Im@hirez.programming.kicks-ass.net Link: https://lore.kernel.org/bpf/20221103120647.796772565@infradead.org --- include/linux/bpf.h | 39 ++++++++++++++++++++++++++++++++++++++- kernel/bpf/dispatcher.c | 22 ++++++++-------------- 2 files changed, 46 insertions(+), 15 deletions(-) diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 5cd95716b441d..74c6f449d81ed 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -27,6 +27,7 @@ #include #include #include +#include struct bpf_verifier_env; struct bpf_verifier_log; @@ -953,6 +954,10 @@ struct bpf_dispatcher { void *rw_image; u32 image_off; struct bpf_ksym ksym; +#ifdef CONFIG_HAVE_STATIC_CALL + struct static_call_key *sc_key; + void *sc_tramp; +#endif }; static __always_inline __nocfi unsigned int bpf_dispatcher_nop_func( @@ -970,6 +975,34 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); + +/* + * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn + * indirection with a direct call to the bpf program. If the architecture does + * not have STATIC_CALL, avoid a double-indirection. + */ +#ifdef CONFIG_HAVE_STATIC_CALL + +#define __BPF_DISPATCHER_SC_INIT(_name) \ + .sc_key = &STATIC_CALL_KEY(_name), \ + .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name), + +#define __BPF_DISPATCHER_SC(name) \ + DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func) + +#define __BPF_DISPATCHER_CALL(name) \ + static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func) + +#define __BPF_DISPATCHER_UPDATE(_d, _new) \ + __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new)) + +#else +#define __BPF_DISPATCHER_SC_INIT(name) +#define __BPF_DISPATCHER_SC(name) +#define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi) +#define __BPF_DISPATCHER_UPDATE(_d, _new) +#endif + #define BPF_DISPATCHER_INIT(_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ .func = &_name##_func, \ @@ -981,25 +1014,29 @@ int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_func .name = #_name, \ .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ }, \ + __BPF_DISPATCHER_SC_INIT(_name##_call) \ } #define DEFINE_BPF_DISPATCHER(name) \ + __BPF_DISPATCHER_SC(name); \ noinline __nocfi unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func) \ { \ - return bpf_func(ctx, insnsi); \ + return __BPF_DISPATCHER_CALL(name); \ } \ EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ struct bpf_dispatcher bpf_dispatcher_##name = \ BPF_DISPATCHER_INIT(bpf_dispatcher_##name); + #define DECLARE_BPF_DISPATCHER(name) \ unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func); \ extern struct bpf_dispatcher bpf_dispatcher_##name; + #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index fa64b80b8bcab..7dfb8d0d5202e 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -4,6 +4,7 @@ #include #include #include +#include /* The BPF dispatcher is a multiway branch code generator. The * dispatcher is a mechanism to avoid the performance penalty of an @@ -104,17 +105,11 @@ static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *b static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) { - void *old, *new, *tmp; - u32 noff; - int err; - - if (!prev_num_progs) { - old = NULL; - noff = 0; - } else { - old = d->image + d->image_off; + void *new, *tmp; + u32 noff = 0; + + if (prev_num_progs) noff = d->image_off ^ (PAGE_SIZE / 2); - } new = d->num_progs ? d->image + noff : NULL; tmp = d->num_progs ? d->rw_image + noff : NULL; @@ -128,11 +123,10 @@ static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) return; } - err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new); - if (err || !new) - return; + __BPF_DISPATCHER_UPDATE(d, new ?: &bpf_dispatcher_nop_func); - d->image_off = noff; + if (new) + d->image_off = noff; } void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, -- GitLab From a90accb358ae33ea982a35595573f7a045993f8b Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:30:24 -0700 Subject: [PATCH 1069/1927] cxl/region: Fix region HPA ordering validation Some regions may not have any address space allocated. Skip them when validating HPA order otherwise a crash like the following may result: devm_cxl_add_region: cxl_acpi cxl_acpi.0: decoder3.4: created region9 BUG: kernel NULL pointer dereference, address: 0000000000000000 [..] RIP: 0010:store_targetN+0x655/0x1740 [cxl_core] [..] Call Trace: kernfs_fop_write_iter+0x144/0x200 vfs_write+0x24a/0x4d0 ksys_write+0x69/0xf0 do_syscall_64+0x3a/0x90 store_targetN+0x655/0x1740: alloc_region_ref at drivers/cxl/core/region.c:676 (inlined by) cxl_port_attach_region at drivers/cxl/core/region.c:850 (inlined by) cxl_region_attach at drivers/cxl/core/region.c:1290 (inlined by) attach_target at drivers/cxl/core/region.c:1410 (inlined by) store_targetN at drivers/cxl/core/region.c:1453 Cc: Fixes: 384e624bb211 ("cxl/region: Attach endpoint decoders") Reviewed-by: Vishal Verma Reviewed-by: Dave Jiang Link: https://lore.kernel.org/r/166752182461.947915.497032805239915067.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/region.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index bb6f4fc84a3ff..d26ca7a6beaee 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -658,6 +658,9 @@ static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port, xa_for_each(&port->regions, index, iter) { struct cxl_region_params *ip = &iter->region->params; + if (!ip->res) + continue; + if (ip->res->start > p->res->start) { dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n", -- GitLab From 0d9e734018d70cecf79e2e4c6082167160a0f13f Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:30:30 -0700 Subject: [PATCH 1070/1927] cxl/region: Fix cxl_region leak, cleanup targets at region delete When a region is deleted any targets that have been previously assigned to that region hold references to it. Trigger those references to drop by detaching all targets at unregister_region() time. Otherwise that region object will leak as userspace has lost the ability to detach targets once region sysfs is torn down. Cc: Fixes: b9686e8c8e39 ("cxl/region: Enable the assignment of endpoint decoders to regions") Reviewed-by: Dave Jiang Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166752183055.947915.17681995648556534844.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/region.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index d26ca7a6beaee..c52465e09f268 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1557,8 +1557,19 @@ static struct cxl_region *to_cxl_region(struct device *dev) static void unregister_region(void *dev) { struct cxl_region *cxlr = to_cxl_region(dev); + struct cxl_region_params *p = &cxlr->params; + int i; device_del(dev); + + /* + * Now that region sysfs is shutdown, the parameter block is now + * read-only, so no need to hold the region rwsem to access the + * region parameters. + */ + for (i = 0; i < p->interleave_ways; i++) + detach_target(cxlr, i); + cxl_region_iomem_release(cxlr); put_device(dev); } -- GitLab From 4d07ae22e79ebc2d7528bbc69daa53b86981cb3a Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:30:36 -0700 Subject: [PATCH 1071/1927] cxl/pmem: Fix cxl_pmem_region and cxl_memdev leak When a cxl_nvdimm object goes through a ->remove() event (device physically removed, nvdimm-bridge disabled, or nvdimm device disabled), then any associated regions must also be disabled. As highlighted by the cxl-create-region.sh test [1], a single device may host multiple regions, but the driver was only tracking one region at a time. This leads to a situation where only the last enabled region per nvdimm device is cleaned up properly. Other regions are leaked, and this also causes cxl_memdev reference leaks. Fix the tracking by allowing cxl_nvdimm objects to track multiple region associations. Cc: Link: https://github.com/pmem/ndctl/blob/main/test/cxl-create-region.sh [1] Reported-by: Vishal Verma Fixes: 04ad63f086d1 ("cxl/region: Introduce cxl_pmem_region objects") Reviewed-by: Dave Jiang Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166752183647.947915.2045230911503793901.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/pmem.c | 2 + drivers/cxl/cxl.h | 2 +- drivers/cxl/pmem.c | 101 ++++++++++++++++++++++++++-------------- 3 files changed, 68 insertions(+), 37 deletions(-) diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index 1d12a8206444e..36aa5070d9024 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -188,6 +188,7 @@ static void cxl_nvdimm_release(struct device *dev) { struct cxl_nvdimm *cxl_nvd = to_cxl_nvdimm(dev); + xa_destroy(&cxl_nvd->pmem_regions); kfree(cxl_nvd); } @@ -230,6 +231,7 @@ static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_memdev *cxlmd) dev = &cxl_nvd->dev; cxl_nvd->cxlmd = cxlmd; + xa_init(&cxl_nvd->pmem_regions); device_initialize(dev); lockdep_set_class(&dev->mutex, &cxl_nvdimm_key); device_set_pm_not_required(dev); diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index f680450f0b16c..1164ad49f3d3a 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -423,7 +423,7 @@ struct cxl_nvdimm { struct device dev; struct cxl_memdev *cxlmd; struct cxl_nvdimm_bridge *bridge; - struct cxl_pmem_region *region; + struct xarray pmem_regions; }; struct cxl_pmem_region_mapping { diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 0bac05d804bc5..4c627d67281a1 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -30,17 +30,20 @@ static void unregister_nvdimm(void *nvdimm) struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_nvdimm_bridge *cxl_nvb = cxl_nvd->bridge; struct cxl_pmem_region *cxlr_pmem; + unsigned long index; device_lock(&cxl_nvb->dev); - cxlr_pmem = cxl_nvd->region; dev_set_drvdata(&cxl_nvd->dev, NULL); - cxl_nvd->region = NULL; - device_unlock(&cxl_nvb->dev); + xa_for_each(&cxl_nvd->pmem_regions, index, cxlr_pmem) { + get_device(&cxlr_pmem->dev); + device_unlock(&cxl_nvb->dev); - if (cxlr_pmem) { device_release_driver(&cxlr_pmem->dev); put_device(&cxlr_pmem->dev); + + device_lock(&cxl_nvb->dev); } + device_unlock(&cxl_nvb->dev); nvdimm_delete(nvdimm); cxl_nvd->bridge = NULL; @@ -366,25 +369,49 @@ static int match_cxl_nvdimm(struct device *dev, void *data) static void unregister_nvdimm_region(void *nd_region) { - struct cxl_nvdimm_bridge *cxl_nvb; - struct cxl_pmem_region *cxlr_pmem; + nvdimm_region_delete(nd_region); +} + +static int cxl_nvdimm_add_region(struct cxl_nvdimm *cxl_nvd, + struct cxl_pmem_region *cxlr_pmem) +{ + int rc; + + rc = xa_insert(&cxl_nvd->pmem_regions, (unsigned long)cxlr_pmem, + cxlr_pmem, GFP_KERNEL); + if (rc) + return rc; + + get_device(&cxlr_pmem->dev); + return 0; +} + +static void cxl_nvdimm_del_region(struct cxl_nvdimm *cxl_nvd, + struct cxl_pmem_region *cxlr_pmem) +{ + /* + * It is possible this is called without a corresponding + * cxl_nvdimm_add_region for @cxlr_pmem + */ + cxlr_pmem = xa_erase(&cxl_nvd->pmem_regions, (unsigned long)cxlr_pmem); + if (cxlr_pmem) + put_device(&cxlr_pmem->dev); +} + +static void release_mappings(void *data) +{ int i; + struct cxl_pmem_region *cxlr_pmem = data; + struct cxl_nvdimm_bridge *cxl_nvb = cxlr_pmem->bridge; - cxlr_pmem = nd_region_provider_data(nd_region); - cxl_nvb = cxlr_pmem->bridge; device_lock(&cxl_nvb->dev); for (i = 0; i < cxlr_pmem->nr_mappings; i++) { struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; struct cxl_nvdimm *cxl_nvd = m->cxl_nvd; - if (cxl_nvd->region) { - put_device(&cxlr_pmem->dev); - cxl_nvd->region = NULL; - } + cxl_nvdimm_del_region(cxl_nvd, cxlr_pmem); } device_unlock(&cxl_nvb->dev); - - nvdimm_region_delete(nd_region); } static void cxlr_pmem_remove_resource(void *res) @@ -422,7 +449,7 @@ static int cxl_pmem_region_probe(struct device *dev) if (!cxl_nvb->nvdimm_bus) { dev_dbg(dev, "nvdimm bus not found\n"); rc = -ENXIO; - goto err; + goto out_nvb; } memset(&mappings, 0, sizeof(mappings)); @@ -431,7 +458,7 @@ static int cxl_pmem_region_probe(struct device *dev) res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL); if (!res) { rc = -ENOMEM; - goto err; + goto out_nvb; } res->name = "Persistent Memory"; @@ -442,11 +469,11 @@ static int cxl_pmem_region_probe(struct device *dev) rc = insert_resource(&iomem_resource, res); if (rc) - goto err; + goto out_nvb; rc = devm_add_action_or_reset(dev, cxlr_pmem_remove_resource, res); if (rc) - goto err; + goto out_nvb; ndr_desc.res = res; ndr_desc.provider_data = cxlr_pmem; @@ -462,7 +489,7 @@ static int cxl_pmem_region_probe(struct device *dev) nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); if (!nd_set) { rc = -ENOMEM; - goto err; + goto out_nvb; } ndr_desc.memregion = cxlr->id; @@ -472,9 +499,13 @@ static int cxl_pmem_region_probe(struct device *dev) info = kmalloc_array(cxlr_pmem->nr_mappings, sizeof(*info), GFP_KERNEL); if (!info) { rc = -ENOMEM; - goto err; + goto out_nvb; } + rc = devm_add_action_or_reset(dev, release_mappings, cxlr_pmem); + if (rc) + goto out_nvd; + for (i = 0; i < cxlr_pmem->nr_mappings; i++) { struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; struct cxl_memdev *cxlmd = m->cxlmd; @@ -486,7 +517,7 @@ static int cxl_pmem_region_probe(struct device *dev) dev_dbg(dev, "[%d]: %s: no cxl_nvdimm found\n", i, dev_name(&cxlmd->dev)); rc = -ENODEV; - goto err; + goto out_nvd; } /* safe to drop ref now with bridge lock held */ @@ -498,10 +529,17 @@ static int cxl_pmem_region_probe(struct device *dev) dev_dbg(dev, "[%d]: %s: no nvdimm found\n", i, dev_name(&cxlmd->dev)); rc = -ENODEV; - goto err; + goto out_nvd; } - cxl_nvd->region = cxlr_pmem; - get_device(&cxlr_pmem->dev); + + /* + * Pin the region per nvdimm device as those may be released + * out-of-order with respect to the region, and a single nvdimm + * maybe associated with multiple regions + */ + rc = cxl_nvdimm_add_region(cxl_nvd, cxlr_pmem); + if (rc) + goto out_nvd; m->cxl_nvd = cxl_nvd; mappings[i] = (struct nd_mapping_desc) { .nvdimm = nvdimm, @@ -527,27 +565,18 @@ static int cxl_pmem_region_probe(struct device *dev) nvdimm_pmem_region_create(cxl_nvb->nvdimm_bus, &ndr_desc); if (!cxlr_pmem->nd_region) { rc = -ENOMEM; - goto err; + goto out_nvd; } rc = devm_add_action_or_reset(dev, unregister_nvdimm_region, cxlr_pmem->nd_region); -out: +out_nvd: kfree(info); +out_nvb: device_unlock(&cxl_nvb->dev); put_device(&cxl_nvb->dev); return rc; - -err: - dev_dbg(dev, "failed to create nvdimm region\n"); - for (i--; i >= 0; i--) { - nvdimm = mappings[i].nvdimm; - cxl_nvd = nvdimm_provider_data(nvdimm); - put_device(&cxl_nvd->region->dev); - cxl_nvd->region = NULL; - } - goto out; } static struct cxl_driver cxl_pmem_region_driver = { -- GitLab From 86e86c3cb63325c12ea99fbce2cc5bafba86bb40 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:30:42 -0700 Subject: [PATCH 1072/1927] tools/testing/cxl: Fix some error exits Fix a few typos where 'goto err_port' was used rather than the object specific cleanup. Reviewed-by: Dave Jiang Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166752184255.947915.16163477849330181425.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- tools/testing/cxl/test/cxl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index a072b2d3e726a..133e4c73d3702 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -695,7 +695,7 @@ static __init int cxl_test_init(void) pdev = platform_device_alloc("cxl_switch_uport", i); if (!pdev) - goto err_port; + goto err_uport; pdev->dev.parent = &root_port->dev; rc = platform_device_add(pdev); @@ -713,7 +713,7 @@ static __init int cxl_test_init(void) pdev = platform_device_alloc("cxl_switch_dport", i); if (!pdev) - goto err_port; + goto err_dport; pdev->dev.parent = &uport->dev; rc = platform_device_add(pdev); -- GitLab From e41c8452b9b204689e68756a3836d1d37b617ad5 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:30:48 -0700 Subject: [PATCH 1073/1927] tools/testing/cxl: Add a single-port host-bridge regression config Jonathan reports that region creation fails when a single-port host-bridge connects to a multi-port switch. Mock up that configuration so a fix can be tested and regression tested going forward. Reported-by: Bobo WL Reported-by: Jonathan Cameron Link: http://lore.kernel.org/r/20221010172057.00001559@huawei.com Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166752184838.947915.2167957540894293891.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- tools/testing/cxl/test/cxl.c | 297 ++++++++++++++++++++++++++++++++--- 1 file changed, 278 insertions(+), 19 deletions(-) diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index 133e4c73d3702..7edce12fd2ce5 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -12,30 +12,62 @@ #include "mock.h" #define NR_CXL_HOST_BRIDGES 2 +#define NR_CXL_SINGLE_HOST 1 #define NR_CXL_ROOT_PORTS 2 #define NR_CXL_SWITCH_PORTS 2 #define NR_CXL_PORT_DECODERS 8 static struct platform_device *cxl_acpi; static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES]; -static struct platform_device - *cxl_root_port[NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS]; -static struct platform_device - *cxl_switch_uport[NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS]; -static struct platform_device - *cxl_switch_dport[NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS * - NR_CXL_SWITCH_PORTS]; -struct platform_device - *cxl_mem[NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS * NR_CXL_SWITCH_PORTS]; +#define NR_MULTI_ROOT (NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS) +static struct platform_device *cxl_root_port[NR_MULTI_ROOT]; +static struct platform_device *cxl_switch_uport[NR_MULTI_ROOT]; +#define NR_MEM_MULTI \ + (NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS * NR_CXL_SWITCH_PORTS) +static struct platform_device *cxl_switch_dport[NR_MEM_MULTI]; + +static struct platform_device *cxl_hb_single[NR_CXL_SINGLE_HOST]; +static struct platform_device *cxl_root_single[NR_CXL_SINGLE_HOST]; +static struct platform_device *cxl_swu_single[NR_CXL_SINGLE_HOST]; +#define NR_MEM_SINGLE (NR_CXL_SINGLE_HOST * NR_CXL_SWITCH_PORTS) +static struct platform_device *cxl_swd_single[NR_MEM_SINGLE]; + +struct platform_device *cxl_mem[NR_MEM_MULTI]; +struct platform_device *cxl_mem_single[NR_MEM_SINGLE]; + + +static inline bool is_multi_bridge(struct device *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) + if (&cxl_host_bridge[i]->dev == dev) + return true; + return false; +} + +static inline bool is_single_bridge(struct device *dev) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++) + if (&cxl_hb_single[i]->dev == dev) + return true; + return false; +} static struct acpi_device acpi0017_mock; -static struct acpi_device host_bridge[NR_CXL_HOST_BRIDGES] = { +static struct acpi_device host_bridge[NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST] = { [0] = { .handle = &host_bridge[0], }, [1] = { .handle = &host_bridge[1], }, + [2] = { + .handle = &host_bridge[2], + }, + }; static bool is_mock_dev(struct device *dev) @@ -45,6 +77,9 @@ static bool is_mock_dev(struct device *dev) for (i = 0; i < ARRAY_SIZE(cxl_mem); i++) if (dev == &cxl_mem[i]->dev) return true; + for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++) + if (dev == &cxl_mem_single[i]->dev) + return true; if (dev == &cxl_acpi->dev) return true; return false; @@ -66,7 +101,7 @@ static bool is_mock_adev(struct acpi_device *adev) static struct { struct acpi_table_cedt cedt; - struct acpi_cedt_chbs chbs[NR_CXL_HOST_BRIDGES]; + struct acpi_cedt_chbs chbs[NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST]; struct { struct acpi_cedt_cfmws cfmws; u32 target[1]; @@ -83,6 +118,10 @@ static struct { struct acpi_cedt_cfmws cfmws; u32 target[2]; } cfmws3; + struct { + struct acpi_cedt_cfmws cfmws; + u32 target[1]; + } cfmws4; } __packed mock_cedt = { .cedt = { .header = { @@ -107,6 +146,14 @@ static struct { .uid = 1, .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20, }, + .chbs[2] = { + .header = { + .type = ACPI_CEDT_TYPE_CHBS, + .length = sizeof(mock_cedt.chbs[0]), + }, + .uid = 2, + .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20, + }, .cfmws0 = { .cfmws = { .header = { @@ -167,13 +214,29 @@ static struct { }, .target = { 0, 1, }, }, + .cfmws4 = { + .cfmws = { + .header = { + .type = ACPI_CEDT_TYPE_CFMWS, + .length = sizeof(mock_cedt.cfmws4), + }, + .interleave_ways = 0, + .granularity = 4, + .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 | + ACPI_CEDT_CFMWS_RESTRICT_PMEM, + .qtg_id = 4, + .window_size = SZ_256M * 4UL, + }, + .target = { 2 }, + }, }; -struct acpi_cedt_cfmws *mock_cfmws[4] = { +struct acpi_cedt_cfmws *mock_cfmws[] = { [0] = &mock_cedt.cfmws0.cfmws, [1] = &mock_cedt.cfmws1.cfmws, [2] = &mock_cedt.cfmws2.cfmws, [3] = &mock_cedt.cfmws3.cfmws, + [4] = &mock_cedt.cfmws4.cfmws, }; struct cxl_mock_res { @@ -304,6 +367,9 @@ static bool is_mock_bridge(struct device *dev) for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) if (dev == &cxl_host_bridge[i]->dev) return true; + for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++) + if (dev == &cxl_hb_single[i]->dev) + return true; return false; } @@ -326,6 +392,18 @@ static bool is_mock_port(struct device *dev) if (dev == &cxl_switch_dport[i]->dev) return true; + for (i = 0; i < ARRAY_SIZE(cxl_root_single); i++) + if (dev == &cxl_root_single[i]->dev) + return true; + + for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++) + if (dev == &cxl_swu_single[i]->dev) + return true; + + for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++) + if (dev == &cxl_swd_single[i]->dev) + return true; + if (is_cxl_memdev(dev)) return is_mock_dev(dev->parent); @@ -561,11 +639,31 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port) int i, array_size; if (port->depth == 1) { - array_size = ARRAY_SIZE(cxl_root_port); - array = cxl_root_port; + if (is_multi_bridge(port->uport)) { + array_size = ARRAY_SIZE(cxl_root_port); + array = cxl_root_port; + } else if (is_single_bridge(port->uport)) { + array_size = ARRAY_SIZE(cxl_root_single); + array = cxl_root_single; + } else { + dev_dbg(&port->dev, "%s: unknown bridge type\n", + dev_name(port->uport)); + return -ENXIO; + } } else if (port->depth == 2) { - array_size = ARRAY_SIZE(cxl_switch_dport); - array = cxl_switch_dport; + struct cxl_port *parent = to_cxl_port(port->dev.parent); + + if (is_multi_bridge(parent->uport)) { + array_size = ARRAY_SIZE(cxl_switch_dport); + array = cxl_switch_dport; + } else if (is_single_bridge(parent->uport)) { + array_size = ARRAY_SIZE(cxl_swd_single); + array = cxl_swd_single; + } else { + dev_dbg(&port->dev, "%s: unknown bridge type\n", + dev_name(port->uport)); + return -ENXIO; + } } else { dev_WARN_ONCE(&port->dev, 1, "unexpected depth %d\n", port->depth); @@ -576,8 +674,12 @@ static int mock_cxl_port_enumerate_dports(struct cxl_port *port) struct platform_device *pdev = array[i]; struct cxl_dport *dport; - if (pdev->dev.parent != port->uport) + if (pdev->dev.parent != port->uport) { + dev_dbg(&port->dev, "%s: mismatch parent %s\n", + dev_name(port->uport), + dev_name(pdev->dev.parent)); continue; + } dport = devm_cxl_add_dport(port, &pdev->dev, pdev->id, CXL_RESOURCE_NONE); @@ -627,6 +729,157 @@ static void mock_companion(struct acpi_device *adev, struct device *dev) #define SZ_512G (SZ_64G * 8) #endif +static __init int cxl_single_init(void) +{ + int i, rc; + + for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++) { + struct acpi_device *adev = + &host_bridge[NR_CXL_HOST_BRIDGES + i]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_host_bridge", + NR_CXL_HOST_BRIDGES + i); + if (!pdev) + goto err_bridge; + + mock_companion(adev, &pdev->dev); + rc = platform_device_add(pdev); + if (rc) { + platform_device_put(pdev); + goto err_bridge; + } + + cxl_hb_single[i] = pdev; + rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj, + "physical_node"); + if (rc) + goto err_bridge; + } + + for (i = 0; i < ARRAY_SIZE(cxl_root_single); i++) { + struct platform_device *bridge = + cxl_hb_single[i % ARRAY_SIZE(cxl_hb_single)]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_root_port", + NR_MULTI_ROOT + i); + if (!pdev) + goto err_port; + pdev->dev.parent = &bridge->dev; + + rc = platform_device_add(pdev); + if (rc) { + platform_device_put(pdev); + goto err_port; + } + cxl_root_single[i] = pdev; + } + + for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++) { + struct platform_device *root_port = cxl_root_single[i]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_switch_uport", + NR_MULTI_ROOT + i); + if (!pdev) + goto err_uport; + pdev->dev.parent = &root_port->dev; + + rc = platform_device_add(pdev); + if (rc) { + platform_device_put(pdev); + goto err_uport; + } + cxl_swu_single[i] = pdev; + } + + for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++) { + struct platform_device *uport = + cxl_swu_single[i % ARRAY_SIZE(cxl_swu_single)]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_switch_dport", + i + NR_MEM_MULTI); + if (!pdev) + goto err_dport; + pdev->dev.parent = &uport->dev; + + rc = platform_device_add(pdev); + if (rc) { + platform_device_put(pdev); + goto err_dport; + } + cxl_swd_single[i] = pdev; + } + + for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++) { + struct platform_device *dport = cxl_swd_single[i]; + struct platform_device *pdev; + + pdev = platform_device_alloc("cxl_mem", NR_MEM_MULTI + i); + if (!pdev) + goto err_mem; + pdev->dev.parent = &dport->dev; + set_dev_node(&pdev->dev, i % 2); + + rc = platform_device_add(pdev); + if (rc) { + platform_device_put(pdev); + goto err_mem; + } + cxl_mem_single[i] = pdev; + } + + return 0; + +err_mem: + for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_mem_single[i]); +err_dport: + for (i = ARRAY_SIZE(cxl_swd_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_swd_single[i]); +err_uport: + for (i = ARRAY_SIZE(cxl_swu_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_swu_single[i]); +err_port: + for (i = ARRAY_SIZE(cxl_root_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_root_single[i]); +err_bridge: + for (i = ARRAY_SIZE(cxl_hb_single) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_hb_single[i]; + + if (!pdev) + continue; + sysfs_remove_link(&pdev->dev.kobj, "physical_node"); + platform_device_unregister(cxl_hb_single[i]); + } + + return rc; +} + +static void cxl_single_exit(void) +{ + int i; + + for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_mem_single[i]); + for (i = ARRAY_SIZE(cxl_swd_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_swd_single[i]); + for (i = ARRAY_SIZE(cxl_swu_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_swu_single[i]); + for (i = ARRAY_SIZE(cxl_root_single) - 1; i >= 0; i--) + platform_device_unregister(cxl_root_single[i]); + for (i = ARRAY_SIZE(cxl_hb_single) - 1; i >= 0; i--) { + struct platform_device *pdev = cxl_hb_single[i]; + + if (!pdev) + continue; + sysfs_remove_link(&pdev->dev.kobj, "physical_node"); + platform_device_unregister(cxl_hb_single[i]); + } +} + static __init int cxl_test_init(void) { int rc, i; @@ -724,7 +977,6 @@ static __init int cxl_test_init(void) cxl_switch_dport[i] = pdev; } - BUILD_BUG_ON(ARRAY_SIZE(cxl_mem) != ARRAY_SIZE(cxl_switch_dport)); for (i = 0; i < ARRAY_SIZE(cxl_mem); i++) { struct platform_device *dport = cxl_switch_dport[i]; struct platform_device *pdev; @@ -743,9 +995,13 @@ static __init int cxl_test_init(void) cxl_mem[i] = pdev; } + rc = cxl_single_init(); + if (rc) + goto err_mem; + cxl_acpi = platform_device_alloc("cxl_acpi", 0); if (!cxl_acpi) - goto err_mem; + goto err_single; mock_companion(&acpi0017_mock, &cxl_acpi->dev); acpi0017_mock.dev.bus = &platform_bus_type; @@ -758,6 +1014,8 @@ static __init int cxl_test_init(void) err_add: platform_device_put(cxl_acpi); +err_single: + cxl_single_exit(); err_mem: for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) platform_device_unregister(cxl_mem[i]); @@ -793,6 +1051,7 @@ static __exit void cxl_test_exit(void) int i; platform_device_unregister(cxl_acpi); + cxl_single_exit(); for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--) platform_device_unregister(cxl_mem[i]); for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--) -- GitLab From e4f6dfa9ef756a3934a4caf618b1e86e9e8e21d0 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:30:54 -0700 Subject: [PATCH 1074/1927] cxl/region: Fix 'distance' calculation with passthrough ports When programming port decode targets, the algorithm wants to ensure that two devices are compatible to be programmed as peers beneath a given port. A compatible peer is a target that shares the same dport, and where that target's interleave position also routes it to the same dport. Compatibility is determined by the device's interleave position being >= to distance. For example, if a given dport can only map every Nth position then positions less than N away from the last target programmed are incompatible. The @distance for the host-bridge's cxl_port in a simple dual-ported host-bridge configuration with 2 direct-attached devices is 1, i.e. An x2 region divided by 2 dports to reach 2 region targets. An x4 region under an x2 host-bridge would need 2 intervening switches where the @distance at the host bridge level is 2 (x4 region divided by 2 switches to reach 4 devices). However, the distance between peers underneath a single ported host-bridge is always zero because there is no limit to the number of devices that can be mapped. In other words, there are no decoders to program in a passthrough, all descendants are mapped and distance only starts matters for the intervening descendant ports of the passthrough port. Add tracking for the number of dports mapped to a port, and use that to detect the passthrough case for calculating @distance. Cc: Reported-by: Bobo WL Reported-by: Jonathan Cameron Link: http://lore.kernel.org/r/20221010172057.00001559@huawei.com Fixes: 27b3f8d13830 ("cxl/region: Program target lists") Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166752185440.947915.6617495912508299445.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/port.c | 11 +++++++++-- drivers/cxl/core/region.c | 9 ++++++++- drivers/cxl/cxl.h | 2 ++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index bffde862de0bf..e7556864ea808 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -811,6 +811,7 @@ static struct cxl_dport *find_dport(struct cxl_port *port, int id) static int add_dport(struct cxl_port *port, struct cxl_dport *new) { struct cxl_dport *dup; + int rc; device_lock_assert(&port->dev); dup = find_dport(port, new->port_id); @@ -821,8 +822,14 @@ static int add_dport(struct cxl_port *port, struct cxl_dport *new) dev_name(dup->dport)); return -EBUSY; } - return xa_insert(&port->dports, (unsigned long)new->dport, new, - GFP_KERNEL); + + rc = xa_insert(&port->dports, (unsigned long)new->dport, new, + GFP_KERNEL); + if (rc) + return rc; + + port->nr_dports++; + return 0; } /* diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index c52465e09f268..c0253de749458 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -990,7 +990,14 @@ static int cxl_port_setup_targets(struct cxl_port *port, if (cxl_rr->nr_targets_set) { int i, distance; - distance = p->nr_targets / cxl_rr->nr_targets; + /* + * Passthrough ports impose no distance requirements between + * peers + */ + if (port->nr_dports == 1) + distance = 0; + else + distance = p->nr_targets / cxl_rr->nr_targets; for (i = 0; i < cxl_rr->nr_targets_set; i++) if (ep->dport == cxlsd->target[i]) { rc = check_last_peer(cxled, ep, cxl_rr, diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 1164ad49f3d3a..ac75554b5d763 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -457,6 +457,7 @@ struct cxl_pmem_region { * @regions: cxl_region_ref instances, regions mapped by this port * @parent_dport: dport that points to this port in the parent * @decoder_ida: allocator for decoder ids + * @nr_dports: number of entries in @dports * @hdm_end: track last allocated HDM decoder instance for allocation ordering * @commit_end: cursor to track highest committed decoder for commit ordering * @component_reg_phys: component register capability base address (optional) @@ -475,6 +476,7 @@ struct cxl_port { struct xarray regions; struct cxl_dport *parent_dport; struct ida decoder_ida; + int nr_dports; int hdm_end; int commit_end; resource_size_t component_reg_phys; -- GitLab From 8f401ec1c8975eabfe4c089de91cbe058deabf71 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 3 Nov 2022 17:31:00 -0700 Subject: [PATCH 1075/1927] cxl/region: Recycle region ids At region creation time the next region-id is atomically cached so that there is predictability of region device names. If that region is destroyed and then a new one is created the region id increments. That ends up looking like a memory leak, or is otherwise surprising that identifiers roll forward even after destroying all previously created regions. Try to reuse rather than free old region ids at region release time. While this fixes a cosmetic issue, the needlessly advancing memory region-id gives the appearance of a memory leak, hence the "Fixes" tag, but no "Cc: stable" tag. Cc: Ben Widawsky Cc: Jonathan Cameron Fixes: 779dd20cfb56 ("cxl/region: Add region creation support") Reviewed-by: Dave Jiang Reviewed-by: Vishal Verma Link: https://lore.kernel.org/r/166752186062.947915.13200195701224993317.stgit@dwillia2-xfh.jf.intel.com Signed-off-by: Dan Williams --- drivers/cxl/core/region.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index c0253de749458..f9ae5ad284ffb 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1534,9 +1534,24 @@ static const struct attribute_group *region_groups[] = { static void cxl_region_release(struct device *dev) { + struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); struct cxl_region *cxlr = to_cxl_region(dev); + int id = atomic_read(&cxlrd->region_id); + + /* + * Try to reuse the recently idled id rather than the cached + * next id to prevent the region id space from increasing + * unnecessarily. + */ + if (cxlr->id < id) + if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { + memregion_free(id); + goto out; + } memregion_free(cxlr->id); +out: + put_device(dev->parent); kfree(cxlr); } @@ -1598,6 +1613,11 @@ static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int i device_initialize(dev); lockdep_set_class(&dev->mutex, &cxl_region_key); dev->parent = &cxlrd->cxlsd.cxld.dev; + /* + * Keep root decoder pinned through cxl_region_release to fixup + * region id allocations + */ + get_device(dev->parent); device_set_pm_not_required(dev); dev->bus = &cxl_bus_type; dev->type = &cxl_region_type; -- GitLab From c4f683731db330460d909bd0ca8d5af876fcdc97 Mon Sep 17 00:00:00 2001 From: Cristian Marussi Date: Mon, 31 Oct 2022 11:40:18 +0000 Subject: [PATCH 1076/1927] hwmon: (scmi) Register explicitly with Thermal Framework Available sensors are enumerated and reported by the SCMI platform server using a 16bit identification number; not all such sensors are of a type supported by hwmon subsystem and, among the supported ones, only a subset could be temperature sensors that have to be registered with the Thermal Framework. Potential clashes between hwmon channels indexes and the underlying real sensors IDs do not play well with the hwmon<-->thermal bridge automatic registration routines and could need a sensible number of fake dummy sensors to be made up in order to keep indexes and IDs in sync. Avoid to use the hwmon<-->thermal bridge dropping the HWMON_C_REGISTER_TZ attribute and instead explicit register temperature sensors directly with the Thermal Framework. Cc: Daniel Lezcano Cc: Guenter Roeck Cc: linux-hwmon@vger.kernel.org Signed-off-by: Cristian Marussi Acked-by: Sudeep Holla Link: https://lore.kernel.org/r/20221031114018.59048-1-cristian.marussi@arm.com Signed-off-by: Guenter Roeck --- drivers/hwmon/scmi-hwmon.c | 116 ++++++++++++++++++++++++++++++++----- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/drivers/hwmon/scmi-hwmon.c b/drivers/hwmon/scmi-hwmon.c index b1329a58ce403..e192f0c671465 100644 --- a/drivers/hwmon/scmi-hwmon.c +++ b/drivers/hwmon/scmi-hwmon.c @@ -20,6 +20,11 @@ struct scmi_sensors { const struct scmi_sensor_info **info[hwmon_max]; }; +struct scmi_thermal_sensor { + const struct scmi_protocol_handle *ph; + const struct scmi_sensor_info *info; +}; + static inline u64 __pow10(u8 x) { u64 r = 1; @@ -64,16 +69,14 @@ static int scmi_hwmon_scale(const struct scmi_sensor_info *sensor, u64 *value) return 0; } -static int scmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type, - u32 attr, int channel, long *val) +static int scmi_hwmon_read_scaled_value(const struct scmi_protocol_handle *ph, + const struct scmi_sensor_info *sensor, + long *val) { int ret; u64 value; - const struct scmi_sensor_info *sensor; - struct scmi_sensors *scmi_sensors = dev_get_drvdata(dev); - sensor = *(scmi_sensors->info[type] + channel); - ret = sensor_ops->reading_get(scmi_sensors->ph, sensor->id, &value); + ret = sensor_ops->reading_get(ph, sensor->id, &value); if (ret) return ret; @@ -84,6 +87,17 @@ static int scmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type, return ret; } +static int scmi_hwmon_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + const struct scmi_sensor_info *sensor; + struct scmi_sensors *scmi_sensors = dev_get_drvdata(dev); + + sensor = *(scmi_sensors->info[type] + channel); + + return scmi_hwmon_read_scaled_value(scmi_sensors->ph, sensor, val); +} + static int scmi_hwmon_read_string(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, const char **str) @@ -122,6 +136,25 @@ static struct hwmon_chip_info scmi_chip_info = { .info = NULL, }; +static int scmi_hwmon_thermal_get_temp(struct thermal_zone_device *tz, + int *temp) +{ + int ret; + long value; + struct scmi_thermal_sensor *th_sensor = tz->devdata; + + ret = scmi_hwmon_read_scaled_value(th_sensor->ph, th_sensor->info, + &value); + if (!ret) + *temp = value; + + return ret; +} + +static const struct thermal_zone_device_ops scmi_hwmon_thermal_ops = { + .get_temp = scmi_hwmon_thermal_get_temp, +}; + static int scmi_hwmon_add_chan_info(struct hwmon_channel_info *scmi_hwmon_chan, struct device *dev, int num, enum hwmon_sensor_types type, u32 config) @@ -149,7 +182,6 @@ static enum hwmon_sensor_types scmi_types[] = { }; static u32 hwmon_attributes[hwmon_max] = { - [hwmon_chip] = HWMON_C_REGISTER_TZ, [hwmon_temp] = HWMON_T_INPUT | HWMON_T_LABEL, [hwmon_in] = HWMON_I_INPUT | HWMON_I_LABEL, [hwmon_curr] = HWMON_C_INPUT | HWMON_C_LABEL, @@ -157,6 +189,43 @@ static u32 hwmon_attributes[hwmon_max] = { [hwmon_energy] = HWMON_E_INPUT | HWMON_E_LABEL, }; +static int scmi_thermal_sensor_register(struct device *dev, + const struct scmi_protocol_handle *ph, + const struct scmi_sensor_info *sensor) +{ + struct scmi_thermal_sensor *th_sensor; + struct thermal_zone_device *tzd; + + th_sensor = devm_kzalloc(dev, sizeof(*th_sensor), GFP_KERNEL); + if (!th_sensor) + return -ENOMEM; + + th_sensor->ph = ph; + th_sensor->info = sensor; + + /* + * Try to register a temperature sensor with the Thermal Framework: + * skip sensors not defined as part of any thermal zone (-ENODEV) but + * report any other errors related to misconfigured zones/sensors. + */ + tzd = devm_thermal_of_zone_register(dev, th_sensor->info->id, th_sensor, + &scmi_hwmon_thermal_ops); + if (IS_ERR(tzd)) { + devm_kfree(dev, th_sensor); + + if (PTR_ERR(tzd) != -ENODEV) + return PTR_ERR(tzd); + + dev_dbg(dev, "Sensor '%s' not attached to any thermal zone.\n", + sensor->name); + } else { + dev_dbg(dev, "Sensor '%s' attached to thermal zone ID:%d\n", + sensor->name, tzd->id); + } + + return 0; +} + static int scmi_hwmon_probe(struct scmi_device *sdev) { int i, idx; @@ -164,7 +233,7 @@ static int scmi_hwmon_probe(struct scmi_device *sdev) enum hwmon_sensor_types type; struct scmi_sensors *scmi_sensors; const struct scmi_sensor_info *sensor; - int nr_count[hwmon_max] = {0}, nr_types = 0; + int nr_count[hwmon_max] = {0}, nr_types = 0, nr_count_temp = 0; const struct hwmon_chip_info *chip_info; struct device *hwdev, *dev = &sdev->dev; struct hwmon_channel_info *scmi_hwmon_chan; @@ -208,10 +277,8 @@ static int scmi_hwmon_probe(struct scmi_device *sdev) } } - if (nr_count[hwmon_temp]) { - nr_count[hwmon_chip]++; - nr_types++; - } + if (nr_count[hwmon_temp]) + nr_count_temp = nr_count[hwmon_temp]; scmi_hwmon_chan = devm_kcalloc(dev, nr_types, sizeof(*scmi_hwmon_chan), GFP_KERNEL); @@ -262,8 +329,31 @@ static int scmi_hwmon_probe(struct scmi_device *sdev) hwdev = devm_hwmon_device_register_with_info(dev, "scmi_sensors", scmi_sensors, chip_info, NULL); + if (IS_ERR(hwdev)) + return PTR_ERR(hwdev); - return PTR_ERR_OR_ZERO(hwdev); + for (i = 0; i < nr_count_temp; i++) { + int ret; + + sensor = *(scmi_sensors->info[hwmon_temp] + i); + if (!sensor) + continue; + + /* + * Warn on any misconfiguration related to thermal zones but + * bail out of probing only on memory errors. + */ + ret = scmi_thermal_sensor_register(dev, ph, sensor); + if (ret) { + if (ret == -ENOMEM) + return ret; + dev_warn(dev, + "Thermal zone misconfigured for %s. err=%d\n", + sensor->name, ret); + } + } + + return 0; } static const struct scmi_device_id scmi_id_table[] = { -- GitLab From 1e699e177e339e462cdc8571e3d0fcf29665608e Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Fri, 4 Nov 2022 16:37:30 -0700 Subject: [PATCH 1077/1927] Revert "hwmon: (pmbus) Add regulator supply into macro" This reverts commit 54cc3dbfc10dc3db7cb1cf49aee4477a8398fbde. Zev Weiss reports that the reverted patch may cause a regulator undercount. Here is his report: ... having regulator-dummy set as a supply on my PMBus regulators (instead of having them as their own top-level regulators without an upstream supply) leads to enable-count underflow errors when disabling them: # echo 0 > /sys/bus/platform/devices/efuse01/state [ 906.094477] regulator-dummy: Underflow of regulator enable count [ 906.100563] Failed to disable vout: -EINVAL [ 136.992676] reg-userspace-consumer efuse01: Failed to configure state: -22 Zev reports that reverting the patch fixes the problem. So let's do that for now. Fixes: 54cc3dbfc10d ("hwmon: (pmbus) Add regulator supply into macro") Cc: Marcello Sylvester Bauer Reported-by: Zev Weiss Signed-off-by: Guenter Roeck --- drivers/hwmon/pmbus/pmbus.h | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/hwmon/pmbus/pmbus.h b/drivers/hwmon/pmbus/pmbus.h index 7daaf0caf4d30..10fb17879f8ed 100644 --- a/drivers/hwmon/pmbus/pmbus.h +++ b/drivers/hwmon/pmbus/pmbus.h @@ -467,7 +467,6 @@ extern const struct regulator_ops pmbus_regulator_ops; #define PMBUS_REGULATOR_STEP(_name, _id, _voltages, _step) \ [_id] = { \ .name = (_name # _id), \ - .supply_name = "vin", \ .id = (_id), \ .of_match = of_match_ptr(_name # _id), \ .regulators_node = of_match_ptr("regulators"), \ -- GitLab From b4c66425771ddb910316c7b4cd7fa0614098ec45 Mon Sep 17 00:00:00 2001 From: Vikas Gupta Date: Thu, 3 Nov 2022 19:33:24 -0400 Subject: [PATCH 1078/1927] bnxt_en: refactor bnxt_cancel_reservations() Introduce bnxt_clear_reservations() to clear the reserved attributes only. This will be used in the next patch to fix PCI AER handling. Signed-off-by: Vikas Gupta Signed-off-by: Michael Chan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 04cf7684f1b0c..3743d9755eb43 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -9983,17 +9983,12 @@ static int bnxt_try_recover_fw(struct bnxt *bp) return -ENODEV; } -int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) +static void bnxt_clear_reservations(struct bnxt *bp, bool fw_reset) { struct bnxt_hw_resc *hw_resc = &bp->hw_resc; - int rc; if (!BNXT_NEW_RM(bp)) - return 0; /* no resource reservations required */ - - rc = bnxt_hwrm_func_resc_qcaps(bp, true); - if (rc) - netdev_err(bp->dev, "resc_qcaps failed\n"); + return; /* no resource reservations required */ hw_resc->resv_cp_rings = 0; hw_resc->resv_stat_ctxs = 0; @@ -10006,6 +10001,20 @@ int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) bp->tx_nr_rings = 0; bp->rx_nr_rings = 0; } +} + +int bnxt_cancel_reservations(struct bnxt *bp, bool fw_reset) +{ + int rc; + + if (!BNXT_NEW_RM(bp)) + return 0; /* no resource reservations required */ + + rc = bnxt_hwrm_func_resc_qcaps(bp, true); + if (rc) + netdev_err(bp->dev, "resc_qcaps failed\n"); + + bnxt_clear_reservations(bp, fw_reset); return rc; } -- GitLab From 0cf736a18a1e804037839bd8df9e36f0efdb8745 Mon Sep 17 00:00:00 2001 From: Vikas Gupta Date: Thu, 3 Nov 2022 19:33:25 -0400 Subject: [PATCH 1079/1927] bnxt_en: fix the handling of PCIE-AER Fix the sequence required for PCIE-AER. While slot reset occurs, firmware might not be ready and the driver needs to check for its recovery. We also need to remap the health registers for some chips and clear the resource reservations. The resources will be allocated again during bnxt_io_resume(). Fixes: fb1e6e562b37 ("bnxt_en: Fix AER recovery.") Signed-off-by: Vikas Gupta Signed-off-by: Michael Chan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 29 ++++++++++++++++++- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 1 + .../net/ethernet/broadcom/bnxt/bnxt_hwrm.c | 3 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 3743d9755eb43..f44f9367b7fd6 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -13922,7 +13922,9 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) pci_ers_result_t result = PCI_ERS_RESULT_DISCONNECT; struct net_device *netdev = pci_get_drvdata(pdev); struct bnxt *bp = netdev_priv(netdev); - int err = 0, off; + int retry = 0; + int err = 0; + int off; netdev_info(bp->dev, "PCI Slot Reset\n"); @@ -13950,11 +13952,36 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev) pci_restore_state(pdev); pci_save_state(pdev); + bnxt_inv_fw_health_reg(bp); + bnxt_try_map_fw_health_reg(bp); + + /* In some PCIe AER scenarios, firmware may take up to + * 10 seconds to become ready in the worst case. + */ + do { + err = bnxt_try_recover_fw(bp); + if (!err) + break; + retry++; + } while (retry < BNXT_FW_SLOT_RESET_RETRY); + + if (err) { + dev_err(&pdev->dev, "Firmware not ready\n"); + goto reset_exit; + } + err = bnxt_hwrm_func_reset(bp); if (!err) result = PCI_ERS_RESULT_RECOVERED; + + bnxt_ulp_irq_stop(bp); + bnxt_clear_int_mode(bp); + err = bnxt_init_int_mode(bp); + bnxt_ulp_irq_restart(bp, err); } +reset_exit: + bnxt_clear_reservations(bp, true); rtnl_unlock(); return result; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index b1b17f9113006..d5fa43cfe5248 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -1621,6 +1621,7 @@ struct bnxt_fw_health { #define BNXT_FW_RETRY 5 #define BNXT_FW_IF_RETRY 10 +#define BNXT_FW_SLOT_RESET_RETRY 4 enum board_idx { BCM57301, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c index b01d42928a53c..132442f16fe67 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwrm.c @@ -476,7 +476,8 @@ static int __hwrm_send(struct bnxt *bp, struct bnxt_hwrm_ctx *ctx) memset(ctx->resp, 0, PAGE_SIZE); req_type = le16_to_cpu(ctx->req->req_type); - if (BNXT_NO_FW_ACCESS(bp) && req_type != HWRM_FUNC_RESET) { + if (BNXT_NO_FW_ACCESS(bp) && + (req_type != HWRM_FUNC_RESET && req_type != HWRM_VER_GET)) { netdev_dbg(bp->dev, "hwrm req_type 0x%x skipped, FW channel down\n", req_type); goto exit; -- GitLab From 6d81ea3765dfa6c8a20822613c81edad1c4a16a0 Mon Sep 17 00:00:00 2001 From: Michael Chan Date: Thu, 3 Nov 2022 19:33:26 -0400 Subject: [PATCH 1080/1927] bnxt_en: Fix possible crash in bnxt_hwrm_set_coal() During the error recovery sequence, the rtnl_lock is not held for the entire duration and some datastructures may be freed during the sequence. Check for the BNXT_STATE_OPEN flag instead of netif_running() to ensure that the device is fully operational before proceeding to reconfigure the coalescing settings. This will fix a possible crash like this: BUG: unable to handle kernel NULL pointer dereference at 0000000000000000 PGD 0 P4D 0 Oops: 0000 [#1] SMP NOPTI CPU: 10 PID: 181276 Comm: ethtool Kdump: loaded Tainted: G IOE --------- - - 4.18.0-348.el8.x86_64 #1 Hardware name: Dell Inc. PowerEdge R740/0F9N89, BIOS 2.3.10 08/15/2019 RIP: 0010:bnxt_hwrm_set_coal+0x1fb/0x2a0 [bnxt_en] Code: c2 66 83 4e 22 08 66 89 46 1c e8 10 cb 00 00 41 83 c6 01 44 39 b3 68 01 00 00 0f 8e a3 00 00 00 48 8b 93 c8 00 00 00 49 63 c6 <48> 8b 2c c2 48 8b 85 b8 02 00 00 48 85 c0 74 2e 48 8b 74 24 08 f6 RSP: 0018:ffffb11c8dcaba50 EFLAGS: 00010246 RAX: 0000000000000000 RBX: ffff8d168a8b0ac0 RCX: 00000000000000c5 RDX: 0000000000000000 RSI: ffff8d162f72c000 RDI: ffff8d168a8b0b28 RBP: 0000000000000000 R08: b6e1f68a12e9a7eb R09: 0000000000000000 R10: 0000000000000001 R11: 0000000000000037 R12: ffff8d168a8b109c R13: ffff8d168a8b10aa R14: 0000000000000000 R15: ffffffffc01ac4e0 FS: 00007f3852e4c740(0000) GS:ffff8d24c0080000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000000 CR3: 000000041b3ee003 CR4: 00000000007706e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 PKRU: 55555554 Call Trace: ethnl_set_coalesce+0x3ce/0x4c0 genl_family_rcv_msg_doit.isra.15+0x10f/0x150 genl_family_rcv_msg+0xb3/0x160 ? coalesce_fill_reply+0x480/0x480 genl_rcv_msg+0x47/0x90 ? genl_family_rcv_msg+0x160/0x160 netlink_rcv_skb+0x4c/0x120 genl_rcv+0x24/0x40 netlink_unicast+0x196/0x230 netlink_sendmsg+0x204/0x3d0 sock_sendmsg+0x4c/0x50 __sys_sendto+0xee/0x160 ? syscall_trace_enter+0x1d3/0x2c0 ? __audit_syscall_exit+0x249/0x2a0 __x64_sys_sendto+0x24/0x30 do_syscall_64+0x5b/0x1a0 entry_SYSCALL_64_after_hwframe+0x65/0xca RIP: 0033:0x7f38524163bb Fixes: 2151fe0830fd ("bnxt_en: Handle RESET_NOTIFY async event from firmware.") Reviewed-by: Somnath Kotur Signed-off-by: Michael Chan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index f57e524c7e30b..8cad15c458b39 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -162,7 +162,7 @@ static int bnxt_set_coalesce(struct net_device *dev, } reset_coalesce: - if (netif_running(dev)) { + if (test_bit(BNXT_STATE_OPEN, &bp->state)) { if (update_stats) { rc = bnxt_close_nic(bp, true, false); if (!rc) -- GitLab From 02597d39145bb0aa81d04bf39b6a913ce9a9d465 Mon Sep 17 00:00:00 2001 From: Alex Barba Date: Thu, 3 Nov 2022 19:33:27 -0400 Subject: [PATCH 1081/1927] bnxt_en: fix potentially incorrect return value for ndo_rx_flow_steer In the bnxt_en driver ndo_rx_flow_steer returns '0' whenever an entry that we are attempting to steer is already found. This is not the correct behavior. The return code should be the value/index that corresponds to the entry. Returning zero all the time causes the RFS records to be incorrect unless entry '0' is the correct one. As flows migrate to different cores this can create entries that are not correct. Fixes: c0c050c58d84 ("bnxt_en: New Broadcom ethernet driver.") Reported-by: Akshay Navgire Signed-off-by: Alex Barba Signed-off-by: Andy Gospodarek Signed-off-by: Michael Chan Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index f44f9367b7fd6..c78b6e9dea2cc 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -12903,8 +12903,8 @@ static int bnxt_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, rcu_read_lock(); hlist_for_each_entry_rcu(fltr, head, hash) { if (bnxt_fltr_match(fltr, new_fltr)) { + rc = fltr->sw_id; rcu_read_unlock(); - rc = 0; goto err_free; } } -- GitLab From b7cbc6740bd6ad5d43345a2504f7e4beff0d709f Mon Sep 17 00:00:00 2001 From: Sean Anderson Date: Thu, 3 Nov 2022 14:28:30 -0400 Subject: [PATCH 1082/1927] net: fman: Unregister ethernet device on removal When the mac device gets removed, it leaves behind the ethernet device. This will result in a segfault next time the ethernet device accesses mac_dev. Remove the ethernet device when we get removed to prevent this. This is not completely reversible, since some resources aren't cleaned up properly, but that can be addressed later. Fixes: 3933961682a3 ("fsl/fman: Add FMan MAC driver") Signed-off-by: Sean Anderson Link: https://lore.kernel.org/r/20221103182831.2248833-1-sean.anderson@seco.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/freescale/fman/mac.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c index 65df308bad97e..13e67f2864be6 100644 --- a/drivers/net/ethernet/freescale/fman/mac.c +++ b/drivers/net/ethernet/freescale/fman/mac.c @@ -487,12 +487,21 @@ static int mac_probe(struct platform_device *_of_dev) return err; } +static int mac_remove(struct platform_device *pdev) +{ + struct mac_device *mac_dev = platform_get_drvdata(pdev); + + platform_device_unregister(mac_dev->priv->eth_dev); + return 0; +} + static struct platform_driver mac_driver = { .driver = { .name = KBUILD_MODNAME, .of_match_table = mac_match, }, .probe = mac_probe, + .remove = mac_remove, }; builtin_platform_driver(mac_driver); -- GitLab From 8abcaeaed38109e5ccaf40218e0e9e387f07bfe6 Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Fri, 28 Oct 2022 09:52:26 +0000 Subject: [PATCH 1083/1927] cifs: always iterate smb sessions using primary channel smb sessions and tcons currently hang off primary channel only. Secondary channels have the lists as empty. Whenever there's a need to iterate sessions or tcons, we should use the list in the corresponding primary channel. Signed-off-by: Shyam Prasad N Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/misc.c | 6 +++++- fs/cifs/smb2misc.c | 12 ++++++++++-- fs/cifs/smb2ops.c | 6 +++++- fs/cifs/smb2transport.c | 6 +++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/fs/cifs/misc.c b/fs/cifs/misc.c index da51ffd029280..3e68d8208cf5e 100644 --- a/fs/cifs/misc.c +++ b/fs/cifs/misc.c @@ -400,6 +400,7 @@ is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv) { struct smb_hdr *buf = (struct smb_hdr *)buffer; struct smb_com_lock_req *pSMB = (struct smb_com_lock_req *)buf; + struct TCP_Server_Info *pserver; struct cifs_ses *ses; struct cifs_tcon *tcon; struct cifsInodeInfo *pCifsInode; @@ -464,9 +465,12 @@ is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv) if (!(pSMB->LockType & LOCKING_ANDX_OPLOCK_RELEASE)) return false; + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(srv) ? srv->primary_server : srv; + /* look up tcon based on tid & uid */ spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(ses, &srv->smb_ses_list, smb_ses_list) { + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { if (tcon->tid != buf->Tid) continue; diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index a387204779660..e73a3c649b877 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -135,6 +135,7 @@ static __u32 get_neg_ctxt_len(struct smb2_hdr *hdr, __u32 len, int smb2_check_message(char *buf, unsigned int len, struct TCP_Server_Info *server) { + struct TCP_Server_Info *pserver; struct smb2_hdr *shdr = (struct smb2_hdr *)buf; struct smb2_pdu *pdu = (struct smb2_pdu *)shdr; int hdr_size = sizeof(struct smb2_hdr); @@ -143,6 +144,9 @@ smb2_check_message(char *buf, unsigned int len, struct TCP_Server_Info *server) __u32 calc_len; /* calculated length */ __u64 mid; + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + /* * Add function to do table lookup of StructureSize by command * ie Validate the wct via smb2_struct_sizes table above @@ -155,7 +159,7 @@ smb2_check_message(char *buf, unsigned int len, struct TCP_Server_Info *server) /* decrypt frame now that it is completely read in */ spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(iter, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(iter, &pserver->smb_ses_list, smb_ses_list) { if (iter->Suid == le64_to_cpu(thdr->SessionId)) { ses = iter; break; @@ -671,6 +675,7 @@ bool smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server) { struct smb2_oplock_break *rsp = (struct smb2_oplock_break *)buffer; + struct TCP_Server_Info *pserver; struct cifs_ses *ses; struct cifs_tcon *tcon; struct cifsInodeInfo *cinode; @@ -691,9 +696,12 @@ smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server) cifs_dbg(FYI, "oplock level 0x%x\n", rsp->OplockLevel); + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + /* look up tcon based on tid & uid */ spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { spin_lock(&tcon->open_file_lock); diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index 4f53fa012936e..cca12eadbb073 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -2302,14 +2302,18 @@ static void smb2_is_network_name_deleted(char *buf, struct TCP_Server_Info *server) { struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + struct TCP_Server_Info *pserver; struct cifs_ses *ses; struct cifs_tcon *tcon; if (shdr->Status != STATUS_NETWORK_NAME_DELETED) return; + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { if (tcon->tid == le32_to_cpu(shdr->Id.SyncId.TreeId)) { spin_lock(&tcon->tc_lock); diff --git a/fs/cifs/smb2transport.c b/fs/cifs/smb2transport.c index 8e3f26e6f6b9b..8a9d9d08cf19e 100644 --- a/fs/cifs/smb2transport.c +++ b/fs/cifs/smb2transport.c @@ -136,9 +136,13 @@ int smb2_get_sign_key(__u64 ses_id, struct TCP_Server_Info *server, u8 *key) static struct cifs_ses * smb2_find_smb_ses_unlocked(struct TCP_Server_Info *server, __u64 ses_id) { + struct TCP_Server_Info *pserver; struct cifs_ses *ses; - list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { if (ses->Suid != ses_id) continue; ++ses->ses_count; -- GitLab From 23d9b9b757e8007204d8f71448ab55d5ef2ae8e5 Mon Sep 17 00:00:00 2001 From: Shyam Prasad N Date: Fri, 28 Oct 2022 10:01:45 +0000 Subject: [PATCH 1084/1927] cifs: avoid unnecessary iteration of tcp sessions In a few places, we do unnecessary iterations of tcp sessions, even when the server struct is provided. The change avoids it and uses the server struct provided. Signed-off-by: Shyam Prasad N Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/smb2misc.c | 69 +++++++++++++++++++++-------------------- fs/cifs/smb2ops.c | 24 +++++++------- fs/cifs/smb2transport.c | 13 ++++---- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/fs/cifs/smb2misc.c b/fs/cifs/smb2misc.c index e73a3c649b877..572293c18e16f 100644 --- a/fs/cifs/smb2misc.c +++ b/fs/cifs/smb2misc.c @@ -612,51 +612,52 @@ smb2_tcon_find_pending_open_lease(struct cifs_tcon *tcon, } static bool -smb2_is_valid_lease_break(char *buffer) +smb2_is_valid_lease_break(char *buffer, struct TCP_Server_Info *server) { struct smb2_lease_break *rsp = (struct smb2_lease_break *)buffer; - struct TCP_Server_Info *server; + struct TCP_Server_Info *pserver; struct cifs_ses *ses; struct cifs_tcon *tcon; struct cifs_pending_open *open; cifs_dbg(FYI, "Checking for lease break\n"); + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + /* look up tcon based on tid & uid */ spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { - list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { - list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { - spin_lock(&tcon->open_file_lock); - cifs_stats_inc( - &tcon->stats.cifs_stats.num_oplock_brks); - if (smb2_tcon_has_lease(tcon, rsp)) { - spin_unlock(&tcon->open_file_lock); - spin_unlock(&cifs_tcp_ses_lock); - return true; - } - open = smb2_tcon_find_pending_open_lease(tcon, - rsp); - if (open) { - __u8 lease_key[SMB2_LEASE_KEY_SIZE]; - struct tcon_link *tlink; - - tlink = cifs_get_tlink(open->tlink); - memcpy(lease_key, open->lease_key, - SMB2_LEASE_KEY_SIZE); - spin_unlock(&tcon->open_file_lock); - spin_unlock(&cifs_tcp_ses_lock); - smb2_queue_pending_open_break(tlink, - lease_key, - rsp->NewLeaseState); - return true; - } + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->open_file_lock); + cifs_stats_inc( + &tcon->stats.cifs_stats.num_oplock_brks); + if (smb2_tcon_has_lease(tcon, rsp)) { spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + return true; + } + open = smb2_tcon_find_pending_open_lease(tcon, + rsp); + if (open) { + __u8 lease_key[SMB2_LEASE_KEY_SIZE]; + struct tcon_link *tlink; + + tlink = cifs_get_tlink(open->tlink); + memcpy(lease_key, open->lease_key, + SMB2_LEASE_KEY_SIZE); + spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + smb2_queue_pending_open_break(tlink, + lease_key, + rsp->NewLeaseState); + return true; + } + spin_unlock(&tcon->open_file_lock); - if (cached_dir_lease_break(tcon, rsp->LeaseKey)) { - spin_unlock(&cifs_tcp_ses_lock); - return true; - } + if (cached_dir_lease_break(tcon, rsp->LeaseKey)) { + spin_unlock(&cifs_tcp_ses_lock); + return true; } } } @@ -689,7 +690,7 @@ smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server) if (rsp->StructureSize != smb2_rsp_struct_sizes[SMB2_OPLOCK_BREAK_HE]) { if (le16_to_cpu(rsp->StructureSize) == 44) - return smb2_is_valid_lease_break(buffer); + return smb2_is_valid_lease_break(buffer, server); else return false; } diff --git a/fs/cifs/smb2ops.c b/fs/cifs/smb2ops.c index cca12eadbb073..880cd494afea4 100644 --- a/fs/cifs/smb2ops.c +++ b/fs/cifs/smb2ops.c @@ -4268,21 +4268,23 @@ init_sg(int num_rqst, struct smb_rqst *rqst, u8 *sign) static int smb2_get_enc_key(struct TCP_Server_Info *server, __u64 ses_id, int enc, u8 *key) { + struct TCP_Server_Info *pserver; struct cifs_ses *ses; u8 *ses_enc_key; + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { - list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { - if (ses->Suid == ses_id) { - spin_lock(&ses->ses_lock); - ses_enc_key = enc ? ses->smb3encryptionkey : - ses->smb3decryptionkey; - memcpy(key, ses_enc_key, SMB3_ENC_DEC_KEY_SIZE); - spin_unlock(&ses->ses_lock); - spin_unlock(&cifs_tcp_ses_lock); - return 0; - } + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (ses->Suid == ses_id) { + spin_lock(&ses->ses_lock); + ses_enc_key = enc ? ses->smb3encryptionkey : + ses->smb3decryptionkey; + memcpy(key, ses_enc_key, SMB3_ENC_DEC_KEY_SIZE); + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + return 0; } } spin_unlock(&cifs_tcp_ses_lock); diff --git a/fs/cifs/smb2transport.c b/fs/cifs/smb2transport.c index 8a9d9d08cf19e..381babc1212c9 100644 --- a/fs/cifs/smb2transport.c +++ b/fs/cifs/smb2transport.c @@ -77,18 +77,19 @@ static int smb2_get_sign_key(__u64 ses_id, struct TCP_Server_Info *server, u8 *key) { struct cifs_chan *chan; + struct TCP_Server_Info *pserver; struct cifs_ses *ses = NULL; - struct TCP_Server_Info *it = NULL; int i; int rc = 0; spin_lock(&cifs_tcp_ses_lock); - list_for_each_entry(it, &cifs_tcp_ses_list, tcp_ses_list) { - list_for_each_entry(ses, &it->smb_ses_list, smb_ses_list) { - if (ses->Suid == ses_id) - goto found; - } + /* If server is a channel, select the primary channel */ + pserver = CIFS_SERVER_IS_CHAN(server) ? server->primary_server : server; + + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (ses->Suid == ses_id) + goto found; } cifs_server_dbg(VFS, "%s: Could not find session 0x%llx\n", __func__, ses_id); -- GitLab From 542228db2f28fdf775b301f2843e1fe486e7c797 Mon Sep 17 00:00:00 2001 From: ChenXiaoSong Date: Fri, 4 Nov 2022 15:44:41 +0800 Subject: [PATCH 1085/1927] cifs: fix use-after-free on the link name xfstests generic/011 reported use-after-free bug as follows: BUG: KASAN: use-after-free in __d_alloc+0x269/0x859 Read of size 15 at addr ffff8880078933a0 by task dirstress/952 CPU: 1 PID: 952 Comm: dirstress Not tainted 6.1.0-rc3+ #77 Call Trace: __dump_stack+0x23/0x29 dump_stack_lvl+0x51/0x73 print_address_description+0x67/0x27f print_report+0x3e/0x5c kasan_report+0x7b/0xa8 kasan_check_range+0x1b2/0x1c1 memcpy+0x22/0x5d __d_alloc+0x269/0x859 d_alloc+0x45/0x20c d_alloc_parallel+0xb2/0x8b2 lookup_open+0x3b8/0x9f9 open_last_lookups+0x63d/0xc26 path_openat+0x11a/0x261 do_filp_open+0xcc/0x168 do_sys_openat2+0x13b/0x3f7 do_sys_open+0x10f/0x146 __se_sys_creat+0x27/0x2e __x64_sys_creat+0x55/0x6a do_syscall_64+0x40/0x96 entry_SYSCALL_64_after_hwframe+0x63/0xcd Allocated by task 952: kasan_save_stack+0x1f/0x42 kasan_set_track+0x21/0x2a kasan_save_alloc_info+0x17/0x1d __kasan_kmalloc+0x7e/0x87 __kmalloc_node_track_caller+0x59/0x155 kstrndup+0x60/0xe6 parse_mf_symlink+0x215/0x30b check_mf_symlink+0x260/0x36a cifs_get_inode_info+0x14e1/0x1690 cifs_revalidate_dentry_attr+0x70d/0x964 cifs_revalidate_dentry+0x36/0x62 cifs_d_revalidate+0x162/0x446 lookup_open+0x36f/0x9f9 open_last_lookups+0x63d/0xc26 path_openat+0x11a/0x261 do_filp_open+0xcc/0x168 do_sys_openat2+0x13b/0x3f7 do_sys_open+0x10f/0x146 __se_sys_creat+0x27/0x2e __x64_sys_creat+0x55/0x6a do_syscall_64+0x40/0x96 entry_SYSCALL_64_after_hwframe+0x63/0xcd Freed by task 950: kasan_save_stack+0x1f/0x42 kasan_set_track+0x21/0x2a kasan_save_free_info+0x1c/0x34 ____kasan_slab_free+0x1c1/0x1d5 __kasan_slab_free+0xe/0x13 __kmem_cache_free+0x29a/0x387 kfree+0xd3/0x10e cifs_fattr_to_inode+0xb6a/0xc8c cifs_get_inode_info+0x3cb/0x1690 cifs_revalidate_dentry_attr+0x70d/0x964 cifs_revalidate_dentry+0x36/0x62 cifs_d_revalidate+0x162/0x446 lookup_open+0x36f/0x9f9 open_last_lookups+0x63d/0xc26 path_openat+0x11a/0x261 do_filp_open+0xcc/0x168 do_sys_openat2+0x13b/0x3f7 do_sys_open+0x10f/0x146 __se_sys_creat+0x27/0x2e __x64_sys_creat+0x55/0x6a do_syscall_64+0x40/0x96 entry_SYSCALL_64_after_hwframe+0x63/0xcd When opened a symlink, link name is from 'inode->i_link', but it may be reset to a new value when revalidate the dentry. If some processes get the link name on the race scenario, then UAF will happen on link name. Fix this by implementing 'get_link' interface to duplicate the link name. Fixes: 76894f3e2f71 ("cifs: improve symlink handling for smb2+") Signed-off-by: ChenXiaoSong Reviewed-by: Paulo Alcantara (SUSE) Signed-off-by: Steve French --- fs/cifs/cifsfs.c | 26 +++++++++++++++++++++++++- fs/cifs/inode.c | 5 ----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index d0b9fec111aac..fe220686bba4f 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -1143,8 +1143,32 @@ const struct inode_operations cifs_file_inode_ops = { .fiemap = cifs_fiemap, }; +const char *cifs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + char *target_path; + + target_path = kmalloc(PATH_MAX, GFP_KERNEL); + if (!target_path) + return ERR_PTR(-ENOMEM); + + spin_lock(&inode->i_lock); + if (likely(CIFS_I(inode)->symlink_target)) { + strscpy(target_path, CIFS_I(inode)->symlink_target, PATH_MAX); + } else { + kfree(target_path); + target_path = ERR_PTR(-EOPNOTSUPP); + } + spin_unlock(&inode->i_lock); + + if (!IS_ERR(target_path)) + set_delayed_call(done, kfree_link, target_path); + + return target_path; +} + const struct inode_operations cifs_symlink_inode_ops = { - .get_link = simple_get_link, + .get_link = cifs_get_link, .permission = cifs_permission, .listxattr = cifs_listxattr, }; diff --git a/fs/cifs/inode.c b/fs/cifs/inode.c index 9bde08d44617f..4e2ca3c6e5c00 100644 --- a/fs/cifs/inode.c +++ b/fs/cifs/inode.c @@ -215,11 +215,6 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr) kfree(cifs_i->symlink_target); cifs_i->symlink_target = fattr->cf_symlink_target; fattr->cf_symlink_target = NULL; - - if (unlikely(!cifs_i->symlink_target)) - inode->i_link = ERR_PTR(-EOPNOTSUPP); - else - inode->i_link = cifs_i->symlink_target; } spin_unlock(&inode->i_lock); -- GitLab From 46653972e3ea64f79e7f8ae3aa41a4d3fdb70a13 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Mon, 31 Oct 2022 19:25:36 +0800 Subject: [PATCH 1086/1927] capabilities: fix undefined behavior in bit shift for CAP_TO_MASK Shifting signed 32-bit value by 31 bits is undefined, so changing significant bit to unsigned. The UBSAN warning calltrace like below: UBSAN: shift-out-of-bounds in security/commoncap.c:1252:2 left shift of 1 by 31 places cannot be represented in type 'int' Call Trace: dump_stack_lvl+0x7d/0xa5 dump_stack+0x15/0x1b ubsan_epilogue+0xe/0x4e __ubsan_handle_shift_out_of_bounds+0x1e7/0x20c cap_task_prctl+0x561/0x6f0 security_task_prctl+0x5a/0xb0 __x64_sys_prctl+0x61/0x8f0 do_syscall_64+0x58/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: e338d263a76a ("Add 64-bit capability support to the kernel") Signed-off-by: Gaosheng Cui Acked-by: Andrew G. Morgan Reviewed-by: Serge Hallyn Signed-off-by: Paul Moore --- include/uapi/linux/capability.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/capability.h b/include/uapi/linux/capability.h index 463d1ba2232ac..3d61a0ae055d4 100644 --- a/include/uapi/linux/capability.h +++ b/include/uapi/linux/capability.h @@ -426,7 +426,7 @@ struct vfs_ns_cap_data { */ #define CAP_TO_INDEX(x) ((x) >> 5) /* 1 << 5 == bits in __u32 */ -#define CAP_TO_MASK(x) (1 << ((x) & 31)) /* mask for indexed __u32 */ +#define CAP_TO_MASK(x) (1U << ((x) & 31)) /* mask for indexed __u32 */ #endif /* _UAPI_LINUX_CAPABILITY_H */ -- GitLab From 0c423e2ffa7edd3f8f9bcf17ce73fa9c7509b99e Mon Sep 17 00:00:00 2001 From: Xian Wang Date: Fri, 4 Nov 2022 13:29:13 -0700 Subject: [PATCH 1087/1927] ALSA: hda/ca0132: add quirk for EVGA Z390 DARK The Z390 DARK mainboard uses a CA0132 audio controller. The quirk is needed to enable surround sound and 3.5mm headphone jack handling in the front audio connector as well as in the rear of the board when in stereo mode. Page 97 of the linked manual contains instructions to setup the controller. Signed-off-by: Xian Wang Cc: stable@vger.kernel.org Link: https://www.evga.com/support/manuals/files/131-CS-E399.pdf Link: https://lore.kernel.org/r/20221104202913.13904-1-dev@xianwang.io Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_ca0132.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c index 9580fe00cbd92..0a292bf271f2e 100644 --- a/sound/pci/hda/patch_ca0132.c +++ b/sound/pci/hda/patch_ca0132.c @@ -1306,6 +1306,7 @@ static const struct snd_pci_quirk ca0132_quirks[] = { SND_PCI_QUIRK(0x1458, 0xA026, "Gigabyte G1.Sniper Z97", QUIRK_R3DI), SND_PCI_QUIRK(0x1458, 0xA036, "Gigabyte GA-Z170X-Gaming 7", QUIRK_R3DI), SND_PCI_QUIRK(0x3842, 0x1038, "EVGA X99 Classified", QUIRK_R3DI), + SND_PCI_QUIRK(0x3842, 0x1055, "EVGA Z390 DARK", QUIRK_R3DI), SND_PCI_QUIRK(0x1102, 0x0013, "Recon3D", QUIRK_R3D), SND_PCI_QUIRK(0x1102, 0x0018, "Recon3D", QUIRK_R3D), SND_PCI_QUIRK(0x1102, 0x0051, "Sound Blaster AE-5", QUIRK_AE5), -- GitLab From 819b885cd886c193782891c4f51bbcab3de119a4 Mon Sep 17 00:00:00 2001 From: John Thomson Date: Sat, 5 Nov 2022 06:52:41 +1000 Subject: [PATCH 1088/1927] phy: ralink: mt7621-pci: add sentinel to quirks table With mt7621 soc_dev_attr fixed to register the soc as a device, kernel will experience an oops in soc_device_match_attr This quirk test was introduced in the staging driver in commit 9445ccb3714c ("staging: mt7621-pci-phy: add quirks for 'E2' revision using 'soc_device_attribute'"). The staging driver was removed, and later re-added in commit d87da32372a0 ("phy: ralink: Add PHY driver for MT7621 PCIe PHY") for kernel 5.11 Link: https://lore.kernel.org/lkml/26ebbed1-0fe9-4af9-8466-65f841d0b382@app.fastmail.com Fixes: d87da32372a0 ("phy: ralink: Add PHY driver for MT7621 PCIe PHY") Signed-off-by: John Thomson Acked-by: Sergio Paracuellos Link: https://lore.kernel.org/r/20221104205242.3440388-2-git@johnthomson.fastmail.com.au Signed-off-by: Vinod Koul --- drivers/phy/ralink/phy-mt7621-pci.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/phy/ralink/phy-mt7621-pci.c b/drivers/phy/ralink/phy-mt7621-pci.c index 5e6530f545b5c..85888ab2d307a 100644 --- a/drivers/phy/ralink/phy-mt7621-pci.c +++ b/drivers/phy/ralink/phy-mt7621-pci.c @@ -280,7 +280,8 @@ static struct phy *mt7621_pcie_phy_of_xlate(struct device *dev, } static const struct soc_device_attribute mt7621_pci_quirks_match[] = { - { .soc_id = "mt7621", .revision = "E2" } + { .soc_id = "mt7621", .revision = "E2" }, + { /* sentinel */ } }; static const struct regmap_config mt7621_pci_phy_regmap_config = { -- GitLab From bdd6b5624c62d0acd350d07564f1c82fe649235f Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Sat, 5 Nov 2022 09:49:26 -0400 Subject: [PATCH 1089/1927] nfsd: fix use-after-free in nfsd_file_do_acquire tracepoint When we fail to insert into the hashtable with a non-retryable error, we'll free the object and then goto out_status. If the tracepoint is enabled, it'll end up accessing the freed object when it tries to grab the fields out of it. Set nf to NULL after freeing it to avoid the issue. Fixes: 243a5263014a ("nfsd: rework hashtable handling in nfsd_do_file_acquire") Reported-by: kernel test robot Reported-by: Dan Carpenter Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/filecache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index adc4e87a71d25..ec3fceb92236e 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -1076,6 +1076,7 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, goto open_file; nfsd_file_slab_free(&nf->nf_rcu); + nf = NULL; if (ret == -EEXIST) goto retry; trace_nfsd_file_insert_err(rqstp, key.inode, may_flags, ret); -- GitLab From 17a0bc9bd697f75cfdf9b378d5eb2d7409c91340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Henriques?= Date: Wed, 12 Oct 2022 14:13:30 +0100 Subject: [PATCH 1090/1927] ext4: fix BUG_ON() when directory entry has invalid rec_len MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rec_len field in the directory entry has to be a multiple of 4. A corrupted filesystem image can be used to hit a BUG() in ext4_rec_len_to_disk(), called from make_indexed_dir(). ------------[ cut here ]------------ kernel BUG at fs/ext4/ext4.h:2413! ... RIP: 0010:make_indexed_dir+0x53f/0x5f0 ... Call Trace: ? add_dirent_to_buf+0x1b2/0x200 ext4_add_entry+0x36e/0x480 ext4_add_nondir+0x2b/0xc0 ext4_create+0x163/0x200 path_openat+0x635/0xe90 do_filp_open+0xb4/0x160 ? __create_object.isra.0+0x1de/0x3b0 ? _raw_spin_unlock+0x12/0x30 do_sys_openat2+0x91/0x150 __x64_sys_open+0x6c/0xa0 do_syscall_64+0x3c/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 The fix simply adds a call to ext4_check_dir_entry() to validate the directory entry, returning -EFSCORRUPTED if the entry is invalid. CC: stable@kernel.org Link: https://bugzilla.kernel.org/show_bug.cgi?id=216540 Signed-off-by: Luís Henriques Link: https://lore.kernel.org/r/20221012131330.32456-1-lhenriques@suse.de Signed-off-by: Theodore Ts'o --- fs/ext4/namei.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 4183a4cb4a21e..be8136aafa22c 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -2259,8 +2259,16 @@ static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname, memset(de, 0, len); /* wipe old data */ de = (struct ext4_dir_entry_2 *) data2; top = data2 + len; - while ((char *)(de2 = ext4_next_entry(de, blocksize)) < top) + while ((char *)(de2 = ext4_next_entry(de, blocksize)) < top) { + if (ext4_check_dir_entry(dir, NULL, de, bh2, data2, len, + (data2 + (blocksize - csum_size) - + (char *) de))) { + brelse(bh2); + brelse(bh); + return -EFSCORRUPTED; + } de = de2; + } de->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) - (char *) de, blocksize); -- GitLab From 1b8f787ef547230a3249bcf897221ef0cc78481b Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Tue, 18 Oct 2022 10:27:01 +0800 Subject: [PATCH 1091/1927] ext4: fix warning in 'ext4_da_release_space' Syzkaller report issue as follows: EXT4-fs (loop0): Free/Dirty block details EXT4-fs (loop0): free_blocks=0 EXT4-fs (loop0): dirty_blocks=0 EXT4-fs (loop0): Block reservation details EXT4-fs (loop0): i_reserved_data_blocks=0 EXT4-fs warning (device loop0): ext4_da_release_space:1527: ext4_da_release_space: ino 18, to_free 1 with only 0 reserved data blocks ------------[ cut here ]------------ WARNING: CPU: 0 PID: 92 at fs/ext4/inode.c:1528 ext4_da_release_space+0x25e/0x370 fs/ext4/inode.c:1524 Modules linked in: CPU: 0 PID: 92 Comm: kworker/u4:4 Not tainted 6.0.0-syzkaller-09423-g493ffd6605b2 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 09/22/2022 Workqueue: writeback wb_workfn (flush-7:0) RIP: 0010:ext4_da_release_space+0x25e/0x370 fs/ext4/inode.c:1528 RSP: 0018:ffffc900015f6c90 EFLAGS: 00010296 RAX: 42215896cd52ea00 RBX: 0000000000000000 RCX: 42215896cd52ea00 RDX: 0000000000000000 RSI: 0000000080000001 RDI: 0000000000000000 RBP: 1ffff1100e907d96 R08: ffffffff816aa79d R09: fffff520002bece5 R10: fffff520002bece5 R11: 1ffff920002bece4 R12: ffff888021fd2000 R13: ffff88807483ecb0 R14: 0000000000000001 R15: ffff88807483e740 FS: 0000000000000000(0000) GS:ffff8880b9a00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 00005555569ba628 CR3: 000000000c88e000 CR4: 00000000003506f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: ext4_es_remove_extent+0x1ab/0x260 fs/ext4/extents_status.c:1461 mpage_release_unused_pages+0x24d/0xef0 fs/ext4/inode.c:1589 ext4_writepages+0x12eb/0x3be0 fs/ext4/inode.c:2852 do_writepages+0x3c3/0x680 mm/page-writeback.c:2469 __writeback_single_inode+0xd1/0x670 fs/fs-writeback.c:1587 writeback_sb_inodes+0xb3b/0x18f0 fs/fs-writeback.c:1870 wb_writeback+0x41f/0x7b0 fs/fs-writeback.c:2044 wb_do_writeback fs/fs-writeback.c:2187 [inline] wb_workfn+0x3cb/0xef0 fs/fs-writeback.c:2227 process_one_work+0x877/0xdb0 kernel/workqueue.c:2289 worker_thread+0xb14/0x1330 kernel/workqueue.c:2436 kthread+0x266/0x300 kernel/kthread.c:376 ret_from_fork+0x1f/0x30 arch/x86/entry/entry_64.S:306 Above issue may happens as follows: ext4_da_write_begin ext4_create_inline_data ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); ext4_set_inode_flag(inode, EXT4_INODE_INLINE_DATA); __ext4_ioctl ext4_ext_migrate -> will lead to eh->eh_entries not zero, and set extent flag ext4_da_write_begin ext4_da_convert_inline_data_to_extent ext4_da_write_inline_data_begin ext4_da_map_blocks ext4_insert_delayed_block if (!ext4_es_scan_clu(inode, &ext4_es_is_delonly, lblk)) if (!ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk)) ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk)); -> will return 1 allocated = true; ext4_es_insert_delayed_block(inode, lblk, allocated); ext4_writepages mpage_map_and_submit_extent(handle, &mpd, &give_up_on_write); -> return -ENOSPC mpage_release_unused_pages(&mpd, give_up_on_write); -> give_up_on_write == 1 ext4_es_remove_extent ext4_da_release_space(inode, reserved); if (unlikely(to_free > ei->i_reserved_data_blocks)) -> to_free == 1 but ei->i_reserved_data_blocks == 0 -> then trigger warning as above To solve above issue, forbid inode do migrate which has inline data. Cc: stable@kernel.org Reported-by: syzbot+c740bb18df70ad00952e@syzkaller.appspotmail.com Signed-off-by: Ye Bin Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20221018022701.683489-1-yebin10@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/migrate.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 0a220ec9862de..a19a9661646eb 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -424,7 +424,8 @@ int ext4_ext_migrate(struct inode *inode) * already is extent-based, error out. */ if (!ext4_has_feature_extents(inode->i_sb) || - (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) + ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || + ext4_has_inline_data(inode)) return -EINVAL; if (S_ISLNK(inode->i_mode) && inode->i_blocks == 0) -- GitLab From 9f2a1d9fb33a2129a9ba29bc61d3f14adb28ddc2 Mon Sep 17 00:00:00 2001 From: Jason Yan Date: Tue, 25 Oct 2022 12:02:06 +0800 Subject: [PATCH 1092/1927] ext4: fix wrong return err in ext4_load_and_init_journal() The return value is wrong in ext4_load_and_init_journal(). The local variable 'err' need to be initialized before goto out. The original code in __ext4_fill_super() is fine because it has two return values 'ret' and 'err' and 'ret' is initialized as -EINVAL. After we factor out ext4_load_and_init_journal(), this code is broken. So fix it by directly returning -EINVAL in the error handler path. Cc: stable@kernel.org Fixes: 9c1dd22d7422 ("ext4: factor out ext4_load_and_init_journal()") Signed-off-by: Jason Yan Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20221025040206.3134773-1-yanaijie@huawei.com Signed-off-by: Theodore Ts'o --- fs/ext4/super.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/ext4/super.c b/fs/ext4/super.c index d733db8a0b026..6ddebc9f1b90e 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -4886,7 +4886,7 @@ static int ext4_load_and_init_journal(struct super_block *sb, flush_work(&sbi->s_error_work); jbd2_journal_destroy(sbi->s_journal); sbi->s_journal = NULL; - return err; + return -EINVAL; } static int ext4_journal_data_mode_check(struct super_block *sb) -- GitLab From 0d043351e5baf3857f915367deba2a518b6a0809 Mon Sep 17 00:00:00 2001 From: Theodore Ts'o Date: Sat, 5 Nov 2022 23:42:36 -0400 Subject: [PATCH 1093/1927] ext4: fix fortify warning in fs/ext4/fast_commit.c:1551 With the new fortify string system, rework the memcpy to avoid this warning: memcpy: detected field-spanning write (size 60) of single field "&raw_inode->i_generation" at fs/ext4/fast_commit.c:1551 (size 4) Cc: stable@kernel.org Fixes: 54d9469bc515 ("fortify: Add run-time WARN for cross-field memcpy()") Signed-off-by: Theodore Ts'o --- fs/ext4/fast_commit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index ef05bfa87798c..0f6d0a80467d7 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -1521,6 +1521,7 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl, struct ext4_iloc iloc; int inode_len, ino, ret, tag = tl->fc_tag; struct ext4_extent_header *eh; + size_t off_gen = offsetof(struct ext4_inode, i_generation); memcpy(&fc_inode, val, sizeof(fc_inode)); @@ -1548,8 +1549,8 @@ static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl, raw_inode = ext4_raw_inode(&iloc); memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); - memcpy(&raw_inode->i_generation, &raw_fc_inode->i_generation, - inode_len - offsetof(struct ext4_inode, i_generation)); + memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen, + inode_len - off_gen); if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) { eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]); if (eh->eh_magic != EXT4_EXT_MAGIC) { -- GitLab From 6dcabcd398946e2b0b776a8310291aeebe1ca0e6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Sun, 6 Nov 2022 13:17:27 -0700 Subject: [PATCH 1094/1927] io_uring: fix typo in io_uring.h comment Just a basic s/thig/this swap, fixing up a typo introduced by a commit added in the 6.1 release. Fixes: 9cda70f622cd ("io_uring: introduce fixed buffer support for io_uring_cmd") Signed-off-by: Jens Axboe --- include/uapi/linux/io_uring.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index ab7458033ee3b..2df3225b562fa 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -222,7 +222,7 @@ enum io_uring_op { /* * sqe->uring_cmd_flags - * IORING_URING_CMD_FIXED use registered buffer; pass thig flag + * IORING_URING_CMD_FIXED use registered buffer; pass this flag * along with setting sqe->buf_index. */ #define IORING_URING_CMD_FIXED (1U << 0) -- GitLab From 328687151b2a29fe93db4736bfb90da134e52fbf Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 4 Nov 2022 23:35:34 -0700 Subject: [PATCH 1095/1927] mm/slab_common: Restore passing "caller" for tracing The "caller" argument was accidentally being ignored in a few places that were recently refactored. Restore these "caller" arguments, instead of _RET_IP_. Fixes: 11e9734bcb6a ("mm/slab_common: unify NUMA and UMA version of tracepoints") Cc: Hyeonggon Yoo <42.hyeyoo@gmail.com> Cc: Vlastimil Babka Cc: Christoph Lameter Cc: Pekka Enberg Cc: David Rientjes Cc: Joonsoo Kim Cc: Andrew Morton Cc: Roman Gushchin Cc: linux-mm@kvack.org Signed-off-by: Kees Cook Acked-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Signed-off-by: Vlastimil Babka --- mm/slab_common.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index 206e59051c1d6..fa784563e1edb 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -941,7 +941,7 @@ void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ret = __kmalloc_large_node(size, flags, node); - trace_kmalloc(_RET_IP_, ret, size, + trace_kmalloc(caller, ret, size, PAGE_SIZE << get_order(size), flags, node); return ret; } @@ -953,7 +953,7 @@ void *__do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller ret = __kmem_cache_alloc_node(s, flags, node, size, caller); ret = kasan_kmalloc(s, ret, size, flags); - trace_kmalloc(_RET_IP_, ret, size, s->size, flags, node); + trace_kmalloc(caller, ret, size, s->size, flags, node); return ret; } -- GitLab From f0c4d9fc9cc9462659728d168387191387e903cc Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 6 Nov 2022 15:07:11 -0800 Subject: [PATCH 1096/1927] Linux 6.1-rc4 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 226ca8a312d94..ac2ec990422d9 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ VERSION = 6 PATCHLEVEL = 1 SUBLEVEL = 0 -EXTRAVERSION = -rc3 +EXTRAVERSION = -rc4 NAME = Hurr durr I'ma ninja sloth # *DOCUMENTATION* -- GitLab From ea6143a86c67110a2c62deaf70d0b7b92e4f865f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:32 +1000 Subject: [PATCH 1097/1927] drm/nouveau/disp: move and extend the role of outp acquire/release methods There are various pieces of information we pass to NVKM about the next modeset, which are generally used while handling supervisor interrupts. We had to start passing in some information about audio requirements a while back to allocate an appropriate SOR in ACQUIRE, so we may as well move all this type of information here for other protocols too. Certain methods will be blocked on non-acquired outputs now, preventing NULL pointer derefs from KMS driver bugs. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/crc.c | 2 +- drivers/gpu/drm/nouveau/dispnv50/disp.c | 98 ++++++------------- drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 10 -- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 32 ++++++ drivers/gpu/drm/nouveau/include/nvif/outp.h | 10 ++ drivers/gpu/drm/nouveau/nouveau_encoder.h | 1 - drivers/gpu/drm/nouveau/nvif/outp.c | 81 ++++++++++++++- .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 18 ---- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 61 ++++++++++++ 9 files changed, 213 insertions(+), 100 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/crc.c b/drivers/gpu/drm/nouveau/dispnv50/crc.c index b834e8a9ae775..9c942fbd836d0 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/crc.c +++ b/drivers/gpu/drm/nouveau/dispnv50/crc.c @@ -463,7 +463,7 @@ void nv50_crc_atomic_set(struct nv50_head *head, if (!outp) return; - func->set_src(head, outp->or, nv50_crc_source_type(outp, asyh->crc.src), + func->set_src(head, outp->outp.or.id, nv50_crc_source_type(outp, asyh->crc.src), &crc->ctx[crc->ctx_idx]); } diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index aa94f8e284dd0..093321a93046e 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -317,52 +317,6 @@ nv50_outp_dump_caps(struct nouveau_drm *drm, outp->base.base.name, outp->caps.dp_interlace); } -static void -nv50_outp_release(struct nouveau_encoder *nv_encoder) -{ - struct nv50_disp *disp = nv50_disp(nv_encoder->base.base.dev); - struct { - struct nv50_disp_mthd_v1 base; - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_RELEASE, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = nv_encoder->dcb->hashm, - }; - - nvif_mthd(&disp->disp->object, 0, &args, sizeof(args)); - nv_encoder->or = -1; - nv_encoder->link = 0; -} - -static int -nv50_outp_acquire(struct nouveau_encoder *nv_encoder, bool hda) -{ - struct nouveau_drm *drm = nouveau_drm(nv_encoder->base.base.dev); - struct nv50_disp *disp = nv50_disp(drm->dev); - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_acquire_v0 info; - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_ACQUIRE, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = nv_encoder->dcb->hashm, - .info.hda = hda, - }; - int ret; - - ret = nvif_mthd(&disp->disp->object, 0, &args, sizeof(args)); - if (ret) { - NV_ERROR(drm, "error acquiring output path: %d\n", ret); - return ret; - } - - nv_encoder->or = args.info.or; - nv_encoder->link = args.info.link; - return 0; -} - static int nv50_outp_atomic_check_view(struct drm_encoder *encoder, struct drm_crtc_state *crtc_state, @@ -489,9 +443,9 @@ nv50_dac_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *st struct nv50_core *core = nv50_disp(encoder->dev)->core; const u32 ctrl = NVDEF(NV507D, DAC_SET_CONTROL, OWNER, NONE); - core->func->dac->ctrl(core, nv_encoder->or, ctrl, NULL); + core->func->dac->ctrl(core, nv_encoder->outp.or.id, ctrl, NULL); nv_encoder->crtc = NULL; - nv50_outp_release(nv_encoder); + nvif_outp_release(&nv_encoder->outp); } static void @@ -516,9 +470,9 @@ nv50_dac_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta ctrl |= NVDEF(NV507D, DAC_SET_CONTROL, PROTOCOL, RGB_CRT); - nv50_outp_acquire(nv_encoder, false); + nvif_outp_acquire_rgb_crt(&nv_encoder->outp); - core->func->dac->ctrl(core, nv_encoder->or, ctrl, asyh); + core->func->dac->ctrl(core, nv_encoder->outp.or.id, ctrl, asyh); asyh->or.depth = 0; nv_encoder->crtc = &nv_crtc->base; @@ -634,7 +588,7 @@ nv50_audio_component_get_eld(struct device *kdev, int port, int dev_id, nv_connector = nouveau_connector(nv_encoder->audio.connector); nv_crtc = nouveau_crtc(nv_encoder->crtc); - if (!nv_crtc || nv_encoder->or != port || nv_crtc->index != dev_id) + if (!nv_crtc || nv_encoder->outp.or.id != port || nv_crtc->index != dev_id) continue; *enabled = nv_encoder->audio.enabled; @@ -724,6 +678,7 @@ nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nv50_disp *disp = nv50_disp(encoder->dev); + struct nvif_outp *outp = &nv_encoder->outp; struct { struct nv50_disp_mthd_v1 base; struct nv50_disp_sor_hda_eld_v0 eld; @@ -743,8 +698,7 @@ nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) } mutex_unlock(&drm->audio.lock); - nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or, - nv_crtc->index); + nv50_audio_component_eld_notify(drm->audio.component, outp->or.id, nv_crtc->index); } static void @@ -755,6 +709,7 @@ nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc, struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); struct nv50_disp *disp = nv50_disp(encoder->dev); + struct nvif_outp *outp = &nv_encoder->outp; struct __packed { struct { struct nv50_disp_mthd_v1 mthd; @@ -783,8 +738,7 @@ nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc, mutex_unlock(&drm->audio.lock); - nv50_audio_component_eld_notify(drm->audio.component, nv_encoder->or, - nv_crtc->index); + nv50_audio_component_eld_notify(drm->audio.component, outp->or.id, nv_crtc->index); } /****************************************************************************** @@ -1107,10 +1061,12 @@ nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st if (WARN_ON(!mstc)) return; - if (!mstm->links++) - nv50_outp_acquire(mstm->outp, false /*XXX: MST audio.*/); + if (!mstm->links++) { + /*XXX: MST audio. */ + nvif_outp_acquire_dp(&mstm->outp->outp, false); + } - if (mstm->outp->link & 1) + if (mstm->outp->outp.or.link & 1) proto = NV917D_SOR_SET_CONTROL_PROTOCOL_DP_A; else proto = NV917D_SOR_SET_CONTROL_PROTOCOL_DP_B; @@ -1405,7 +1361,7 @@ nv50_mstm_prepare(struct drm_atomic_state *state, if (mstm->disabled) { if (!mstm->links) - nv50_outp_release(mstm->outp); + nvif_outp_release(&mstm->outp->outp); mstm->disabled = false; } } @@ -1623,7 +1579,7 @@ nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head, asyh->or.depth = depth; } - core->func->sor->ctrl(core, nv_encoder->or, nv_encoder->ctrl, asyh); + core->func->sor->ctrl(core, nv_encoder->outp.or.id, nv_encoder->ctrl, asyh); } /* TODO: Should we extend this to PWM-only backlights? @@ -1667,7 +1623,7 @@ nv50_sor_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *st nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0); nv50_audio_disable(encoder, nv_crtc); nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc); - nv50_outp_release(nv_encoder); + nvif_outp_release(&nv_encoder->outp); nv_encoder->crtc = NULL; } @@ -1707,11 +1663,11 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta disp->disp->object.oclass >= GF110_DISP) && drm_detect_monitor_audio(nv_connector->edid)) hda = true; - nv50_outp_acquire(nv_encoder, hda); switch (nv_encoder->dcb->type) { case DCB_OUTPUT_TMDS: - if (nv_encoder->link & 1) { + nvif_outp_acquire_tmds(&nv_encoder->outp, hda); + if (nv_encoder->outp.or.link & 1) { proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_A; /* Only enable dual-link if: * - Need to (i.e. rate > 165MHz) @@ -1758,12 +1714,14 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta lvds.lvds.script |= 0x0200; } + nvif_outp_acquire_lvds(&nv_encoder->outp); nvif_mthd(&disp->disp->object, 0, &lvds, sizeof(lvds)); break; case DCB_OUTPUT_DP: + nvif_outp_acquire_dp(&nv_encoder->outp, hda); depth = nv50_dp_bpc_to_depth(asyh->or.bpc); - if (nv_encoder->link & 1) + if (nv_encoder->outp.or.link & 1) proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_A; else proto = NV887D_SOR_SET_CONTROL_PROTOCOL_DP_B; @@ -1921,9 +1879,9 @@ nv50_pior_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *s struct nv50_core *core = nv50_disp(encoder->dev)->core; const u32 ctrl = NVDEF(NV507D, PIOR_SET_CONTROL, OWNER, NONE); - core->func->pior->ctrl(core, nv_encoder->or, ctrl, NULL); + core->func->pior->ctrl(core, nv_encoder->outp.or.id, ctrl, NULL); nv_encoder->crtc = NULL; - nv50_outp_release(nv_encoder); + nvif_outp_release(&nv_encoder->outp); } static void @@ -1944,8 +1902,6 @@ nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st break; } - nv50_outp_acquire(nv_encoder, false); - switch (asyh->or.bpc) { case 10: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_30_444; break; case 8: asyh->or.depth = NV837D_PIOR_SET_CONTROL_PIXEL_DEPTH_BPP_24_444; break; @@ -1955,15 +1911,19 @@ nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st switch (nv_encoder->dcb->type) { case DCB_OUTPUT_TMDS: + ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC); + nvif_outp_acquire_tmds(&nv_encoder->outp, false); + break; case DCB_OUTPUT_DP: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC); + nvif_outp_acquire_dp(&nv_encoder->outp, false); break; default: BUG(); break; } - core->func->pior->ctrl(core, nv_encoder->or, ctrl, asyh); + core->func->pior->ctrl(core, nv_encoder->outp.or.id, ctrl, asyh); nv_encoder->crtc = &nv_crtc->base; } diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index 56affb606adfb..37e669b9c4dcd 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -28,8 +28,6 @@ struct nv50_disp_scanoutpos_v0 { struct nv50_disp_mthd_v1 { __u8 version; -#define NV50_DISP_MTHD_V1_ACQUIRE 0x01 -#define NV50_DISP_MTHD_V1_RELEASE 0x02 #define NV50_DISP_MTHD_V1_SOR_HDA_ELD 0x21 #define NV50_DISP_MTHD_V1_SOR_HDMI_PWR 0x22 #define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT 0x23 @@ -41,14 +39,6 @@ struct nv50_disp_mthd_v1 { __u8 pad06[2]; }; -struct nv50_disp_acquire_v0 { - __u8 version; - __u8 or; - __u8 link; - __u8 hda; - __u8 pad04[4]; -}; - struct nv50_disp_sor_hda_eld_v0 { __u8 version; __u8 pad01[7]; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index 243bd35d942f9..8bc00a8c525af 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -11,6 +11,8 @@ union nvif_outp_args { }; #define NVIF_OUTP_V0_LOAD_DETECT 0x00 +#define NVIF_OUTP_V0_ACQUIRE 0x01 +#define NVIF_OUTP_V0_RELEASE 0x02 union nvif_outp_load_detect_args { struct nvif_outp_load_detect_v0 { @@ -20,4 +22,34 @@ union nvif_outp_load_detect_args { __u32 data; /*TODO: move vbios loadval parsing into nvkm */ } v0; }; + +union nvif_outp_acquire_args { + struct nvif_outp_acquire_v0 { + __u8 version; +#define NVIF_OUTP_ACQUIRE_V0_RGB_CRT 0x00 +#define NVIF_OUTP_ACQUIRE_V0_TV 0x01 +#define NVIF_OUTP_ACQUIRE_V0_TMDS 0x02 +#define NVIF_OUTP_ACQUIRE_V0_LVDS 0x03 +#define NVIF_OUTP_ACQUIRE_V0_DP 0x04 + __u8 proto; + __u8 or; + __u8 link; + __u8 pad04[4]; + union { + struct { + __u8 hda; + __u8 pad01[7]; + } tmds; + struct { + __u8 hda; + __u8 pad01[7]; + } dp; + }; + } v0; +}; + +union nvif_outp_release_args { + struct nvif_outp_release_vn { + } vn; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index 0d6aa07a9184a..2a45b57b1f75c 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -6,9 +6,19 @@ struct nvif_disp; struct nvif_outp { struct nvif_object object; + + struct { + int id; + int link; + } or; }; int nvif_outp_ctor(struct nvif_disp *, const char *name, int id, struct nvif_outp *); void nvif_outp_dtor(struct nvif_outp *); int nvif_outp_load_detect(struct nvif_outp *, u32 loadval); +int nvif_outp_acquire_rgb_crt(struct nvif_outp *); +int nvif_outp_acquire_tmds(struct nvif_outp *, bool hda); +int nvif_outp_acquire_lvds(struct nvif_outp *); +int nvif_outp_acquire_dp(struct nvif_outp *, bool hda); +void nvif_outp_release(struct nvif_outp *); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h index b72e5783a00f3..b6c51fc60d137 100644 --- a/drivers/gpu/drm/nouveau/nouveau_encoder.h +++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h @@ -48,7 +48,6 @@ struct nouveau_encoder { struct dcb_output *dcb; struct nvif_outp outp; int or; - int link; struct i2c_adapter *i2c; struct nvkm_i2c_aux *aux; diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 7bfe91a8d6f9d..bd20f75045dc7 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -26,6 +26,81 @@ #include #include +void +nvif_outp_release(struct nvif_outp *outp) +{ + int ret = nvif_mthd(&outp->object, NVIF_OUTP_V0_RELEASE, NULL, 0); + NVIF_ERRON(ret, &outp->object, "[RELEASE]"); + outp->or.id = -1; +} + +static inline int +nvif_outp_acquire(struct nvif_outp *outp, u8 proto, struct nvif_outp_acquire_v0 *args) +{ + int ret; + + args->version = 0; + args->proto = proto; + + ret = nvif_mthd(&outp->object, NVIF_OUTP_V0_ACQUIRE, args, sizeof(*args)); + if (ret) + return ret; + + outp->or.id = args->or; + outp->or.link = args->link; + return 0; +} + +int +nvif_outp_acquire_dp(struct nvif_outp *outp, bool hda) +{ + struct nvif_outp_acquire_v0 args; + int ret; + + args.dp.hda = hda; + + ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_DP, &args); + NVIF_ERRON(ret, &outp->object, + "[ACQUIRE proto:DP hda:%d] or:%d link:%d", args.dp.hda, args.or, args.link); + return ret; +} + +int +nvif_outp_acquire_lvds(struct nvif_outp *outp) +{ + struct nvif_outp_acquire_v0 args; + int ret; + + ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_LVDS, &args); + NVIF_ERRON(ret, &outp->object, "[ACQUIRE proto:LVDS] or:%d link:%d", args.or, args.link); + return ret; +} + +int +nvif_outp_acquire_tmds(struct nvif_outp *outp, bool hda) +{ + struct nvif_outp_acquire_v0 args; + int ret; + + args.tmds.hda = hda; + + ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_TMDS, &args); + NVIF_ERRON(ret, &outp->object, + "[ACQUIRE proto:TMDS hda:%d] or:%d link:%d", args.tmds.hda, args.or, args.link); + return ret; +} + +int +nvif_outp_acquire_rgb_crt(struct nvif_outp *outp) +{ + struct nvif_outp_acquire_v0 args; + int ret; + + ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_RGB_CRT, &args); + NVIF_ERRON(ret, &outp->object, "[ACQUIRE proto:RGB_CRT] or:%d", args.or); + return ret; +} + int nvif_outp_load_detect(struct nvif_outp *outp, u32 loadval) { @@ -58,5 +133,9 @@ nvif_outp_ctor(struct nvif_disp *disp, const char *name, int id, struct nvif_out ret = nvif_object_ctor(&disp->object, name ?: "nvifOutp", id, NVIF_CLASS_OUTP, &args, sizeof(args), &outp->object); NVIF_ERRON(ret, &disp->object, "[NEW outp id:%d]", id); - return ret; + if (ret) + return ret; + + outp->or.id = -1; + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index 0af45ccd140c5..341f244cddb71 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -91,24 +91,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) } switch (mthd * !!outp) { - case NV50_DISP_MTHD_V1_ACQUIRE: { - union { - struct nv50_disp_acquire_v0 v0; - } *args = data; - int ret = -ENOSYS; - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.hda); - if (ret == 0) { - args->v0.or = outp->ior->id; - args->v0.link = outp->ior->asy.link; - } - } - return ret; - } - break; - case NV50_DISP_MTHD_V1_RELEASE: - nvkm_outp_release(outp, NVKM_OUTP_USER); - return 0; case NV50_DISP_MTHD_V1_SOR_HDA_ELD: { union { struct nv50_disp_sor_hda_eld_v0 v0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index abedb3e863610..edbed699ade2b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -25,6 +25,49 @@ #include +static int +nvkm_uoutp_mthd_release(struct nvkm_outp *outp, void *argv, u32 argc) +{ + union nvif_outp_release_args *args = argv; + + if (argc != sizeof(args->vn)) + return -ENOSYS; + + nvkm_outp_release(outp, NVKM_OUTP_USER); + return 0; +} + +static int +nvkm_uoutp_mthd_acquire(struct nvkm_outp *outp, void *argv, u32 argc) +{ + union nvif_outp_acquire_args *args = argv; + int ret; + + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + switch (args->v0.proto) { + case NVIF_OUTP_ACQUIRE_V0_RGB_CRT: + case NVIF_OUTP_ACQUIRE_V0_LVDS: + ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, false); + break; + case NVIF_OUTP_ACQUIRE_V0_TMDS: + case NVIF_OUTP_ACQUIRE_V0_DP: + ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.dp.hda); + break; + default: + ret = -EINVAL; + break; + } + + if (ret) + return ret; + + args->v0.or = outp->ior->id; + args->v0.link = outp->ior->asy.link; + return 0; +} + static int nvkm_uoutp_mthd_load_detect(struct nvkm_outp *outp, void *argv, u32 argc) { @@ -48,11 +91,24 @@ nvkm_uoutp_mthd_load_detect(struct nvkm_outp *outp, void *argv, u32 argc) return ret; } +static int +nvkm_uoutp_mthd_acquired(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc) +{ + switch (mthd) { + case NVIF_OUTP_V0_RELEASE : return nvkm_uoutp_mthd_release (outp, argv, argc); + default: + break; + } + + return -EINVAL; +} + static int nvkm_uoutp_mthd_noacquire(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc) { switch (mthd) { case NVIF_OUTP_V0_LOAD_DETECT: return nvkm_uoutp_mthd_load_detect(outp, argv, argc); + case NVIF_OUTP_V0_ACQUIRE : return nvkm_uoutp_mthd_acquire (outp, argv, argc); default: break; } @@ -73,6 +129,11 @@ nvkm_uoutp_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) if (ret <= 0) goto done; + if (outp->ior) + ret = nvkm_uoutp_mthd_acquired(outp, mthd, argv, argc); + else + ret = -EIO; + done: mutex_unlock(&disp->super.mutex); return ret; -- GitLab From 9793083f1dd9da8dda0ef68e90934dd7d112203b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:32 +1000 Subject: [PATCH 1098/1927] drm/nouveau/disp: move LVDS protocol information into acquire Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 32 ++++++------------- drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 8 ----- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 5 +++ drivers/gpu/drm/nouveau/include/nvif/outp.h | 2 +- .../drm/nouveau/include/nvkm/engine/disp.h | 8 +---- drivers/gpu/drm/nouveau/nvif/outp.c | 9 ++++-- .../gpu/drm/nouveau/nvkm/engine/disp/nv50.c | 10 +++--- .../gpu/drm/nouveau/nvkm/engine/disp/outp.h | 5 +++ .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 16 ---------- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 16 +++++++++- 10 files changed, 50 insertions(+), 61 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 093321a93046e..0a8404686f169 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -1635,15 +1635,6 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta struct nv50_head_atom *asyh = nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base)); struct drm_display_mode *mode = &asyh->state.adjusted_mode; - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_lvds_script_v0 lvds; - } lvds = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = nv_encoder->dcb->hashm, - }; struct nv50_disp *disp = nv50_disp(encoder->dev); struct drm_device *dev = encoder->dev; struct nouveau_drm *drm = nouveau_drm(dev); @@ -1652,7 +1643,7 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta struct nouveau_backlight *backlight; #endif struct nvbios *bios = &drm->vbios; - bool hda = false; + bool lvds_dual = false, lvds_8bpc = false, hda = false; u8 proto = NV507D_SOR_SET_CONTROL_PROTOCOL_CUSTOM; u8 depth = NV837D_SOR_SET_CONTROL_PIXEL_DEPTH_DEFAULT; @@ -1689,33 +1680,30 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta proto = NV507D_SOR_SET_CONTROL_PROTOCOL_LVDS_CUSTOM; if (bios->fp_no_ddc) { - if (bios->fp.dual_link) - lvds.lvds.script |= 0x0100; - if (bios->fp.if_is_24bit) - lvds.lvds.script |= 0x0200; + lvds_dual = bios->fp.dual_link; + lvds_8bpc = bios->fp.if_is_24bit; } else { if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) { if (((u8 *)nv_connector->edid)[121] == 2) - lvds.lvds.script |= 0x0100; + lvds_dual = true; } else if (mode->clock >= bios->fp.duallink_transition_clk) { - lvds.lvds.script |= 0x0100; + lvds_dual = true; } - if (lvds.lvds.script & 0x0100) { + if (lvds_dual) { if (bios->fp.strapless_is_24bit & 2) - lvds.lvds.script |= 0x0200; + lvds_8bpc = true; } else { if (bios->fp.strapless_is_24bit & 1) - lvds.lvds.script |= 0x0200; + lvds_8bpc = true; } if (asyh->or.bpc == 8) - lvds.lvds.script |= 0x0200; + lvds_8bpc = true; } - nvif_outp_acquire_lvds(&nv_encoder->outp); - nvif_mthd(&disp->disp->object, 0, &lvds, sizeof(lvds)); + nvif_outp_acquire_lvds(&nv_encoder->outp, lvds_dual, lvds_8bpc); break; case DCB_OUTPUT_DP: nvif_outp_acquire_dp(&nv_encoder->outp, hda); diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index 37e669b9c4dcd..f371fc7a1d100 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -30,7 +30,6 @@ struct nv50_disp_mthd_v1 { __u8 version; #define NV50_DISP_MTHD_V1_SOR_HDA_ELD 0x21 #define NV50_DISP_MTHD_V1_SOR_HDMI_PWR 0x22 -#define NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT 0x23 #define NV50_DISP_MTHD_V1_SOR_DP_MST_LINK 0x25 #define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI 0x26 __u8 method; @@ -58,13 +57,6 @@ struct nv50_disp_sor_hdmi_pwr_v0 { __u8 pad07[1]; }; -struct nv50_disp_sor_lvds_script_v0 { - __u8 version; - __u8 pad01[1]; - __u16 script; - __u8 pad04[4]; -}; - struct nv50_disp_sor_dp_mst_link_v0 { __u8 version; __u8 state; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index 8bc00a8c525af..4fcb9e4f5d766 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -40,6 +40,11 @@ union nvif_outp_acquire_args { __u8 hda; __u8 pad01[7]; } tmds; + struct { + __u8 dual; + __u8 bpc8; + __u8 pad02[6]; + } lvds; struct { __u8 hda; __u8 pad01[7]; diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index 2a45b57b1f75c..e247d441afc1d 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -18,7 +18,7 @@ void nvif_outp_dtor(struct nvif_outp *); int nvif_outp_load_detect(struct nvif_outp *, u32 loadval); int nvif_outp_acquire_rgb_crt(struct nvif_outp *); int nvif_outp_acquire_tmds(struct nvif_outp *, bool hda); -int nvif_outp_acquire_lvds(struct nvif_outp *); +int nvif_outp_acquire_lvds(struct nvif_outp *, bool dual, bool bpc8); int nvif_outp_acquire_dp(struct nvif_outp *, bool hda); void nvif_outp_release(struct nvif_outp *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h index 8b5d8a434be86..eaf10f5d505e1 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h @@ -31,13 +31,7 @@ struct nvkm_disp { struct { unsigned long mask; int nr; - } wndw, head, dac; - - struct { - unsigned long mask; - int nr; - u32 lvdsconf; - } sor; + } wndw, head, dac, sor; struct { unsigned long mask; diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index bd20f75045dc7..58cd629657fb9 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -66,13 +66,18 @@ nvif_outp_acquire_dp(struct nvif_outp *outp, bool hda) } int -nvif_outp_acquire_lvds(struct nvif_outp *outp) +nvif_outp_acquire_lvds(struct nvif_outp *outp, bool dual, bool bpc8) { struct nvif_outp_acquire_v0 args; int ret; + args.lvds.dual = dual; + args.lvds.bpc8 = bpc8; + ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_LVDS, &args); - NVIF_ERRON(ret, &outp->object, "[ACQUIRE proto:LVDS] or:%d link:%d", args.or, args.link); + NVIF_ERRON(ret, &outp->object, + "[ACQUIRE proto:LVDS dual:%d 8bpc:%d] or:%d link:%d", + args.lvds.dual, args.lvds.bpc8, args.or, args.link); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c index a46e13cc9ff13..55ac6d70e9500 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c @@ -1238,6 +1238,8 @@ nv50_disp_super_2_2(struct nvkm_disp *disp, struct nvkm_head *head) if (!ior) return; + outp = ior->asy.outp; + /* For some reason, NVIDIA decided not to: * * A) Give dual-link LVDS a separate EVO protocol, like for TMDS. @@ -1247,13 +1249,13 @@ nv50_disp_super_2_2(struct nvkm_disp *disp, struct nvkm_head *head) * Override the values we usually read from HW with the same * data we pass though an ioctl instead. */ - if (ior->type == SOR && ior->asy.proto == LVDS) { - head->asy.or.depth = (disp->sor.lvdsconf & 0x0200) ? 24 : 18; - ior->asy.link = (disp->sor.lvdsconf & 0x0100) ? 3 : 1; + if (outp && ior->type == SOR && ior->asy.proto == LVDS) { + head->asy.or.depth = outp->lvds.bpc8 ? 24 : 18; + ior->asy.link = outp->lvds.dual ? 3 : 1; } /* Handle any link training, etc. */ - if ((outp = ior->asy.outp) && outp->func->acquire) + if (outp && outp->func->acquire) outp->func->acquire(outp); /* Execute OnInt2 IED script. */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h index 3f3924c419571..16aadcedcbda3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h @@ -27,6 +27,11 @@ struct nvkm_outp { struct nvkm_ior *ior; union { + struct { + bool dual; + bool bpc8; + } lvds; + struct { struct nvbios_dpout info; u8 version; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index 341f244cddb71..f028ab17f8cfd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -169,22 +169,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) return 0; } break; - case NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT: { - union { - struct nv50_disp_sor_lvds_script_v0 v0; - } *args = data; - int ret = -ENOSYS; - nvif_ioctl(object, "disp sor lvds script size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "disp sor lvds script " - "vers %d name %04x\n", - args->v0.version, args->v0.script); - disp->sor.lvdsconf = args->v0.script; - return 0; - } else - return ret; - } - break; case NV50_DISP_MTHD_V1_SOR_DP_MST_LINK: { union { struct nv50_disp_sor_dp_mst_link_v0 v0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index edbed699ade2b..7e0cef0f80a9a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -37,6 +37,18 @@ nvkm_uoutp_mthd_release(struct nvkm_outp *outp, void *argv, u32 argc) return 0; } +static int +nvkm_uoutp_mthd_acquire_lvds(struct nvkm_outp *outp, bool dual, bool bpc8) +{ + if (outp->info.type != DCB_OUTPUT_LVDS) + return -EINVAL; + + outp->lvds.dual = dual; + outp->lvds.bpc8 = bpc8; + + return nvkm_outp_acquire(outp, NVKM_OUTP_USER, false); +} + static int nvkm_uoutp_mthd_acquire(struct nvkm_outp *outp, void *argv, u32 argc) { @@ -48,13 +60,15 @@ nvkm_uoutp_mthd_acquire(struct nvkm_outp *outp, void *argv, u32 argc) switch (args->v0.proto) { case NVIF_OUTP_ACQUIRE_V0_RGB_CRT: - case NVIF_OUTP_ACQUIRE_V0_LVDS: ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, false); break; case NVIF_OUTP_ACQUIRE_V0_TMDS: case NVIF_OUTP_ACQUIRE_V0_DP: ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.dp.hda); break; + case NVIF_OUTP_ACQUIRE_V0_LVDS: + ret = nvkm_uoutp_mthd_acquire_lvds(outp, args->v0.lvds.dual, args->v0.lvds.bpc8); + break; default: ret = -EINVAL; break; -- GitLab From 59dc2a7e7c6a0d4a2a2c4dafea874af484fc0592 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 31 Oct 2022 11:17:45 +0100 Subject: [PATCH 1099/1927] ACPI: video: Improve Chromebook checks 2 improvements for the Chromebook handling in acpi_video_get_backlight_type(): 1. Also check for the "GOOG000C" ACPI HID used on some models 2. Move the Chromebook check to above the ACPI-video check normally Chromebooks don't have ACPI video backlight support, but when flashed with upstream coreboot builds they may have ACPI video backlight support, but native should still be used/preferred then. Suggested-by: Mr. Chromebox Signed-off-by: Hans de Goede --- drivers/acpi/video_detect.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 9cd8797d12bb4..06aaec2e378bd 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -670,7 +670,7 @@ static const struct dmi_system_id video_detect_dmi_table[] = { static bool google_cros_ec_present(void) { - return acpi_dev_found("GOOG0004"); + return acpi_dev_found("GOOG0004") || acpi_dev_found("GOOG000C"); } /* @@ -718,6 +718,10 @@ static enum acpi_backlight_type __acpi_video_get_backlight_type(bool native) if (apple_gmux_present()) return acpi_backlight_apple_gmux; + /* Chromebooks should always prefer native backlight control. */ + if (google_cros_ec_present() && native_available) + return acpi_backlight_native; + /* On systems with ACPI video use either native or ACPI video. */ if (video_caps & ACPI_VIDEO_BACKLIGHT) { /* @@ -735,13 +739,6 @@ static enum acpi_backlight_type __acpi_video_get_backlight_type(bool native) return acpi_backlight_video; } - /* - * Chromebooks that don't have backlight handle in ACPI table - * are supposed to use native backlight if it's available. - */ - if (google_cros_ec_present() && native_available) - return acpi_backlight_native; - /* No ACPI video (old hw), use vendor specific fw methods. */ return acpi_backlight_vendor; } -- GitLab From 6973091d1b50ab4042f6a2d495f59e9db3662ab8 Mon Sep 17 00:00:00 2001 From: Nico Boehr Date: Tue, 11 Oct 2022 18:07:12 +0200 Subject: [PATCH 1100/1927] KVM: s390: pv: don't allow userspace to set the clock under PV When running under PV, the guest's TOD clock is under control of the ultravisor and the hypervisor isn't allowed to change it. Hence, don't allow userspace to change the guest's TOD clock by returning -EOPNOTSUPP. When userspace changes the guest's TOD clock, KVM updates its kvm.arch.epoch field and, in addition, the epoch field in all state descriptions of all VCPUs. But, under PV, the ultravisor will ignore the epoch field in the state description and simply overwrite it on next SIE exit with the actual guest epoch. This leads to KVM having an incorrect view of the guest's TOD clock: it has updated its internal kvm.arch.epoch field, but the ultravisor ignores the field in the state description. Whenever a guest is now waiting for a clock comparator, KVM will incorrectly calculate the time when the guest should wake up, possibly causing the guest to sleep for much longer than expected. With this change, kvm_s390_set_tod() will now take the kvm->lock to be able to call kvm_s390_pv_is_protected(). Since kvm_s390_set_tod_clock() also takes kvm->lock, use __kvm_s390_set_tod_clock() instead. The function kvm_s390_set_tod_clock is now unused, hence remove it. Update the documentation to indicate the TOD clock attr calls can now return -EOPNOTSUPP. Fixes: 0f3035047140 ("KVM: s390: protvirt: Do only reset registers that are accessible") Reported-by: Marc Hartmayer Signed-off-by: Nico Boehr Reviewed-by: Claudio Imbrenda Reviewed-by: Janosch Frank Link: https://lore.kernel.org/r/20221011160712.928239-2-nrb@linux.ibm.com Message-Id: <20221011160712.928239-2-nrb@linux.ibm.com> Signed-off-by: Janosch Frank --- Documentation/virt/kvm/devices/vm.rst | 3 +++ arch/s390/kvm/kvm-s390.c | 26 +++++++++++++++++--------- arch/s390/kvm/kvm-s390.h | 1 - 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Documentation/virt/kvm/devices/vm.rst b/Documentation/virt/kvm/devices/vm.rst index 0aa5b1cfd700c..60acc39e0e937 100644 --- a/Documentation/virt/kvm/devices/vm.rst +++ b/Documentation/virt/kvm/devices/vm.rst @@ -215,6 +215,7 @@ KVM_S390_VM_TOD_EXT). :Parameters: address of a buffer in user space to store the data (u8) to :Returns: -EFAULT if the given address is not accessible from kernel space; -EINVAL if setting the TOD clock extension to != 0 is not supported + -EOPNOTSUPP for a PV guest (TOD managed by the ultravisor) 3.2. ATTRIBUTE: KVM_S390_VM_TOD_LOW ----------------------------------- @@ -224,6 +225,7 @@ the POP (u64). :Parameters: address of a buffer in user space to store the data (u64) to :Returns: -EFAULT if the given address is not accessible from kernel space + -EOPNOTSUPP for a PV guest (TOD managed by the ultravisor) 3.3. ATTRIBUTE: KVM_S390_VM_TOD_EXT ----------------------------------- @@ -237,6 +239,7 @@ it, it is stored as 0 and not allowed to be set to a value != 0. (kvm_s390_vm_tod_clock) to :Returns: -EFAULT if the given address is not accessible from kernel space; -EINVAL if setting the TOD clock extension to != 0 is not supported + -EOPNOTSUPP for a PV guest (TOD managed by the ultravisor) 4. GROUP: KVM_S390_VM_CRYPTO ============================ diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 45d4b8182b073..bc491a73815c3 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -1207,6 +1207,8 @@ static int kvm_s390_vm_get_migration(struct kvm *kvm, return 0; } +static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod); + static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr) { struct kvm_s390_vm_tod_clock gtod; @@ -1216,7 +1218,7 @@ static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr) if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx) return -EINVAL; - kvm_s390_set_tod_clock(kvm, >od); + __kvm_s390_set_tod_clock(kvm, >od); VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx", gtod.epoch_idx, gtod.tod); @@ -1247,7 +1249,7 @@ static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) sizeof(gtod.tod))) return -EFAULT; - kvm_s390_set_tod_clock(kvm, >od); + __kvm_s390_set_tod_clock(kvm, >od); VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod); return 0; } @@ -1259,6 +1261,16 @@ static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr) if (attr->flags) return -EINVAL; + mutex_lock(&kvm->lock); + /* + * For protected guests, the TOD is managed by the ultravisor, so trying + * to change it will never bring the expected results. + */ + if (kvm_s390_pv_is_protected(kvm)) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + switch (attr->attr) { case KVM_S390_VM_TOD_EXT: ret = kvm_s390_set_tod_ext(kvm, attr); @@ -1273,6 +1285,9 @@ static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr) ret = -ENXIO; break; } + +out_unlock: + mutex_unlock(&kvm->lock); return ret; } @@ -4377,13 +4392,6 @@ static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_t preempt_enable(); } -void kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod) -{ - mutex_lock(&kvm->lock); - __kvm_s390_set_tod_clock(kvm, gtod); - mutex_unlock(&kvm->lock); -} - int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod) { if (!mutex_trylock(&kvm->lock)) diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index f6fd668f887e8..4755492dfabc6 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -363,7 +363,6 @@ int kvm_s390_handle_sigp(struct kvm_vcpu *vcpu); int kvm_s390_handle_sigp_pei(struct kvm_vcpu *vcpu); /* implemented in kvm-s390.c */ -void kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod); int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod); long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable); int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long addr); -- GitLab From b6662e37772715447aeff2538444ff291e02ea31 Mon Sep 17 00:00:00 2001 From: Rafael Mendonca Date: Tue, 25 Oct 2022 22:32:33 -0300 Subject: [PATCH 1101/1927] KVM: s390: pci: Fix allocation size of aift kzdev elements The 'kzdev' field of struct 'zpci_aift' is an array of pointers to 'kvm_zdev' structs. Allocate the proper size accordingly. Reported by Coccinelle: WARNING: Use correct pointer type argument for sizeof Fixes: 98b1d33dac5f ("KVM: s390: pci: do initial setup for AEN interpretation") Signed-off-by: Rafael Mendonca Reviewed-by: Christian Borntraeger Reviewed-by: Matthew Rosato Link: https://lore.kernel.org/r/20221026013234.960859-1-rafaelmendsr@gmail.com Message-Id: <20221026013234.960859-1-rafaelmendsr@gmail.com> Signed-off-by: Janosch Frank --- arch/s390/kvm/pci.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index c50c1645c0aec..ded1af2ddae99 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -126,7 +126,7 @@ int kvm_s390_pci_aen_init(u8 nisc) return -EPERM; mutex_lock(&aift->aift_lock); - aift->kzdev = kcalloc(ZPCI_NR_DEVICES, sizeof(struct kvm_zdev), + aift->kzdev = kcalloc(ZPCI_NR_DEVICES, sizeof(struct kvm_zdev *), GFP_KERNEL); if (!aift->kzdev) { rc = -ENOMEM; -- GitLab From 531705a765493655472c993627106e19f7e5a6d2 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Thu, 3 Nov 2022 17:05:37 +0800 Subject: [PATCH 1102/1927] net: lapbether: fix issue of dev reference count leakage in lapbeth_device_event() When following tests are performed, it will cause dev reference counting leakage. a)ip link add bond2 type bond mode balance-rr b)ip link set bond2 up c)ifenslave -f bond2 rose1 d)ip link del bond2 When new bond device is created, the default type of the bond device is ether. And the bond device is up, lapbeth_device_event() receives the message and creates a new lapbeth device. In this case, the reference count value of dev is hold once. But after "ifenslave -f bond2 rose1" command is executed, the type of the bond device is changed to rose. When the bond device is unregistered, lapbeth_device_event() will not put the dev reference count. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Zhengchao Shao Signed-off-by: David S. Miller --- drivers/net/wan/lapbether.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/wan/lapbether.c b/drivers/net/wan/lapbether.c index 960f1393595cc..cb360dca32509 100644 --- a/drivers/net/wan/lapbether.c +++ b/drivers/net/wan/lapbether.c @@ -446,7 +446,7 @@ static int lapbeth_device_event(struct notifier_block *this, if (dev_net(dev) != &init_net) return NOTIFY_DONE; - if (!dev_is_ethdev(dev)) + if (!dev_is_ethdev(dev) && !lapbeth_get_x25_dev(dev)) return NOTIFY_DONE; switch (event) { -- GitLab From 85cbaf032d3cd9f595152625eda5d4ecb1d6d78d Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Thu, 3 Nov 2022 17:09:05 +0800 Subject: [PATCH 1103/1927] hamradio: fix issue of dev reference count leakage in bpq_device_event() When following tests are performed, it will cause dev reference counting leakage. a)ip link add bond2 type bond mode balance-rr b)ip link set bond2 up c)ifenslave -f bond2 rose1 d)ip link del bond2 When new bond device is created, the default type of the bond device is ether. And the bond device is up, bpq_device_event() receives the message and creates a new bpq device. In this case, the reference count value of dev is hold once. But after "ifenslave -f bond2 rose1" command is executed, the type of the bond device is changed to rose. When the bond device is unregistered, bpq_device_event() will not put the dev reference count. Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Signed-off-by: Zhengchao Shao Signed-off-by: David S. Miller --- drivers/net/hamradio/bpqether.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/hamradio/bpqether.c b/drivers/net/hamradio/bpqether.c index 30af0081e2bef..83a16d10eedbc 100644 --- a/drivers/net/hamradio/bpqether.c +++ b/drivers/net/hamradio/bpqether.c @@ -533,7 +533,7 @@ static int bpq_device_event(struct notifier_block *this, if (!net_eq(dev_net(dev), &init_net)) return NOTIFY_DONE; - if (!dev_is_ethdev(dev)) + if (!dev_is_ethdev(dev) && !bpq_get_ax25_dev(dev)) return NOTIFY_DONE; switch (event) { -- GitLab From f25caaca424703d5a0607310f0452f978f1f78d9 Mon Sep 17 00:00:00 2001 From: HW He Date: Thu, 3 Nov 2022 18:40:00 +0800 Subject: [PATCH 1104/1927] net: wwan: iosm: fix memory leak in ipc_wwan_dellink IOSM driver registers network device without setting the needs_free_netdev flag, and does NOT call free_netdev() when unregisters network device, which causes a memory leak. This patch sets needs_free_netdev to true when registers network device, which makes netdev subsystem call free_netdev() automatically after unregister_netdevice(). Fixes: 2a54f2c77934 ("net: iosm: net driver") Signed-off-by: HW He Reviewed-by: Loic Poulain Signed-off-by: Zhaoping Shu Signed-off-by: David S. Miller --- drivers/net/wwan/iosm/iosm_ipc_wwan.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wwan/iosm/iosm_ipc_wwan.c b/drivers/net/wwan/iosm/iosm_ipc_wwan.c index 2f1f8b5d5b595..0108d8d01ff26 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_wwan.c +++ b/drivers/net/wwan/iosm/iosm_ipc_wwan.c @@ -168,6 +168,7 @@ static void ipc_wwan_setup(struct net_device *iosm_dev) iosm_dev->max_mtu = ETH_MAX_MTU; iosm_dev->flags = IFF_POINTOPOINT | IFF_NOARP; + iosm_dev->needs_free_netdev = true; iosm_dev->netdev_ops = &ipc_inm_ops; } -- GitLab From 668205b9c9f94d5ed6ab00cce9a46a654c2b5d16 Mon Sep 17 00:00:00 2001 From: HW He Date: Thu, 3 Nov 2022 18:54:19 +0800 Subject: [PATCH 1105/1927] net: wwan: mhi: fix memory leak in mhi_mbim_dellink MHI driver registers network device without setting the needs_free_netdev flag, and does NOT call free_netdev() when unregisters network device, which causes a memory leak. This patch sets needs_free_netdev to true when registers network device, which makes netdev subsystem call free_netdev() automatically after unregister_netdevice(). Fixes: aa730a9905b7 ("net: wwan: Add MHI MBIM network driver") Signed-off-by: HW He Signed-off-by: Zhaoping Shu Signed-off-by: David S. Miller --- drivers/net/wwan/mhi_wwan_mbim.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wwan/mhi_wwan_mbim.c b/drivers/net/wwan/mhi_wwan_mbim.c index 6872782e8dd89..ef70bb7c88ad6 100644 --- a/drivers/net/wwan/mhi_wwan_mbim.c +++ b/drivers/net/wwan/mhi_wwan_mbim.c @@ -582,6 +582,7 @@ static void mhi_mbim_setup(struct net_device *ndev) ndev->min_mtu = ETH_MIN_MTU; ndev->max_mtu = MHI_MAX_BUF_SZ - ndev->needed_headroom; ndev->tx_queue_len = 1000; + ndev->needs_free_netdev = true; } static const struct wwan_ops mhi_mbim_wwan_ops = { -- GitLab From f9027f88f7d1dc2b0ed1afbf28f22992d72d7efe Mon Sep 17 00:00:00 2001 From: Zhaoping Shu Date: Thu, 3 Nov 2022 19:08:49 +0800 Subject: [PATCH 1106/1927] net: wwan: iosm: Remove unnecessary if_mutex lock These WWAN network interface operations (create/delete/open/close) are already protected by RTNL lock, i.e., wwan_ops.newlink(), wwan_ops.dellink(), net_device_ops.ndo_open() and net_device.ndo_stop() are called with RTNL lock held. Therefore, this patch removes the unnecessary if_mutex. Signed-off-by: Zhaoping Shu Signed-off-by: David S. Miller --- drivers/net/wwan/iosm/iosm_ipc_wwan.c | 42 ++++----------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/drivers/net/wwan/iosm/iosm_ipc_wwan.c b/drivers/net/wwan/iosm/iosm_ipc_wwan.c index 0108d8d01ff26..4c9022a93e018 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_wwan.c +++ b/drivers/net/wwan/iosm/iosm_ipc_wwan.c @@ -40,13 +40,11 @@ struct iosm_netdev_priv { * @ipc_imem: Pointer to imem data-struct * @sub_netlist: List of active netdevs * @dev: Pointer device structure - * @if_mutex: Mutex used for add and remove interface id */ struct iosm_wwan { struct iosm_imem *ipc_imem; struct iosm_netdev_priv __rcu *sub_netlist[IP_MUX_SESSION_END + 1]; struct device *dev; - struct mutex if_mutex; /* Mutex used for add and remove interface id */ }; /* Bring-up the wwan net link */ @@ -55,14 +53,11 @@ static int ipc_wwan_link_open(struct net_device *netdev) struct iosm_netdev_priv *priv = wwan_netdev_drvpriv(netdev); struct iosm_wwan *ipc_wwan = priv->ipc_wwan; int if_id = priv->if_id; - int ret; if (if_id < IP_MUX_SESSION_START || if_id >= ARRAY_SIZE(ipc_wwan->sub_netlist)) return -EINVAL; - mutex_lock(&ipc_wwan->if_mutex); - /* get channel id */ priv->ch_id = ipc_imem_sys_wwan_open(ipc_wwan->ipc_imem, if_id); @@ -70,8 +65,7 @@ static int ipc_wwan_link_open(struct net_device *netdev) dev_err(ipc_wwan->dev, "cannot connect wwan0 & id %d to the IPC mem layer", if_id); - ret = -ENODEV; - goto out; + return -ENODEV; } /* enable tx path, DL data may follow */ @@ -80,10 +74,7 @@ static int ipc_wwan_link_open(struct net_device *netdev) dev_dbg(ipc_wwan->dev, "Channel id %d allocated to if_id %d", priv->ch_id, priv->if_id); - ret = 0; -out: - mutex_unlock(&ipc_wwan->if_mutex); - return ret; + return 0; } /* Bring-down the wwan net link */ @@ -93,11 +84,9 @@ static int ipc_wwan_link_stop(struct net_device *netdev) netif_stop_queue(netdev); - mutex_lock(&priv->ipc_wwan->if_mutex); ipc_imem_sys_wwan_close(priv->ipc_wwan->ipc_imem, priv->if_id, priv->ch_id); priv->ch_id = -1; - mutex_unlock(&priv->ipc_wwan->if_mutex); return 0; } @@ -190,26 +179,17 @@ static int ipc_wwan_newlink(void *ctxt, struct net_device *dev, priv->netdev = dev; priv->ipc_wwan = ipc_wwan; - mutex_lock(&ipc_wwan->if_mutex); - if (rcu_access_pointer(ipc_wwan->sub_netlist[if_id])) { - err = -EBUSY; - goto out_unlock; - } + if (rcu_access_pointer(ipc_wwan->sub_netlist[if_id])) + return -EBUSY; err = register_netdevice(dev); if (err) - goto out_unlock; + return err; rcu_assign_pointer(ipc_wwan->sub_netlist[if_id], priv); - mutex_unlock(&ipc_wwan->if_mutex); - netif_device_attach(dev); return 0; - -out_unlock: - mutex_unlock(&ipc_wwan->if_mutex); - return err; } static void ipc_wwan_dellink(void *ctxt, struct net_device *dev, @@ -223,17 +203,12 @@ static void ipc_wwan_dellink(void *ctxt, struct net_device *dev, if_id >= ARRAY_SIZE(ipc_wwan->sub_netlist))) return; - mutex_lock(&ipc_wwan->if_mutex); - if (WARN_ON(rcu_access_pointer(ipc_wwan->sub_netlist[if_id]) != priv)) - goto unlock; + return; RCU_INIT_POINTER(ipc_wwan->sub_netlist[if_id], NULL); /* unregistering includes synchronize_net() */ unregister_netdevice_queue(dev, head); - -unlock: - mutex_unlock(&ipc_wwan->if_mutex); } static const struct wwan_ops iosm_wwan_ops = { @@ -324,12 +299,9 @@ struct iosm_wwan *ipc_wwan_init(struct iosm_imem *ipc_imem, struct device *dev) ipc_wwan->dev = dev; ipc_wwan->ipc_imem = ipc_imem; - mutex_init(&ipc_wwan->if_mutex); - /* WWAN core will create a netdev for the default IP MUX channel */ if (wwan_register_ops(ipc_wwan->dev, &iosm_wwan_ops, ipc_wwan, IP_MUX_SESSION_DEFAULT)) { - mutex_destroy(&ipc_wwan->if_mutex); kfree(ipc_wwan); return NULL; } @@ -342,7 +314,5 @@ void ipc_wwan_deinit(struct iosm_wwan *ipc_wwan) /* This call will remove all child netdev(s) */ wwan_unregister_ops(ipc_wwan->dev); - mutex_destroy(&ipc_wwan->if_mutex); - kfree(ipc_wwan); } -- GitLab From cf53db768a8790fdaae2fa3a81322b080285f7e5 Mon Sep 17 00:00:00 2001 From: Yuan Can Date: Thu, 3 Nov 2022 01:47:05 +0000 Subject: [PATCH 1107/1927] drm/vc4: Fix missing platform_unregister_drivers() call in vc4_drm_register() A problem about modprobe vc4 failed is triggered with the following log given: [ 420.327987] Error: Driver 'vc4_hvs' is already registered, aborting... [ 420.333904] failed to register platform driver vc4_hvs_driver [vc4]: -16 modprobe: ERROR: could not insert 'vc4': Device or resource busy The reason is that vc4_drm_register() returns platform_driver_register() directly without checking its return value, if platform_driver_register() fails, it returns without unregistering all the vc4 drivers, resulting the vc4 can never be installed later. A simple call graph is shown as below: vc4_drm_register() platform_register_drivers() # all vc4 drivers are registered platform_driver_register() driver_register() bus_add_driver() priv = kzalloc(...) # OOM happened # return without unregister drivers Fixing this problem by checking the return value of platform_driver_register() and do platform_unregister_drivers() if error happened. Fixes: c8b75bca92cb ("drm/vc4: Add KMS support for Raspberry Pi.") Signed-off-by: Yuan Can Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20221103014705.109322-1-yuancan@huawei.com --- drivers/gpu/drm/vc4/vc4_drv.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c index 2027063fdc309..8c329c071c62d 100644 --- a/drivers/gpu/drm/vc4/vc4_drv.c +++ b/drivers/gpu/drm/vc4/vc4_drv.c @@ -476,7 +476,12 @@ static int __init vc4_drm_register(void) if (ret) return ret; - return platform_driver_register(&vc4_platform_driver); + ret = platform_driver_register(&vc4_platform_driver); + if (ret) + platform_unregister_drivers(component_drivers, + ARRAY_SIZE(component_drivers)); + + return ret; } static void __exit vc4_drm_unregister(void) -- GitLab From 0c175da7b0378445f5ef53904247cfbfb87e0b78 Mon Sep 17 00:00:00 2001 From: Lu Wei Date: Fri, 4 Nov 2022 10:27:23 +0800 Subject: [PATCH 1108/1927] tcp: prohibit TCP_REPAIR_OPTIONS if data was already sent If setsockopt with option name of TCP_REPAIR_OPTIONS and opt_code of TCPOPT_SACK_PERM is called to enable sack after data is sent and dupacks are received , it will trigger a warning in function tcp_verify_left_out() as follows: ============================================ WARNING: CPU: 8 PID: 0 at net/ipv4/tcp_input.c:2132 tcp_timeout_mark_lost+0x154/0x160 tcp_enter_loss+0x2b/0x290 tcp_retransmit_timer+0x50b/0x640 tcp_write_timer_handler+0x1c8/0x340 tcp_write_timer+0xe5/0x140 call_timer_fn+0x3a/0x1b0 __run_timers.part.0+0x1bf/0x2d0 run_timer_softirq+0x43/0xb0 __do_softirq+0xfd/0x373 __irq_exit_rcu+0xf6/0x140 The warning is caused in the following steps: 1. a socket named socketA is created 2. socketA enters repair mode without build a connection 3. socketA calls connect() and its state is changed to TCP_ESTABLISHED directly 4. socketA leaves repair mode 5. socketA calls sendmsg() to send data, packets_out and sack_outs(dup ack receives) increase 6. socketA enters repair mode again 7. socketA calls setsockopt with TCPOPT_SACK_PERM to enable sack 8. retransmit timer expires, it calls tcp_timeout_mark_lost(), lost_out increases 9. sack_outs + lost_out > packets_out triggers since lost_out and sack_outs increase repeatly In function tcp_timeout_mark_lost(), tp->sacked_out will be cleared if Step7 not happen and the warning will not be triggered. As suggested by Denis and Eric, TCP_REPAIR_OPTIONS should be prohibited if data was already sent. socket-tcp tests in CRIU has been tested as follows: $ sudo ./test/zdtm.py run -t zdtm/static/socket-tcp* --keep-going \ --ignore-taint socket-tcp* represent all socket-tcp tests in test/zdtm/static/. Fixes: b139ba4e90dc ("tcp: Repair connection-time negotiated parameters") Signed-off-by: Lu Wei Reviewed-by: Eric Dumazet Signed-off-by: David S. Miller --- net/ipv4/tcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index ef14efa1fb70e..54836a6b81d6f 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -3647,7 +3647,7 @@ int do_tcp_setsockopt(struct sock *sk, int level, int optname, case TCP_REPAIR_OPTIONS: if (!tp->repair) err = -EINVAL; - else if (sk->sk_state == TCP_ESTABLISHED) + else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent) err = tcp_repair_options_est(sk, optval, optlen); else err = -EPERM; -- GitLab From 454d61a56d5e4c8cd9905f81d475d23f86f370af Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Fri, 4 Nov 2022 18:06:29 +0100 Subject: [PATCH 1109/1927] ACPI: video: Make acpi_video_backlight_use_native() always return true Testing has shown that there are quite a few laptop models which rely on native backlight control and which do not support ACPI video bus backlight control, causing __acpi_video_get_backlight_type() to return vendor. Known Windows laptop models affected by this are: Acer Aspire 1640 HP Compaq nc6120 IBM ThinkPad X40 System76 Starling Star1 and the following MacBook models are affected too: Apple MacBook 2.1 Apple MacBook 4.1 Apple MacBook Pro 7.1 the list of affected Windows laptop models is likely just the top of the iceberg. So for now lets undo the change to not register native backlight class devices when __acpi_video_get_backlight_type() != native. Since as part of the backlight-detect refactor the detection code now relies on the GPU drivers calling acpi_video_backlight_use_native() to learn that native backlight support is available we cannot just remove the acpi_video_backlight_use_native() calls from the GPU drivers. Instead modify acpi_video_backlight_use_native() to always return true for now. This is meant as a temporary work-around, which will be removed again when the heuristics from __acpi_video_get_backlight_type() have been improved so that they will return native on affected models. Reported-by: Matthew Garrett Reported-by: John Warriner Reported-by: Scott Ostrander Reported-by: Matthias Rampke Reported-by: Milan Hodoscek Signed-off-by: Hans de Goede --- drivers/acpi/video_detect.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 06aaec2e378bd..c02e960cdbe33 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -751,6 +751,18 @@ EXPORT_SYMBOL(acpi_video_get_backlight_type); bool acpi_video_backlight_use_native(void) { - return __acpi_video_get_backlight_type(true) == acpi_backlight_native; + /* + * Call __acpi_video_get_backlight_type() to let it know that + * a native backlight is available. + */ + __acpi_video_get_backlight_type(true); + + /* + * For now just always return true. There is a whole bunch of laptop + * models where (video_caps & ACPI_VIDEO_BACKLIGHT) is false causing + * __acpi_video_get_backlight_type() to return vendor, while these + * models only have a native backlight control. + */ + return true; } EXPORT_SYMBOL(acpi_video_backlight_use_native); -- GitLab From f46acc1efd4b5846de9fa05f966e504f328f34a6 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 31 Oct 2022 21:20:59 +0100 Subject: [PATCH 1110/1927] ACPI: video: Add backlight=native DMI quirk for Dell G15 5515 The Dell G15 5515 has the WMI interface (and WMI call returns) expected by the nvidia-wmi-ec-backlight interface. But the backlight class device registered by the nvidia-wmi-ec-backlight driver does not actually work. The amdgpu_bl0 native GPU backlight class device does actually work, add a backlight=native DMI quirk for this. Reported-by: Iris Reviewed-by: Daniel Dadap Signed-off-by: Hans de Goede --- Changes in v2: - Add a comment that this needs to be revisited when dynamic-mux support gets added (suggested by: Daniel Dadap) --- drivers/acpi/video_detect.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index c02e960cdbe33..b2a6162876387 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -645,6 +645,20 @@ static const struct dmi_system_id video_detect_dmi_table[] = { }, }, + /* + * Models which have nvidia-ec-wmi support, but should not use it. + * Note this indicates a likely firmware bug on these models and should + * be revisited if/when Linux gets support for dynamic mux mode. + */ + { + .callback = video_detect_force_native, + /* Dell G15 5515 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Dell G15 5515"), + }, + }, + /* * Desktops which falsely report a backlight and which our heuristics * for this do not catch. -- GitLab From a231224a601c1924b9df620281ad04472900d75f Mon Sep 17 00:00:00 2001 From: Manyi Li Date: Tue, 18 Oct 2022 17:53:23 +0800 Subject: [PATCH 1111/1927] platform/x86: ideapad-laptop: Disable touchpad_switch Ideapads for "Lenovo Yoga 3 Pro 1370" and "ZhaoYang K4e-IML" do not use EC to switch touchpad. Reading VPCCMD_R_TOUCHPAD will return zero thus touchpad may be blocked unexpectedly. Signed-off-by: Manyi Li Link: https://lore.kernel.org/r/20221018095323.14591-1-limanyi@uniontech.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/ideapad-laptop.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/drivers/platform/x86/ideapad-laptop.c b/drivers/platform/x86/ideapad-laptop.c index abd0c81d62c40..33b3dfdd1b08f 100644 --- a/drivers/platform/x86/ideapad-laptop.c +++ b/drivers/platform/x86/ideapad-laptop.c @@ -1533,6 +1533,24 @@ static const struct dmi_system_id hw_rfkill_list[] = { {} }; +static const struct dmi_system_id no_touchpad_switch_list[] = { + { + .ident = "Lenovo Yoga 3 Pro 1370", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "Lenovo YOGA 3"), + }, + }, + { + .ident = "ZhaoYang K4e-IML", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_VERSION, "ZhaoYang K4e-IML"), + }, + }, + {} +}; + static void ideapad_check_features(struct ideapad_private *priv) { acpi_handle handle = priv->adev->handle; @@ -1541,7 +1559,12 @@ static void ideapad_check_features(struct ideapad_private *priv) priv->features.hw_rfkill_switch = dmi_check_system(hw_rfkill_list); /* Most ideapads with ELAN0634 touchpad don't use EC touchpad switch */ - priv->features.touchpad_ctrl_via_ec = !acpi_dev_present("ELAN0634", NULL, -1); + if (acpi_dev_present("ELAN0634", NULL, -1)) + priv->features.touchpad_ctrl_via_ec = 0; + else if (dmi_check_system(no_touchpad_switch_list)) + priv->features.touchpad_ctrl_via_ec = 0; + else + priv->features.touchpad_ctrl_via_ec = 1; if (!read_ec_data(handle, VPCCMD_R_FAN, &val)) priv->features.fan_mode = true; -- GitLab From 0df044b34bf33e7e35c32b3bf6747fde6279c162 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Tue, 25 Oct 2022 16:11:31 +0200 Subject: [PATCH 1112/1927] platform/x86: touchscreen_dmi: Add info for the RCA Cambio W101 v2 2-in-1 Add touchscreen info for the RCA Cambio W101 v2 2-in-1. Link: https://github.com/onitake/gsl-firmware/discussions/193 Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20221025141131.509211-1-hdegoede@redhat.com --- drivers/platform/x86/touchscreen_dmi.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/drivers/platform/x86/touchscreen_dmi.c b/drivers/platform/x86/touchscreen_dmi.c index bc97bfa8e8a65..baae3120efd05 100644 --- a/drivers/platform/x86/touchscreen_dmi.c +++ b/drivers/platform/x86/touchscreen_dmi.c @@ -770,6 +770,22 @@ static const struct ts_dmi_data predia_basic_data = { .properties = predia_basic_props, }; +static const struct property_entry rca_cambio_w101_v2_props[] = { + PROPERTY_ENTRY_U32("touchscreen-min-x", 4), + PROPERTY_ENTRY_U32("touchscreen-min-y", 20), + PROPERTY_ENTRY_U32("touchscreen-size-x", 1644), + PROPERTY_ENTRY_U32("touchscreen-size-y", 874), + PROPERTY_ENTRY_BOOL("touchscreen-swapped-x-y"), + PROPERTY_ENTRY_STRING("firmware-name", "gsl1680-rca-cambio-w101-v2.fw"), + PROPERTY_ENTRY_U32("silead,max-fingers", 10), + { } +}; + +static const struct ts_dmi_data rca_cambio_w101_v2_data = { + .acpi_name = "MSSL1680:00", + .properties = rca_cambio_w101_v2_props, +}; + static const struct property_entry rwc_nanote_p8_props[] = { PROPERTY_ENTRY_U32("touchscreen-min-y", 46), PROPERTY_ENTRY_U32("touchscreen-size-x", 1728), @@ -1409,6 +1425,15 @@ const struct dmi_system_id touchscreen_dmi_table[] = { DMI_EXACT_MATCH(DMI_BOARD_NAME, "0E57"), }, }, + { + /* RCA Cambio W101 v2 */ + /* https://github.com/onitake/gsl-firmware/discussions/193 */ + .driver_data = (void *)&rca_cambio_w101_v2_data, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "RCA"), + DMI_MATCH(DMI_PRODUCT_NAME, "W101SA23T1"), + }, + }, { /* RWC NANOTE P8 */ .driver_data = (void *)&rwc_nanote_p8_data, -- GitLab From 1598bfa8e1faa932de42e1ee7628a1c4c4263f0a Mon Sep 17 00:00:00 2001 From: Jorge Lopez Date: Fri, 28 Oct 2022 10:55:27 -0500 Subject: [PATCH 1113/1927] platform/x86: hp_wmi: Fix rfkill causing soft blocked wifi After upgrading BIOS to U82 01.02.01 Rev.A, the console is flooded strange char "^@" which printed out every second and makes login nearly impossible. Also the below messages were shown both in console and journal/dmesg every second: usb 1-3: Device not responding to setup address. usb 1-3: device not accepting address 4, error -71 usb 1-3: device descriptor read/all, error -71 usb usb1-port3: unable to enumerate USB device Wifi is soft blocked by checking rfkill. When unblocked manually, after few seconds it would be soft blocked again. So I was suspecting something triggered rfkill to soft block wifi. At the end it was fixed by removing hp_wmi module. The root cause is the way hp-wmi driver handles command 1B on post-2009 BIOS. In pre-2009 BIOS, command 1Bh return 0x4 to indicate that BIOS no longer controls the power for the wireless devices. Signed-off-by: Jorge Lopez Link: https://bugzilla.kernel.org/show_bug.cgi?id=216468 Reviewed-by: Mario Limonciello Link: https://lore.kernel.org/r/20221028155527.7724-1-jorge.lopez2@hp.com Cc: stable@vger.kernel.org Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/hp-wmi.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/platform/x86/hp-wmi.c b/drivers/platform/x86/hp-wmi.c index 627a6d0eaf831..12449038bed12 100644 --- a/drivers/platform/x86/hp-wmi.c +++ b/drivers/platform/x86/hp-wmi.c @@ -1300,8 +1300,16 @@ static int __init hp_wmi_bios_setup(struct platform_device *device) wwan_rfkill = NULL; rfkill2_count = 0; - if (hp_wmi_rfkill_setup(device)) - hp_wmi_rfkill2_setup(device); + /* + * In pre-2009 BIOS, command 1Bh return 0x4 to indicate that + * BIOS no longer controls the power for the wireless + * devices. All features supported by this command will no + * longer be supported. + */ + if (!hp_wmi_bios_2009_later()) { + if (hp_wmi_rfkill_setup(device)) + hp_wmi_rfkill2_setup(device); + } err = hp_wmi_hwmon_init(); -- GitLab From bcdfa1f77ea7f67368d20384932a9d1e3047ddd2 Mon Sep 17 00:00:00 2001 From: "David E. Box" Date: Fri, 4 Nov 2022 20:42:28 -0700 Subject: [PATCH 1114/1927] platform/x86/intel/pmt: Sapphire Rapids PMT errata fix On Sapphire Rapids, due to a hardware issue affecting the PUNIT telemetry region, reads that are not done in QWORD quantities and alignment may return incorrect data. Use a custom 64-bit copy for this region. Signed-off-by: David E. Box Link: https://lore.kernel.org/r/20221105034228.1376677-1-david.e.box@linux.intel.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/intel/pmt/class.c | 31 +++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/drivers/platform/x86/intel/pmt/class.c b/drivers/platform/x86/intel/pmt/class.c index 53d7fd2943b4c..46598dcb634aa 100644 --- a/drivers/platform/x86/intel/pmt/class.c +++ b/drivers/platform/x86/intel/pmt/class.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -19,6 +20,7 @@ #define PMT_XA_START 0 #define PMT_XA_MAX INT_MAX #define PMT_XA_LIMIT XA_LIMIT(PMT_XA_START, PMT_XA_MAX) +#define GUID_SPR_PUNIT 0x9956f43f bool intel_pmt_is_early_client_hw(struct device *dev) { @@ -33,6 +35,29 @@ bool intel_pmt_is_early_client_hw(struct device *dev) } EXPORT_SYMBOL_GPL(intel_pmt_is_early_client_hw); +static inline int +pmt_memcpy64_fromio(void *to, const u64 __iomem *from, size_t count) +{ + int i, remain; + u64 *buf = to; + + if (!IS_ALIGNED((unsigned long)from, 8)) + return -EFAULT; + + for (i = 0; i < count/8; i++) + buf[i] = readq(&from[i]); + + /* Copy any remaining bytes */ + remain = count % 8; + if (remain) { + u64 tmp = readq(&from[i]); + + memcpy(&buf[i], &tmp, remain); + } + + return count; +} + /* * sysfs */ @@ -54,7 +79,11 @@ intel_pmt_read(struct file *filp, struct kobject *kobj, if (count > entry->size - off) count = entry->size - off; - memcpy_fromio(buf, entry->base + off, count); + if (entry->guid == GUID_SPR_PUNIT) + /* PUNIT on SPR only supports aligned 64-bit read */ + count = pmt_memcpy64_fromio(buf, entry->base + off, count); + else + memcpy_fromio(buf, entry->base + off, count); return count; } -- GitLab From a977ece5773b6746b814aac410da4776023db239 Mon Sep 17 00:00:00 2001 From: Ivan Hu Date: Wed, 2 Nov 2022 10:05:48 +0800 Subject: [PATCH 1115/1927] platform/x86/intel/hid: Add some ACPI device IDs Add INTC1076 (JasonLake), INTC1077 (MeteorLake) and INTC1078 (RaptorLake) devices IDs. Signed-off-by: Ivan Hu Link: https://lore.kernel.org/r/20221102020548.5225-1-ivan.hu@canonical.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/intel/hid.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 79cff1fc675c2..b6313ecd190c0 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -27,6 +27,9 @@ static const struct acpi_device_id intel_hid_ids[] = { {"INTC1051", 0}, {"INTC1054", 0}, {"INTC1070", 0}, + {"INTC1076", 0}, + {"INTC1077", 0}, + {"INTC1078", 0}, {"", 0}, }; MODULE_DEVICE_TABLE(acpi, intel_hid_ids); -- GitLab From 53eb64c88f17b14b324fbdfd417f56c5d3fa6fee Mon Sep 17 00:00:00 2001 From: Andy Shevchenko Date: Fri, 4 Nov 2022 17:49:16 +0200 Subject: [PATCH 1116/1927] platform/x86: p2sb: Don't fail if unknown CPU is found We have accessing P2SB from a very few places for quite known hardware. When a new SoC appears in intel-family.h it's not obvious that it needs to be added to p2sb.c as well. Instead, provide default BDF and refactor p2sb_get_devfn() to always succeed. If in the future we would need to exclude something, we may add a list of unsupported IDs. Without this change the iTCO on Intel Comet Lake SoCs became unavailable: i801_smbus 0000:00:1f.4: failed to create iTCO device Fixes: 5c7b9167ddf8 ("i2c: i801: convert to use common P2SB accessor") Reported-and-tested-by: Jarkko Nikula Signed-off-by: Andy Shevchenko Link: https://lore.kernel.org/r/20221104154916.35231-1-andriy.shevchenko@linux.intel.com Reviewed-by: Hans de Goede Signed-off-by: Hans de Goede --- drivers/platform/x86/p2sb.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/drivers/platform/x86/p2sb.c b/drivers/platform/x86/p2sb.c index 384d0962ae93a..1cf2471d54dde 100644 --- a/drivers/platform/x86/p2sb.c +++ b/drivers/platform/x86/p2sb.c @@ -19,26 +19,23 @@ #define P2SBC 0xe0 #define P2SBC_HIDE BIT(8) +#define P2SB_DEVFN_DEFAULT PCI_DEVFN(31, 1) + static const struct x86_cpu_id p2sb_cpu_ids[] = { X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, PCI_DEVFN(13, 0)), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, PCI_DEVFN(31, 1)), - X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, PCI_DEVFN(31, 1)), - X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, PCI_DEVFN(31, 1)), - X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, PCI_DEVFN(31, 1)), - X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, PCI_DEVFN(31, 1)), - X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, PCI_DEVFN(31, 1)), {} }; static int p2sb_get_devfn(unsigned int *devfn) { + unsigned int fn = P2SB_DEVFN_DEFAULT; const struct x86_cpu_id *id; id = x86_match_cpu(p2sb_cpu_ids); - if (!id) - return -ENODEV; + if (id) + fn = (unsigned int)id->driver_data; - *devfn = (unsigned int)id->driver_data; + *devfn = fn; return 0; } -- GitLab From c23fb2c82267638f9d206cb96bb93e1f93ad7828 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Fri, 4 Nov 2022 11:32:16 +0100 Subject: [PATCH 1117/1927] ipv6: addrlabel: fix infoleak when sending struct ifaddrlblmsg to network When copying a `struct ifaddrlblmsg` to the network, __ifal_reserved remained uninitialized, resulting in a 1-byte infoleak: BUG: KMSAN: kernel-network-infoleak in __netdev_start_xmit ./include/linux/netdevice.h:4841 __netdev_start_xmit ./include/linux/netdevice.h:4841 netdev_start_xmit ./include/linux/netdevice.h:4857 xmit_one net/core/dev.c:3590 dev_hard_start_xmit+0x1dc/0x800 net/core/dev.c:3606 __dev_queue_xmit+0x17e8/0x4350 net/core/dev.c:4256 dev_queue_xmit ./include/linux/netdevice.h:3009 __netlink_deliver_tap_skb net/netlink/af_netlink.c:307 __netlink_deliver_tap+0x728/0xad0 net/netlink/af_netlink.c:325 netlink_deliver_tap net/netlink/af_netlink.c:338 __netlink_sendskb net/netlink/af_netlink.c:1263 netlink_sendskb+0x1d9/0x200 net/netlink/af_netlink.c:1272 netlink_unicast+0x56d/0xf50 net/netlink/af_netlink.c:1360 nlmsg_unicast ./include/net/netlink.h:1061 rtnl_unicast+0x5a/0x80 net/core/rtnetlink.c:758 ip6addrlbl_get+0xfad/0x10f0 net/ipv6/addrlabel.c:628 rtnetlink_rcv_msg+0xb33/0x1570 net/core/rtnetlink.c:6082 ... Uninit was created at: slab_post_alloc_hook+0x118/0xb00 mm/slab.h:742 slab_alloc_node mm/slub.c:3398 __kmem_cache_alloc_node+0x4f2/0x930 mm/slub.c:3437 __do_kmalloc_node mm/slab_common.c:954 __kmalloc_node_track_caller+0x117/0x3d0 mm/slab_common.c:975 kmalloc_reserve net/core/skbuff.c:437 __alloc_skb+0x27a/0xab0 net/core/skbuff.c:509 alloc_skb ./include/linux/skbuff.h:1267 nlmsg_new ./include/net/netlink.h:964 ip6addrlbl_get+0x490/0x10f0 net/ipv6/addrlabel.c:608 rtnetlink_rcv_msg+0xb33/0x1570 net/core/rtnetlink.c:6082 netlink_rcv_skb+0x299/0x550 net/netlink/af_netlink.c:2540 rtnetlink_rcv+0x26/0x30 net/core/rtnetlink.c:6109 netlink_unicast_kernel net/netlink/af_netlink.c:1319 netlink_unicast+0x9ab/0xf50 net/netlink/af_netlink.c:1345 netlink_sendmsg+0xebc/0x10f0 net/netlink/af_netlink.c:1921 ... This patch ensures that the reserved field is always initialized. Reported-by: syzbot+3553517af6020c4f2813f1003fe76ef3cbffe98d@syzkaller.appspotmail.com Fixes: 2a8cc6c89039 ("[IPV6] ADDRCONF: Support RFC3484 configurable address selection policy table.") Signed-off-by: Alexander Potapenko Reviewed-by: David Ahern Signed-off-by: David S. Miller --- net/ipv6/addrlabel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/ipv6/addrlabel.c b/net/ipv6/addrlabel.c index 8a22486cf2702..17ac45aa7194c 100644 --- a/net/ipv6/addrlabel.c +++ b/net/ipv6/addrlabel.c @@ -437,6 +437,7 @@ static void ip6addrlbl_putmsg(struct nlmsghdr *nlh, { struct ifaddrlblmsg *ifal = nlmsg_data(nlh); ifal->ifal_family = AF_INET6; + ifal->__ifal_reserved = 0; ifal->ifal_prefixlen = prefixlen; ifal->ifal_flags = 0; ifal->ifal_index = ifindex; -- GitLab From ebb5fd38f41132e6924cb33b647337f4a5d5360c Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 26 Oct 2022 12:42:03 -0700 Subject: [PATCH 1118/1927] mmc: cqhci: Provide helper for resetting both SDHCI and CQHCI Several SDHCI drivers need to deactivate command queueing in their reset hook (see sdhci_cqhci_reset() / sdhci-pci-core.c, for example), and several more are coming. Those reset implementations have some small subtleties (e.g., ordering of initialization of SDHCI vs. CQHCI might leave us resetting with a NULL ->cqe_private), and are often identical across different host drivers. We also don't want to force a dependency between SDHCI and CQHCI, or vice versa; non-SDHCI drivers use CQHCI, and SDHCI drivers might support command queueing through some other means. So, implement a small helper, to avoid repeating the same mistakes in different drivers. Simply stick it in a header, because it's so small it doesn't deserve its own module right now, and inlining to each driver is pretty reasonable. This is marked for -stable, as it is an important prerequisite patch for several SDHCI controller bugfixes that follow. Cc: Signed-off-by: Brian Norris Acked-by: Adrian Hunter Reviewed-by: Florian Fainelli Link: https://lore.kernel.org/r/20221026124150.v4.1.Ie85faa09432bfe1b0890d8c24ff95e17f3097317@changeid Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-cqhci.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 drivers/mmc/host/sdhci-cqhci.h diff --git a/drivers/mmc/host/sdhci-cqhci.h b/drivers/mmc/host/sdhci-cqhci.h new file mode 100644 index 0000000000000..cf8e7ba71bbd7 --- /dev/null +++ b/drivers/mmc/host/sdhci-cqhci.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright 2022 The Chromium OS Authors + * + * Support that applies to the combination of SDHCI and CQHCI, while not + * expressing a dependency between the two modules. + */ + +#ifndef __MMC_HOST_SDHCI_CQHCI_H__ +#define __MMC_HOST_SDHCI_CQHCI_H__ + +#include "cqhci.h" +#include "sdhci.h" + +static inline void sdhci_and_cqhci_reset(struct sdhci_host *host, u8 mask) +{ + if ((host->mmc->caps2 & MMC_CAP2_CQE) && (mask & SDHCI_RESET_ALL) && + host->mmc->cqe_private) + cqhci_deactivate(host->mmc); + + sdhci_reset(host, mask); +} + +#endif /* __MMC_HOST_SDHCI_CQHCI_H__ */ -- GitLab From 5d249ac37fc2396e8acc1adb0650cdacae5a990d Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 26 Oct 2022 12:42:04 -0700 Subject: [PATCH 1119/1927] mmc: sdhci-of-arasan: Fix SDHCI_RESET_ALL for CQHCI SDHCI_RESET_ALL resets will reset the hardware CQE state, but we aren't tracking that properly in software. When out of sync, we may trigger various timeouts. It's not typical to perform resets while CQE is enabled, but one particular case I hit commonly enough: mmc_suspend() -> mmc_power_off(). Typically we will eventually deactivate CQE (cqhci_suspend() -> cqhci_deactivate()), but that's not guaranteed -- in particular, if we perform a partial (e.g., interrupted) system suspend. The same bug was already found and fixed for two other drivers, in v5.7 and v5.9: 5cf583f1fb9c ("mmc: sdhci-msm: Deactivate CQE during SDHC reset") df57d73276b8 ("mmc: sdhci-pci: Fix SDHCI_RESET_ALL for CQHCI for Intel GLK-based controllers") The latter is especially prescient, saying "other drivers using CQHCI might benefit from a similar change, if they also have CQHCI reset by SDHCI_RESET_ALL." So like these other patches, deactivate CQHCI when resetting the controller. Do this via the new sdhci_and_cqhci_reset() helper. This patch depends on (and should not compile without) the patch entitled "mmc: cqhci: Provide helper for resetting both SDHCI and CQHCI". Fixes: 84362d79f436 ("mmc: sdhci-of-arasan: Add CQHCI support for arasan,sdhci-5.1") Cc: Signed-off-by: Brian Norris Reviewed-by: Guenter Roeck Acked-by: Adrian Hunter Link: https://lore.kernel.org/r/20221026124150.v4.2.I29f6a2189e84e35ad89c1833793dca9e36c64297@changeid Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-of-arasan.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/sdhci-of-arasan.c b/drivers/mmc/host/sdhci-of-arasan.c index 3997cad1f793d..cfb891430174a 100644 --- a/drivers/mmc/host/sdhci-of-arasan.c +++ b/drivers/mmc/host/sdhci-of-arasan.c @@ -25,6 +25,7 @@ #include #include "cqhci.h" +#include "sdhci-cqhci.h" #include "sdhci-pltfm.h" #define SDHCI_ARASAN_VENDOR_REGISTER 0x78 @@ -366,7 +367,7 @@ static void sdhci_arasan_reset(struct sdhci_host *host, u8 mask) struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_arasan_data *sdhci_arasan = sdhci_pltfm_priv(pltfm_host); - sdhci_reset(host, mask); + sdhci_and_cqhci_reset(host, mask); if (sdhci_arasan->quirks & SDHCI_ARASAN_QUIRK_FORCE_CDTEST) { ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); -- GitLab From 56baa208f91061ff27ec2d93fbc483f624d373b4 Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 26 Oct 2022 12:42:05 -0700 Subject: [PATCH 1120/1927] mmc: sdhci-brcmstb: Fix SDHCI_RESET_ALL for CQHCI [[ NOTE: this is completely untested by the author, but included solely because, as noted in commit df57d73276b8 ("mmc: sdhci-pci: Fix SDHCI_RESET_ALL for CQHCI for Intel GLK-based controllers"), "other drivers using CQHCI might benefit from a similar change, if they also have CQHCI reset by SDHCI_RESET_ALL." We've now seen the same bug on at least MSM, Arasan, and Intel hardware. ]] SDHCI_RESET_ALL resets will reset the hardware CQE state, but we aren't tracking that properly in software. When out of sync, we may trigger various timeouts. It's not typical to perform resets while CQE is enabled, but this may occur in some suspend or error recovery scenarios. Include this fix by way of the new sdhci_and_cqhci_reset() helper. I only patch the bcm7216 variant even though others potentially *could* provide the 'supports-cqe' property (and thus enable CQHCI), because d46ba2d17f90 ("mmc: sdhci-brcmstb: Add support for Command Queuing (CQE)") and some Broadcom folks confirm that only the 7216 variant actually supports it. This patch depends on (and should not compile without) the patch entitled "mmc: cqhci: Provide helper for resetting both SDHCI and CQHCI". Fixes: d46ba2d17f90 ("mmc: sdhci-brcmstb: Add support for Command Queuing (CQE)") Signed-off-by: Brian Norris Reviewed-by: Florian Fainelli Acked-by: Adrian Hunter Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221026124150.v4.3.I6a715feab6d01f760455865e968ecf0d85036018@changeid Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-brcmstb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/sdhci-brcmstb.c b/drivers/mmc/host/sdhci-brcmstb.c index aff36a933ebec..55d8bd232695c 100644 --- a/drivers/mmc/host/sdhci-brcmstb.c +++ b/drivers/mmc/host/sdhci-brcmstb.c @@ -12,6 +12,7 @@ #include #include +#include "sdhci-cqhci.h" #include "sdhci-pltfm.h" #include "cqhci.h" @@ -55,7 +56,7 @@ static void brcmstb_reset(struct sdhci_host *host, u8 mask) struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_brcmstb_priv *priv = sdhci_pltfm_priv(pltfm_host); - sdhci_reset(host, mask); + sdhci_and_cqhci_reset(host, mask); /* Reset will clear this, so re-enable it */ if (priv->flags & BRCMSTB_PRIV_FLAGS_GATE_CLOCK) -- GitLab From fb1dec44c6750bb414f47b929c8c175a1a127c31 Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 26 Oct 2022 12:42:06 -0700 Subject: [PATCH 1121/1927] mms: sdhci-esdhc-imx: Fix SDHCI_RESET_ALL for CQHCI [[ NOTE: this is completely untested by the author, but included solely because, as noted in commit df57d73276b8 ("mmc: sdhci-pci: Fix SDHCI_RESET_ALL for CQHCI for Intel GLK-based controllers"), "other drivers using CQHCI might benefit from a similar change, if they also have CQHCI reset by SDHCI_RESET_ALL." We've now seen the same bug on at least MSM, Arasan, and Intel hardware. ]] SDHCI_RESET_ALL resets will reset the hardware CQE state, but we aren't tracking that properly in software. When out of sync, we may trigger various timeouts. It's not typical to perform resets while CQE is enabled, but this may occur in some suspend or error recovery scenarios. Include this fix by way of the new sdhci_and_cqhci_reset() helper. This patch depends on (and should not compile without) the patch entitled "mmc: cqhci: Provide helper for resetting both SDHCI and CQHCI". Fixes: bb6e358169bf ("mmc: sdhci-esdhc-imx: add CMDQ support") Signed-off-by: Brian Norris Reviewed-by: Haibo Chen Acked-by: Adrian Hunter Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221026124150.v4.4.I7d01f9ad11bacdc9213dee61b7918982aea39115@changeid Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-esdhc-imx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index 747df79d90eef..89225faa242ab 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -25,6 +25,7 @@ #include #include #include +#include "sdhci-cqhci.h" #include "sdhci-pltfm.h" #include "sdhci-esdhc.h" #include "cqhci.h" @@ -1288,7 +1289,7 @@ static void esdhc_set_uhs_signaling(struct sdhci_host *host, unsigned timing) static void esdhc_reset(struct sdhci_host *host, u8 mask) { - sdhci_reset(host, mask); + sdhci_and_cqhci_reset(host, mask); sdhci_writel(host, host->ier, SDHCI_INT_ENABLE); sdhci_writel(host, host->ier, SDHCI_SIGNAL_ENABLE); -- GitLab From 836078449464e6af3b66ae6652dae79af176f21e Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 26 Oct 2022 12:42:07 -0700 Subject: [PATCH 1122/1927] mmc: sdhci-tegra: Fix SDHCI_RESET_ALL for CQHCI [[ NOTE: this is completely untested by the author, but included solely because, as noted in commit df57d73276b8 ("mmc: sdhci-pci: Fix SDHCI_RESET_ALL for CQHCI for Intel GLK-based controllers"), "other drivers using CQHCI might benefit from a similar change, if they also have CQHCI reset by SDHCI_RESET_ALL." We've now seen the same bug on at least MSM, Arasan, and Intel hardware. ]] SDHCI_RESET_ALL resets will reset the hardware CQE state, but we aren't tracking that properly in software. When out of sync, we may trigger various timeouts. It's not typical to perform resets while CQE is enabled, but this may occur in some suspend or error recovery scenarios. Include this fix by way of the new sdhci_and_cqhci_reset() helper. This patch depends on (and should not compile without) the patch entitled "mmc: cqhci: Provide helper for resetting both SDHCI and CQHCI". Fixes: 3c4019f97978 ("mmc: tegra: HW Command Queue Support for Tegra SDMMC") Signed-off-by: Brian Norris Acked-by: Adrian Hunter Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221026124150.v4.5.I418c9eaaf754880fcd2698113e8c3ef821a944d7@changeid Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-tegra.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mmc/host/sdhci-tegra.c b/drivers/mmc/host/sdhci-tegra.c index 413925bce0ca8..c71000a07656e 100644 --- a/drivers/mmc/host/sdhci-tegra.c +++ b/drivers/mmc/host/sdhci-tegra.c @@ -28,6 +28,7 @@ #include +#include "sdhci-cqhci.h" #include "sdhci-pltfm.h" #include "cqhci.h" @@ -367,7 +368,7 @@ static void tegra_sdhci_reset(struct sdhci_host *host, u8 mask) const struct sdhci_tegra_soc_data *soc_data = tegra_host->soc_data; u32 misc_ctrl, clk_ctrl, pad_ctrl; - sdhci_reset(host, mask); + sdhci_and_cqhci_reset(host, mask); if (!(mask & SDHCI_RESET_ALL)) return; -- GitLab From 162503fd1c3a1d4e14dbe7f399c1d1bec1c8abbc Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 26 Oct 2022 12:42:08 -0700 Subject: [PATCH 1123/1927] mmc: sdhci_am654: Fix SDHCI_RESET_ALL for CQHCI [[ NOTE: this is completely untested by the author, but included solely because, as noted in commit df57d73276b8 ("mmc: sdhci-pci: Fix SDHCI_RESET_ALL for CQHCI for Intel GLK-based controllers"), "other drivers using CQHCI might benefit from a similar change, if they also have CQHCI reset by SDHCI_RESET_ALL." We've now seen the same bug on at least MSM, Arasan, and Intel hardware. ]] SDHCI_RESET_ALL resets will reset the hardware CQE state, but we aren't tracking that properly in software. When out of sync, we may trigger various timeouts. It's not typical to perform resets while CQE is enabled, but this may occur in some suspend or error recovery scenarios. Include this fix by way of the new sdhci_and_cqhci_reset() helper. This patch depends on (and should not compile without) the patch entitled "mmc: cqhci: Provide helper for resetting both SDHCI and CQHCI". Fixes: f545702b74f9 ("mmc: sdhci_am654: Add Support for Command Queuing Engine to J721E") Signed-off-by: Brian Norris Acked-by: Adrian Hunter Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20221026124150.v4.6.I35ca9d6220ba48304438b992a76647ca8e5b126f@changeid Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci_am654.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c index 8f1023480e12c..c2333c7acac9b 100644 --- a/drivers/mmc/host/sdhci_am654.c +++ b/drivers/mmc/host/sdhci_am654.c @@ -15,6 +15,7 @@ #include #include "cqhci.h" +#include "sdhci-cqhci.h" #include "sdhci-pltfm.h" /* CTL_CFG Registers */ @@ -378,7 +379,7 @@ static void sdhci_am654_reset(struct sdhci_host *host, u8 mask) struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); struct sdhci_am654_data *sdhci_am654 = sdhci_pltfm_priv(pltfm_host); - sdhci_reset(host, mask); + sdhci_and_cqhci_reset(host, mask); if (sdhci_am654->quirks & SDHCI_AM654_QUIRK_FORCE_CDTEST) { ctrl = sdhci_readb(host, SDHCI_HOST_CONTROL); @@ -464,7 +465,7 @@ static struct sdhci_ops sdhci_am654_ops = { .set_clock = sdhci_am654_set_clock, .write_b = sdhci_am654_write_b, .irq = sdhci_am654_cqhci_irq, - .reset = sdhci_reset, + .reset = sdhci_and_cqhci_reset, }; static const struct sdhci_pltfm_data sdhci_am654_pdata = { @@ -494,7 +495,7 @@ static struct sdhci_ops sdhci_j721e_8bit_ops = { .set_clock = sdhci_am654_set_clock, .write_b = sdhci_am654_write_b, .irq = sdhci_am654_cqhci_irq, - .reset = sdhci_reset, + .reset = sdhci_and_cqhci_reset, }; static const struct sdhci_pltfm_data sdhci_j721e_8bit_pdata = { -- GitLab From a3335faebe1608f3e78e24f09501c9f7d5ebf87a Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Mon, 31 Oct 2022 11:30:53 +0800 Subject: [PATCH 1124/1927] can: af_can: can_exit(): add missing dev_remove_pack() of canxl_packet In can_init(), dev_add_pack(&canxl_packet) is added but not removed in can_exit(). It breaks the packet handler list and can make kernel panic when can_init() is called for the second time. | > modprobe can && rmmod can | > rmmod xxx && modprobe can | | BUG: unable to handle page fault for address: fffffbfff807d7f4 | RIP: 0010:dev_add_pack+0x133/0x1f0 | Call Trace: | | can_init+0xaa/0x1000 [can] | do_one_initcall+0xd3/0x4e0 | ... Fixes: fb08cba12b52 ("can: canxl: update CAN infrastructure for CAN XL frames") Signed-off-by: Chen Zhongjin Acked-by: Oliver Hartkopp Link: https://lore.kernel.org/all/20221031033053.37849-1-chenzhongjin@huawei.com [mkl: adjust subject and commit message] Signed-off-by: Marc Kleine-Budde --- net/can/af_can.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/can/af_can.c b/net/can/af_can.c index 9503ab10f9b8f..5e9e3e1e9825f 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -902,6 +902,7 @@ static __init int can_init(void) static __exit void can_exit(void) { /* protocol unregister */ + dev_remove_pack(&canxl_packet); dev_remove_pack(&canfd_packet); dev_remove_pack(&can_packet); sock_unregister(PF_CAN); -- GitLab From 8aa59e355949442c408408c2d836e561794c40a1 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Fri, 28 Oct 2022 16:56:50 +0800 Subject: [PATCH 1125/1927] can: af_can: fix NULL pointer dereference in can_rx_register() It causes NULL pointer dereference when testing as following: (a) use syscall(__NR_socket, 0x10ul, 3ul, 0) to create netlink socket. (b) use syscall(__NR_sendmsg, ...) to create bond link device and vxcan link device, and bind vxcan device to bond device (can also use ifenslave command to bind vxcan device to bond device). (c) use syscall(__NR_socket, 0x1dul, 3ul, 1) to create CAN socket. (d) use syscall(__NR_bind, ...) to bind the bond device to CAN socket. The bond device invokes the can-raw protocol registration interface to receive CAN packets. However, ml_priv is not allocated to the dev, dev_rcv_lists is assigned to NULL in can_rx_register(). In this case, it will occur the NULL pointer dereference issue. The following is the stack information: BUG: kernel NULL pointer dereference, address: 0000000000000008 PGD 122a4067 P4D 122a4067 PUD 1223c067 PMD 0 Oops: 0000 [#1] PREEMPT SMP RIP: 0010:can_rx_register+0x12d/0x1e0 Call Trace: raw_enable_filters+0x8d/0x120 raw_enable_allfilters+0x3b/0x130 raw_bind+0x118/0x4f0 __sys_bind+0x163/0x1a0 __x64_sys_bind+0x1e/0x30 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: 4e096a18867a ("net: introduce CAN specific pointer in the struct net_device") Signed-off-by: Zhengchao Shao Reviewed-by: Marc Kleine-Budde Link: https://lore.kernel.org/all/20221028085650.170470-1-shaozhengchao@huawei.com Signed-off-by: Marc Kleine-Budde --- net/can/af_can.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/can/af_can.c b/net/can/af_can.c index 5e9e3e1e9825f..27dcdcc0b8089 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -450,7 +450,7 @@ int can_rx_register(struct net *net, struct net_device *dev, canid_t can_id, /* insert new receiver (dev,canid,mask) -> (func,data) */ - if (dev && dev->type != ARPHRD_CAN) + if (dev && (dev->type != ARPHRD_CAN || !can_get_ml_priv(dev))) return -ENODEV; if (dev && !net_eq(net, dev_net(dev))) -- GitLab From 866337865f3747c68a3e7bb837611e39cec1ecd6 Mon Sep 17 00:00:00 2001 From: Oliver Hartkopp Date: Fri, 4 Nov 2022 15:25:51 +0100 Subject: [PATCH 1126/1927] can: isotp: fix tx state handling for echo tx processing In commit 4b7fe92c0690 ("can: isotp: add local echo tx processing for consecutive frames") the data flow for consecutive frames (CF) has been reworked to improve the reliability of long data transfers. This rework did not touch the transmission and the tx state changes of single frame (SF) transfers which likely led to the WARN in the isotp_tx_timer_handler() catching a wrong tx state. This patch makes use of the improved frame processing for SF frames and sets the ISOTP_SENDING state in isotp_sendmsg() within the cmpxchg() condition handling. A review of the state machine and the timer handling additionally revealed a missing echo timeout handling in the case of the burst mode in isotp_rcv_echo() and removes a potential timer configuration uncertainty in isotp_rcv_fc() when the receiver requests consecutive frames. Fixes: 4b7fe92c0690 ("can: isotp: add local echo tx processing for consecutive frames") Link: https://lore.kernel.org/linux-can/CAO4mrfe3dG7cMP1V5FLUkw7s+50c9vichigUMQwsxX4M=45QEw@mail.gmail.com/T/#u Reported-by: Wei Chen Cc: stable@vger.kernel.org # v6.0 Signed-off-by: Oliver Hartkopp Link: https://lore.kernel.org/all/20221104142551.16924-1-socketcan@hartkopp.net Signed-off-by: Marc Kleine-Budde --- net/can/isotp.c | 71 ++++++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 33 deletions(-) diff --git a/net/can/isotp.c b/net/can/isotp.c index a9d1357f8489f..608f8c24ae46b 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -111,6 +111,9 @@ MODULE_ALIAS("can-proto-6"); #define ISOTP_FC_WT 1 /* wait */ #define ISOTP_FC_OVFLW 2 /* overflow */ +#define ISOTP_FC_TIMEOUT 1 /* 1 sec */ +#define ISOTP_ECHO_TIMEOUT 2 /* 2 secs */ + enum { ISOTP_IDLE = 0, ISOTP_WAIT_FIRST_FC, @@ -258,7 +261,8 @@ static int isotp_send_fc(struct sock *sk, int ae, u8 flowstatus) so->lastrxcf_tstamp = ktime_set(0, 0); /* start rx timeout watchdog */ - hrtimer_start(&so->rxtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT); + hrtimer_start(&so->rxtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), + HRTIMER_MODE_REL_SOFT); return 0; } @@ -344,6 +348,8 @@ static int check_pad(struct isotp_sock *so, struct canfd_frame *cf, return 0; } +static void isotp_send_cframe(struct isotp_sock *so); + static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) { struct sock *sk = &so->sk; @@ -398,14 +404,15 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) case ISOTP_FC_CTS: so->tx.bs = 0; so->tx.state = ISOTP_SENDING; - /* start cyclic timer for sending CF frame */ - hrtimer_start(&so->txtimer, so->tx_gap, + /* send CF frame and enable echo timeout handling */ + hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); + isotp_send_cframe(so); break; case ISOTP_FC_WT: /* start timer to wait for next FC frame */ - hrtimer_start(&so->txtimer, ktime_set(1, 0), + hrtimer_start(&so->txtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); break; @@ -600,7 +607,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, /* perform blocksize handling, if enabled */ if (!so->rxfc.bs || ++so->rx.bs < so->rxfc.bs) { /* start rx timeout watchdog */ - hrtimer_start(&so->rxtimer, ktime_set(1, 0), + hrtimer_start(&so->rxtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); return 0; } @@ -829,7 +836,7 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) struct isotp_sock *so = isotp_sk(sk); struct canfd_frame *cf = (struct canfd_frame *)skb->data; - /* only handle my own local echo skb's */ + /* only handle my own local echo CF/SF skb's (no FF!) */ if (skb->sk != sk || so->cfecho != *(u32 *)cf->data) return; @@ -849,13 +856,16 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { /* stop and wait for FC with timeout */ so->tx.state = ISOTP_WAIT_FC; - hrtimer_start(&so->txtimer, ktime_set(1, 0), + hrtimer_start(&so->txtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); return; } /* no gap between data frames needed => use burst mode */ if (!so->tx_gap) { + /* enable echo timeout handling */ + hrtimer_start(&so->txtimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), + HRTIMER_MODE_REL_SOFT); isotp_send_cframe(so); return; } @@ -879,7 +889,7 @@ static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) /* start timeout for unlikely lost echo skb */ hrtimer_set_expires(&so->txtimer, ktime_add(ktime_get(), - ktime_set(2, 0))); + ktime_set(ISOTP_ECHO_TIMEOUT, 0))); restart = HRTIMER_RESTART; /* push out the next consecutive frame */ @@ -907,7 +917,8 @@ static enum hrtimer_restart isotp_tx_timer_handler(struct hrtimer *hrtimer) break; default: - WARN_ON_ONCE(1); + WARN_ONCE(1, "can-isotp: tx timer state %08X cfecho %08X\n", + so->tx.state, so->cfecho); } return restart; @@ -923,7 +934,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) struct canfd_frame *cf; int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; int wait_tx_done = (so->opt.flags & CAN_ISOTP_WAIT_TX_DONE) ? 1 : 0; - s64 hrtimer_sec = 0; + s64 hrtimer_sec = ISOTP_ECHO_TIMEOUT; int off; int err; @@ -942,6 +953,8 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) err = wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); if (err) goto err_out; + + so->tx.state = ISOTP_SENDING; } if (!size || size > MAX_MSG_LENGTH) { @@ -986,6 +999,10 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) cf = (struct canfd_frame *)skb->data; skb_put_zero(skb, so->ll.mtu); + /* cfecho should have been zero'ed by init / former isotp_rcv_echo() */ + if (so->cfecho) + pr_notice_once("can-isotp: uninit cfecho %08X\n", so->cfecho); + /* check for single frame transmission depending on TX_DL */ if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) { /* The message size generally fits into a SingleFrame - good. @@ -1011,11 +1028,8 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) else cf->data[ae] |= size; - so->tx.state = ISOTP_IDLE; - wake_up_interruptible(&so->wait); - - /* don't enable wait queue for a single frame transmission */ - wait_tx_done = 0; + /* set CF echo tag for isotp_rcv_echo() (SF-mode) */ + so->cfecho = *(u32 *)cf->data; } else { /* send first frame */ @@ -1031,31 +1045,23 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) /* disable wait for FCs due to activated block size */ so->txfc.bs = 0; - /* cfecho should have been zero'ed by init */ - if (so->cfecho) - pr_notice_once("can-isotp: no fc cfecho %08X\n", - so->cfecho); - - /* set consecutive frame echo tag */ + /* set CF echo tag for isotp_rcv_echo() (CF-mode) */ so->cfecho = *(u32 *)cf->data; - - /* switch directly to ISOTP_SENDING state */ - so->tx.state = ISOTP_SENDING; - - /* start timeout for unlikely lost echo skb */ - hrtimer_sec = 2; } else { /* standard flow control check */ so->tx.state = ISOTP_WAIT_FIRST_FC; /* start timeout for FC */ - hrtimer_sec = 1; - } + hrtimer_sec = ISOTP_FC_TIMEOUT; - hrtimer_start(&so->txtimer, ktime_set(hrtimer_sec, 0), - HRTIMER_MODE_REL_SOFT); + /* no CF echo tag for isotp_rcv_echo() (FF-mode) */ + so->cfecho = 0; + } } + hrtimer_start(&so->txtimer, ktime_set(hrtimer_sec, 0), + HRTIMER_MODE_REL_SOFT); + /* send the first or only CAN frame */ cf->flags = so->ll.tx_flags; @@ -1068,8 +1074,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) __func__, ERR_PTR(err)); /* no transmission -> no timeout monitoring */ - if (hrtimer_sec) - hrtimer_cancel(&so->txtimer); + hrtimer_cancel(&so->txtimer); /* reset consecutive frame echo tag */ so->cfecho = 0; -- GitLab From 3eb3d283e8579a22b81dd2ac3987b77465b2a22f Mon Sep 17 00:00:00 2001 From: Oliver Hartkopp Date: Fri, 4 Nov 2022 08:50:00 +0100 Subject: [PATCH 1127/1927] can: j1939: j1939_send_one(): fix missing CAN header initialization The read access to struct canxl_frame::len inside of a j1939 created skbuff revealed a missing initialization of reserved and later filled elements in struct can_frame. This patch initializes the 8 byte CAN header with zero. Fixes: 9d71dd0c7009 ("can: add support of SAE J1939 protocol") Cc: Oleksij Rempel Link: https://lore.kernel.org/linux-can/20221104052235.GA6474@pengutronix.de Reported-by: syzbot+d168ec0caca4697e03b1@syzkaller.appspotmail.com Signed-off-by: Oliver Hartkopp Link: https://lore.kernel.org/all/20221104075000.105414-1-socketcan@hartkopp.net Cc: stable@vger.kernel.org Signed-off-by: Marc Kleine-Budde --- net/can/j1939/main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c index 144c86b0e3ffe..821d4ff303b35 100644 --- a/net/can/j1939/main.c +++ b/net/can/j1939/main.c @@ -336,6 +336,9 @@ int j1939_send_one(struct j1939_priv *priv, struct sk_buff *skb) /* re-claim the CAN_HDR from the SKB */ cf = skb_push(skb, J1939_CAN_HDR); + /* initialize header structure */ + memset(cf, 0, J1939_CAN_HDR); + /* make it a full can frame again */ skb_put(skb, J1939_CAN_FTR + (8 - dlc)); -- GitLab From ae64438be1923e3c1102d90fd41db7afcfaf54cc Mon Sep 17 00:00:00 2001 From: Oliver Hartkopp Date: Wed, 2 Nov 2022 10:54:31 +0100 Subject: [PATCH 1128/1927] can: dev: fix skb drop check In commit a6d190f8c767 ("can: skb: drop tx skb if in listen only mode") the priv->ctrlmode element is read even on virtual CAN interfaces that do not create the struct can_priv at startup. This out-of-bounds read may lead to CAN frame drops for virtual CAN interfaces like vcan and vxcan. This patch mainly reverts the original commit and adds a new helper for CAN interface drivers that provide the required information in struct can_priv. Fixes: a6d190f8c767 ("can: skb: drop tx skb if in listen only mode") Reported-by: Dariusz Stojaczyk Cc: Vincent Mailhol Cc: Max Staudt Signed-off-by: Oliver Hartkopp Acked-by: Vincent Mailhol Link: https://lore.kernel.org/all/20221102095431.36831-1-socketcan@hartkopp.net Cc: stable@vger.kernel.org # 6.0.x [mkl: patch pch_can, too] Signed-off-by: Marc Kleine-Budde --- drivers/net/can/at91_can.c | 2 +- drivers/net/can/c_can/c_can_main.c | 2 +- drivers/net/can/can327.c | 2 +- drivers/net/can/cc770/cc770.c | 2 +- drivers/net/can/ctucanfd/ctucanfd_base.c | 2 +- drivers/net/can/dev/skb.c | 10 +--------- drivers/net/can/flexcan/flexcan-core.c | 2 +- drivers/net/can/grcan.c | 2 +- drivers/net/can/ifi_canfd/ifi_canfd.c | 2 +- drivers/net/can/janz-ican3.c | 2 +- drivers/net/can/kvaser_pciefd.c | 2 +- drivers/net/can/m_can/m_can.c | 2 +- drivers/net/can/mscan/mscan.c | 2 +- drivers/net/can/pch_can.c | 2 +- drivers/net/can/peak_canfd/peak_canfd.c | 2 +- drivers/net/can/rcar/rcar_can.c | 2 +- drivers/net/can/rcar/rcar_canfd.c | 2 +- drivers/net/can/sja1000/sja1000.c | 2 +- drivers/net/can/slcan/slcan-core.c | 2 +- drivers/net/can/softing/softing_main.c | 2 +- drivers/net/can/spi/hi311x.c | 2 +- drivers/net/can/spi/mcp251x.c | 2 +- drivers/net/can/spi/mcp251xfd/mcp251xfd-tx.c | 2 +- drivers/net/can/sun4i_can.c | 2 +- drivers/net/can/ti_hecc.c | 2 +- drivers/net/can/usb/ems_usb.c | 2 +- drivers/net/can/usb/esd_usb.c | 2 +- drivers/net/can/usb/etas_es58x/es58x_core.c | 2 +- drivers/net/can/usb/gs_usb.c | 2 +- drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c | 2 +- drivers/net/can/usb/mcba_usb.c | 2 +- drivers/net/can/usb/peak_usb/pcan_usb_core.c | 2 +- drivers/net/can/usb/ucan.c | 2 +- drivers/net/can/usb/usb_8dev.c | 2 +- drivers/net/can/xilinx_can.c | 2 +- include/linux/can/dev.h | 16 ++++++++++++++++ 36 files changed, 51 insertions(+), 43 deletions(-) diff --git a/drivers/net/can/at91_can.c b/drivers/net/can/at91_can.c index 3a2d109a3792f..199cb200f2bdd 100644 --- a/drivers/net/can/at91_can.c +++ b/drivers/net/can/at91_can.c @@ -452,7 +452,7 @@ static netdev_tx_t at91_start_xmit(struct sk_buff *skb, struct net_device *dev) unsigned int mb, prio; u32 reg_mid, reg_mcr; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; mb = get_tx_next_mb(priv); diff --git a/drivers/net/can/c_can/c_can_main.c b/drivers/net/can/c_can/c_can_main.c index d6605dbb7737b..c63f7fc1e6917 100644 --- a/drivers/net/can/c_can/c_can_main.c +++ b/drivers/net/can/c_can/c_can_main.c @@ -457,7 +457,7 @@ static netdev_tx_t c_can_start_xmit(struct sk_buff *skb, struct c_can_tx_ring *tx_ring = &priv->tx; u32 idx, obj, cmd = IF_COMM_TX; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; if (c_can_tx_busy(priv, tx_ring)) diff --git a/drivers/net/can/can327.c b/drivers/net/can/can327.c index 0aa1af31d0fe4..0941977807761 100644 --- a/drivers/net/can/can327.c +++ b/drivers/net/can/can327.c @@ -813,7 +813,7 @@ static netdev_tx_t can327_netdev_start_xmit(struct sk_buff *skb, struct can327 *elm = netdev_priv(dev); struct can_frame *frame = (struct can_frame *)skb->data; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; /* We shouldn't get here after a hardware fault: diff --git a/drivers/net/can/cc770/cc770.c b/drivers/net/can/cc770/cc770.c index 0b9dfc76e769c..30909f3aab576 100644 --- a/drivers/net/can/cc770/cc770.c +++ b/drivers/net/can/cc770/cc770.c @@ -429,7 +429,7 @@ static netdev_tx_t cc770_start_xmit(struct sk_buff *skb, struct net_device *dev) struct cc770_priv *priv = netdev_priv(dev); unsigned int mo = obj2msgobj(CC770_OBJ_TX); - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; netif_stop_queue(dev); diff --git a/drivers/net/can/ctucanfd/ctucanfd_base.c b/drivers/net/can/ctucanfd/ctucanfd_base.c index b8da15ea6ad9a..64c349fd46007 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_base.c +++ b/drivers/net/can/ctucanfd/ctucanfd_base.c @@ -600,7 +600,7 @@ static netdev_tx_t ctucan_start_xmit(struct sk_buff *skb, struct net_device *nde bool ok; unsigned long flags; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; if (unlikely(!CTU_CAN_FD_TXTNF(priv))) { diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 791a51e2f5d64..241ec636e91fd 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -5,7 +5,6 @@ */ #include -#include #include #define MOD_DESC "CAN device driver interface" @@ -337,8 +336,6 @@ static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb) /* Drop a given socketbuffer if it does not contain a valid CAN frame. */ bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb) { - struct can_priv *priv = netdev_priv(dev); - switch (ntohs(skb->protocol)) { case ETH_P_CAN: if (!can_is_can_skb(skb)) @@ -359,13 +356,8 @@ bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb) goto inval_skb; } - if (!can_skb_headroom_valid(dev, skb)) { + if (!can_skb_headroom_valid(dev, skb)) goto inval_skb; - } else if (priv->ctrlmode & CAN_CTRLMODE_LISTENONLY) { - netdev_info_once(dev, - "interface in listen only mode, dropping skb\n"); - goto inval_skb; - } return false; diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c index 5ee38e586fd80..9bdadd716f4e0 100644 --- a/drivers/net/can/flexcan/flexcan-core.c +++ b/drivers/net/can/flexcan/flexcan-core.c @@ -742,7 +742,7 @@ static netdev_tx_t flexcan_start_xmit(struct sk_buff *skb, struct net_device *de u32 ctrl = FLEXCAN_MB_CODE_TX_DATA | ((can_fd_len2dlc(cfd->len)) << 16); int i; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; netif_stop_queue(dev); diff --git a/drivers/net/can/grcan.c b/drivers/net/can/grcan.c index 6c37aab93eb3a..4bedcc3eea0d6 100644 --- a/drivers/net/can/grcan.c +++ b/drivers/net/can/grcan.c @@ -1345,7 +1345,7 @@ static netdev_tx_t grcan_start_xmit(struct sk_buff *skb, unsigned long flags; u32 oneshotmode = priv->can.ctrlmode & CAN_CTRLMODE_ONE_SHOT; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; /* Trying to transmit in silent mode will generate error interrupts, but diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c index 8d42b7e6661f2..07eaf724a5727 100644 --- a/drivers/net/can/ifi_canfd/ifi_canfd.c +++ b/drivers/net/can/ifi_canfd/ifi_canfd.c @@ -860,7 +860,7 @@ static netdev_tx_t ifi_canfd_start_xmit(struct sk_buff *skb, u32 txst, txid, txdlc; int i; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; /* Check if the TX buffer is full */ diff --git a/drivers/net/can/janz-ican3.c b/drivers/net/can/janz-ican3.c index 71a2caae07579..0732a50921418 100644 --- a/drivers/net/can/janz-ican3.c +++ b/drivers/net/can/janz-ican3.c @@ -1693,7 +1693,7 @@ static netdev_tx_t ican3_xmit(struct sk_buff *skb, struct net_device *ndev) void __iomem *desc_addr; unsigned long flags; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; spin_lock_irqsave(&mod->lock, flags); diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c index 4e9680c8eb344..bcad11709bc98 100644 --- a/drivers/net/can/kvaser_pciefd.c +++ b/drivers/net/can/kvaser_pciefd.c @@ -772,7 +772,7 @@ static netdev_tx_t kvaser_pciefd_start_xmit(struct sk_buff *skb, int nwords; u8 count; - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; nwords = kvaser_pciefd_prepare_tx_packet(&packet, can, skb); diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c index dcb582563d5e4..00d11e95fd98e 100644 --- a/drivers/net/can/m_can/m_can.c +++ b/drivers/net/can/m_can/m_can.c @@ -1721,7 +1721,7 @@ static netdev_tx_t m_can_start_xmit(struct sk_buff *skb, { struct m_can_classdev *cdev = netdev_priv(dev); - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; if (cdev->is_peripheral) { diff --git a/drivers/net/can/mscan/mscan.c b/drivers/net/can/mscan/mscan.c index 2119fbb287efc..a6829cdc0e81f 100644 --- a/drivers/net/can/mscan/mscan.c +++ b/drivers/net/can/mscan/mscan.c @@ -191,7 +191,7 @@ static netdev_tx_t mscan_start_xmit(struct sk_buff *skb, struct net_device *dev) int i, rtr, buf_id; u32 can_id; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; out_8(®s->cantier, 0); diff --git a/drivers/net/can/pch_can.c b/drivers/net/can/pch_can.c index 0558ff67ec6ab..2a44b2803e555 100644 --- a/drivers/net/can/pch_can.c +++ b/drivers/net/can/pch_can.c @@ -882,7 +882,7 @@ static netdev_tx_t pch_xmit(struct sk_buff *skb, struct net_device *ndev) int i; u32 id2; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; tx_obj_no = priv->tx_obj; diff --git a/drivers/net/can/peak_canfd/peak_canfd.c b/drivers/net/can/peak_canfd/peak_canfd.c index f8420cc1d9075..31c9c127e24bb 100644 --- a/drivers/net/can/peak_canfd/peak_canfd.c +++ b/drivers/net/can/peak_canfd/peak_canfd.c @@ -651,7 +651,7 @@ static netdev_tx_t peak_canfd_start_xmit(struct sk_buff *skb, int room_left; u8 len; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; msg_size = ALIGN(sizeof(*msg) + cf->len, 4); diff --git a/drivers/net/can/rcar/rcar_can.c b/drivers/net/can/rcar/rcar_can.c index 6ee968c59ac90..cc43c9c5e38c5 100644 --- a/drivers/net/can/rcar/rcar_can.c +++ b/drivers/net/can/rcar/rcar_can.c @@ -590,7 +590,7 @@ static netdev_tx_t rcar_can_start_xmit(struct sk_buff *skb, struct can_frame *cf = (struct can_frame *)skb->data; u32 data, i; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; if (cf->can_id & CAN_EFF_FLAG) /* Extended frame format */ diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 198da643ee6dd..d530e986f7fab 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -1481,7 +1481,7 @@ static netdev_tx_t rcar_canfd_start_xmit(struct sk_buff *skb, unsigned long flags; u32 ch = priv->channel; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; if (cf->can_id & CAN_EFF_FLAG) { diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c index 1bb1129b04503..aac5956e4a532 100644 --- a/drivers/net/can/sja1000/sja1000.c +++ b/drivers/net/can/sja1000/sja1000.c @@ -291,7 +291,7 @@ static netdev_tx_t sja1000_start_xmit(struct sk_buff *skb, u8 cmd_reg_val = 0x00; int i; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; netif_stop_queue(dev); diff --git a/drivers/net/can/slcan/slcan-core.c b/drivers/net/can/slcan/slcan-core.c index 8d13fdf8c28a4..fbb34139daa1a 100644 --- a/drivers/net/can/slcan/slcan-core.c +++ b/drivers/net/can/slcan/slcan-core.c @@ -594,7 +594,7 @@ static netdev_tx_t slcan_netdev_xmit(struct sk_buff *skb, { struct slcan *sl = netdev_priv(dev); - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; spin_lock(&sl->lock); diff --git a/drivers/net/can/softing/softing_main.c b/drivers/net/can/softing/softing_main.c index a5ef57f415f73..c72f505d29fee 100644 --- a/drivers/net/can/softing/softing_main.c +++ b/drivers/net/can/softing/softing_main.c @@ -60,7 +60,7 @@ static netdev_tx_t softing_netdev_start_xmit(struct sk_buff *skb, struct can_frame *cf = (struct can_frame *)skb->data; uint8_t buf[DPRAM_TX_SIZE]; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; spin_lock(&card->spin); diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index b87dc420428d9..e1b8533a602e2 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -373,7 +373,7 @@ static netdev_tx_t hi3110_hard_start_xmit(struct sk_buff *skb, return NETDEV_TX_BUSY; } - if (can_dropped_invalid_skb(net, skb)) + if (can_dev_dropped_skb(net, skb)) return NETDEV_TX_OK; netif_stop_queue(net); diff --git a/drivers/net/can/spi/mcp251x.c b/drivers/net/can/spi/mcp251x.c index 24883a65ca66a..79c4bab5f7246 100644 --- a/drivers/net/can/spi/mcp251x.c +++ b/drivers/net/can/spi/mcp251x.c @@ -789,7 +789,7 @@ static netdev_tx_t mcp251x_hard_start_xmit(struct sk_buff *skb, return NETDEV_TX_BUSY; } - if (can_dropped_invalid_skb(net, skb)) + if (can_dev_dropped_skb(net, skb)) return NETDEV_TX_OK; netif_stop_queue(net); diff --git a/drivers/net/can/spi/mcp251xfd/mcp251xfd-tx.c b/drivers/net/can/spi/mcp251xfd/mcp251xfd-tx.c index ffb6c36b7d9bd..160528d3cc26b 100644 --- a/drivers/net/can/spi/mcp251xfd/mcp251xfd-tx.c +++ b/drivers/net/can/spi/mcp251xfd/mcp251xfd-tx.c @@ -172,7 +172,7 @@ netdev_tx_t mcp251xfd_start_xmit(struct sk_buff *skb, u8 tx_head; int err; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; if (mcp251xfd_tx_busy(priv, tx_ring)) diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index 525309da1320a..2b78f9197681b 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -429,7 +429,7 @@ static netdev_tx_t sun4ican_start_xmit(struct sk_buff *skb, struct net_device *d canid_t id; int i; - if (can_dropped_invalid_skb(dev, skb)) + if (can_dev_dropped_skb(dev, skb)) return NETDEV_TX_OK; netif_stop_queue(dev); diff --git a/drivers/net/can/ti_hecc.c b/drivers/net/can/ti_hecc.c index b218fb3c6b760..27700f72eac25 100644 --- a/drivers/net/can/ti_hecc.c +++ b/drivers/net/can/ti_hecc.c @@ -470,7 +470,7 @@ static netdev_tx_t ti_hecc_xmit(struct sk_buff *skb, struct net_device *ndev) u32 mbxno, mbx_mask, data; unsigned long flags; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; mbxno = get_tx_head_mb(priv); diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index d31191686a549..050c0b49938a4 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -747,7 +747,7 @@ static netdev_tx_t ems_usb_start_xmit(struct sk_buff *skb, struct net_device *ne size_t size = CPC_HEADER_SIZE + CPC_MSG_HEADER_LEN + sizeof(struct cpc_can_msg); - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; /* create a URB, and a buffer for it, and copy the data to the URB */ diff --git a/drivers/net/can/usb/esd_usb.c b/drivers/net/can/usb/esd_usb.c index 1bcfad11b1e44..81b88e9e5bdc0 100644 --- a/drivers/net/can/usb/esd_usb.c +++ b/drivers/net/can/usb/esd_usb.c @@ -725,7 +725,7 @@ static netdev_tx_t esd_usb_start_xmit(struct sk_buff *skb, int ret = NETDEV_TX_OK; size_t size = sizeof(struct esd_usb_msg); - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; /* create a URB, and a buffer for it, and copy the data to the URB */ diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index 51294b7170405..25f863b4f5f06 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -1913,7 +1913,7 @@ static netdev_tx_t es58x_start_xmit(struct sk_buff *skb, unsigned int frame_len; int ret; - if (can_dropped_invalid_skb(netdev, skb)) { + if (can_dev_dropped_skb(netdev, skb)) { if (priv->tx_urb) goto xmit_commit; return NETDEV_TX_OK; diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index f0065d40eb241..9c2c25fde3d14 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -723,7 +723,7 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb, unsigned int idx; struct gs_tx_context *txc; - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; /* find an empty context to keep track of transmission */ diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c index e91648ed73862..802e27c0ecedb 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_core.c @@ -570,7 +570,7 @@ static netdev_tx_t kvaser_usb_start_xmit(struct sk_buff *skb, unsigned int i; unsigned long flags; - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; urb = usb_alloc_urb(0, GFP_ATOMIC); diff --git a/drivers/net/can/usb/mcba_usb.c b/drivers/net/can/usb/mcba_usb.c index 69346c63021fe..218b098b261df 100644 --- a/drivers/net/can/usb/mcba_usb.c +++ b/drivers/net/can/usb/mcba_usb.c @@ -311,7 +311,7 @@ static netdev_tx_t mcba_usb_start_xmit(struct sk_buff *skb, .cmd_id = MBCA_CMD_TRANSMIT_MESSAGE_EV }; - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; ctx = mcba_usb_get_free_ctx(priv, cf); diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index 225697d70a9aa..1d996d3320fef 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -351,7 +351,7 @@ static netdev_tx_t peak_usb_ndo_start_xmit(struct sk_buff *skb, int i, err; size_t size = dev->adapter->tx_buffer_size; - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; for (i = 0; i < PCAN_USB_MAX_TX_URBS; i++) diff --git a/drivers/net/can/usb/ucan.c b/drivers/net/can/usb/ucan.c index 7c35f50fda4ee..67c2ff407d066 100644 --- a/drivers/net/can/usb/ucan.c +++ b/drivers/net/can/usb/ucan.c @@ -1120,7 +1120,7 @@ static netdev_tx_t ucan_start_xmit(struct sk_buff *skb, struct can_frame *cf = (struct can_frame *)skb->data; /* check skb */ - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; /* allocate a context and slow down tx path, if fifo state is low */ diff --git a/drivers/net/can/usb/usb_8dev.c b/drivers/net/can/usb/usb_8dev.c index 64c00abe91cf0..8a5596ce4e463 100644 --- a/drivers/net/can/usb/usb_8dev.c +++ b/drivers/net/can/usb/usb_8dev.c @@ -602,7 +602,7 @@ static netdev_tx_t usb_8dev_start_xmit(struct sk_buff *skb, int i, err; size_t size = sizeof(struct usb_8dev_tx_msg); - if (can_dropped_invalid_skb(netdev, skb)) + if (can_dev_dropped_skb(netdev, skb)) return NETDEV_TX_OK; /* create a URB, and a buffer for it, and copy the data to the URB */ diff --git a/drivers/net/can/xilinx_can.c b/drivers/net/can/xilinx_can.c index 5d3172795ad01..43c812ea1de02 100644 --- a/drivers/net/can/xilinx_can.c +++ b/drivers/net/can/xilinx_can.c @@ -743,7 +743,7 @@ static netdev_tx_t xcan_start_xmit(struct sk_buff *skb, struct net_device *ndev) struct xcan_priv *priv = netdev_priv(ndev); int ret; - if (can_dropped_invalid_skb(ndev, skb)) + if (can_dev_dropped_skb(ndev, skb)) return NETDEV_TX_OK; if (priv->devtype.flags & XCAN_FLAG_TX_MAILBOXES) diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h index 58f5431a55598..982ba245eb41c 100644 --- a/include/linux/can/dev.h +++ b/include/linux/can/dev.h @@ -152,6 +152,22 @@ static inline bool can_is_canxl_dev_mtu(unsigned int mtu) return (mtu >= CANXL_MIN_MTU && mtu <= CANXL_MAX_MTU); } +/* drop skb if it does not contain a valid CAN frame for sending */ +static inline bool can_dev_dropped_skb(struct net_device *dev, struct sk_buff *skb) +{ + struct can_priv *priv = netdev_priv(dev); + + if (priv->ctrlmode & CAN_CTRLMODE_LISTENONLY) { + netdev_info_once(dev, + "interface in listen only mode, dropping skb\n"); + kfree_skb(skb); + dev->stats.tx_dropped++; + return true; + } + + return can_dropped_invalid_skb(dev, skb); +} + void can_setup(struct net_device *dev); struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max, -- GitLab From 8b043dfb3dc7c32f9c2c0c93e3c2de346ee5e358 Mon Sep 17 00:00:00 2001 From: Geert Uytterhoeven Date: Fri, 28 Oct 2022 12:06:45 +0200 Subject: [PATCH 1129/1927] can: rcar_canfd: Add missing ECC error checks for channels 2-7 When introducing support for R-Car V3U, which has 8 instead of 2 channels, the ECC error bitmask was extended to take into account the extra channels, but rcar_canfd_global_error() was not updated to act upon the extra bits. Replace the RCANFD_GERFL_EEF[01] macros by a new macro that takes the channel number, fixing R-Car V3U while simplifying the code. Fixes: 45721c406dcf50d4 ("can: rcar_canfd: Add support for r8a779a0 SoC") Signed-off-by: Geert Uytterhoeven Reviewed-by: Biju Das Link: https://lore.kernel.org/all/4edb2ea46cc64d0532a08a924179827481e14b4f.1666951503.git.geert+renesas@glider.be Cc: stable@vger.kernel.org Signed-off-by: Marc Kleine-Budde --- drivers/net/can/rcar/rcar_canfd.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index d530e986f7fab..b306cf554634f 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -81,8 +81,7 @@ enum rcanfd_chip_id { /* RSCFDnCFDGERFL / RSCFDnGERFL */ #define RCANFD_GERFL_EEF0_7 GENMASK(23, 16) -#define RCANFD_GERFL_EEF1 BIT(17) -#define RCANFD_GERFL_EEF0 BIT(16) +#define RCANFD_GERFL_EEF(ch) BIT(16 + (ch)) #define RCANFD_GERFL_CMPOF BIT(3) /* CAN FD only */ #define RCANFD_GERFL_THLES BIT(2) #define RCANFD_GERFL_MES BIT(1) @@ -90,7 +89,7 @@ enum rcanfd_chip_id { #define RCANFD_GERFL_ERR(gpriv, x) \ ((x) & (reg_v3u(gpriv, RCANFD_GERFL_EEF0_7, \ - RCANFD_GERFL_EEF0 | RCANFD_GERFL_EEF1) | \ + RCANFD_GERFL_EEF(0) | RCANFD_GERFL_EEF(1)) | \ RCANFD_GERFL_MES | \ ((gpriv)->fdmode ? RCANFD_GERFL_CMPOF : 0))) @@ -936,12 +935,8 @@ static void rcar_canfd_global_error(struct net_device *ndev) u32 ridx = ch + RCANFD_RFFIFO_IDX; gerfl = rcar_canfd_read(priv->base, RCANFD_GERFL); - if ((gerfl & RCANFD_GERFL_EEF0) && (ch == 0)) { - netdev_dbg(ndev, "Ch0: ECC Error flag\n"); - stats->tx_dropped++; - } - if ((gerfl & RCANFD_GERFL_EEF1) && (ch == 1)) { - netdev_dbg(ndev, "Ch1: ECC Error flag\n"); + if (gerfl & RCANFD_GERFL_EEF(ch)) { + netdev_dbg(ndev, "Ch%u: ECC Error flag\n", ch); stats->tx_dropped++; } if (gerfl & RCANFD_GERFL_MES) { -- GitLab From f90daa975911961b65070ec72bd7dd8d448f9ef7 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 28 Oct 2022 16:50:26 +0100 Subject: [PATCH 1130/1927] drm/i915/dmabuf: fix sg_table handling in map_dma_buf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need to iterate over the original entries here for the sg_table, pulling out the struct page for each one, to be remapped. However currently this incorrectly iterates over the final dma mapped entries, which is likely just one gigantic sg entry if the iommu is enabled, leading to us only mapping the first struct page (and any physically contiguous pages following it), even if there is potentially lots more data to follow. Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7306 Fixes: 1286ff739773 ("i915: add dmabuf/prime buffer sharing support.") Signed-off-by: Matthew Auld Cc: Lionel Landwerlin Cc: Tvrtko Ursulin Cc: Ville Syrjälä Cc: Michael J. Ruhl Cc: # v3.5+ Reviewed-by: Michael J. Ruhl Link: https://patchwork.freedesktop.org/patch/msgid/20221028155029.494736-1-matthew.auld@intel.com (cherry picked from commit 28d52f99bbca7227008cf580c9194c9b3516968e) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index f5062d0c63336..824971a1ceece 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -40,13 +40,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme goto err; } - ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL); + ret = sg_alloc_table(st, obj->mm.pages->orig_nents, GFP_KERNEL); if (ret) goto err_free; src = obj->mm.pages->sgl; dst = st->sgl; - for (i = 0; i < obj->mm.pages->nents; i++) { + for (i = 0; i < obj->mm.pages->orig_nents; i++) { sg_set_page(dst, sg_page(src), src->length, 0); dst = sg_next(dst); src = sg_next(src); -- GitLab From 4ff4ebac3f1378f4ba6e11fe5ad4a4ac590bb8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jouni=20H=C3=B6gander?= Date: Mon, 24 Oct 2022 08:46:49 +0300 Subject: [PATCH 1131/1927] drm/i915/psr: Send update also on invalidate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we are observing mouse cursor stuttering when using xrandr --scaling=1.2x1.2. X scaling/transformation seems to be doing fronbuffer rendering. When moving mouse cursor X seems to perform several invalidates and only one DirtyFB. I.e. it seems to be assuming updates are sent to panel while drawing is done. Earlier we were disabling PSR in frontbuffer invalidate call back (when drawing in X started). PSR was re-enabled in frontbuffer flush callback (dirtyfb ioctl). This was working fine with X scaling/transformation. Now we are just enabling continuous full frame (cff) in PSR invalidate callback. Enabling cff doesn't trigger any updates. It just configures PSR to send full frame when updates are sent. I.e. there are no updates on screen before PSR flush callback is made. X seems to be doing several updates in frontbuffer before doing dirtyfb ioctl. Fix this by sending single update on every invalidate callback. Cc: José Roberto de Souza Cc: Ville Syrjälä Cc: Mika Kahola Fixes: 805f04d42a6b ("drm/i915/display/psr: Use continuos full frame to handle frontbuffer invalidations") Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/6679 Signed-off-by: Jouni Högander Reported-by: Brian J. Tarricone Tested-by: Brian J. Tarricone Reviewed-by: Mika Kahola Reviewed-by: José Roberto de Souza Signed-off-by: José Roberto de Souza Link: https://patchwork.freedesktop.org/patch/msgid/20221024054649.31299-1-jouni.hogander@intel.com (cherry picked from commit d755f89220a2b49bc90b7b520bb6edeb4adb5f01) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/display/intel_psr.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index d4cce627d7a87..15c3e448aa0e6 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -2201,8 +2201,11 @@ static void _psr_invalidate_handle(struct intel_dp *intel_dp) if (intel_dp->psr.psr2_sel_fetch_enabled) { u32 val; - if (intel_dp->psr.psr2_sel_fetch_cff_enabled) + if (intel_dp->psr.psr2_sel_fetch_cff_enabled) { + /* Send one update otherwise lag is observed in screen */ + intel_de_write(dev_priv, CURSURFLIVE(intel_dp->psr.pipe), 0); return; + } val = man_trk_ctl_enable_bit_get(dev_priv) | man_trk_ctl_partial_frame_bit_get(dev_priv) | -- GitLab From 19b168136395150a4a6e011f944eb30d3d85094b Mon Sep 17 00:00:00 2001 From: Niranjana Vishwanathapura Date: Tue, 1 Nov 2022 22:14:16 -0700 Subject: [PATCH 1132/1927] drm/i915: Do not set cache_dirty for DGFX Currently on DG1, which does not have LLC, we hit the below warning while rebinding an userptr invalidated object. WARNING: CPU: 4 PID: 13008 at drivers/gpu/drm/i915/gem/i915_gem_pages.c:34 __i915_gem_object_set_pages+0x296/0x2d0 [i915] ... RIP: 0010:__i915_gem_object_set_pages+0x296/0x2d0 [i915] ... Call Trace: i915_gem_userptr_get_pages+0x175/0x1a0 [i915] ____i915_gem_object_get_pages+0x32/0xb0 [i915] i915_gem_object_userptr_submit_init+0x286/0x470 [i915] eb_lookup_vmas+0x2ff/0xcf0 [i915] ? __intel_wakeref_get_first+0x55/0xb0 [i915] i915_gem_do_execbuffer+0x785/0x21d0 [i915] i915_gem_execbuffer2_ioctl+0xe7/0x3d0 [i915] We shouldn't be setting the obj->cache_dirty for DGFX, fix it. Fixes: d70af57944a1 ("drm/i915/shmem: ensure flush during swap-in on non-LLC") Suggested-by: Matthew Auld Reported-by: Niranjana Vishwanathapura Signed-off-by: Niranjana Vishwanathapura Acked-by: Nirmoy Das Reviewed-by: Matthew Auld Reviewed-by: Andi Shyti Signed-off-by: Andi Shyti Link: https://patchwork.freedesktop.org/patch/msgid/20221102051416.27327-1-niranjana.vishwanathapura@intel.com (cherry picked from commit 0aeec60c76ca2631696b4228f3fc99fe3a80013d) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 11125c32dd35d..2f7804492cd5c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -369,14 +369,14 @@ __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, __start_cpu_write(obj); /* - * On non-LLC platforms, force the flush-on-acquire if this is ever + * On non-LLC igfx platforms, force the flush-on-acquire if this is ever * swapped-in. Our async flush path is not trust worthy enough yet(and * happens in the wrong order), and with some tricks it's conceivable * for userspace to change the cache-level to I915_CACHE_NONE after the * pages are swapped-in, and since execbuf binds the object before doing * the async flush, we have a race window. */ - if (!HAS_LLC(i915)) + if (!HAS_LLC(i915) && !IS_DGFX(i915)) obj->cache_dirty = true; } -- GitLab From 178e31ce82d0308a9e5f2f15bfb4493160136729 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 28 Oct 2022 14:06:35 +0100 Subject: [PATCH 1133/1927] drm/i915/userptr: restore probe_range behaviour The conversion looks harmless, however the addr value is updated inside the loop with the previous vm_end, which then incorrectly leads to for_each_vma_range() iterating over stuff outside the range we care about. Fix this by storing the end value separately. Also fix the case where the range doesn't intersect with any vma, or if the vma itself doesn't extend the entire range, which must mean we have hole at the end. Both should result in an error, as per the previous behaviour. v2: Fix the cases where the range is empty, or if there's a hole at the end of the range Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7247 Testcase: igt@gem_userptr_blits@probe Fixes: f683b9d61319 ("i915: use the VMA iterator") Reported-by: kernel test robot Signed-off-by: Matthew Auld Cc: Tvrtko Ursulin Cc: Matthew Wilcox (Oracle) Cc: Liam R. Howlett Cc: Vlastimil Babka Cc: Yu Zhao Reviewed-by: Liam R. Howlett Reviewed-by: Andrzej Hajda Link: https://patchwork.freedesktop.org/patch/msgid/20221028130635.465839-1-matthew.auld@intel.com (cherry picked from commit 6f7de35b50860c345babf8ed0aa0d75f9315eee4) Signed-off-by: Tvrtko Ursulin --- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index f34e01a7fefb9..ba14b18d65f38 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -428,9 +428,10 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) { VMA_ITERATOR(vmi, mm, addr); struct vm_area_struct *vma; + unsigned long end = addr + len; mmap_read_lock(mm); - for_each_vma_range(vmi, vma, addr + len) { + for_each_vma_range(vmi, vma, end) { /* Check for holes, note that we also update the addr below */ if (vma->vm_start > addr) break; @@ -442,7 +443,7 @@ probe_range(struct mm_struct *mm, unsigned long addr, unsigned long len) } mmap_read_unlock(mm); - if (vma) + if (vma || addr < end) return -EFAULT; return 0; } -- GitLab From 0fca385d6ebc3cabb20f67bcf8a71f1448bdc001 Mon Sep 17 00:00:00 2001 From: Liu Shixin Date: Thu, 3 Nov 2022 16:33:01 +0800 Subject: [PATCH 1134/1927] btrfs: fix match incorrectly in dev_args_match_device syzkaller found a failed assertion: assertion failed: (args->devid != (u64)-1) || args->missing, in fs/btrfs/volumes.c:6921 This can be triggered when we set devid to (u64)-1 by ioctl. In this case, the match of devid will be skipped and the match of device may succeed incorrectly. Patch 562d7b1512f7 introduced this function which is used to match device. This function contains two matching scenarios, we can distinguish them by checking the value of args->missing rather than check whether args->devid and args->uuid is default value. Reported-by: syzbot+031687116258450f9853@syzkaller.appspotmail.com Fixes: 562d7b1512f7 ("btrfs: handle device lookup with btrfs_dev_lookup_args") CC: stable@vger.kernel.org # 5.16+ Reviewed-by: Nikolay Borisov Signed-off-by: Liu Shixin Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index a8d4bc6a19379..f09d09c259f5f 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -6918,18 +6918,18 @@ static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args, static bool dev_args_match_device(const struct btrfs_dev_lookup_args *args, const struct btrfs_device *device) { - ASSERT((args->devid != (u64)-1) || args->missing); + if (args->missing) { + if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) && + !device->bdev) + return true; + return false; + } - if ((args->devid != (u64)-1) && device->devid != args->devid) + if (device->devid != args->devid) return false; if (args->uuid && memcmp(device->uuid, args->uuid, BTRFS_UUID_SIZE) != 0) return false; - if (!args->missing) - return true; - if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) && - !device->bdev) - return true; - return false; + return true; } /* -- GitLab From 9b2f20344d450137d015b380ff0c2e2a6a170135 Mon Sep 17 00:00:00 2001 From: Zhang Xiaoxu Date: Tue, 1 Nov 2022 10:53:54 +0800 Subject: [PATCH 1135/1927] btrfs: selftests: fix wrong error check in btrfs_free_dummy_root() The btrfs_alloc_dummy_root() uses ERR_PTR as the error return value rather than NULL, if error happened, there will be a NULL pointer dereference: BUG: KASAN: null-ptr-deref in btrfs_free_dummy_root+0x21/0x50 [btrfs] Read of size 8 at addr 000000000000002c by task insmod/258926 CPU: 2 PID: 258926 Comm: insmod Tainted: G W 6.1.0-rc2+ #5 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.14.0-1.fc33 04/01/2014 Call Trace: dump_stack_lvl+0x34/0x44 kasan_report+0xb7/0x140 kasan_check_range+0x145/0x1a0 btrfs_free_dummy_root+0x21/0x50 [btrfs] btrfs_test_free_space_cache+0x1a8c/0x1add [btrfs] btrfs_run_sanity_tests+0x65/0x80 [btrfs] init_btrfs_fs+0xec/0x154 [btrfs] do_one_initcall+0x87/0x2a0 do_init_module+0xdf/0x320 load_module+0x3006/0x3390 __do_sys_finit_module+0x113/0x1b0 do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Fixes: aaedb55bc08f ("Btrfs: add tests for btrfs_get_extent") CC: stable@vger.kernel.org # 4.9+ Reviewed-by: Anand Jain Signed-off-by: Zhang Xiaoxu Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/tests/btrfs-tests.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c index 9c478fa256f65..d43cb5242fec9 100644 --- a/fs/btrfs/tests/btrfs-tests.c +++ b/fs/btrfs/tests/btrfs-tests.c @@ -200,7 +200,7 @@ void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info) void btrfs_free_dummy_root(struct btrfs_root *root) { - if (!root) + if (IS_ERR_OR_NULL(root)) return; /* Will be freed by btrfs_free_fs_roots */ if (WARN_ON(test_bit(BTRFS_ROOT_IN_RADIX, &root->state))) -- GitLab From 8bb808c6ad91ec3d332f072ce8f8aa4b16e307e0 Mon Sep 17 00:00:00 2001 From: David Sterba Date: Thu, 3 Nov 2022 14:39:01 +0100 Subject: [PATCH 1136/1927] btrfs: don't print stack trace when transaction is aborted due to ENOMEM Add ENOMEM among the error codes that don't print stack trace on transaction abort. We've got several reports from syzbot that detects stacks as errors but caused by limiting memory. As this is an artificial condition we don't need to know where exactly the error happens, the abort and error cleanup will continue like e.g. for EIO. As the transaction aborts code needs to be inline in a lot of code, the implementation cases about minimal bloat. The error codes are in a separate function and the WARN uses the condition directly. This increases the code size by 571 bytes on release build. Alternatives considered: add -ENOMEM among the errors, this increases size by 2340 bytes, various attempts to combine the WARN and helper calls, increase by 700 or more bytes. Example syzbot reports (error -12): - https://syzkaller.appspot.com/bug?extid=5244d35be7f589cf093e - https://syzkaller.appspot.com/bug?extid=9c37714c07194d816417 Signed-off-by: David Sterba --- fs/btrfs/ctree.c | 16 ++++++++++++++++ fs/btrfs/ctree.h | 11 +++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index b39b339fbf96c..a9543f01184cb 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -113,6 +113,22 @@ noinline void btrfs_release_path(struct btrfs_path *p) } } +/* + * We want the transaction abort to print stack trace only for errors where the + * cause could be a bug, eg. due to ENOSPC, and not for common errors that are + * caused by external factors. + */ +bool __cold abort_should_print_stack(int errno) +{ + switch (errno) { + case -EIO: + case -EROFS: + case -ENOMEM: + return false; + } + return true; +} + /* * safely gets a reference on the root node of a tree. A lock * is not taken, so a concurrent writer may put a different node diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index f677b49df8ae0..9e6d48ff45972 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -3796,9 +3796,11 @@ void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, const char *function, unsigned int line, int errno, bool first_hit); +bool __cold abort_should_print_stack(int errno); + /* * Call btrfs_abort_transaction as early as possible when an error condition is - * detected, that way the exact line number is reported. + * detected, that way the exact stack trace is reported for some errors. */ #define btrfs_abort_transaction(trans, errno) \ do { \ @@ -3807,10 +3809,11 @@ do { \ if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ &((trans)->fs_info->fs_state))) { \ first = true; \ - if ((errno) != -EIO && (errno) != -EROFS) { \ - WARN(1, KERN_DEBUG \ + if (WARN(abort_should_print_stack(errno), \ + KERN_DEBUG \ "BTRFS: Transaction aborted (error %d)\n", \ - (errno)); \ + (errno))) { \ + /* Stack trace printed. */ \ } else { \ btrfs_debug((trans)->fs_info, \ "Transaction aborted (error %d)", \ -- GitLab From b75b51f886e9dd8cdfca1392ad43f4e542611c00 Mon Sep 17 00:00:00 2001 From: Qu Wenruo Date: Mon, 7 Nov 2022 07:23:26 +0800 Subject: [PATCH 1137/1927] Revert "btrfs: scrub: use larger block size for data extent scrub" This reverts commit 786672e9e1a39a231806313e3c445c236588ceef. [BUG] Since commit 786672e9e1a3 ("btrfs: scrub: use larger block size for data extent scrub"), btrfs scrub no longer reports errors if the corruption is not in the first sector of a STRIPE_LEN. The following script can expose the problem: mkfs.btrfs -f $dev mount $dev $mnt xfs_io -f -c "pwrite -S 0xff 0 8k" $mnt/foobar umount $mnt # 13631488 is the logical bytenr of above 8K extent btrfs-map-logical -l 13631488 -b 4096 $dev mirror 1 logical 13631488 physical 13631488 device /dev/test/scratch1 # Corrupt the 2nd sector of that extent xfs_io -f -c "pwrite -S 0x00 13635584 4k" $dev mount $dev $mnt btrfs scrub start -B $mnt scrub done for 54e63f9f-0c30-4c84-a33b-5c56014629b7 Scrub started: Mon Nov 7 07:18:27 2022 Status: finished Duration: 0:00:00 Total to scrub: 536.00MiB Rate: 0.00B/s Error summary: no errors found <<< [CAUSE] That offending commit enlarges the data extent scrub size from sector size to BTRFS_STRIPE_LEN, to avoid extra scrub_block to be allocated. But unfortunately the data extent scrub is still heavily relying on the fact that there is only one scrub_sector per scrub_block. Thus it will only check the first sector, and ignoring the remaining sectors. Furthermore the error reporting is not able to handle multiple sectors either. [FIX] For now just revert the offending commit. The consequence is just extra memory usage during scrub. We will need a proper change to make the remaining data scrub path to handle multiple sectors before we enlarging the data scrub size. Reported-by: Li Zhang Signed-off-by: Qu Wenruo Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index f260c53829e5e..b659c67af1e09 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -2672,17 +2672,11 @@ static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map, u8 csum[BTRFS_CSUM_SIZE]; u32 blocksize; - /* - * Block size determines how many scrub_block will be allocated. Here - * we use BTRFS_STRIPE_LEN (64KiB) as default limit, so we won't - * allocate too many scrub_block, while still won't cause too large - * bios for large extents. - */ if (flags & BTRFS_EXTENT_FLAG_DATA) { if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) blocksize = map->stripe_len; else - blocksize = BTRFS_STRIPE_LEN; + blocksize = sctx->fs_info->sectorsize; spin_lock(&sctx->stat_lock); sctx->stat.data_extents_scrubbed++; sctx->stat.data_bytes_scrubbed += len; -- GitLab From 21e61ec6d0bb786818490e926aa9aeb4de95ad0d Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Fri, 4 Nov 2022 07:12:33 -0700 Subject: [PATCH 1138/1927] btrfs: zoned: clone zoned device info when cloning a device When cloning a btrfs_device, we're not cloning the associated btrfs_zoned_device_info structure of the device in case of a zoned filesystem. Later on this leads to a NULL pointer dereference when accessing the device's zone_info for instance when setting a zone as active. This was uncovered by fstests' testcase btrfs/161. CC: stable@vger.kernel.org # 5.15+ Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/volumes.c | 12 ++++++++++++ fs/btrfs/zoned.c | 40 ++++++++++++++++++++++++++++++++++++++++ fs/btrfs/zoned.h | 11 +++++++++++ 3 files changed, 63 insertions(+) diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index f09d09c259f5f..3cb968ede6759 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1011,6 +1011,18 @@ static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig) rcu_assign_pointer(device->name, name); } + if (orig_dev->zone_info) { + struct btrfs_zoned_device_info *zone_info; + + zone_info = btrfs_clone_dev_zone_info(orig_dev); + if (!zone_info) { + btrfs_free_device(device); + ret = -ENOMEM; + goto error; + } + device->zone_info = zone_info; + } + list_add(&device->dev_list, &fs_devices->devices); device->fs_devices = fs_devices; fs_devices->num_devices++; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index e2d073b08a7d2..1912abf6d0207 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -639,6 +639,46 @@ void btrfs_destroy_dev_zone_info(struct btrfs_device *device) device->zone_info = NULL; } +struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev) +{ + struct btrfs_zoned_device_info *zone_info; + + zone_info = kmemdup(orig_dev->zone_info, sizeof(*zone_info), GFP_KERNEL); + if (!zone_info) + return NULL; + + zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->seq_zones) + goto out; + + bitmap_copy(zone_info->seq_zones, orig_dev->zone_info->seq_zones, + zone_info->nr_zones); + + zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->empty_zones) + goto out; + + bitmap_copy(zone_info->empty_zones, orig_dev->zone_info->empty_zones, + zone_info->nr_zones); + + zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->active_zones) + goto out; + + bitmap_copy(zone_info->active_zones, orig_dev->zone_info->active_zones, + zone_info->nr_zones); + zone_info->zone_cache = NULL; + + return zone_info; + +out: + bitmap_free(zone_info->seq_zones); + bitmap_free(zone_info->empty_zones); + bitmap_free(zone_info->active_zones); + kfree(zone_info); + return NULL; +} + int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) { diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index e17462db3a842..8bd16d40b7c65 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -36,6 +36,7 @@ int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info); int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache); void btrfs_destroy_dev_zone_info(struct btrfs_device *device); +struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev); int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info); int btrfs_check_mountopts_zoned(struct btrfs_fs_info *info); int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, @@ -103,6 +104,16 @@ static inline int btrfs_get_dev_zone_info(struct btrfs_device *device, static inline void btrfs_destroy_dev_zone_info(struct btrfs_device *device) { } +/* + * In case the kernel is compiled without CONFIG_BLK_DEV_ZONED we'll never call + * into btrfs_clone_dev_zone_info() so it's safe to return NULL here. + */ +static inline struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info( + struct btrfs_device *orig_dev) +{ + return NULL; +} + static inline int btrfs_check_zoned_mode(const struct btrfs_fs_info *fs_info) { if (!btrfs_is_zoned(fs_info)) -- GitLab From a8d1b1647bf8244a5f270538e9e636e2657fffa3 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Fri, 4 Nov 2022 07:12:34 -0700 Subject: [PATCH 1139/1927] btrfs: zoned: initialize device's zone info for seeding When performing seeding on a zoned filesystem it is necessary to initialize each zoned device's btrfs_zoned_device_info structure, otherwise mounting the filesystem will cause a NULL pointer dereference. This was uncovered by fstests' testcase btrfs/163. CC: stable@vger.kernel.org # 5.15+ Signed-off-by: Johannes Thumshirn Signed-off-by: David Sterba --- fs/btrfs/disk-io.c | 4 +++- fs/btrfs/volumes.c | 11 +++++++++-- fs/btrfs/volumes.h | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 4b28263c3d329..d99bf7c646110 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -2551,7 +2551,9 @@ static int btrfs_read_roots(struct btrfs_fs_info *fs_info) fs_info->dev_root = root; } /* Initialize fs_info for all devices in any case */ - btrfs_init_devices_late(fs_info); + ret = btrfs_init_devices_late(fs_info); + if (ret) + goto out; /* * This tree can share blocks with some other fs tree during relocation diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 3cb968ede6759..635f45f1a2ef8 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -7756,10 +7756,11 @@ int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info) return ret; } -void btrfs_init_devices_late(struct btrfs_fs_info *fs_info) +int btrfs_init_devices_late(struct btrfs_fs_info *fs_info) { struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; struct btrfs_device *device; + int ret = 0; fs_devices->fs_info = fs_info; @@ -7768,12 +7769,18 @@ void btrfs_init_devices_late(struct btrfs_fs_info *fs_info) device->fs_info = fs_info; list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { - list_for_each_entry(device, &seed_devs->devices, dev_list) + list_for_each_entry(device, &seed_devs->devices, dev_list) { device->fs_info = fs_info; + ret = btrfs_get_dev_zone_info(device, false); + if (ret) + break; + } seed_devs->fs_info = fs_info; } mutex_unlock(&fs_devices->device_list_mutex); + + return ret; } static u64 btrfs_dev_stats_value(const struct extent_buffer *eb, diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index f8b668dc8bf81..099def5613b87 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -671,7 +671,7 @@ int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index); int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info, struct btrfs_ioctl_get_dev_stats *stats); -void btrfs_init_devices_late(struct btrfs_fs_info *fs_info); +int btrfs_init_devices_late(struct btrfs_fs_info *fs_info); int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info); int btrfs_run_dev_stats(struct btrfs_trans_handle *trans); void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev); -- GitLab From c62f6bec53e63b11112e1ebce6bbaa39ce6f6706 Mon Sep 17 00:00:00 2001 From: Johannes Thumshirn Date: Fri, 4 Nov 2022 07:12:35 -0700 Subject: [PATCH 1140/1927] btrfs: zoned: fix locking imbalance on scrub If we're doing device replace on a zoned filesystem and discover in scrub_enumerate_chunks() that we don't have to copy the block group it is unlocked before it gets skipped. But as the block group hasn't yet been locked before it leads to a locking imbalance. To fix this simply remove the unlock. This was uncovered by fstests' testcase btrfs/163. Fixes: 9283b9e09a6d ("btrfs: remove lock protection for BLOCK_GROUP_FLAG_TO_COPY") Signed-off-by: Johannes Thumshirn Reviewed-by: David Sterba Signed-off-by: David Sterba --- fs/btrfs/scrub.c | 1 - 1 file changed, 1 deletion(-) diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index b659c67af1e09..196c4c6ed1ed8 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -3911,7 +3911,6 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { if (!test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags)) { - spin_unlock(&cache->lock); btrfs_put_block_group(cache); goto skip; } -- GitLab From 314d34fe7f0a5836cb0472950c1f17744b4efde8 Mon Sep 17 00:00:00 2001 From: Chen Zhongjin Date: Mon, 31 Oct 2022 21:40:31 +0800 Subject: [PATCH 1141/1927] ASoC: soc-utils: Remove __exit for snd_soc_util_exit() snd_soc_util_exit() is called in __init snd_soc_init() for cleanup. Remove the __exit annotation for it to fix the build warning: WARNING: modpost: sound/soc/snd-soc-core.o: section mismatch in reference: init_module (section: .init.text) -> snd_soc_util_exit (section: .exit.text) Fixes: 6ec27c53886c ("ASoC: core: Fix use-after-free in snd_soc_exit()") Signed-off-by: Chen Zhongjin Link: https://lore.kernel.org/r/20221031134031.256511-1-chenzhongjin@huawei.com Signed-off-by: Mark Brown --- sound/soc/soc-utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sound/soc/soc-utils.c b/sound/soc/soc-utils.c index a3b6df2378b40..a4dba0b751e76 100644 --- a/sound/soc/soc-utils.c +++ b/sound/soc/soc-utils.c @@ -264,7 +264,7 @@ int __init snd_soc_util_init(void) return ret; } -void __exit snd_soc_util_exit(void) +void snd_soc_util_exit(void) { platform_driver_unregister(&soc_dummy_driver); platform_device_unregister(soc_dummy_dev); -- GitLab From 3d59eaef49ca2db581156a7b77c9afc0546eefc0 Mon Sep 17 00:00:00 2001 From: Peter Ujfalusi Date: Mon, 7 Nov 2022 11:04:33 +0200 Subject: [PATCH 1142/1927] ASoC: SOF: topology: No need to assign core ID if token parsing failed Move the return value check before attempting to assign the core ID to the swidget since we are going to fail the sof_widget_ready() and free up swidget anyways. Fixes: 909dadf21aae ("ASoC: SOF: topology: Make DAI widget parsing IPC agnostic") Signed-off-by: Peter Ujfalusi Reviewed-by: Pierre-Louis Bossart Reviewed-by: Ranjani Sridharan Link: https://lore.kernel.org/r/20221107090433.5146-1-peter.ujfalusi@linux.intel.com Signed-off-by: Mark Brown --- sound/soc/sof/topology.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 38855dd60617d..6a0e7f3b50234 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -1344,16 +1344,6 @@ static int sof_widget_ready(struct snd_soc_component *scomp, int index, break; } - if (sof_debug_check_flag(SOF_DBG_DISABLE_MULTICORE)) { - swidget->core = SOF_DSP_PRIMARY_CORE; - } else { - int core = sof_get_token_value(SOF_TKN_COMP_CORE_ID, swidget->tuples, - swidget->num_tuples); - - if (core >= 0) - swidget->core = core; - } - /* check token parsing reply */ if (ret < 0) { dev_err(scomp->dev, @@ -1365,6 +1355,16 @@ static int sof_widget_ready(struct snd_soc_component *scomp, int index, return ret; } + if (sof_debug_check_flag(SOF_DBG_DISABLE_MULTICORE)) { + swidget->core = SOF_DSP_PRIMARY_CORE; + } else { + int core = sof_get_token_value(SOF_TKN_COMP_CORE_ID, swidget->tuples, + swidget->num_tuples); + + if (core >= 0) + swidget->core = core; + } + /* bind widget to external event */ if (tw->event_type) { if (widget_ops[w->id].bind_event) { -- GitLab From 89cdb224f2abe37ec4ac21ba0d9ddeb5a6a9cf68 Mon Sep 17 00:00:00 2001 From: Zhu Ning Date: Fri, 28 Oct 2022 10:04:56 +0800 Subject: [PATCH 1143/1927] ASoC: sof_es8336: reduce pop noise on speaker The Speaker GPIO needs to be turned on slightly behind the codec turned on. It also need to be turned off slightly before the codec turned down. Current code uses delay in DAPM_EVENT to do it but the mdelay delays the DAPM itself and thus has no effect. A delayed_work is added to turn on the speaker. The Speaker is turned off in .trigger since trigger is called slightly before the DAPM events. Signed-off-by: Zhu Ning ------------ v1: cancel delayed work while disabling speaker. Link: https://lore.kernel.org/r/20221028020456.90286-1-zhuning0077@gmail.com Signed-off-by: Mark Brown --- sound/soc/intel/boards/sof_es8336.c | 60 ++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/sound/soc/intel/boards/sof_es8336.c b/sound/soc/intel/boards/sof_es8336.c index fbb42e54947a8..70713e4b07dc1 100644 --- a/sound/soc/intel/boards/sof_es8336.c +++ b/sound/soc/intel/boards/sof_es8336.c @@ -63,6 +63,7 @@ struct sof_es8336_private { struct snd_soc_jack jack; struct list_head hdmi_pcm_list; bool speaker_en; + struct delayed_work pcm_pop_work; }; struct sof_hdmi_pcm { @@ -111,6 +112,46 @@ static void log_quirks(struct device *dev) dev_info(dev, "quirk headset at mic1 port enabled\n"); } +static void pcm_pop_work_events(struct work_struct *work) +{ + struct sof_es8336_private *priv = + container_of(work, struct sof_es8336_private, pcm_pop_work.work); + + gpiod_set_value_cansleep(priv->gpio_speakers, priv->speaker_en); + + if (quirk & SOF_ES8336_HEADPHONE_GPIO) + gpiod_set_value_cansleep(priv->gpio_headphone, priv->speaker_en); + +} + +static int sof_8336_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_card *card = rtd->card; + struct sof_es8336_private *priv = snd_soc_card_get_drvdata(card); + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + case SNDRV_PCM_TRIGGER_RESUME: + break; + + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_STOP: + if (priv->speaker_en == false) + if (substream->stream == 0) { + cancel_delayed_work(&priv->pcm_pop_work); + gpiod_set_value_cansleep(priv->gpio_speakers, true); + } + break; + default: + return -EINVAL; + } + + return 0; +} + static int sof_es8316_speaker_power_event(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol, int event) { @@ -122,19 +163,7 @@ static int sof_es8316_speaker_power_event(struct snd_soc_dapm_widget *w, priv->speaker_en = !SND_SOC_DAPM_EVENT_ON(event); - if (SND_SOC_DAPM_EVENT_ON(event)) - msleep(70); - - gpiod_set_value_cansleep(priv->gpio_speakers, priv->speaker_en); - - if (!(quirk & SOF_ES8336_HEADPHONE_GPIO)) - return 0; - - if (SND_SOC_DAPM_EVENT_ON(event)) - msleep(70); - - gpiod_set_value_cansleep(priv->gpio_headphone, priv->speaker_en); - + queue_delayed_work(system_wq, &priv->pcm_pop_work, msecs_to_jiffies(70)); return 0; } @@ -344,6 +373,7 @@ static int sof_es8336_hw_params(struct snd_pcm_substream *substream, /* machine stream operations */ static struct snd_soc_ops sof_es8336_ops = { .hw_params = sof_es8336_hw_params, + .trigger = sof_8336_trigger, }; static struct snd_soc_dai_link_component platform_component[] = { @@ -723,7 +753,8 @@ static int sof_es8336_probe(struct platform_device *pdev) } INIT_LIST_HEAD(&priv->hdmi_pcm_list); - + INIT_DELAYED_WORK(&priv->pcm_pop_work, + pcm_pop_work_events); snd_soc_card_set_drvdata(card, priv); if (mach->mach_params.dmic_num > 0) { @@ -752,6 +783,7 @@ static int sof_es8336_remove(struct platform_device *pdev) struct snd_soc_card *card = platform_get_drvdata(pdev); struct sof_es8336_private *priv = snd_soc_card_get_drvdata(card); + cancel_delayed_work(&priv->pcm_pop_work); gpiod_put(priv->gpio_speakers); device_remove_software_node(priv->codec_dev); put_device(priv->codec_dev); -- GitLab From bee55f2e7a44e7a7676e264b42f026e34bd244d9 Mon Sep 17 00:00:00 2001 From: Quentin Schulz Date: Tue, 18 Oct 2022 14:17:23 +0200 Subject: [PATCH 1144/1927] pinctrl: rockchip: list all pins in a possible mux route for PX30 The mux routes are incomplete for the PX30. This was discovered because we had a HW design using cif-clkoutm1 with the correct pinmux in the Device Tree but the clock would still not work. There are actually two muxing required: the pin muxing (performed by the usual Device Tree pinctrl nodes) and the "function" muxing (m0 vs m1; performed by the mux routing inside the driver). The pin muxing was correct but the function muxing was not. This adds the missing pins and their configuration for the mux routes that are already specified in the driver. Note that there are some "conflicts": it is possible *in Device Tree* to (attempt to) mux the pins for e.g. clkoutm1 and clkinm0 at the same time but this is actually not possible in hardware (because both share the same bit for the function muxing). Since it is an impossible hardware design, it is not deemed necessary to prevent the user from attempting to "misconfigure" the pins/functions. Fixes: 87065ca9b8e5 ("pinctrl: rockchip: Add pinctrl support for PX30") Signed-off-by: Quentin Schulz Link: https://lore.kernel.org/r/20221017-upstream-px30-cif-clkoutm1-v1-0-4ea1389237f7@theobroma-systems.com Signed-off-by: Linus Walleij --- drivers/pinctrl/pinctrl-rockchip.c | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index 53bdfc40f0558..da974ff2d75d0 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -679,14 +679,54 @@ static void rockchip_get_recalced_mux(struct rockchip_pin_bank *bank, int pin, } static struct rockchip_mux_route_data px30_mux_route_data[] = { + RK_MUXROUTE_SAME(2, RK_PB4, 1, 0x184, BIT(16 + 7)), /* cif-d0m0 */ + RK_MUXROUTE_SAME(3, RK_PA1, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d0m1 */ + RK_MUXROUTE_SAME(2, RK_PB6, 1, 0x184, BIT(16 + 7)), /* cif-d1m0 */ + RK_MUXROUTE_SAME(3, RK_PA2, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d1m1 */ RK_MUXROUTE_SAME(2, RK_PA0, 1, 0x184, BIT(16 + 7)), /* cif-d2m0 */ RK_MUXROUTE_SAME(3, RK_PA3, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d2m1 */ + RK_MUXROUTE_SAME(2, RK_PA1, 1, 0x184, BIT(16 + 7)), /* cif-d3m0 */ + RK_MUXROUTE_SAME(3, RK_PA5, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d3m1 */ + RK_MUXROUTE_SAME(2, RK_PA2, 1, 0x184, BIT(16 + 7)), /* cif-d4m0 */ + RK_MUXROUTE_SAME(3, RK_PA7, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d4m1 */ + RK_MUXROUTE_SAME(2, RK_PA3, 1, 0x184, BIT(16 + 7)), /* cif-d5m0 */ + RK_MUXROUTE_SAME(3, RK_PB0, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d5m1 */ + RK_MUXROUTE_SAME(2, RK_PA4, 1, 0x184, BIT(16 + 7)), /* cif-d6m0 */ + RK_MUXROUTE_SAME(3, RK_PB1, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d6m1 */ + RK_MUXROUTE_SAME(2, RK_PA5, 1, 0x184, BIT(16 + 7)), /* cif-d7m0 */ + RK_MUXROUTE_SAME(3, RK_PB4, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d7m1 */ + RK_MUXROUTE_SAME(2, RK_PA6, 1, 0x184, BIT(16 + 7)), /* cif-d8m0 */ + RK_MUXROUTE_SAME(3, RK_PB6, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d8m1 */ + RK_MUXROUTE_SAME(2, RK_PA7, 1, 0x184, BIT(16 + 7)), /* cif-d9m0 */ + RK_MUXROUTE_SAME(3, RK_PB7, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d9m1 */ + RK_MUXROUTE_SAME(2, RK_PB7, 1, 0x184, BIT(16 + 7)), /* cif-d10m0 */ + RK_MUXROUTE_SAME(3, RK_PC6, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d10m1 */ + RK_MUXROUTE_SAME(2, RK_PC0, 1, 0x184, BIT(16 + 7)), /* cif-d11m0 */ + RK_MUXROUTE_SAME(3, RK_PC7, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-d11m1 */ + RK_MUXROUTE_SAME(2, RK_PB0, 1, 0x184, BIT(16 + 7)), /* cif-vsyncm0 */ + RK_MUXROUTE_SAME(3, RK_PD1, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-vsyncm1 */ + RK_MUXROUTE_SAME(2, RK_PB1, 1, 0x184, BIT(16 + 7)), /* cif-hrefm0 */ + RK_MUXROUTE_SAME(3, RK_PD2, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-hrefm1 */ + RK_MUXROUTE_SAME(2, RK_PB2, 1, 0x184, BIT(16 + 7)), /* cif-clkinm0 */ + RK_MUXROUTE_SAME(3, RK_PD3, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-clkinm1 */ + RK_MUXROUTE_SAME(2, RK_PB3, 1, 0x184, BIT(16 + 7)), /* cif-clkoutm0 */ + RK_MUXROUTE_SAME(3, RK_PD0, 3, 0x184, BIT(16 + 7) | BIT(7)), /* cif-clkoutm1 */ RK_MUXROUTE_SAME(3, RK_PC6, 2, 0x184, BIT(16 + 8)), /* pdm-m0 */ RK_MUXROUTE_SAME(2, RK_PC6, 1, 0x184, BIT(16 + 8) | BIT(8)), /* pdm-m1 */ + RK_MUXROUTE_SAME(3, RK_PD3, 2, 0x184, BIT(16 + 8)), /* pdm-sdi0m0 */ + RK_MUXROUTE_SAME(2, RK_PC5, 2, 0x184, BIT(16 + 8) | BIT(8)), /* pdm-sdi0m1 */ RK_MUXROUTE_SAME(1, RK_PD3, 2, 0x184, BIT(16 + 10)), /* uart2-rxm0 */ RK_MUXROUTE_SAME(2, RK_PB6, 2, 0x184, BIT(16 + 10) | BIT(10)), /* uart2-rxm1 */ + RK_MUXROUTE_SAME(1, RK_PD2, 2, 0x184, BIT(16 + 10)), /* uart2-txm0 */ + RK_MUXROUTE_SAME(2, RK_PB4, 2, 0x184, BIT(16 + 10) | BIT(10)), /* uart2-txm1 */ RK_MUXROUTE_SAME(0, RK_PC1, 2, 0x184, BIT(16 + 9)), /* uart3-rxm0 */ RK_MUXROUTE_SAME(1, RK_PB7, 2, 0x184, BIT(16 + 9) | BIT(9)), /* uart3-rxm1 */ + RK_MUXROUTE_SAME(0, RK_PC0, 2, 0x184, BIT(16 + 9)), /* uart3-txm0 */ + RK_MUXROUTE_SAME(1, RK_PB6, 2, 0x184, BIT(16 + 9) | BIT(9)), /* uart3-txm1 */ + RK_MUXROUTE_SAME(0, RK_PC2, 2, 0x184, BIT(16 + 9)), /* uart3-ctsm0 */ + RK_MUXROUTE_SAME(1, RK_PB4, 2, 0x184, BIT(16 + 9) | BIT(9)), /* uart3-ctsm1 */ + RK_MUXROUTE_SAME(0, RK_PC3, 2, 0x184, BIT(16 + 9)), /* uart3-rtsm0 */ + RK_MUXROUTE_SAME(1, RK_PB5, 2, 0x184, BIT(16 + 9) | BIT(9)), /* uart3-rtsm1 */ }; static struct rockchip_mux_route_data rv1126_mux_route_data[] = { -- GitLab From c4299907c09a638c0a30f029338d07941c049d73 Mon Sep 17 00:00:00 2001 From: Steven Price Date: Thu, 3 Nov 2022 11:40:36 +0000 Subject: [PATCH 1145/1927] drm/panfrost: Remove type name from internal struct again Commit 72655fb942c1 ("drm/panfrost: replace endian-specific types with native ones") accidentally reverted part of the parent commit 7228d9d79248 ("drm/panfrost: Remove type name from internal structs") leading to the situation that the Panfrost UAPI header still doesn't compile correctly in C++. Revert the accidental revert and pass me a brown paper bag. Reported-by: Alyssa Rosenzweig Fixes: 72655fb942c1 ("drm/panfrost: replace endian-specific types with native ones") Signed-off-by: Steven Price Reviewed-by: Alyssa Rosenzweig Link: https://patchwork.freedesktop.org/patch/msgid/20221103114036.1581854-1-steven.price@arm.com --- include/uapi/drm/panfrost_drm.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h index 6f93c915cc88a..9f231d40a146a 100644 --- a/include/uapi/drm/panfrost_drm.h +++ b/include/uapi/drm/panfrost_drm.h @@ -254,7 +254,7 @@ struct panfrost_dump_object_header { __u64 nbos; } reghdr; - struct pan_bomap_hdr { + struct { __u32 valid; __u64 iova; __u32 data[2]; -- GitLab From 7df140e84a75c89962feef659d686303d3ce75e5 Mon Sep 17 00:00:00 2001 From: Christian Marangi Date: Fri, 21 Oct 2022 18:53:04 +0200 Subject: [PATCH 1146/1927] mtd: rawnand: qcom: handle ret from parse with codeword_fixup With use_codeword_fixup enabled, any return from mtd_device_parse_register gets overwritten. Aside from the clear bug, this is also problematic as a parser can EPROBE_DEFER and because this is not correctly handled, the nand is never rescanned later in the bootup process. An example of this problem is when smem requires additional time to be probed and nandc use qcomsmempart as parser. Parser will return EPROBE_DEFER but in the current code this ret gets overwritten by qcom_nand_host_parse_boot_partitions and qcom_nand_host_init_and_register return 0. Correctly handle the return code from mtd_device_parse_register so that any error from this function is not ignored. Fixes: 862bdedd7f4b ("mtd: nand: raw: qcom_nandc: add support for unprotected spare data pages") Cc: stable@vger.kernel.org # v6.0+ Signed-off-by: Christian Marangi Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20221021165304.19991-1-ansuelsmth@gmail.com --- drivers/mtd/nand/raw/qcom_nandc.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/mtd/nand/raw/qcom_nandc.c b/drivers/mtd/nand/raw/qcom_nandc.c index 8f80019a9f016..198a44794d2dc 100644 --- a/drivers/mtd/nand/raw/qcom_nandc.c +++ b/drivers/mtd/nand/raw/qcom_nandc.c @@ -3167,16 +3167,18 @@ static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc, ret = mtd_device_parse_register(mtd, probes, NULL, NULL, 0); if (ret) - nand_cleanup(chip); + goto err; if (nandc->props->use_codeword_fixup) { ret = qcom_nand_host_parse_boot_partitions(nandc, host, dn); - if (ret) { - nand_cleanup(chip); - return ret; - } + if (ret) + goto err; } + return 0; + +err: + nand_cleanup(chip); return ret; } -- GitLab From 83f0869e9bf3333d778d62f055b0f8e1de1cc812 Mon Sep 17 00:00:00 2001 From: Adam Borowski Date: Mon, 24 Oct 2022 11:20:26 +0200 Subject: [PATCH 1147/1927] mtd: rawnand: placate "$VARIABLE is used uninitialized" warnings The compiler is not smart enough to notice that it's impossible for them to be actually used uninitialized. Which exact variables trip here varies depending on random surrounding code; none triggered in 6.1-rc1 but 6.1-rc2 fails on three of these five, despite variables declared in the very same line having identical flow. Signed-off-by: Adam Borowski Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20221024092026.42123-1-kilobyte@angband.pl --- drivers/mtd/nand/raw/nand_base.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mtd/nand/raw/nand_base.c b/drivers/mtd/nand/raw/nand_base.c index 33f2c98a030eb..c3cc660399255 100644 --- a/drivers/mtd/nand/raw/nand_base.c +++ b/drivers/mtd/nand/raw/nand_base.c @@ -5834,7 +5834,7 @@ nand_match_ecc_req(struct nand_chip *chip, int req_step = requirements->step_size; int req_strength = requirements->strength; int req_corr, step_size, strength, nsteps, ecc_bytes, ecc_bytes_total; - int best_step, best_strength, best_ecc_bytes; + int best_step = 0, best_strength = 0, best_ecc_bytes = 0; int best_ecc_bytes_total = INT_MAX; int i, j; @@ -5915,7 +5915,7 @@ nand_maximize_ecc(struct nand_chip *chip, int step_size, strength, nsteps, ecc_bytes, corr; int best_corr = 0; int best_step = 0; - int best_strength, best_ecc_bytes; + int best_strength = 0, best_ecc_bytes = 0; int i, j; for (i = 0; i < caps->nstepinfos; i++) { -- GitLab From c717b9b7d6de9e024e47f7cd5bbff49f581d3db9 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Mon, 7 Nov 2022 10:15:20 +0100 Subject: [PATCH 1148/1927] mtd: onenand: omap2: add dependency on GPMC OMAP2 OneNAND driver uses gpmc_omap_onenand_set_timings() provided by OMAP_GPMC driver, so the latter cannot be module if OneNAND driver is built-in: /usr/bin/arm-linux-gnueabi-ld: drivers/mtd/nand/onenand/onenand_omap2.o: in function `omap2_onenand_probe': onenand_omap2.c:(.text+0x520): undefined reference to `gpmc_omap_onenand_set_timings' The OMAP_GPMC is also a runtime dependency. Reported-by: kernel test robot Fixes: 854fd9209b20 ("memory: omap-gpmc: Allow building as a module") Signed-off-by: Krzysztof Kozlowski Reviewed-by: Roger Quadros Signed-off-by: Miquel Raynal Link: https://lore.kernel.org/linux-mtd/20221107091520.127053-1-krzysztof.kozlowski@linaro.org --- drivers/mtd/nand/onenand/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/mtd/nand/onenand/Kconfig b/drivers/mtd/nand/onenand/Kconfig index 34d9a7a82ad4e..c94bf483541e1 100644 --- a/drivers/mtd/nand/onenand/Kconfig +++ b/drivers/mtd/nand/onenand/Kconfig @@ -26,6 +26,7 @@ config MTD_ONENAND_OMAP2 tristate "OneNAND on OMAP2/OMAP3 support" depends on ARCH_OMAP2 || ARCH_OMAP3 || (COMPILE_TEST && ARM) depends on OF || COMPILE_TEST + depends on OMAP_GPMC help Support for a OneNAND flash device connected to an OMAP2/OMAP3 SoC via the GPMC memory controller. -- GitLab From c18c20f16219516b12a4f2fd29c25e06be97e064 Mon Sep 17 00:00:00 2001 From: Vlastimil Babka Date: Mon, 7 Nov 2022 17:11:27 +0100 Subject: [PATCH 1149/1927] mm, slab: remove duplicate kernel-doc comment for ksize() Akira reports: > "make htmldocs" reports duplicate C declaration of ksize() as follows: > /linux/Documentation/core-api/mm-api:43: ./mm/slab_common.c:1428: WARNING: Duplicate C declaration, also defined at core-api/mm-api:212. > Declaration is '.. c:function:: size_t ksize (const void *objp)'. > This is due to the kernel-doc comment for ksize() declaration added in > include/linux/slab.h by commit 05a940656e1e ("slab: Introduce > kmalloc_size_roundup()"). There is an older kernel-doc comment for ksize() definition in mm/slab_common.c, which is not only duplicated, but also contradicts the new one - the additional storage discovered by ksize() should not be used by callers anymore. Delete the old kernel-doc. Reported-by: Akira Yokosawa Link: https://lore.kernel.org/all/d33440f6-40cf-9747-3340-e54ffaf7afb8@gmail.com/ Fixes: 05a940656e1e ("slab: Introduce kmalloc_size_roundup()") Cc: Kees Cook Signed-off-by: Vlastimil Babka --- mm/slab_common.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/mm/slab_common.c b/mm/slab_common.c index fa784563e1edb..0042fb2730d1e 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -1409,20 +1409,6 @@ void kfree_sensitive(const void *p) } EXPORT_SYMBOL(kfree_sensitive); -/** - * ksize - get the actual amount of memory allocated for a given object - * @objp: Pointer to the object - * - * kmalloc may internally round up allocations and return more memory - * than requested. ksize() can be used to determine the actual amount of - * memory allocated. The caller may use this additional memory, even though - * a smaller amount of memory was initially specified with the kmalloc call. - * The caller must guarantee that objp points to a valid object previously - * allocated with either kmalloc() or kmem_cache_alloc(). The object - * must not be freed during the duration of the call. - * - * Return: size of the actual memory used by @objp in bytes - */ size_t ksize(const void *objp) { size_t size; -- GitLab From b8ebf250997c5fb253582f42bfe98673801ebebd Mon Sep 17 00:00:00 2001 From: Tetsuo Handa Date: Mon, 7 Nov 2022 10:21:40 -0800 Subject: [PATCH 1150/1927] Input: iforce - invert valid length check when fetching device IDs syzbot is reporting uninitialized value at iforce_init_device() [1], for commit 6ac0aec6b0a6 ("Input: iforce - allow callers supply data buffer when fetching device IDs") is checking that valid length is shorter than bytes to read. Since iforce_get_id_packet() stores valid length when returning 0, the caller needs to check that valid length is longer than or equals to bytes to read. Reported-by: syzbot Signed-off-by: Tetsuo Handa Fixes: 6ac0aec6b0a6 ("Input: iforce - allow callers supply data buffer when fetching device IDs") Link: https://lore.kernel.org/r/531fb432-7396-ad37-ecba-3e42e7f56d5c@I-love.SAKURA.ne.jp Cc: stable@vger.kernel.org Signed-off-by: Dmitry Torokhov --- drivers/input/joystick/iforce/iforce-main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/input/joystick/iforce/iforce-main.c b/drivers/input/joystick/iforce/iforce-main.c index b86de1312512b..84b87526b7ba3 100644 --- a/drivers/input/joystick/iforce/iforce-main.c +++ b/drivers/input/joystick/iforce/iforce-main.c @@ -273,22 +273,22 @@ int iforce_init_device(struct device *parent, u16 bustype, * Get device info. */ - if (!iforce_get_id_packet(iforce, 'M', buf, &len) || len < 3) + if (!iforce_get_id_packet(iforce, 'M', buf, &len) && len >= 3) input_dev->id.vendor = get_unaligned_le16(buf + 1); else dev_warn(&iforce->dev->dev, "Device does not respond to id packet M\n"); - if (!iforce_get_id_packet(iforce, 'P', buf, &len) || len < 3) + if (!iforce_get_id_packet(iforce, 'P', buf, &len) && len >= 3) input_dev->id.product = get_unaligned_le16(buf + 1); else dev_warn(&iforce->dev->dev, "Device does not respond to id packet P\n"); - if (!iforce_get_id_packet(iforce, 'B', buf, &len) || len < 3) + if (!iforce_get_id_packet(iforce, 'B', buf, &len) && len >= 3) iforce->device_memory.end = get_unaligned_le16(buf + 1); else dev_warn(&iforce->dev->dev, "Device does not respond to id packet B\n"); - if (!iforce_get_id_packet(iforce, 'N', buf, &len) || len < 2) + if (!iforce_get_id_packet(iforce, 'N', buf, &len) && len >= 2) ff_effects = buf[1]; else dev_warn(&iforce->dev->dev, "Device does not respond to id packet N\n"); -- GitLab From 8e9ada1d0e72b4737df400fe1bba48dc42a68df7 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 7 Nov 2022 10:30:25 -0800 Subject: [PATCH 1151/1927] Input: soc_button_array - add use_low_level_irq module parameter It seems that the Windows drivers for the ACPI0011 soc_button_array device use low level triggered IRQs rather then using edge triggering. Some ACPI tables depend on this, directly poking the GPIO controller's registers to clear the trigger type when closing a laptop's/2-in-1's lid and re-instating the trigger when opening the lid again. Linux sets the edge/level on which to trigger to both low+high since it is using edge type IRQs, the ACPI tables then ends up also setting the bit for level IRQs and since both low and high level have been selected by Linux we get an IRQ storm leading to soft lockups. As a workaround for this the soc_button_array already contains a DMI quirk table with device models known to have this issue. Add a module parameter for this so that users can easily test if their device is affected too and so that they can use the module parameter as a workaround. Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20221106215320.67109-1-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/misc/soc_button_array.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/input/misc/soc_button_array.c b/drivers/input/misc/soc_button_array.c index 480476121c010..50497dd05027c 100644 --- a/drivers/input/misc/soc_button_array.c +++ b/drivers/input/misc/soc_button_array.c @@ -18,6 +18,10 @@ #include #include +static bool use_low_level_irq; +module_param(use_low_level_irq, bool, 0444); +MODULE_PARM_DESC(use_low_level_irq, "Use low-level triggered IRQ instead of edge triggered"); + struct soc_button_info { const char *name; int acpi_index; @@ -164,7 +168,8 @@ soc_button_device_create(struct platform_device *pdev, } /* See dmi_use_low_level_irq[] comment */ - if (!autorepeat && dmi_check_system(dmi_use_low_level_irq)) { + if (!autorepeat && (use_low_level_irq || + dmi_check_system(dmi_use_low_level_irq))) { irq_set_irq_type(irq, IRQ_TYPE_LEVEL_LOW); gpio_keys[n_buttons].irq = irq; gpio_keys[n_buttons].gpio = -ENOENT; -- GitLab From e13757f52496444b994a7ac67b6e517a15d89bbc Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 7 Nov 2022 10:30:41 -0800 Subject: [PATCH 1152/1927] Input: soc_button_array - add Acer Switch V 10 to dmi_use_low_level_irq[] Like on the Acer Switch 10 SW5-012, the Acer Switch V 10 SW5-017's _LID method messes with home- and power-button GPIO IRQ settings, causing an IRQ storm. Add a quirk entry for the Acer Switch V 10 to the dmi_use_low_level_irq[] DMI quirk list, to use low-level IRQs on this model, fixing the IRQ storm. Signed-off-by: Hans de Goede Link: https://lore.kernel.org/r/20221106215320.67109-2-hdegoede@redhat.com Signed-off-by: Dmitry Torokhov --- drivers/input/misc/soc_button_array.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/input/misc/soc_button_array.c b/drivers/input/misc/soc_button_array.c index 50497dd05027c..09489380afda7 100644 --- a/drivers/input/misc/soc_button_array.c +++ b/drivers/input/misc/soc_button_array.c @@ -77,6 +77,13 @@ static const struct dmi_system_id dmi_use_low_level_irq[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Aspire SW5-012"), }, }, + { + /* Acer Switch V 10 SW5-017, same issue as Acer Switch 10 SW5-012. */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_MATCH(DMI_PRODUCT_NAME, "SW5-017"), + }, + }, { /* * Acer One S1003. _LID method messes with power-button GPIO -- GitLab From 62fa3ce05d5d73c5eccc40b2db493f55fecfc446 Mon Sep 17 00:00:00 2001 From: Brian King Date: Wed, 26 Oct 2022 13:13:56 -0500 Subject: [PATCH 1153/1927] scsi: ibmvfc: Avoid path failures during live migration Fix an issue reported when performing a live migration when multipath is configured with a short fast fail timeout of 5 seconds and also to have no_path_retry set to fail. In this scenario, all paths would go into the devloss state while the ibmvfc driver went through discovery to log back in. On a loaded system, the discovery might take longer than 5 seconds, which was resulting in all paths being marked failed, which then resulted in a read only filesystem. This patch changes the migration code in ibmvfc to avoid deleting rports at all in this scenario, so we avoid losing all paths. Signed-off-by: Brian King Link: https://lore.kernel.org/r/20221026181356.148517-1-brking@linux.vnet.ibm.com Signed-off-by: Martin K. Petersen --- drivers/scsi/ibmvscsi/ibmvfc.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/scsi/ibmvscsi/ibmvfc.c b/drivers/scsi/ibmvscsi/ibmvfc.c index 00684e11976be..1a0c0b7289d26 100644 --- a/drivers/scsi/ibmvscsi/ibmvfc.c +++ b/drivers/scsi/ibmvscsi/ibmvfc.c @@ -708,8 +708,13 @@ static void ibmvfc_init_host(struct ibmvfc_host *vhost) memset(vhost->async_crq.msgs.async, 0, PAGE_SIZE); vhost->async_crq.cur = 0; - list_for_each_entry(tgt, &vhost->targets, queue) - ibmvfc_del_tgt(tgt); + list_for_each_entry(tgt, &vhost->targets, queue) { + if (vhost->client_migrated) + tgt->need_login = 1; + else + ibmvfc_del_tgt(tgt); + } + scsi_block_requests(vhost->host); ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_INIT); vhost->job_step = ibmvfc_npiv_login; @@ -3235,9 +3240,12 @@ static void ibmvfc_handle_crq(struct ibmvfc_crq *crq, struct ibmvfc_host *vhost, /* We need to re-setup the interpartition connection */ dev_info(vhost->dev, "Partition migrated, Re-enabling adapter\n"); vhost->client_migrated = 1; + + scsi_block_requests(vhost->host); ibmvfc_purge_requests(vhost, DID_REQUEUE); - ibmvfc_link_down(vhost, IBMVFC_LINK_DOWN); + ibmvfc_set_host_state(vhost, IBMVFC_LINK_DOWN); ibmvfc_set_host_action(vhost, IBMVFC_HOST_ACTION_REENABLE); + wake_up(&vhost->work_wait_q); } else if (crq->format == IBMVFC_PARTNER_FAILED || crq->format == IBMVFC_PARTNER_DEREGISTER) { dev_err(vhost->dev, "Host partner adapter deregistered or failed (rc=%d)\n", crq->format); ibmvfc_purge_requests(vhost, DID_ERROR); -- GitLab From 5fd974d164b4240652259e7058e2c72a68662cb0 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 2 Nov 2022 10:10:43 -0700 Subject: [PATCH 1154/1927] drm/i915/mtl: add initial definitions for GSC CS Starting on MTL, the GSC is no longer managed with direct MMIO access, but we instead have a dedicated command streamer for it. As a first step for adding support for this CS, add the required definitions. Note that, although it is now a CS, the GSC retains its old class:instance value (OTHER_CLASS instance 6) Bspec: 65308, 45605 Signed-off-by: Daniele Ceraolo Spurio Cc: Matt Roper Reviewed-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20221102171047.2787951-2-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/i915/gt/intel_engine_cs.c | 8 ++++++++ drivers/gpu/drm/i915/gt/intel_engine_types.h | 1 + drivers/gpu/drm/i915/gt/intel_engine_user.c | 1 + drivers/gpu/drm/i915/i915_reg.h | 1 + 4 files changed, 11 insertions(+) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 3b7d750ad0541..e0fbfac039797 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -244,6 +244,13 @@ static const struct engine_info intel_engines[] = { { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE } } }, + [GSC0] = { + .class = OTHER_CLASS, + .instance = OTHER_GSC_INSTANCE, + .mmio_bases = { + { .graphics_ver = 12, .base = MTL_GSC_RING_BASE } + } + }, }; /** @@ -324,6 +331,7 @@ u32 intel_engine_context_size(struct intel_gt *gt, u8 class) case VIDEO_DECODE_CLASS: case VIDEO_ENHANCEMENT_CLASS: case COPY_ENGINE_CLASS: + case OTHER_CLASS: if (GRAPHICS_VER(gt->i915) < 8) return 0; return GEN8_LR_CONTEXT_OTHER_SIZE; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 6b5d4ea22b673..4fd54fb8810fb 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -136,6 +136,7 @@ enum intel_engine_id { CCS2, CCS3, #define _CCS(n) (CCS0 + (n)) + GSC0, I915_NUM_ENGINES #define INVALID_ENGINE ((enum intel_engine_id)-1) }; diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c index 46a174f8aa007..79312b7346909 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_user.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c @@ -140,6 +140,7 @@ const char *intel_engine_class_repr(u8 class) [COPY_ENGINE_CLASS] = "bcs", [VIDEO_DECODE_CLASS] = "vcs", [VIDEO_ENHANCEMENT_CLASS] = "vecs", + [OTHER_CLASS] = "other", [COMPUTE_CLASS] = "ccs", }; diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 1c0da50c0dc73..d056c3117ef2b 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -970,6 +970,7 @@ #define GEN11_VEBOX2_RING_BASE 0x1d8000 #define XEHP_VEBOX3_RING_BASE 0x1e8000 #define XEHP_VEBOX4_RING_BASE 0x1f8000 +#define MTL_GSC_RING_BASE 0x11a000 #define GEN12_COMPUTE0_RING_BASE 0x1a000 #define GEN12_COMPUTE1_RING_BASE 0x1c000 #define GEN12_COMPUTE2_RING_BASE 0x1e000 -- GitLab From c9c12ba72e740e3adb5a2287f6d0372fa45721c3 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 2 Nov 2022 10:10:44 -0700 Subject: [PATCH 1155/1927] drm/i915/mtl: pass the GSC CS info to the GuC We need to tell the GuC that the GSC CS is there. Signed-off-by: Daniele Ceraolo Spurio Cc: Matt Roper Reviewed-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20221102171047.2787951-3-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c | 11 +++++------ drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h | 7 +++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c index a419d60166c87..a7f737c4792e2 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c @@ -488,6 +488,11 @@ static void fill_engine_enable_masks(struct intel_gt *gt, info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt)); info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt)); info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt)); + + /* The GSC engine is an instance (6) of OTHER_CLASS */ + if (gt->engine[GSC0]) + info_map_write(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS], + BIT(gt->engine[GSC0]->instance)); } #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32)) @@ -529,9 +534,6 @@ static int guc_prep_golden_context(struct intel_guc *guc) } for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { - if (engine_class == OTHER_CLASS) - continue; - guc_class = engine_class_to_guc_class(engine_class); if (!info_map_read(&info_map, engine_enabled_masks[guc_class])) @@ -609,9 +611,6 @@ static void guc_init_golden_context(struct intel_guc *guc) addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset; for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) { - if (engine_class == OTHER_CLASS) - continue; - guc_class = engine_class_to_guc_class(engine_class); if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class])) continue; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h index 968ebd79dce70..4ae5fc2f60023 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h @@ -47,7 +47,8 @@ #define GUC_VIDEOENHANCE_CLASS 2 #define GUC_BLITTER_CLASS 3 #define GUC_COMPUTE_CLASS 4 -#define GUC_LAST_ENGINE_CLASS GUC_COMPUTE_CLASS +#define GUC_GSC_OTHER_CLASS 5 +#define GUC_LAST_ENGINE_CLASS GUC_GSC_OTHER_CLASS #define GUC_MAX_ENGINE_CLASSES 16 #define GUC_MAX_INSTANCES_PER_CLASS 32 @@ -169,6 +170,7 @@ static u8 engine_class_guc_class_map[] = { [COPY_ENGINE_CLASS] = GUC_BLITTER_CLASS, [VIDEO_DECODE_CLASS] = GUC_VIDEO_CLASS, [VIDEO_ENHANCEMENT_CLASS] = GUC_VIDEOENHANCE_CLASS, + [OTHER_CLASS] = GUC_GSC_OTHER_CLASS, [COMPUTE_CLASS] = GUC_COMPUTE_CLASS, }; @@ -178,12 +180,13 @@ static u8 guc_class_engine_class_map[] = { [GUC_VIDEO_CLASS] = VIDEO_DECODE_CLASS, [GUC_VIDEOENHANCE_CLASS] = VIDEO_ENHANCEMENT_CLASS, [GUC_COMPUTE_CLASS] = COMPUTE_CLASS, + [GUC_GSC_OTHER_CLASS] = OTHER_CLASS, }; static inline u8 engine_class_to_guc_class(u8 class) { BUILD_BUG_ON(ARRAY_SIZE(engine_class_guc_class_map) != MAX_ENGINE_CLASS + 1); - GEM_BUG_ON(class > MAX_ENGINE_CLASS || class == OTHER_CLASS); + GEM_BUG_ON(class > MAX_ENGINE_CLASS); return engine_class_guc_class_map[class]; } -- GitLab From c07ee636901d1496caf81594f90fc68e9a9c7ba5 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 2 Nov 2022 10:10:45 -0700 Subject: [PATCH 1156/1927] drm/i915/mtl: add GSC CS interrupt support The GSC CS re-uses the same interrupt bits that the GSC used in older platforms. This means that we can now have an engine interrupt coming out of OTHER_CLASS, so we need to handle that appropriately. v2: clean up the if statement for the engine irq (Tvrtko) Signed-off-by: Daniele Ceraolo Spurio Cc: Matt Roper Cc: Tvrtko Ursulin Reviewed-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20221102171047.2787951-4-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/i915/gt/intel_gt_irq.c | 75 ++++++++++++++------------ 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index f26882fdc24c9..b197f0e9794f1 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -81,35 +81,27 @@ gen11_other_irq_handler(struct intel_gt *gt, const u8 instance, instance, iir); } -static void -gen11_engine_irq_handler(struct intel_gt *gt, const u8 class, - const u8 instance, const u16 iir) +static struct intel_gt *pick_gt(struct intel_gt *gt, u8 class, u8 instance) { - struct intel_engine_cs *engine; - - /* - * Platforms with standalone media have their media engines in another - * GT. - */ - if (MEDIA_VER(gt->i915) >= 13 && - (class == VIDEO_DECODE_CLASS || class == VIDEO_ENHANCEMENT_CLASS)) { - if (!gt->i915->media_gt) - goto err; + struct intel_gt *media_gt = gt->i915->media_gt; - gt = gt->i915->media_gt; + /* we expect the non-media gt to be passed in */ + GEM_BUG_ON(gt == media_gt); + + if (!media_gt) + return gt; + + switch (class) { + case VIDEO_DECODE_CLASS: + case VIDEO_ENHANCEMENT_CLASS: + return media_gt; + case OTHER_CLASS: + if (instance == OTHER_GSC_INSTANCE && HAS_ENGINE(media_gt, GSC0)) + return media_gt; + fallthrough; + default: + return gt; } - - if (instance <= MAX_ENGINE_INSTANCE) - engine = gt->engine_class[class][instance]; - else - engine = NULL; - - if (likely(engine)) - return intel_engine_cs_irq(engine, iir); - -err: - WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n", - class, instance); } static void @@ -122,8 +114,17 @@ gen11_gt_identity_handler(struct intel_gt *gt, const u32 identity) if (unlikely(!intr)) return; - if (class <= COPY_ENGINE_CLASS || class == COMPUTE_CLASS) - return gen11_engine_irq_handler(gt, class, instance, intr); + /* + * Platforms with standalone media have the media and GSC engines in + * another GT. + */ + gt = pick_gt(gt, class, instance); + + if (class <= MAX_ENGINE_CLASS && instance <= MAX_ENGINE_INSTANCE) { + struct intel_engine_cs *engine = gt->engine_class[class][instance]; + if (engine) + return intel_engine_cs_irq(engine, intr); + } if (class == OTHER_CLASS) return gen11_other_irq_handler(gt, instance, intr); @@ -206,7 +207,7 @@ void gen11_gt_irq_reset(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, 0); if (CCS_MASK(gt)) intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, 0); - if (HAS_HECI_GSC(gt->i915)) + if (HAS_HECI_GSC(gt->i915) || HAS_ENGINE(gt, GSC0)) intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, 0); /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ @@ -233,7 +234,7 @@ void gen11_gt_irq_reset(struct intel_gt *gt) intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~0); if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3)) intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~0); - if (HAS_HECI_GSC(gt->i915)) + if (HAS_HECI_GSC(gt->i915) || HAS_ENGINE(gt, GSC0)) intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~0); intel_uncore_write(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE, 0); @@ -249,7 +250,7 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) { struct intel_uncore *uncore = gt->uncore; u32 irqs = GT_RENDER_USER_INTERRUPT; - const u32 gsc_mask = GSC_IRQ_INTF(0) | GSC_IRQ_INTF(1); + u32 gsc_mask = 0; u32 dmask; u32 smask; @@ -261,6 +262,11 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) dmask = irqs << 16 | irqs; smask = irqs << 16; + if (HAS_ENGINE(gt, GSC0)) + gsc_mask = irqs; + else if (HAS_HECI_GSC(gt->i915)) + gsc_mask = GSC_IRQ_INTF(0) | GSC_IRQ_INTF(1); + BUILD_BUG_ON(irqs & 0xffff0000); /* Enable RCS, BCS, VCS and VECS class interrupts. */ @@ -268,9 +274,8 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) intel_uncore_write(uncore, GEN11_VCS_VECS_INTR_ENABLE, dmask); if (CCS_MASK(gt)) intel_uncore_write(uncore, GEN12_CCS_RSVD_INTR_ENABLE, smask); - if (HAS_HECI_GSC(gt->i915)) - intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, - gsc_mask); + if (gsc_mask) + intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_ENABLE, gsc_mask); /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ intel_uncore_write(uncore, GEN11_RCS0_RSVD_INTR_MASK, ~smask); @@ -296,7 +301,7 @@ void gen11_gt_irq_postinstall(struct intel_gt *gt) intel_uncore_write(uncore, GEN12_CCS0_CCS1_INTR_MASK, ~dmask); if (HAS_ENGINE(gt, CCS2) || HAS_ENGINE(gt, CCS3)) intel_uncore_write(uncore, GEN12_CCS2_CCS3_INTR_MASK, ~dmask); - if (HAS_HECI_GSC(gt->i915)) + if (gsc_mask) intel_uncore_write(uncore, GEN11_GUNIT_CSME_INTR_MASK, ~gsc_mask); /* -- GitLab From ef8281abb149c1ed66fe80e28faca4e350ff4c60 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 2 Nov 2022 10:10:46 -0700 Subject: [PATCH 1157/1927] drm/i915/mtl: add GSC CS reset support The GSC CS has its own dedicated bit in the GDRST register. Bspec: 52549 Signed-off-by: Daniele Ceraolo Spurio Cc: Matt Roper Reviewed-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20221102171047.2787951-5-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/i915/gt/intel_engine_cs.c | 1 + drivers/gpu/drm/i915/gt/intel_gt_regs.h | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index e0fbfac039797..f63829abf66c9 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -423,6 +423,7 @@ static u32 get_reset_domain(u8 ver, enum intel_engine_id id) [CCS1] = GEN11_GRDOM_RENDER, [CCS2] = GEN11_GRDOM_RENDER, [CCS3] = GEN11_GRDOM_RENDER, + [GSC0] = GEN12_GRDOM_GSC, }; GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) || !engine_reset_domains[id]); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h index 70177d3f2e945..8aa06b0327e51 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h @@ -643,6 +643,7 @@ #define XEHPC_GRDOM_BLT3 REG_BIT(26) #define XEHPC_GRDOM_BLT2 REG_BIT(25) #define XEHPC_GRDOM_BLT1 REG_BIT(24) +#define GEN12_GRDOM_GSC REG_BIT(21) #define GEN11_GRDOM_SFC3 REG_BIT(20) #define GEN11_GRDOM_SFC2 REG_BIT(19) #define GEN11_GRDOM_SFC1 REG_BIT(18) -- GitLab From 194babe26bdcf6b9dec98907dc13f319baf01e43 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 2 Nov 2022 10:10:47 -0700 Subject: [PATCH 1158/1927] drm/i915/mtl: don't expose GSC command streamer to the user There is no userspace user for this CS yet, we only need it for internal kernel ops (e.g. HuC, PXP), so don't expose it. v2: even if it's not exposed, rename the engine so it is easier to identify in the debug logs (Matt) Signed-off-by: Daniele Ceraolo Spurio Cc: Matt Roper Reviewed-by: Matt Roper Link: https://patchwork.freedesktop.org/patch/msgid/20221102171047.2787951-6-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/i915/gt/intel_engine_user.c | 27 ++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c index 79312b7346909..cd4f1b126f753 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_user.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c @@ -191,6 +191,15 @@ static void add_legacy_ring(struct legacy_ring *ring, ring->instance++; } +static void engine_rename(struct intel_engine_cs *engine, const char *name, u16 instance) +{ + char old[sizeof(engine->name)]; + + memcpy(old, engine->name, sizeof(engine->name)); + scnprintf(engine->name, sizeof(engine->name), "%s%u", name, instance); + drm_dbg(&engine->i915->drm, "renamed %s to %s\n", old, engine->name); +} + void intel_engines_driver_register(struct drm_i915_private *i915) { struct legacy_ring ring = {}; @@ -206,11 +215,19 @@ void intel_engines_driver_register(struct drm_i915_private *i915) struct intel_engine_cs *engine = container_of((struct rb_node *)it, typeof(*engine), uabi_node); - char old[sizeof(engine->name)]; if (intel_gt_has_unrecoverable_error(engine->gt)) continue; /* ignore incomplete engines */ + /* + * We don't want to expose the GSC engine to the users, but we + * still rename it so it is easier to identify in the debug logs + */ + if (engine->id == GSC0) { + engine_rename(engine, "gsc", 0); + continue; + } + GEM_BUG_ON(engine->class >= ARRAY_SIZE(uabi_classes)); engine->uabi_class = uabi_classes[engine->class]; @@ -220,11 +237,9 @@ void intel_engines_driver_register(struct drm_i915_private *i915) i915->engine_uabi_class_count[engine->uabi_class]++; /* Replace the internal name with the final user facing name */ - memcpy(old, engine->name, sizeof(engine->name)); - scnprintf(engine->name, sizeof(engine->name), "%s%u", - intel_engine_class_repr(engine->class), - engine->uabi_instance); - DRM_DEBUG_DRIVER("renamed %s to %s\n", old, engine->name); + engine_rename(engine, + intel_engine_class_repr(engine->class), + engine->uabi_instance); rb_link_node(&engine->uabi_node, prev, p); rb_insert_color(&engine->uabi_node, &i915->uabi_engines); -- GitLab From 8b693ea26c209757a4c96cf4463cc597a3625e19 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 2 Nov 2022 14:43:10 -0700 Subject: [PATCH 1159/1927] drm/i915/guc: don't hardcode BCS0 in guc_hang selftest On MTL there are no BCS engines on the media GT, so we can't always use BCS0 in the test. There is no actual reason to use a BCS engine over an engine of a different class, so switch to using any available engine. Signed-off-by: Daniele Ceraolo Spurio Cc: John Harrison Reviewed-by: John Harrison Acked-by: Nirmoy Das Link: https://patchwork.freedesktop.org/patch/msgid/20221102214310.2829310-1-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c index 01f8cd3c31340..d91b58f704039 100644 --- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c +++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c @@ -35,11 +35,14 @@ static int intel_hang_guc(void *arg) struct i915_request *rq; intel_wakeref_t wakeref; struct i915_gpu_error *global = >->i915->gpu_error; - struct intel_engine_cs *engine; + struct intel_engine_cs *engine = intel_selftest_find_any_engine(gt); unsigned int reset_count; u32 guc_status; u32 old_beat; + if (!engine) + return 0; + ctx = kernel_context(gt->i915, NULL); if (IS_ERR(ctx)) { drm_err(>->i915->drm, "Failed get kernel context: %ld\n", PTR_ERR(ctx)); @@ -48,14 +51,13 @@ static int intel_hang_guc(void *arg) wakeref = intel_runtime_pm_get(gt->uncore->rpm); - ce = intel_context_create(gt->engine[BCS0]); + ce = intel_context_create(engine); if (IS_ERR(ce)) { ret = PTR_ERR(ce); drm_err(>->i915->drm, "Failed to create spinner request: %d\n", ret); goto err; } - engine = ce->engine; reset_count = i915_reset_count(global); old_beat = engine->props.heartbeat_interval_ms; -- GitLab From 59f2f4b8a757412fce372f6d0767bdb55da127a8 Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Mon, 7 Nov 2022 20:11:42 +0000 Subject: [PATCH 1160/1927] fs/userfaultfd: Fix maple tree iterator in userfaultfd_unregister() When iterating the VMAs, the maple state needs to be invalidated if the tree is modified by a split or merge to ensure the maple tree node contained in the maple state is still valid. These invalidations were missed, so add them to the paths which alter the tree. Reported-by: syzbot+0d2014e4da2ccced5b41@syzkaller.appspotmail.com Fixes: 69dbe6daf104 (userfaultfd: use maple tree iterator to iterate VMAs) Signed-off-by: Liam R. Howlett Signed-off-by: Linus Torvalds --- fs/userfaultfd.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 07c81ab3fd4dd..98ac37e34e3d4 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -1630,17 +1630,20 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx, NULL_VM_UFFD_CTX, anon_vma_name(vma)); if (prev) { vma = prev; + mas_pause(&mas); goto next; } if (vma->vm_start < start) { ret = split_vma(mm, vma, start, 1); if (ret) break; + mas_pause(&mas); } if (vma->vm_end > end) { ret = split_vma(mm, vma, end, 0); if (ret) break; + mas_pause(&mas); } next: /* -- GitLab From a679120edfcf3d63f066f53afd425d51b480e533 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Mon, 7 Nov 2022 10:07:11 -0700 Subject: [PATCH 1161/1927] bpf: Add explicit cast to 'void *' for __BPF_DISPATCHER_UPDATE() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When building with clang: kernel/bpf/dispatcher.c:126:33: error: pointer type mismatch ('void *' and 'unsigned int (*)(const void *, const struct bpf_insn *, bpf_func_t)' (aka 'unsigned int (*)(const void *, const struct bpf_insn *, unsigned int (*)(const void *, const struct bpf_insn *))')) [-Werror,-Wpointer-type-mismatch] __BPF_DISPATCHER_UPDATE(d, new ?: &bpf_dispatcher_nop_func); ~~~ ^ ~~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/bpf.h:1045:54: note: expanded from macro '__BPF_DISPATCHER_UPDATE' __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new)) ^~~~ 1 error generated. The warning is pointing out that the type of new ('void *') and &bpf_dispatcher_nop_func are not compatible, which could have side effects coming out of a conditional operator due to promotion rules. Add the explicit cast to 'void *' to make it clear that this is expected, as __BPF_DISPATCHER_UPDATE() expands to a call to __static_call_update(), which expects a 'void *' as its final argument. Fixes: c86df29d11df ("bpf: Convert BPF_DISPATCHER to use static_call() (not ftrace)") Link: https://github.com/ClangBuiltLinux/linux/issues/1755 Reported-by: kernel test robot Reported-by: "kernelci.org bot" Signed-off-by: Nathan Chancellor Acked-by: Björn Töpel Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20221107170711.42409-1-nathan@kernel.org Signed-off-by: Martin KaFai Lau --- kernel/bpf/dispatcher.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c index 7dfb8d0d5202e..c19719f48ce06 100644 --- a/kernel/bpf/dispatcher.c +++ b/kernel/bpf/dispatcher.c @@ -123,7 +123,7 @@ static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) return; } - __BPF_DISPATCHER_UPDATE(d, new ?: &bpf_dispatcher_nop_func); + __BPF_DISPATCHER_UPDATE(d, new ?: (void *)&bpf_dispatcher_nop_func); if (new) d->image_off = noff; -- GitLab From ed4314f7729714d788698ade4f9905ee5378ebc0 Mon Sep 17 00:00:00 2001 From: Rasmus Villemoes Date: Fri, 4 Nov 2022 09:30:04 +0100 Subject: [PATCH 1162/1927] net: stmmac: dwmac-meson8b: fix meson8b_devm_clk_prepare_enable() There are two problems with meson8b_devm_clk_prepare_enable(), introduced in commit a54dc4a49045 ("net: stmmac: dwmac-meson8b: Make the clock enabling code re-usable"): - It doesn't pass the clk argument, but instead always the rgmii_tx_clk of the device. - It silently ignores the return value of devm_add_action_or_reset(). The former didn't become an actual bug until another user showed up in the next commit 9308c47640d5 ("net: stmmac: dwmac-meson8b: add support for the RX delay configuration"). The latter means the callers could end up with the clock not actually prepared/enabled. Fixes: a54dc4a49045 ("net: stmmac: dwmac-meson8b: Make the clock enabling code re-usable") Signed-off-by: Rasmus Villemoes Reviewed-by: Martin Blumenstingl Link: https://lore.kernel.org/r/20221104083004.2212520-1-linux@rasmusvillemoes.dk Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c index c7a6588d9398b..e8b507f88fbce 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-meson8b.c @@ -272,11 +272,9 @@ static int meson8b_devm_clk_prepare_enable(struct meson8b_dwmac *dwmac, if (ret) return ret; - devm_add_action_or_reset(dwmac->dev, - (void(*)(void *))clk_disable_unprepare, - dwmac->rgmii_tx_clk); - - return 0; + return devm_add_action_or_reset(dwmac->dev, + (void(*)(void *))clk_disable_unprepare, + clk); } static int meson8b_init_rgmii_delays(struct meson8b_dwmac *dwmac) -- GitLab From 5d7bebf2dfb0dc97aac1fbace0910e557ecdb16f Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Mon, 7 Nov 2022 20:48:28 +0800 Subject: [PATCH 1163/1927] scsi: scsi_transport_sas: Fix error handling in sas_phy_add() If transport_add_device() fails in sas_phy_add(), the kernel will crash trying to delete the device in transport_remove_device() called from sas_remove_host(). Unable to handle kernel NULL pointer dereference at virtual address 0000000000000108 CPU: 61 PID: 42829 Comm: rmmod Kdump: loaded Tainted: G W 6.1.0-rc1+ #173 pstate: 60000005 (nZCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : device_del+0x54/0x3d0 lr : device_del+0x37c/0x3d0 Call trace: device_del+0x54/0x3d0 attribute_container_class_device_del+0x28/0x38 transport_remove_classdev+0x6c/0x80 attribute_container_device_trigger+0x108/0x110 transport_remove_device+0x28/0x38 sas_phy_delete+0x30/0x60 [scsi_transport_sas] do_sas_phy_delete+0x6c/0x80 [scsi_transport_sas] device_for_each_child+0x68/0xb0 sas_remove_children+0x40/0x50 [scsi_transport_sas] sas_remove_host+0x20/0x38 [scsi_transport_sas] hisi_sas_remove+0x40/0x68 [hisi_sas_main] hisi_sas_v2_remove+0x20/0x30 [hisi_sas_v2_hw] platform_remove+0x2c/0x60 Fix this by checking and handling return value of transport_add_device() in sas_phy_add(). Fixes: c7ebbbce366c ("[SCSI] SAS transport class") Suggested-by: John Garry Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221107124828.115557-1-yangyingliang@huawei.com Reviewed-by: John Garry Reviewed-by: Jason Yan Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_transport_sas.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c index 2f88c61216eea..74b99f2b0b74a 100644 --- a/drivers/scsi/scsi_transport_sas.c +++ b/drivers/scsi/scsi_transport_sas.c @@ -722,12 +722,17 @@ int sas_phy_add(struct sas_phy *phy) int error; error = device_add(&phy->dev); - if (!error) { - transport_add_device(&phy->dev); - transport_configure_device(&phy->dev); + if (error) + return error; + + error = transport_add_device(&phy->dev); + if (error) { + device_del(&phy->dev); + return error; } + transport_configure_device(&phy->dev); - return error; + return 0; } EXPORT_SYMBOL(sas_phy_add); -- GitLab From ec683f02a150b9c4428f08accd387c8c216ea0e5 Mon Sep 17 00:00:00 2001 From: Miquel Raynal Date: Fri, 4 Nov 2022 17:21:47 +0100 Subject: [PATCH 1164/1927] dt-bindings: net: tsnep: Fix typo on generic nvmem property While working on the nvmem description I figured out this file had the "nvmem-cell-names" property name misspelled. Fix the typo, as "nvmem-cells-names" has never existed. Fixes: 603094b2cdb7 ("dt-bindings: net: Add tsnep Ethernet controller") Signed-off-by: Miquel Raynal Reviewed-by: Gerhard Engleder Acked-by: Rob Herring Link: https://lore.kernel.org/r/20221104162147.1288230-1-miquel.raynal@bootlin.com Signed-off-by: Jakub Kicinski --- Documentation/devicetree/bindings/net/engleder,tsnep.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml index 5bd964a46a9d0..a6921e805e37e 100644 --- a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml +++ b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml @@ -47,7 +47,7 @@ properties: nvmem-cells: true - nvmem-cells-names: true + nvmem-cell-names: true phy-connection-type: enum: -- GitLab From 02f5999e652952d69c341a03d4313310703fd7f1 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Sat, 5 Nov 2022 14:34:42 +0800 Subject: [PATCH 1165/1927] octeontx2-pf: fix build error when CONFIG_OCTEONTX2_PF=y If CONFIG_MACSEC=m and CONFIG_OCTEONTX2_PF=y, it leads a build error: ld: drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.o: in function `otx2_pfaf_mbox_up_handler': otx2_pf.c:(.text+0x181c): undefined reference to `cn10k_handle_mcs_event' ld: drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.o: in function `otx2_probe': otx2_pf.c:(.text+0x437e): undefined reference to `cn10k_mcs_init' ld: drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.o: in function `otx2_remove': otx2_pf.c:(.text+0x5031): undefined reference to `cn10k_mcs_free' ld: drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.o: in function `otx2_mbox_up_handler_mcs_intr_notify': otx2_pf.c:(.text+0x5f11): undefined reference to `cn10k_handle_mcs_event' Make CONFIG_OCTEONTX2_PF depends on CONFIG_MACSEC to fix it. Because it has empty stub functions of cn10k, CONFIG_OCTEONTX2_PF can be enabled if CONFIG_MACSEC is disabled Fixes: c54ffc73601c ("octeontx2-pf: mcs: Introduce MACSEC hardware offloading") Reported-by: kernel test robot Signed-off-by: Yang Yingliang Link: https://lore.kernel.org/r/20221105063442.2013981-1-yangyingliang@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/marvell/octeontx2/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig index 993ac180a5db8..6b4f640163f71 100644 --- a/drivers/net/ethernet/marvell/octeontx2/Kconfig +++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig @@ -32,6 +32,7 @@ config OCTEONTX2_PF tristate "Marvell OcteonTX2 NIC Physical Function driver" select OCTEONTX2_MBOX select NET_DEVLINK + depends on MACSEC || !MACSEC depends on (64BIT && COMPILE_TEST) || ARM64 select DIMLIB depends on PCI -- GitLab From 8d820bc9d12b8beebca836cceaf2bbe68216c2f8 Mon Sep 17 00:00:00 2001 From: YueHaibing Date: Sat, 5 Nov 2022 17:02:45 +0800 Subject: [PATCH 1166/1927] net: broadcom: Fix BCMGENET Kconfig While BCMGENET select BROADCOM_PHY as y, but PTP_1588_CLOCK_OPTIONAL is m, kconfig warning and build errors: WARNING: unmet direct dependencies detected for BROADCOM_PHY Depends on [m]: NETDEVICES [=y] && PHYLIB [=y] && PTP_1588_CLOCK_OPTIONAL [=m] Selected by [y]: - BCMGENET [=y] && NETDEVICES [=y] && ETHERNET [=y] && NET_VENDOR_BROADCOM [=y] && HAS_IOMEM [=y] && ARCH_BCM2835 [=y] drivers/net/phy/broadcom.o: In function `bcm54xx_suspend': broadcom.c:(.text+0x6ac): undefined reference to `bcm_ptp_stop' drivers/net/phy/broadcom.o: In function `bcm54xx_phy_probe': broadcom.c:(.text+0x784): undefined reference to `bcm_ptp_probe' drivers/net/phy/broadcom.o: In function `bcm54xx_config_init': broadcom.c:(.text+0xd4c): undefined reference to `bcm_ptp_config_init' Fixes: 99addbe31f55 ("net: broadcom: Select BROADCOM_PHY for BCMGENET") Signed-off-by: YueHaibing Acked-by: Florian Fainelli Link: https://lore.kernel.org/r/20221105090245.8508-1-yuehaibing@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/broadcom/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index f4e1ca68d8310..55dfdb34e37b8 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -77,7 +77,7 @@ config BCMGENET select BCM7XXX_PHY select MDIO_BCM_UNIMAC select DIMLIB - select BROADCOM_PHY if ARCH_BCM2835 + select BROADCOM_PHY if (ARCH_BCM2835 && PTP_1588_CLOCK_OPTIONAL) help This driver supports the built-in Ethernet MACs found in the Broadcom BCM7xxx Set Top Box family chipset. -- GitLab From ecb8c2580d37dbb641451049376d80c8afaa387f Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Wed, 2 Nov 2022 12:32:48 -0700 Subject: [PATCH 1167/1927] scsi: scsi_debug: Make the READ CAPACITY response compliant with ZBC From ZBC-1: - RC BASIS = 0: The RETURNED LOGICAL BLOCK ADDRESS field indicates the highest LBA of a contiguous range of zones that are not sequential write required zones starting with the first zone. - RC BASIS = 1: The RETURNED LOGICAL BLOCK ADDRESS field indicates the LBA of the last logical block on the logical unit. The current scsi_debug READ CAPACITY response does not comply with the above if there are one or more sequential write required zones. SCSI initiators need a way to retrieve the largest valid LBA from SCSI devices. Reporting the largest valid LBA if there are one or more sequential zones requires to set the RC BASIS field in the READ CAPACITY response to one. Hence this patch. Cc: Douglas Gilbert Cc: Damien Le Moal Suggested-by: Damien Le Moal Signed-off-by: Bart Van Assche Link: https://lore.kernel.org/r/20221102193248.3177608-1-bvanassche@acm.org Reviewed-by: Damien Le Moal Acked-by: Douglas Gilbert Signed-off-by: Martin K. Petersen --- drivers/scsi/scsi_debug.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 697fc57bc711f..629853662b820 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -1899,6 +1899,13 @@ static int resp_readcap16(struct scsi_cmnd *scp, arr[14] |= 0x40; } + /* + * Since the scsi_debug READ CAPACITY implementation always reports the + * total disk capacity, set RC BASIS = 1 for host-managed ZBC devices. + */ + if (devip->zmodel == BLK_ZONED_HM) + arr[12] |= 1 << 4; + arr[15] = sdebug_lowest_aligned & 0xff; if (have_dif_prot) { -- GitLab From 1c075b192fe41030457cd4a5f7dea730412bca40 Mon Sep 17 00:00:00 2001 From: Xin Long Date: Fri, 4 Nov 2022 16:48:53 -0400 Subject: [PATCH 1168/1927] tipc: fix the msg->req tlv len check in tipc_nl_compat_name_table_dump_header This is a follow-up for commit 974cb0e3e7c9 ("tipc: fix uninit-value in tipc_nl_compat_name_table_dump") where it should have type casted sizeof(..) to int to work when TLV_GET_DATA_LEN() returns a negative value. syzbot reported a call trace because of it: BUG: KMSAN: uninit-value in ... tipc_nl_compat_name_table_dump+0x841/0xea0 net/tipc/netlink_compat.c:934 __tipc_nl_compat_dumpit+0xab2/0x1320 net/tipc/netlink_compat.c:238 tipc_nl_compat_dumpit+0x991/0xb50 net/tipc/netlink_compat.c:321 tipc_nl_compat_recv+0xb6e/0x1640 net/tipc/netlink_compat.c:1324 genl_family_rcv_msg_doit net/netlink/genetlink.c:731 [inline] genl_family_rcv_msg net/netlink/genetlink.c:775 [inline] genl_rcv_msg+0x103f/0x1260 net/netlink/genetlink.c:792 netlink_rcv_skb+0x3a5/0x6c0 net/netlink/af_netlink.c:2501 genl_rcv+0x3c/0x50 net/netlink/genetlink.c:803 netlink_unicast_kernel net/netlink/af_netlink.c:1319 [inline] netlink_unicast+0xf3b/0x1270 net/netlink/af_netlink.c:1345 netlink_sendmsg+0x1288/0x1440 net/netlink/af_netlink.c:1921 sock_sendmsg_nosec net/socket.c:714 [inline] sock_sendmsg net/socket.c:734 [inline] Reported-by: syzbot+e5dbaaa238680ce206ea@syzkaller.appspotmail.com Fixes: 974cb0e3e7c9 ("tipc: fix uninit-value in tipc_nl_compat_name_table_dump") Signed-off-by: Xin Long Link: https://lore.kernel.org/r/ccd6a7ea801b15aec092c3b532a883b4c5708695.1667594933.git.lucien.xin@gmail.com Signed-off-by: Jakub Kicinski --- net/tipc/netlink_compat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c index fc68733673ba6..dfea27a906f2f 100644 --- a/net/tipc/netlink_compat.c +++ b/net/tipc/netlink_compat.c @@ -880,7 +880,7 @@ static int tipc_nl_compat_name_table_dump_header(struct tipc_nl_compat_msg *msg) }; ntq = (struct tipc_name_table_query *)TLV_DATA(msg->req); - if (TLV_GET_DATA_LEN(msg->req) < sizeof(struct tipc_name_table_query)) + if (TLV_GET_DATA_LEN(msg->req) < (int)sizeof(struct tipc_name_table_query)) return -EINVAL; depth = ntohl(ntq->depth); -- GitLab From 9f0b773210c27a8f5d98ddb2fc4ba60a42a3285f Mon Sep 17 00:00:00 2001 From: Xin Long Date: Fri, 4 Nov 2022 17:45:15 -0400 Subject: [PATCH 1169/1927] sctp: remove the unnecessary sinfo_stream check in sctp_prsctp_prune_unsent Since commit 5bbbbe32a431 ("sctp: introduce stream scheduler foundations"), sctp_stream_outq_migrate() has been called in sctp_stream_init/update to removes those chunks to streams higher than the new max. There is no longer need to do such check in sctp_prsctp_prune_unsent(). Signed-off-by: Xin Long Signed-off-by: Jakub Kicinski --- net/sctp/outqueue.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index e213aaf45d67c..c99fe3dc19bc6 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -384,6 +384,7 @@ static int sctp_prsctp_prune_unsent(struct sctp_association *asoc, { struct sctp_outq *q = &asoc->outqueue; struct sctp_chunk *chk, *temp; + struct sctp_stream_out *sout; q->sched->unsched_all(&asoc->stream); @@ -398,12 +399,9 @@ static int sctp_prsctp_prune_unsent(struct sctp_association *asoc, sctp_sched_dequeue_common(q, chk); asoc->sent_cnt_removable--; asoc->abandoned_unsent[SCTP_PR_INDEX(PRIO)]++; - if (chk->sinfo.sinfo_stream < asoc->stream.outcnt) { - struct sctp_stream_out *streamout = - SCTP_SO(&asoc->stream, chk->sinfo.sinfo_stream); - streamout->ext->abandoned_unsent[SCTP_PR_INDEX(PRIO)]++; - } + sout = SCTP_SO(&asoc->stream, chk->sinfo.sinfo_stream); + sout->ext->abandoned_unsent[SCTP_PR_INDEX(PRIO)]++; msg_len -= chk->skb->truesize + sizeof(struct sctp_chunk); sctp_chunk_free(chk); -- GitLab From 2f201ae14ae0f91dbf1cffea7bb1e29e81d4d108 Mon Sep 17 00:00:00 2001 From: Xin Long Date: Fri, 4 Nov 2022 17:45:16 -0400 Subject: [PATCH 1170/1927] sctp: clear out_curr if all frag chunks of current msg are pruned A crash was reported by Zhen Chen: list_del corruption, ffffa035ddf01c18->next is NULL WARNING: CPU: 1 PID: 250682 at lib/list_debug.c:49 __list_del_entry_valid+0x59/0xe0 RIP: 0010:__list_del_entry_valid+0x59/0xe0 Call Trace: sctp_sched_dequeue_common+0x17/0x70 [sctp] sctp_sched_fcfs_dequeue+0x37/0x50 [sctp] sctp_outq_flush_data+0x85/0x360 [sctp] sctp_outq_uncork+0x77/0xa0 [sctp] sctp_cmd_interpreter.constprop.0+0x164/0x1450 [sctp] sctp_side_effects+0x37/0xe0 [sctp] sctp_do_sm+0xd0/0x230 [sctp] sctp_primitive_SEND+0x2f/0x40 [sctp] sctp_sendmsg_to_asoc+0x3fa/0x5c0 [sctp] sctp_sendmsg+0x3d5/0x440 [sctp] sock_sendmsg+0x5b/0x70 and in sctp_sched_fcfs_dequeue() it dequeued a chunk from stream out_curr outq while this outq was empty. Normally stream->out_curr must be set to NULL once all frag chunks of current msg are dequeued, as we can see in sctp_sched_dequeue_done(). However, in sctp_prsctp_prune_unsent() as it is not a proper dequeue, sctp_sched_dequeue_done() is not called to do this. This patch is to fix it by simply setting out_curr to NULL when the last frag chunk of current msg is dequeued from out_curr stream in sctp_prsctp_prune_unsent(). Fixes: 5bbbbe32a431 ("sctp: introduce stream scheduler foundations") Reported-by: Zhen Chen Tested-by: Caowangbao Signed-off-by: Xin Long Signed-off-by: Jakub Kicinski --- net/sctp/outqueue.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index c99fe3dc19bc6..20831079fb09e 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -403,6 +403,11 @@ static int sctp_prsctp_prune_unsent(struct sctp_association *asoc, sout = SCTP_SO(&asoc->stream, chk->sinfo.sinfo_stream); sout->ext->abandoned_unsent[SCTP_PR_INDEX(PRIO)]++; + /* clear out_curr if all frag chunks are pruned */ + if (asoc->stream.out_curr == sout && + list_is_last(&chk->frag_list, &chk->msg->chunks)) + asoc->stream.out_curr = NULL; + msg_len -= chk->skb->truesize + sizeof(struct sctp_chunk); sctp_chunk_free(chk); if (msg_len <= 0) -- GitLab From b3d726cb8497c6b12106fd617d46eef11763ea86 Mon Sep 17 00:00:00 2001 From: Doug Brown Date: Mon, 5 Sep 2022 17:07:09 -0700 Subject: [PATCH 1171/1927] dmaengine: pxa_dma: use platform_get_irq_optional The first IRQ is required, but IRQs 1 through (nb_phy_chans - 1) are optional, because on some platforms (e.g. PXA168) there is a single IRQ shared between all channels. This change inhibits a flood of "IRQ index # not found" messages at startup. Tested on a PXA168-based device. Fixes: 7723f4c5ecdb ("driver core: platform: Add an error message to platform_get_irq*()") Signed-off-by: Doug Brown Link: https://lore.kernel.org/r/20220906000709.52705-1-doug@schmorgal.com Signed-off-by: Vinod Koul --- drivers/dma/pxa_dma.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/pxa_dma.c b/drivers/dma/pxa_dma.c index e7034f6f3994a..22a392fe6d32b 100644 --- a/drivers/dma/pxa_dma.c +++ b/drivers/dma/pxa_dma.c @@ -1247,14 +1247,14 @@ static int pxad_init_phys(struct platform_device *op, return -ENOMEM; for (i = 0; i < nb_phy_chans; i++) - if (platform_get_irq(op, i) > 0) + if (platform_get_irq_optional(op, i) > 0) nr_irq++; for (i = 0; i < nb_phy_chans; i++) { phy = &pdev->phys[i]; phy->base = pdev->base; phy->idx = i; - irq = platform_get_irq(op, i); + irq = platform_get_irq_optional(op, i); if ((nr_irq > 1) && (irq > 0)) ret = devm_request_irq(&op->dev, irq, pxad_chan_handler, -- GitLab From e8dbd6445dd6b38c4c50410a86f13158486ee99a Mon Sep 17 00:00:00 2001 From: Xiaochen Shen Date: Sat, 1 Oct 2022 04:15:27 +0800 Subject: [PATCH 1172/1927] dmaengine: idxd: Fix max batch size for Intel IAA >From Intel IAA spec [1], Intel IAA does not support batch processing. Two batch related default values for IAA are incorrect in current code: (1) The max batch size of device is set during device initialization, that indicates batch is supported. It should be always 0 on IAA. (2) The max batch size of work queue is set to WQ_DEFAULT_MAX_BATCH (32) as the default value regardless of Intel DSA or IAA device during work queue setup and cleanup. It should be always 0 on IAA. Fix the issues by setting the max batch size of device and max batch size of work queue to 0 on IAA device, that means batch is not supported. [1]: https://cdrdv2.intel.com/v1/dl/getContent/721858 Fixes: 23084545dbb0 ("dmaengine: idxd: set max_xfer and max_batch for RO device") Fixes: 92452a72ebdf ("dmaengine: idxd: set defaults for wq configs") Fixes: bfe1d56091c1 ("dmaengine: idxd: Init and probe for Intel data accelerators") Signed-off-by: Xiaochen Shen Reviewed-by: Dave Jiang Reviewed-by: Fenghua Yu Link: https://lore.kernel.org/r/20220930201528.18621-2-xiaochen.shen@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/device.c | 6 +++--- drivers/dma/idxd/idxd.h | 32 ++++++++++++++++++++++++++++++++ drivers/dma/idxd/init.c | 4 ++-- drivers/dma/idxd/sysfs.c | 2 +- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 2c1e6f6daa628..670b71927db0b 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -390,7 +390,7 @@ static void idxd_wq_disable_cleanup(struct idxd_wq *wq) clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags); memset(wq->name, 0, WQ_NAME_SIZE); wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; - wq->max_batch_size = WQ_DEFAULT_MAX_BATCH; + idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); if (wq->opcap_bmap) bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); } @@ -869,7 +869,7 @@ static int idxd_wq_config_write(struct idxd_wq *wq) /* bytes 12-15 */ wq->wqcfg->max_xfer_shift = ilog2(wq->max_xfer_bytes); - wq->wqcfg->max_batch_shift = ilog2(wq->max_batch_size); + idxd_wqcfg_set_max_batch_shift(idxd->data->type, wq->wqcfg, ilog2(wq->max_batch_size)); /* bytes 32-63 */ if (idxd->hw.wq_cap.op_config && wq->opcap_bmap) { @@ -1051,7 +1051,7 @@ static int idxd_wq_load_config(struct idxd_wq *wq) wq->priority = wq->wqcfg->priority; wq->max_xfer_bytes = 1ULL << wq->wqcfg->max_xfer_shift; - wq->max_batch_size = 1ULL << wq->wqcfg->max_batch_shift; + idxd_wq_set_max_batch_size(idxd->data->type, wq, 1U << wq->wqcfg->max_batch_shift); for (i = 0; i < WQCFG_STRIDES(idxd); i++) { wqcfg_offset = WQCFG_OFFSET(idxd, wq->id, i); diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h index 1196ab342f011..7ced8d283d98b 100644 --- a/drivers/dma/idxd/idxd.h +++ b/drivers/dma/idxd/idxd.h @@ -548,6 +548,38 @@ static inline int idxd_wq_refcount(struct idxd_wq *wq) return wq->client_count; }; +/* + * Intel IAA does not support batch processing. + * The max batch size of device, max batch size of wq and + * max batch shift of wqcfg should be always 0 on IAA. + */ +static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd, + u32 max_batch_size) +{ + if (idxd_type == IDXD_TYPE_IAX) + idxd->max_batch_size = 0; + else + idxd->max_batch_size = max_batch_size; +} + +static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq, + u32 max_batch_size) +{ + if (idxd_type == IDXD_TYPE_IAX) + wq->max_batch_size = 0; + else + wq->max_batch_size = max_batch_size; +} + +static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg, + u32 max_batch_shift) +{ + if (idxd_type == IDXD_TYPE_IAX) + wqcfg->max_batch_shift = 0; + else + wqcfg->max_batch_shift = max_batch_shift; +} + int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv, struct module *module, const char *mod_name); #define idxd_driver_register(driver) \ diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c index 2b18d512cbfc9..09cbf0c179ba9 100644 --- a/drivers/dma/idxd/init.c +++ b/drivers/dma/idxd/init.c @@ -183,7 +183,7 @@ static int idxd_setup_wqs(struct idxd_device *idxd) init_completion(&wq->wq_dead); init_completion(&wq->wq_resurrect); wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; - wq->max_batch_size = WQ_DEFAULT_MAX_BATCH; + idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); if (!wq->wqcfg) { @@ -418,7 +418,7 @@ static void idxd_read_caps(struct idxd_device *idxd) idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); - idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; + idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); if (idxd->hw.gen_cap.config_en) set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c index bdaccf9e04363..7269bd54554f6 100644 --- a/drivers/dma/idxd/sysfs.c +++ b/drivers/dma/idxd/sysfs.c @@ -1046,7 +1046,7 @@ static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribu if (batch_size > idxd->max_batch_size) return -EINVAL; - wq->max_batch_size = (u32)batch_size; + idxd_wq_set_max_batch_size(idxd->data->type, wq, (u32)batch_size); return count; } -- GitLab From 0b8c97a1d8c1bb6a853b8bb1778e8fef17b86fc9 Mon Sep 17 00:00:00 2001 From: Fengqian Gao Date: Fri, 30 Sep 2022 11:28:35 +0800 Subject: [PATCH 1173/1927] dmaengine: idxd: fix RO device state error after been disabled/reset When IDXD is not configurable, that means its WQ, engine, and group configurations cannot be changed. But it can be disabled and its state should be set as disabled regardless it's configurable or not. Fix this by setting device state IDXD_DEV_DISABLED for read-only device as well in idxd_device_clear_state(). Fixes: cf4ac3fef338 ("dmaengine: idxd: fix lockdep warning on device driver removal") Signed-off-by: Fengqian Gao Reviewed-by: Xiaochen Shen Reviewed-by: Dave Jiang Reviewed-by: Fenghua Yu Link: https://lore.kernel.org/r/20220930032835.2290-1-fengqian.gao@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/device.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c index 670b71927db0b..6f44fa8f78a5d 100644 --- a/drivers/dma/idxd/device.c +++ b/drivers/dma/idxd/device.c @@ -730,13 +730,21 @@ static void idxd_device_wqs_clear_state(struct idxd_device *idxd) void idxd_device_clear_state(struct idxd_device *idxd) { - if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) - return; + /* IDXD is always disabled. Other states are cleared only when IDXD is configurable. */ + if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { + /* + * Clearing wq state is protected by wq lock. + * So no need to be protected by device lock. + */ + idxd_device_wqs_clear_state(idxd); + + spin_lock(&idxd->dev_lock); + idxd_groups_clear_state(idxd); + idxd_engines_clear_state(idxd); + } else { + spin_lock(&idxd->dev_lock); + } - idxd_device_wqs_clear_state(idxd); - spin_lock(&idxd->dev_lock); - idxd_groups_clear_state(idxd); - idxd_engines_clear_state(idxd); idxd->state = IDXD_DEV_DISABLED; spin_unlock(&idxd->dev_lock); } -- GitLab From 8454f880c24bca0d9d4bfb6ed4a4a5429a4d9b20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Povi=C5=A1er?= Date: Wed, 19 Oct 2022 15:23:23 +0200 Subject: [PATCH 1174/1927] dmaengine: apple-admac: Fix grabbing of channels in of_xlate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The of_xlate callback is supposed to return the channel after already having 'grabbed' it for private use, so fill that in. Fixes: b127315d9a78 ("dmaengine: apple-admac: Add Apple ADMAC driver") Signed-off-by: Martin Povišer Link: https://lore.kernel.org/r/20221019132324.8585-1-povik+lin@cutebit.org Signed-off-by: Vinod Koul --- drivers/dma/apple-admac.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/apple-admac.c b/drivers/dma/apple-admac.c index 317ca76ccafd2..a2cc520225d32 100644 --- a/drivers/dma/apple-admac.c +++ b/drivers/dma/apple-admac.c @@ -493,7 +493,7 @@ static struct dma_chan *admac_dma_of_xlate(struct of_phandle_args *dma_spec, return NULL; } - return &ad->channels[index].chan; + return dma_get_slave_channel(&ad->channels[index].chan); } static int admac_drain_reports(struct admac_data *ad, int channo) -- GitLab From 081195d17a0c4c636da2b869bd5809d42e8cbb13 Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Mon, 24 Oct 2022 21:50:09 +0200 Subject: [PATCH 1175/1927] dmaengine: mv_xor_v2: Fix a resource leak in mv_xor_v2_remove() A clk_prepare_enable() call in the probe is not balanced by a corresponding clk_disable_unprepare() in the remove function. Add the missing call. Fixes: 3cd2c313f1d6 ("dmaengine: mv_xor_v2: Fix clock resource by adding a register clock") Signed-off-by: Christophe JAILLET Link: https://lore.kernel.org/r/e9e3837a680c9bd2438e4db2b83270c6c052d005.1666640987.git.christophe.jaillet@wanadoo.fr Signed-off-by: Vinod Koul --- drivers/dma/mv_xor_v2.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/mv_xor_v2.c b/drivers/dma/mv_xor_v2.c index f629ef6fd3c2a..113834e1167b6 100644 --- a/drivers/dma/mv_xor_v2.c +++ b/drivers/dma/mv_xor_v2.c @@ -893,6 +893,7 @@ static int mv_xor_v2_remove(struct platform_device *pdev) tasklet_kill(&xor_dev->irq_tasklet); clk_disable_unprepare(xor_dev->clk); + clk_disable_unprepare(xor_dev->reg_clk); return 0; } -- GitLab From ac2b9f34f02052709aea7b34bb2a165e1853eb41 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Thu, 20 Oct 2022 14:28:27 +0800 Subject: [PATCH 1176/1927] dmaengine: ti: k3-udma-glue: fix memory leak when register device fail If device_register() fails, it should call put_device() to give up reference, the name allocated in dev_set_name() can be freed in callback function kobject_cleanup(). Fixes: 5b65781d06ea ("dmaengine: ti: k3-udma-glue: Add support for K3 PKTDMA") Signed-off-by: Yang Yingliang Acked-by: Peter Ujfalusi Link: https://lore.kernel.org/r/20221020062827.2914148-1-yangyingliang@huawei.com Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma-glue.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c index 4fdd9f06b7235..4f1aeb81e9c7f 100644 --- a/drivers/dma/ti/k3-udma-glue.c +++ b/drivers/dma/ti/k3-udma-glue.c @@ -299,6 +299,7 @@ struct k3_udma_glue_tx_channel *k3_udma_glue_request_tx_chn(struct device *dev, ret = device_register(&tx_chn->common.chan_dev); if (ret) { dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&tx_chn->common.chan_dev); tx_chn->common.chan_dev.parent = NULL; goto err; } @@ -917,6 +918,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name, ret = device_register(&rx_chn->common.chan_dev); if (ret) { dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&rx_chn->common.chan_dev); rx_chn->common.chan_dev.parent = NULL; goto err; } @@ -1048,6 +1050,7 @@ k3_udma_glue_request_remote_rx_chn(struct device *dev, const char *name, ret = device_register(&rx_chn->common.chan_dev); if (ret) { dev_err(dev, "Channel Device registration failed %d\n", ret); + put_device(&rx_chn->common.chan_dev); rx_chn->common.chan_dev.parent = NULL; goto err; } -- GitLab From 140fd5e74a1cc7413df88c735fff5ec64d33c1d3 Mon Sep 17 00:00:00 2001 From: Amelie Delaunay Date: Mon, 24 Oct 2022 10:36:11 +0200 Subject: [PATCH 1177/1927] dmaengine: stm32-dma: fix potential race between pause and resume When disabling dma channel, a TCF flag is set and as TCIE is enabled, an interrupt is raised. On a busy system, the interrupt may have latency and the user can ask for dmaengine_resume while stm32-dma driver has not yet managed the complete pause (backup of registers to restore state in resume). To avoid such a case, instead of waiting the interrupt to backup the registers, do it just after disabling the channel and discard Transfer Complete interrupt in case the channel is paused. Fixes: 099a9a94be0e ("dmaengine: stm32-dma: add device_pause/device_resume support") Signed-off-by: Amelie Delaunay Link: https://lore.kernel.org/r/20221024083611.132588-1-amelie.delaunay@foss.st.com Signed-off-by: Vinod Koul --- drivers/dma/stm32-dma.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/dma/stm32-dma.c b/drivers/dma/stm32-dma.c index 4891a1767e5aa..37674029cb427 100644 --- a/drivers/dma/stm32-dma.c +++ b/drivers/dma/stm32-dma.c @@ -675,6 +675,8 @@ static void stm32_dma_handle_chan_paused(struct stm32_dma_chan *chan) chan->chan_reg.dma_sndtr = stm32_dma_read(dmadev, STM32_DMA_SNDTR(chan->id)); + chan->status = DMA_PAUSED; + dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan); } @@ -789,9 +791,7 @@ static irqreturn_t stm32_dma_chan_irq(int irq, void *devid) if (status & STM32_DMA_TCI) { stm32_dma_irq_clear(chan, STM32_DMA_TCI); if (scr & STM32_DMA_SCR_TCIE) { - if (chan->status == DMA_PAUSED && !(scr & STM32_DMA_SCR_EN)) - stm32_dma_handle_chan_paused(chan); - else + if (chan->status != DMA_PAUSED) stm32_dma_handle_chan_done(chan, scr); } status &= ~STM32_DMA_TCI; @@ -838,13 +838,11 @@ static int stm32_dma_pause(struct dma_chan *c) return -EPERM; spin_lock_irqsave(&chan->vchan.lock, flags); + ret = stm32_dma_disable_chan(chan); - /* - * A transfer complete flag is set to indicate the end of transfer due to the stream - * interruption, so wait for interrupt - */ if (!ret) - chan->status = DMA_PAUSED; + stm32_dma_handle_chan_paused(chan); + spin_unlock_irqrestore(&chan->vchan.lock, flags); return ret; -- GitLab From f1171bbdd2ba2a50ee64bb198a78c268a5baf5f1 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:35 +0300 Subject: [PATCH 1178/1927] dmaengine: at_hdmac: Fix at_lli struct definition Those hardware registers are all of 32 bits, while dma_addr_t ca be of type u64 or u32 depending on CONFIG_ARCH_DMA_ADDR_T_64BIT. Force u32 to comply with what the hardware expects. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-2-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac_regs.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index 4d1ebc040031c..d4d382d746078 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -186,13 +186,13 @@ /* LLI == Linked List Item; aka DMA buffer descriptor */ struct at_lli { /* values that are not changed by hardware */ - dma_addr_t saddr; - dma_addr_t daddr; + u32 saddr; + u32 daddr; /* value that may get written back: */ - u32 ctrla; + u32 ctrla; /* more values that are not changed by hardware */ - u32 ctrlb; - dma_addr_t dscr; /* chain to next lli */ + u32 ctrlb; + u32 dscr; /* chain to next lli */ }; /** -- GitLab From 7176a6a8982d311e50a7c1168868d26e65bbba19 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:36 +0300 Subject: [PATCH 1179/1927] dmaengine: at_hdmac: Don't start transactions at tx_submit level tx_submit is supposed to push the current transaction descriptor to a pending queue, waiting for issue_pending() to be called. issue_pending() must start the transfer, not tx_submit(), thus remove atc_dostart() from atc_tx_submit(). Clients of at_xdmac that assume that tx_submit() starts the transfer must be updated and call dma_async_issue_pending() if they miss to call it. The vdbg print was moved to after the lock is released. It is desirable to do the prints without the lock held if possible, and because the if statement disappears there's no reason why to do the print while holding the lock. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-3-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 5a50423b7378e..3f71f4d2f467c 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -691,19 +691,11 @@ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) spin_lock_irqsave(&atchan->lock, flags); cookie = dma_cookie_assign(tx); - if (list_empty(&atchan->active_list)) { - dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n", - desc->txd.cookie); - atc_dostart(atchan, desc); - list_add_tail(&desc->desc_node, &atchan->active_list); - } else { - dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n", - desc->txd.cookie); - list_add_tail(&desc->desc_node, &atchan->queue); - } - + list_add_tail(&desc->desc_node, &atchan->queue); spin_unlock_irqrestore(&atchan->lock, flags); + dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n", + desc->txd.cookie); return cookie; } -- GitLab From 8a47221fc28417ff8a32a4f92d4448a56c3cf7e1 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:37 +0300 Subject: [PATCH 1180/1927] dmaengine: at_hdmac: Start transfer for cyclic channels in issue_pending Cyclic channels must too call issue_pending in order to start a transfer. Start the transfer in issue_pending regardless of the type of channel. This wrongly worked before, because in the past the transfer was started at tx_submit level when only a desc in the transfer list. Fixes: 53830cc75974 ("dmaengine: at_hdmac: add cyclic DMA operation support") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-4-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 3f71f4d2f467c..e9d0c3632868f 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -1536,10 +1536,6 @@ static void atc_issue_pending(struct dma_chan *chan) dev_vdbg(chan2dev(chan), "issue_pending\n"); - /* Not needed for cyclic transfers */ - if (atc_chan_is_cyclic(atchan)) - return; - atc_advance_work(atchan); } -- GitLab From fcd37565efdaffeac179d0f0ce980ac79bfdf569 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:38 +0300 Subject: [PATCH 1181/1927] dmaengine: at_hdmac: Fix premature completion of desc in issue_pending Multiple calls to atc_issue_pending() could result in a premature completion of a descriptor from the atchan->active list, as the method always completed the first active descriptor from the list. Instead, issue_pending() should just take the first transaction descriptor from the pending queue, move it to active_list and start the transfer. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-5-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index e9d0c3632868f..cb5522417db65 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -1527,16 +1527,26 @@ atc_tx_status(struct dma_chan *chan, } /** - * atc_issue_pending - try to finish work + * atc_issue_pending - takes the first transaction descriptor in the pending + * queue and starts the transfer. * @chan: target DMA channel */ static void atc_issue_pending(struct dma_chan *chan) { - struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_desc *desc; + unsigned long flags; dev_vdbg(chan2dev(chan), "issue_pending\n"); - atc_advance_work(atchan); + spin_lock_irqsave(&atchan->lock, flags); + if (atc_chan_is_enabled(atchan) || list_empty(&atchan->queue)) + return spin_unlock_irqrestore(&atchan->lock, flags); + + desc = atc_first_queued(atchan); + list_move_tail(&desc->desc_node, &atchan->active_list); + atc_dostart(atchan, desc); + spin_unlock_irqrestore(&atchan->lock, flags); } /** -- GitLab From f645f85ae1104f8bd882f962ac0a69a1070076dd Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:39 +0300 Subject: [PATCH 1182/1927] dmaengine: at_hdmac: Do not call the complete callback on device_terminate_all The method was wrong because it violated the dmaengine API. For aborted transfers the complete callback should not be called. Fix the behavior and do not call the complete callback on device_terminate_all. Fixes: 808347f6a317 ("dmaengine: at_hdmac: add DMA slave transfers") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-6-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index cb5522417db65..11816484843ee 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -1437,11 +1437,8 @@ static int atc_terminate_all(struct dma_chan *chan) struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); int chan_id = atchan->chan_common.chan_id; - struct at_desc *desc, *_desc; unsigned long flags; - LIST_HEAD(list); - dev_vdbg(chan2dev(chan), "%s\n", __func__); /* @@ -1460,15 +1457,11 @@ static int atc_terminate_all(struct dma_chan *chan) cpu_relax(); /* active_list entries will end up before queued entries */ - list_splice_init(&atchan->queue, &list); - list_splice_init(&atchan->active_list, &list); + list_splice_tail_init(&atchan->queue, &atchan->free_list); + list_splice_tail_init(&atchan->active_list, &atchan->free_list); spin_unlock_irqrestore(&atchan->lock, flags); - /* Flush all pending and queued descriptors */ - list_for_each_entry_safe(desc, _desc, &list, desc_node) - atc_chain_complete(atchan, desc); - clear_bit(ATC_IS_PAUSED, &atchan->status); /* if channel dedicated to cyclic operations, free it */ clear_bit(ATC_IS_CYCLIC, &atchan->status); -- GitLab From 6e5ad28d16f082efeae3d0bd2e31f24bed218019 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:40 +0300 Subject: [PATCH 1183/1927] dmaengine: at_hdmac: Protect atchan->status with the channel lock Now that the complete callback call was removed from device_terminate_all(), we can protect the atchan->status with the channel lock. The atomic bitops on atchan->status do not substitute proper locking on the status, as one could still modify the status after the lock was dropped in atc_terminate_all() but before the atomic bitops were executed. Fixes: 078a6506141a ("dmaengine: at_hdmac: Fix deadlocks") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-7-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 11816484843ee..deb4c6027436c 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -1460,12 +1460,12 @@ static int atc_terminate_all(struct dma_chan *chan) list_splice_tail_init(&atchan->queue, &atchan->free_list); list_splice_tail_init(&atchan->active_list, &atchan->free_list); - spin_unlock_irqrestore(&atchan->lock, flags); - clear_bit(ATC_IS_PAUSED, &atchan->status); /* if channel dedicated to cyclic operations, free it */ clear_bit(ATC_IS_CYCLIC, &atchan->status); + spin_unlock_irqrestore(&atchan->lock, flags); + return 0; } -- GitLab From c6babed879fbe82796a601bf097649e07382db46 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:41 +0300 Subject: [PATCH 1184/1927] dmaengine: at_hdmac: Fix concurrency problems by removing atc_complete_all() atc_complete_all() had concurrency bugs, thus remove it: 1/ atc_complete_all() in its entirety was buggy, as when the atchan->queue list (the one that contains descriptors that are not yet issued to the hardware) contained descriptors, it fired just the first from the atchan->queue, but moved all the desc from atchan->queue to atchan->active_list and considered them all as fired. This could result in calling the completion of a descriptor that was not yet issued to the hardware. 2/ when in tasklet at atc_advance_work() time, atchan->active_list was queried without holding the lock of the chan. This can result in atchan->active_list concurrency problems between the tasklet and issue_pending(). Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-8-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 49 ++++-------------------------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index deb4c6027436c..f1e6fa6af6c25 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -485,42 +485,6 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) dma_run_dependencies(txd); } -/** - * atc_complete_all - finish work for all transactions - * @atchan: channel to complete transactions for - * - * Eventually submit queued descriptors if any - * - * Assume channel is idle while calling this function - * Called with atchan->lock held and bh disabled - */ -static void atc_complete_all(struct at_dma_chan *atchan) -{ - struct at_desc *desc, *_desc; - LIST_HEAD(list); - unsigned long flags; - - dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); - - spin_lock_irqsave(&atchan->lock, flags); - - /* - * Submit queued descriptors ASAP, i.e. before we go through - * the completed ones. - */ - if (!list_empty(&atchan->queue)) - atc_dostart(atchan, atc_first_queued(atchan)); - /* empty active_list now it is completed */ - list_splice_init(&atchan->active_list, &list); - /* empty queue list by moving descriptors (if any) to active_list */ - list_splice_init(&atchan->queue, &atchan->active_list); - - spin_unlock_irqrestore(&atchan->lock, flags); - - list_for_each_entry_safe(desc, _desc, &list, desc_node) - atc_chain_complete(atchan, desc); -} - /** * atc_advance_work - at the end of a transaction, move forward * @atchan: channel where the transaction ended @@ -528,25 +492,20 @@ static void atc_complete_all(struct at_dma_chan *atchan) static void atc_advance_work(struct at_dma_chan *atchan) { unsigned long flags; - int ret; dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); spin_lock_irqsave(&atchan->lock, flags); - ret = atc_chan_is_enabled(atchan); + if (atc_chan_is_enabled(atchan) || list_empty(&atchan->active_list)) + return spin_unlock_irqrestore(&atchan->lock, flags); spin_unlock_irqrestore(&atchan->lock, flags); - if (ret) - return; - - if (list_empty(&atchan->active_list) || - list_is_singular(&atchan->active_list)) - return atc_complete_all(atchan); atc_chain_complete(atchan, atc_first_active(atchan)); /* advance work */ spin_lock_irqsave(&atchan->lock, flags); - atc_dostart(atchan, atc_first_active(atchan)); + if (!list_empty(&atchan->active_list)) + atc_dostart(atchan, atc_first_active(atchan)); spin_unlock_irqrestore(&atchan->lock, flags); } -- GitLab From 06988949df8c3007ad82036d3606d8ae72ed9000 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:42 +0300 Subject: [PATCH 1185/1927] dmaengine: at_hdmac: Fix concurrency over descriptor The descriptor was added to the free_list before calling the callback, which could result in reissuing of the same descriptor and calling of a single callback for both. Move the decriptor to the free list after the callback is invoked. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-9-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index f1e6fa6af6c25..2012ecc578262 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -469,11 +469,8 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) desc->memset_buffer = false; } - /* move children to free_list */ - list_splice_init(&desc->tx_list, &atchan->free_list); - /* move myself to free_list */ - list_move(&desc->desc_node, &atchan->free_list); - + /* Remove transfer node from the active list. */ + list_del_init(&desc->desc_node); spin_unlock_irqrestore(&atchan->lock, flags); dma_descriptor_unmap(txd); @@ -483,6 +480,13 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) dmaengine_desc_get_callback_invoke(txd, NULL); dma_run_dependencies(txd); + + spin_lock_irqsave(&atchan->lock, flags); + /* move children to free_list */ + list_splice_init(&desc->tx_list, &atchan->free_list); + /* add myself to free_list */ + list_add(&desc->desc_node, &atchan->free_list); + spin_unlock_irqrestore(&atchan->lock, flags); } /** -- GitLab From 6ba826cbb57d675f447b59323204d1473bbd5593 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:43 +0300 Subject: [PATCH 1186/1927] dmaengine: at_hdmac: Free the memset buf without holding the chan lock There's no need to hold the channel lock when freeing the memset buf, as the operation has already completed. Free the memset buf without holding the channel lock. Fixes: 4d112426c344 ("dmaengine: hdmac: Add memset capabilities") Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-10-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 2012ecc578262..0fb44f622d35e 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -462,13 +462,6 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) if (!atc_chan_is_cyclic(atchan)) dma_cookie_complete(txd); - /* If the transfer was a memset, free our temporary buffer */ - if (desc->memset_buffer) { - dma_pool_free(atdma->memset_pool, desc->memset_vaddr, - desc->memset_paddr); - desc->memset_buffer = false; - } - /* Remove transfer node from the active list. */ list_del_init(&desc->desc_node); spin_unlock_irqrestore(&atchan->lock, flags); @@ -487,6 +480,13 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* add myself to free_list */ list_add(&desc->desc_node, &atchan->free_list); spin_unlock_irqrestore(&atchan->lock, flags); + + /* If the transfer was a memset, free our temporary buffer */ + if (desc->memset_buffer) { + dma_pool_free(atdma->memset_pool, desc->memset_vaddr, + desc->memset_paddr); + desc->memset_buffer = false; + } } /** -- GitLab From 03ed9ba357cc78116164b90b87f45eacab60b561 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:44 +0300 Subject: [PATCH 1187/1927] dmaengine: at_hdmac: Fix concurrency over the active list The tasklet (atc_advance_work()) did not held the channel lock when retrieving the first active descriptor, causing concurrency problems if issue_pending() was called in between. If issue_pending() was called exactly after the lock was released in the tasklet (atc_advance_work()), atc_chain_complete() could complete a descriptor for which the controller has not yet raised an interrupt. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-11-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 0fb44f622d35e..b53a9fc15dd9b 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -462,8 +462,6 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) if (!atc_chan_is_cyclic(atchan)) dma_cookie_complete(txd); - /* Remove transfer node from the active list. */ - list_del_init(&desc->desc_node); spin_unlock_irqrestore(&atchan->lock, flags); dma_descriptor_unmap(txd); @@ -495,6 +493,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) */ static void atc_advance_work(struct at_dma_chan *atchan) { + struct at_desc *desc; unsigned long flags; dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); @@ -502,9 +501,12 @@ static void atc_advance_work(struct at_dma_chan *atchan) spin_lock_irqsave(&atchan->lock, flags); if (atc_chan_is_enabled(atchan) || list_empty(&atchan->active_list)) return spin_unlock_irqrestore(&atchan->lock, flags); - spin_unlock_irqrestore(&atchan->lock, flags); - atc_chain_complete(atchan, atc_first_active(atchan)); + desc = atc_first_active(atchan); + /* Remove the transfer node from the active list. */ + list_del_init(&desc->desc_node); + spin_unlock_irqrestore(&atchan->lock, flags); + atc_chain_complete(atchan, desc); /* advance work */ spin_lock_irqsave(&atchan->lock, flags); -- GitLab From ba2423633ba646e1df20e30cb3cf35495c16f173 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:45 +0300 Subject: [PATCH 1188/1927] dmaengine: at_hdmac: Fix descriptor handling when issuing it to hardware As it was before, the descriptor was issued to the hardware without adding it to the active (issued) list. This could result in a completion of other descriptor, or/and in the descriptor never being completed. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-12-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index b53a9fc15dd9b..9e5a30396c1c1 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -510,8 +510,11 @@ static void atc_advance_work(struct at_dma_chan *atchan) /* advance work */ spin_lock_irqsave(&atchan->lock, flags); - if (!list_empty(&atchan->active_list)) - atc_dostart(atchan, atc_first_active(atchan)); + if (!list_empty(&atchan->active_list)) { + desc = atc_first_queued(atchan); + list_move_tail(&desc->desc_node, &atchan->active_list); + atc_dostart(atchan, desc); + } spin_unlock_irqrestore(&atchan->lock, flags); } @@ -523,6 +526,7 @@ static void atc_advance_work(struct at_dma_chan *atchan) static void atc_handle_error(struct at_dma_chan *atchan) { struct at_desc *bad_desc; + struct at_desc *desc; struct at_desc *child; unsigned long flags; @@ -540,8 +544,11 @@ static void atc_handle_error(struct at_dma_chan *atchan) list_splice_init(&atchan->queue, atchan->active_list.prev); /* Try to restart the controller */ - if (!list_empty(&atchan->active_list)) - atc_dostart(atchan, atc_first_active(atchan)); + if (!list_empty(&atchan->active_list)) { + desc = atc_first_queued(atchan); + list_move_tail(&desc->desc_node, &atchan->active_list); + atc_dostart(atchan, desc); + } /* * KERN_CRITICAL may seem harsh, but since this only happens -- GitLab From ef2cb4f0ce479f77607b04c4b0414bf32f863ee8 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:46 +0300 Subject: [PATCH 1189/1927] dmaengine: at_hdmac: Fix completion of unissued descriptor in case of errors In case the controller detected an error, the code took the chance to move all the queued (submitted) descriptors to the active (issued) list. This was wrong as if there were any descriptors in the submitted list they were moved to the issued list without actually issuing them to the controller, thus a completion could be raised without even fireing the descriptor. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-13-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 9e5a30396c1c1..80eeb4fb88efd 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -539,10 +539,6 @@ static void atc_handle_error(struct at_dma_chan *atchan) bad_desc = atc_first_active(atchan); list_del_init(&bad_desc->desc_node); - /* As we are stopped, take advantage to push queued descriptors - * in active_list */ - list_splice_init(&atchan->queue, atchan->active_list.prev); - /* Try to restart the controller */ if (!list_empty(&atchan->active_list)) { desc = atc_first_queued(atchan); -- GitLab From 580ee84405c27d6ed419abe4d2b3de1968abdafd Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:47 +0300 Subject: [PATCH 1190/1927] dmaengine: at_hdmac: Don't allow CPU to reorder channel enable at_hdmac uses __raw_writel for register writes. In the absence of a barrier, the CPU may reorder the register operations. Introduce a write memory barrier so that the CPU does not reorder the channel enable, thus the start of the transfer, without making sure that all the pre-required register fields are already written. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Reported-by: Peter Rosin Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Link: https://lore.kernel.org/lkml/13c6c9a2-6db5-c3bf-349b-4c127ad3496a@axentia.se/ Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-14-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 80eeb4fb88efd..968a5aba47cd4 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -256,6 +256,8 @@ static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) ATC_SPIP_BOUNDARY(first->boundary)); channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | ATC_DPIP_BOUNDARY(first->boundary)); + /* Don't allow CPU to reorder channel enable. */ + wmb(); dma_writel(atdma, CHER, atchan->mask); vdbg_dump_regs(atchan); -- GitLab From 28cbe5a0a46a6637adbda52337d7b2777fc04027 Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:48 +0300 Subject: [PATCH 1191/1927] dmaengine: at_hdmac: Fix impossible condition The iterator can not be greater than ATC_MAX_DSCR_TRIALS, as the for loop will stop when i == ATC_MAX_DSCR_TRIALS. While here, use the common "i" name for the iterator. Fixes: 93dce3a6434f ("dmaengine: at_hdmac: fix residue computation") Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-15-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index 968a5aba47cd4..afcbad3e17186 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -318,7 +318,8 @@ static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) struct at_desc *desc_first = atc_first_active(atchan); struct at_desc *desc; int ret; - u32 ctrla, dscr, trials; + u32 ctrla, dscr; + unsigned int i; /* * If the cookie doesn't match to the currently running transfer then @@ -388,7 +389,7 @@ static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) dscr = channel_readl(atchan, DSCR); rmb(); /* ensure DSCR is read before CTRLA */ ctrla = channel_readl(atchan, CTRLA); - for (trials = 0; trials < ATC_MAX_DSCR_TRIALS; ++trials) { + for (i = 0; i < ATC_MAX_DSCR_TRIALS; ++i) { u32 new_dscr; rmb(); /* ensure DSCR is read after CTRLA */ @@ -414,7 +415,7 @@ static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie) rmb(); /* ensure DSCR is read before CTRLA */ ctrla = channel_readl(atchan, CTRLA); } - if (unlikely(trials >= ATC_MAX_DSCR_TRIALS)) + if (unlikely(i == ATC_MAX_DSCR_TRIALS)) return -ETIMEDOUT; /* for the first descriptor we can be more accurate */ -- GitLab From c47e6403fa099f200868d6b106701cb42d181d2b Mon Sep 17 00:00:00 2001 From: Tudor Ambarus Date: Tue, 25 Oct 2022 12:02:49 +0300 Subject: [PATCH 1192/1927] dmaengine: at_hdmac: Check return code of dma_async_device_register dma_async_device_register() can fail, check the return code and display an error. Fixes: dc78baa2b90b ("dmaengine: at_hdmac: new driver for the Atmel AHB DMA Controller") Signed-off-by: Tudor Ambarus Cc: stable@vger.kernel.org Acked-by: Nicolas Ferre Link: https://lore.kernel.org/r/20221025090306.297886-1-tudor.ambarus@microchip.com Link: https://lore.kernel.org/r/20221025090306.297886-16-tudor.ambarus@microchip.com Signed-off-by: Vinod Koul --- drivers/dma/at_hdmac.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index afcbad3e17186..858bd64f13135 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -1928,7 +1928,11 @@ static int __init at_dma_probe(struct platform_device *pdev) dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", plat_dat->nr_channels); - dma_async_device_register(&atdma->dma_common); + err = dma_async_device_register(&atdma->dma_common); + if (err) { + dev_err(&pdev->dev, "Unable to register: %d.\n", err); + goto err_dma_async_device_register; + } /* * Do not return an error if the dmac node is not present in order to @@ -1948,6 +1952,7 @@ static int __init at_dma_probe(struct platform_device *pdev) err_of_dma_controller_register: dma_async_device_unregister(&atdma->dma_common); +err_dma_async_device_register: dma_pool_destroy(atdma->memset_pool); err_memset_pool_create: dma_pool_destroy(atdma->dma_desc_pool); -- GitLab From ea045fd344cb15c164e9ffc8b8cffb6883df8475 Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Mon, 7 Nov 2022 13:02:29 +0900 Subject: [PATCH 1193/1927] ata: libata-scsi: fix SYNCHRONIZE CACHE (16) command failure SAT SCSI/ATA Translation specification requires SCSI SYNCHRONIZE CACHE (10) and (16) commands both shall be translated to ATA flush command. Also, ZBC Zoned Block Commands specification mandates SYNCHRONIZE CACHE (16) command support. However, libata translates only SYNCHRONIZE CACHE (10). This results in SYNCHRONIZE CACHE (16) command failures on SATA drives and then libata translation does not conform to ZBC. To avoid the failure, add support for SYNCHRONIZE CACHE (16). Signed-off-by: Shin'ichiro Kawasaki Cc: stable@vger.kernel.org Reviewed-by: Christoph Hellwig Reviewed-by: Martin K. Petersen Signed-off-by: Damien Le Moal --- drivers/ata/libata-scsi.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index e2ebb0b065e27..61cd4e90e4e78 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -3264,6 +3264,7 @@ static unsigned int ata_scsiop_maint_in(struct ata_scsi_args *args, u8 *rbuf) case REPORT_LUNS: case REQUEST_SENSE: case SYNCHRONIZE_CACHE: + case SYNCHRONIZE_CACHE_16: case REZERO_UNIT: case SEEK_6: case SEEK_10: @@ -3922,6 +3923,7 @@ static inline ata_xlat_func_t ata_get_xlat_func(struct ata_device *dev, u8 cmd) return ata_scsi_write_same_xlat; case SYNCHRONIZE_CACHE: + case SYNCHRONIZE_CACHE_16: if (ata_try_flush_cache(dev)) return ata_scsi_flush_xlat; break; @@ -4145,6 +4147,7 @@ void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd) * turning this into a no-op. */ case SYNCHRONIZE_CACHE: + case SYNCHRONIZE_CACHE_16: fallthrough; /* no-op's, complete with success */ -- GitLab From e5126de138caef0eedb3d6431059c0c5581a1a5d Mon Sep 17 00:00:00 2001 From: Yue Hu Date: Fri, 21 Oct 2022 16:53:25 +0800 Subject: [PATCH 1194/1927] erofs: fix general protection fault when reading fragment As syzbot reported [1], the fragment feature sb flag is not set, so packed_inode != NULL needs to be checked in z_erofs_read_fragment(). [1] https://lore.kernel.org/all/0000000000002e7a8905eb841ddd@google.com/ Reported-by: syzbot+3faecbfd845a895c04cb@syzkaller.appspotmail.com Fixes: b15b2e307c3a ("erofs: support on-disk compressed fragments data") Signed-off-by: Yue Hu Reviewed-by: Gao Xiang Reviewed-by: Chao Yu Link: https://lore.kernel.org/r/20221021085325.25788-1-zbestahu@gmail.com Signed-off-by: Gao Xiang --- fs/erofs/zdata.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index c7f24fc7efd59..d6caf275be776 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -660,6 +660,9 @@ static int z_erofs_read_fragment(struct inode *inode, erofs_off_t pos, u8 *src, *dst; unsigned int i, cnt; + if (!packed_inode) + return -EFSCORRUPTED; + pos += EROFS_I(inode)->z_fragmentoff; for (i = 0; i < len; i += cnt) { cnt = min_t(unsigned int, len - i, -- GitLab From 75e43355cbe4d5948a79bd592f2ffecb9f75f75d Mon Sep 17 00:00:00 2001 From: Jingbo Xu Date: Fri, 4 Nov 2022 13:40:27 +0800 Subject: [PATCH 1195/1927] erofs: put metabuf in error path in fscache mode For tail packing layout, put metabuf when error is encountered. Fixes: 1ae9470c3e14 ("erofs: clean up .read_folio() and .readahead() in fscache mode") Signed-off-by: Jingbo Xu Reviewed-by: Gao Xiang Reviewed-by: Jia Zhu Reviewed-by: Chao Yu Link: https://lore.kernel.org/r/20221104054028.52208-2-jefflexu@linux.alibaba.com Signed-off-by: Gao Xiang --- fs/erofs/fscache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index fe05bc51f9f2f..83559008bfa89 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -287,8 +287,10 @@ static int erofs_fscache_data_read(struct address_space *mapping, return PTR_ERR(src); iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, PAGE_SIZE); - if (copy_to_iter(src + offset, size, &iter) != size) + if (copy_to_iter(src + offset, size, &iter) != size) { + erofs_put_metabuf(&buf); return -EFAULT; + } iov_iter_zero(PAGE_SIZE - size, &iter); erofs_put_metabuf(&buf); return PAGE_SIZE; -- GitLab From e6d9f9ba111b56154f1b1120252aff269cebd49c Mon Sep 17 00:00:00 2001 From: Jingbo Xu Date: Fri, 4 Nov 2022 13:40:28 +0800 Subject: [PATCH 1196/1927] erofs: get correct count for unmapped range in fscache mode For unmapped range, the returned map.m_llen is zero, and thus the calculated count is unexpected zero. Prior to the refactoring introduced by commit 1ae9470c3e14 ("erofs: clean up .read_folio() and .readahead() in fscache mode"), only the readahead routine suffers from this. With the refactoring of making .read_folio() and .readahead() calling one common routine, both read_folio and readahead have this issue now. Fix this by calculating count separately in unmapped condition. Fixes: c665b394b9e8 ("erofs: implement fscache-based data readahead") Fixes: 1ae9470c3e14 ("erofs: clean up .read_folio() and .readahead() in fscache mode") Signed-off-by: Jingbo Xu Reviewed-by: Gao Xiang Reviewed-by: Chao Yu Link: https://lore.kernel.org/r/20221104054028.52208-3-jefflexu@linux.alibaba.com Signed-off-by: Gao Xiang --- fs/erofs/fscache.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index 83559008bfa89..260fa4737fc00 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -296,15 +296,16 @@ static int erofs_fscache_data_read(struct address_space *mapping, return PAGE_SIZE; } - count = min_t(size_t, map.m_llen - (pos - map.m_la), len); - DBG_BUGON(!count || count % PAGE_SIZE); - if (!(map.m_flags & EROFS_MAP_MAPPED)) { + count = len; iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, count); iov_iter_zero(count, &iter); return count; } + count = min_t(size_t, map.m_llen - (pos - map.m_la), len); + DBG_BUGON(!count || count % PAGE_SIZE); + mdev = (struct erofs_map_dev) { .m_deviceid = map.m_deviceid, .m_pa = map.m_pa, -- GitLab From 971cb608d1c5d95533a43b549bb8ec9637f10043 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Tue, 8 Nov 2022 07:58:23 +0100 Subject: [PATCH 1197/1927] ALSA: usb-audio: Yet more regression for for the delayed card registration Although we tried to fix the regression for the recent changes with the delayed card registration, it doesn't seem covering the all cases; e.g. on Roland EDIROL M-100FX, where the generic quirk for Roland devices is applied, it misses the card registration because the detection of the last interface (apparently for MIDI) fails. This patch is an attempt to recover from those failures by calling the card register also at the error path for the secondary interfaces. The card register condition is also extended to match with the old check in the previous patch, too (i.e. the simple check of the interface number) for catching the probe with errors. Fixes: 39efc9c8a973 ("ALSA: usb-audio: Fix last interface check for registration") Cc: Link: https://bugzilla.suse.com/show_bug.cgi?id=1205111 Link: https://lore.kernel.org/r/20221108065824.14418-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/usb/card.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/sound/usb/card.c b/sound/usb/card.c index a5ed11ea11456..26268ffb82742 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -742,6 +742,18 @@ get_alias_quirk(struct usb_device *dev, unsigned int id) return NULL; } +/* register card if we reach to the last interface or to the specified + * one given via option + */ +static int try_to_register_card(struct snd_usb_audio *chip, int ifnum) +{ + if (check_delayed_register_option(chip) == ifnum || + chip->last_iface == ifnum || + usb_interface_claimed(usb_ifnum_to_if(chip->dev, chip->last_iface))) + return snd_card_register(chip->card); + return 0; +} + /* * probe the active usb device * @@ -880,15 +892,9 @@ static int usb_audio_probe(struct usb_interface *intf, chip->need_delayed_register = false; /* clear again */ } - /* register card if we reach to the last interface or to the specified - * one given via option - */ - if (check_delayed_register_option(chip) == ifnum || - usb_interface_claimed(usb_ifnum_to_if(dev, chip->last_iface))) { - err = snd_card_register(chip->card); - if (err < 0) - goto __error; - } + err = try_to_register_card(chip, ifnum); + if (err < 0) + goto __error_no_register; if (chip->quirk_flags & QUIRK_FLAG_SHARE_MEDIA_DEVICE) { /* don't want to fail when snd_media_device_create() fails */ @@ -907,6 +913,11 @@ static int usb_audio_probe(struct usb_interface *intf, return 0; __error: + /* in the case of error in secondary interface, still try to register */ + if (chip) + try_to_register_card(chip, ifnum); + + __error_no_register: if (chip) { /* chip->active is inside the chip->card object, * decrement before memory is possibly returned. -- GitLab From ea83ec50393a6993fa09315e4200240c938c21b3 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Tue, 8 Nov 2022 07:58:24 +0100 Subject: [PATCH 1198/1927] ALSA: usb-audio: Remove redundant workaround for Roland quirk The recent fix for the delayed card registration made the current workaround for QUIRK_AUTODETECT superfluous, since the card registration itself is delayed until the last interface probe. This patch drops the redundant workaround in create_autodetect_quirks() for simplification. Fixes: 39efc9c8a973 ("ALSA: usb-audio: Fix last interface check for registration") Link: https://bugzilla.suse.com/show_bug.cgi?id=1205111 Link: https://lore.kernel.org/r/20221108065824.14418-2-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/usb/quirks.c | 44 +++----------------------------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index eadac586bcc83..2dd19a5c3026e 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -376,7 +376,8 @@ static int create_auto_midi_quirk(struct snd_usb_audio *chip, static int create_autodetect_quirk(struct snd_usb_audio *chip, struct usb_interface *iface, - struct usb_driver *driver) + struct usb_driver *driver, + const struct snd_usb_audio_quirk *quirk) { int err; @@ -386,45 +387,6 @@ static int create_autodetect_quirk(struct snd_usb_audio *chip, return err; } -static int create_autodetect_quirks(struct snd_usb_audio *chip, - struct usb_interface *iface, - struct usb_driver *driver, - const struct snd_usb_audio_quirk *quirk) -{ - int probed_ifnum = get_iface_desc(iface->altsetting)->bInterfaceNumber; - int ifcount, ifnum, err; - - err = create_autodetect_quirk(chip, iface, driver); - if (err < 0) - return err; - - /* - * ALSA PCM playback/capture devices cannot be registered in two steps, - * so we have to claim the other corresponding interface here. - */ - ifcount = chip->dev->actconfig->desc.bNumInterfaces; - for (ifnum = 0; ifnum < ifcount; ifnum++) { - if (ifnum == probed_ifnum || quirk->ifnum >= 0) - continue; - iface = usb_ifnum_to_if(chip->dev, ifnum); - if (!iface || - usb_interface_claimed(iface) || - get_iface_desc(iface->altsetting)->bInterfaceClass != - USB_CLASS_VENDOR_SPEC) - continue; - - err = create_autodetect_quirk(chip, iface, driver); - if (err >= 0) { - err = usb_driver_claim_interface(driver, iface, - USB_AUDIO_IFACE_UNUSED); - if (err < 0) - return err; - } - } - - return 0; -} - /* * Create a stream for an Edirol UA-700/UA-25/UA-4FX interface. * The only way to detect the sample rate is by looking at wMaxPacketSize. @@ -554,7 +516,7 @@ int snd_usb_create_quirk(struct snd_usb_audio *chip, static const quirk_func_t quirk_funcs[] = { [QUIRK_IGNORE_INTERFACE] = ignore_interface_quirk, [QUIRK_COMPOSITE] = create_composite_quirk, - [QUIRK_AUTODETECT] = create_autodetect_quirks, + [QUIRK_AUTODETECT] = create_autodetect_quirk, [QUIRK_MIDI_STANDARD_INTERFACE] = create_any_midi_quirk, [QUIRK_MIDI_FIXED_ENDPOINT] = create_any_midi_quirk, [QUIRK_MIDI_YAMAHA] = create_any_midi_quirk, -- GitLab From 28adfe402909dd8bca741c72619eae6f52364987 Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 21 May 2022 13:10:59 +0200 Subject: [PATCH 1199/1927] drm/i915/gvt: fix typo in comment Spelling mistake (triple letters) in comment. Detected with the help of Coccinelle. Signed-off-by: Julia Lawall Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20220521111145.81697-49-Julia.Lawall@inria.fr Acked-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/gtt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c index b4f69364f9a13..ce0eb03709c3f 100644 --- a/drivers/gpu/drm/i915/gvt/gtt.c +++ b/drivers/gpu/drm/i915/gvt/gtt.c @@ -2341,7 +2341,7 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off, gvt_vgpu_err("fail to populate guest ggtt entry\n"); /* guest driver may read/write the entry when partial * update the entry in this situation p2m will fail - * settting the shadow entry to point to a scratch page + * setting the shadow entry to point to a scratch page */ ops->set_pfn(&m, gvt->gtt.scratch_mfn); } else -- GitLab From 63ba856bf1e3627fd2c86cc6e7229d92dd3e887e Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Tue, 24 May 2022 16:37:33 +0800 Subject: [PATCH 1200/1927] drm/i915/gvt: Fix kernel-doc Fix the following W=1 kernel warnings: drivers/gpu/drm/i915/gvt/handlers.c:3066: warning: expecting prototype for intel_t_default_mmio_write(). Prototype was for intel_vgpu_default_mmio_write() instead. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20220524083733.67148-2-jiapeng.chong@linux.alibaba.com Acked-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/handlers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c index e7e33f95cc515..c1490b657c147 100644 --- a/drivers/gpu/drm/i915/gvt/handlers.c +++ b/drivers/gpu/drm/i915/gvt/handlers.c @@ -3052,7 +3052,7 @@ int intel_vgpu_default_mmio_read(struct intel_vgpu *vgpu, unsigned int offset, } /** - * intel_t_default_mmio_write - default MMIO write handler + * intel_vgpu_default_mmio_write() - default MMIO write handler * @vgpu: a vGPU * @offset: access offset * @p_data: write data buffer -- GitLab From 400c0563bf2a9328eda5c4d67b3369100364c5c8 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Tue, 24 May 2022 16:37:32 +0800 Subject: [PATCH 1201/1927] drm/i915/gvt: Fix kernel-doc Fix the following W=1 kernel warnings: drivers/gpu/drm/i915/gvt/mmio_context.c:560: warning: expecting prototype for intel_gvt_switch_render_mmio(). Prototype was for intel_gvt_switch_mmio() instead. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20220524083733.67148-1-jiapeng.chong@linux.alibaba.com Acked-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/mmio_context.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/gvt/mmio_context.c b/drivers/gpu/drm/i915/gvt/mmio_context.c index c85bafe7539ee..1c6e941c96666 100644 --- a/drivers/gpu/drm/i915/gvt/mmio_context.c +++ b/drivers/gpu/drm/i915/gvt/mmio_context.c @@ -546,7 +546,7 @@ static void switch_mmio(struct intel_vgpu *pre, } /** - * intel_gvt_switch_render_mmio - switch mmio context of specific engine + * intel_gvt_switch_mmio - switch mmio context of specific engine * @pre: the last vGPU that own the engine * @next: the vGPU to switch to * @engine: the engine -- GitLab From 63dfc0c0fd48c371fa7b0ce718cf0d02f572f404 Mon Sep 17 00:00:00 2001 From: Jiapeng Chong Date: Thu, 2 Jun 2022 15:35:19 +0800 Subject: [PATCH 1202/1927] drm/i915/gvt: Fix kernel-doc Fix the following W=1 kernel warnings: drivers/gpu/drm/i915/gvt/aperture_gm.c:308: warning: expecting prototype for inte_gvt_free_vgpu_resource(). Prototype was for intel_vgpu_free_resource() instead. drivers/gpu/drm/i915/gvt/aperture_gm.c:344: warning: expecting prototype for intel_alloc_vgpu_resource(). Prototype was for intel_vgpu_alloc_resource() instead. Reported-by: Abaci Robot Signed-off-by: Jiapeng Chong Signed-off-by: Zhenyu Wang Link: http://patchwork.freedesktop.org/patch/msgid/20220602073519.22363-1-jiapeng.chong@linux.alibaba.com Acked-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/aperture_gm.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c index 557f3314291a8..3b81a6d35a7b2 100644 --- a/drivers/gpu/drm/i915/gvt/aperture_gm.c +++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c @@ -298,7 +298,7 @@ static int alloc_resource(struct intel_vgpu *vgpu, } /** - * inte_gvt_free_vgpu_resource - free HW resource owned by a vGPU + * intel_vgpu_free_resource() - free HW resource owned by a vGPU * @vgpu: a vGPU * * This function is used to free the HW resource owned by a vGPU. @@ -328,7 +328,7 @@ void intel_vgpu_reset_resource(struct intel_vgpu *vgpu) } /** - * intel_alloc_vgpu_resource - allocate HW resource for a vGPU + * intel_vgpu_alloc_resource() - allocate HW resource for a vGPU * @vgpu: vGPU * @param: vGPU creation params * -- GitLab From 38e0d3fd1ee170ba95f908e1606fbb8763a98593 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Mon, 7 Nov 2022 16:04:54 +0200 Subject: [PATCH 1203/1927] drm/i915/pxp: use <> instead of "" for headers in include/ Headers in include/ should be included using the system header #include syntax. Fixes: 887a193b4fb1 ("drm/i915/pxp: add huc authentication and loading command") Cc: Tomas Winkler Cc: Vitaly Lubart Cc: Daniele Ceraolo Spurio Cc: Alan Previn Signed-off-by: Jani Nikula Reviewed-by: Daniele Ceraolo Spurio Link: https://patchwork.freedesktop.org/patch/msgid/20221107140454.2680954-1-jani.nikula@intel.com --- drivers/gpu/drm/i915/pxp/intel_pxp_huc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c b/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c index 7ec36d94e7583..f6a3f53a1d225 100644 --- a/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c +++ b/drivers/gpu/drm/i915/pxp/intel_pxp_huc.c @@ -3,7 +3,8 @@ * Copyright(c) 2021-2022, Intel Corporation. All rights reserved. */ -#include "drm/i915_drm.h" +#include + #include "i915_drv.h" #include "gem/i915_gem_region.h" -- GitLab From 308451d9c7fece33d9551230cb8e5eb7f3914988 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Mon, 19 Sep 2022 15:32:58 +0200 Subject: [PATCH 1204/1927] drm: panel-orientation-quirks: Add quirk for Nanote UMPC-01 The Nanote UMPC-01 is a mini laptop with a 1200x1920 portrait screen mounted in a landscape oriented clamshell case. Add a quirk for this. Signed-off-by: Hans de Goede Acked-by: Simon Ser Link: https://patchwork.freedesktop.org/patch/msgid/20220919133258.711639-1-hdegoede@redhat.com --- drivers/gpu/drm/drm_panel_orientation_quirks.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c index 8a0c0e0bb5bd2..f0f6fa3065218 100644 --- a/drivers/gpu/drm/drm_panel_orientation_quirks.c +++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c @@ -319,6 +319,12 @@ static const struct dmi_system_id orientation_data[] = { DMI_MATCH(DMI_BIOS_VERSION, "BLADE_21"), }, .driver_data = (void *)&lcd1200x1920_rightside_up, + }, { /* Nanote UMPC-01 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "RWC CO.,LTD"), + DMI_MATCH(DMI_PRODUCT_NAME, "UMPC-01"), + }, + .driver_data = (void *)&lcd1200x1920_rightside_up, }, { /* OneGX1 Pro */ .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "SYSTEM_MANUFACTURER"), -- GitLab From 653f2d94fcda200b02bd79cea2e0307b26c1b747 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 6 Nov 2022 22:50:52 +0100 Subject: [PATCH 1205/1927] drm: panel-orientation-quirks: Add quirk for Acer Switch V 10 (SW5-017) Like the Acer Switch One 10 S1003, for which there already is a quirk, the Acer Switch V 10 (SW5-017) has a 800x1280 portrait screen mounted in the tablet part of a landscape oriented 2-in-1. Add a quirk for this. Cc: Rudolf Polzer Signed-off-by: Hans de Goede Acked-by: Simon Ser Link: https://patchwork.freedesktop.org/patch/msgid/20221106215052.66995-1-hdegoede@redhat.com --- drivers/gpu/drm/drm_panel_orientation_quirks.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c index f0f6fa3065218..52d8800a8ab86 100644 --- a/drivers/gpu/drm/drm_panel_orientation_quirks.c +++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c @@ -134,6 +134,12 @@ static const struct dmi_system_id orientation_data[] = { DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "One S1003"), }, .driver_data = (void *)&lcd800x1280_rightside_up, + }, { /* Acer Switch V 10 (SW5-017) */ + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Acer"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "SW5-017"), + }, + .driver_data = (void *)&lcd800x1280_rightside_up, }, { /* Anbernic Win600 */ .matches = { DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Anbernic"), -- GitLab From fdcc4c22b7ab20e90b97f8bc6225d876b72b8f16 Mon Sep 17 00:00:00 2001 From: Evan Quan Date: Tue, 8 Nov 2022 16:47:46 +0800 Subject: [PATCH 1206/1927] ALSA: hda/hdmi - enable runtime pm for more AMD display audio We are able to power down the GPU and audio via the GPU driver so flag these asics as supporting runtime pm. Signed-off-by: Evan Quan Cc: Link: https://lore.kernel.org/r/20221108084746.583058-1-evan.quan@amd.com Signed-off-by: Takashi Iwai --- sound/pci/hda/hda_intel.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c index 92f66bbc5c91e..87002670c0c92 100644 --- a/sound/pci/hda/hda_intel.c +++ b/sound/pci/hda/hda_intel.c @@ -2711,6 +2711,9 @@ static const struct pci_device_id azx_ids[] = { { PCI_DEVICE(0x1002, 0xab28), .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS | AZX_DCAPS_PM_RUNTIME }, + { PCI_DEVICE(0x1002, 0xab30), + .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS | + AZX_DCAPS_PM_RUNTIME }, { PCI_DEVICE(0x1002, 0xab38), .driver_data = AZX_DRIVER_ATIHDMI_NS | AZX_DCAPS_PRESET_ATI_HDMI_NS | AZX_DCAPS_PM_RUNTIME }, -- GitLab From bb88f9695460bec25aa30ba9072595025cf6c8af Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 31 Oct 2022 10:35:13 +0100 Subject: [PATCH 1207/1927] perf: Improve missing SIGTRAP checking To catch missing SIGTRAP we employ a WARN in __perf_event_overflow(), which fires if pending_sigtrap was already set: returning to user space without consuming pending_sigtrap, and then having the event fire again would re-enter the kernel and trigger the WARN. This, however, seemed to miss the case where some events not associated with progress in the user space task can fire and the interrupt handler runs before the IRQ work meant to consume pending_sigtrap (and generate the SIGTRAP). syzbot gifted us this stack trace: | WARNING: CPU: 0 PID: 3607 at kernel/events/core.c:9313 __perf_event_overflow | Modules linked in: | CPU: 0 PID: 3607 Comm: syz-executor100 Not tainted 6.1.0-rc2-syzkaller-00073-g88619e77b33d #0 | Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/11/2022 | RIP: 0010:__perf_event_overflow+0x498/0x540 kernel/events/core.c:9313 | <...> | Call Trace: | | perf_swevent_hrtimer+0x34f/0x3c0 kernel/events/core.c:10729 | __run_hrtimer kernel/time/hrtimer.c:1685 [inline] | __hrtimer_run_queues+0x1c6/0xfb0 kernel/time/hrtimer.c:1749 | hrtimer_interrupt+0x31c/0x790 kernel/time/hrtimer.c:1811 | local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1096 [inline] | __sysvec_apic_timer_interrupt+0x17c/0x640 arch/x86/kernel/apic/apic.c:1113 | sysvec_apic_timer_interrupt+0x40/0xc0 arch/x86/kernel/apic/apic.c:1107 | asm_sysvec_apic_timer_interrupt+0x16/0x20 arch/x86/include/asm/idtentry.h:649 | <...> | In this case, syzbot produced a program with event type PERF_TYPE_SOFTWARE and config PERF_COUNT_SW_CPU_CLOCK. The hrtimer manages to fire again before the IRQ work got a chance to run, all while never having returned to user space. Improve the WARN to check for real progress in user space: approximate this by storing a 32-bit hash of the current IP into pending_sigtrap, and if an event fires while pending_sigtrap still matches the previous IP, we assume no progress (false negatives are possible given we could return to user space and trigger again on the same IP). Fixes: ca6c21327c6a ("perf: Fix missing SIGTRAPs") Reported-by: syzbot+b8ded3e2e2c6adde4990@syzkaller.appspotmail.com Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://lkml.kernel.org/r/20221031093513.3032814-1-elver@google.com --- kernel/events/core.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/kernel/events/core.c b/kernel/events/core.c index 4ec3717003d55..884871427a94f 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -9306,14 +9306,27 @@ static int __perf_event_overflow(struct perf_event *event, } if (event->attr.sigtrap) { - /* - * Should not be able to return to user space without processing - * pending_sigtrap (kernel events can overflow multiple times). - */ - WARN_ON_ONCE(event->pending_sigtrap && event->attr.exclude_kernel); + unsigned int pending_id = 1; + + if (regs) + pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1; if (!event->pending_sigtrap) { - event->pending_sigtrap = 1; + event->pending_sigtrap = pending_id; local_inc(&event->ctx->nr_pending); + } else if (event->attr.exclude_kernel) { + /* + * Should not be able to return to user space without + * consuming pending_sigtrap; with exceptions: + * + * 1. Where !exclude_kernel, events can overflow again + * in the kernel without returning to user space. + * + * 2. Events that can overflow again before the IRQ- + * work without user space progress (e.g. hrtimer). + * To approximate progress (with false negatives), + * check 32-bit hash of the current IP. + */ + WARN_ON_ONCE(event->pending_sigtrap != pending_id); } event->pending_addr = data->addr; irq_work_queue(&event->pending_irq); -- GitLab From 625b74460ec0978979f883fbee117e1b97e6e35e Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Mon, 7 Nov 2022 16:54:13 +0000 Subject: [PATCH 1208/1927] drm/i915/ttm: fix uaf with lmem_userfault_list handling In the fault handler, make sure we check if the BO maps lmem after we schedule the migration, since the current resource might change from lmem to smem, if the pages are in the non-cpu visible portion of lmem. This then leads to adding the object to the lmem_userfault_list even though the current resource is no longer lmem. If we then destroy the object, the list might still contain a link to the now free object, since we only remove it if the object is still in lmem. Closes: https://gitlab.freedesktop.org/drm/intel/-/issues/7469 Fixes: ad74457a6b5a ("drm/i915/dgfx: Release mmap on rpm suspend") Signed-off-by: Matthew Auld Cc: Anshuman Gupta Cc: Rodrigo Vivi Cc: Andrzej Hajda Cc: Nirmoy Das Reviewed-by: Andrzej Hajda Link: https://patchwork.freedesktop.org/patch/msgid/20221107165414.56970-1-matthew.auld@intel.com --- drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 25129af70f70f..9421dc4dc98f6 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -1048,9 +1048,6 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) return VM_FAULT_SIGBUS; } - if (i915_ttm_cpu_maps_iomem(bo->resource)) - wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); - if (!i915_ttm_resource_mappable(bo->resource)) { int err = -ENODEV; int i; @@ -1078,6 +1075,9 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) } } + if (i915_ttm_cpu_maps_iomem(bo->resource)) + wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm); + if (drm_dev_enter(dev, &idx)) { ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, TTM_BO_VM_NUM_PREFAULT); -- GitLab From ccb0e02787d0f80d0081c446aec3756dd8f7bfe0 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Mon, 7 Nov 2022 16:54:14 +0000 Subject: [PATCH 1209/1927] drm/i915/ttm: add some sanity checks for lmem_userfault_list Rather than getting some hard to debug uaf, add some warns to hopefully catch issues with userfault_count being non-zero when destroying the object. Also if we somehow add an object to lmem_userfault_list that somehow doesn't map lmem. References: https://gitlab.freedesktop.org/drm/intel/-/issues/7469 Signed-off-by: Matthew Auld Cc: Anshuman Gupta Cc: Rodrigo Vivi Cc: Andrzej Hajda Cc: Nirmoy Das Reviewed-by: Andrzej Hajda Link: https://patchwork.freedesktop.org/patch/msgid/20221107165414.56970-2-matthew.auld@intel.com --- drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 9421dc4dc98f6..2c8b2d5ae9035 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -1098,6 +1098,8 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf) spin_lock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); list_add(&obj->userfault_link, &to_i915(obj->base.dev)->runtime_pm.lmem_userfault_list); spin_unlock(&to_i915(obj->base.dev)->runtime_pm.lmem_userfault_lock); + + GEM_WARN_ON(!i915_ttm_cpu_maps_iomem(bo->resource)); } if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) @@ -1180,6 +1182,8 @@ static void i915_ttm_unmap_virtual(struct drm_i915_gem_object *obj) } } + GEM_WARN_ON(obj->userfault_count); + ttm_bo_unmap_virtual(i915_gem_to_ttm(obj)); if (wakeref) -- GitLab From 3faf7e14ec0c3462c2d747fa6793b8645d1391df Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 7 Nov 2022 09:14:45 +0800 Subject: [PATCH 1210/1927] net: lapbether: fix issue of invalid opcode in lapbeth_open() If lapb_register() failed when lapb device goes to up for the first time, the NAPI is not disabled. As a result, the invalid opcode issue is reported when the lapb device goes to up for the second time. The stack info is as follows: [ 1958.311422][T11356] kernel BUG at net/core/dev.c:6442! [ 1958.312206][T11356] invalid opcode: 0000 [#1] PREEMPT SMP KASAN [ 1958.315979][T11356] RIP: 0010:napi_enable+0x16a/0x1f0 [ 1958.332310][T11356] Call Trace: [ 1958.332817][T11356] [ 1958.336135][T11356] lapbeth_open+0x18/0x90 [ 1958.337446][T11356] __dev_open+0x258/0x490 [ 1958.341672][T11356] __dev_change_flags+0x4d4/0x6a0 [ 1958.345325][T11356] dev_change_flags+0x93/0x160 [ 1958.346027][T11356] devinet_ioctl+0x1276/0x1bf0 [ 1958.346738][T11356] inet_ioctl+0x1c8/0x2d0 [ 1958.349638][T11356] sock_ioctl+0x5d1/0x750 [ 1958.356059][T11356] __x64_sys_ioctl+0x3ec/0x1790 [ 1958.365594][T11356] do_syscall_64+0x35/0x80 [ 1958.366239][T11356] entry_SYSCALL_64_after_hwframe+0x46/0xb0 [ 1958.377381][T11356] Fixes: 514e1150da9c ("net: x25: Queue received packets in the drivers instead of per-CPU queues") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221107011445.207372-1-shaozhengchao@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/wan/lapbether.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/wan/lapbether.c b/drivers/net/wan/lapbether.c index cb360dca32509..d62a904d2e422 100644 --- a/drivers/net/wan/lapbether.c +++ b/drivers/net/wan/lapbether.c @@ -325,6 +325,7 @@ static int lapbeth_open(struct net_device *dev) err = lapb_register(dev, &lapbeth_callbacks); if (err != LAPB_OK) { + napi_disable(&lapbeth->napi); pr_err("lapb_register error: %d\n", err); return -ENODEV; } -- GitLab From b0c09c7f08c2467b2089bdf4adb2fbbc2464f4a8 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 7 Nov 2022 09:21:59 +0800 Subject: [PATCH 1211/1927] net: ethernet: mtk-star-emac: disable napi when connect and start PHY failed in mtk_star_enable() When failed to connect to and start PHY in mtk_star_enable() for opening device, napi isn't disabled. When open mtk star device next time, it will reports a invalid opcode issue. Fix it. Only be compiled, not be tested. Fixes: 8c7bd5a454ff ("net: ethernet: mtk-star-emac: new driver") Signed-off-by: Zhengchao Shao Reviewed-by: Leon Romanovsky Link: https://lore.kernel.org/r/20221107012159.211387-1-shaozhengchao@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/mediatek/mtk_star_emac.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/ethernet/mediatek/mtk_star_emac.c b/drivers/net/ethernet/mediatek/mtk_star_emac.c index 7e890f81148e6..7050351250b7a 100644 --- a/drivers/net/ethernet/mediatek/mtk_star_emac.c +++ b/drivers/net/ethernet/mediatek/mtk_star_emac.c @@ -1026,6 +1026,8 @@ static int mtk_star_enable(struct net_device *ndev) return 0; err_free_irq: + napi_disable(&priv->rx_napi); + napi_disable(&priv->tx_napi); free_irq(ndev->irq, ndev); err_free_skbs: mtk_star_free_rx_skbs(priv); -- GitLab From f0dfc4c88ef39be0ba736aa0ce6119263fc19aeb Mon Sep 17 00:00:00 2001 From: Ratheesh Kannoth Date: Mon, 7 Nov 2022 09:05:05 +0530 Subject: [PATCH 1212/1927] octeontx2-pf: Fix SQE threshold checking Current way of checking available SQE count which is based on HW updated SQB count could result in driver submitting an SQE even before CQE for the previously transmitted SQE at the same index is processed in NAPI resulting losing SKB pointers, hence a leak. Fix this by checking a consumer index which is updated once CQE is processed. Fixes: 3ca6c4c882a7 ("octeontx2-pf: Add packet transmission support") Signed-off-by: Ratheesh Kannoth Reviewed-by: Sunil Kovvuri Goutham Link: https://lore.kernel.org/r/20221107033505.2491464-1-rkannoth@marvell.com Signed-off-by: Paolo Abeni --- .../marvell/octeontx2/nic/otx2_common.c | 1 + .../marvell/octeontx2/nic/otx2_txrx.c | 32 +++++++++++-------- .../marvell/octeontx2/nic/otx2_txrx.h | 1 + 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 9ac9e6615ae71..9e10e7471b887 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -898,6 +898,7 @@ static int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura) } sq->head = 0; + sq->cons_head = 0; sq->sqe_per_sqb = (pfvf->hw.sqb_size / sq->sqe_size) - 1; sq->num_sqbs = (qset->sqe_cnt + sq->sqe_per_sqb) / sq->sqe_per_sqb; /* Set SQE threshold to 10% of total SQEs */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c index 5ec11d71bf606..ef10aef3cda02 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c @@ -441,6 +441,7 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int budget) { int tx_pkts = 0, tx_bytes = 0, qidx; + struct otx2_snd_queue *sq; struct nix_cqe_tx_s *cqe; int processed_cqe = 0; @@ -451,6 +452,9 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, return 0; process_cqe: + qidx = cq->cq_idx - pfvf->hw.rx_queues; + sq = &pfvf->qset.sq[qidx]; + while (likely(processed_cqe < budget) && cq->pend_cqe) { cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq); if (unlikely(!cqe)) { @@ -458,18 +462,20 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf, return 0; break; } + if (cq->cq_type == CQ_XDP) { - qidx = cq->cq_idx - pfvf->hw.rx_queues; - otx2_xdp_snd_pkt_handler(pfvf, &pfvf->qset.sq[qidx], - cqe); + otx2_xdp_snd_pkt_handler(pfvf, sq, cqe); } else { - otx2_snd_pkt_handler(pfvf, cq, - &pfvf->qset.sq[cq->cint_idx], - cqe, budget, &tx_pkts, &tx_bytes); + otx2_snd_pkt_handler(pfvf, cq, sq, cqe, budget, + &tx_pkts, &tx_bytes); } + cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID; processed_cqe++; cq->pend_cqe--; + + sq->cons_head++; + sq->cons_head &= (sq->sqe_cnt - 1); } /* Free CQEs to HW */ @@ -1072,17 +1078,17 @@ bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq, { struct netdev_queue *txq = netdev_get_tx_queue(netdev, qidx); struct otx2_nic *pfvf = netdev_priv(netdev); - int offset, num_segs, free_sqe; + int offset, num_segs, free_desc; struct nix_sqe_hdr_s *sqe_hdr; - /* Check if there is room for new SQE. - * 'Num of SQBs freed to SQ's pool - SQ's Aura count' - * will give free SQE count. + /* Check if there is enough room between producer + * and consumer index. */ - free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb; + free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1); + if (free_desc < sq->sqe_thresh) + return false; - if (free_sqe < sq->sqe_thresh || - free_sqe < otx2_get_sqe_count(pfvf, skb)) + if (free_desc < otx2_get_sqe_count(pfvf, skb)) return false; num_segs = skb_shinfo(skb)->nr_frags + 1; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h index fbe62bbfb789a..93cac2c2664c2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h @@ -79,6 +79,7 @@ struct sg_list { struct otx2_snd_queue { u8 aura_id; u16 head; + u16 cons_head; u16 sqe_size; u32 sqe_cnt; u16 num_sqbs; -- GitLab From e0b26b9482461e9528552f54fa662c2269f75b3f Mon Sep 17 00:00:00 2001 From: Ma Jun Date: Wed, 14 Sep 2022 20:53:31 +0800 Subject: [PATCH 1213/1927] drm/amdgpu: Fix the lpfn checking condition in drm buddy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because the value of man->size is changed during suspend/resume process, use mgr->mm.size instead of man->size here for lpfn checking. Signed-off-by: Ma Jun Suggested-by: Christian König Link: https://patchwork.freedesktop.org/patch/msgid/20220914125331.2467162-1-Jun.Ma2@amd.com Signed-off-by: Christian König --- drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 73a517bcf5c11..80dd1343594c7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -435,7 +435,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, if (place->flags & TTM_PL_FLAG_TOPDOWN) vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; - if (fpfn || lpfn != man->size) + if (fpfn || lpfn != mgr->mm.size) /* Allocate blocks in desired range */ vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; -- GitLab From f04a2862f9c3f64962b8709c75d788efba6df26b Mon Sep 17 00:00:00 2001 From: Anjana Hari Date: Thu, 3 Nov 2022 11:10:51 -0700 Subject: [PATCH 1214/1927] pinctrl: qcom: sc8280xp: Rectify UFS reset pins UFS reset pin offsets are wrongly configured for SC8280XP, correcting the same for both UFS instances here. Signed-off-by: Anjana Hari Signed-off-by: Bjorn Andersson Reviewed-by: Andrew Halaney Tested-by: Andrew Halaney # QDrive3 Link: https://lore.kernel.org/r/20221103181051.26912-1-quic_bjorande@quicinc.com Signed-off-by: Linus Walleij --- drivers/pinctrl/qcom/pinctrl-sc8280xp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c index aa2075390f3eb..e96c00686a25b 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c +++ b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c @@ -1873,8 +1873,8 @@ static const struct msm_pingroup sc8280xp_groups[] = { [225] = PINGROUP(225, hs3_mi2s, phase_flag, _, _, _, _, egpio), [226] = PINGROUP(226, hs3_mi2s, phase_flag, _, _, _, _, egpio), [227] = PINGROUP(227, hs3_mi2s, phase_flag, _, _, _, _, egpio), - [228] = UFS_RESET(ufs_reset, 0xf1004), - [229] = UFS_RESET(ufs1_reset, 0xf3004), + [228] = UFS_RESET(ufs_reset, 0xf1000), + [229] = UFS_RESET(ufs1_reset, 0xf3000), [230] = SDC_QDSD_PINGROUP(sdc2_clk, 0xe8000, 14, 6), [231] = SDC_QDSD_PINGROUP(sdc2_cmd, 0xe8000, 11, 3), [232] = SDC_QDSD_PINGROUP(sdc2_data, 0xe8000, 9, 0), -- GitLab From 2f01a612d4758b45f775dbb88a49cf534ba47275 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Tue, 8 Nov 2022 15:07:21 +0100 Subject: [PATCH 1215/1927] ALSA: usb-audio: Add quirk entry for M-Audio Micro M-Audio Micro (0762:201a) defines the descriptor as vendor-specific, while the content seems class-compliant. Just overriding the probe makes the device working. Reported-by: Ash Logan Cc: Link: https://lore.kernel.org/r/7ecd4417-d860-4773-c1c1-b07433342390@heyquark.com Link: https://lore.kernel.org/r/20221108140721.24248-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/usb/quirks-table.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index 06dfdd45cff8c..874fcf245747f 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -2049,6 +2049,10 @@ YAMAHA_DEVICE(0x7010, "UB99"), } } }, +{ + /* M-Audio Micro */ + USB_DEVICE_VENDOR_SPEC(0x0763, 0x201a), +}, { USB_DEVICE_VENDOR_SPEC(0x0763, 0x2030), .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { -- GitLab From f02be9002c480cd3ec0fcf184ad27cf531bd6ece Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Tue, 8 Nov 2022 18:34:34 +0800 Subject: [PATCH 1216/1927] block, bfq: fix null pointer dereference in bfq_bio_bfqg() Out test found a following problem in kernel 5.10, and the same problem should exist in mainline: BUG: kernel NULL pointer dereference, address: 0000000000000094 PGD 0 P4D 0 Oops: 0000 [#1] SMP CPU: 7 PID: 155 Comm: kworker/7:1 Not tainted 5.10.0-01932-g19e0ace2ca1d-dirty 4 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS ?-20190727_073836-b4 Workqueue: kthrotld blk_throtl_dispatch_work_fn RIP: 0010:bfq_bio_bfqg+0x52/0xc0 Code: 94 00 00 00 00 75 2e 48 8b 40 30 48 83 05 35 06 c8 0b 01 48 85 c0 74 3d 4b RSP: 0018:ffffc90001a1fba0 EFLAGS: 00010002 RAX: ffff888100d60400 RBX: ffff8881132e7000 RCX: 0000000000000000 RDX: 0000000000000017 RSI: ffff888103580a18 RDI: ffff888103580a18 RBP: ffff8881132e7000 R08: 0000000000000000 R09: ffffc90001a1fe10 R10: 0000000000000a20 R11: 0000000000034320 R12: 0000000000000000 R13: ffff888103580a18 R14: ffff888114447000 R15: 0000000000000000 FS: 0000000000000000(0000) GS:ffff88881fdc0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000094 CR3: 0000000100cdb000 CR4: 00000000000006e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: bfq_bic_update_cgroup+0x3c/0x350 ? ioc_create_icq+0x42/0x270 bfq_init_rq+0xfd/0x1060 bfq_insert_requests+0x20f/0x1cc0 ? ioc_create_icq+0x122/0x270 blk_mq_sched_insert_requests+0x86/0x1d0 blk_mq_flush_plug_list+0x193/0x2a0 blk_flush_plug_list+0x127/0x170 blk_finish_plug+0x31/0x50 blk_throtl_dispatch_work_fn+0x151/0x190 process_one_work+0x27c/0x5f0 worker_thread+0x28b/0x6b0 ? rescuer_thread+0x590/0x590 kthread+0x153/0x1b0 ? kthread_flush_work+0x170/0x170 ret_from_fork+0x1f/0x30 Modules linked in: CR2: 0000000000000094 ---[ end trace e2e59ac014314547 ]--- RIP: 0010:bfq_bio_bfqg+0x52/0xc0 Code: 94 00 00 00 00 75 2e 48 8b 40 30 48 83 05 35 06 c8 0b 01 48 85 c0 74 3d 4b RSP: 0018:ffffc90001a1fba0 EFLAGS: 00010002 RAX: ffff888100d60400 RBX: ffff8881132e7000 RCX: 0000000000000000 RDX: 0000000000000017 RSI: ffff888103580a18 RDI: ffff888103580a18 RBP: ffff8881132e7000 R08: 0000000000000000 R09: ffffc90001a1fe10 R10: 0000000000000a20 R11: 0000000000034320 R12: 0000000000000000 R13: ffff888103580a18 R14: ffff888114447000 R15: 0000000000000000 FS: 0000000000000000(0000) GS:ffff88881fdc0000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 0000000000000094 CR3: 0000000100cdb000 CR4: 00000000000006e0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Root cause is quite complex: 1) use bfq elevator for the test device. 2) create a cgroup CG 3) config blk throtl in CG blkg_conf_prep blkg_create 4) create a thread T1 and issue async io in CG: bio_init bio_associate_blkg ... submit_bio submit_bio_noacct blk_throtl_bio -> io is throttled // io submit is done 5) switch elevator: bfq_exit_queue blkcg_deactivate_policy list_for_each_entry(blkg, &q->blkg_list, q_node) blkg->pd[] = NULL // bfq policy is removed 5) thread t1 exist, then remove the cgroup CG: blkcg_unpin_online blkcg_destroy_blkgs blkg_destroy list_del_init(&blkg->q_node) // blkg is removed from queue list 6) switch elevator back to bfq bfq_init_queue bfq_create_group_hierarchy blkcg_activate_policy list_for_each_entry_reverse(blkg, &q->blkg_list) // blkg is removed from list, hence bfq policy is still NULL 7) throttled io is dispatched to bfq: bfq_insert_requests bfq_init_rq bfq_bic_update_cgroup bfq_bio_bfqg bfqg = blkg_to_bfqg(blkg) // bfqg is NULL because bfq policy is NULL The problem is only possible in bfq because only bfq can be deactivated and activated while queue is online, while others can only be deactivated while the device is removed. Fix the problem in bfq by checking if blkg is online before calling blkg_to_bfqg(). Signed-off-by: Yu Kuai Reviewed-by: Jan Kara Link: https://lore.kernel.org/r/20221108103434.2853269-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index 144bca006463d..7d624a3a3f0f8 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -610,6 +610,10 @@ struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio) struct bfq_group *bfqg; while (blkg) { + if (!blkg->online) { + blkg = blkg->parent; + continue; + } bfqg = blkg_to_bfqg(blkg); if (bfqg->online) { bio_associate_blkg_from_css(bio, &blkg->blkcg->css); -- GitLab From f829230dd51974c1f4478900ed30bb77ba530b40 Mon Sep 17 00:00:00 2001 From: Serge Semin Date: Mon, 7 Nov 2022 23:39:44 +0300 Subject: [PATCH 1217/1927] block: sed-opal: kmalloc the cmd/resp buffers In accordance with [1] the DMA-able memory buffers must be cacheline-aligned otherwise the cache writing-back and invalidation performed during the mapping may cause the adjacent data being lost. It's specifically required for the DMA-noncoherent platforms [2]. Seeing the opal_dev.{cmd,resp} buffers are implicitly used for DMAs in the NVME and SCSI/SD drivers in framework of the nvme_sec_submit() and sd_sec_submit() methods respectively they must be cacheline-aligned to prevent the denoted problem. One of the option to guarantee that is to kmalloc the buffers [2]. Let's explicitly allocate them then instead of embedding into the opal_dev structure instance. Note this fix was inspired by the commit c94b7f9bab22 ("nvme-hwmon: kmalloc the NVME SMART log buffer"). [1] Documentation/core-api/dma-api.rst [2] Documentation/core-api/dma-api-howto.rst Fixes: 455a7b238cd6 ("block: Add Sed-opal library") Signed-off-by: Serge Semin Reviewed-by: Christoph Hellwig Link: https://lore.kernel.org/r/20221107203944.31686-1-Sergey.Semin@baikalelectronics.ru Signed-off-by: Jens Axboe --- block/sed-opal.c | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/block/sed-opal.c b/block/sed-opal.c index 2c5327a0543a6..9bdb833e5817d 100644 --- a/block/sed-opal.c +++ b/block/sed-opal.c @@ -87,8 +87,8 @@ struct opal_dev { u64 lowest_lba; size_t pos; - u8 cmd[IO_BUFFER_LENGTH]; - u8 resp[IO_BUFFER_LENGTH]; + u8 *cmd; + u8 *resp; struct parsed_resp parsed; size_t prev_d_len; @@ -2175,6 +2175,8 @@ void free_opal_dev(struct opal_dev *dev) return; clean_opal_dev(dev); + kfree(dev->resp); + kfree(dev->cmd); kfree(dev); } EXPORT_SYMBOL(free_opal_dev); @@ -2187,6 +2189,18 @@ struct opal_dev *init_opal_dev(void *data, sec_send_recv *send_recv) if (!dev) return NULL; + /* + * Presumably DMA-able buffers must be cache-aligned. Kmalloc makes + * sure the allocated buffer is DMA-safe in that regard. + */ + dev->cmd = kmalloc(IO_BUFFER_LENGTH, GFP_KERNEL); + if (!dev->cmd) + goto err_free_dev; + + dev->resp = kmalloc(IO_BUFFER_LENGTH, GFP_KERNEL); + if (!dev->resp) + goto err_free_cmd; + INIT_LIST_HEAD(&dev->unlk_lst); mutex_init(&dev->dev_lock); dev->flags = 0; @@ -2194,11 +2208,21 @@ struct opal_dev *init_opal_dev(void *data, sec_send_recv *send_recv) dev->send_recv = send_recv; if (check_opal_support(dev) != 0) { pr_debug("Opal is not supported on this device\n"); - kfree(dev); - return NULL; + goto err_free_resp; } return dev; + +err_free_resp: + kfree(dev->resp); + +err_free_cmd: + kfree(dev->cmd); + +err_free_dev: + kfree(dev); + + return NULL; } EXPORT_SYMBOL(init_opal_dev); -- GitLab From ce9e57feeed81d17d5e80ed86f516ff0d39c3867 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 7 Nov 2022 12:30:32 +0800 Subject: [PATCH 1218/1927] drivers: net: xgene: disable napi when register irq failed in xgene_enet_open() When failed to register irq in xgene_enet_open() for opening device, napi isn't disabled. When open xgene device next time, it will reports a invalid opcode issue. Fix it. Only be compiled, not be tested. Fixes: aeb20b6b3f4e ("drivers: net: xgene: fix: ifconfig up/down crash") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221107043032.357673-1-shaozhengchao@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/apm/xgene/xgene_enet_main.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c index d6cfea65a7140..390671640388b 100644 --- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c +++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c @@ -1004,8 +1004,10 @@ static int xgene_enet_open(struct net_device *ndev) xgene_enet_napi_enable(pdata); ret = xgene_enet_register_irq(ndev); - if (ret) + if (ret) { + xgene_enet_napi_disable(pdata); return ret; + } if (ndev->phydev) { phy_start(ndev->phydev); -- GitLab From cd136706b4f925aa5d316642543babac90d45910 Mon Sep 17 00:00:00 2001 From: Linus Walleij Date: Mon, 7 Nov 2022 10:07:53 +0100 Subject: [PATCH 1219/1927] USB: bcma: Make GPIO explicitly optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit What the code does is to not check the return value from devm_gpiod_get() and then avoid using an erroneous GPIO descriptor with IS_ERR_OR_NULL(). This will miss real errors from the GPIO core that should not be ignored, such as probe deferral. Instead request the GPIO as explicitly optional, which means that if it doesn't exist, the descriptor returned will be NULL. Then we can add error handling and also avoid just doing this on the device tree path, and simplify the site where the optional GPIO descriptor is used. There were some problems with cleaning up this GPIO descriptor use in the past, but this is the proper way to deal with it. Cc: Rafał Miłecki Cc: Chuhong Yuan Signed-off-by: Linus Walleij Cc: stable Link: https://lore.kernel.org/r/20221107090753.1404679-1-linus.walleij@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/bcma-hcd.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/usb/host/bcma-hcd.c b/drivers/usb/host/bcma-hcd.c index 2df52f75f6b3c..7558cc4d90cc6 100644 --- a/drivers/usb/host/bcma-hcd.c +++ b/drivers/usb/host/bcma-hcd.c @@ -285,7 +285,7 @@ static void bcma_hci_platform_power_gpio(struct bcma_device *dev, bool val) { struct bcma_hcd_device *usb_dev = bcma_get_drvdata(dev); - if (IS_ERR_OR_NULL(usb_dev->gpio_desc)) + if (!usb_dev->gpio_desc) return; gpiod_set_value(usb_dev->gpio_desc, val); @@ -406,9 +406,11 @@ static int bcma_hcd_probe(struct bcma_device *core) return -ENOMEM; usb_dev->core = core; - if (core->dev.of_node) - usb_dev->gpio_desc = devm_gpiod_get(&core->dev, "vcc", - GPIOD_OUT_HIGH); + usb_dev->gpio_desc = devm_gpiod_get_optional(&core->dev, "vcc", + GPIOD_OUT_HIGH); + if (IS_ERR(usb_dev->gpio_desc)) + return dev_err_probe(&core->dev, PTR_ERR(usb_dev->gpio_desc), + "error obtaining VCC GPIO"); switch (core->id.id) { case BCMA_CORE_USB20_HOST: -- GitLab From ffb9da4a04c69567bad717707b6fdfbc4c216ef4 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Mon, 7 Nov 2022 18:45:44 -0800 Subject: [PATCH 1220/1927] usb: dwc3: gadget: Return -ESHUTDOWN on ep disable The usb_request API clearly noted that removed requests due to disabled endpoint should have -ESHUTDOWN status returned. Don't change this behavior. Fixes: b44c0e7fef51 ("usb: dwc3: gadget: conditionally remove requests") Signed-off-by: Thinh Nguyen Link: https://lore.kernel.org/r/3421859485cb32d77e2068549679a6c07a7797bc.1667875427.git.Thinh.Nguyen@synopsys.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/gadget.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 5fe2d136dff51..026d4029bda6b 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1029,7 +1029,7 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) dep->endpoint.desc = NULL; } - dwc3_remove_requests(dwc, dep, -ECONNRESET); + dwc3_remove_requests(dwc, dep, -ESHUTDOWN); dep->stream_capable = false; dep->type = 0; -- GitLab From 5c294de36e7fb3e0cba0c4e1ef9a5f57bc080d0f Mon Sep 17 00:00:00 2001 From: Johan Hovold Date: Thu, 3 Nov 2022 15:46:48 +0100 Subject: [PATCH 1221/1927] Revert "usb: dwc3: disable USB core PHY management" This reverts commit 6000b8d900cd5f52fbcd0776d0cc396e88c8c2ea. The offending commit disabled the USB core PHY management as the dwc3 already manages the PHYs in question. Unfortunately some platforms have started relying on having USB core also controlling the PHY and this is specifically currently needed on some Exynos platforms for PHY calibration or connected device may fail to enumerate. The PHY calibration was previously handled in the dwc3 driver, but to work around some issues related to how the dwc3 driver interacts with xhci (e.g. using multiple drivers) this was moved to USB core by commits 34c7ed72f4f0 ("usb: core: phy: add support for PHY calibration") and a0a465569b45 ("usb: dwc3: remove generic PHY calibrate() calls"). The same PHY obviously should not be controlled from two different places, which for example do no agree on the PHY mode or power state during suspend, but as the offending patch was backported to stable, let's revert it for now. Reported-by: Stefan Agner Link: https://lore.kernel.org/lkml/808bdba846bb60456adf10a3016911ee@agner.ch/ Fixes: 6000b8d900cd ("usb: dwc3: disable USB core PHY management") Cc: stable@vger.kernel.org Signed-off-by: Johan Hovold Tested-by: Marek Szyprowski Acked-by: Thinh Nguyen Link: https://lore.kernel.org/r/20221103144648.14197-1-johan+linaro@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/host.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/drivers/usb/dwc3/host.c b/drivers/usb/dwc3/host.c index a7154fe8206d1..f6f13e7f1ba14 100644 --- a/drivers/usb/dwc3/host.c +++ b/drivers/usb/dwc3/host.c @@ -11,13 +11,8 @@ #include #include -#include "../host/xhci-plat.h" #include "core.h" -static const struct xhci_plat_priv dwc3_xhci_plat_priv = { - .quirks = XHCI_SKIP_PHY_INIT, -}; - static void dwc3_host_fill_xhci_irq_res(struct dwc3 *dwc, int irq, char *name) { @@ -97,11 +92,6 @@ int dwc3_host_init(struct dwc3 *dwc) goto err; } - ret = platform_device_add_data(xhci, &dwc3_xhci_plat_priv, - sizeof(dwc3_xhci_plat_priv)); - if (ret) - goto err; - memset(props, 0, sizeof(struct property_entry) * ARRAY_SIZE(props)); if (dwc->usb3_lpm_capable) -- GitLab From 341fd15e2e18c24d5c738496cfc3d7a272241201 Mon Sep 17 00:00:00 2001 From: Yassine Oudjana Date: Mon, 7 Nov 2022 18:33:17 +0300 Subject: [PATCH 1222/1927] extcon: usbc-tusb320: Call the Type-C IRQ handler only if a port is registered Commit bf7571c00dca ("extcon: usbc-tusb320: Add USB TYPE-C support") added an optional Type-C interface to the driver but missed to check if it is in use when calling the IRQ handler. This causes an oops on devices currently using the old extcon interface. Check if a Type-C port is registered before calling the Type-C IRQ handler. Fixes: bf7571c00dca ("extcon: usbc-tusb320: Add USB TYPE-C support") Signed-off-by: Yassine Oudjana Reviewed-by: Marek Vasut Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20221107153317.657803-1-y.oudjana@protonmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/extcon/extcon-usbc-tusb320.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/extcon/extcon-usbc-tusb320.c b/drivers/extcon/extcon-usbc-tusb320.c index 41041ff0fadbb..2a120d8d3c272 100644 --- a/drivers/extcon/extcon-usbc-tusb320.c +++ b/drivers/extcon/extcon-usbc-tusb320.c @@ -327,7 +327,13 @@ static irqreturn_t tusb320_irq_handler(int irq, void *dev_id) return IRQ_NONE; tusb320_extcon_irq_handler(priv, reg); - tusb320_typec_irq_handler(priv, reg); + + /* + * Type-C support is optional. Only call the Type-C handler if a + * port had been registered previously. + */ + if (priv->port) + tusb320_typec_irq_handler(priv, reg); regmap_write(priv->regmap, TUSB320_REG9, reg); -- GitLab From 40bf8f162d0f95e0716e479d7db41443d931765c Mon Sep 17 00:00:00 2001 From: Rajat Khandelwal Date: Mon, 24 Oct 2022 22:46:11 +0530 Subject: [PATCH 1223/1927] usb: typec: mux: Enter safe mode only when pins need to be reconfigured There is no point to enter safe mode during DP/TBT configuration if the DP/TBT was already configured in mux. This is because safe mode is only applicable when there is a need to reconfigure the pins in order to avoid damage within/to port partner. In some chrome systems, IOM/mux is already configured before OS comes up. Thus, when driver is probed, it blindly enters safe mode due to PD negotiations but only after gfx driver lowers dp_phy_ownership, will the IOM complete safe mode and send an ack to PMC. Since, that never happens, we see IPC timeout. Hence, allow safe mode only when pin reconfiguration is not required, which makes sense. Fixes: 43d596e32276 ("usb: typec: intel_pmc_mux: Check the port status before connect") Cc: stable Signed-off-by: Rajat Khandelwal Signed-off-by: Lee Shawn C Reviewed-by: Heikki Krogerus Link: https://lore.kernel.org/r/20221024171611.181468-1-rajat.khandelwal@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/mux/intel_pmc_mux.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/usb/typec/mux/intel_pmc_mux.c b/drivers/usb/typec/mux/intel_pmc_mux.c index e1f4df7238bf4..fdbf3694e21f4 100644 --- a/drivers/usb/typec/mux/intel_pmc_mux.c +++ b/drivers/usb/typec/mux/intel_pmc_mux.c @@ -369,13 +369,24 @@ pmc_usb_mux_usb4(struct pmc_usb_port *port, struct typec_mux_state *state) return pmc_usb_command(port, (void *)&req, sizeof(req)); } -static int pmc_usb_mux_safe_state(struct pmc_usb_port *port) +static int pmc_usb_mux_safe_state(struct pmc_usb_port *port, + struct typec_mux_state *state) { u8 msg; if (IOM_PORT_ACTIVITY_IS(port->iom_status, SAFE_MODE)) return 0; + if ((IOM_PORT_ACTIVITY_IS(port->iom_status, DP) || + IOM_PORT_ACTIVITY_IS(port->iom_status, DP_MFD)) && + state->alt && state->alt->svid == USB_TYPEC_DP_SID) + return 0; + + if ((IOM_PORT_ACTIVITY_IS(port->iom_status, TBT) || + IOM_PORT_ACTIVITY_IS(port->iom_status, ALT_MODE_TBT_USB)) && + state->alt && state->alt->svid == USB_TYPEC_TBT_SID) + return 0; + msg = PMC_USB_SAFE_MODE; msg |= port->usb3_port << PMC_USB_MSG_USB3_PORT_SHIFT; @@ -443,7 +454,7 @@ pmc_usb_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state) return 0; if (state->mode == TYPEC_STATE_SAFE) - return pmc_usb_mux_safe_state(port); + return pmc_usb_mux_safe_state(port, state); if (state->mode == TYPEC_STATE_USB) return pmc_usb_connect(port, port->role); -- GitLab From 6d8fc203b28ff8f6115fbe5eaf584de8b824f4fa Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Wed, 2 Nov 2022 17:15:42 +0100 Subject: [PATCH 1224/1927] usb: typec: tipd: Prevent uninitialized event{1,2} in IRQ handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If reading TPS_REG_INT_EVENT1/2 fails in the interrupt handler event1 and event2 may be uninitialized when they are used to determine IRQ_HANDLED vs. IRQ_NONE in the error path. Fixes: c7260e29dd20 ("usb: typec: tipd: Add short-circuit for no irqs") Fixes: 45188f27b3d0 ("usb: typec: tipd: Add support for Apple CD321X") Cc: stable Signed-off-by: Sven Peter Reviewed-by: Eric Curtin Reviewed-by: Heikki Krogerus Reviewed-by: Guido Günther Link: https://lore.kernel.org/r/20221102161542.30669-1-sven@svenpeter.dev Signed-off-by: Greg Kroah-Hartman --- drivers/usb/typec/tipd/core.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index b637e8b378b3a..2a77bab948f54 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -474,7 +474,7 @@ static void tps6598x_handle_plug_event(struct tps6598x *tps, u32 status) static irqreturn_t cd321x_interrupt(int irq, void *data) { struct tps6598x *tps = data; - u64 event; + u64 event = 0; u32 status; int ret; @@ -519,8 +519,8 @@ static irqreturn_t cd321x_interrupt(int irq, void *data) static irqreturn_t tps6598x_interrupt(int irq, void *data) { struct tps6598x *tps = data; - u64 event1; - u64 event2; + u64 event1 = 0; + u64 event2 = 0; u32 status; int ret; -- GitLab From 2081b3bd0c11757725dcab9ba5d38e1bddb03459 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 3 Nov 2022 18:00:15 +0100 Subject: [PATCH 1225/1927] arm64: fix rodata=full again Commit 2e8cff0a0eee87b2 ("arm64: fix rodata=full") addressed a couple of issues with the rodata= kernel command line option, which is not a simple boolean on arm64, and inadvertently got broken due to changes in the generic bool handling. Unfortunately, the resulting code never clears the rodata_full boolean variable if it defaults to true and rodata=on or rodata=off is passed, as the generic code is not aware of the existence of this variable. Given the way this code is plumbed together, clearing rodata_full when returning false from arch_parse_debug_rodata() may result in inconsistencies if the generic code decides that it cannot parse the right hand side, so the best way to deal with this is to only take rodata_full in account if rodata_enabled is also true. Fixes: 2e8cff0a0eee ("arm64: fix rodata=full") Cc: # 6.0.x Signed-off-by: Ard Biesheuvel Acked-by: Will Deacon Link: https://lore.kernel.org/r/20221103170015.4124426-1-ardb@kernel.org Signed-off-by: Catalin Marinas --- arch/arm64/mm/pageattr.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c index d107c3d434e22..5922178d7a064 100644 --- a/arch/arm64/mm/pageattr.c +++ b/arch/arm64/mm/pageattr.c @@ -26,7 +26,7 @@ bool can_set_direct_map(void) * mapped at page granularity, so that it is possible to * protect/unprotect single pages. */ - return rodata_full || debug_pagealloc_enabled() || + return (rodata_enabled && rodata_full) || debug_pagealloc_enabled() || IS_ENABLED(CONFIG_KFENCE); } @@ -102,7 +102,8 @@ static int change_memory_common(unsigned long addr, int numpages, * If we are manipulating read-only permissions, apply the same * change to the linear mapping of the pages that back this VM area. */ - if (rodata_full && (pgprot_val(set_mask) == PTE_RDONLY || + if (rodata_enabled && + rodata_full && (pgprot_val(set_mask) == PTE_RDONLY || pgprot_val(clear_mask) == PTE_RDONLY)) { for (i = 0; i < area->nr_pages; i++) { __change_memory_common((u64)page_address(area->pages[i]), -- GitLab From 50256e4793a5e5ab77703c82a47344ad2e774a59 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 8 Nov 2022 11:23:11 -0500 Subject: [PATCH 1226/1927] nfsd: put the export reference in nfsd4_verify_deleg_dentry nfsd_lookup_dentry returns an export reference in addition to the dentry ref. Ensure that we put it too. Link: https://bugzilla.redhat.com/show_bug.cgi?id=2138866 Fixes: 876c553cb410 ("NFSD: verify the opened dentry after setting a delegation") Reported-by: Yongcheng Yang Signed-off-by: Jeff Layton Signed-off-by: Chuck Lever --- fs/nfsd/nfs4state.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 198d7abf34e45..4f4706f7139bb 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -5382,6 +5382,7 @@ nfsd4_verify_deleg_dentry(struct nfsd4_open *open, struct nfs4_file *fp, if (err) return -EAGAIN; + exp_put(exp); dput(child); if (child != file_dentry(fp->fi_deleg_file->nf_file)) return -EAGAIN; -- GitLab From eb86559a691cea5fa63e57a03ec3dc9c31e97955 Mon Sep 17 00:00:00 2001 From: Wang Yufen Date: Tue, 8 Nov 2022 13:11:31 +0800 Subject: [PATCH 1227/1927] bpf: Fix memory leaks in __check_func_call kmemleak reports this issue: unreferenced object 0xffff88817139d000 (size 2048): comm "test_progs", pid 33246, jiffies 4307381979 (age 45851.820s) hex dump (first 32 bytes): 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<0000000045f075f0>] kmalloc_trace+0x27/0xa0 [<0000000098b7c90a>] __check_func_call+0x316/0x1230 [<00000000b4c3c403>] check_helper_call+0x172e/0x4700 [<00000000aa3875b7>] do_check+0x21d8/0x45e0 [<000000001147357b>] do_check_common+0x767/0xaf0 [<00000000b5a595b4>] bpf_check+0x43e3/0x5bc0 [<0000000011e391b1>] bpf_prog_load+0xf26/0x1940 [<0000000007f765c0>] __sys_bpf+0xd2c/0x3650 [<00000000839815d6>] __x64_sys_bpf+0x75/0xc0 [<00000000946ee250>] do_syscall_64+0x3b/0x90 [<0000000000506b7f>] entry_SYSCALL_64_after_hwframe+0x63/0xcd The root case here is: In function prepare_func_exit(), the callee is not released in the abnormal scenario after "state->curframe--;". To fix, move "state->curframe--;" to the very bottom of the function, right when we free callee and reset frame[] pointer to NULL, as Andrii suggested. In addition, function __check_func_call() has a similar problem. In the abnormal scenario before "state->curframe++;", the callee also should be released by free_func_state(). Fixes: 69c087ba6225 ("bpf: Add bpf_for_each_map_elem() helper") Fixes: fd978bf7fd31 ("bpf: Add reference tracking to verifier") Signed-off-by: Wang Yufen Link: https://lore.kernel.org/r/1667884291-15666-1-git-send-email-wangyufen@huawei.com Signed-off-by: Martin KaFai Lau --- kernel/bpf/verifier.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 225666307bba3..264b3dc714cc4 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6745,11 +6745,11 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn /* Transfer references to the callee */ err = copy_reference_state(callee, caller); if (err) - return err; + goto err_out; err = set_callee_state_cb(env, caller, callee, *insn_idx); if (err) - return err; + goto err_out; clear_caller_saved_regs(env, caller->regs); @@ -6766,6 +6766,11 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn print_verifier_state(env, callee, true); } return 0; + +err_out: + free_func_state(callee); + state->frame[state->curframe + 1] = NULL; + return err; } int map_set_for_each_callback_args(struct bpf_verifier_env *env, @@ -6979,8 +6984,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EINVAL; } - state->curframe--; - caller = state->frame[state->curframe]; + caller = state->frame[state->curframe - 1]; if (callee->in_callback_fn) { /* enforce R0 return value range [0, 1]. */ struct tnum range = callee->callback_ret_range; @@ -7019,7 +7023,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) } /* clear everything in the callee */ free_func_state(callee); - state->frame[state->curframe + 1] = NULL; + state->frame[state->curframe--] = NULL; return 0; } -- GitLab From 0fc8c2acbfc789a977a50a4a9812a8e4b37958ce Mon Sep 17 00:00:00 2001 From: Dylan Yudaken Date: Tue, 8 Nov 2022 07:30:16 -0800 Subject: [PATCH 1228/1927] io_uring: calculate CQEs from the user visible value io_cqring_wait (and it's wake function io_has_work) used cached_cq_tail in order to calculate the number of CQEs. cached_cq_tail is set strictly before the user visible rings->cq.tail However as far as userspace is concerned, if io_uring_enter(2) is called with a minimum number of events, they will verify by checking rings->cq.tail. It is therefore possible for io_uring_enter(2) to return early with fewer events visible to the user. Instead make the wait functions read from the user visible value, so there will be no discrepency. This is triggered eventually by the following reproducer: struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; unsigned int cqe_ready; struct io_uring ring; int ret, i; ret = io_uring_queue_init(N, &ring, 0); assert(!ret); while(true) { for (i = 0; i < N; i++) { sqe = io_uring_get_sqe(&ring); io_uring_prep_nop(sqe); sqe->flags |= IOSQE_ASYNC; } ret = io_uring_submit(&ring); assert(ret == N); do { ret = io_uring_wait_cqes(&ring, &cqe, N, NULL, NULL); } while(ret == -EINTR); cqe_ready = io_uring_cq_ready(&ring); assert(!ret); assert(cqe_ready == N); io_uring_cq_advance(&ring, N); } Fixes: ad3eb2c89fb2 ("io_uring: split overflow state into SQ and CQ side") Signed-off-by: Dylan Yudaken Link: https://lore.kernel.org/r/20221108153016.1854297-1-dylany@meta.com Signed-off-by: Jens Axboe --- io_uring/io_uring.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index ac8c488e3077d..4a1e482747cc5 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -176,6 +176,11 @@ static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx) return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head); } +static inline unsigned int __io_cqring_events_user(struct io_ring_ctx *ctx) +{ + return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head); +} + static bool io_match_linked(struct io_kiocb *head) { struct io_kiocb *req; @@ -2315,7 +2320,7 @@ static inline bool io_has_work(struct io_ring_ctx *ctx) static inline bool io_should_wake(struct io_wait_queue *iowq) { struct io_ring_ctx *ctx = iowq->ctx; - int dist = ctx->cached_cq_tail - (int) iowq->cq_tail; + int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail; /* * Wake up if we have enough events, or if a timeout occurred since we @@ -2399,7 +2404,8 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events, return ret; io_cqring_overflow_flush(ctx); - if (io_cqring_events(ctx) >= min_events) + /* if user messes with these they will just get an early return */ + if (__io_cqring_events_user(ctx) >= min_events) return 0; } while (ret > 0); -- GitLab From 000f8870a47bdc36730357883b6aef42bced91ee Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 8 Nov 2022 10:49:34 -0700 Subject: [PATCH 1229/1927] vmlinux.lds.h: Fix placement of '.data..decrypted' section Commit d4c639990036 ("vmlinux.lds.h: Avoid orphan section with !SMP") fixed an orphan section warning by adding the '.data..decrypted' section to the linker script under the PERCPU_DECRYPTED_SECTION define but that placement introduced a panic with !SMP, as the percpu sections are not instantiated with that configuration so attempting to access variables defined with DEFINE_PER_CPU_DECRYPTED() will result in a page fault. Move the '.data..decrypted' section to the DATA_MAIN define so that the variables in it are properly instantiated at boot time with CONFIG_SMP=n. Cc: stable@vger.kernel.org Fixes: d4c639990036 ("vmlinux.lds.h: Avoid orphan section with !SMP") Link: https://lore.kernel.org/cbbd3548-880c-d2ca-1b67-5bb93b291d5f@huawei.com/ Debugged-by: Ard Biesheuvel Reported-by: Zhao Wenhui Tested-by: xiafukun Signed-off-by: Nathan Chancellor Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20221108174934.3384275-1-nathan@kernel.org --- include/asm-generic/vmlinux.lds.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index c15de165ec8ff..f40a0e078bd08 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -341,6 +341,7 @@ #define DATA_DATA \ *(.xiptext) \ *(DATA_MAIN) \ + *(.data..decrypted) \ *(.ref.data) \ *(.data..shared_aligned) /* percpu related */ \ MEM_KEEP(init.data*) \ @@ -989,7 +990,6 @@ #ifdef CONFIG_AMD_MEM_ENCRYPT #define PERCPU_DECRYPTED_SECTION \ . = ALIGN(PAGE_SIZE); \ - *(.data..decrypted) \ *(.data..percpu..decrypted) \ . = ALIGN(PAGE_SIZE); #else -- GitLab From 26c263bf1847d4dadba016a0457c4c5f446407bf Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Tue, 8 Nov 2022 09:30:52 -0800 Subject: [PATCH 1230/1927] Input: i8042 - apply probe defer to more ASUS ZenBook models There are yet a few more ASUS ZenBook models that require the deferred probe. At least, there are different ZenBook UX325x and UX425x models. Let's extend the DMI matching table entries for adapting those missing models. Signed-off-by: Takashi Iwai Link: https://lore.kernel.org/r/20221108142027.28480-1-tiwai@suse.de Signed-off-by: Dmitry Torokhov --- drivers/input/serio/i8042-acpipnpio.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/input/serio/i8042-acpipnpio.h b/drivers/input/serio/i8042-acpipnpio.h index 0778dc03cd9e0..46f8a694291ed 100644 --- a/drivers/input/serio/i8042-acpipnpio.h +++ b/drivers/input/serio/i8042-acpipnpio.h @@ -115,18 +115,18 @@ static const struct dmi_system_id i8042_dmi_quirk_table[] __initconst = { .driver_data = (void *)(SERIO_QUIRK_NOMUX | SERIO_QUIRK_RESET_NEVER) }, { - /* ASUS ZenBook UX425UA */ + /* ASUS ZenBook UX425UA/QA */ .matches = { DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), - DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX425UA"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX425"), }, .driver_data = (void *)(SERIO_QUIRK_PROBE_DEFER | SERIO_QUIRK_RESET_NEVER) }, { - /* ASUS ZenBook UM325UA */ + /* ASUS ZenBook UM325UA/QA */ .matches = { DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), - DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX325UA_UM325UA"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZenBook UX325"), }, .driver_data = (void *)(SERIO_QUIRK_PROBE_DEFER | SERIO_QUIRK_RESET_NEVER) }, -- GitLab From 6191de8b175fad1a877ae8ed4be699a5022b9c03 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 31 Oct 2022 19:11:49 +0100 Subject: [PATCH 1231/1927] s390/configs: move CONFIG_DEBUG_INFO_BTF into btf.config addon config CONFIG_DEBUG_INFO_BTF significantly increases compile time for the kernel. E.g. when changing a single C file compile time for a new bzImage is increased by ~50% if BTF debug info is generated. Therefore remove CONFIG_DEBUG_INFO_BTF from all defconfigs and introduce a btf.config addon config file. Quickly enabling CONFIG_DEBUG_INFO_BTF into the current kernel config can be done by simply invoking make btf.config Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/configs/btf.config | 1 + arch/s390/configs/debug_defconfig | 1 - arch/s390/configs/defconfig | 1 - arch/s390/configs/zfcpdump_defconfig | 1 - 4 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 arch/s390/configs/btf.config diff --git a/arch/s390/configs/btf.config b/arch/s390/configs/btf.config new file mode 100644 index 0000000000000..39227b4511aff --- /dev/null +++ b/arch/s390/configs/btf.config @@ -0,0 +1 @@ +CONFIG_DEBUG_INFO_BTF=y diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index e33dc2d6fd046..63807bd0b5364 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -797,7 +797,6 @@ CONFIG_CMA_SIZE_MBYTES=0 CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y CONFIG_DEBUG_INFO_DWARF4=y -CONFIG_DEBUG_INFO_BTF=y CONFIG_GDB_SCRIPTS=y CONFIG_HEADERS_INSTALL=y CONFIG_DEBUG_SECTION_MISMATCH=y diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 9cc8f024f3dfe..4f9a982474423 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -781,7 +781,6 @@ CONFIG_CMA_SIZE_MBYTES=0 CONFIG_PRINTK_TIME=y CONFIG_DYNAMIC_DEBUG=y CONFIG_DEBUG_INFO_DWARF4=y -CONFIG_DEBUG_INFO_BTF=y CONFIG_GDB_SCRIPTS=y CONFIG_DEBUG_SECTION_MISMATCH=y CONFIG_MAGIC_SYSRQ=y diff --git a/arch/s390/configs/zfcpdump_defconfig b/arch/s390/configs/zfcpdump_defconfig index a5576b8d4081e..5fe9948be6442 100644 --- a/arch/s390/configs/zfcpdump_defconfig +++ b/arch/s390/configs/zfcpdump_defconfig @@ -74,7 +74,6 @@ CONFIG_PRINTK_TIME=y # CONFIG_SYMBOLIC_ERRNAME is not set CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_INFO_DWARF4=y -CONFIG_DEBUG_INFO_BTF=y CONFIG_DEBUG_FS=y CONFIG_PANIC_ON_OOPS=y # CONFIG_SCHED_DEBUG is not set -- GitLab From 9afea696a04af29b114b38aec734e28b03e93dae Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 31 Oct 2022 19:40:31 +0100 Subject: [PATCH 1232/1927] s390/configs: add kasan.config addon config file Add kasan.config addon config file which allows to easily enable KASAN into the current kernel config. Reviewed-by: Alexander Gordeev Signed-off-by: Heiko Carstens Signed-off-by: Alexander Gordeev --- arch/s390/configs/kasan.config | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 arch/s390/configs/kasan.config diff --git a/arch/s390/configs/kasan.config b/arch/s390/configs/kasan.config new file mode 100644 index 0000000000000..700a8b25c3ffc --- /dev/null +++ b/arch/s390/configs/kasan.config @@ -0,0 +1,3 @@ +CONFIG_KASAN=y +CONFIG_KASAN_INLINE=y +CONFIG_KASAN_VMALLOC=y -- GitLab From 80ddf5ce1c9291cb175d52ed1227134ad48c47ee Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Sun, 30 Oct 2022 19:22:02 +0100 Subject: [PATCH 1233/1927] s390: always build relocatable kernel Nathan Chancellor reported several link errors on s390 with CONFIG_RELOCATABLE disabled, after binutils commit 906f69cf65da ("IBM zSystems: Issue error for *DBL relocs on misaligned symbols"). The binutils commit reveals potential miscompiles that might have happened already before with linker script defined symbols at odd addresses. A similar bug was recently fixed in the kernel with commit c9305b6c1f52 ("s390: fix nospec table alignments"). See https://github.com/ClangBuiltLinux/linux/issues/1747 for an analysis from Ulich Weigand. Therefore always build a relocatable kernel to avoid this problem. There is hardly any use-case for non-relocatable kernels, so this shouldn't be controversial. Link: https://github.com/ClangBuiltLinux/linux/issues/1747 Signed-off-by: Heiko Carstens Reported-by: Nathan Chancellor Tested-by: Nathan Chancellor Link: https://lore.kernel.org/r/20221030182202.2062705-1-hca@linux.ibm.com Signed-off-by: Alexander Gordeev --- arch/s390/Kconfig | 6 +++--- arch/s390/Makefile | 2 -- arch/s390/boot/Makefile | 3 +-- arch/s390/boot/startup.c | 3 +-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 318fce77601d3..de575af02ffea 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -568,8 +568,7 @@ config EXPOLINE_FULL endchoice config RELOCATABLE - bool "Build a relocatable kernel" - default y + def_bool y help This builds a kernel image that retains relocation information so it can be loaded at an arbitrary address. @@ -578,10 +577,11 @@ config RELOCATABLE bootup process. The relocations make the kernel image about 15% larger (compressed 10%), but are discarded at runtime. + Note: this option exists only for documentation purposes, please do + not remove it. config RANDOMIZE_BASE bool "Randomize the address of the kernel image (KASLR)" - depends on RELOCATABLE default y help In support of Kernel Address Space Layout Randomization (KASLR), diff --git a/arch/s390/Makefile b/arch/s390/Makefile index de6d8b2ea4d8f..b3235ab0ace83 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -14,10 +14,8 @@ KBUILD_AFLAGS_MODULE += -fPIC KBUILD_CFLAGS_MODULE += -fPIC KBUILD_AFLAGS += -m64 KBUILD_CFLAGS += -m64 -ifeq ($(CONFIG_RELOCATABLE),y) KBUILD_CFLAGS += -fPIE LDFLAGS_vmlinux := -pie -endif aflags_dwarf := -Wa,-gdwarf-2 KBUILD_AFLAGS_DECOMPRESSOR := $(CLANG_FLAGS) -m64 -D__ASSEMBLY__ ifndef CONFIG_AS_IS_LLVM diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index 883357a211a3b..d52c3e2e16bc5 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -37,9 +37,8 @@ CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char obj-y := head.o als.o startup.o mem_detect.o ipl_parm.o ipl_report.o obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o -obj-y += version.o pgm_check_info.o ctype.o ipl_data.o +obj-y += version.o pgm_check_info.o ctype.o ipl_data.o machine_kexec_reloc.o obj-$(findstring y, $(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) $(CONFIG_PGSTE)) += uv.o -obj-$(CONFIG_RELOCATABLE) += machine_kexec_reloc.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o obj-y += $(if $(CONFIG_KERNEL_UNCOMPRESSED),,decompressor.o) info.o obj-$(CONFIG_KERNEL_ZSTD) += clz_ctz.o diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 6e7f01ca53e6d..47ca3264c0230 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -291,8 +291,7 @@ void startup_kernel(void) clear_bss_section(); copy_bootdata(); - if (IS_ENABLED(CONFIG_RELOCATABLE)) - handle_relocs(__kaslr_offset); + handle_relocs(__kaslr_offset); if (__kaslr_offset) { /* -- GitLab From d68cc25b7c7fb3034c5a5b5f350a0b858c6d5a45 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Sun, 6 Nov 2022 22:48:04 +0100 Subject: [PATCH 1234/1927] usb: dwc3: Do not get extcon device when usb-role-switch is used The change breaks device tree based platforms with PHY device and use usb-role-switch instead of an extcon switch. extcon_find_edev_by_node() will return EPROBE_DEFER if it can not find a device so probing without an extcon device will be deferred indefinitely. Fix this by explicitly checking for usb-role-switch. At least the out-of-tree USB3 support on Apple silicon based platforms using dwc3 with tipd USB Type-C and PD controller is affected by this issue. Fixes: d182c2e1bc92 ("usb: dwc3: Don't switch OTG -> peripheral if extcon is present") Cc: stable@kernel.org Signed-off-by: Janne Grunau Acked-by: Thinh Nguyen Reviewed-by: Sven Peter Link: https://lore.kernel.org/r/20221106214804.2814-1-j@jannau.net Signed-off-by: Greg Kroah-Hartman --- drivers/usb/dwc3/core.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index c0e7c76dc5c8b..1f348bc867c22 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -1710,6 +1710,16 @@ static struct extcon_dev *dwc3_get_extcon(struct dwc3 *dwc) if (device_property_read_string(dev, "linux,extcon-name", &name) == 0) return extcon_get_extcon_dev(name); + /* + * Check explicitly if "usb-role-switch" is used since + * extcon_find_edev_by_node() can not be used to check the absence of + * an extcon device. In the absence of an device it will always return + * EPROBE_DEFER. + */ + if (IS_ENABLED(CONFIG_USB_ROLE_SWITCH) && + device_property_read_bool(dev, "usb-role-switch")) + return NULL; + /* * Try to get an extcon device from the USB PHY controller's "port" * node. Check if it has the "port" node first, to avoid printing the -- GitLab From f0861f49bd946ff94fce4f82509c45e167f63690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Borys=20Pop=C5=82awski?= Date: Wed, 5 Oct 2022 00:59:03 +0200 Subject: [PATCH 1235/1927] x86/sgx: Add overflow check in sgx_validate_offset_length() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sgx_validate_offset_length() function verifies "offset" and "length" arguments provided by userspace, but was missing an overflow check on their addition. Add it. Fixes: c6d26d370767 ("x86/sgx: Add SGX_IOC_ENCLAVE_ADD_PAGES") Signed-off-by: Borys Popławski Signed-off-by: Borislav Petkov Reviewed-by: Jarkko Sakkinen Cc: stable@vger.kernel.org # v5.11+ Link: https://lore.kernel.org/r/0d91ac79-6d84-abed-5821-4dbe59fa1a38@invisiblethingslab.com --- arch/x86/kernel/cpu/sgx/ioctl.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/kernel/cpu/sgx/ioctl.c b/arch/x86/kernel/cpu/sgx/ioctl.c index ebe79d60619f2..da8b8ea6b063d 100644 --- a/arch/x86/kernel/cpu/sgx/ioctl.c +++ b/arch/x86/kernel/cpu/sgx/ioctl.c @@ -356,6 +356,9 @@ static int sgx_validate_offset_length(struct sgx_encl *encl, if (!length || !IS_ALIGNED(length, PAGE_SIZE)) return -EINVAL; + if (offset + length < offset) + return -EINVAL; + if (offset + length - PAGE_SIZE >= encl->size) return -EINVAL; -- GitLab From 84d1b2013272947ad9b13025df89226d8fa31cc5 Mon Sep 17 00:00:00 2001 From: Namhyung Kim Date: Mon, 7 Nov 2022 13:33:06 -0800 Subject: [PATCH 1236/1927] perf stat: Fix crash with --per-node --metric-only in CSV mode The following command will get segfault due to missing aggr_header_csv for AGGR_NODE: $ sudo perf stat -a --per-node -x, --metric-only true Committer testing: Before this patch: # perf stat -a --per-node -x, --metric-only true Segmentation fault (core dumped) # After: # gdb perf -bash: gdb: command not found # perf stat -a --per-node -x, --metric-only true node,Ghz,frontend cycles idle,backend cycles idle,insn per cycle,branch-misses of all branches, N0,32,0.335,2.10,0.65,0.69,0.03,1.92, # Fixes: 86895b480a2f10c7 ("perf stat: Add --per-node agregation support") Signed-off-by: Namhyung Kim Cc: Adrian Hunter Cc: Ian Rogers Cc: James Clark Cc: Jiri Olsa Cc: Kan Liang Cc: Peter Zijlstra Cc: Xing Zhengjun Link: http://lore.kernel.org/lkml/20221107213314.3239159-2-namhyung@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/stat-display.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 5c47ee9963a7c..9ba0f08582f06 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -559,7 +559,7 @@ static void printout(struct perf_stat_config *config, struct aggr_cpu_id id, int [AGGR_CORE] = 2, [AGGR_THREAD] = 1, [AGGR_UNSET] = 0, - [AGGR_NODE] = 0, + [AGGR_NODE] = 1, }; pm = config->metric_only ? print_metric_only_csv : print_metric_csv; @@ -1124,6 +1124,7 @@ static int aggr_header_lens[] = { [AGGR_SOCKET] = 12, [AGGR_NONE] = 6, [AGGR_THREAD] = 24, + [AGGR_NODE] = 6, [AGGR_GLOBAL] = 0, }; @@ -1133,6 +1134,7 @@ static const char *aggr_header_csv[] = { [AGGR_SOCKET] = "socket,cpus", [AGGR_NONE] = "cpu,", [AGGR_THREAD] = "comm-pid,", + [AGGR_NODE] = "node,", [AGGR_GLOBAL] = "" }; -- GitLab From ad353b710c7493df3d4fc2d3a51819126bed2e81 Mon Sep 17 00:00:00 2001 From: Athira Rajeev Date: Tue, 18 Oct 2022 14:26:04 +0530 Subject: [PATCH 1237/1927] perf stat: Fix printing os->prefix in CSV metrics output 'perf stat' with CSV output option prints an extra empty string as first field in metrics output line. Sample output below: # ./perf stat -x, --per-socket -a -C 1 ls S0,1,1.78,msec,cpu-clock,1785146,100.00,0.973,CPUs utilized S0,1,26,,context-switches,1781750,100.00,0.015,M/sec S0,1,1,,cpu-migrations,1780526,100.00,0.561,K/sec S0,1,1,,page-faults,1779060,100.00,0.561,K/sec S0,1,875807,,cycles,1769826,100.00,0.491,GHz S0,1,85281,,stalled-cycles-frontend,1767512,100.00,9.74,frontend cycles idle S0,1,576839,,stalled-cycles-backend,1766260,100.00,65.86,backend cycles idle S0,1,288430,,instructions,1762246,100.00,0.33,insn per cycle ====> ,S0,1,,,,,,,2.00,stalled cycles per insn The above command line uses field separator as "," via "-x," option and per-socket option displays socket value as first field. But here the last line for "stalled cycles per insn" has "," in the beginning. Sample output using interval mode: # ./perf stat -I 1000 -x, --per-socket -a -C 1 ls 0.001813453,S0,1,1.87,msec,cpu-clock,1872052,100.00,0.002,CPUs utilized 0.001813453,S0,1,2,,context-switches,1868028,100.00,1.070,K/sec ------ 0.001813453,S0,1,85379,,instructions,1856754,100.00,0.32,insn per cycle ====> 0.001813453,,S0,1,,,,,,,1.34,stalled cycles per insn Above result also has an extra CSV separator after the timestamp. Patch addresses extra field separator in the beginning of the metric output line. The counter stats are displayed by function "perf_stat__print_shadow_stats" in code "util/stat-shadow.c". While printing the stats info for "stalled cycles per insn", function "new_line_csv" is used as new_line callback. The new_line_csv function has check for "os->prefix" and if prefix is not null, it will be printed along with cvs separator. Snippet from "new_line_csv": if (os->prefix) fprintf(os->fh, "%s%s", os->prefix, config->csv_sep); Here os->prefix gets printed followed by "," which is the cvs separator. The os->prefix is used in interval mode option ( -I ), to print time stamp on every new line. But prefix is already set to contain CSV separator when used in interval mode for CSV option. Reference: Function "static void print_interval" Snippet: sprintf(prefix, "%6lu.%09lu%s", ts->tv_sec, ts->tv_nsec, config->csv_sep); Also if prefix is not assigned (if not used with -I option), it gets set to empty string. Reference: function printout() in util/stat-display.c Snippet: .prefix = prefix ? prefix : "", Since prefix already set to contain cvs_sep in interval option, patch removes printing config->csv_sep in new_line_csv function to avoid printing extra field. After the patch: # ./perf stat -x, --per-socket -a -C 1 ls S0,1,2.04,msec,cpu-clock,2045202,100.00,1.013,CPUs utilized S0,1,2,,context-switches,2041444,100.00,979.289,/sec S0,1,0,,cpu-migrations,2040820,100.00,0.000,/sec S0,1,2,,page-faults,2040288,100.00,979.289,/sec S0,1,254589,,cycles,2036066,100.00,0.125,GHz S0,1,82481,,stalled-cycles-frontend,2032420,100.00,32.40,frontend cycles idle S0,1,113170,,stalled-cycles-backend,2031722,100.00,44.45,backend cycles idle S0,1,88766,,instructions,2030942,100.00,0.35,insn per cycle S0,1,,,,,,,1.27,stalled cycles per insn Fixes: 92a61f6412d3a09d ("perf stat: Implement CSV metrics output") Reported-by: Disha Goel Reviewed-By: Kajol Jain Signed-off-by: Athira Jajeev Tested-by: Disha Goel Cc: Andi Kleen Cc: Ian Rogers Cc: James Clark Cc: Jiri Olsa Cc: linuxppc-dev@lists.ozlabs.org Cc: Madhavan Srinivasan Cc: Michael Ellerman Cc: Nageswara R Sastry Cc: Namhyung Kim Link: https://lore.kernel.org/r/20221018085605.63834-1-atrajeev@linux.vnet.ibm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/util/stat-display.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index 9ba0f08582f06..ba66bb7fc1ca7 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -273,7 +273,7 @@ static void new_line_csv(struct perf_stat_config *config, void *ctx) fputc('\n', os->fh); if (os->prefix) - fprintf(os->fh, "%s%s", os->prefix, config->csv_sep); + fprintf(os->fh, "%s", os->prefix); aggr_printout(config, os->evsel, os->id, os->nr); for (i = 0; i < os->nfields; i++) fputs(config->csv_sep, os->fh); -- GitLab From 20ebc4a649b82e6ad892684c76ea1e8dd786d336 Mon Sep 17 00:00:00 2001 From: James Clark Date: Fri, 28 Oct 2022 13:19:13 +0100 Subject: [PATCH 1238/1927] perf test: Fix skipping branch stack sampling test Commit f4a2aade6809c657 ("perf tests powerpc: Fix branch stack sampling test to include sanity check for branch filter") added a skip if certain branch options aren't available. But the change added both -b (--branch-any) and --branch-filter options at the same time, which will always result in a failure on any platform because the arguments can't be used together. Fix this by removing -b (--branch-any) and leaving --branch-filter which already specifies 'any'. Also add warning messages to the test and perf tool. Output on x86 before this fix: $ sudo ./perf test branch 108: Check branch stack sampling : Skip After: $ sudo ./perf test branch 108: Check branch stack sampling : Ok Fixes: f4a2aade6809c657 ("perf tests powerpc: Fix branch stack sampling test to include sanity check for branch filter") Signed-off-by: James Clark Tested-by: Athira Jajeev Cc: Alexander Shishkin Cc: Anshuman.Khandual@arm.com Cc: Ingo Molnar Cc: Jiri Olsa Cc: Kajol Jain Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20221028121913.745307-1-james.clark@arm.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/tests/shell/test_brstack.sh | 5 ++++- tools/perf/util/parse-branch-options.c | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh index ec801cffae6bc..d7ff5c4b4da4c 100755 --- a/tools/perf/tests/shell/test_brstack.sh +++ b/tools/perf/tests/shell/test_brstack.sh @@ -13,7 +13,10 @@ fi # skip the test if the hardware doesn't support branch stack sampling # and if the architecture doesn't support filter types: any,save_type,u -perf record -b -o- -B --branch-filter any,save_type,u true > /dev/null 2>&1 || exit 2 +if ! perf record -o- --no-buildid --branch-filter any,save_type,u -- true > /dev/null 2>&1 ; then + echo "skip: system doesn't support filter types: any,save_type,u" + exit 2 +fi TMPDIR=$(mktemp -d /tmp/__perf_test.program.XXXXX) diff --git a/tools/perf/util/parse-branch-options.c b/tools/perf/util/parse-branch-options.c index 00588b9db474e..31faf2bb49ff1 100644 --- a/tools/perf/util/parse-branch-options.c +++ b/tools/perf/util/parse-branch-options.c @@ -102,8 +102,10 @@ parse_branch_stack(const struct option *opt, const char *str, int unset) /* * cannot set it twice, -b + --branch-filter for instance */ - if (*mode) + if (*mode) { + pr_err("Error: Can't use --branch-any (-b) with --branch-filter (-j).\n"); return -1; + } return parse_branch_str(str, mode); } -- GitLab From 94d957ae513fc420d0a5a9bac815eb49ffebb56f Mon Sep 17 00:00:00 2001 From: Donglin Peng Date: Thu, 3 Nov 2022 02:27:04 -0700 Subject: [PATCH 1239/1927] perf tools: Add the include/perf/ directory to .gitignore Commit 3af1dfdd51e06697 ("perf build: Move perf_dlfilters.h in the source tree") moved perf_dlfilters.h to the include/perf/ directory while include/perf is ignored because it has 'perf' in the name. Newly created files in the include/perf/ directory will be ignored. Testing: Before: $ touch tools/perf/include/perf/junk $ git status | grep junk $ git check-ignore -v tools/perf/include/perf/junk tools/perf/.gitignore:6:perf tools/perf/include/perf/junk After: $ git status | grep junk tools/perf/include/perf/junk $ git check-ignore -v tools/perf/include/perf/junk Add !include/perf/ to perf's .gitignore file. Fixes: 3af1dfdd51e06697 ("perf build: Move perf_dlfilters.h in the source tree") Signed-off-by: Donglin Peng Acked-by: Adrian Hunter Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20221103092704.173391-1-dolinux.peng@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore index a653311d96938..fd7a6ff9e7aae 100644 --- a/tools/perf/.gitignore +++ b/tools/perf/.gitignore @@ -4,6 +4,7 @@ PERF-GUI-VARS PERF-VERSION-FILE FEATURE-DUMP perf +!include/perf/ perf-read-vdso32 perf-read-vdsox32 perf-help -- GitLab From 03832a32bf8ff0a8305d94ddd3979835a807248f Mon Sep 17 00:00:00 2001 From: Ziyang Xuan Date: Thu, 3 Nov 2022 09:12:02 +0800 Subject: [PATCH 1240/1927] netfilter: nfnetlink: fix potential dead lock in nfnetlink_rcv_msg() When type is NFNL_CB_MUTEX and -EAGAIN error occur in nfnetlink_rcv_msg(), it does not execute nfnl_unlock(). That would trigger potential dead lock. Fixes: 50f2db9e368f ("netfilter: nfnetlink: consolidate callback types") Signed-off-by: Ziyang Xuan Signed-off-by: Florian Westphal --- net/netfilter/nfnetlink.c | 1 + 1 file changed, 1 insertion(+) diff --git a/net/netfilter/nfnetlink.c b/net/netfilter/nfnetlink.c index 9c44518cb70ff..6d18fb3468683 100644 --- a/net/netfilter/nfnetlink.c +++ b/net/netfilter/nfnetlink.c @@ -294,6 +294,7 @@ static int nfnetlink_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh, nfnl_lock(subsys_id); if (nfnl_dereference_protected(subsys_id) != ss || nfnetlink_find_client(type, ss) != nc) { + nfnl_unlock(subsys_id); err = -EAGAIN; break; } -- GitLab From 03c1f1ef1584c981935fab2fa0c45d3e43e2c235 Mon Sep 17 00:00:00 2001 From: Shigeru Yoshida Date: Thu, 3 Nov 2022 22:08:49 +0900 Subject: [PATCH 1241/1927] netfilter: Cleanup nft_net->module_list from nf_tables_exit_net() syzbot reported a warning like below [1]: WARNING: CPU: 3 PID: 9 at net/netfilter/nf_tables_api.c:10096 nf_tables_exit_net+0x71c/0x840 Modules linked in: CPU: 2 PID: 9 Comm: kworker/u8:0 Tainted: G W 6.1.0-rc3-00072-g8e5423e991e8 #47 Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.0-1.fc36 04/01/2014 Workqueue: netns cleanup_net RIP: 0010:nf_tables_exit_net+0x71c/0x840 ... Call Trace: ? __nft_release_table+0xfc0/0xfc0 ops_exit_list+0xb5/0x180 cleanup_net+0x506/0xb10 ? unregister_pernet_device+0x80/0x80 process_one_work+0xa38/0x1730 ? pwq_dec_nr_in_flight+0x2b0/0x2b0 ? rwlock_bug.part.0+0x90/0x90 ? _raw_spin_lock_irq+0x46/0x50 worker_thread+0x67e/0x10e0 ? process_one_work+0x1730/0x1730 kthread+0x2e5/0x3a0 ? kthread_complete_and_exit+0x40/0x40 ret_from_fork+0x1f/0x30 In nf_tables_exit_net(), there is a case where nft_net->commit_list is empty but nft_net->module_list is not empty. Such a case occurs with the following scenario: 1. nfnetlink_rcv_batch() is called 2. nf_tables_newset() returns -EAGAIN and NFNL_BATCH_FAILURE bit is set to status 3. nf_tables_abort() is called with NFNL_ABORT_AUTOLOAD (nft_net->commit_list is released, but nft_net->module_list is not because of NFNL_ABORT_AUTOLOAD flag) 4. Jump to replay label 5. netlink_skb_clone() fails and returns from the function (this is caused by fault injection in the reproducer of syzbot) This patch fixes this issue by calling __nf_tables_abort() when nft_net->module_list is not empty in nf_tables_exit_net(). Fixes: eb014de4fd41 ("netfilter: nf_tables: autoload modules from the abort path") Link: https://syzkaller.appspot.com/bug?id=802aba2422de4218ad0c01b46c9525cc9d4e4aa3 [1] Reported-by: syzbot+178efee9e2d7f87f5103@syzkaller.appspotmail.com Signed-off-by: Shigeru Yoshida Signed-off-by: Florian Westphal --- net/netfilter/nf_tables_api.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 76bd4d03dbda4..e7152d599d73f 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -10090,7 +10090,8 @@ static void __net_exit nf_tables_exit_net(struct net *net) struct nftables_pernet *nft_net = nft_pernet(net); mutex_lock(&nft_net->commit_mutex); - if (!list_empty(&nft_net->commit_list)) + if (!list_empty(&nft_net->commit_list) || + !list_empty(&nft_net->module_list)) __nf_tables_abort(net, NFNL_ABORT_NONE); __nft_release_tables(net); mutex_unlock(&nft_net->commit_mutex); -- GitLab From f530bc60a30bee47ff51b7fb71511fdd058b774a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:33 +1000 Subject: [PATCH 1242/1927] drm/nouveau/disp: move HDMI config into acquire + infoframe methods v2: - fix typo in sorhdmi/g84 struct initialiser (kbuild test robot) v3: - less convoluted flow control in nvkm_uoutp_mthd_acquire_tmds() (lyude) v4: - we don't support hdmi on original nv50, don't try Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 172 +++++++----------- drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 14 -- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 24 ++- drivers/gpu/drm/nouveau/include/nvif/outp.h | 5 +- drivers/gpu/drm/nouveau/nvif/outp.c | 28 ++- .../gpu/drm/nouveau/nvkm/engine/disp/g84.c | 91 +++++---- .../gpu/drm/nouveau/nvkm/engine/disp/ga102.c | 5 +- .../gpu/drm/nouveau/nvkm/engine/disp/gf119.c | 93 ++++++---- .../gpu/drm/nouveau/nvkm/engine/disp/gk104.c | 85 +++++---- .../gpu/drm/nouveau/nvkm/engine/disp/gm107.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/disp/gm200.c | 13 +- .../gpu/drm/nouveau/nvkm/engine/disp/gp100.c | 5 +- .../gpu/drm/nouveau/nvkm/engine/disp/gt215.c | 89 +++++---- .../gpu/drm/nouveau/nvkm/engine/disp/gv100.c | 95 ++++++---- .../gpu/drm/nouveau/nvkm/engine/disp/ior.h | 23 ++- .../gpu/drm/nouveau/nvkm/engine/disp/mcp77.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/disp/mcp89.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/disp/outp.h | 3 + .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 45 ----- .../gpu/drm/nouveau/nvkm/engine/disp/tu102.c | 5 +- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 74 ++++++++ 21 files changed, 503 insertions(+), 378 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 0a8404686f169..edf8996702877 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -744,123 +745,85 @@ nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc, /****************************************************************************** * HDMI *****************************************************************************/ -static void -nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) -{ - struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_disp *disp = nv50_disp(encoder->dev); - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_hdmi_pwr_v0 pwr; - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | - (0x0100 << nv_crtc->index), - }; - - nvif_mthd(&disp->disp->object, 0, &args, sizeof(args)); -} - static void nv50_hdmi_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc, struct nouveau_connector *nv_connector, struct drm_atomic_state *state, - struct drm_display_mode *mode) + struct drm_display_mode *mode, bool hda) { struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_disp *disp = nv50_disp(encoder->dev); - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_hdmi_pwr_v0 pwr; - u8 infoframes[2 * 17]; /* two frames, up to 17 bytes each */ - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | - (0x0100 << nv_crtc->index), - .pwr.state = 1, - .pwr.rekey = 56, /* binary driver, and tegra, constant */ - }; - struct drm_hdmi_info *hdmi; + struct drm_hdmi_info *hdmi = &nv_connector->base.display_info.hdmi; + union hdmi_infoframe infoframe; + const u8 rekey = 56; /* binary driver, and tegra, constant */ + u8 config, scdc = 0; u32 max_ac_packet; - union hdmi_infoframe avi_frame; - union hdmi_infoframe vendor_frame; - bool high_tmds_clock_ratio = false, scrambling = false; - u8 config; - int ret; - int size; - - if (!drm_detect_hdmi_monitor(nv_connector->edid)) - return; - - hdmi = &nv_connector->base.display_info.hdmi; - - ret = drm_hdmi_avi_infoframe_from_display_mode(&avi_frame.avi, - &nv_connector->base, mode); - if (!ret) { - drm_hdmi_avi_infoframe_quant_range(&avi_frame.avi, - &nv_connector->base, mode, - HDMI_QUANTIZATION_RANGE_FULL); - /* We have an AVI InfoFrame, populate it to the display */ - args.pwr.avi_infoframe_length - = hdmi_infoframe_pack(&avi_frame, args.infoframes, 17); - } - - ret = drm_hdmi_vendor_infoframe_from_display_mode(&vendor_frame.vendor.hdmi, - &nv_connector->base, mode); - if (!ret) { - /* We have a Vendor InfoFrame, populate it to the display */ - args.pwr.vendor_infoframe_length - = hdmi_infoframe_pack(&vendor_frame, - args.infoframes - + args.pwr.avi_infoframe_length, - 17); - } + struct { + struct nvif_outp_infoframe_v0 infoframe; + u8 data[17]; + } args; + int ret, size; max_ac_packet = mode->htotal - mode->hdisplay; - max_ac_packet -= args.pwr.rekey; + max_ac_packet -= rekey; max_ac_packet -= 18; /* constant from tegra */ - args.pwr.max_ac_packet = max_ac_packet / 32; + max_ac_packet /= 32; if (hdmi->scdc.scrambling.supported) { - high_tmds_clock_ratio = mode->clock > 340000; - scrambling = high_tmds_clock_ratio || - hdmi->scdc.scrambling.low_rates; - } + const bool high_tmds_clock_ratio = mode->clock > 340000; - args.pwr.scdc = - NV50_DISP_SOR_HDMI_PWR_V0_SCDC_SCRAMBLE * scrambling | - NV50_DISP_SOR_HDMI_PWR_V0_SCDC_DIV_BY_4 * high_tmds_clock_ratio; + ret = drm_scdc_readb(nv_encoder->i2c, SCDC_TMDS_CONFIG, &config); + if (ret < 0) { + NV_ERROR(drm, "Failure to read SCDC_TMDS_CONFIG: %d\n", ret); + return; + } - size = sizeof(args.base) - + sizeof(args.pwr) - + args.pwr.avi_infoframe_length - + args.pwr.vendor_infoframe_length; - nvif_mthd(&disp->disp->object, 0, &args, size); + config &= ~(SCDC_TMDS_BIT_CLOCK_RATIO_BY_40 | SCDC_SCRAMBLING_ENABLE); + if (high_tmds_clock_ratio || hdmi->scdc.scrambling.low_rates) + config |= SCDC_SCRAMBLING_ENABLE; + if (high_tmds_clock_ratio) + config |= SCDC_TMDS_BIT_CLOCK_RATIO_BY_40; - nv50_audio_enable(encoder, nv_crtc, nv_connector, state, mode); + ret = drm_scdc_writeb(nv_encoder->i2c, SCDC_TMDS_CONFIG, config); + if (ret < 0) + NV_ERROR(drm, "Failure to write SCDC_TMDS_CONFIG = 0x%02x: %d\n", + config, ret); - /* If SCDC is supported by the downstream monitor, update - * divider / scrambling settings to what we programmed above. - */ - if (!hdmi->scdc.scrambling.supported) - return; + if (high_tmds_clock_ratio || hdmi->scdc.scrambling.low_rates) + scdc |= NVIF_OUTP_ACQUIRE_V0_TMDS_HDMI_SCDC_SCRAMBLE; + if (high_tmds_clock_ratio) + scdc |= NVIF_OUTP_ACQUIRE_V0_TMDS_HDMI_SCDC_DIV_BY_4; + } - ret = drm_scdc_readb(nv_encoder->i2c, SCDC_TMDS_CONFIG, &config); - if (ret < 0) { - NV_ERROR(drm, "Failure to read SCDC_TMDS_CONFIG: %d\n", ret); + ret = nvif_outp_acquire_tmds(&nv_encoder->outp, nv_crtc->index, true, + max_ac_packet, rekey, scdc, hda); + if (ret) return; + + /* AVI InfoFrame. */ + args.infoframe.version = 0; + args.infoframe.head = nv_crtc->index; + + if (!drm_hdmi_avi_infoframe_from_display_mode(&infoframe.avi, &nv_connector->base, mode)) { + drm_hdmi_avi_infoframe_quant_range(&infoframe.avi, &nv_connector->base, mode, + HDMI_QUANTIZATION_RANGE_FULL); + + size = hdmi_infoframe_pack(&infoframe, args.data, 17); + } else { + size = 0; } - config &= ~(SCDC_TMDS_BIT_CLOCK_RATIO_BY_40 | SCDC_SCRAMBLING_ENABLE); - config |= SCDC_TMDS_BIT_CLOCK_RATIO_BY_40 * high_tmds_clock_ratio; - config |= SCDC_SCRAMBLING_ENABLE * scrambling; - ret = drm_scdc_writeb(nv_encoder->i2c, SCDC_TMDS_CONFIG, config); - if (ret < 0) - NV_ERROR(drm, "Failure to write SCDC_TMDS_CONFIG = 0x%02x: %d\n", - config, ret); + + nvif_outp_infoframe(&nv_encoder->outp, NVIF_OUTP_INFOFRAME_V0_AVI, &args.infoframe, size); + + /* Vendor InfoFrame. */ + if (!drm_hdmi_vendor_infoframe_from_display_mode(&infoframe.vendor.hdmi, + &nv_connector->base, mode)) + size = hdmi_infoframe_pack(&infoframe, args.data, 17); + else + size = 0; + + nvif_outp_infoframe(&nv_encoder->outp, NVIF_OUTP_INFOFRAME_V0_VSI, &args.infoframe, size); + + nv50_audio_enable(encoder, nv_crtc, nv_connector, state, mode); } /****************************************************************************** @@ -1622,7 +1585,6 @@ nv50_sor_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *st nv_encoder->update(nv_encoder, nv_crtc->index, NULL, 0, 0); nv50_audio_disable(encoder, nv_crtc); - nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc); nvif_outp_release(&nv_encoder->outp); nv_encoder->crtc = NULL; } @@ -1636,6 +1598,7 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta nv50_head_atom(drm_atomic_get_new_crtc_state(state, &nv_crtc->base)); struct drm_display_mode *mode = &asyh->state.adjusted_mode; struct nv50_disp *disp = nv50_disp(encoder->dev); + struct nvif_outp *outp = &nv_encoder->outp; struct drm_device *dev = encoder->dev; struct nouveau_drm *drm = nouveau_drm(dev); struct nouveau_connector *nv_connector; @@ -1657,7 +1620,12 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta switch (nv_encoder->dcb->type) { case DCB_OUTPUT_TMDS: - nvif_outp_acquire_tmds(&nv_encoder->outp, hda); + if (disp->disp->object.oclass == NV50_DISP || + !drm_detect_hdmi_monitor(nv_connector->edid)) + nvif_outp_acquire_tmds(outp, nv_crtc->index, false, 0, 0, 0, false); + else + nv50_hdmi_enable(encoder, nv_crtc, nv_connector, state, mode, hda); + if (nv_encoder->outp.or.link & 1) { proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_A; /* Only enable dual-link if: @@ -1673,8 +1641,6 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta } else { proto = NV507D_SOR_SET_CONTROL_PROTOCOL_SINGLE_TMDS_B; } - - nv50_hdmi_enable(&nv_encoder->base.base, nv_crtc, nv_connector, state, mode); break; case DCB_OUTPUT_LVDS: proto = NV507D_SOR_SET_CONTROL_PROTOCOL_LVDS_CUSTOM; @@ -1900,7 +1866,7 @@ nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st switch (nv_encoder->dcb->type) { case DCB_OUTPUT_TMDS: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC); - nvif_outp_acquire_tmds(&nv_encoder->outp, false); + nvif_outp_acquire_tmds(&nv_encoder->outp, false, false, 0, 0, 0, false); break; case DCB_OUTPUT_DP: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC); diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index f371fc7a1d100..6389558dda1fd 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -29,7 +29,6 @@ struct nv50_disp_scanoutpos_v0 { struct nv50_disp_mthd_v1 { __u8 version; #define NV50_DISP_MTHD_V1_SOR_HDA_ELD 0x21 -#define NV50_DISP_MTHD_V1_SOR_HDMI_PWR 0x22 #define NV50_DISP_MTHD_V1_SOR_DP_MST_LINK 0x25 #define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI 0x26 __u8 method; @@ -44,19 +43,6 @@ struct nv50_disp_sor_hda_eld_v0 { __u8 data[]; }; -struct nv50_disp_sor_hdmi_pwr_v0 { - __u8 version; - __u8 state; - __u8 max_ac_packet; - __u8 rekey; - __u8 avi_infoframe_length; - __u8 vendor_infoframe_length; -#define NV50_DISP_SOR_HDMI_PWR_V0_SCDC_SCRAMBLE (1 << 0) -#define NV50_DISP_SOR_HDMI_PWR_V0_SCDC_DIV_BY_4 (1 << 1) - __u8 scdc; - __u8 pad07[1]; -}; - struct nv50_disp_sor_dp_mst_link_v0 { __u8 version; __u8 state; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index 4fcb9e4f5d766..06ed372cbe5a1 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -13,6 +13,7 @@ union nvif_outp_args { #define NVIF_OUTP_V0_LOAD_DETECT 0x00 #define NVIF_OUTP_V0_ACQUIRE 0x01 #define NVIF_OUTP_V0_RELEASE 0x02 +#define NVIF_OUTP_V0_INFOFRAME 0x03 union nvif_outp_load_detect_args { struct nvif_outp_load_detect_v0 { @@ -37,8 +38,15 @@ union nvif_outp_acquire_args { __u8 pad04[4]; union { struct { - __u8 hda; - __u8 pad01[7]; + __u8 head; + __u8 hdmi; + __u8 hdmi_max_ac_packet; + __u8 hdmi_rekey; +#define NVIF_OUTP_ACQUIRE_V0_TMDS_HDMI_SCDC_SCRAMBLE (1 << 0) +#define NVIF_OUTP_ACQUIRE_V0_TMDS_HDMI_SCDC_DIV_BY_4 (1 << 1) + __u8 hdmi_scdc; + __u8 hdmi_hda; + __u8 pad06[2]; } tmds; struct { __u8 dual; @@ -57,4 +65,16 @@ union nvif_outp_release_args { struct nvif_outp_release_vn { } vn; }; + +union nvif_outp_infoframe_args { + struct nvif_outp_infoframe_v0 { + __u8 version; +#define NVIF_OUTP_INFOFRAME_V0_AVI 0 +#define NVIF_OUTP_INFOFRAME_V0_VSI 1 + __u8 type; + __u8 head; + __u8 pad03[5]; + __u8 data[]; + } v0; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index e247d441afc1d..dd7e603d13c9a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -2,6 +2,7 @@ #ifndef __NVIF_OUTP_H__ #define __NVIF_OUTP_H__ #include +#include struct nvif_disp; struct nvif_outp { @@ -17,8 +18,10 @@ int nvif_outp_ctor(struct nvif_disp *, const char *name, int id, struct nvif_out void nvif_outp_dtor(struct nvif_outp *); int nvif_outp_load_detect(struct nvif_outp *, u32 loadval); int nvif_outp_acquire_rgb_crt(struct nvif_outp *); -int nvif_outp_acquire_tmds(struct nvif_outp *, bool hda); +int nvif_outp_acquire_tmds(struct nvif_outp *, int head, + bool hdmi, u8 max_ac_packet, u8 rekey, u8 scdc, bool hda); int nvif_outp_acquire_lvds(struct nvif_outp *, bool dual, bool bpc8); int nvif_outp_acquire_dp(struct nvif_outp *, bool hda); void nvif_outp_release(struct nvif_outp *); +int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_v0 *, u32 size); #endif diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 58cd629657fb9..987bc3a9e7bf3 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -24,7 +24,18 @@ #include #include -#include + +int +nvif_outp_infoframe(struct nvif_outp *outp, u8 type, struct nvif_outp_infoframe_v0 *args, u32 size) +{ + int ret; + + args->type = type; + + ret = nvif_mthd(&outp->object, NVIF_OUTP_V0_INFOFRAME, args, sizeof(*args) + size); + NVIF_ERRON(ret, &outp->object, "[INFOFRAME type:%d size:%d]", type, size); + return ret; +} void nvif_outp_release(struct nvif_outp *outp) @@ -82,16 +93,25 @@ nvif_outp_acquire_lvds(struct nvif_outp *outp, bool dual, bool bpc8) } int -nvif_outp_acquire_tmds(struct nvif_outp *outp, bool hda) +nvif_outp_acquire_tmds(struct nvif_outp *outp, int head, + bool hdmi, u8 max_ac_packet, u8 rekey, u8 scdc, bool hda) { struct nvif_outp_acquire_v0 args; int ret; - args.tmds.hda = hda; + args.tmds.head = head; + args.tmds.hdmi = hdmi; + args.tmds.hdmi_max_ac_packet = max_ac_packet; + args.tmds.hdmi_rekey = rekey; + args.tmds.hdmi_scdc = scdc; + args.tmds.hdmi_hda = hda; ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_TMDS, &args); NVIF_ERRON(ret, &outp->object, - "[ACQUIRE proto:TMDS hda:%d] or:%d link:%d", args.tmds.hda, args.or, args.link); + "[ACQUIRE proto:TMDS head:%d hdmi:%d max_ac_packet:%d rekey:%d scdc:%d hda:%d]" + " or:%d link:%d", args.tmds.head, args.tmds.hdmi, args.tmds.hdmi_max_ac_packet, + args.tmds.hdmi_rekey, args.tmds.hdmi_scdc, args.tmds.hdmi_hda, + args.or, args.link); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c index 4966a51af3d7f..23ae451ba473d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/g84.c @@ -29,9 +29,54 @@ #include -void -g84_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, - u8 rekey, u8 *avi, u8 avi_size, u8 *vendor, u8 vendor_size) +static void +g84_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe vsi; + const u32 hoff = head * 0x800; + + nvkm_mask(device, 0x61653c + hoff, 0x00010001, 0x00010000); + if (!size) + return; + + pack_hdmi_infoframe(&vsi, data, size); + + nvkm_wr32(device, 0x616544 + hoff, vsi.header); + nvkm_wr32(device, 0x616548 + hoff, vsi.subpack0_low); + nvkm_wr32(device, 0x61654c + hoff, vsi.subpack0_high); + /* Is there a second (or up to fourth?) set of subpack registers here? */ + /* nvkm_wr32(device, 0x616550 + hoff, vsi.subpack1_low); */ + /* nvkm_wr32(device, 0x616554 + hoff, vsi.subpack1_high); */ + + nvkm_mask(device, 0x61653c + hoff, 0x00010001, 0x00010001); +} + +static void +g84_sor_hdmi_infoframe_avi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe avi; + const u32 hoff = head * 0x800; + + pack_hdmi_infoframe(&avi, data, size); + + nvkm_mask(device, 0x616520 + hoff, 0x00000001, 0x00000000); + if (!size) + return; + + nvkm_wr32(device, 0x616528 + hoff, avi.header); + nvkm_wr32(device, 0x61652c + hoff, avi.subpack0_low); + nvkm_wr32(device, 0x616530 + hoff, avi.subpack0_high); + nvkm_wr32(device, 0x616534 + hoff, avi.subpack1_low); + nvkm_wr32(device, 0x616538 + hoff, avi.subpack1_high); + + nvkm_mask(device, 0x616520 + hoff, 0x00000001, 0x00000001); +} + + +static void +g84_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, u8 rekey) { struct nvkm_device *device = ior->disp->engine.subdev.device; const u32 ctrl = 0x40000000 * enable | @@ -39,31 +84,13 @@ g84_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, max_ac_packet << 16 | rekey; const u32 hoff = head * 0x800; - struct packed_hdmi_infoframe avi_infoframe; - struct packed_hdmi_infoframe vendor_infoframe; - - pack_hdmi_infoframe(&avi_infoframe, avi, avi_size); - pack_hdmi_infoframe(&vendor_infoframe, vendor, vendor_size); if (!(ctrl & 0x40000000)) { nvkm_mask(device, 0x6165a4 + hoff, 0x40000000, 0x00000000); - nvkm_mask(device, 0x61653c + hoff, 0x00000001, 0x00000000); - nvkm_mask(device, 0x616520 + hoff, 0x00000001, 0x00000000); nvkm_mask(device, 0x616500 + hoff, 0x00000001, 0x00000000); return; } - /* AVI InfoFrame */ - nvkm_mask(device, 0x616520 + hoff, 0x00000001, 0x00000000); - if (avi_size) { - nvkm_wr32(device, 0x616528 + hoff, avi_infoframe.header); - nvkm_wr32(device, 0x61652c + hoff, avi_infoframe.subpack0_low); - nvkm_wr32(device, 0x616530 + hoff, avi_infoframe.subpack0_high); - nvkm_wr32(device, 0x616534 + hoff, avi_infoframe.subpack1_low); - nvkm_wr32(device, 0x616538 + hoff, avi_infoframe.subpack1_high); - nvkm_mask(device, 0x616520 + hoff, 0x00000001, 0x00000001); - } - /* Audio InfoFrame */ nvkm_mask(device, 0x616500 + hoff, 0x00000001, 0x00000000); nvkm_wr32(device, 0x616508 + hoff, 0x000a0184); @@ -71,17 +98,6 @@ g84_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, nvkm_wr32(device, 0x616510 + hoff, 0x00000000); nvkm_mask(device, 0x616500 + hoff, 0x00000001, 0x00000001); - /* Vendor InfoFrame */ - nvkm_mask(device, 0x61653c + hoff, 0x00010001, 0x00010000); - if (vendor_size) { - nvkm_wr32(device, 0x616544 + hoff, vendor_infoframe.header); - nvkm_wr32(device, 0x616548 + hoff, vendor_infoframe.subpack0_low); - nvkm_wr32(device, 0x61654c + hoff, vendor_infoframe.subpack0_high); - /* Is there a second (or up to fourth?) set of subpack registers here? */ - /* nvkm_wr32(device, 0x616550 + hoff, vendor_infoframe->subpack1_low); */ - /* nvkm_wr32(device, 0x616554 + hoff, vendor_infoframe->subpack1_high); */ - nvkm_mask(device, 0x61653c + hoff, 0x00010001, 0x00010001); - } nvkm_mask(device, 0x6165d0 + hoff, 0x00070001, 0x00010001); /* SPARE, HW_CTS */ nvkm_mask(device, 0x616568 + hoff, 0x00010101, 0x00000000); /* ACR_CTRL, ?? */ @@ -96,14 +112,19 @@ g84_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, nvkm_mask(device, 0x6165a4 + hoff, 0x5f1f007f, ctrl); } +const struct nvkm_ior_func_hdmi +g84_sor_hdmi = { + .ctrl = g84_sor_hdmi_ctrl, + .infoframe_avi = g84_sor_hdmi_infoframe_avi, + .infoframe_vsi = g84_sor_hdmi_infoframe_vsi, +}; + static const struct nvkm_ior_func g84_sor = { .state = nv50_sor_state, .power = nv50_sor_power, .clock = nv50_sor_clock, - .hdmi = { - .ctrl = g84_sor_hdmi_ctrl, - }, + .hdmi = &g84_sor_hdmi, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c index 7489d0d7fce0c..52099b75f52ae 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c @@ -105,10 +105,7 @@ ga102_sor = { .state = gv100_sor_state, .power = nv50_sor_power, .clock = ga102_sor_clock, - .hdmi = { - .ctrl = gv100_sor_hdmi_ctrl, - .scdc = gm200_sor_hdmi_scdc, - }, + .hdmi = &gv100_sor_hdmi, .dp = &ga102_sor_dp, .hda = &gv100_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c index 39822f1b5b950..a48e9bdf4cd07 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c @@ -202,19 +202,61 @@ gf119_sor_dp = { }; static void -gf119_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, - u8 rekey, u8 *avi, u8 avi_size, u8 *vendor, u8 vendor_size) +gf119_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe vsi; + const u32 hoff = head * 0x800; + + pack_hdmi_infoframe(&vsi, data, size); + + nvkm_mask(device, 0x616730 + hoff, 0x00010001, 0x00010000); + if (!size) + return; + + /* + * These appear to be the audio infoframe registers, + * but no other set of infoframe registers has yet + * been found. + */ + nvkm_wr32(device, 0x616738 + hoff, vsi.header); + nvkm_wr32(device, 0x61673c + hoff, vsi.subpack0_low); + nvkm_wr32(device, 0x616740 + hoff, vsi.subpack0_high); + /* Is there a second (or further?) set of subpack registers here? */ + + nvkm_mask(device, 0x616730 + hoff, 0x00000001, 0x00000001); +} + +static void +gf119_sor_hdmi_infoframe_avi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe avi; + const u32 hoff = head * 0x800; + + pack_hdmi_infoframe(&avi, data, size); + + nvkm_mask(device, 0x616714 + hoff, 0x00000001, 0x00000000); + if (!size) + return; + + nvkm_wr32(device, 0x61671c + hoff, avi.header); + nvkm_wr32(device, 0x616720 + hoff, avi.subpack0_low); + nvkm_wr32(device, 0x616724 + hoff, avi.subpack0_high); + nvkm_wr32(device, 0x616728 + hoff, avi.subpack1_low); + nvkm_wr32(device, 0x61672c + hoff, avi.subpack1_high); + + nvkm_mask(device, 0x616714 + hoff, 0x00000001, 0x00000001); +} + +static void +gf119_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, u8 rekey) { struct nvkm_device *device = ior->disp->engine.subdev.device; const u32 ctrl = 0x40000000 * enable | max_ac_packet << 16 | rekey; const u32 hoff = head * 0x800; - struct packed_hdmi_infoframe avi_infoframe; - struct packed_hdmi_infoframe vendor_infoframe; - - pack_hdmi_infoframe(&avi_infoframe, avi, avi_size); - pack_hdmi_infoframe(&vendor_infoframe, vendor, vendor_size); if (!(ctrl & 0x40000000)) { nvkm_mask(device, 0x616798 + hoff, 0x40000000, 0x00000000); @@ -224,32 +266,6 @@ gf119_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe return; } - /* AVI InfoFrame */ - nvkm_mask(device, 0x616714 + hoff, 0x00000001, 0x00000000); - if (avi_size) { - nvkm_wr32(device, 0x61671c + hoff, avi_infoframe.header); - nvkm_wr32(device, 0x616720 + hoff, avi_infoframe.subpack0_low); - nvkm_wr32(device, 0x616724 + hoff, avi_infoframe.subpack0_high); - nvkm_wr32(device, 0x616728 + hoff, avi_infoframe.subpack1_low); - nvkm_wr32(device, 0x61672c + hoff, avi_infoframe.subpack1_high); - nvkm_mask(device, 0x616714 + hoff, 0x00000001, 0x00000001); - } - - /* GENERIC(?) / Vendor InfoFrame? */ - nvkm_mask(device, 0x616730 + hoff, 0x00010001, 0x00010000); - if (vendor_size) { - /* - * These appear to be the audio infoframe registers, - * but no other set of infoframe registers has yet - * been found. - */ - nvkm_wr32(device, 0x616738 + hoff, vendor_infoframe.header); - nvkm_wr32(device, 0x61673c + hoff, vendor_infoframe.subpack0_low); - nvkm_wr32(device, 0x616740 + hoff, vendor_infoframe.subpack0_high); - /* Is there a second (or further?) set of subpack registers here? */ - nvkm_mask(device, 0x616730 + hoff, 0x00000001, 0x00000001); - } - /* ??? InfoFrame? */ nvkm_mask(device, 0x6167a4 + hoff, 0x00000001, 0x00000000); nvkm_wr32(device, 0x6167ac + hoff, 0x00000010); @@ -259,6 +275,13 @@ gf119_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe nvkm_mask(device, 0x616798 + hoff, 0x401f007f, ctrl); } +static const struct nvkm_ior_func_hdmi +gf119_sor_hdmi = { + .ctrl = gf119_sor_hdmi_ctrl, + .infoframe_avi = gf119_sor_hdmi_infoframe_avi, + .infoframe_vsi = gf119_sor_hdmi_infoframe_vsi, +}; + void gf119_sor_clock(struct nvkm_ior *sor) { @@ -305,9 +328,7 @@ gf119_sor = { .state = gf119_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gf119_sor_hdmi_ctrl, - }, + .hdmi = &gf119_sor_hdmi, .dp = &gf119_sor_dp, .hda = &gf119_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c index 7248e9ec835e3..876a21a0cebb7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c @@ -30,8 +30,51 @@ #include void -gk104_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, - u8 rekey, u8 *avi, u8 avi_size, u8 *vendor, u8 vendor_size) +gk104_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe vsi; + const u32 hoff = head * 0x400; + + pack_hdmi_infoframe(&vsi, data, size); + + /* GENERIC(?) / Vendor InfoFrame? */ + nvkm_mask(device, 0x690100 + hoff, 0x00010001, 0x00000000); + if (!size) + return; + + nvkm_wr32(device, 0x690108 + hoff, vsi.header); + nvkm_wr32(device, 0x69010c + hoff, vsi.subpack0_low); + nvkm_wr32(device, 0x690110 + hoff, vsi.subpack0_high); + /* Is there a second (or further?) set of subpack registers here? */ + nvkm_mask(device, 0x690100 + hoff, 0x00000001, 0x00000001); +} + +void +gk104_sor_hdmi_infoframe_avi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe avi; + const u32 hoff = head * 0x400; + + pack_hdmi_infoframe(&avi, data, size); + + /* AVI InfoFrame */ + nvkm_mask(device, 0x690000 + hoff, 0x00000001, 0x00000000); + if (!size) + return; + + nvkm_wr32(device, 0x690008 + hoff, avi.header); + nvkm_wr32(device, 0x69000c + hoff, avi.subpack0_low); + nvkm_wr32(device, 0x690010 + hoff, avi.subpack0_high); + nvkm_wr32(device, 0x690014 + hoff, avi.subpack1_low); + nvkm_wr32(device, 0x690018 + hoff, avi.subpack1_high); + + nvkm_mask(device, 0x690000 + hoff, 0x00000001, 0x00000001); +} + +void +gk104_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, u8 rekey) { struct nvkm_device *device = ior->disp->engine.subdev.device; const u32 ctrl = 0x40000000 * enable | @@ -39,11 +82,6 @@ gk104_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe rekey; const u32 hoff = head * 0x800; const u32 hdmi = head * 0x400; - struct packed_hdmi_infoframe avi_infoframe; - struct packed_hdmi_infoframe vendor_infoframe; - - pack_hdmi_infoframe(&avi_infoframe, avi, avi_size); - pack_hdmi_infoframe(&vendor_infoframe, vendor, vendor_size); if (!(ctrl & 0x40000000)) { nvkm_mask(device, 0x616798 + hoff, 0x40000000, 0x00000000); @@ -53,28 +91,6 @@ gk104_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe return; } - /* AVI InfoFrame */ - nvkm_mask(device, 0x690000 + hdmi, 0x00000001, 0x00000000); - if (avi_size) { - nvkm_wr32(device, 0x690008 + hdmi, avi_infoframe.header); - nvkm_wr32(device, 0x69000c + hdmi, avi_infoframe.subpack0_low); - nvkm_wr32(device, 0x690010 + hdmi, avi_infoframe.subpack0_high); - nvkm_wr32(device, 0x690014 + hdmi, avi_infoframe.subpack1_low); - nvkm_wr32(device, 0x690018 + hdmi, avi_infoframe.subpack1_high); - nvkm_mask(device, 0x690000 + hdmi, 0x00000001, 0x00000001); - } - - /* GENERIC(?) / Vendor InfoFrame? */ - nvkm_mask(device, 0x690100 + hdmi, 0x00010001, 0x00000000); - if (vendor_size) { - nvkm_wr32(device, 0x690108 + hdmi, vendor_infoframe.header); - nvkm_wr32(device, 0x69010c + hdmi, vendor_infoframe.subpack0_low); - nvkm_wr32(device, 0x690110 + hdmi, vendor_infoframe.subpack0_high); - /* Is there a second (or further?) set of subpack registers here? */ - nvkm_mask(device, 0x690100 + hdmi, 0x00000001, 0x00000001); - } - - /* ??? InfoFrame? */ nvkm_mask(device, 0x6900c0 + hdmi, 0x00000001, 0x00000000); nvkm_wr32(device, 0x6900cc + hdmi, 0x00000010); @@ -87,14 +103,19 @@ gk104_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe nvkm_mask(device, 0x616798 + hoff, 0x401f007f, ctrl); } +const struct nvkm_ior_func_hdmi +gk104_sor_hdmi = { + .ctrl = gk104_sor_hdmi_ctrl, + .infoframe_avi = gk104_sor_hdmi_infoframe_avi, + .infoframe_vsi = gk104_sor_hdmi_infoframe_vsi, +}; + static const struct nvkm_ior_func gk104_sor = { .state = gf119_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gk104_sor_hdmi_ctrl, - }, + .hdmi = &gk104_sor_hdmi, .dp = &gf119_sor_dp, .hda = &gf119_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c index 9e9ef49bd8acf..b4d8e868616fa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c @@ -70,9 +70,7 @@ gm107_sor = { .state = gf119_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gk104_sor_hdmi_ctrl, - }, + .hdmi = &gk104_sor_hdmi, .dp = &gm107_sor_dp, .hda = &gf119_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c index 4ecc8f98af6e9..562ebae57d445 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c @@ -79,6 +79,14 @@ gm200_sor_hdmi_scdc(struct nvkm_ior *ior, u8 scdc) ior->tmds.high_speed = !!(scdc & 0x2); } +const struct nvkm_ior_func_hdmi +gm200_sor_hdmi = { + .ctrl = gk104_sor_hdmi_ctrl, + .scdc = gm200_sor_hdmi_scdc, + .infoframe_avi = gk104_sor_hdmi_infoframe_avi, + .infoframe_vsi = gk104_sor_hdmi_infoframe_vsi, +}; + void gm200_sor_route_set(struct nvkm_outp *outp, struct nvkm_ior *ior) { @@ -131,10 +139,7 @@ gm200_sor = { .state = gf119_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gk104_sor_hdmi_ctrl, - .scdc = gm200_sor_hdmi_scdc, - }, + .hdmi = &gm200_sor_hdmi, .dp = &gm200_sor_dp, .hda = &gf119_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c index 7172a9dfd89b5..7f1eb4332040b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c @@ -37,10 +37,7 @@ gp100_sor = { .state = gf119_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gk104_sor_hdmi_ctrl, - .scdc = gm200_sor_hdmi_scdc, - }, + .hdmi = &gm200_sor_hdmi, .dp = &gm200_sor_dp, .hda = &gf119_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c index 70c49e7af9cfd..a2c7c6f83dcdb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gt215.c @@ -92,9 +92,53 @@ gt215_sor_dp = { .watermark = g94_sor_dp_watermark, }; -void -gt215_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, - u8 rekey, u8 *avi, u8 avi_size, u8 *vendor, u8 vendor_size) +static void +gt215_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe vsi; + const u32 soff = nv50_ior_base(ior); + + pack_hdmi_infoframe(&vsi, data, size); + + nvkm_mask(device, 0x61c53c + soff, 0x00010001, 0x00010000); + if (!size) + return; + + nvkm_wr32(device, 0x61c544 + soff, vsi.header); + nvkm_wr32(device, 0x61c548 + soff, vsi.subpack0_low); + nvkm_wr32(device, 0x61c54c + soff, vsi.subpack0_high); + /* Is there a second (or up to fourth?) set of subpack registers here? */ + /* nvkm_wr32(device, 0x61c550 + soff, vsi.subpack1_low); */ + /* nvkm_wr32(device, 0x61c554 + soff, vsi.subpack1_high); */ + + nvkm_mask(device, 0x61c53c + soff, 0x00010001, 0x00010001); +} + +static void +gt215_sor_hdmi_infoframe_avi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe avi; + const u32 soff = nv50_ior_base(ior); + + pack_hdmi_infoframe(&avi, data, size); + + nvkm_mask(device, 0x61c520 + soff, 0x00000001, 0x00000000); + if (size) + return; + + nvkm_wr32(device, 0x61c528 + soff, avi.header); + nvkm_wr32(device, 0x61c52c + soff, avi.subpack0_low); + nvkm_wr32(device, 0x61c530 + soff, avi.subpack0_high); + nvkm_wr32(device, 0x61c534 + soff, avi.subpack1_low); + nvkm_wr32(device, 0x61c538 + soff, avi.subpack1_high); + + nvkm_mask(device, 0x61c520 + soff, 0x00000001, 0x00000001); +} + +static void +gt215_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, u8 rekey) { struct nvkm_device *device = ior->disp->engine.subdev.device; const u32 ctrl = 0x40000000 * enable | @@ -102,11 +146,6 @@ gt215_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe max_ac_packet << 16 | rekey; const u32 soff = nv50_ior_base(ior); - struct packed_hdmi_infoframe avi_infoframe; - struct packed_hdmi_infoframe vendor_infoframe; - - pack_hdmi_infoframe(&avi_infoframe, avi, avi_size); - pack_hdmi_infoframe(&vendor_infoframe, vendor, vendor_size); if (!(ctrl & 0x40000000)) { nvkm_mask(device, 0x61c5a4 + soff, 0x40000000, 0x00000000); @@ -116,17 +155,6 @@ gt215_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe return; } - /* AVI InfoFrame */ - nvkm_mask(device, 0x61c520 + soff, 0x00000001, 0x00000000); - if (avi_size) { - nvkm_wr32(device, 0x61c528 + soff, avi_infoframe.header); - nvkm_wr32(device, 0x61c52c + soff, avi_infoframe.subpack0_low); - nvkm_wr32(device, 0x61c530 + soff, avi_infoframe.subpack0_high); - nvkm_wr32(device, 0x61c534 + soff, avi_infoframe.subpack1_low); - nvkm_wr32(device, 0x61c538 + soff, avi_infoframe.subpack1_high); - nvkm_mask(device, 0x61c520 + soff, 0x00000001, 0x00000001); - } - /* Audio InfoFrame */ nvkm_mask(device, 0x61c500 + soff, 0x00000001, 0x00000000); nvkm_wr32(device, 0x61c508 + soff, 0x000a0184); @@ -134,18 +162,6 @@ gt215_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe nvkm_wr32(device, 0x61c510 + soff, 0x00000000); nvkm_mask(device, 0x61c500 + soff, 0x00000001, 0x00000001); - /* Vendor InfoFrame */ - nvkm_mask(device, 0x61c53c + soff, 0x00010001, 0x00010000); - if (vendor_size) { - nvkm_wr32(device, 0x61c544 + soff, vendor_infoframe.header); - nvkm_wr32(device, 0x61c548 + soff, vendor_infoframe.subpack0_low); - nvkm_wr32(device, 0x61c54c + soff, vendor_infoframe.subpack0_high); - /* Is there a second (or up to fourth?) set of subpack registers here? */ - /* nvkm_wr32(device, 0x61c550 + soff, vendor_infoframe.subpack1_low); */ - /* nvkm_wr32(device, 0x61c554 + soff, vendor_infoframe.subpack1_high); */ - nvkm_mask(device, 0x61c53c + soff, 0x00010001, 0x00010001); - } - nvkm_mask(device, 0x61c5d0 + soff, 0x00070001, 0x00010001); /* SPARE, HW_CTS */ nvkm_mask(device, 0x61c568 + soff, 0x00010101, 0x00000000); /* ACR_CTRL, ?? */ nvkm_mask(device, 0x61c578 + soff, 0x80000000, 0x80000000); /* ACR_0441_ENABLE */ @@ -159,14 +175,19 @@ gt215_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe nvkm_mask(device, 0x61c5a4 + soff, 0x5f1f007f, ctrl); } +const struct nvkm_ior_func_hdmi +gt215_sor_hdmi = { + .ctrl = gt215_sor_hdmi_ctrl, + .infoframe_avi = gt215_sor_hdmi_infoframe_avi, + .infoframe_vsi = gt215_sor_hdmi_infoframe_vsi, +}; + static const struct nvkm_ior_func gt215_sor = { .state = g94_sor_state, .power = nv50_sor_power, .clock = nv50_sor_clock, - .hdmi = { - .ctrl = gt215_sor_hdmi_ctrl, - }, + .hdmi = >215_sor_hdmi, .dp = >215_sor_dp, .hda = >215_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c index 6b9d49270fa7a..115d0997fd62f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c @@ -96,9 +96,54 @@ gv100_sor_dp = { .watermark = gv100_sor_dp_watermark, }; -void -gv100_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, - u8 rekey, u8 *avi, u8 avi_size, u8 *vendor, u8 vendor_size) +static void +gv100_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe vsi; + const u32 hoff = head * 0x400; + + pack_hdmi_infoframe(&vsi, data, size); + + nvkm_mask(device, 0x6f0100 + hoff, 0x00010001, 0x00000000); + if (!size) + return; + + nvkm_wr32(device, 0x6f0108 + hoff, vsi.header); + nvkm_wr32(device, 0x6f010c + hoff, vsi.subpack0_low); + nvkm_wr32(device, 0x6f0110 + hoff, vsi.subpack0_high); + nvkm_wr32(device, 0x6f0114 + hoff, 0x00000000); + nvkm_wr32(device, 0x6f0118 + hoff, 0x00000000); + nvkm_wr32(device, 0x6f011c + hoff, 0x00000000); + nvkm_wr32(device, 0x6f0120 + hoff, 0x00000000); + nvkm_wr32(device, 0x6f0124 + hoff, 0x00000000); + nvkm_mask(device, 0x6f0100 + hoff, 0x00000001, 0x00000001); +} + +static void +gv100_sor_hdmi_infoframe_avi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + struct packed_hdmi_infoframe avi; + const u32 hoff = head * 0x400; + + pack_hdmi_infoframe(&avi, data, size); + + nvkm_mask(device, 0x6f0000 + hoff, 0x00000001, 0x00000000); + if (!size) + return; + + nvkm_wr32(device, 0x6f0008 + hoff, avi.header); + nvkm_wr32(device, 0x6f000c + hoff, avi.subpack0_low); + nvkm_wr32(device, 0x6f0010 + hoff, avi.subpack0_high); + nvkm_wr32(device, 0x6f0014 + hoff, avi.subpack1_low); + nvkm_wr32(device, 0x6f0018 + hoff, avi.subpack1_high); + + nvkm_mask(device, 0x6f0000 + hoff, 0x00000001, 0x00000001); +} + +static void +gv100_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packet, u8 rekey) { struct nvkm_device *device = ior->disp->engine.subdev.device; const u32 ctrl = 0x40000000 * enable | @@ -106,11 +151,6 @@ gv100_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe rekey; const u32 hoff = head * 0x800; const u32 hdmi = head * 0x400; - struct packed_hdmi_infoframe avi_infoframe; - struct packed_hdmi_infoframe vendor_infoframe; - - pack_hdmi_infoframe(&avi_infoframe, avi, avi_size); - pack_hdmi_infoframe(&vendor_infoframe, vendor, vendor_size); if (!(ctrl & 0x40000000)) { nvkm_mask(device, 0x6165c0 + hoff, 0x40000000, 0x00000000); @@ -120,32 +160,6 @@ gv100_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe return; } - /* AVI InfoFrame (AVI). */ - nvkm_mask(device, 0x6f0000 + hdmi, 0x00000001, 0x00000000); - if (avi_size) { - nvkm_wr32(device, 0x6f0008 + hdmi, avi_infoframe.header); - nvkm_wr32(device, 0x6f000c + hdmi, avi_infoframe.subpack0_low); - nvkm_wr32(device, 0x6f0010 + hdmi, avi_infoframe.subpack0_high); - nvkm_wr32(device, 0x6f0014 + hdmi, avi_infoframe.subpack1_low); - nvkm_wr32(device, 0x6f0018 + hdmi, avi_infoframe.subpack1_high); - nvkm_mask(device, 0x6f0000 + hdmi, 0x00000001, 0x00000001); - } - - /* Vendor-specific InfoFrame (VSI). */ - nvkm_mask(device, 0x6f0100 + hdmi, 0x00010001, 0x00000000); - if (vendor_size) { - nvkm_wr32(device, 0x6f0108 + hdmi, vendor_infoframe.header); - nvkm_wr32(device, 0x6f010c + hdmi, vendor_infoframe.subpack0_low); - nvkm_wr32(device, 0x6f0110 + hdmi, vendor_infoframe.subpack0_high); - nvkm_wr32(device, 0x6f0114 + hdmi, 0x00000000); - nvkm_wr32(device, 0x6f0118 + hdmi, 0x00000000); - nvkm_wr32(device, 0x6f011c + hdmi, 0x00000000); - nvkm_wr32(device, 0x6f0120 + hdmi, 0x00000000); - nvkm_wr32(device, 0x6f0124 + hdmi, 0x00000000); - nvkm_mask(device, 0x6f0100 + hdmi, 0x00000001, 0x00000001); - } - - /* General Control (GCP). */ nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000); nvkm_wr32(device, 0x6f00cc + hdmi, 0x00000010); @@ -158,6 +172,14 @@ gv100_sor_hdmi_ctrl(struct nvkm_ior *ior, int head, bool enable, u8 max_ac_packe nvkm_mask(device, 0x6165c0 + hoff, 0x401f007f, ctrl); } +const struct nvkm_ior_func_hdmi +gv100_sor_hdmi = { + .ctrl = gv100_sor_hdmi_ctrl, + .scdc = gm200_sor_hdmi_scdc, + .infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, +}; + void gv100_sor_state(struct nvkm_ior *sor, struct nvkm_ior_state *state) { @@ -190,10 +212,7 @@ gv100_sor = { .state = gv100_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gv100_sor_hdmi_ctrl, - .scdc = gm200_sor_hdmi_scdc, - }, + .hdmi = &gv100_sor_hdmi, .dp = &gv100_sor_dp, .hda = &gv100_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index 671c4674ffcc3..da1b1a626ef21 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -63,12 +63,12 @@ struct nvkm_ior_func { void (*war_2)(struct nvkm_ior *); void (*war_3)(struct nvkm_ior *); - struct { - void (*ctrl)(struct nvkm_ior *, int head, bool enable, - u8 max_ac_packet, u8 rekey, u8 *avi, u8 avi_size, - u8 *vendor, u8 vendor_size); + const struct nvkm_ior_func_hdmi { + void (*ctrl)(struct nvkm_ior *, int head, bool enable, u8 max_ac_packet, u8 rekey); void (*scdc)(struct nvkm_ior *, u8 scdc); - } hdmi; + void (*infoframe_avi)(struct nvkm_ior *, int head, void *data, u32 size); + void (*infoframe_vsi)(struct nvkm_ior *, int head, void *data, u32 size); + } *hdmi; const struct nvkm_ior_func_dp { u8 lanes[4]; @@ -124,9 +124,10 @@ void nv50_sor_power(struct nvkm_ior *, bool, bool, bool, bool, bool); void nv50_sor_clock(struct nvkm_ior *); int g84_sor_new(struct nvkm_disp *, int); -void g84_sor_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); +extern const struct nvkm_ior_func_hdmi g84_sor_hdmi; int g94_sor_cnt(struct nvkm_disp *, unsigned long *); + void g94_sor_state(struct nvkm_ior *, struct nvkm_ior_state *); extern const struct nvkm_ior_func_dp g94_sor_dp; int g94_sor_dp_links(struct nvkm_ior *, struct nvkm_i2c_aux *); @@ -137,7 +138,7 @@ void g94_sor_dp_audio_sym(struct nvkm_ior *, int, u16, u32); void g94_sor_dp_activesym(struct nvkm_ior *, int, u8, u8, u8, u8); void g94_sor_dp_watermark(struct nvkm_ior *, int, u8); -void gt215_sor_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); +extern const struct nvkm_ior_func_hdmi gt215_sor_hdmi; void gt215_sor_dp_audio(struct nvkm_ior *, int, bool); extern const struct nvkm_ior_func_hda gt215_sor_hda; @@ -156,12 +157,16 @@ void gf119_sor_hda_hpd(struct nvkm_ior *, int, bool); void gf119_sor_hda_eld(struct nvkm_ior *, int, u8 *, u8); int gk104_sor_new(struct nvkm_disp *, int); -void gk104_sor_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); +extern const struct nvkm_ior_func_hdmi gk104_sor_hdmi; +void gk104_sor_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8); +void gk104_sor_hdmi_infoframe_avi(struct nvkm_ior *, int, void *, u32); +void gk104_sor_hdmi_infoframe_vsi(struct nvkm_ior *, int, void *, u32); void gm107_sor_dp_pattern(struct nvkm_ior *, int); void gm200_sor_route_set(struct nvkm_outp *, struct nvkm_ior *); int gm200_sor_route_get(struct nvkm_outp *, int *); +extern const struct nvkm_ior_func_hdmi gm200_sor_hdmi; void gm200_sor_hdmi_scdc(struct nvkm_ior *, u8); extern const struct nvkm_ior_func_dp gm200_sor_dp; void gm200_sor_dp_drive(struct nvkm_ior *, int, int, int, int, int); @@ -170,7 +175,7 @@ int gp100_sor_new(struct nvkm_disp *, int); int gv100_sor_cnt(struct nvkm_disp *, unsigned long *); void gv100_sor_state(struct nvkm_ior *, struct nvkm_ior_state *); -void gv100_sor_hdmi_ctrl(struct nvkm_ior *, int, bool, u8, u8, u8 *, u8 , u8 *, u8); +extern const struct nvkm_ior_func_hdmi gv100_sor_hdmi; void gv100_sor_dp_audio(struct nvkm_ior *, int, bool); void gv100_sor_dp_audio_sym(struct nvkm_ior *, int, u16, u32); void gv100_sor_dp_watermark(struct nvkm_ior *, int, u8); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c index 916b1d477b0bf..841e3b69fcaf3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp77.c @@ -31,9 +31,7 @@ mcp77_sor = { .state = g94_sor_state, .power = nv50_sor_power, .clock = nv50_sor_clock, - .hdmi = { - .ctrl = g84_sor_hdmi_ctrl, - }, + .hdmi = &g84_sor_hdmi, .dp = &g94_sor_dp, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c index a5a0b94393744..f96ba4752655c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/mcp89.c @@ -44,9 +44,7 @@ mcp89_sor = { .state = g94_sor_state, .power = nv50_sor_power, .clock = nv50_sor_clock, - .hdmi = { - .ctrl = gt215_sor_hdmi_ctrl, - }, + .hdmi = >215_sor_hdmi, .dp = &mcp89_sor_dp, .hda = >215_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h index 16aadcedcbda3..eb9cd11468256 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h @@ -60,6 +60,9 @@ struct nvkm_outp { }; struct nvkm_object object; + struct { + struct nvkm_head *head; + } asy; }; int nvkm_outp_new_(const struct nvkm_outp_func *, struct nvkm_disp *, int index, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index f028ab17f8cfd..9c481fa49bc56 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -121,51 +121,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) ior->func->hda->hpd(ior, hidx, false); } - return 0; - } - break; - case NV50_DISP_MTHD_V1_SOR_HDMI_PWR: { - union { - struct nv50_disp_sor_hdmi_pwr_v0 v0; - } *args = data; - u8 *vendor, vendor_size; - u8 *avi, avi_size; - int ret = -ENOSYS; - - nvif_ioctl(object, "disp sor hdmi ctrl size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { - nvif_ioctl(object, "disp sor hdmi ctrl vers %d state %d " - "max_ac_packet %d rekey %d scdc %d\n", - args->v0.version, args->v0.state, - args->v0.max_ac_packet, args->v0.rekey, - args->v0.scdc); - if (args->v0.max_ac_packet > 0x1f || args->v0.rekey > 0x7f) - return -EINVAL; - if ((args->v0.avi_infoframe_length - + args->v0.vendor_infoframe_length) > size) - return -EINVAL; - else - if ((args->v0.avi_infoframe_length - + args->v0.vendor_infoframe_length) < size) - return -E2BIG; - avi = data; - avi_size = args->v0.avi_infoframe_length; - vendor = avi + avi_size; - vendor_size = args->v0.vendor_infoframe_length; - } else - return ret; - - if (!outp->ior->func->hdmi.ctrl) - return -ENODEV; - - outp->ior->func->hdmi.ctrl(outp->ior, hidx, args->v0.state, - args->v0.max_ac_packet, - args->v0.rekey, avi, avi_size, - vendor, vendor_size); - - if (outp->ior->func->hdmi.scdc) - outp->ior->func->hdmi.scdc(outp->ior, args->v0.scdc); - return 0; } break; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c index e4ad1a6f6c884..f5242a6722794 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c @@ -88,10 +88,7 @@ tu102_sor = { .state = gv100_sor_state, .power = nv50_sor_power, .clock = gf119_sor_clock, - .hdmi = { - .ctrl = gv100_sor_hdmi_ctrl, - .scdc = gm200_sor_hdmi_scdc, - }, + .hdmi = &gv100_sor_hdmi, .dp = &tu102_sor_dp, .hda = &gv100_sor_hda, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index 7e0cef0f80a9a..f82100b1a45d4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -21,22 +21,88 @@ */ #define nvkm_uoutp(p) container_of((p), struct nvkm_outp, object) #include "outp.h" +#include "head.h" #include "ior.h" #include +static int +nvkm_uoutp_mthd_infoframe(struct nvkm_outp *outp, void *argv, u32 argc) +{ + struct nvkm_ior *ior = outp->ior; + union nvif_outp_infoframe_args *args = argv; + + if (argc < sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + if (!nvkm_head_find(outp->disp, args->v0.head)) + return -EINVAL; + + switch (ior->func->hdmi ? args->v0.type : 0xff) { + case NVIF_OUTP_INFOFRAME_V0_AVI: + ior->func->hdmi->infoframe_avi(ior, args->v0.head, argv, argc); + return 0; + case NVIF_OUTP_INFOFRAME_V0_VSI: + ior->func->hdmi->infoframe_vsi(ior, args->v0.head, argv, argc); + return 0; + default: + break; + } + + return -EINVAL; +} + static int nvkm_uoutp_mthd_release(struct nvkm_outp *outp, void *argv, u32 argc) { + struct nvkm_head *head = outp->asy.head; + struct nvkm_ior *ior = outp->ior; union nvif_outp_release_args *args = argv; if (argc != sizeof(args->vn)) return -ENOSYS; + if (ior->func->hdmi && head) { + ior->func->hdmi->infoframe_avi(ior, head->id, NULL, 0); + ior->func->hdmi->infoframe_vsi(ior, head->id, NULL, 0); + ior->func->hdmi->ctrl(ior, head->id, false, 0, 0); + } + nvkm_outp_release(outp, NVKM_OUTP_USER); return 0; } +static int +nvkm_uoutp_mthd_acquire_tmds(struct nvkm_outp *outp, u8 head, u8 hdmi, u8 hdmi_max_ac_packet, + u8 hdmi_rekey, u8 hdmi_scdc, u8 hdmi_hda) +{ + struct nvkm_ior *ior; + int ret; + + if (!(outp->asy.head = nvkm_head_find(outp->disp, head))) + return -EINVAL; + + ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, hdmi && hdmi_hda); + if (ret) + return ret; + + ior = outp->ior; + + if (hdmi) { + if (!ior->func->hdmi || + hdmi_max_ac_packet > 0x1f || hdmi_rekey > 0x7f || + (hdmi_scdc && !ior->func->hdmi->scdc)) { + nvkm_outp_release(outp, NVKM_OUTP_USER); + return -EINVAL; + } + + ior->func->hdmi->ctrl(ior, head, hdmi, hdmi_max_ac_packet, hdmi_rekey); + if (ior->func->hdmi->scdc) + ior->func->hdmi->scdc(ior, hdmi_scdc); + } + + return 0; +} + static int nvkm_uoutp_mthd_acquire_lvds(struct nvkm_outp *outp, bool dual, bool bpc8) { @@ -63,6 +129,13 @@ nvkm_uoutp_mthd_acquire(struct nvkm_outp *outp, void *argv, u32 argc) ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, false); break; case NVIF_OUTP_ACQUIRE_V0_TMDS: + ret = nvkm_uoutp_mthd_acquire_tmds(outp, args->v0.tmds.head, + args->v0.tmds.hdmi, + args->v0.tmds.hdmi_max_ac_packet, + args->v0.tmds.hdmi_rekey, + args->v0.tmds.hdmi_scdc, + args->v0.tmds.hdmi_hda); + break; case NVIF_OUTP_ACQUIRE_V0_DP: ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.dp.hda); break; @@ -110,6 +183,7 @@ nvkm_uoutp_mthd_acquired(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc) { switch (mthd) { case NVIF_OUTP_V0_RELEASE : return nvkm_uoutp_mthd_release (outp, argv, argc); + case NVIF_OUTP_V0_INFOFRAME : return nvkm_uoutp_mthd_infoframe (outp, argv, argc); default: break; } -- GitLab From a9f5d7721923f6016578167640b8dd5cb0c5a698 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:33 +1000 Subject: [PATCH 1243/1927] drm/nouveau/disp: move HDA ELD method Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 50 +++++++------------ drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 7 --- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 10 ++++ drivers/gpu/drm/nouveau/include/nvif/outp.h | 1 + drivers/gpu/drm/nouveau/nvif/outp.c | 21 ++++++++ .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 33 ------------ .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 30 +++++++++++ 7 files changed, 81 insertions(+), 71 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index edf8996702877..c61617acaf87c 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -673,29 +673,33 @@ nv50_audio_component_fini(struct nouveau_drm *drm) /****************************************************************************** * Audio *****************************************************************************/ +static bool +nv50_audio_supported(struct drm_encoder *encoder) +{ + struct nv50_disp *disp = nv50_disp(encoder->dev); + + if (disp->disp->object.oclass <= GT200_DISP || + disp->disp->object.oclass == GT206_DISP) + return false; + + return true; +} + static void nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc) { struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_disp *disp = nv50_disp(encoder->dev); struct nvif_outp *outp = &nv_encoder->outp; - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_hda_eld_v0 eld; - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_HDA_ELD, - .base.hasht = nv_encoder->dcb->hasht, - .base.hashm = (0xf0ff & nv_encoder->dcb->hashm) | - (0x0100 << nv_crtc->index), - }; + + if (!nv50_audio_supported(encoder)) + return; mutex_lock(&drm->audio.lock); if (nv_encoder->audio.enabled) { nv_encoder->audio.enabled = false; nv_encoder->audio.connector = NULL; - nvif_mthd(&disp->disp->object, 0, &args, sizeof(args)); + nvif_outp_hda_eld(&nv_encoder->outp, nv_crtc->index, NULL, 0); } mutex_unlock(&drm->audio.lock); @@ -709,31 +713,15 @@ nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc, { struct nouveau_drm *drm = nouveau_drm(encoder->dev); struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder); - struct nv50_disp *disp = nv50_disp(encoder->dev); struct nvif_outp *outp = &nv_encoder->outp; - struct __packed { - struct { - struct nv50_disp_mthd_v1 mthd; - struct nv50_disp_sor_hda_eld_v0 eld; - } base; - u8 data[sizeof(nv_connector->base.eld)]; - } args = { - .base.mthd.version = 1, - .base.mthd.method = NV50_DISP_MTHD_V1_SOR_HDA_ELD, - .base.mthd.hasht = nv_encoder->dcb->hasht, - .base.mthd.hashm = (0xf0ff & nv_encoder->dcb->hashm) | - (0x0100 << nv_crtc->index), - }; - if (!drm_detect_monitor_audio(nv_connector->edid)) + if (!nv50_audio_supported(encoder) || !drm_detect_monitor_audio(nv_connector->edid)) return; mutex_lock(&drm->audio.lock); - memcpy(args.data, nv_connector->base.eld, sizeof(args.data)); - - nvif_mthd(&disp->disp->object, 0, &args, - sizeof(args.base) + drm_eld_size(args.data)); + nvif_outp_hda_eld(&nv_encoder->outp, nv_crtc->index, nv_connector->base.eld, + drm_eld_size(nv_connector->base.eld)); nv_encoder->audio.enabled = true; nv_encoder->audio.connector = &nv_connector->base; diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index 6389558dda1fd..a71317733748a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -28,7 +28,6 @@ struct nv50_disp_scanoutpos_v0 { struct nv50_disp_mthd_v1 { __u8 version; -#define NV50_DISP_MTHD_V1_SOR_HDA_ELD 0x21 #define NV50_DISP_MTHD_V1_SOR_DP_MST_LINK 0x25 #define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI 0x26 __u8 method; @@ -37,12 +36,6 @@ struct nv50_disp_mthd_v1 { __u8 pad06[2]; }; -struct nv50_disp_sor_hda_eld_v0 { - __u8 version; - __u8 pad01[7]; - __u8 data[]; -}; - struct nv50_disp_sor_dp_mst_link_v0 { __u8 version; __u8 state; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index 06ed372cbe5a1..938d384f5b1fa 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -14,6 +14,7 @@ union nvif_outp_args { #define NVIF_OUTP_V0_ACQUIRE 0x01 #define NVIF_OUTP_V0_RELEASE 0x02 #define NVIF_OUTP_V0_INFOFRAME 0x03 +#define NVIF_OUTP_V0_HDA_ELD 0x04 union nvif_outp_load_detect_args { struct nvif_outp_load_detect_v0 { @@ -77,4 +78,13 @@ union nvif_outp_infoframe_args { __u8 data[]; } v0; }; + +union nvif_outp_hda_eld_args { + struct nvif_outp_hda_eld_v0 { + __u8 version; + __u8 head; + __u8 pad02[6]; + __u8 data[]; + } v0; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index dd7e603d13c9a..c8879d3428c8d 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -24,4 +24,5 @@ int nvif_outp_acquire_lvds(struct nvif_outp *, bool dual, bool bpc8); int nvif_outp_acquire_dp(struct nvif_outp *, bool hda); void nvif_outp_release(struct nvif_outp *); int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_v0 *, u32 size); +int nvif_outp_hda_eld(struct nvif_outp *, int head, void *data, u32 size); #endif diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 987bc3a9e7bf3..3a7c2d573e698 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -25,6 +25,27 @@ #include +int +nvif_outp_hda_eld(struct nvif_outp *outp, int head, void *data, u32 size) +{ + struct { + struct nvif_outp_hda_eld_v0 mthd; + u8 data[128]; + } args; + int ret; + + if (WARN_ON(size > ARRAY_SIZE(args.data))) + return -EINVAL; + + args.mthd.version = 0; + args.mthd.head = head; + + memcpy(args.data, data, size); + ret = nvif_mthd(&outp->object, NVIF_OUTP_V0_HDA_ELD, &args, sizeof(args.mthd) + size); + NVIF_ERRON(ret, &outp->object, "[HDA_ELD head:%d size:%d]", head, size); + return ret; +} + int nvif_outp_infoframe(struct nvif_outp *outp, u8 type, struct nvif_outp_infoframe_v0 *args, u32 size) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index 9c481fa49bc56..a5bb15b62a0ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -91,39 +91,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) } switch (mthd * !!outp) { - case NV50_DISP_MTHD_V1_SOR_HDA_ELD: { - union { - struct nv50_disp_sor_hda_eld_v0 v0; - } *args = data; - struct nvkm_ior *ior = outp->ior; - int ret = -ENOSYS; - - nvif_ioctl(object, "disp sor hda eld size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { - nvif_ioctl(object, "disp sor hda eld vers %d\n", - args->v0.version); - if (size > 0x60) - return -E2BIG; - } else - return ret; - - if (!ior->hda) - return -ENODEV; - - if (size && args->v0.data[0]) { - if (outp->info.type == DCB_OUTPUT_DP) - ior->func->dp->audio(ior, hidx, true); - ior->func->hda->hpd(ior, hidx, true); - ior->func->hda->eld(ior, hidx, data, size); - } else { - if (outp->info.type == DCB_OUTPUT_DP) - ior->func->dp->audio(ior, hidx, false); - ior->func->hda->hpd(ior, hidx, false); - } - - return 0; - } - break; case NV50_DISP_MTHD_V1_SOR_DP_MST_LINK: { union { struct nv50_disp_sor_dp_mst_link_v0 v0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index f82100b1a45d4..052b2d14f47a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -26,6 +26,35 @@ #include +static int +nvkm_uoutp_mthd_hda_eld(struct nvkm_outp *outp, void *argv, u32 argc) +{ + struct nvkm_ior *ior = outp->ior; + union nvif_outp_hda_eld_args *args = argv; + + if (argc < sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + argc -= sizeof(args->v0); + + if (!ior->hda || !nvkm_head_find(outp->disp, args->v0.head)) + return -EINVAL; + if (argc > 0x60) + return -E2BIG; + + if (argc && args->v0.data[0]) { + if (outp->info.type == DCB_OUTPUT_DP) + ior->func->dp->audio(ior, args->v0.head, true); + ior->func->hda->hpd(ior, args->v0.head, true); + ior->func->hda->eld(ior, args->v0.head, args->v0.data, argc); + } else { + if (outp->info.type == DCB_OUTPUT_DP) + ior->func->dp->audio(ior, args->v0.head, false); + ior->func->hda->hpd(ior, args->v0.head, false); + } + + return 0; +} + static int nvkm_uoutp_mthd_infoframe(struct nvkm_outp *outp, void *argv, u32 argc) { @@ -184,6 +213,7 @@ nvkm_uoutp_mthd_acquired(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc) switch (mthd) { case NVIF_OUTP_V0_RELEASE : return nvkm_uoutp_mthd_release (outp, argv, argc); case NVIF_OUTP_V0_INFOFRAME : return nvkm_uoutp_mthd_infoframe (outp, argv, argc); + case NVIF_OUTP_V0_HDA_ELD : return nvkm_uoutp_mthd_hda_eld (outp, argv, argc); default: break; } -- GitLab From 8134437213316a58d1844b87e2042ebf1fd9962c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:33 +1000 Subject: [PATCH 1244/1927] drm/nouveau/disp: move DP link config into acquire Aside from fixing MST->SST switching (KMS never turned off MST link config), this should preserve existing behaviour for the moment, but provide a path for the KMS driver to have more explicit control of the DP link, which has been requested by Lyude. More research into modeset/supervisor interactions is needed before we can have fully explicit control from the KMS driver. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 34 +++---------------- drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 7 ---- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 6 +++- drivers/gpu/drm/nouveau/include/nvif/outp.h | 3 +- drivers/gpu/drm/nouveau/nvif/outp.c | 10 ++++-- drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c | 19 ++++++++++- .../gpu/drm/nouveau/nvkm/engine/disp/outp.h | 2 ++ .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 15 -------- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 29 ++++++++++++++-- 9 files changed, 65 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index c61617acaf87c..ca58c6b65339e 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -1014,7 +1014,7 @@ nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st if (!mstm->links++) { /*XXX: MST audio. */ - nvif_outp_acquire_dp(&mstm->outp->outp, false); + nvif_outp_acquire_dp(&mstm->outp->outp, mstm->outp->dp.dpcd, 0, 0, false, true); } if (mstm->outp->outp.or.link & 1) @@ -1380,26 +1380,6 @@ nv50_mstm_remove(struct nv50_mstm *mstm) drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, false); } -static int -nv50_mstm_enable(struct nv50_mstm *mstm, int state) -{ - struct nouveau_encoder *outp = mstm->outp; - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_dp_mst_link_v0 mst; - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK, - .base.hasht = outp->dcb->hasht, - .base.hashm = outp->dcb->hashm, - .mst.state = state, - }; - struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev); - struct nvif_object *disp = &drm->display->disp.object; - - return nvif_mthd(disp, 0, &args, sizeof(args)); -} - int nv50_mstm_detect(struct nouveau_encoder *outp) { @@ -1420,15 +1400,9 @@ nv50_mstm_detect(struct nouveau_encoder *outp) return ret; /* And start enabling */ - ret = nv50_mstm_enable(mstm, true); - if (ret) - return ret; - ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, true); - if (ret) { - nv50_mstm_enable(mstm, false); + if (ret) return ret; - } mstm->is_mst = true; return 1; @@ -1660,7 +1634,7 @@ nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *sta nvif_outp_acquire_lvds(&nv_encoder->outp, lvds_dual, lvds_8bpc); break; case DCB_OUTPUT_DP: - nvif_outp_acquire_dp(&nv_encoder->outp, hda); + nvif_outp_acquire_dp(&nv_encoder->outp, nv_encoder->dp.dpcd, 0, 0, hda, false); depth = nv50_dp_bpc_to_depth(asyh->or.bpc); if (nv_encoder->outp.or.link & 1) @@ -1858,7 +1832,7 @@ nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st break; case DCB_OUTPUT_DP: ctrl |= NVDEF(NV507D, PIOR_SET_CONTROL, PROTOCOL, EXT_TMDS_ENC); - nvif_outp_acquire_dp(&nv_encoder->outp, false); + nvif_outp_acquire_dp(&nv_encoder->outp, nv_encoder->dp.dpcd, 0, 0, false, false); break; default: BUG(); diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index a71317733748a..15e30fbdc2152 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -28,7 +28,6 @@ struct nv50_disp_scanoutpos_v0 { struct nv50_disp_mthd_v1 { __u8 version; -#define NV50_DISP_MTHD_V1_SOR_DP_MST_LINK 0x25 #define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI 0x26 __u8 method; __u16 hasht; @@ -36,12 +35,6 @@ struct nv50_disp_mthd_v1 { __u8 pad06[2]; }; -struct nv50_disp_sor_dp_mst_link_v0 { - __u8 version; - __u8 state; - __u8 pad02[6]; -}; - struct nv50_disp_sor_dp_mst_vcpi_v0 { __u8 version; __u8 pad01[1]; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index 938d384f5b1fa..c67f03d52d8cb 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -55,8 +55,12 @@ union nvif_outp_acquire_args { __u8 pad02[6]; } lvds; struct { + __u8 link_nr; /* 0 = highest possible. */ + __u8 link_bw; /* 0 = highest possible, DP BW code otherwise. */ __u8 hda; - __u8 pad01[7]; + __u8 mst; + __u8 pad04[4]; + __u8 dpcd[16]; } dp; }; } v0; diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index c8879d3428c8d..88fd2b9703041 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -21,7 +21,8 @@ int nvif_outp_acquire_rgb_crt(struct nvif_outp *); int nvif_outp_acquire_tmds(struct nvif_outp *, int head, bool hdmi, u8 max_ac_packet, u8 rekey, u8 scdc, bool hda); int nvif_outp_acquire_lvds(struct nvif_outp *, bool dual, bool bpc8); -int nvif_outp_acquire_dp(struct nvif_outp *, bool hda); +int nvif_outp_acquire_dp(struct nvif_outp *, u8 dpcd[16], + int link_nr, int link_bw, bool hda, bool mst); void nvif_outp_release(struct nvif_outp *); int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_v0 *, u32 size); int nvif_outp_hda_eld(struct nvif_outp *, int head, void *data, u32 size); diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 3a7c2d573e698..1ae5b19a8025c 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -84,16 +84,22 @@ nvif_outp_acquire(struct nvif_outp *outp, u8 proto, struct nvif_outp_acquire_v0 } int -nvif_outp_acquire_dp(struct nvif_outp *outp, bool hda) +nvif_outp_acquire_dp(struct nvif_outp *outp, u8 dpcd[16], + int link_nr, int link_bw, bool hda, bool mst) { struct nvif_outp_acquire_v0 args; int ret; + args.dp.link_nr = link_nr; + args.dp.link_bw = link_bw; args.dp.hda = hda; + args.dp.mst = mst; + memcpy(args.dp.dpcd, dpcd, sizeof(args.dp.dpcd)); ret = nvif_outp_acquire(outp, NVIF_OUTP_ACQUIRE_V0_DP, &args); NVIF_ERRON(ret, &outp->object, - "[ACQUIRE proto:DP hda:%d] or:%d link:%d", args.dp.hda, args.or, args.link); + "[ACQUIRE proto:DP link_nr:%d link_bw:%02x hda:%d mst:%d] or:%d link:%d", + args.dp.link_nr, args.dp.link_bw, args.dp.hda, args.dp.mst, args.or, args.link); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c index c1b3206f27e64..f1887b58f472d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c @@ -287,7 +287,7 @@ nvkm_dp_train_links(struct nvkm_outp *outp, int rate) u8 sink[2], data; int ret; - OUTP_DBG(outp, "training %d x %d MB/s", ior->dp.nr, ior->dp.bw * 27); + OUTP_DBG(outp, "training %dx%02x", ior->dp.nr, ior->dp.bw); /* Intersect misc. capabilities of the OR and sink. */ if (disp->engine.subdev.device->chipset < 0x110) @@ -455,6 +455,21 @@ nvkm_dp_train(struct nvkm_outp *outp, u32 dataKBps) /* Link training. */ OUTP_DBG(outp, "training"); nvkm_dp_train_init(outp); + + /* Validate and train at configuration requested (if any) on ACQUIRE. */ + if (outp->dp.lt.nr) { + for (nr = outp->dp.links; ret < 0 && nr; nr >>= 1) { + for (rate = 0; nr == outp->dp.lt.nr && rate < outp->dp.rates; rate++) { + if (outp->dp.rate[rate].rate / 27000 == outp->dp.lt.bw) { + ior->dp.bw = outp->dp.rate[rate].rate / 27000; + ior->dp.nr = nr; + ret = nvkm_dp_train_links(outp, rate); + } + } + } + } + + /* Otherwise, loop through all valid link configurations that support the data rate. */ for (nr = outp->dp.links; ret < 0 && nr; nr >>= 1) { for (rate = 0; ret < 0 && rate < outp->dp.rates; rate++) { if (outp->dp.rate[rate].rate * nr >= dataKBps || WARN_ON(!ior->dp.nr)) { @@ -465,6 +480,8 @@ nvkm_dp_train(struct nvkm_outp *outp, u32 dataKBps) } } } + + /* Finish up. */ nvkm_dp_train_fini(outp); if (ret < 0) OUTP_ERR(outp, "training failed"); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h index eb9cd11468256..d7c989e1cd01b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h @@ -54,6 +54,8 @@ struct nvkm_outp { struct mutex mutex; struct { atomic_t done; + u8 nr; + u8 bw; bool mst; } lt; } dp; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index a5bb15b62a0ca..2f42940367a0f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -91,21 +91,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) } switch (mthd * !!outp) { - case NV50_DISP_MTHD_V1_SOR_DP_MST_LINK: { - union { - struct nv50_disp_sor_dp_mst_link_v0 v0; - } *args = data; - int ret = -ENOSYS; - nvif_ioctl(object, "disp sor dp mst link size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "disp sor dp mst link vers %d state %d\n", - args->v0.version, args->v0.state); - outp->dp.lt.mst = !!args->v0.state; - return 0; - } else - return ret; - } - break; case NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI: { union { struct nv50_disp_sor_dp_mst_vcpi_v0 v0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index 052b2d14f47a8..61d41b326f43c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -100,6 +100,23 @@ nvkm_uoutp_mthd_release(struct nvkm_outp *outp, void *argv, u32 argc) return 0; } +static int +nvkm_uoutp_mthd_acquire_dp(struct nvkm_outp *outp, u8 dpcd[16], + u8 link_nr, u8 link_bw, bool hda, bool mst) +{ + int ret; + + ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, hda); + if (ret) + return ret; + + memcpy(outp->dp.dpcd, dpcd, sizeof(outp->dp.dpcd)); + outp->dp.lt.nr = link_nr; + outp->dp.lt.bw = link_bw; + outp->dp.lt.mst = mst; + return 0; +} + static int nvkm_uoutp_mthd_acquire_tmds(struct nvkm_outp *outp, u8 head, u8 hdmi, u8 hdmi_max_ac_packet, u8 hdmi_rekey, u8 hdmi_scdc, u8 hdmi_hda) @@ -152,6 +169,8 @@ nvkm_uoutp_mthd_acquire(struct nvkm_outp *outp, void *argv, u32 argc) if (argc != sizeof(args->v0) || args->v0.version != 0) return -ENOSYS; + if (outp->ior) + return -EBUSY; switch (args->v0.proto) { case NVIF_OUTP_ACQUIRE_V0_RGB_CRT: @@ -165,12 +184,16 @@ nvkm_uoutp_mthd_acquire(struct nvkm_outp *outp, void *argv, u32 argc) args->v0.tmds.hdmi_scdc, args->v0.tmds.hdmi_hda); break; - case NVIF_OUTP_ACQUIRE_V0_DP: - ret = nvkm_outp_acquire(outp, NVKM_OUTP_USER, args->v0.dp.hda); - break; case NVIF_OUTP_ACQUIRE_V0_LVDS: ret = nvkm_uoutp_mthd_acquire_lvds(outp, args->v0.lvds.dual, args->v0.lvds.bpc8); break; + case NVIF_OUTP_ACQUIRE_V0_DP: + ret = nvkm_uoutp_mthd_acquire_dp(outp, args->v0.dp.dpcd, + args->v0.dp.link_nr, + args->v0.dp.link_bw, + args->v0.dp.hda != 0, + args->v0.dp.mst != 0); + break; default: ret = -EINVAL; break; -- GitLab From a62b749390630fd02525ed8abd29323319f9096e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:34 +1000 Subject: [PATCH 1245/1927] drm/nouveau/disp: add method to control DPAUX pad power This removes the need for NVKM to track DP HPD events, as the KMS driver follows them already, and has better information available. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 9 ++ drivers/gpu/drm/nouveau/include/nvif/outp.h | 1 + drivers/gpu/drm/nouveau/nouveau_dp.c | 10 ++- drivers/gpu/drm/nouveau/nvif/outp.c | 14 +++ drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c | 90 +++++++++---------- drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h | 1 + .../gpu/drm/nouveau/nvkm/engine/disp/outp.h | 4 +- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 15 ++++ 8 files changed, 93 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index c67f03d52d8cb..d1583271f5d4d 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -15,6 +15,7 @@ union nvif_outp_args { #define NVIF_OUTP_V0_RELEASE 0x02 #define NVIF_OUTP_V0_INFOFRAME 0x03 #define NVIF_OUTP_V0_HDA_ELD 0x04 +#define NVIF_OUTP_V0_DP_AUX_PWR 0x05 union nvif_outp_load_detect_args { struct nvif_outp_load_detect_v0 { @@ -91,4 +92,12 @@ union nvif_outp_hda_eld_args { __u8 data[]; } v0; }; + +union nvif_outp_dp_aux_pwr_args { + struct nvif_outp_dp_aux_pwr_v0 { + __u8 version; + __u8 state; + __u8 pad02[6]; + } v0; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index 88fd2b9703041..1c960f6fdd7a1 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -26,4 +26,5 @@ int nvif_outp_acquire_dp(struct nvif_outp *, u8 dpcd[16], void nvif_outp_release(struct nvif_outp *); int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_v0 *, u32 size); int nvif_outp_hda_eld(struct nvif_outp *, int head, void *data, u32 size); +int nvif_outp_dp_aux_pwr(struct nvif_outp *, bool enable); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c index 20db8ea1a0baf..b7104e676eb29 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dp.c +++ b/drivers/gpu/drm/nouveau/nouveau_dp.c @@ -140,12 +140,17 @@ nouveau_dp_detect(struct nouveau_connector *nv_connector, * TODO: look into checking this before probing I2C to detect DVI/HDMI */ hpd = nvif_conn_hpd_status(&nv_connector->conn); - if (hpd == NVIF_CONN_HPD_STATUS_NOT_PRESENT) + if (hpd == NVIF_CONN_HPD_STATUS_NOT_PRESENT) { + nvif_outp_dp_aux_pwr(&nv_encoder->outp, false); goto out; + } + nvif_outp_dp_aux_pwr(&nv_encoder->outp, true); status = nouveau_dp_probe_dpcd(nv_connector, nv_encoder); - if (status == connector_status_disconnected) + if (status == connector_status_disconnected) { + nvif_outp_dp_aux_pwr(&nv_encoder->outp, false); goto out; + } /* If we're in MST mode, we're done here */ if (mstm && mstm->can_mst && mstm->is_mst) { @@ -193,6 +198,7 @@ nouveau_dp_detect(struct nouveau_connector *nv_connector, ret = NOUVEAU_DP_MST; goto out; } else if (ret != 0) { + nvif_outp_dp_aux_pwr(&nv_encoder->outp, false); goto out; } } diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 1ae5b19a8025c..9bd9cc21a0396 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -25,6 +25,20 @@ #include +int +nvif_outp_dp_aux_pwr(struct nvif_outp *outp, bool enable) +{ + struct nvif_outp_dp_aux_pwr_v0 args; + int ret; + + args.version = 0; + args.state = enable; + + ret = nvif_object_mthd(&outp->object, NVIF_OUTP_V0_DP_AUX_PWR, &args, sizeof(args)); + NVIF_ERRON(ret, &outp->object, "[DP_AUX_PWR state:%d]", args.state); + return ret; +} + int nvif_outp_hda_eld(struct nvif_outp *outp, int head, void *data, u32 size) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c index f1887b58f472d..92c9faecffae8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c @@ -612,18 +612,38 @@ nvkm_dp_enable_supported_link_rates(struct nvkm_outp *outp) return outp->dp.rates != 0; } -static bool -nvkm_dp_enable(struct nvkm_outp *outp, bool enable) +void +nvkm_dp_enable(struct nvkm_outp *outp, bool auxpwr) { + struct nvkm_gpio *gpio = outp->disp->engine.subdev.device->gpio; struct nvkm_i2c_aux *aux = outp->dp.aux; - if (enable) { - if (!outp->dp.present) { - OUTP_DBG(outp, "aux power -> always"); - nvkm_i2c_aux_monitor(aux, true); - outp->dp.present = true; + if (auxpwr && !outp->dp.aux_pwr) { + /* eDP panels need powering on by us (if the VBIOS doesn't default it + * to on) before doing any AUX channel transactions. LVDS panel power + * is handled by the SOR itself, and not required for LVDS DDC. + */ + if (outp->conn->info.type == DCB_CONNECTOR_eDP) { + int power = nvkm_gpio_get(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff); + if (power == 0) { + nvkm_gpio_set(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff, 1); + outp->dp.aux_pwr_pu = true; + } + + /* We delay here unconditionally, even if already powered, + * because some laptop panels having a significant resume + * delay before the panel begins responding. + * + * This is likely a bit of a hack, but no better idea for + * handling this at the moment. + */ + msleep(300); } + OUTP_DBG(outp, "aux power -> always"); + nvkm_i2c_aux_monitor(aux, true); + outp->dp.aux_pwr = true; + /* Detect any LTTPRs before reading DPCD receiver caps. */ if (!nvkm_rdaux(aux, DPCD_LTTPR_REV, outp->dp.lttpr, sizeof(outp->dp.lttpr)) && outp->dp.lttpr[0] >= 0x14 && outp->dp.lttpr[2]) { @@ -676,19 +696,24 @@ nvkm_dp_enable(struct nvkm_outp *outp, bool enable) outp->dp.rates++; } } - - return true; } - } - - if (outp->dp.present) { + } else + if (!auxpwr && outp->dp.aux_pwr) { OUTP_DBG(outp, "aux power -> demand"); nvkm_i2c_aux_monitor(aux, false); - outp->dp.present = false; - } + outp->dp.aux_pwr = false; + atomic_set(&outp->dp.lt.done, 0); - atomic_set(&outp->dp.lt.done, 0); - return false; + /* Restore eDP panel GPIO to its prior state if we changed it, as + * it could potentially interfere with other outputs. + */ + if (outp->conn->info.type == DCB_CONNECTOR_eDP) { + if (outp->dp.aux_pwr_pu) { + nvkm_gpio_set(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff, 0); + outp->dp.aux_pwr_pu = false; + } + } + } } static int @@ -705,8 +730,6 @@ nvkm_dp_hpd(struct nvkm_notify *notify) if (atomic_read(&outp->dp.lt.done)) outp->func->acquire(outp); rep.mask |= NVIF_NOTIFY_CONN_V0_IRQ; - } else { - nvkm_dp_enable(outp, true); } if (line->mask & NVKM_I2C_UNPLUG) @@ -728,37 +751,8 @@ nvkm_dp_fini(struct nvkm_outp *outp) static void nvkm_dp_init(struct nvkm_outp *outp) { - struct nvkm_gpio *gpio = outp->disp->engine.subdev.device->gpio; - + nvkm_dp_enable(outp, outp->dp.enabled); nvkm_notify_put(&outp->conn->hpd); - - /* eDP panels need powering on by us (if the VBIOS doesn't default it - * to on) before doing any AUX channel transactions. LVDS panel power - * is handled by the SOR itself, and not required for LVDS DDC. - */ - if (outp->conn->info.type == DCB_CONNECTOR_eDP) { - int power = nvkm_gpio_get(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff); - if (power == 0) - nvkm_gpio_set(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff, 1); - - /* We delay here unconditionally, even if already powered, - * because some laptop panels having a significant resume - * delay before the panel begins responding. - * - * This is likely a bit of a hack, but no better idea for - * handling this at the moment. - */ - msleep(300); - - /* If the eDP panel can't be detected, we need to restore - * the panel power GPIO to avoid breaking another output. - */ - if (!nvkm_dp_enable(outp, true) && power == 0) - nvkm_gpio_set(gpio, 0, DCB_GPIO_PANEL_POWER, 0xff, 0); - } else { - nvkm_dp_enable(outp, true); - } - nvkm_notify_get(&outp->dp.hpd); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h index 1d86baa6a424a..9a6be43916bcc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.h @@ -6,6 +6,7 @@ int nvkm_dp_new(struct nvkm_disp *, int index, struct dcb_output *, struct nvkm_outp **); void nvkm_dp_disable(struct nvkm_outp *, struct nvkm_ior *); +void nvkm_dp_enable(struct nvkm_outp *, bool auxpwr); /* DPCD Receiver Capabilities */ #define DPCD_RC00_DPCD_REV 0x00000 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h index d7c989e1cd01b..66def8ae31655 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h @@ -39,7 +39,9 @@ struct nvkm_outp { struct nvkm_i2c_aux *aux; struct nvkm_notify hpd; - bool present; + bool enabled; + bool aux_pwr; + bool aux_pwr_pu; u8 lttpr[6]; u8 lttprs; u8 dpcd[16]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index 61d41b326f43c..3bfc3e39778e2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -21,11 +21,25 @@ */ #define nvkm_uoutp(p) container_of((p), struct nvkm_outp, object) #include "outp.h" +#include "dp.h" #include "head.h" #include "ior.h" #include +static int +nvkm_uoutp_mthd_dp_aux_pwr(struct nvkm_outp *outp, void *argv, u32 argc) +{ + union nvif_outp_dp_aux_pwr_args *args = argv; + + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + outp->dp.enabled = !!args->v0.state; + nvkm_dp_enable(outp, outp->dp.enabled); + return 0; +} + static int nvkm_uoutp_mthd_hda_eld(struct nvkm_outp *outp, void *argv, u32 argc) { @@ -250,6 +264,7 @@ nvkm_uoutp_mthd_noacquire(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc switch (mthd) { case NVIF_OUTP_V0_LOAD_DETECT: return nvkm_uoutp_mthd_load_detect(outp, argv, argc); case NVIF_OUTP_V0_ACQUIRE : return nvkm_uoutp_mthd_acquire (outp, argv, argc); + case NVIF_OUTP_V0_DP_AUX_PWR : return nvkm_uoutp_mthd_dp_aux_pwr (outp, argv, argc); default: break; } -- GitLab From c61b3a2b2d9bb36698f8c2f65aa41ba183815264 Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Wed, 26 Oct 2022 15:13:29 +0000 Subject: [PATCH 1246/1927] maple_tree: remove pointer to pointer use in mas_alloc_nodes() There is a more direct and cleaner way of implementing the same functional code. Remove the confusing and unnecessary use of pointers here. Link: https://lkml.kernel.org/r/20221026151241.4031117-1-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Suggested-by: Lukas Bulwahn Signed-off-by: Andrew Morton --- lib/maple_tree.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index fbde494444b89..72e3b6a9c021a 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -1209,7 +1209,6 @@ static inline void mas_push_node(struct ma_state *mas, struct maple_node *used) static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) { struct maple_alloc *node; - struct maple_alloc **nodep = &mas->alloc; unsigned long allocated = mas_allocated(mas); unsigned long success = allocated; unsigned int requested = mas_alloc_req(mas); @@ -1263,8 +1262,7 @@ static inline void mas_alloc_nodes(struct ma_state *mas, gfp_t gfp) node->node_count--; success += count; - nodep = &node->slot[0]; - node = *nodep; + node = node->slot[0]; requested -= count; } mas->alloc->total = success; -- GitLab From 9a887877ef981e5a185a84339603300cf2eb1900 Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Wed, 26 Oct 2022 15:14:31 +0000 Subject: [PATCH 1247/1927] maple_tree: mas_anode_descend() clang-analyzer cleanup clang-analyzer reported some Dead Stores in mas_anode_descend(). Upon inspection, there were a few clean ups that would make the code cleaner: The count variable was set from the mt_slots array and then updated but never used again. Just use the array reference directly. Also stop updating the type since it isn't used after the update. Stop setting the gaps pointer to NULL at the start since it is always set before the loop begins. Link: https://lkml.kernel.org/r/20221026151413.4032730-1-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Suggested-by: Lukas Bulwahn Signed-off-by: Andrew Morton --- lib/maple_tree.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 72e3b6a9c021a..4c7eef927f1a0 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -4968,8 +4968,9 @@ static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) { enum maple_type type = mte_node_type(mas->node); unsigned long pivot, min, gap = 0; - unsigned char count, offset; - unsigned long *gaps = NULL, *pivots = ma_pivots(mas_mn(mas), type); + unsigned char offset; + unsigned long *gaps; + unsigned long *pivots = ma_pivots(mas_mn(mas), type); void __rcu **slots = ma_slots(mas_mn(mas), type); bool found = false; @@ -4980,9 +4981,8 @@ static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) gaps = ma_gaps(mte_to_node(mas->node), type); offset = mas->offset; - count = mt_slots[type]; min = mas_safe_min(mas, pivots, offset); - for (; offset < count; offset++) { + for (; offset < mt_slots[type]; offset++) { pivot = mas_safe_pivot(mas, pivots, offset, type); if (offset && !pivot) break; @@ -5008,8 +5008,6 @@ static inline bool mas_anode_descend(struct ma_state *mas, unsigned long size) mas->min = min; mas->max = pivot; offset = 0; - type = mte_node_type(mas->node); - count = mt_slots[type]; break; } } -- GitLab From 120b116208a0877227fc82e3f0df81e7a3ed4ab1 Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Fri, 28 Oct 2022 18:04:30 +0000 Subject: [PATCH 1248/1927] maple_tree: reorganize testing to restore module testing Along the development cycle, the testing code support for module/in-kernel compiles was removed. Restore this functionality by moving any internal API tests to the userspace side, as well as threading tests. Fix the lockdep issues and add a way to reduce memory usage so the tests can complete with KASAN + memleak detection. Make the tests work on 32 bit hosts where possible and detect 32 bit hosts in the radix test suite. [akpm@linux-foundation.org: fix module export] [akpm@linux-foundation.org: fix it some more] [liam.howlett@oracle.com: fix compile warnings on 32bit build in check_find()] Link: https://lkml.kernel.org/r/20221107203816.1260327-1-Liam.Howlett@oracle.com Link: https://lkml.kernel.org/r/20221028180415.3074673-1-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Signed-off-by: Andrew Morton --- include/linux/maple_tree.h | 7 + lib/Kconfig.debug | 4 + lib/Makefile | 1 + lib/maple_tree.c | 38 +- lib/test_maple_tree.c | 37926 +--------------- tools/testing/radix-tree/.gitignore | 1 + tools/testing/radix-tree/Makefile | 19 +- tools/testing/radix-tree/generated/autoconf.h | 2 +- tools/testing/radix-tree/linux.c | 4 + tools/testing/radix-tree/maple.c | 35770 +++++++++++++++ 10 files changed, 37029 insertions(+), 36743 deletions(-) diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 2effab72add10..e594db58a0f14 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -638,6 +638,12 @@ static inline void mt_set_in_rcu(struct maple_tree *mt) } } +static inline unsigned int mt_height(const struct maple_tree *mt) + +{ + return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; +} + void *mt_find(struct maple_tree *mt, unsigned long *index, unsigned long max); void *mt_find_after(struct maple_tree *mt, unsigned long *index, unsigned long max); @@ -664,6 +670,7 @@ extern atomic_t maple_tree_tests_passed; void mt_dump(const struct maple_tree *mt); void mt_validate(struct maple_tree *mt); +void mt_cache_shrink(void); #define MT_BUG_ON(__tree, __x) do { \ atomic_inc(&maple_tree_tests_run); \ if (__x) { \ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 29280072dc0e4..be69844e40e64 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2241,6 +2241,10 @@ config TEST_UUID config TEST_XARRAY tristate "Test the XArray code at runtime" +config TEST_MAPLE_TREE + select DEBUG_MAPLE_TREE + tristate "Test the Maple Tree code at runtime" + config TEST_RHASHTABLE tristate "Perform selftest on resizable hash table" help diff --git a/lib/Makefile b/lib/Makefile index 161d6a724ff71..59bd7c2f793a7 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -85,6 +85,7 @@ obj-$(CONFIG_TEST_BITMAP) += test_bitmap.o obj-$(CONFIG_TEST_STRSCPY) += test_strscpy.o obj-$(CONFIG_TEST_UUID) += test_uuid.o obj-$(CONFIG_TEST_XARRAY) += test_xarray.o +obj-$(CONFIG_TEST_MAPLE_TREE) += test_maple_tree.o obj-$(CONFIG_TEST_PARMAN) += test_parman.o obj-$(CONFIG_TEST_KMOD) += test_kmod.o obj-$(CONFIG_TEST_DEBUG_VIRTUAL) += test_debug_virtual.o diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 4c7eef927f1a0..f23f11da4113b 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -183,10 +183,6 @@ static void ma_free_rcu(struct maple_node *node) call_rcu(&node->rcu, mt_free_rcu); } -static unsigned int mt_height(const struct maple_tree *mt) -{ - return (mt->ma_flags & MT_FLAGS_HEIGHT_MASK) >> MT_FLAGS_HEIGHT_OFFSET; -} static void mas_set_height(struct ma_state *mas) { @@ -5061,6 +5057,7 @@ void *mas_walk(struct ma_state *mas) return entry; } +EXPORT_SYMBOL_GPL(mas_walk); static inline bool mas_rewind_node(struct ma_state *mas) { @@ -5272,6 +5269,7 @@ int mas_empty_area(struct ma_state *mas, unsigned long min, mas->last = mas->index + size - 1; return 0; } +EXPORT_SYMBOL_GPL(mas_empty_area); /* * mas_empty_area_rev() - Get the highest address within the range that is @@ -5335,6 +5333,7 @@ int mas_empty_area_rev(struct ma_state *mas, unsigned long min, mas->index = mas->last - size + 1; return 0; } +EXPORT_SYMBOL_GPL(mas_empty_area_rev); static inline int mas_alloc(struct ma_state *mas, void *entry, unsigned long size, unsigned long *index) @@ -5656,6 +5655,7 @@ void *mas_store(struct ma_state *mas, void *entry) mas_wr_store_entry(&wr_mas); return wr_mas.content; } +EXPORT_SYMBOL_GPL(mas_store); /** * mas_store_gfp() - Store a value into the tree. @@ -5682,6 +5682,7 @@ int mas_store_gfp(struct ma_state *mas, void *entry, gfp_t gfp) return 0; } +EXPORT_SYMBOL_GPL(mas_store_gfp); /** * mas_store_prealloc() - Store a value into the tree using memory @@ -5699,6 +5700,7 @@ void mas_store_prealloc(struct ma_state *mas, void *entry) BUG_ON(mas_is_err(mas)); mas_destroy(mas); } +EXPORT_SYMBOL_GPL(mas_store_prealloc); /** * mas_preallocate() - Preallocate enough nodes for a store operation @@ -5768,6 +5770,7 @@ void mas_destroy(struct ma_state *mas) } mas->alloc = NULL; } +EXPORT_SYMBOL_GPL(mas_destroy); /* * mas_expected_entries() - Set the expected number of entries that will be inserted. @@ -5829,6 +5832,7 @@ int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries) return ret; } +EXPORT_SYMBOL_GPL(mas_expected_entries); /** * mas_next() - Get the next entry. @@ -6009,6 +6013,7 @@ void *mas_find(struct ma_state *mas, unsigned long max) /* Retries on dead nodes handled by mas_next_entry */ return mas_next_entry(mas, max); } +EXPORT_SYMBOL_GPL(mas_find); /** * mas_find_rev: On the first call, find the first non-null entry at or below @@ -6055,7 +6060,7 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min) /* Retries on dead nodes handled by mas_next_entry */ return mas_prev_entry(mas, min); } -EXPORT_SYMBOL_GPL(mas_find); +EXPORT_SYMBOL_GPL(mas_find_rev); /** * mas_erase() - Find the range in which index resides and erase the entire @@ -6537,8 +6542,27 @@ static inline int mas_dead_node(struct ma_state *mas, unsigned long index) mas_rewalk(mas, index); return 1; } -#endif /* not defined __KERNEL__ */ +void mt_cache_shrink(void) +{ +} +#else +/* + * mt_cache_shrink() - For testing, don't use this. + * + * Certain testcases can trigger an OOM when combined with other memory + * debugging configuration options. This function is used to reduce the + * possibility of an out of memory even due to kmem_cache objects remaining + * around for longer than usual. + */ +void mt_cache_shrink(void) +{ + kmem_cache_shrink(maple_node_cache); + +} +EXPORT_SYMBOL_GPL(mt_cache_shrink); + +#endif /* not defined __KERNEL__ */ /* * mas_get_slot() - Get the entry in the maple state node stored at @offset. * @mas: The maple state @@ -6812,6 +6836,7 @@ void mt_dump(const struct maple_tree *mt) else if (entry) mt_dump_node(mt, entry, 0, mt_max[mte_node_type(entry)], 0); } +EXPORT_SYMBOL_GPL(mt_dump); /* * Calculate the maximum gap in a node and check if that's what is reported in @@ -7122,5 +7147,6 @@ void mt_validate(struct maple_tree *mt) rcu_read_unlock(); } +EXPORT_SYMBOL_GPL(mt_validate); #endif /* CONFIG_DEBUG_MAPLE_TREE */ diff --git a/lib/test_maple_tree.c b/lib/test_maple_tree.c index 4f69e009a015d..f425f169ef089 100644 --- a/lib/test_maple_tree.c +++ b/lib/test_maple_tree.c @@ -1,24 +1,35 @@ // SPDX-License-Identifier: GPL-2.0+ /* * test_maple_tree.c: Test the maple tree API - * Copyright (c) 2018 Liam R. Howlett + * Copyright (c) 2018-2022 Oracle Corporation * Author: Liam R. Howlett + * + * Any tests that only require the interface of the tree. */ #include #include -#include -#include #define MTREE_ALLOC_MAX 0x2000000000000Ul +#ifndef CONFIG_DEBUG_MAPLE_TREE #define CONFIG_DEBUG_MAPLE_TREE +#endif #define CONFIG_MAPLE_SEARCH +#define MAPLE_32BIT (MAPLE_NODE_SLOTS > 31) + /* #define BENCH_SLOT_STORE */ /* #define BENCH_NODE_STORE */ /* #define BENCH_AWALK */ /* #define BENCH_WALK */ /* #define BENCH_MT_FOR_EACH */ /* #define BENCH_FORK */ + +#ifdef __KERNEL__ +#define mt_set_non_kernel(x) do {} while (0) +#define mt_zero_nr_tallocated(x) do {} while (0) +#else +#define cond_resched() do {} while (0) +#endif static int mtree_insert_index(struct maple_tree *mt, unsigned long index, gfp_t gfp) { @@ -65,6 +76,7 @@ static void *mtree_test_erase(struct maple_tree *mt, unsigned long index) return mtree_erase(mt, index); } +#if defined(CONFIG_64BIT) static noinline void check_mtree_alloc_range(struct maple_tree *mt, unsigned long start, unsigned long end, unsigned long size, unsigned long expected, int eret, void *ptr) @@ -98,6 +110,7 @@ static noinline void check_mtree_alloc_rrange(struct maple_tree *mt, MT_BUG_ON(mt, result != expected); } +#endif static noinline void check_load(struct maple_tree *mt, unsigned long index, void *ptr) @@ -150,12 +163,6 @@ static noinline void check_insert(struct maple_tree *mt, unsigned long index, MT_BUG_ON(mt, ret != 0); } -static noinline void check_erase(struct maple_tree *mt, unsigned long index, - void *ptr) -{ - MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr); -} - static noinline void check_dup_insert(struct maple_tree *mt, unsigned long index, void *ptr) { @@ -172,41 +179,6 @@ void check_index_load(struct maple_tree *mt, unsigned long index) return check_load(mt, index, xa_mk_value(index & LONG_MAX)); } -static noinline void check_nomem(struct maple_tree *mt) -{ - MA_STATE(ms, mt, 1, 1); - - MT_BUG_ON(mt, !mtree_empty(mt)); - /* Ensure no bypassing of allocation failures */ - mt_set_non_kernel(0); - - /* Storing something at 1 requires memory allocation */ - MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM); - /* Storing something at 0 does not */ - MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0); - - /* - * Simulate two threads racing; the first one fails to allocate - * memory to insert an entry at 1, then the second one succeeds - * in allocating memory to insert an entry at 2. The first one - * then needs to free the node it allocated. LeakSanitizer will - * notice this, as will the 'nr_allocated' debugging aid in the - * userspace test suite. - */ - mtree_lock(mt); - mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */ - MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM)); - mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ - MT_BUG_ON(mt, ms.node != MAS_START); - mtree_unlock(mt); - MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0); - mtree_lock(mt); - mas_store(&ms, &ms); /* insert 1 -> &ms */ - mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ - mtree_unlock(mt); - mtree_destroy(mt); -} - static inline int not_empty(struct maple_node *node) { int i; @@ -221,350 +193,6 @@ static inline int not_empty(struct maple_node *node) return 0; } -static noinline void check_new_node(struct maple_tree *mt) -{ - - struct maple_node *mn, *mn2, *mn3; - struct maple_alloc *smn; - struct maple_node *nodes[100]; - int i, j, total; - - MA_STATE(mas, mt, 0, 0); - - /* Try allocating 3 nodes */ - mtree_lock(mt); - /* request 3 nodes to be allocated. */ - mas_node_count(&mas, 3); - /* Allocation request of 3. */ - MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); - /* Allocate failed. */ - MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - - MT_BUG_ON(mt, mas_allocated(&mas) != 3); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, mas.alloc == NULL); - MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); - mas_push_node(&mas, mn); - mas_nomem(&mas, GFP_KERNEL); /* free */ - mtree_unlock(mt); - - - /* Try allocating 1 node, then 2 more */ - mtree_lock(mt); - /* Set allocation request to 1. */ - mas_set_alloc_req(&mas, 1); - /* Check Allocation request of 1. */ - MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); - mas_set_err(&mas, -ENOMEM); - /* Validate allocation request. */ - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - /* Eat the requested node. */ - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, mn->slot[0] != NULL); - MT_BUG_ON(mt, mn->slot[1] != NULL); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - - ma_free_rcu(mn); - mas.node = MAS_START; - mas_nomem(&mas, GFP_KERNEL); - /* Allocate 3 nodes, will fail. */ - mas_node_count(&mas, 3); - /* Drop the lock and allocate 3 nodes. */ - mas_nomem(&mas, GFP_KERNEL); - /* Ensure 3 are allocated. */ - MT_BUG_ON(mt, mas_allocated(&mas) != 3); - /* Allocation request of 0. */ - MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); - - MT_BUG_ON(mt, mas.alloc == NULL); - MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); - MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); - /* Ensure we counted 3. */ - MT_BUG_ON(mt, mas_allocated(&mas) != 3); - /* Free. */ - mas_nomem(&mas, GFP_KERNEL); - - /* Set allocation request to 1. */ - mas_set_alloc_req(&mas, 1); - MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); - mas_set_err(&mas, -ENOMEM); - /* Validate allocation request. */ - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - MT_BUG_ON(mt, mas_allocated(&mas) != 1); - /* Check the node is only one node. */ - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, mn->slot[0] != NULL); - MT_BUG_ON(mt, mn->slot[1] != NULL); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - mas_push_node(&mas, mn); - MT_BUG_ON(mt, mas_allocated(&mas) != 1); - MT_BUG_ON(mt, mas.alloc->node_count); - - mas_set_alloc_req(&mas, 2); /* request 2 more. */ - MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); - mas_set_err(&mas, -ENOMEM); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - MT_BUG_ON(mt, mas_allocated(&mas) != 3); - MT_BUG_ON(mt, mas.alloc == NULL); - MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); - MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); - for (i = 2; i >= 0; i--) { - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, mas_allocated(&mas) != i); - MT_BUG_ON(mt, !mn); - MT_BUG_ON(mt, not_empty(mn)); - ma_free_rcu(mn); - } - - total = 64; - mas_set_alloc_req(&mas, total); /* request 2 more. */ - MT_BUG_ON(mt, mas_alloc_req(&mas) != total); - mas_set_err(&mas, -ENOMEM); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - for (i = total; i > 0; i--) { - unsigned int e = 0; /* expected node_count */ - - if (i >= 35) - e = i - 35; - else if (i >= 5) - e = i - 5; - else if (i >= 2) - e = i - 2; - MT_BUG_ON(mt, mas.alloc->node_count != e); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); - MT_BUG_ON(mt, !mn); - ma_free_rcu(mn); - } - - total = 100; - for (i = 1; i < total; i++) { - mas_set_alloc_req(&mas, i); - mas_set_err(&mas, -ENOMEM); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - for (j = i; j > 0; j--) { - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); - MT_BUG_ON(mt, !mn); - MT_BUG_ON(mt, not_empty(mn)); - mas_push_node(&mas, mn); - MT_BUG_ON(mt, mas_allocated(&mas) != j); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); - ma_free_rcu(mn); - } - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - - mas_set_alloc_req(&mas, i); - mas_set_err(&mas, -ENOMEM); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - for (j = 0; j <= i/2; j++) { - MT_BUG_ON(mt, mas_allocated(&mas) != i - j); - nodes[j] = mas_pop_node(&mas); - MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); - } - - while (j) { - j--; - mas_push_node(&mas, nodes[j]); - MT_BUG_ON(mt, mas_allocated(&mas) != i - j); - } - MT_BUG_ON(mt, mas_allocated(&mas) != i); - for (j = 0; j <= i/2; j++) { - MT_BUG_ON(mt, mas_allocated(&mas) != i - j); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - ma_free_rcu(mn); - MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); - } - MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); - - } - - /* Set allocation request. */ - total = 500; - mas_node_count(&mas, total); - /* Drop the lock and allocate the nodes. */ - mas_nomem(&mas, GFP_KERNEL); - MT_BUG_ON(mt, !mas.alloc); - i = 1; - smn = mas.alloc; - while (i < total) { - for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) { - i++; - MT_BUG_ON(mt, !smn->slot[j]); - if (i == total) - break; - } - smn = smn->slot[0]; /* next. */ - } - MT_BUG_ON(mt, mas_allocated(&mas) != total); - mas_nomem(&mas, GFP_KERNEL); /* Free. */ - - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - for (i = 1; i < 128; i++) { - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ - for (j = i; j > 0; j--) { /*Free the requests */ - mn = mas_pop_node(&mas); /* get the next node. */ - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, not_empty(mn)); - ma_free_rcu(mn); - } - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - } - - for (i = 1; i < MAPLE_NODE_MASK + 1; i++) { - MA_STATE(mas2, mt, 0, 0); - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ - for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */ - mn = mas_pop_node(&mas); /* get the next node. */ - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, not_empty(mn)); - mas_push_node(&mas2, mn); - MT_BUG_ON(mt, mas_allocated(&mas2) != j); - } - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - MT_BUG_ON(mt, mas_allocated(&mas2) != i); - - for (j = i; j > 0; j--) { /*Free the requests */ - MT_BUG_ON(mt, mas_allocated(&mas2) != j); - mn = mas_pop_node(&mas2); /* get the next node. */ - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, not_empty(mn)); - ma_free_rcu(mn); - } - MT_BUG_ON(mt, mas_allocated(&mas2) != 0); - } - - - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ - MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); - MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); - - mn = mas_pop_node(&mas); /* get the next node. */ - MT_BUG_ON(mt, mn == NULL); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); - MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2); - - mas_push_node(&mas, mn); - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); - MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); - - /* Check the limit of pop/push/pop */ - mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ - MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); - MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); - MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); - MT_BUG_ON(mt, mas_alloc_req(&mas)); - MT_BUG_ON(mt, mas.alloc->node_count); - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); - MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); - mas_push_node(&mas, mn); - MT_BUG_ON(mt, mas.alloc->node_count); - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - ma_free_rcu(mn); - for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) { - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, not_empty(mn)); - ma_free_rcu(mn); - } - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - - - for (i = 3; i < MAPLE_NODE_MASK * 3; i++) { - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mn = mas_pop_node(&mas); /* get the next node. */ - mas_push_node(&mas, mn); /* put it back */ - mas_destroy(&mas); - - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mn = mas_pop_node(&mas); /* get the next node. */ - mn2 = mas_pop_node(&mas); /* get the next node. */ - mas_push_node(&mas, mn); /* put them back */ - mas_push_node(&mas, mn2); - mas_destroy(&mas); - - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mn = mas_pop_node(&mas); /* get the next node. */ - mn2 = mas_pop_node(&mas); /* get the next node. */ - mn3 = mas_pop_node(&mas); /* get the next node. */ - mas_push_node(&mas, mn); /* put them back */ - mas_push_node(&mas, mn2); - mas_push_node(&mas, mn3); - mas_destroy(&mas); - - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mn = mas_pop_node(&mas); /* get the next node. */ - ma_free_rcu(mn); - mas_destroy(&mas); - - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, i); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mn = mas_pop_node(&mas); /* get the next node. */ - ma_free_rcu(mn); - mn = mas_pop_node(&mas); /* get the next node. */ - ma_free_rcu(mn); - mn = mas_pop_node(&mas); /* get the next node. */ - ma_free_rcu(mn); - mas_destroy(&mas); - } - - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, 5); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - MT_BUG_ON(mt, mas_allocated(&mas) != 5); - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, 10); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mas.node = MAS_START; - MT_BUG_ON(mt, mas_allocated(&mas) != 10); - mas_destroy(&mas); - - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); - mas.node = MA_ERROR(-ENOMEM); - mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ - mas_nomem(&mas, GFP_KERNEL); /* Fill request */ - mas.node = MAS_START; - MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); - mas_destroy(&mas); - - mtree_unlock(mt); -} static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, bool verbose) @@ -588,6 +216,7 @@ static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, } check_load(mt, max + 1, NULL); +#ifndef __KERNEL__ if (verbose) { rcu_barrier(); mt_dump(mt); @@ -595,6 +224,7 @@ static noinline void check_rev_seq(struct maple_tree *mt, unsigned long max, __func__, max, mt_get_alloc_size()/1024, mt_nr_allocated(), mt_nr_tallocated()); } +#endif } static noinline void check_seq(struct maple_tree *mt, unsigned long max, @@ -614,6 +244,8 @@ static noinline void check_seq(struct maple_tree *mt, unsigned long max, MT_BUG_ON(mt, !mt_height(mt)); check_load(mt, i + 1, NULL); } + +#ifndef __KERNEL__ if (verbose) { rcu_barrier(); mt_dump(mt); @@ -621,6 +253,7 @@ static noinline void check_seq(struct maple_tree *mt, unsigned long max, max, mt_get_alloc_size()/1024, mt_nr_allocated(), mt_nr_tallocated()); } +#endif } static noinline void check_lb_not_empty(struct maple_tree *mt) @@ -651,10 +284,15 @@ static noinline void check_lower_bound_split(struct maple_tree *mt) static noinline void check_upper_bound_split(struct maple_tree *mt) { unsigned long i, j; - unsigned long huge = 4000UL * 1000 * 1000; + unsigned long huge; MT_BUG_ON(mt, !mtree_empty(mt)); + if (MAPLE_32BIT) + huge = 2147483647UL; + else + huge = 4000UL * 1000 * 1000; + i = 4096; while (i < huge) { check_insert(mt, i, (void *) i); @@ -687,6 +325,7 @@ static noinline void check_rev_find(struct maple_tree *mt) mtree_store_range(mt, i*10, i*10 + 5, xa_mk_value(i), GFP_KERNEL); + rcu_read_lock(); mas_set(&mas, 1000); val = mas_find_rev(&mas, 1000); MT_BUG_ON(mt, val != xa_mk_value(100)); @@ -712,13 +351,15 @@ static noinline void check_rev_find(struct maple_tree *mt) MT_BUG_ON(mt, val != xa_mk_value(0)); val = mas_find_rev(&mas, 0); MT_BUG_ON(mt, val != NULL); + rcu_read_unlock(); } static noinline void check_find(struct maple_tree *mt) { unsigned long val = 0; - unsigned long count = 20; + unsigned long count; unsigned long max; + unsigned long top; unsigned long last = 0, index = 0; void *entry, *entry2; @@ -727,6 +368,18 @@ static noinline void check_find(struct maple_tree *mt) /* Insert 0. */ MT_BUG_ON(mt, mtree_insert_index(mt, val++, GFP_KERNEL)); +#if defined(CONFIG_64BIT) + top = 4398046511104UL; +#else + top = ULONG_MAX; +#endif + + if (MAPLE_32BIT) { + count = 15; + } else { + count = 20; + } + for (int i = 0; i <= count; i++) { if (val != 64) MT_BUG_ON(mt, mtree_insert_index(mt, val, GFP_KERNEL)); @@ -805,12 +458,17 @@ static noinline void check_find(struct maple_tree *mt) index = 0; MT_BUG_ON(mt, mtree_insert_index(mt, ULONG_MAX, GFP_KERNEL)); mt_for_each(mt, entry, index, ULONG_MAX) { - if (val == 4398046511104) - MT_BUG_ON(mt, entry != - xa_mk_value(ULONG_MAX & LONG_MAX)); + if (val == top) + MT_BUG_ON(mt, entry != xa_mk_value(LONG_MAX)); else MT_BUG_ON(mt, xa_mk_value(val) != entry); - val <<= 2; + + /* Workaround for 32bit */ + if ((val << 2) < val) + val = ULONG_MAX; + else + val <<= 2; + if (val == 64) /* Skip zero entry. */ val <<= 2; /* For zero check. */ @@ -842,11 +500,16 @@ static noinline void check_find(struct maple_tree *mt) mas_for_each(&mas, entry, ULONG_MAX) { if (val == 64) MT_BUG_ON(mt, entry != XA_ZERO_ENTRY); - else if (val == 4398046511104) - MT_BUG_ON(mt, entry != xa_mk_value(ULONG_MAX & LONG_MAX)); + else if (val == top) + MT_BUG_ON(mt, entry != xa_mk_value(LONG_MAX)); else MT_BUG_ON(mt, xa_mk_value(val) != entry); - val <<= 2; + + /* Workaround for 32bit */ + if ((val << 2) < val) + val = ULONG_MAX; + else + val <<= 2; /* For zero check. */ if (!val) @@ -951,36548 +614,1350 @@ static noinline void check_find_2(struct maple_tree *mt) /*MT_BUG_ON(mt, !mtree_empty(mt)); */ } -#define erase_ptr(i) entry[i%2] -#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2]) -#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2]) -#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2]) -static noinline void check_erase_testset(struct maple_tree *mt) +#if defined(CONFIG_64BIT) +static noinline void check_alloc_rev_range(struct maple_tree *mt) { - unsigned long set[] = { 5015, 5014, 5017, 25, 1000, - 1001, 1002, 1003, 1005, 0, - 6003, 6002, 6008, 6012, 6015, - 7003, 7002, 7008, 7012, 7015, - 8003, 8002, 8008, 8012, 8015, - 9003, 9002, 9008, 9012, 9015, - 10003, 10002, 10008, 10012, 10015, - 11003, 11002, 11008, 11012, 11015, - 12003, 12002, 12008, 12012, 12015, - 13003, 13002, 13008, 13012, 13015, - 14003, 14002, 14008, 14012, 14015, - 15003, 15002, 15008, 15012, 15015, - }; - - - void *ptr = &set; - void *entry[2] = { ptr, mt }; - void *root_node; - - - rcu_register_thread(); - mt_set_in_rcu(mt); - for (int i = 0; i < 4; i++) - erase_check_insert(mt, i); - for (int i = 0; i < 4; i++) - erase_check_load(mt, i); - - mt_set_non_kernel(2); - erase_check_erase(mt, 1); - erase_check_load(mt, 0); - check_load(mt, set[1], NULL); - for (int i = 2; i < 4; i++) - erase_check_load(mt, i); - - - erase_check_erase(mt, 2); - erase_check_load(mt, 0); - check_load(mt, set[1], NULL); - check_load(mt, set[2], NULL); - - erase_check_insert(mt, 1); - erase_check_insert(mt, 2); - - for (int i = 0; i < 4; i++) - erase_check_load(mt, i); - - /* Check erase and load without an allocation. */ - erase_check_load(mt, 3); - erase_check_erase(mt, 1); - erase_check_load(mt, 0); - check_load(mt, set[1], NULL); - for (int i = 2; i < 4; i++) - erase_check_load(mt, i); - /* - * Set the newly erased node. This will produce a different allocated - * node to avoid busy slots. + * Generated by: + * cat /proc/self/maps | awk '{print $1}'| + * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' */ - root_node = mt->ma_root; - erase_check_insert(mt, 1); - - erase_check_load(mt, 0); - check_load(mt, 5016, NULL); - erase_check_load(mt, 1); - check_load(mt, 5013, NULL); - erase_check_load(mt, 2); - check_load(mt, 5018, NULL); - erase_check_load(mt, 3); - - erase_check_erase(mt, 2); /* erase 5017 to check append */ - erase_check_load(mt, 0); - check_load(mt, 5016, NULL); - erase_check_load(mt, 1); - check_load(mt, 5013, NULL); - check_load(mt, set[2], NULL); - check_load(mt, 5018, NULL); - - erase_check_load(mt, 3); - - root_node = mt->ma_root; - erase_check_insert(mt, 2); - - erase_check_load(mt, 0); - check_load(mt, 5016, NULL); - erase_check_load(mt, 1); - check_load(mt, 5013, NULL); - erase_check_load(mt, 2); - check_load(mt, 5018, NULL); - erase_check_load(mt, 3); - mt_set_non_kernel(1); - erase_check_erase(mt, 2); /* erase 5017 to check append */ - erase_check_load(mt, 0); - check_load(mt, 5016, NULL); - check_load(mt, set[2], NULL); - erase_check_erase(mt, 0); /* erase 5015 to check append */ - check_load(mt, set[0], NULL); - check_load(mt, 5016, NULL); - erase_check_insert(mt, 4); /* 1000 < Should not split. */ - check_load(mt, set[0], NULL); - check_load(mt, 5016, NULL); - erase_check_load(mt, 1); - check_load(mt, 5013, NULL); - check_load(mt, set[2], NULL); - check_load(mt, 5018, NULL); - erase_check_load(mt, 4); - check_load(mt, 999, NULL); - check_load(mt, 1001, NULL); - erase_check_load(mt, 4); - if (mt_in_rcu(mt)) - MT_BUG_ON(mt, root_node == mt->ma_root); - else - MT_BUG_ON(mt, root_node != mt->ma_root); + unsigned long range[] = { + /* Inclusive , Exclusive. */ + 0x565234af2000, 0x565234af4000, + 0x565234af4000, 0x565234af9000, + 0x565234af9000, 0x565234afb000, + 0x565234afc000, 0x565234afd000, + 0x565234afd000, 0x565234afe000, + 0x565235def000, 0x565235e10000, + 0x7f36d4bfd000, 0x7f36d4ee2000, + 0x7f36d4ee2000, 0x7f36d4f04000, + 0x7f36d4f04000, 0x7f36d504c000, + 0x7f36d504c000, 0x7f36d5098000, + 0x7f36d5098000, 0x7f36d5099000, + 0x7f36d5099000, 0x7f36d509d000, + 0x7f36d509d000, 0x7f36d509f000, + 0x7f36d509f000, 0x7f36d50a5000, + 0x7f36d50b9000, 0x7f36d50db000, + 0x7f36d50db000, 0x7f36d50dc000, + 0x7f36d50dc000, 0x7f36d50fa000, + 0x7f36d50fa000, 0x7f36d5102000, + 0x7f36d5102000, 0x7f36d5103000, + 0x7f36d5103000, 0x7f36d5104000, + 0x7f36d5104000, 0x7f36d5105000, + 0x7fff5876b000, 0x7fff5878d000, + 0x7fff5878e000, 0x7fff58791000, + 0x7fff58791000, 0x7fff58793000, + }; - /* Should not have split. */ - MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root)); + unsigned long holes[] = { + /* + * Note: start of hole is INCLUSIVE + * end of hole is EXCLUSIVE + * (opposite of the above table.) + * Start of hole, end of hole, size of hole (+1) + */ + 0x565234afb000, 0x565234afc000, 0x1000, + 0x565234afe000, 0x565235def000, 0x12F1000, + 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, + }; + /* + * req_range consists of 4 values. + * 1. min index + * 2. max index + * 3. size + * 4. number that should be returned. + * 5. return value + */ + unsigned long req_range[] = { + 0x565234af9000, /* Min */ + 0x7fff58791000, /* Max */ + 0x1000, /* Size */ + 0x7fff5878d << 12, /* First rev hole of size 0x1000 */ + 0, /* Return value success. */ - /* Coalesce testing */ - erase_check_insert(mt, 0); - erase_check_insert(mt, 2); + 0x0, /* Min */ + 0x565234AF1 << 12, /* Max */ + 0x3000, /* Size */ + 0x565234AEE << 12, /* max - 3. */ + 0, /* Return value success. */ - for (int i = 5; i < 25; i++) { - erase_check_insert(mt, i); - for (int j = i; j >= 0; j--) - erase_check_load(mt, j); - } + 0x0, /* Min */ + -1, /* Max */ + 0x1000, /* Size */ + 562949953421311 << 12,/* First rev hole of size 0x1000 */ + 0, /* Return value success. */ - erase_check_erase(mt, 14); /*6015 */ - for (int i = 0; i < 25; i++) { - if (i == 14) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } - erase_check_erase(mt, 16); /*7002 */ - for (int i = 0; i < 25; i++) { - if (i == 16 || i == 14) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } + 0x0, /* Min */ + 0x7F36D510A << 12, /* Max */ + 0x4000, /* Size */ + 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */ + 0, /* Return value success. */ + /* Ascend test. */ + 0x0, + 34148798629 << 12, + 19 << 12, + 34148797418 << 12, + 0x0, - mt_set_non_kernel(1); - erase_check_erase(mt, 13); /*6012 */ - for (int i = 0; i < 25; i++) { - if (i == 16 || i == 14 || i == 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } + /* Too big test. */ + 0x0, + 18446744073709551615UL, + 562915594369134UL << 12, + 0x0, + -EBUSY, - erase_check_erase(mt, 15); /*7003 */ - for (int i = 0; i < 25; i++) { - if (i <= 16 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } + }; - mt_set_non_kernel(2); - erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */ - for (int i = 0; i < 25; i++) { - if (i <= 17 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } + int i, range_count = ARRAY_SIZE(range); + int req_range_count = ARRAY_SIZE(req_range); + unsigned long min = 0; - erase_check_erase(mt, 18); /*7012 */ - for (int i = 0; i < 25; i++) { - if (i <= 18 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } + MA_STATE(mas, mt, 0, 0); - mt_set_non_kernel(2); - erase_check_erase(mt, 19); /*7015 */ - for (int i = 0; i < 25; i++) { - if (i <= 19 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } + mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, + GFP_KERNEL); +#define DEBUG_REV_RANGE 0 + for (i = 0; i < range_count; i += 2) { + /* Inclusive, Inclusive (with the -1) */ - erase_check_erase(mt, 20); /*8003 */ - for (int i = 0; i < 25; i++) { - if (i <= 20 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); +#if DEBUG_REV_RANGE + pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12, + (range[i + 1] >> 12) - 1); +#endif + check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, + xa_mk_value(range[i] >> 12), 0); + mt_validate(mt); } - erase_check_erase(mt, 21); /*8002 */ - for (int i = 0; i < 25; i++) { - if (i <= 21 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); - } - mt_set_non_kernel(2); - erase_check_erase(mt, 22); /*8008 */ - for (int i = 0; i < 25; i++) { - if (i <= 22 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); + mas_lock(&mas); + for (i = 0; i < ARRAY_SIZE(holes); i += 3) { +#if DEBUG_REV_RANGE + pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n", + min, holes[i+1]>>12, holes[i+2]>>12, + holes[i] >> 12); +#endif + MT_BUG_ON(mt, mas_empty_area_rev(&mas, min, + holes[i+1] >> 12, + holes[i+2] >> 12)); +#if DEBUG_REV_RANGE + pr_debug("Found %lu %lu\n", mas.index, mas.last); + pr_debug("gap %lu %lu\n", (holes[i] >> 12), + (holes[i+1] >> 12)); +#endif + MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12)); + MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12)); + min = holes[i+1] >> 12; + mas_reset(&mas); } - for (int i = 23; i < 25; i++) - erase_check_erase(mt, i); - for (int i = 0; i < 25; i++) { - if (i <= 25 && i >= 13) - check_load(mt, set[i], NULL); - else - erase_check_load(mt, i); + mas_unlock(&mas); + for (i = 0; i < req_range_count; i += 5) { +#if DEBUG_REV_RANGE + pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n", + req_range[i] >> 12, + (req_range[i + 1] >> 12) - 1, + req_range[i+2] >> 12, + req_range[i+3] >> 12); +#endif + check_mtree_alloc_rrange(mt, + req_range[i] >> 12, /* start */ + req_range[i+1] >> 12, /* end */ + req_range[i+2] >> 12, /* size */ + req_range[i+3] >> 12, /* expected address */ + req_range[i+4], /* expected return */ + xa_mk_value(req_range[i] >> 12)); /* pointer */ + mt_validate(mt); } - /* Shrinking tree test. */ + mt_set_non_kernel(1); + mtree_erase(mt, 34148798727); /* create a deleted range. */ + check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414, + 34148798725, 0, mt); - for (int i = 13; i < ARRAY_SIZE(set); i++) - erase_check_insert(mt, i); + mtree_destroy(mt); +} - mt_set_non_kernel(99); - for (int i = 18; i < ARRAY_SIZE(set); i++) { - erase_check_erase(mt, i); - for (int j = 0; j < ARRAY_SIZE(set); j++) { - if (j < 18 || j > i) - erase_check_load(mt, j); - else - check_load(mt, set[j], NULL); - } - } - mt_set_non_kernel(35); - for (int i = 0; i < 18; i++) { - erase_check_erase(mt, i); - for (int j = 0; j < ARRAY_SIZE(set); j++) { - if (j < 18 && j > i) - erase_check_load(mt, j); - else - check_load(mt, set[j], NULL); - } - } - erase_check_insert(mt, 8); - erase_check_insert(mt, 9); - erase_check_erase(mt, 8); - rcu_unregister_thread(); -} - -#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \ - a[(i)], a[(i + 1)], ptr) -#define STORE 1 -#define SNULL 2 -#define ERASE 3 -#define ec_type_str(x) \ - (((x) == STORE) ? \ - "STORE" : \ - (((x) == SNULL) ? \ - "SNULL" : "ERASE") \ - ) -#define check_erase2_debug 0 -void *mas_next(struct ma_state *mas, unsigned long max); - -/* Calculate the overwritten entries. */ -int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end, - void *s_entry, unsigned long s_min, - void *e_entry, unsigned long e_max, - unsigned long *set, int i, bool null_entry) -{ - int count = 0, span = 0; - unsigned long retry = 0; - void *entry; - struct ma_state tmp; - - - /* count slots */ - memcpy(&tmp, mas_start, sizeof(tmp)); - entry = mas_next(&tmp, mas_end->last); - while (entry) { - BUG_ON(retry > 50); /* stop infinite retry on testing. */ - if (xa_is_zero(s_entry)) { - retry++; - continue; - } - count++; - span++; - entry = mas_next(&tmp, mas_end->last); - } - - if (null_entry) { - /* Check splitting end. */ - if (e_entry && (e_max > mas_end->last)) - count--; - - /* check overwrite of entire start */ - if (s_entry && (s_min == mas_start->index)) - count++; - } else { /* !null_entry (store) */ - bool esplit = e_max > mas_end->last; - bool ssplit = s_min != mas_start->index; - - if (s_entry && e_entry) { - if (esplit && ssplit) - count--; - else if (ssplit) - count--; - else if (esplit) { - if (span) - count--; - } - } else if (s_entry && !e_entry) { - if (ssplit) - count--; - } else if (!s_entry && e_entry) { - if (esplit) - count--; - count--; - } else { - count--; - } - } - return count; -} - -/* - * mas_node_walk() - Walk a maple node to offset of the index. - * @mas: The maple state - * @type: The maple node type - * @*range_min: Pointer to store the minimum range of the offset - * @*range_max: Pointer to store the maximum range of the offset - * - * The offset will be stored in the maple state. - * - */ -static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, - enum maple_type type, unsigned long *range_min, - unsigned long *range_max) - -{ - unsigned long *pivots; - unsigned char count; - unsigned long prev, max; - unsigned char offset; - unsigned long index; - - if (unlikely(ma_is_dense(type))) { - (*range_max) = (*range_min) = mas->index; - if (unlikely(ma_dead_node(node))) - return; - - mas->offset = mas->index = mas->min; - return; - } - - pivots = ma_pivots(node, type); - max = pivots[0]; - if (unlikely(ma_dead_node(node))) - return; - - offset = 0; - prev = mas->min; - index = mas->index; - if (unlikely(index <= max)) - goto offset_zero; - - count = mt_pivots[type]; - while (++offset < count) { - prev = max; - max = pivots[offset]; - if (unlikely(ma_dead_node(node))) - return; - - if (index <= max) - goto offset_found; - else if (unlikely(!max)) - goto mas_max; - } - - prev = max; -mas_max: - max = mas->max; -offset_found: - prev++; -offset_zero: - mas->offset = offset; - if (ma_is_leaf(type)) { - *range_max = max; - *range_min = prev; - } else { - mas->max = max; - mas->min = prev; - } -} - -/* - * mas_descend_walk(): Locates a value and sets the mas->node and slot - * accordingly. range_min and range_max are set to the range which the entry is - * valid. - * @mas: The maple state - * @*range_min: A pointer to store the minimum of the range - * @*range_max: A pointer to store the maximum of the range - * - * Check mas->node is still valid on return of any value. - * - * Return: true if pointing to a valid node and offset. False otherwise. - */ -static inline bool mas_descend_walk(struct ma_state *mas, - unsigned long *range_min, unsigned long *range_max) -{ - struct maple_enode *next; - struct maple_node *node; - enum maple_type type; - - next = mas->node; - while (true) { - node = mte_to_node(next); - type = mte_node_type(next); - mas_node_walk(mas, node, type, range_min, range_max); - next = mas_slot(mas, ma_slots(node, type), mas->offset); - if (unlikely(ma_dead_node(node))) - return false; - - if (unlikely(ma_is_leaf(type))) - return true; - - /* Descend. */ - mas->node = next; - } - return false; -} - -/* - * mas_tree_walk() - Walk to @mas->index and set the range values. - * @mas: The maple state. - * @*range_min: The minimum range to be set. - * @*range_max: The maximum range to be set. - * - * Ranges are only valid if there is a valid entry at @mas->index. - * - * Return: True if a value exists, false otherwise. - */ -static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, - unsigned long *range_max) +static noinline void check_alloc_range(struct maple_tree *mt) { - bool ret; - -retry: - ret = false; - mas_start(mas); - if (mas_is_none(mas)) - goto not_found; - - if (mas_is_ptr(mas)) { - *range_min = *range_max = 0; - if (!mas->index) - return true; - - goto not_found; - } - - ret = mas_descend_walk(mas, range_min, range_max); - if (unlikely(mte_dead_node(mas->node))) { - mas->node = MAS_START; - goto retry; - } - - return ret; + /* + * Generated by: + * cat /proc/self/maps|awk '{print $1}'| + * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' + */ -not_found: - mas->offset = MAPLE_NODE_SLOTS; - return false; -} + unsigned long range[] = { + /* Inclusive , Exclusive. */ + 0x565234af2000, 0x565234af4000, + 0x565234af4000, 0x565234af9000, + 0x565234af9000, 0x565234afb000, + 0x565234afc000, 0x565234afd000, + 0x565234afd000, 0x565234afe000, + 0x565235def000, 0x565235e10000, + 0x7f36d4bfd000, 0x7f36d4ee2000, + 0x7f36d4ee2000, 0x7f36d4f04000, + 0x7f36d4f04000, 0x7f36d504c000, + 0x7f36d504c000, 0x7f36d5098000, + 0x7f36d5098000, 0x7f36d5099000, + 0x7f36d5099000, 0x7f36d509d000, + 0x7f36d509d000, 0x7f36d509f000, + 0x7f36d509f000, 0x7f36d50a5000, + 0x7f36d50b9000, 0x7f36d50db000, + 0x7f36d50db000, 0x7f36d50dc000, + 0x7f36d50dc000, 0x7f36d50fa000, + 0x7f36d50fa000, 0x7f36d5102000, + 0x7f36d5102000, 0x7f36d5103000, + 0x7f36d5103000, 0x7f36d5104000, + 0x7f36d5104000, 0x7f36d5105000, + 0x7fff5876b000, 0x7fff5878d000, + 0x7fff5878e000, 0x7fff58791000, + 0x7fff58791000, 0x7fff58793000, + }; + unsigned long holes[] = { + /* Start of hole, end of hole, size of hole (+1) */ + 0x565234afb000, 0x565234afc000, 0x1000, + 0x565234afe000, 0x565235def000, 0x12F1000, + 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, + }; -static inline void *mas_range_load(struct ma_state *mas, - unsigned long *range_min, unsigned long *range_max) + /* + * req_range consists of 4 values. + * 1. min index + * 2. max index + * 3. size + * 4. number that should be returned. + * 5. return value + */ + unsigned long req_range[] = { + 0x565234af9000, /* Min */ + 0x7fff58791000, /* Max */ + 0x1000, /* Size */ + 0x565234afb000, /* First hole in our data of size 1000. */ + 0, /* Return value success. */ -{ - void *entry = NULL; - unsigned long index = mas->index; + 0x0, /* Min */ + 0x7fff58791000, /* Max */ + 0x1F00, /* Size */ + 0x0, /* First hole in our data of size 2000. */ + 0, /* Return value success. */ - if (mas_is_none(mas) || mas_is_paused(mas)) - mas->node = MAS_START; -retry: - if (mas_tree_walk(mas, range_min, range_max)) - if (unlikely(mas->node == MAS_ROOT)) - return mas_root(mas); + /* Test ascend. */ + 34148797436 << 12, /* Min */ + 0x7fff587AF000, /* Max */ + 0x3000, /* Size */ + 34148798629 << 12, /* Expected location */ + 0, /* Return value success. */ - if (likely(mas->offset != MAPLE_NODE_SLOTS)) - entry = mas_get_slot(mas, mas->offset); + /* Test failing. */ + 34148798623 << 12, /* Min */ + 34148798683 << 12, /* Max */ + 0x15000, /* Size */ + 0, /* Expected location */ + -EBUSY, /* Return value failed. */ - if (mas_dead_node(mas, index)) - goto retry; + /* Test filling entire gap. */ + 34148798623 << 12, /* Min */ + 0x7fff587AF000, /* Max */ + 0x10000, /* Size */ + 34148798632 << 12, /* Expected location */ + 0, /* Return value success. */ - return entry; -} -static noinline void check_erase2_testset(struct maple_tree *mt, - unsigned long *set, unsigned long size) -{ - int entry_count = 0; - int check = 0; - void *foo; - unsigned long addr = 0; - void *s_entry = NULL, *e_entry = NULL; + /* Test walking off the end of root. */ + 0, /* Min */ + -1, /* Max */ + -1, /* Size */ + 0, /* Expected location */ + -EBUSY, /* Return value failure. */ + /* Test looking for too large a hole across entire range. */ + 0, /* Min */ + -1, /* Max */ + 4503599618982063UL << 12, /* Size */ + 34359052178 << 12, /* Expected location */ + -EBUSY, /* Return failure. */ + }; + int i, range_count = ARRAY_SIZE(range); + int req_range_count = ARRAY_SIZE(req_range); + unsigned long min = 0x565234af2000; MA_STATE(mas, mt, 0, 0); - for (int i = 0; i < size; i += 3) { - unsigned long s_min, s_max; - unsigned long e_min, e_max; - void *value = NULL; - - MA_STATE(mas_start, mt, set[i+1], set[i+1]); - MA_STATE(mas_end, mt, set[i+2], set[i+2]); - mt_set_non_kernel(127); -#if check_erase2_debug - pr_err("%s: %d %s %lu - %lu\n", __func__, i, - ec_type_str(set[i]), - set[i+1], set[i+2]); -#endif - s_entry = mas_range_load(&mas_start, &s_min, &s_max); - e_entry = mas_range_load(&mas_end, &e_min, &e_max); - - switch (set[i]) { - case SNULL: - if ((s_min == set[i+1]) && (s_max == set[i+2])) { - if (s_entry) - entry_count--; - } else if ((s_min != set[i+1]) && (s_max != set[i+2])) { - entry_count++; - } else if ((mas_start.node != mas_end.node) || - (mas_start.offset != mas_end.offset)) { - entry_count -= - mas_ce2_over_count(&mas_start, &mas_end, - s_entry, s_min, - e_entry, e_max, set, i, - true); - } - - - erase_check_store_range(mt, set, i + 1, value); - break; - case STORE: - value = xa_mk_value(set[i + 1]); - if (mas_start.offset > mt_slot_count(mas_start.node)) { - entry_count++; /* appending an entry. */ - } else if ((s_min == e_min) && (s_max == e_max)) { - if (!entry_count) - entry_count++; - - else if (s_entry) { - if (e_max > mas_end.last) - entry_count++; - - if (s_min < mas_start.index) - entry_count++; - - } else { - entry_count++; - } - } else { - entry_count -= - mas_ce2_over_count(&mas_start, &mas_end, - s_entry, s_min, - e_entry, e_max, set, i, - false); - } - - erase_check_store_range(mt, set, i + 1, value); - break; - case ERASE: - if (!s_entry) - break; - check_erase(mt, set[i+1], xa_mk_value(set[i+1])); - entry_count--; - break; - } - mt_validate(mt); - if (entry_count) - MT_BUG_ON(mt, !mt_height(mt)); -#if check_erase2_debug > 1 + mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, + GFP_KERNEL); + for (i = 0; i < range_count; i += 2) { +#define DEBUG_ALLOC_RANGE 0 +#if DEBUG_ALLOC_RANGE + pr_debug("\tInsert %lu-%lu\n", range[i] >> 12, + (range[i + 1] >> 12) - 1); mt_dump(mt); #endif -#if check_erase2_debug - pr_err("Done\n"); -#endif + check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, + xa_mk_value(range[i] >> 12), 0); + mt_validate(mt); + } - check = 0; - addr = 0; - mt_for_each(mt, foo, addr, ULONG_MAX) { - check++; -#if check_erase2_debug > 2 - pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check); -#endif - if (check > entry_count) - break; - } -#if check_erase2_debug > 2 - pr_err("mt_for_each %d and count %d\n", check, entry_count); -#endif - MT_BUG_ON(mt, check != entry_count); + mas_lock(&mas); + for (i = 0; i < ARRAY_SIZE(holes); i += 3) { - check = 0; - addr = 0; +#if DEBUG_ALLOC_RANGE + pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12, + holes[i+1] >> 12, holes[i+2] >> 12, + min, holes[i+1]); +#endif + MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12, + holes[i+1] >> 12, + holes[i+2] >> 12)); + MT_BUG_ON(mt, mas.index != holes[i] >> 12); + min = holes[i+1]; mas_reset(&mas); - mas.index = 0; - rcu_read_lock(); - mas_for_each(&mas, foo, ULONG_MAX) { - if (xa_is_zero(foo)) { - if (addr == mas.index) { - mt_dump(mas.tree); - pr_err("retry failed %lu - %lu\n", - mas.index, mas.last); - MT_BUG_ON(mt, 1); - } - addr = mas.index; - continue; - } -#if check_erase2_debug > 2 - pr_err("mas: %lu -> %p\n", mas.index, foo); + } + mas_unlock(&mas); + for (i = 0; i < req_range_count; i += 5) { +#if DEBUG_ALLOC_RANGE + pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n", + i/5, req_range[i] >> 12, req_range[i + 1] >> 12, + req_range[i + 2] >> 12, req_range[i + 3] >> 12, + req_range[i], req_range[i+1]); #endif - check++; - if (check > entry_count) - break; - } - rcu_read_unlock(); -#if check_erase2_debug > 2 - pr_err("mas_for_each %d and count %d\n", check, entry_count); + check_mtree_alloc_range(mt, + req_range[i] >> 12, /* start */ + req_range[i+1] >> 12, /* end */ + req_range[i+2] >> 12, /* size */ + req_range[i+3] >> 12, /* expected address */ + req_range[i+4], /* expected return */ + xa_mk_value(req_range[i] >> 12)); /* pointer */ mt_validate(mt); +#if DEBUG_ALLOC_RANGE + mt_dump(mt); #endif - - MT_BUG_ON(mt, check != entry_count); - - MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); } -} + mtree_destroy(mt); +} +#endif -/* These tests were pulled from kvm tests. */ -static noinline void check_erase2_sets(struct maple_tree *mt) +static noinline void check_ranges(struct maple_tree *mt) { - void *entry; - unsigned long start = 0; - unsigned long set[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140721266458624, 140737488351231, -ERASE, 140721266458624, 140737488351231, -STORE, 140721266458624, 140721266462719, -STORE, 94735788949504, 94735789121535, -ERASE, 94735788949504, 94735789121535, -STORE, 94735788949504, 94735788965887, -STORE, 94735788965888, 94735789121535, -ERASE, 94735788965888, 94735789121535, -STORE, 94735788965888, 94735789068287, -STORE, 94735789068288, 94735789109247, -STORE, 94735789109248, 94735789121535, -STORE, 140253902692352, 140253902864383, -ERASE, 140253902692352, 140253902864383, -STORE, 140253902692352, 140253902696447, -STORE, 140253902696448, 140253902864383, - }; - unsigned long set2[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140735933583360, 140737488351231, -ERASE, 140735933583360, 140737488351231, -STORE, 140735933583360, 140735933587455, -STORE, 94811003260928, 94811003432959, -ERASE, 94811003260928, 94811003432959, -STORE, 94811003260928, 94811003277311, -STORE, 94811003277312, 94811003432959, -ERASE, 94811003277312, 94811003432959, -STORE, 94811003277312, 94811003379711, -STORE, 94811003379712, 94811003420671, -STORE, 94811003420672, 94811003432959, -STORE, 140277094653952, 140277094825983, -ERASE, 140277094653952, 140277094825983, -STORE, 140277094653952, 140277094658047, -STORE, 140277094658048, 140277094825983, -ERASE, 140277094658048, 140277094825983, -STORE, 140277094658048, 140277094780927, -STORE, 140277094780928, 140277094813695, -STORE, 140277094813696, 140277094821887, -STORE, 140277094821888, 140277094825983, -STORE, 140735933906944, 140735933911039, - }; - unsigned long set3[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140735790264320, 140737488351231, -ERASE, 140735790264320, 140737488351231, -STORE, 140735790264320, 140735790268415, -STORE, 94016597282816, 94016597454847, -ERASE, 94016597282816, 94016597454847, -STORE, 94016597282816, 94016597299199, -STORE, 94016597299200, 94016597454847, -ERASE, 94016597299200, 94016597454847, -STORE, 94016597299200, 94016597401599, -STORE, 94016597401600, 94016597442559, -STORE, 94016597442560, 94016597454847, -STORE, 140496959283200, 140496959455231, -ERASE, 140496959283200, 140496959455231, -STORE, 140496959283200, 140496959287295, -STORE, 140496959287296, 140496959455231, -ERASE, 140496959287296, 140496959455231, -STORE, 140496959287296, 140496959410175, -STORE, 140496959410176, 140496959442943, -STORE, 140496959442944, 140496959451135, -STORE, 140496959451136, 140496959455231, -STORE, 140735791718400, 140735791722495, -STORE, 140735791706112, 140735791718399, -STORE, 47135835713536, 47135835721727, -STORE, 47135835721728, 47135835729919, -STORE, 47135835729920, 47135835893759, -ERASE, 47135835729920, 47135835893759, -STORE, 47135835729920, 47135835742207, -STORE, 47135835742208, 47135835893759, -STORE, 47135835840512, 47135835893759, -STORE, 47135835742208, 47135835840511, -ERASE, 47135835742208, 47135835840511, -STORE, 47135835742208, 47135835840511, -STORE, 47135835885568, 47135835893759, -STORE, 47135835840512, 47135835885567, -ERASE, 47135835840512, 47135835885567, -STORE, 47135835840512, 47135835893759, -ERASE, 47135835840512, 47135835893759, -STORE, 47135835840512, 47135835885567, -STORE, 47135835885568, 47135835893759, - }; - - unsigned long set4[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140728251703296, 140737488351231, -ERASE, 140728251703296, 140737488351231, -STORE, 140728251703296, 140728251707391, -STORE, 94668429205504, 94668429377535, -ERASE, 94668429205504, 94668429377535, -STORE, 94668429205504, 94668429221887, -STORE, 94668429221888, 94668429377535, -ERASE, 94668429221888, 94668429377535, -STORE, 94668429221888, 94668429324287, -STORE, 94668429324288, 94668429365247, -STORE, 94668429365248, 94668429377535, -STORE, 47646523273216, 47646523445247, -ERASE, 47646523273216, 47646523445247, -STORE, 47646523273216, 47646523277311, -STORE, 47646523277312, 47646523445247, -ERASE, 47646523277312, 47646523445247, -STORE, 47646523277312, 47646523400191, - }; - - unsigned long set5[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140726874062848, 140737488351231, -ERASE, 140726874062848, 140737488351231, -STORE, 140726874062848, 140726874066943, -STORE, 94248892870656, 94248893042687, -ERASE, 94248892870656, 94248893042687, -STORE, 94248892870656, 94248892887039, -STORE, 94248892887040, 94248893042687, -ERASE, 94248892887040, 94248893042687, -STORE, 94248892887040, 94248892989439, -STORE, 94248892989440, 94248893030399, -STORE, 94248893030400, 94248893042687, -STORE, 47884786266112, 47884786438143, -ERASE, 47884786266112, 47884786438143, -STORE, 47884786266112, 47884786270207, -STORE, 47884786270208, 47884786438143, -ERASE, 47884786270208, 47884786438143, -STORE, 47884786270208, 47884786393087, -STORE, 47884786393088, 47884786425855, -STORE, 47884786425856, 47884786434047, -STORE, 47884786434048, 47884786438143, -STORE, 140726874513408, 140726874517503, -STORE, 140726874501120, 140726874513407, -STORE, 47884786438144, 47884786446335, -STORE, 47884786446336, 47884786454527, -STORE, 47884786454528, 47884786618367, -ERASE, 47884786454528, 47884786618367, -STORE, 47884786454528, 47884786466815, -STORE, 47884786466816, 47884786618367, -STORE, 47884786565120, 47884786618367, -STORE, 47884786466816, 47884786565119, -ERASE, 47884786466816, 47884786565119, -STORE, 47884786466816, 47884786565119, -STORE, 47884786610176, 47884786618367, -STORE, 47884786565120, 47884786610175, -ERASE, 47884786565120, 47884786610175, -STORE, 47884786565120, 47884786618367, -ERASE, 47884786565120, 47884786618367, -STORE, 47884786565120, 47884786610175, -STORE, 47884786610176, 47884786618367, -ERASE, 47884786610176, 47884786618367, -STORE, 47884786610176, 47884786618367, -STORE, 47884786618368, 47884789669887, -STORE, 47884787163136, 47884789669887, -STORE, 47884786618368, 47884787163135, -ERASE, 47884787163136, 47884789669887, -STORE, 47884787163136, 47884789448703, -STORE, 47884789448704, 47884789669887, -STORE, 47884788858880, 47884789448703, -STORE, 47884787163136, 47884788858879, -ERASE, 47884787163136, 47884788858879, -STORE, 47884787163136, 47884788858879, -STORE, 47884789444608, 47884789448703, -STORE, 47884788858880, 47884789444607, -ERASE, 47884788858880, 47884789444607, -STORE, 47884788858880, 47884789444607, -STORE, 47884789653504, 47884789669887, -STORE, 47884789448704, 47884789653503, -ERASE, 47884789448704, 47884789653503, -STORE, 47884789448704, 47884789653503, -ERASE, 47884789653504, 47884789669887, -STORE, 47884789653504, 47884789669887, -STORE, 47884789669888, 47884791508991, -STORE, 47884789809152, 47884791508991, -STORE, 47884789669888, 47884789809151, -ERASE, 47884789809152, 47884791508991, -STORE, 47884789809152, 47884791468031, -STORE, 47884791468032, 47884791508991, -STORE, 47884791152640, 47884791468031, -STORE, 47884789809152, 47884791152639, -ERASE, 47884789809152, 47884791152639, -STORE, 47884789809152, 47884791152639, -STORE, 47884791463936, 47884791468031, -STORE, 47884791152640, 47884791463935, -ERASE, 47884791152640, 47884791463935, -STORE, 47884791152640, 47884791463935, -STORE, 47884791492608, 47884791508991, -STORE, 47884791468032, 47884791492607, -ERASE, 47884791468032, 47884791492607, -STORE, 47884791468032, 47884791492607, -ERASE, 47884791492608, 47884791508991, -STORE, 47884791492608, 47884791508991, -STORE, 47884791508992, 47884791644159, -ERASE, 47884791508992, 47884791644159, -STORE, 47884791508992, 47884791533567, -STORE, 47884791533568, 47884791644159, -STORE, 47884791595008, 47884791644159, -STORE, 47884791533568, 47884791595007, -ERASE, 47884791533568, 47884791595007, -STORE, 47884791533568, 47884791595007, -STORE, 47884791619584, 47884791644159, -STORE, 47884791595008, 47884791619583, -ERASE, 47884791595008, 47884791619583, -STORE, 47884791595008, 47884791644159, -ERASE, 47884791595008, 47884791644159, -STORE, 47884791595008, 47884791619583, -STORE, 47884791619584, 47884791644159, -STORE, 47884791627776, 47884791644159, -STORE, 47884791619584, 47884791627775, -ERASE, 47884791619584, 47884791627775, -STORE, 47884791619584, 47884791627775, -ERASE, 47884791627776, 47884791644159, -STORE, 47884791627776, 47884791644159, -STORE, 47884791644160, 47884791664639, -ERASE, 47884791644160, 47884791664639, -STORE, 47884791644160, 47884791648255, -STORE, 47884791648256, 47884791664639, -STORE, 47884791652352, 47884791664639, -STORE, 47884791648256, 47884791652351, -ERASE, 47884791648256, 47884791652351, -STORE, 47884791648256, 47884791652351, -STORE, 47884791656448, 47884791664639, -STORE, 47884791652352, 47884791656447, -ERASE, 47884791652352, 47884791656447, -STORE, 47884791652352, 47884791664639, -ERASE, 47884791652352, 47884791664639, -STORE, 47884791652352, 47884791656447, -STORE, 47884791656448, 47884791664639, -ERASE, 47884791656448, 47884791664639, -STORE, 47884791656448, 47884791664639, -STORE, 47884791664640, 47884791672831, -ERASE, 47884791468032, 47884791492607, -STORE, 47884791468032, 47884791484415, -STORE, 47884791484416, 47884791492607, -ERASE, 47884791656448, 47884791664639, -STORE, 47884791656448, 47884791660543, -STORE, 47884791660544, 47884791664639, -ERASE, 47884791619584, 47884791627775, -STORE, 47884791619584, 47884791623679, -STORE, 47884791623680, 47884791627775, - }; + int i, val, val2; + unsigned long r[] = { + 10, 15, + 20, 25, + 17, 22, /* Overlaps previous range. */ + 9, 1000, /* Huge. */ + 100, 200, + 45, 168, + 118, 128, + }; - unsigned long set6[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140722999021568, 140737488351231, -ERASE, 140722999021568, 140737488351231, -STORE, 140722999021568, 140722999025663, -STORE, 94901500268544, 94901500440575, -ERASE, 94901500268544, 94901500440575, -STORE, 94901500268544, 94901500284927, -STORE, 94901500284928, 94901500440575, -ERASE, 94901500284928, 94901500440575, -STORE, 94901500284928, 94901500387327, -STORE, 94901500387328, 94901500428287, -STORE, 94901500428288, 94901500440575, -STORE, 47430426660864, 47430426832895, -ERASE, 47430426660864, 47430426832895, -STORE, 47430426660864, 47430426664959, -STORE, 47430426664960, 47430426832895, -ERASE, 47430426664960, 47430426832895, -STORE, 47430426664960, 47430426787839, -STORE, 47430426787840, 47430426820607, -STORE, 47430426820608, 47430426828799, -STORE, 47430426828800, 47430426832895, -STORE, 140722999115776, 140722999119871, -STORE, 140722999103488, 140722999115775, -STORE, 47430426832896, 47430426841087, -STORE, 47430426841088, 47430426849279, -STORE, 47430426849280, 47430427013119, -ERASE, 47430426849280, 47430427013119, -STORE, 47430426849280, 47430426861567, -STORE, 47430426861568, 47430427013119, -STORE, 47430426959872, 47430427013119, -STORE, 47430426861568, 47430426959871, -ERASE, 47430426861568, 47430426959871, -STORE, 47430426861568, 47430426959871, -STORE, 47430427004928, 47430427013119, -STORE, 47430426959872, 47430427004927, -ERASE, 47430426959872, 47430427004927, -STORE, 47430426959872, 47430427013119, -ERASE, 47430426959872, 47430427013119, -STORE, 47430426959872, 47430427004927, -STORE, 47430427004928, 47430427013119, -ERASE, 47430427004928, 47430427013119, -STORE, 47430427004928, 47430427013119, -STORE, 47430427013120, 47430430064639, -STORE, 47430427557888, 47430430064639, -STORE, 47430427013120, 47430427557887, -ERASE, 47430427557888, 47430430064639, -STORE, 47430427557888, 47430429843455, -STORE, 47430429843456, 47430430064639, -STORE, 47430429253632, 47430429843455, -STORE, 47430427557888, 47430429253631, -ERASE, 47430427557888, 47430429253631, -STORE, 47430427557888, 47430429253631, -STORE, 47430429839360, 47430429843455, -STORE, 47430429253632, 47430429839359, -ERASE, 47430429253632, 47430429839359, -STORE, 47430429253632, 47430429839359, -STORE, 47430430048256, 47430430064639, -STORE, 47430429843456, 47430430048255, -ERASE, 47430429843456, 47430430048255, -STORE, 47430429843456, 47430430048255, -ERASE, 47430430048256, 47430430064639, -STORE, 47430430048256, 47430430064639, -STORE, 47430430064640, 47430431903743, -STORE, 47430430203904, 47430431903743, -STORE, 47430430064640, 47430430203903, -ERASE, 47430430203904, 47430431903743, -STORE, 47430430203904, 47430431862783, -STORE, 47430431862784, 47430431903743, -STORE, 47430431547392, 47430431862783, -STORE, 47430430203904, 47430431547391, -ERASE, 47430430203904, 47430431547391, -STORE, 47430430203904, 47430431547391, -STORE, 47430431858688, 47430431862783, -STORE, 47430431547392, 47430431858687, -ERASE, 47430431547392, 47430431858687, -STORE, 47430431547392, 47430431858687, -STORE, 47430431887360, 47430431903743, -STORE, 47430431862784, 47430431887359, -ERASE, 47430431862784, 47430431887359, -STORE, 47430431862784, 47430431887359, -ERASE, 47430431887360, 47430431903743, -STORE, 47430431887360, 47430431903743, -STORE, 47430431903744, 47430432038911, -ERASE, 47430431903744, 47430432038911, -STORE, 47430431903744, 47430431928319, -STORE, 47430431928320, 47430432038911, -STORE, 47430431989760, 47430432038911, -STORE, 47430431928320, 47430431989759, -ERASE, 47430431928320, 47430431989759, -STORE, 47430431928320, 47430431989759, -STORE, 47430432014336, 47430432038911, -STORE, 47430431989760, 47430432014335, -ERASE, 47430431989760, 47430432014335, -STORE, 47430431989760, 47430432038911, -ERASE, 47430431989760, 47430432038911, -STORE, 47430431989760, 47430432014335, -STORE, 47430432014336, 47430432038911, -STORE, 47430432022528, 47430432038911, -STORE, 47430432014336, 47430432022527, -ERASE, 47430432014336, 47430432022527, -STORE, 47430432014336, 47430432022527, -ERASE, 47430432022528, 47430432038911, -STORE, 47430432022528, 47430432038911, -STORE, 47430432038912, 47430432059391, -ERASE, 47430432038912, 47430432059391, -STORE, 47430432038912, 47430432043007, -STORE, 47430432043008, 47430432059391, -STORE, 47430432047104, 47430432059391, -STORE, 47430432043008, 47430432047103, -ERASE, 47430432043008, 47430432047103, -STORE, 47430432043008, 47430432047103, -STORE, 47430432051200, 47430432059391, -STORE, 47430432047104, 47430432051199, -ERASE, 47430432047104, 47430432051199, -STORE, 47430432047104, 47430432059391, -ERASE, 47430432047104, 47430432059391, -STORE, 47430432047104, 47430432051199, -STORE, 47430432051200, 47430432059391, -ERASE, 47430432051200, 47430432059391, -STORE, 47430432051200, 47430432059391, -STORE, 47430432059392, 47430432067583, -ERASE, 47430431862784, 47430431887359, -STORE, 47430431862784, 47430431879167, -STORE, 47430431879168, 47430431887359, -ERASE, 47430432051200, 47430432059391, -STORE, 47430432051200, 47430432055295, -STORE, 47430432055296, 47430432059391, -ERASE, 47430432014336, 47430432022527, -STORE, 47430432014336, 47430432018431, -STORE, 47430432018432, 47430432022527, - }; - unsigned long set7[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140729808330752, 140737488351231, -ERASE, 140729808330752, 140737488351231, -STORE, 140729808330752, 140729808334847, -STORE, 94629632020480, 94629632192511, -ERASE, 94629632020480, 94629632192511, -STORE, 94629632020480, 94629632036863, -STORE, 94629632036864, 94629632192511, -ERASE, 94629632036864, 94629632192511, -STORE, 94629632036864, 94629632139263, -STORE, 94629632139264, 94629632180223, -STORE, 94629632180224, 94629632192511, -STORE, 47439981776896, 47439981948927, -ERASE, 47439981776896, 47439981948927, -STORE, 47439981776896, 47439981780991, -STORE, 47439981780992, 47439981948927, -ERASE, 47439981780992, 47439981948927, -STORE, 47439981780992, 47439981903871, -STORE, 47439981903872, 47439981936639, -STORE, 47439981936640, 47439981944831, -STORE, 47439981944832, 47439981948927, -STORE, 140729808474112, 140729808478207, -STORE, 140729808461824, 140729808474111, -STORE, 47439981948928, 47439981957119, -STORE, 47439981957120, 47439981965311, -STORE, 47439981965312, 47439982129151, -ERASE, 47439981965312, 47439982129151, -STORE, 47439981965312, 47439981977599, -STORE, 47439981977600, 47439982129151, -STORE, 47439982075904, 47439982129151, -STORE, 47439981977600, 47439982075903, -ERASE, 47439981977600, 47439982075903, -STORE, 47439981977600, 47439982075903, -STORE, 47439982120960, 47439982129151, -STORE, 47439982075904, 47439982120959, -ERASE, 47439982075904, 47439982120959, -STORE, 47439982075904, 47439982129151, -ERASE, 47439982075904, 47439982129151, -STORE, 47439982075904, 47439982120959, -STORE, 47439982120960, 47439982129151, -ERASE, 47439982120960, 47439982129151, -STORE, 47439982120960, 47439982129151, -STORE, 47439982129152, 47439985180671, -STORE, 47439982673920, 47439985180671, -STORE, 47439982129152, 47439982673919, -ERASE, 47439982673920, 47439985180671, -STORE, 47439982673920, 47439984959487, -STORE, 47439984959488, 47439985180671, -STORE, 47439984369664, 47439984959487, -STORE, 47439982673920, 47439984369663, -ERASE, 47439982673920, 47439984369663, -STORE, 47439982673920, 47439984369663, -STORE, 47439984955392, 47439984959487, -STORE, 47439984369664, 47439984955391, -ERASE, 47439984369664, 47439984955391, -STORE, 47439984369664, 47439984955391, -STORE, 47439985164288, 47439985180671, -STORE, 47439984959488, 47439985164287, -ERASE, 47439984959488, 47439985164287, -STORE, 47439984959488, 47439985164287, -ERASE, 47439985164288, 47439985180671, -STORE, 47439985164288, 47439985180671, -STORE, 47439985180672, 47439987019775, -STORE, 47439985319936, 47439987019775, -STORE, 47439985180672, 47439985319935, -ERASE, 47439985319936, 47439987019775, -STORE, 47439985319936, 47439986978815, -STORE, 47439986978816, 47439987019775, -STORE, 47439986663424, 47439986978815, -STORE, 47439985319936, 47439986663423, -ERASE, 47439985319936, 47439986663423, -STORE, 47439985319936, 47439986663423, -STORE, 47439986974720, 47439986978815, -STORE, 47439986663424, 47439986974719, -ERASE, 47439986663424, 47439986974719, -STORE, 47439986663424, 47439986974719, -STORE, 47439987003392, 47439987019775, -STORE, 47439986978816, 47439987003391, -ERASE, 47439986978816, 47439987003391, -STORE, 47439986978816, 47439987003391, -ERASE, 47439987003392, 47439987019775, -STORE, 47439987003392, 47439987019775, -STORE, 47439987019776, 47439987154943, -ERASE, 47439987019776, 47439987154943, -STORE, 47439987019776, 47439987044351, -STORE, 47439987044352, 47439987154943, -STORE, 47439987105792, 47439987154943, -STORE, 47439987044352, 47439987105791, -ERASE, 47439987044352, 47439987105791, -STORE, 47439987044352, 47439987105791, -STORE, 47439987130368, 47439987154943, -STORE, 47439987105792, 47439987130367, -ERASE, 47439987105792, 47439987130367, -STORE, 47439987105792, 47439987154943, -ERASE, 47439987105792, 47439987154943, -STORE, 47439987105792, 47439987130367, -STORE, 47439987130368, 47439987154943, -STORE, 47439987138560, 47439987154943, -STORE, 47439987130368, 47439987138559, -ERASE, 47439987130368, 47439987138559, -STORE, 47439987130368, 47439987138559, -ERASE, 47439987138560, 47439987154943, -STORE, 47439987138560, 47439987154943, -STORE, 47439987154944, 47439987175423, -ERASE, 47439987154944, 47439987175423, -STORE, 47439987154944, 47439987159039, -STORE, 47439987159040, 47439987175423, -STORE, 47439987163136, 47439987175423, -STORE, 47439987159040, 47439987163135, -ERASE, 47439987159040, 47439987163135, -STORE, 47439987159040, 47439987163135, -STORE, 47439987167232, 47439987175423, -STORE, 47439987163136, 47439987167231, -ERASE, 47439987163136, 47439987167231, -STORE, 47439987163136, 47439987175423, -ERASE, 47439987163136, 47439987175423, -STORE, 47439987163136, 47439987167231, -STORE, 47439987167232, 47439987175423, -ERASE, 47439987167232, 47439987175423, -STORE, 47439987167232, 47439987175423, -STORE, 47439987175424, 47439987183615, -ERASE, 47439986978816, 47439987003391, -STORE, 47439986978816, 47439986995199, -STORE, 47439986995200, 47439987003391, -ERASE, 47439987167232, 47439987175423, -STORE, 47439987167232, 47439987171327, -STORE, 47439987171328, 47439987175423, -ERASE, 47439987130368, 47439987138559, -STORE, 47439987130368, 47439987134463, -STORE, 47439987134464, 47439987138559, - }; - unsigned long set8[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140722482974720, 140737488351231, -ERASE, 140722482974720, 140737488351231, -STORE, 140722482974720, 140722482978815, -STORE, 94121505034240, 94121505206271, -ERASE, 94121505034240, 94121505206271, -STORE, 94121505034240, 94121505050623, -STORE, 94121505050624, 94121505206271, -ERASE, 94121505050624, 94121505206271, -STORE, 94121505050624, 94121505153023, -STORE, 94121505153024, 94121505193983, -STORE, 94121505193984, 94121505206271, -STORE, 47708483284992, 47708483457023, -ERASE, 47708483284992, 47708483457023, -STORE, 47708483284992, 47708483289087, -STORE, 47708483289088, 47708483457023, -ERASE, 47708483289088, 47708483457023, -STORE, 47708483289088, 47708483411967, -STORE, 47708483411968, 47708483444735, -STORE, 47708483444736, 47708483452927, -STORE, 47708483452928, 47708483457023, -STORE, 140722483142656, 140722483146751, -STORE, 140722483130368, 140722483142655, -STORE, 47708483457024, 47708483465215, -STORE, 47708483465216, 47708483473407, -STORE, 47708483473408, 47708483637247, -ERASE, 47708483473408, 47708483637247, -STORE, 47708483473408, 47708483485695, -STORE, 47708483485696, 47708483637247, -STORE, 47708483584000, 47708483637247, -STORE, 47708483485696, 47708483583999, -ERASE, 47708483485696, 47708483583999, -STORE, 47708483485696, 47708483583999, -STORE, 47708483629056, 47708483637247, -STORE, 47708483584000, 47708483629055, -ERASE, 47708483584000, 47708483629055, -STORE, 47708483584000, 47708483637247, -ERASE, 47708483584000, 47708483637247, -STORE, 47708483584000, 47708483629055, -STORE, 47708483629056, 47708483637247, -ERASE, 47708483629056, 47708483637247, -STORE, 47708483629056, 47708483637247, -STORE, 47708483637248, 47708486688767, -STORE, 47708484182016, 47708486688767, -STORE, 47708483637248, 47708484182015, -ERASE, 47708484182016, 47708486688767, -STORE, 47708484182016, 47708486467583, -STORE, 47708486467584, 47708486688767, -STORE, 47708485877760, 47708486467583, -STORE, 47708484182016, 47708485877759, -ERASE, 47708484182016, 47708485877759, -STORE, 47708484182016, 47708485877759, -STORE, 47708486463488, 47708486467583, -STORE, 47708485877760, 47708486463487, -ERASE, 47708485877760, 47708486463487, -STORE, 47708485877760, 47708486463487, -STORE, 47708486672384, 47708486688767, -STORE, 47708486467584, 47708486672383, -ERASE, 47708486467584, 47708486672383, -STORE, 47708486467584, 47708486672383, -ERASE, 47708486672384, 47708486688767, -STORE, 47708486672384, 47708486688767, -STORE, 47708486688768, 47708488527871, -STORE, 47708486828032, 47708488527871, -STORE, 47708486688768, 47708486828031, -ERASE, 47708486828032, 47708488527871, -STORE, 47708486828032, 47708488486911, -STORE, 47708488486912, 47708488527871, -STORE, 47708488171520, 47708488486911, -STORE, 47708486828032, 47708488171519, -ERASE, 47708486828032, 47708488171519, -STORE, 47708486828032, 47708488171519, -STORE, 47708488482816, 47708488486911, -STORE, 47708488171520, 47708488482815, -ERASE, 47708488171520, 47708488482815, -STORE, 47708488171520, 47708488482815, -STORE, 47708488511488, 47708488527871, -STORE, 47708488486912, 47708488511487, -ERASE, 47708488486912, 47708488511487, -STORE, 47708488486912, 47708488511487, -ERASE, 47708488511488, 47708488527871, -STORE, 47708488511488, 47708488527871, -STORE, 47708488527872, 47708488663039, -ERASE, 47708488527872, 47708488663039, -STORE, 47708488527872, 47708488552447, -STORE, 47708488552448, 47708488663039, -STORE, 47708488613888, 47708488663039, -STORE, 47708488552448, 47708488613887, -ERASE, 47708488552448, 47708488613887, -STORE, 47708488552448, 47708488613887, -STORE, 47708488638464, 47708488663039, -STORE, 47708488613888, 47708488638463, -ERASE, 47708488613888, 47708488638463, -STORE, 47708488613888, 47708488663039, -ERASE, 47708488613888, 47708488663039, -STORE, 47708488613888, 47708488638463, -STORE, 47708488638464, 47708488663039, -STORE, 47708488646656, 47708488663039, -STORE, 47708488638464, 47708488646655, -ERASE, 47708488638464, 47708488646655, -STORE, 47708488638464, 47708488646655, -ERASE, 47708488646656, 47708488663039, -STORE, 47708488646656, 47708488663039, -STORE, 47708488663040, 47708488683519, -ERASE, 47708488663040, 47708488683519, -STORE, 47708488663040, 47708488667135, -STORE, 47708488667136, 47708488683519, -STORE, 47708488671232, 47708488683519, -STORE, 47708488667136, 47708488671231, -ERASE, 47708488667136, 47708488671231, -STORE, 47708488667136, 47708488671231, -STORE, 47708488675328, 47708488683519, -STORE, 47708488671232, 47708488675327, -ERASE, 47708488671232, 47708488675327, -STORE, 47708488671232, 47708488683519, -ERASE, 47708488671232, 47708488683519, -STORE, 47708488671232, 47708488675327, -STORE, 47708488675328, 47708488683519, -ERASE, 47708488675328, 47708488683519, -STORE, 47708488675328, 47708488683519, -STORE, 47708488683520, 47708488691711, -ERASE, 47708488486912, 47708488511487, -STORE, 47708488486912, 47708488503295, -STORE, 47708488503296, 47708488511487, -ERASE, 47708488675328, 47708488683519, -STORE, 47708488675328, 47708488679423, -STORE, 47708488679424, 47708488683519, -ERASE, 47708488638464, 47708488646655, -STORE, 47708488638464, 47708488642559, -STORE, 47708488642560, 47708488646655, - }; + MT_BUG_ON(mt, !mtree_empty(mt)); + check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0); + check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0); + check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST); + MT_BUG_ON(mt, !mt_height(mt)); + /* Store */ + check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0); + check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0); + check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); + MT_BUG_ON(mt, mt_height(mt)); - unsigned long set9[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140736427839488, 140737488351231, -ERASE, 140736427839488, 140736427839488, -STORE, 140736427839488, 140736427843583, -STORE, 94071213395968, 94071213567999, -ERASE, 94071213395968, 94071213395968, -STORE, 94071213395968, 94071213412351, -STORE, 94071213412352, 94071213567999, -ERASE, 94071213412352, 94071213412352, -STORE, 94071213412352, 94071213514751, -STORE, 94071213514752, 94071213555711, -STORE, 94071213555712, 94071213567999, -STORE, 139968410644480, 139968410816511, -ERASE, 139968410644480, 139968410644480, -STORE, 139968410644480, 139968410648575, -STORE, 139968410648576, 139968410816511, -ERASE, 139968410648576, 139968410648576, -STORE, 139968410648576, 139968410771455, -STORE, 139968410771456, 139968410804223, -STORE, 139968410804224, 139968410812415, -STORE, 139968410812416, 139968410816511, -STORE, 140736429277184, 140736429281279, -STORE, 140736429264896, 140736429277183, -STORE, 47664384352256, 47664384360447, -STORE, 47664384360448, 47664384368639, -STORE, 47664384368640, 47664384532479, -ERASE, 47664384368640, 47664384368640, -STORE, 47664384368640, 47664384380927, -STORE, 47664384380928, 47664384532479, -STORE, 47664384479232, 47664384532479, -STORE, 47664384380928, 47664384479231, -ERASE, 47664384380928, 47664384380928, -STORE, 47664384380928, 47664384479231, -STORE, 47664384524288, 47664384532479, -STORE, 47664384479232, 47664384524287, -ERASE, 47664384479232, 47664384479232, -STORE, 47664384479232, 47664384532479, -ERASE, 47664384479232, 47664384479232, -STORE, 47664384479232, 47664384524287, -STORE, 47664384524288, 47664384532479, -ERASE, 47664384524288, 47664384524288, -STORE, 47664384524288, 47664384532479, -STORE, 47664384532480, 47664387583999, -STORE, 47664385077248, 47664387583999, -STORE, 47664384532480, 47664385077247, -ERASE, 47664385077248, 47664385077248, -STORE, 47664385077248, 47664387362815, -STORE, 47664387362816, 47664387583999, -STORE, 47664386772992, 47664387362815, -STORE, 47664385077248, 47664386772991, -ERASE, 47664385077248, 47664385077248, -STORE, 47664385077248, 47664386772991, -STORE, 47664387358720, 47664387362815, -STORE, 47664386772992, 47664387358719, -ERASE, 47664386772992, 47664386772992, -STORE, 47664386772992, 47664387358719, -STORE, 47664387567616, 47664387583999, -STORE, 47664387362816, 47664387567615, -ERASE, 47664387362816, 47664387362816, -STORE, 47664387362816, 47664387567615, -ERASE, 47664387567616, 47664387567616, -STORE, 47664387567616, 47664387583999, -STORE, 47664387584000, 47664389423103, -STORE, 47664387723264, 47664389423103, -STORE, 47664387584000, 47664387723263, -ERASE, 47664387723264, 47664387723264, -STORE, 47664387723264, 47664389382143, -STORE, 47664389382144, 47664389423103, -STORE, 47664389066752, 47664389382143, -STORE, 47664387723264, 47664389066751, -ERASE, 47664387723264, 47664387723264, -STORE, 47664387723264, 47664389066751, -STORE, 47664389378048, 47664389382143, -STORE, 47664389066752, 47664389378047, -ERASE, 47664389066752, 47664389066752, -STORE, 47664389066752, 47664389378047, -STORE, 47664389406720, 47664389423103, -STORE, 47664389382144, 47664389406719, -ERASE, 47664389382144, 47664389382144, -STORE, 47664389382144, 47664389406719, -ERASE, 47664389406720, 47664389406720, -STORE, 47664389406720, 47664389423103, -STORE, 47664389423104, 47664389558271, -ERASE, 47664389423104, 47664389423104, -STORE, 47664389423104, 47664389447679, -STORE, 47664389447680, 47664389558271, -STORE, 47664389509120, 47664389558271, -STORE, 47664389447680, 47664389509119, -ERASE, 47664389447680, 47664389447680, -STORE, 47664389447680, 47664389509119, -STORE, 47664389533696, 47664389558271, -STORE, 47664389509120, 47664389533695, -ERASE, 47664389509120, 47664389509120, -STORE, 47664389509120, 47664389558271, -ERASE, 47664389509120, 47664389509120, -STORE, 47664389509120, 47664389533695, -STORE, 47664389533696, 47664389558271, -STORE, 47664389541888, 47664389558271, -STORE, 47664389533696, 47664389541887, -ERASE, 47664389533696, 47664389533696, -STORE, 47664389533696, 47664389541887, -ERASE, 47664389541888, 47664389541888, -STORE, 47664389541888, 47664389558271, -STORE, 47664389558272, 47664389578751, -ERASE, 47664389558272, 47664389558272, -STORE, 47664389558272, 47664389562367, -STORE, 47664389562368, 47664389578751, -STORE, 47664389566464, 47664389578751, -STORE, 47664389562368, 47664389566463, -ERASE, 47664389562368, 47664389562368, -STORE, 47664389562368, 47664389566463, -STORE, 47664389570560, 47664389578751, -STORE, 47664389566464, 47664389570559, -ERASE, 47664389566464, 47664389566464, -STORE, 47664389566464, 47664389578751, -ERASE, 47664389566464, 47664389566464, -STORE, 47664389566464, 47664389570559, -STORE, 47664389570560, 47664389578751, -ERASE, 47664389570560, 47664389570560, -STORE, 47664389570560, 47664389578751, -STORE, 47664389578752, 47664389586943, -ERASE, 47664389382144, 47664389382144, -STORE, 47664389382144, 47664389398527, -STORE, 47664389398528, 47664389406719, -ERASE, 47664389570560, 47664389570560, -STORE, 47664389570560, 47664389574655, -STORE, 47664389574656, 47664389578751, -ERASE, 47664389533696, 47664389533696, -STORE, 47664389533696, 47664389537791, -STORE, 47664389537792, 47664389541887, -ERASE, 47664387362816, 47664387362816, -STORE, 47664387362816, 47664387559423, -STORE, 47664387559424, 47664387567615, -ERASE, 47664384524288, 47664384524288, -STORE, 47664384524288, 47664384528383, -STORE, 47664384528384, 47664384532479, -ERASE, 94071213555712, 94071213555712, -STORE, 94071213555712, 94071213563903, -STORE, 94071213563904, 94071213567999, -ERASE, 139968410804224, 139968410804224, -STORE, 139968410804224, 139968410808319, -STORE, 139968410808320, 139968410812415, -ERASE, 47664384352256, 47664384352256, -STORE, 94071244402688, 94071244537855, -STORE, 140737488347136, 140737488351231, -STORE, 140728271503360, 140737488351231, -ERASE, 140728271503360, 140728271503360, -STORE, 140728271503360, 140728271507455, -STORE, 94410361982976, 94410362155007, -ERASE, 94410361982976, 94410361982976, -STORE, 94410361982976, 94410361999359, -STORE, 94410361999360, 94410362155007, -ERASE, 94410361999360, 94410361999360, -STORE, 94410361999360, 94410362101759, -STORE, 94410362101760, 94410362142719, -STORE, 94410362142720, 94410362155007, -STORE, 140351953997824, 140351954169855, -ERASE, 140351953997824, 140351953997824, -STORE, 140351953997824, 140351954001919, -STORE, 140351954001920, 140351954169855, -ERASE, 140351954001920, 140351954001920, -STORE, 140351954001920, 140351954124799, -STORE, 140351954124800, 140351954157567, -STORE, 140351954157568, 140351954165759, -STORE, 140351954165760, 140351954169855, -STORE, 140728272429056, 140728272433151, -STORE, 140728272416768, 140728272429055, -STORE, 47280840998912, 47280841007103, -STORE, 47280841007104, 47280841015295, -STORE, 47280841015296, 47280841179135, -ERASE, 47280841015296, 47280841015296, -STORE, 47280841015296, 47280841027583, -STORE, 47280841027584, 47280841179135, -STORE, 47280841125888, 47280841179135, -STORE, 47280841027584, 47280841125887, -ERASE, 47280841027584, 47280841027584, -STORE, 47280841027584, 47280841125887, -STORE, 47280841170944, 47280841179135, -STORE, 47280841125888, 47280841170943, -ERASE, 47280841125888, 47280841125888, -STORE, 47280841125888, 47280841179135, -ERASE, 47280841125888, 47280841125888, -STORE, 47280841125888, 47280841170943, -STORE, 47280841170944, 47280841179135, -ERASE, 47280841170944, 47280841170944, -STORE, 47280841170944, 47280841179135, -STORE, 47280841179136, 47280844230655, -STORE, 47280841723904, 47280844230655, -STORE, 47280841179136, 47280841723903, -ERASE, 47280841723904, 47280841723904, -STORE, 47280841723904, 47280844009471, -STORE, 47280844009472, 47280844230655, -STORE, 47280843419648, 47280844009471, -STORE, 47280841723904, 47280843419647, -ERASE, 47280841723904, 47280841723904, -STORE, 47280841723904, 47280843419647, -STORE, 47280844005376, 47280844009471, -STORE, 47280843419648, 47280844005375, -ERASE, 47280843419648, 47280843419648, -STORE, 47280843419648, 47280844005375, -STORE, 47280844214272, 47280844230655, -STORE, 47280844009472, 47280844214271, -ERASE, 47280844009472, 47280844009472, -STORE, 47280844009472, 47280844214271, -ERASE, 47280844214272, 47280844214272, -STORE, 47280844214272, 47280844230655, -STORE, 47280844230656, 47280846069759, -STORE, 47280844369920, 47280846069759, -STORE, 47280844230656, 47280844369919, -ERASE, 47280844369920, 47280844369920, -STORE, 47280844369920, 47280846028799, -STORE, 47280846028800, 47280846069759, -STORE, 47280845713408, 47280846028799, -STORE, 47280844369920, 47280845713407, -ERASE, 47280844369920, 47280844369920, -STORE, 47280844369920, 47280845713407, -STORE, 47280846024704, 47280846028799, -STORE, 47280845713408, 47280846024703, -ERASE, 47280845713408, 47280845713408, -STORE, 47280845713408, 47280846024703, -STORE, 47280846053376, 47280846069759, -STORE, 47280846028800, 47280846053375, -ERASE, 47280846028800, 47280846028800, -STORE, 47280846028800, 47280846053375, -ERASE, 47280846053376, 47280846053376, -STORE, 47280846053376, 47280846069759, -STORE, 47280846069760, 47280846204927, -ERASE, 47280846069760, 47280846069760, -STORE, 47280846069760, 47280846094335, -STORE, 47280846094336, 47280846204927, -STORE, 47280846155776, 47280846204927, -STORE, 47280846094336, 47280846155775, -ERASE, 47280846094336, 47280846094336, -STORE, 47280846094336, 47280846155775, -STORE, 47280846180352, 47280846204927, -STORE, 47280846155776, 47280846180351, -ERASE, 47280846155776, 47280846155776, -STORE, 47280846155776, 47280846204927, -ERASE, 47280846155776, 47280846155776, -STORE, 47280846155776, 47280846180351, -STORE, 47280846180352, 47280846204927, -STORE, 47280846188544, 47280846204927, -STORE, 47280846180352, 47280846188543, -ERASE, 47280846180352, 47280846180352, -STORE, 47280846180352, 47280846188543, -ERASE, 47280846188544, 47280846188544, -STORE, 47280846188544, 47280846204927, -STORE, 47280846204928, 47280846225407, -ERASE, 47280846204928, 47280846204928, -STORE, 47280846204928, 47280846209023, -STORE, 47280846209024, 47280846225407, -STORE, 47280846213120, 47280846225407, -STORE, 47280846209024, 47280846213119, -ERASE, 47280846209024, 47280846209024, -STORE, 47280846209024, 47280846213119, -STORE, 47280846217216, 47280846225407, -STORE, 47280846213120, 47280846217215, -ERASE, 47280846213120, 47280846213120, -STORE, 47280846213120, 47280846225407, -ERASE, 47280846213120, 47280846213120, -STORE, 47280846213120, 47280846217215, -STORE, 47280846217216, 47280846225407, -ERASE, 47280846217216, 47280846217216, -STORE, 47280846217216, 47280846225407, -STORE, 47280846225408, 47280846233599, -ERASE, 47280846028800, 47280846028800, -STORE, 47280846028800, 47280846045183, -STORE, 47280846045184, 47280846053375, -ERASE, 47280846217216, 47280846217216, -STORE, 47280846217216, 47280846221311, -STORE, 47280846221312, 47280846225407, -ERASE, 47280846180352, 47280846180352, -STORE, 47280846180352, 47280846184447, -STORE, 47280846184448, 47280846188543, -ERASE, 47280844009472, 47280844009472, -STORE, 47280844009472, 47280844206079, -STORE, 47280844206080, 47280844214271, -ERASE, 47280841170944, 47280841170944, -STORE, 47280841170944, 47280841175039, -STORE, 47280841175040, 47280841179135, -ERASE, 94410362142720, 94410362142720, -STORE, 94410362142720, 94410362150911, -STORE, 94410362150912, 94410362155007, -ERASE, 140351954157568, 140351954157568, -STORE, 140351954157568, 140351954161663, -STORE, 140351954161664, 140351954165759, -ERASE, 47280840998912, 47280840998912, -STORE, 94410379456512, 94410379591679, -STORE, 140737488347136, 140737488351231, -STORE, 140732946362368, 140737488351231, -ERASE, 140732946362368, 140732946362368, -STORE, 140732946362368, 140732946366463, -STORE, 94352937934848, 94352938106879, -ERASE, 94352937934848, 94352937934848, -STORE, 94352937934848, 94352937951231, -STORE, 94352937951232, 94352938106879, -ERASE, 94352937951232, 94352937951232, -STORE, 94352937951232, 94352938053631, -STORE, 94352938053632, 94352938094591, -STORE, 94352938094592, 94352938106879, -STORE, 140595518742528, 140595518914559, -ERASE, 140595518742528, 140595518742528, -STORE, 140595518742528, 140595518746623, -STORE, 140595518746624, 140595518914559, -ERASE, 140595518746624, 140595518746624, -STORE, 140595518746624, 140595518869503, -STORE, 140595518869504, 140595518902271, -STORE, 140595518902272, 140595518910463, -STORE, 140595518910464, 140595518914559, -STORE, 140732947468288, 140732947472383, -STORE, 140732947456000, 140732947468287, -STORE, 47037276254208, 47037276262399, -STORE, 47037276262400, 47037276270591, -STORE, 47037276270592, 47037276434431, -ERASE, 47037276270592, 47037276270592, -STORE, 47037276270592, 47037276282879, -STORE, 47037276282880, 47037276434431, -STORE, 47037276381184, 47037276434431, -STORE, 47037276282880, 47037276381183, -ERASE, 47037276282880, 47037276282880, -STORE, 47037276282880, 47037276381183, -STORE, 47037276426240, 47037276434431, -STORE, 47037276381184, 47037276426239, -ERASE, 47037276381184, 47037276381184, -STORE, 47037276381184, 47037276434431, -ERASE, 47037276381184, 47037276381184, -STORE, 47037276381184, 47037276426239, -STORE, 47037276426240, 47037276434431, -ERASE, 47037276426240, 47037276426240, -STORE, 47037276426240, 47037276434431, -STORE, 47037276434432, 47037279485951, -STORE, 47037276979200, 47037279485951, -STORE, 47037276434432, 47037276979199, -ERASE, 47037276979200, 47037276979200, -STORE, 47037276979200, 47037279264767, -STORE, 47037279264768, 47037279485951, -STORE, 47037278674944, 47037279264767, -STORE, 47037276979200, 47037278674943, -ERASE, 47037276979200, 47037276979200, -STORE, 47037276979200, 47037278674943, -STORE, 47037279260672, 47037279264767, -STORE, 47037278674944, 47037279260671, -ERASE, 47037278674944, 47037278674944, -STORE, 47037278674944, 47037279260671, -STORE, 47037279469568, 47037279485951, -STORE, 47037279264768, 47037279469567, -ERASE, 47037279264768, 47037279264768, -STORE, 47037279264768, 47037279469567, -ERASE, 47037279469568, 47037279469568, -STORE, 47037279469568, 47037279485951, -STORE, 47037279485952, 47037281325055, -STORE, 47037279625216, 47037281325055, -STORE, 47037279485952, 47037279625215, -ERASE, 47037279625216, 47037279625216, -STORE, 47037279625216, 47037281284095, -STORE, 47037281284096, 47037281325055, -STORE, 47037280968704, 47037281284095, -STORE, 47037279625216, 47037280968703, -ERASE, 47037279625216, 47037279625216, -STORE, 47037279625216, 47037280968703, -STORE, 47037281280000, 47037281284095, -STORE, 47037280968704, 47037281279999, -ERASE, 47037280968704, 47037280968704, -STORE, 47037280968704, 47037281279999, -STORE, 47037281308672, 47037281325055, -STORE, 47037281284096, 47037281308671, -ERASE, 47037281284096, 47037281284096, -STORE, 47037281284096, 47037281308671, -ERASE, 47037281308672, 47037281308672, -STORE, 47037281308672, 47037281325055, -STORE, 47037281325056, 47037281460223, -ERASE, 47037281325056, 47037281325056, -STORE, 47037281325056, 47037281349631, -STORE, 47037281349632, 47037281460223, -STORE, 47037281411072, 47037281460223, -STORE, 47037281349632, 47037281411071, -ERASE, 47037281349632, 47037281349632, -STORE, 47037281349632, 47037281411071, -STORE, 47037281435648, 47037281460223, -STORE, 47037281411072, 47037281435647, -ERASE, 47037281411072, 47037281411072, -STORE, 47037281411072, 47037281460223, -ERASE, 47037281411072, 47037281411072, -STORE, 47037281411072, 47037281435647, -STORE, 47037281435648, 47037281460223, -STORE, 47037281443840, 47037281460223, -STORE, 47037281435648, 47037281443839, -ERASE, 47037281435648, 47037281435648, -STORE, 47037281435648, 47037281443839, -ERASE, 47037281443840, 47037281443840, -STORE, 47037281443840, 47037281460223, -STORE, 47037281460224, 47037281480703, -ERASE, 47037281460224, 47037281460224, -STORE, 47037281460224, 47037281464319, -STORE, 47037281464320, 47037281480703, -STORE, 47037281468416, 47037281480703, -STORE, 47037281464320, 47037281468415, -ERASE, 47037281464320, 47037281464320, -STORE, 47037281464320, 47037281468415, -STORE, 47037281472512, 47037281480703, -STORE, 47037281468416, 47037281472511, -ERASE, 47037281468416, 47037281468416, -STORE, 47037281468416, 47037281480703, -ERASE, 47037281468416, 47037281468416, -STORE, 47037281468416, 47037281472511, -STORE, 47037281472512, 47037281480703, -ERASE, 47037281472512, 47037281472512, -STORE, 47037281472512, 47037281480703, -STORE, 47037281480704, 47037281488895, -ERASE, 47037281284096, 47037281284096, -STORE, 47037281284096, 47037281300479, -STORE, 47037281300480, 47037281308671, -ERASE, 47037281472512, 47037281472512, -STORE, 47037281472512, 47037281476607, -STORE, 47037281476608, 47037281480703, -ERASE, 47037281435648, 47037281435648, -STORE, 47037281435648, 47037281439743, -STORE, 47037281439744, 47037281443839, -ERASE, 47037279264768, 47037279264768, -STORE, 47037279264768, 47037279461375, -STORE, 47037279461376, 47037279469567, -ERASE, 47037276426240, 47037276426240, -STORE, 47037276426240, 47037276430335, -STORE, 47037276430336, 47037276434431, -ERASE, 94352938094592, 94352938094592, -STORE, 94352938094592, 94352938102783, -STORE, 94352938102784, 94352938106879, -ERASE, 140595518902272, 140595518902272, -STORE, 140595518902272, 140595518906367, -STORE, 140595518906368, 140595518910463, -ERASE, 47037276254208, 47037276254208, -STORE, 94352938438656, 94352938573823, -STORE, 140737488347136, 140737488351231, -STORE, 140733506027520, 140737488351231, -ERASE, 140733506027520, 140733506027520, -STORE, 140733506027520, 140733506031615, -STORE, 94150123073536, 94150123245567, -ERASE, 94150123073536, 94150123073536, -STORE, 94150123073536, 94150123089919, -STORE, 94150123089920, 94150123245567, -ERASE, 94150123089920, 94150123089920, -STORE, 94150123089920, 94150123192319, -STORE, 94150123192320, 94150123233279, -STORE, 94150123233280, 94150123245567, -STORE, 140081290375168, 140081290547199, -ERASE, 140081290375168, 140081290375168, -STORE, 140081290375168, 140081290379263, -STORE, 140081290379264, 140081290547199, -ERASE, 140081290379264, 140081290379264, -STORE, 140081290379264, 140081290502143, -STORE, 140081290502144, 140081290534911, -STORE, 140081290534912, 140081290543103, -STORE, 140081290543104, 140081290547199, -STORE, 140733506707456, 140733506711551, -STORE, 140733506695168, 140733506707455, -STORE, 47551504621568, 47551504629759, -STORE, 47551504629760, 47551504637951, -STORE, 47551504637952, 47551504801791, -ERASE, 47551504637952, 47551504637952, -STORE, 47551504637952, 47551504650239, -STORE, 47551504650240, 47551504801791, -STORE, 47551504748544, 47551504801791, -STORE, 47551504650240, 47551504748543, -ERASE, 47551504650240, 47551504650240, -STORE, 47551504650240, 47551504748543, -STORE, 47551504793600, 47551504801791, -STORE, 47551504748544, 47551504793599, -ERASE, 47551504748544, 47551504748544, -STORE, 47551504748544, 47551504801791, -ERASE, 47551504748544, 47551504748544, -STORE, 47551504748544, 47551504793599, -STORE, 47551504793600, 47551504801791, -ERASE, 47551504793600, 47551504793600, -STORE, 47551504793600, 47551504801791, -STORE, 47551504801792, 47551507853311, -STORE, 47551505346560, 47551507853311, -STORE, 47551504801792, 47551505346559, -ERASE, 47551505346560, 47551505346560, -STORE, 47551505346560, 47551507632127, -STORE, 47551507632128, 47551507853311, -STORE, 47551507042304, 47551507632127, -STORE, 47551505346560, 47551507042303, -ERASE, 47551505346560, 47551505346560, -STORE, 47551505346560, 47551507042303, -STORE, 47551507628032, 47551507632127, -STORE, 47551507042304, 47551507628031, -ERASE, 47551507042304, 47551507042304, -STORE, 47551507042304, 47551507628031, -STORE, 47551507836928, 47551507853311, -STORE, 47551507632128, 47551507836927, -ERASE, 47551507632128, 47551507632128, -STORE, 47551507632128, 47551507836927, -ERASE, 47551507836928, 47551507836928, -STORE, 47551507836928, 47551507853311, -STORE, 47551507853312, 47551509692415, -STORE, 47551507992576, 47551509692415, -STORE, 47551507853312, 47551507992575, -ERASE, 47551507992576, 47551507992576, -STORE, 47551507992576, 47551509651455, -STORE, 47551509651456, 47551509692415, -STORE, 47551509336064, 47551509651455, -STORE, 47551507992576, 47551509336063, -ERASE, 47551507992576, 47551507992576, -STORE, 47551507992576, 47551509336063, -STORE, 47551509647360, 47551509651455, -STORE, 47551509336064, 47551509647359, -ERASE, 47551509336064, 47551509336064, -STORE, 47551509336064, 47551509647359, -STORE, 47551509676032, 47551509692415, -STORE, 47551509651456, 47551509676031, -ERASE, 47551509651456, 47551509651456, -STORE, 47551509651456, 47551509676031, -ERASE, 47551509676032, 47551509676032, -STORE, 47551509676032, 47551509692415, -STORE, 47551509692416, 47551509827583, -ERASE, 47551509692416, 47551509692416, -STORE, 47551509692416, 47551509716991, -STORE, 47551509716992, 47551509827583, -STORE, 47551509778432, 47551509827583, -STORE, 47551509716992, 47551509778431, -ERASE, 47551509716992, 47551509716992, -STORE, 47551509716992, 47551509778431, -STORE, 47551509803008, 47551509827583, -STORE, 47551509778432, 47551509803007, -ERASE, 47551509778432, 47551509778432, -STORE, 47551509778432, 47551509827583, -ERASE, 47551509778432, 47551509778432, -STORE, 47551509778432, 47551509803007, -STORE, 47551509803008, 47551509827583, -STORE, 47551509811200, 47551509827583, -STORE, 47551509803008, 47551509811199, -ERASE, 47551509803008, 47551509803008, -STORE, 47551509803008, 47551509811199, -ERASE, 47551509811200, 47551509811200, -STORE, 47551509811200, 47551509827583, -STORE, 47551509827584, 47551509848063, -ERASE, 47551509827584, 47551509827584, -STORE, 47551509827584, 47551509831679, -STORE, 47551509831680, 47551509848063, -STORE, 47551509835776, 47551509848063, -STORE, 47551509831680, 47551509835775, -ERASE, 47551509831680, 47551509831680, -STORE, 47551509831680, 47551509835775, -STORE, 47551509839872, 47551509848063, -STORE, 47551509835776, 47551509839871, -ERASE, 47551509835776, 47551509835776, -STORE, 47551509835776, 47551509848063, -ERASE, 47551509835776, 47551509835776, -STORE, 47551509835776, 47551509839871, -STORE, 47551509839872, 47551509848063, -ERASE, 47551509839872, 47551509839872, -STORE, 47551509839872, 47551509848063, -STORE, 47551509848064, 47551509856255, -ERASE, 47551509651456, 47551509651456, -STORE, 47551509651456, 47551509667839, -STORE, 47551509667840, 47551509676031, -ERASE, 47551509839872, 47551509839872, -STORE, 47551509839872, 47551509843967, -STORE, 47551509843968, 47551509848063, -ERASE, 47551509803008, 47551509803008, -STORE, 47551509803008, 47551509807103, -STORE, 47551509807104, 47551509811199, -ERASE, 47551507632128, 47551507632128, -STORE, 47551507632128, 47551507828735, -STORE, 47551507828736, 47551507836927, -ERASE, 47551504793600, 47551504793600, -STORE, 47551504793600, 47551504797695, -STORE, 47551504797696, 47551504801791, -ERASE, 94150123233280, 94150123233280, -STORE, 94150123233280, 94150123241471, -STORE, 94150123241472, 94150123245567, -ERASE, 140081290534912, 140081290534912, -STORE, 140081290534912, 140081290539007, -STORE, 140081290539008, 140081290543103, -ERASE, 47551504621568, 47551504621568, -STORE, 94150148112384, 94150148247551, -STORE, 140737488347136, 140737488351231, -STORE, 140734389334016, 140737488351231, -ERASE, 140734389334016, 140734389334016, -STORE, 140734389334016, 140734389338111, -STORE, 94844636606464, 94844636778495, -ERASE, 94844636606464, 94844636606464, -STORE, 94844636606464, 94844636622847, -STORE, 94844636622848, 94844636778495, -ERASE, 94844636622848, 94844636622848, -STORE, 94844636622848, 94844636725247, -STORE, 94844636725248, 94844636766207, -STORE, 94844636766208, 94844636778495, -STORE, 139922765217792, 139922765389823, -ERASE, 139922765217792, 139922765217792, -STORE, 139922765217792, 139922765221887, -STORE, 139922765221888, 139922765389823, -ERASE, 139922765221888, 139922765221888, -STORE, 139922765221888, 139922765344767, -STORE, 139922765344768, 139922765377535, -STORE, 139922765377536, 139922765385727, -STORE, 139922765385728, 139922765389823, -STORE, 140734389678080, 140734389682175, -STORE, 140734389665792, 140734389678079, -STORE, 47710029778944, 47710029787135, -STORE, 47710029787136, 47710029795327, -STORE, 47710029795328, 47710029959167, -ERASE, 47710029795328, 47710029795328, -STORE, 47710029795328, 47710029807615, -STORE, 47710029807616, 47710029959167, -STORE, 47710029905920, 47710029959167, -STORE, 47710029807616, 47710029905919, -ERASE, 47710029807616, 47710029807616, -STORE, 47710029807616, 47710029905919, -STORE, 47710029950976, 47710029959167, -STORE, 47710029905920, 47710029950975, -ERASE, 47710029905920, 47710029905920, -STORE, 47710029905920, 47710029959167, -ERASE, 47710029905920, 47710029905920, -STORE, 47710029905920, 47710029950975, -STORE, 47710029950976, 47710029959167, -ERASE, 47710029950976, 47710029950976, -STORE, 47710029950976, 47710029959167, -STORE, 47710029959168, 47710033010687, -STORE, 47710030503936, 47710033010687, -STORE, 47710029959168, 47710030503935, -ERASE, 47710030503936, 47710030503936, -STORE, 47710030503936, 47710032789503, -STORE, 47710032789504, 47710033010687, -STORE, 47710032199680, 47710032789503, -STORE, 47710030503936, 47710032199679, -ERASE, 47710030503936, 47710030503936, -STORE, 47710030503936, 47710032199679, -STORE, 47710032785408, 47710032789503, -STORE, 47710032199680, 47710032785407, -ERASE, 47710032199680, 47710032199680, -STORE, 47710032199680, 47710032785407, -STORE, 47710032994304, 47710033010687, -STORE, 47710032789504, 47710032994303, -ERASE, 47710032789504, 47710032789504, -STORE, 47710032789504, 47710032994303, -ERASE, 47710032994304, 47710032994304, -STORE, 47710032994304, 47710033010687, -STORE, 47710033010688, 47710034849791, -STORE, 47710033149952, 47710034849791, -STORE, 47710033010688, 47710033149951, -ERASE, 47710033149952, 47710033149952, -STORE, 47710033149952, 47710034808831, -STORE, 47710034808832, 47710034849791, -STORE, 47710034493440, 47710034808831, -STORE, 47710033149952, 47710034493439, -ERASE, 47710033149952, 47710033149952, -STORE, 47710033149952, 47710034493439, -STORE, 47710034804736, 47710034808831, -STORE, 47710034493440, 47710034804735, -ERASE, 47710034493440, 47710034493440, -STORE, 47710034493440, 47710034804735, -STORE, 47710034833408, 47710034849791, -STORE, 47710034808832, 47710034833407, -ERASE, 47710034808832, 47710034808832, -STORE, 47710034808832, 47710034833407, -ERASE, 47710034833408, 47710034833408, -STORE, 47710034833408, 47710034849791, -STORE, 47710034849792, 47710034984959, -ERASE, 47710034849792, 47710034849792, -STORE, 47710034849792, 47710034874367, -STORE, 47710034874368, 47710034984959, -STORE, 47710034935808, 47710034984959, -STORE, 47710034874368, 47710034935807, -ERASE, 47710034874368, 47710034874368, -STORE, 47710034874368, 47710034935807, -STORE, 47710034960384, 47710034984959, -STORE, 47710034935808, 47710034960383, -ERASE, 47710034935808, 47710034935808, -STORE, 47710034935808, 47710034984959, -ERASE, 47710034935808, 47710034935808, -STORE, 47710034935808, 47710034960383, -STORE, 47710034960384, 47710034984959, -STORE, 47710034968576, 47710034984959, -STORE, 47710034960384, 47710034968575, -ERASE, 47710034960384, 47710034960384, -STORE, 47710034960384, 47710034968575, -ERASE, 47710034968576, 47710034968576, -STORE, 47710034968576, 47710034984959, -STORE, 47710034984960, 47710035005439, -ERASE, 47710034984960, 47710034984960, -STORE, 47710034984960, 47710034989055, -STORE, 47710034989056, 47710035005439, -STORE, 47710034993152, 47710035005439, -STORE, 47710034989056, 47710034993151, -ERASE, 47710034989056, 47710034989056, -STORE, 47710034989056, 47710034993151, -STORE, 47710034997248, 47710035005439, -STORE, 47710034993152, 47710034997247, -ERASE, 47710034993152, 47710034993152, -STORE, 47710034993152, 47710035005439, -ERASE, 47710034993152, 47710034993152, -STORE, 47710034993152, 47710034997247, -STORE, 47710034997248, 47710035005439, -ERASE, 47710034997248, 47710034997248, -STORE, 47710034997248, 47710035005439, -STORE, 47710035005440, 47710035013631, -ERASE, 47710034808832, 47710034808832, -STORE, 47710034808832, 47710034825215, -STORE, 47710034825216, 47710034833407, -ERASE, 47710034997248, 47710034997248, -STORE, 47710034997248, 47710035001343, -STORE, 47710035001344, 47710035005439, -ERASE, 47710034960384, 47710034960384, -STORE, 47710034960384, 47710034964479, -STORE, 47710034964480, 47710034968575, -ERASE, 47710032789504, 47710032789504, -STORE, 47710032789504, 47710032986111, -STORE, 47710032986112, 47710032994303, -ERASE, 47710029950976, 47710029950976, -STORE, 47710029950976, 47710029955071, -STORE, 47710029955072, 47710029959167, -ERASE, 94844636766208, 94844636766208, -STORE, 94844636766208, 94844636774399, -STORE, 94844636774400, 94844636778495, -ERASE, 139922765377536, 139922765377536, -STORE, 139922765377536, 139922765381631, -STORE, 139922765381632, 139922765385727, -ERASE, 47710029778944, 47710029778944, -STORE, 94844641775616, 94844641910783, -STORE, 140737488347136, 140737488351231, -STORE, 140732213886976, 140737488351231, -ERASE, 140732213886976, 140732213886976, -STORE, 140732213886976, 140732213891071, -STORE, 94240508887040, 94240509059071, -ERASE, 94240508887040, 94240508887040, -STORE, 94240508887040, 94240508903423, -STORE, 94240508903424, 94240509059071, -ERASE, 94240508903424, 94240508903424, -STORE, 94240508903424, 94240509005823, -STORE, 94240509005824, 94240509046783, -STORE, 94240509046784, 94240509059071, -STORE, 140275106516992, 140275106689023, -ERASE, 140275106516992, 140275106516992, -STORE, 140275106516992, 140275106521087, -STORE, 140275106521088, 140275106689023, -ERASE, 140275106521088, 140275106521088, -STORE, 140275106521088, 140275106643967, -STORE, 140275106643968, 140275106676735, -STORE, 140275106676736, 140275106684927, -STORE, 140275106684928, 140275106689023, -STORE, 140732213977088, 140732213981183, -STORE, 140732213964800, 140732213977087, -STORE, 47357688479744, 47357688487935, -STORE, 47357688487936, 47357688496127, -STORE, 47357688496128, 47357688659967, -ERASE, 47357688496128, 47357688496128, -STORE, 47357688496128, 47357688508415, -STORE, 47357688508416, 47357688659967, -STORE, 47357688606720, 47357688659967, -STORE, 47357688508416, 47357688606719, -ERASE, 47357688508416, 47357688508416, -STORE, 47357688508416, 47357688606719, -STORE, 47357688651776, 47357688659967, -STORE, 47357688606720, 47357688651775, -ERASE, 47357688606720, 47357688606720, -STORE, 47357688606720, 47357688659967, -ERASE, 47357688606720, 47357688606720, -STORE, 47357688606720, 47357688651775, -STORE, 47357688651776, 47357688659967, -ERASE, 47357688651776, 47357688651776, -STORE, 47357688651776, 47357688659967, -STORE, 47357688659968, 47357691711487, -STORE, 47357689204736, 47357691711487, -STORE, 47357688659968, 47357689204735, -ERASE, 47357689204736, 47357689204736, -STORE, 47357689204736, 47357691490303, -STORE, 47357691490304, 47357691711487, -STORE, 47357690900480, 47357691490303, -STORE, 47357689204736, 47357690900479, -ERASE, 47357689204736, 47357689204736, -STORE, 47357689204736, 47357690900479, -STORE, 47357691486208, 47357691490303, -STORE, 47357690900480, 47357691486207, -ERASE, 47357690900480, 47357690900480, -STORE, 47357690900480, 47357691486207, -STORE, 47357691695104, 47357691711487, -STORE, 47357691490304, 47357691695103, -ERASE, 47357691490304, 47357691490304, -STORE, 47357691490304, 47357691695103, -ERASE, 47357691695104, 47357691695104, -STORE, 47357691695104, 47357691711487, -STORE, 47357691711488, 47357693550591, -STORE, 47357691850752, 47357693550591, -STORE, 47357691711488, 47357691850751, -ERASE, 47357691850752, 47357691850752, -STORE, 47357691850752, 47357693509631, -STORE, 47357693509632, 47357693550591, -STORE, 47357693194240, 47357693509631, -STORE, 47357691850752, 47357693194239, -ERASE, 47357691850752, 47357691850752, -STORE, 47357691850752, 47357693194239, -STORE, 47357693505536, 47357693509631, -STORE, 47357693194240, 47357693505535, -ERASE, 47357693194240, 47357693194240, -STORE, 47357693194240, 47357693505535, -STORE, 47357693534208, 47357693550591, -STORE, 47357693509632, 47357693534207, -ERASE, 47357693509632, 47357693509632, -STORE, 47357693509632, 47357693534207, -ERASE, 47357693534208, 47357693534208, -STORE, 47357693534208, 47357693550591, -STORE, 47357693550592, 47357693685759, -ERASE, 47357693550592, 47357693550592, -STORE, 47357693550592, 47357693575167, -STORE, 47357693575168, 47357693685759, -STORE, 47357693636608, 47357693685759, -STORE, 47357693575168, 47357693636607, -ERASE, 47357693575168, 47357693575168, -STORE, 47357693575168, 47357693636607, -STORE, 47357693661184, 47357693685759, -STORE, 47357693636608, 47357693661183, -ERASE, 47357693636608, 47357693636608, -STORE, 47357693636608, 47357693685759, -ERASE, 47357693636608, 47357693636608, -STORE, 47357693636608, 47357693661183, -STORE, 47357693661184, 47357693685759, -STORE, 47357693669376, 47357693685759, -STORE, 47357693661184, 47357693669375, -ERASE, 47357693661184, 47357693661184, -STORE, 47357693661184, 47357693669375, -ERASE, 47357693669376, 47357693669376, -STORE, 47357693669376, 47357693685759, -STORE, 47357693685760, 47357693706239, -ERASE, 47357693685760, 47357693685760, -STORE, 47357693685760, 47357693689855, -STORE, 47357693689856, 47357693706239, -STORE, 47357693693952, 47357693706239, -STORE, 47357693689856, 47357693693951, -ERASE, 47357693689856, 47357693689856, -STORE, 47357693689856, 47357693693951, -STORE, 47357693698048, 47357693706239, -STORE, 47357693693952, 47357693698047, -ERASE, 47357693693952, 47357693693952, -STORE, 47357693693952, 47357693706239, -ERASE, 47357693693952, 47357693693952, -STORE, 47357693693952, 47357693698047, -STORE, 47357693698048, 47357693706239, -ERASE, 47357693698048, 47357693698048, -STORE, 47357693698048, 47357693706239, -STORE, 47357693706240, 47357693714431, -ERASE, 47357693509632, 47357693509632, -STORE, 47357693509632, 47357693526015, -STORE, 47357693526016, 47357693534207, -ERASE, 47357693698048, 47357693698048, -STORE, 47357693698048, 47357693702143, -STORE, 47357693702144, 47357693706239, -ERASE, 47357693661184, 47357693661184, -STORE, 47357693661184, 47357693665279, -STORE, 47357693665280, 47357693669375, -ERASE, 47357691490304, 47357691490304, -STORE, 47357691490304, 47357691686911, -STORE, 47357691686912, 47357691695103, -ERASE, 47357688651776, 47357688651776, -STORE, 47357688651776, 47357688655871, -STORE, 47357688655872, 47357688659967, -ERASE, 94240509046784, 94240509046784, -STORE, 94240509046784, 94240509054975, -STORE, 94240509054976, 94240509059071, -ERASE, 140275106676736, 140275106676736, -STORE, 140275106676736, 140275106680831, -STORE, 140275106680832, 140275106684927, -ERASE, 47357688479744, 47357688479744, -STORE, 94240518361088, 94240518496255, -STORE, 140737488347136, 140737488351231, -STORE, 140732688277504, 140737488351231, -ERASE, 140732688277504, 140732688277504, -STORE, 140732688277504, 140732688281599, -STORE, 94629171351552, 94629172064255, -ERASE, 94629171351552, 94629171351552, -STORE, 94629171351552, 94629171400703, -STORE, 94629171400704, 94629172064255, -ERASE, 94629171400704, 94629171400704, -STORE, 94629171400704, 94629171945471, -STORE, 94629171945472, 94629172043775, -STORE, 94629172043776, 94629172064255, -STORE, 139770707644416, 139770707816447, -ERASE, 139770707644416, 139770707644416, -STORE, 139770707644416, 139770707648511, -STORE, 139770707648512, 139770707816447, -ERASE, 139770707648512, 139770707648512, -STORE, 139770707648512, 139770707771391, -STORE, 139770707771392, 139770707804159, -STORE, 139770707804160, 139770707812351, -STORE, 139770707812352, 139770707816447, -STORE, 140732689121280, 140732689125375, -STORE, 140732689108992, 140732689121279, -STORE, 47862087352320, 47862087360511, -STORE, 47862087360512, 47862087368703, -STORE, 47862087368704, 47862087475199, -STORE, 47862087385088, 47862087475199, -STORE, 47862087368704, 47862087385087, -ERASE, 47862087385088, 47862087385088, -STORE, 47862087385088, 47862087458815, -STORE, 47862087458816, 47862087475199, -STORE, 47862087438336, 47862087458815, -STORE, 47862087385088, 47862087438335, -ERASE, 47862087385088, 47862087385088, -STORE, 47862087385088, 47862087438335, -STORE, 47862087454720, 47862087458815, -STORE, 47862087438336, 47862087454719, -ERASE, 47862087438336, 47862087438336, -STORE, 47862087438336, 47862087454719, -STORE, 47862087467008, 47862087475199, -STORE, 47862087458816, 47862087467007, -ERASE, 47862087458816, 47862087458816, -STORE, 47862087458816, 47862087467007, -ERASE, 47862087467008, 47862087467008, -STORE, 47862087467008, 47862087475199, -STORE, 47862087475200, 47862089314303, -STORE, 47862087614464, 47862089314303, -STORE, 47862087475200, 47862087614463, -ERASE, 47862087614464, 47862087614464, -STORE, 47862087614464, 47862089273343, -STORE, 47862089273344, 47862089314303, -STORE, 47862088957952, 47862089273343, -STORE, 47862087614464, 47862088957951, -ERASE, 47862087614464, 47862087614464, -STORE, 47862087614464, 47862088957951, -STORE, 47862089269248, 47862089273343, -STORE, 47862088957952, 47862089269247, -ERASE, 47862088957952, 47862088957952, -STORE, 47862088957952, 47862089269247, -STORE, 47862089297920, 47862089314303, -STORE, 47862089273344, 47862089297919, -ERASE, 47862089273344, 47862089273344, -STORE, 47862089273344, 47862089297919, -ERASE, 47862089297920, 47862089297920, -STORE, 47862089297920, 47862089314303, -STORE, 47862089297920, 47862089326591, -ERASE, 47862089273344, 47862089273344, -STORE, 47862089273344, 47862089289727, -STORE, 47862089289728, 47862089297919, -ERASE, 47862087458816, 47862087458816, -STORE, 47862087458816, 47862087462911, -STORE, 47862087462912, 47862087467007, -ERASE, 94629172043776, 94629172043776, -STORE, 94629172043776, 94629172060159, -STORE, 94629172060160, 94629172064255, -ERASE, 139770707804160, 139770707804160, -STORE, 139770707804160, 139770707808255, -STORE, 139770707808256, 139770707812351, -ERASE, 47862087352320, 47862087352320, -STORE, 94629197533184, 94629197668351, -STORE, 140737488347136, 140737488351231, -STORE, 140727540711424, 140737488351231, -ERASE, 140727540711424, 140727540711424, -STORE, 140727540711424, 140727540715519, -STORE, 94299865313280, 94299866025983, -ERASE, 94299865313280, 94299865313280, -STORE, 94299865313280, 94299865362431, -STORE, 94299865362432, 94299866025983, -ERASE, 94299865362432, 94299865362432, -STORE, 94299865362432, 94299865907199, -STORE, 94299865907200, 94299866005503, -STORE, 94299866005504, 94299866025983, -STORE, 140680268763136, 140680268935167, -ERASE, 140680268763136, 140680268763136, -STORE, 140680268763136, 140680268767231, -STORE, 140680268767232, 140680268935167, -ERASE, 140680268767232, 140680268767232, -STORE, 140680268767232, 140680268890111, -STORE, 140680268890112, 140680268922879, -STORE, 140680268922880, 140680268931071, -STORE, 140680268931072, 140680268935167, -STORE, 140727541424128, 140727541428223, -STORE, 140727541411840, 140727541424127, -STORE, 46952526233600, 46952526241791, -STORE, 46952526241792, 46952526249983, -STORE, 46952526249984, 46952526356479, -STORE, 46952526266368, 46952526356479, -STORE, 46952526249984, 46952526266367, -ERASE, 46952526266368, 46952526266368, -STORE, 46952526266368, 46952526340095, -STORE, 46952526340096, 46952526356479, -STORE, 46952526319616, 46952526340095, -STORE, 46952526266368, 46952526319615, -ERASE, 46952526266368, 46952526266368, -STORE, 46952526266368, 46952526319615, -STORE, 46952526336000, 46952526340095, -STORE, 46952526319616, 46952526335999, -ERASE, 46952526319616, 46952526319616, -STORE, 46952526319616, 46952526335999, -STORE, 46952526348288, 46952526356479, -STORE, 46952526340096, 46952526348287, -ERASE, 46952526340096, 46952526340096, -STORE, 46952526340096, 46952526348287, -ERASE, 46952526348288, 46952526348288, -STORE, 46952526348288, 46952526356479, -STORE, 46952526356480, 46952528195583, -STORE, 46952526495744, 46952528195583, -STORE, 46952526356480, 46952526495743, -ERASE, 46952526495744, 46952526495744, -STORE, 46952526495744, 46952528154623, -STORE, 46952528154624, 46952528195583, -STORE, 46952527839232, 46952528154623, -STORE, 46952526495744, 46952527839231, -ERASE, 46952526495744, 46952526495744, -STORE, 46952526495744, 46952527839231, -STORE, 46952528150528, 46952528154623, -STORE, 46952527839232, 46952528150527, -ERASE, 46952527839232, 46952527839232, -STORE, 46952527839232, 46952528150527, -STORE, 46952528179200, 46952528195583, -STORE, 46952528154624, 46952528179199, -ERASE, 46952528154624, 46952528154624, -STORE, 46952528154624, 46952528179199, -ERASE, 46952528179200, 46952528179200, -STORE, 46952528179200, 46952528195583, -STORE, 46952528179200, 46952528207871, -ERASE, 46952528154624, 46952528154624, -STORE, 46952528154624, 46952528171007, -STORE, 46952528171008, 46952528179199, -ERASE, 46952526340096, 46952526340096, -STORE, 46952526340096, 46952526344191, -STORE, 46952526344192, 46952526348287, -ERASE, 94299866005504, 94299866005504, -STORE, 94299866005504, 94299866021887, -STORE, 94299866021888, 94299866025983, -ERASE, 140680268922880, 140680268922880, -STORE, 140680268922880, 140680268926975, -STORE, 140680268926976, 140680268931071, -ERASE, 46952526233600, 46952526233600, -STORE, 140737488347136, 140737488351231, -STORE, 140722874793984, 140737488351231, -ERASE, 140722874793984, 140722874793984, -STORE, 140722874793984, 140722874798079, -STORE, 94448916213760, 94448916926463, -ERASE, 94448916213760, 94448916213760, -STORE, 94448916213760, 94448916262911, -STORE, 94448916262912, 94448916926463, -ERASE, 94448916262912, 94448916262912, -STORE, 94448916262912, 94448916807679, -STORE, 94448916807680, 94448916905983, -STORE, 94448916905984, 94448916926463, -STORE, 140389117046784, 140389117218815, -ERASE, 140389117046784, 140389117046784, -STORE, 140389117046784, 140389117050879, -STORE, 140389117050880, 140389117218815, -ERASE, 140389117050880, 140389117050880, -STORE, 140389117050880, 140389117173759, -STORE, 140389117173760, 140389117206527, -STORE, 140389117206528, 140389117214719, -STORE, 140389117214720, 140389117218815, -STORE, 140722875297792, 140722875301887, -STORE, 140722875285504, 140722875297791, -STORE, 47243677949952, 47243677958143, -STORE, 47243677958144, 47243677966335, -STORE, 47243677966336, 47243678072831, -STORE, 47243677982720, 47243678072831, -STORE, 47243677966336, 47243677982719, -ERASE, 47243677982720, 47243677982720, -STORE, 47243677982720, 47243678056447, -STORE, 47243678056448, 47243678072831, -STORE, 47243678035968, 47243678056447, -STORE, 47243677982720, 47243678035967, -ERASE, 47243677982720, 47243677982720, -STORE, 47243677982720, 47243678035967, -STORE, 47243678052352, 47243678056447, -STORE, 47243678035968, 47243678052351, -ERASE, 47243678035968, 47243678035968, -STORE, 47243678035968, 47243678052351, -STORE, 47243678064640, 47243678072831, -STORE, 47243678056448, 47243678064639, -ERASE, 47243678056448, 47243678056448, -STORE, 47243678056448, 47243678064639, -ERASE, 47243678064640, 47243678064640, -STORE, 47243678064640, 47243678072831, -STORE, 47243678072832, 47243679911935, -STORE, 47243678212096, 47243679911935, -STORE, 47243678072832, 47243678212095, -ERASE, 47243678212096, 47243678212096, -STORE, 47243678212096, 47243679870975, -STORE, 47243679870976, 47243679911935, -STORE, 47243679555584, 47243679870975, -STORE, 47243678212096, 47243679555583, -ERASE, 47243678212096, 47243678212096, -STORE, 47243678212096, 47243679555583, -STORE, 47243679866880, 47243679870975, -STORE, 47243679555584, 47243679866879, -ERASE, 47243679555584, 47243679555584, -STORE, 47243679555584, 47243679866879, -STORE, 47243679895552, 47243679911935, -STORE, 47243679870976, 47243679895551, -ERASE, 47243679870976, 47243679870976, -STORE, 47243679870976, 47243679895551, -ERASE, 47243679895552, 47243679895552, -STORE, 47243679895552, 47243679911935, -STORE, 47243679895552, 47243679924223, -ERASE, 47243679870976, 47243679870976, -STORE, 47243679870976, 47243679887359, -STORE, 47243679887360, 47243679895551, -ERASE, 47243678056448, 47243678056448, -STORE, 47243678056448, 47243678060543, -STORE, 47243678060544, 47243678064639, -ERASE, 94448916905984, 94448916905984, -STORE, 94448916905984, 94448916922367, -STORE, 94448916922368, 94448916926463, -ERASE, 140389117206528, 140389117206528, -STORE, 140389117206528, 140389117210623, -STORE, 140389117210624, 140389117214719, -ERASE, 47243677949952, 47243677949952, -STORE, 140737488347136, 140737488351231, -STORE, 140733068505088, 140737488351231, -ERASE, 140733068505088, 140733068505088, -STORE, 140733068505088, 140733068509183, -STORE, 94207145750528, 94207146463231, -ERASE, 94207145750528, 94207145750528, -STORE, 94207145750528, 94207145799679, -STORE, 94207145799680, 94207146463231, -ERASE, 94207145799680, 94207145799680, -STORE, 94207145799680, 94207146344447, -STORE, 94207146344448, 94207146442751, -STORE, 94207146442752, 94207146463231, -STORE, 140684504911872, 140684505083903, -ERASE, 140684504911872, 140684504911872, -STORE, 140684504911872, 140684504915967, -STORE, 140684504915968, 140684505083903, -ERASE, 140684504915968, 140684504915968, -STORE, 140684504915968, 140684505038847, -STORE, 140684505038848, 140684505071615, -STORE, 140684505071616, 140684505079807, -STORE, 140684505079808, 140684505083903, -STORE, 140733068607488, 140733068611583, -STORE, 140733068595200, 140733068607487, -STORE, 46948290084864, 46948290093055, -STORE, 46948290093056, 46948290101247, -STORE, 46948290101248, 46948290207743, -STORE, 46948290117632, 46948290207743, -STORE, 46948290101248, 46948290117631, -ERASE, 46948290117632, 46948290117632, -STORE, 46948290117632, 46948290191359, -STORE, 46948290191360, 46948290207743, -STORE, 46948290170880, 46948290191359, -STORE, 46948290117632, 46948290170879, -ERASE, 46948290117632, 46948290117632, -STORE, 46948290117632, 46948290170879, -STORE, 46948290187264, 46948290191359, -STORE, 46948290170880, 46948290187263, -ERASE, 46948290170880, 46948290170880, -STORE, 46948290170880, 46948290187263, -STORE, 46948290199552, 46948290207743, -STORE, 46948290191360, 46948290199551, -ERASE, 46948290191360, 46948290191360, -STORE, 46948290191360, 46948290199551, -ERASE, 46948290199552, 46948290199552, -STORE, 46948290199552, 46948290207743, -STORE, 46948290207744, 46948292046847, -STORE, 46948290347008, 46948292046847, -STORE, 46948290207744, 46948290347007, -ERASE, 46948290347008, 46948290347008, -STORE, 46948290347008, 46948292005887, -STORE, 46948292005888, 46948292046847, -STORE, 46948291690496, 46948292005887, -STORE, 46948290347008, 46948291690495, -ERASE, 46948290347008, 46948290347008, -STORE, 46948290347008, 46948291690495, -STORE, 46948292001792, 46948292005887, -STORE, 46948291690496, 46948292001791, -ERASE, 46948291690496, 46948291690496, -STORE, 46948291690496, 46948292001791, -STORE, 46948292030464, 46948292046847, -STORE, 46948292005888, 46948292030463, -ERASE, 46948292005888, 46948292005888, -STORE, 46948292005888, 46948292030463, -ERASE, 46948292030464, 46948292030464, -STORE, 46948292030464, 46948292046847, -STORE, 46948292030464, 46948292059135, -ERASE, 46948292005888, 46948292005888, -STORE, 46948292005888, 46948292022271, -STORE, 46948292022272, 46948292030463, -ERASE, 46948290191360, 46948290191360, -STORE, 46948290191360, 46948290195455, -STORE, 46948290195456, 46948290199551, -ERASE, 94207146442752, 94207146442752, -STORE, 94207146442752, 94207146459135, -STORE, 94207146459136, 94207146463231, -ERASE, 140684505071616, 140684505071616, -STORE, 140684505071616, 140684505075711, -STORE, 140684505075712, 140684505079807, -ERASE, 46948290084864, 46948290084864, -STORE, 140737488347136, 140737488351231, -STORE, 140726367158272, 140737488351231, -ERASE, 140726367158272, 140726367158272, -STORE, 140726367158272, 140726367162367, -STORE, 94436124106752, 94436124819455, -ERASE, 94436124106752, 94436124106752, -STORE, 94436124106752, 94436124155903, -STORE, 94436124155904, 94436124819455, -ERASE, 94436124155904, 94436124155904, -STORE, 94436124155904, 94436124700671, -STORE, 94436124700672, 94436124798975, -STORE, 94436124798976, 94436124819455, -STORE, 140049025044480, 140049025216511, -ERASE, 140049025044480, 140049025044480, -STORE, 140049025044480, 140049025048575, -STORE, 140049025048576, 140049025216511, -ERASE, 140049025048576, 140049025048576, -STORE, 140049025048576, 140049025171455, -STORE, 140049025171456, 140049025204223, -STORE, 140049025204224, 140049025212415, -STORE, 140049025212416, 140049025216511, -STORE, 140726367256576, 140726367260671, -STORE, 140726367244288, 140726367256575, -STORE, 47583769952256, 47583769960447, -STORE, 47583769960448, 47583769968639, -STORE, 47583769968640, 47583770075135, -STORE, 47583769985024, 47583770075135, -STORE, 47583769968640, 47583769985023, -ERASE, 47583769985024, 47583769985024, -STORE, 47583769985024, 47583770058751, -STORE, 47583770058752, 47583770075135, -STORE, 47583770038272, 47583770058751, -STORE, 47583769985024, 47583770038271, -ERASE, 47583769985024, 47583769985024, -STORE, 47583769985024, 47583770038271, -STORE, 47583770054656, 47583770058751, -STORE, 47583770038272, 47583770054655, -ERASE, 47583770038272, 47583770038272, -STORE, 47583770038272, 47583770054655, -STORE, 47583770066944, 47583770075135, -STORE, 47583770058752, 47583770066943, -ERASE, 47583770058752, 47583770058752, -STORE, 47583770058752, 47583770066943, -ERASE, 47583770066944, 47583770066944, -STORE, 47583770066944, 47583770075135, -STORE, 47583770075136, 47583771914239, -STORE, 47583770214400, 47583771914239, -STORE, 47583770075136, 47583770214399, -ERASE, 47583770214400, 47583770214400, -STORE, 47583770214400, 47583771873279, -STORE, 47583771873280, 47583771914239, -STORE, 47583771557888, 47583771873279, -STORE, 47583770214400, 47583771557887, -ERASE, 47583770214400, 47583770214400, -STORE, 47583770214400, 47583771557887, -STORE, 47583771869184, 47583771873279, -STORE, 47583771557888, 47583771869183, -ERASE, 47583771557888, 47583771557888, -STORE, 47583771557888, 47583771869183, -STORE, 47583771897856, 47583771914239, -STORE, 47583771873280, 47583771897855, -ERASE, 47583771873280, 47583771873280, -STORE, 47583771873280, 47583771897855, -ERASE, 47583771897856, 47583771897856, -STORE, 47583771897856, 47583771914239, -STORE, 47583771897856, 47583771926527, -ERASE, 47583771873280, 47583771873280, -STORE, 47583771873280, 47583771889663, -STORE, 47583771889664, 47583771897855, -ERASE, 47583770058752, 47583770058752, -STORE, 47583770058752, 47583770062847, -STORE, 47583770062848, 47583770066943, -ERASE, 94436124798976, 94436124798976, -STORE, 94436124798976, 94436124815359, -STORE, 94436124815360, 94436124819455, -ERASE, 140049025204224, 140049025204224, -STORE, 140049025204224, 140049025208319, -STORE, 140049025208320, 140049025212415, -ERASE, 47583769952256, 47583769952256, -STORE, 140737488347136, 140737488351231, -STORE, 140727116099584, 140737488351231, -ERASE, 140727116099584, 140727116099584, -STORE, 140727116099584, 140727116103679, -STORE, 94166319734784, 94166320447487, -ERASE, 94166319734784, 94166319734784, -STORE, 94166319734784, 94166319783935, -STORE, 94166319783936, 94166320447487, -ERASE, 94166319783936, 94166319783936, -STORE, 94166319783936, 94166320328703, -STORE, 94166320328704, 94166320427007, -STORE, 94166320427008, 94166320447487, -STORE, 139976559542272, 139976559714303, -ERASE, 139976559542272, 139976559542272, -STORE, 139976559542272, 139976559546367, -STORE, 139976559546368, 139976559714303, -ERASE, 139976559546368, 139976559546368, -STORE, 139976559546368, 139976559669247, -STORE, 139976559669248, 139976559702015, -STORE, 139976559702016, 139976559710207, -STORE, 139976559710208, 139976559714303, -STORE, 140727116222464, 140727116226559, -STORE, 140727116210176, 140727116222463, -STORE, 47656235454464, 47656235462655, -STORE, 47656235462656, 47656235470847, -STORE, 47656235470848, 47656235577343, -STORE, 47656235487232, 47656235577343, -STORE, 47656235470848, 47656235487231, -ERASE, 47656235487232, 47656235487232, -STORE, 47656235487232, 47656235560959, -STORE, 47656235560960, 47656235577343, -STORE, 47656235540480, 47656235560959, -STORE, 47656235487232, 47656235540479, -ERASE, 47656235487232, 47656235487232, -STORE, 47656235487232, 47656235540479, -STORE, 47656235556864, 47656235560959, -STORE, 47656235540480, 47656235556863, -ERASE, 47656235540480, 47656235540480, -STORE, 47656235540480, 47656235556863, -STORE, 47656235569152, 47656235577343, -STORE, 47656235560960, 47656235569151, -ERASE, 47656235560960, 47656235560960, -STORE, 47656235560960, 47656235569151, -ERASE, 47656235569152, 47656235569152, -STORE, 47656235569152, 47656235577343, -STORE, 47656235577344, 47656237416447, -STORE, 47656235716608, 47656237416447, -STORE, 47656235577344, 47656235716607, -ERASE, 47656235716608, 47656235716608, -STORE, 47656235716608, 47656237375487, -STORE, 47656237375488, 47656237416447, -STORE, 47656237060096, 47656237375487, -STORE, 47656235716608, 47656237060095, -ERASE, 47656235716608, 47656235716608, -STORE, 47656235716608, 47656237060095, -STORE, 47656237371392, 47656237375487, -STORE, 47656237060096, 47656237371391, -ERASE, 47656237060096, 47656237060096, -STORE, 47656237060096, 47656237371391, -STORE, 47656237400064, 47656237416447, -STORE, 47656237375488, 47656237400063, -ERASE, 47656237375488, 47656237375488, -STORE, 47656237375488, 47656237400063, -ERASE, 47656237400064, 47656237400064, -STORE, 47656237400064, 47656237416447, -STORE, 47656237400064, 47656237428735, -ERASE, 47656237375488, 47656237375488, -STORE, 47656237375488, 47656237391871, -STORE, 47656237391872, 47656237400063, -ERASE, 47656235560960, 47656235560960, -STORE, 47656235560960, 47656235565055, -STORE, 47656235565056, 47656235569151, -ERASE, 94166320427008, 94166320427008, -STORE, 94166320427008, 94166320443391, -STORE, 94166320443392, 94166320447487, -ERASE, 139976559702016, 139976559702016, -STORE, 139976559702016, 139976559706111, -STORE, 139976559706112, 139976559710207, -ERASE, 47656235454464, 47656235454464, -STORE, 94166332153856, 94166332289023, -STORE, 140737488347136, 140737488351231, -STORE, 140726412816384, 140737488351231, -ERASE, 140726412816384, 140726412816384, -STORE, 140726412816384, 140726412820479, -STORE, 94094884507648, 94094885220351, -ERASE, 94094884507648, 94094884507648, -STORE, 94094884507648, 94094884556799, -STORE, 94094884556800, 94094885220351, -ERASE, 94094884556800, 94094884556800, -STORE, 94094884556800, 94094885101567, -STORE, 94094885101568, 94094885199871, -STORE, 94094885199872, 94094885220351, -STORE, 139773773938688, 139773774110719, -ERASE, 139773773938688, 139773773938688, -STORE, 139773773938688, 139773773942783, -STORE, 139773773942784, 139773774110719, -ERASE, 139773773942784, 139773773942784, -STORE, 139773773942784, 139773774065663, -STORE, 139773774065664, 139773774098431, -STORE, 139773774098432, 139773774106623, -STORE, 139773774106624, 139773774110719, -STORE, 140726412963840, 140726412967935, -STORE, 140726412951552, 140726412963839, -STORE, 47859021058048, 47859021066239, -STORE, 47859021066240, 47859021074431, -STORE, 47859021074432, 47859021180927, -STORE, 47859021090816, 47859021180927, -STORE, 47859021074432, 47859021090815, -ERASE, 47859021090816, 47859021090816, -STORE, 47859021090816, 47859021164543, -STORE, 47859021164544, 47859021180927, -STORE, 47859021144064, 47859021164543, -STORE, 47859021090816, 47859021144063, -ERASE, 47859021090816, 47859021090816, -STORE, 47859021090816, 47859021144063, -STORE, 47859021160448, 47859021164543, -STORE, 47859021144064, 47859021160447, -ERASE, 47859021144064, 47859021144064, -STORE, 47859021144064, 47859021160447, -STORE, 47859021172736, 47859021180927, -STORE, 47859021164544, 47859021172735, -ERASE, 47859021164544, 47859021164544, -STORE, 47859021164544, 47859021172735, -ERASE, 47859021172736, 47859021172736, -STORE, 47859021172736, 47859021180927, -STORE, 47859021180928, 47859023020031, -STORE, 47859021320192, 47859023020031, -STORE, 47859021180928, 47859021320191, -ERASE, 47859021320192, 47859021320192, -STORE, 47859021320192, 47859022979071, -STORE, 47859022979072, 47859023020031, -STORE, 47859022663680, 47859022979071, -STORE, 47859021320192, 47859022663679, -ERASE, 47859021320192, 47859021320192, -STORE, 47859021320192, 47859022663679, -STORE, 47859022974976, 47859022979071, -STORE, 47859022663680, 47859022974975, -ERASE, 47859022663680, 47859022663680, -STORE, 47859022663680, 47859022974975, -STORE, 47859023003648, 47859023020031, -STORE, 47859022979072, 47859023003647, -ERASE, 47859022979072, 47859022979072, -STORE, 47859022979072, 47859023003647, -ERASE, 47859023003648, 47859023003648, -STORE, 47859023003648, 47859023020031, -STORE, 47859023003648, 47859023032319, -ERASE, 47859022979072, 47859022979072, -STORE, 47859022979072, 47859022995455, -STORE, 47859022995456, 47859023003647, -ERASE, 47859021164544, 47859021164544, -STORE, 47859021164544, 47859021168639, -STORE, 47859021168640, 47859021172735, -ERASE, 94094885199872, 94094885199872, -STORE, 94094885199872, 94094885216255, -STORE, 94094885216256, 94094885220351, -ERASE, 139773774098432, 139773774098432, -STORE, 139773774098432, 139773774102527, -STORE, 139773774102528, 139773774106623, -ERASE, 47859021058048, 47859021058048, -STORE, 94094901108736, 94094901243903, -STORE, 140737488347136, 140737488351231, -STORE, 140736567963648, 140737488351231, -ERASE, 140736567963648, 140736567963648, -STORE, 140736567963648, 140736567967743, -STORE, 94924425748480, 94924426461183, -ERASE, 94924425748480, 94924425748480, -STORE, 94924425748480, 94924425797631, -STORE, 94924425797632, 94924426461183, -ERASE, 94924425797632, 94924425797632, -STORE, 94924425797632, 94924426342399, -STORE, 94924426342400, 94924426440703, -STORE, 94924426440704, 94924426461183, -STORE, 140042126319616, 140042126491647, -ERASE, 140042126319616, 140042126319616, -STORE, 140042126319616, 140042126323711, -STORE, 140042126323712, 140042126491647, -ERASE, 140042126323712, 140042126323712, -STORE, 140042126323712, 140042126446591, -STORE, 140042126446592, 140042126479359, -STORE, 140042126479360, 140042126487551, -STORE, 140042126487552, 140042126491647, -STORE, 140736568672256, 140736568676351, -STORE, 140736568659968, 140736568672255, -STORE, 47590668677120, 47590668685311, -STORE, 47590668685312, 47590668693503, -STORE, 47590668693504, 47590668799999, -STORE, 47590668709888, 47590668799999, -STORE, 47590668693504, 47590668709887, -ERASE, 47590668709888, 47590668709888, -STORE, 47590668709888, 47590668783615, -STORE, 47590668783616, 47590668799999, -STORE, 47590668763136, 47590668783615, -STORE, 47590668709888, 47590668763135, -ERASE, 47590668709888, 47590668709888, -STORE, 47590668709888, 47590668763135, -STORE, 47590668779520, 47590668783615, -STORE, 47590668763136, 47590668779519, -ERASE, 47590668763136, 47590668763136, -STORE, 47590668763136, 47590668779519, -STORE, 47590668791808, 47590668799999, -STORE, 47590668783616, 47590668791807, -ERASE, 47590668783616, 47590668783616, -STORE, 47590668783616, 47590668791807, -ERASE, 47590668791808, 47590668791808, -STORE, 47590668791808, 47590668799999, -STORE, 47590668800000, 47590670639103, -STORE, 47590668939264, 47590670639103, -STORE, 47590668800000, 47590668939263, -ERASE, 47590668939264, 47590668939264, -STORE, 47590668939264, 47590670598143, -STORE, 47590670598144, 47590670639103, -STORE, 47590670282752, 47590670598143, -STORE, 47590668939264, 47590670282751, -ERASE, 47590668939264, 47590668939264, -STORE, 47590668939264, 47590670282751, -STORE, 47590670594048, 47590670598143, -STORE, 47590670282752, 47590670594047, -ERASE, 47590670282752, 47590670282752, -STORE, 47590670282752, 47590670594047, -STORE, 47590670622720, 47590670639103, -STORE, 47590670598144, 47590670622719, -ERASE, 47590670598144, 47590670598144, -STORE, 47590670598144, 47590670622719, -ERASE, 47590670622720, 47590670622720, -STORE, 47590670622720, 47590670639103, -STORE, 47590670622720, 47590670651391, -ERASE, 47590670598144, 47590670598144, -STORE, 47590670598144, 47590670614527, -STORE, 47590670614528, 47590670622719, -ERASE, 47590668783616, 47590668783616, -STORE, 47590668783616, 47590668787711, -STORE, 47590668787712, 47590668791807, -ERASE, 94924426440704, 94924426440704, -STORE, 94924426440704, 94924426457087, -STORE, 94924426457088, 94924426461183, -ERASE, 140042126479360, 140042126479360, -STORE, 140042126479360, 140042126483455, -STORE, 140042126483456, 140042126487551, -ERASE, 47590668677120, 47590668677120, -STORE, 140737488347136, 140737488351231, -STORE, 140733281439744, 140737488351231, -ERASE, 140733281439744, 140733281439744, -STORE, 140733281439744, 140733281443839, -STORE, 94490667069440, 94490667782143, -ERASE, 94490667069440, 94490667069440, -STORE, 94490667069440, 94490667118591, -STORE, 94490667118592, 94490667782143, -ERASE, 94490667118592, 94490667118592, -STORE, 94490667118592, 94490667663359, -STORE, 94490667663360, 94490667761663, -STORE, 94490667761664, 94490667782143, -STORE, 139878215118848, 139878215290879, -ERASE, 139878215118848, 139878215118848, -STORE, 139878215118848, 139878215122943, -STORE, 139878215122944, 139878215290879, -ERASE, 139878215122944, 139878215122944, -STORE, 139878215122944, 139878215245823, -STORE, 139878215245824, 139878215278591, -STORE, 139878215278592, 139878215286783, -STORE, 139878215286784, 139878215290879, -STORE, 140733281464320, 140733281468415, -STORE, 140733281452032, 140733281464319, -STORE, 47754579877888, 47754579886079, -STORE, 47754579886080, 47754579894271, -STORE, 47754579894272, 47754580000767, -STORE, 47754579910656, 47754580000767, -STORE, 47754579894272, 47754579910655, -ERASE, 47754579910656, 47754579910656, -STORE, 47754579910656, 47754579984383, -STORE, 47754579984384, 47754580000767, -STORE, 47754579963904, 47754579984383, -STORE, 47754579910656, 47754579963903, -ERASE, 47754579910656, 47754579910656, -STORE, 47754579910656, 47754579963903, -STORE, 47754579980288, 47754579984383, -STORE, 47754579963904, 47754579980287, -ERASE, 47754579963904, 47754579963904, -STORE, 47754579963904, 47754579980287, -STORE, 47754579992576, 47754580000767, -STORE, 47754579984384, 47754579992575, -ERASE, 47754579984384, 47754579984384, -STORE, 47754579984384, 47754579992575, -ERASE, 47754579992576, 47754579992576, -STORE, 47754579992576, 47754580000767, -STORE, 47754580000768, 47754581839871, -STORE, 47754580140032, 47754581839871, -STORE, 47754580000768, 47754580140031, -ERASE, 47754580140032, 47754580140032, -STORE, 47754580140032, 47754581798911, -STORE, 47754581798912, 47754581839871, -STORE, 47754581483520, 47754581798911, -STORE, 47754580140032, 47754581483519, -ERASE, 47754580140032, 47754580140032, -STORE, 47754580140032, 47754581483519, -STORE, 47754581794816, 47754581798911, -STORE, 47754581483520, 47754581794815, -ERASE, 47754581483520, 47754581483520, -STORE, 47754581483520, 47754581794815, -STORE, 47754581823488, 47754581839871, -STORE, 47754581798912, 47754581823487, -ERASE, 47754581798912, 47754581798912, -STORE, 47754581798912, 47754581823487, -ERASE, 47754581823488, 47754581823488, -STORE, 47754581823488, 47754581839871, -STORE, 47754581823488, 47754581852159, -ERASE, 47754581798912, 47754581798912, -STORE, 47754581798912, 47754581815295, -STORE, 47754581815296, 47754581823487, -ERASE, 47754579984384, 47754579984384, -STORE, 47754579984384, 47754579988479, -STORE, 47754579988480, 47754579992575, -ERASE, 94490667761664, 94490667761664, -STORE, 94490667761664, 94490667778047, -STORE, 94490667778048, 94490667782143, -ERASE, 139878215278592, 139878215278592, -STORE, 139878215278592, 139878215282687, -STORE, 139878215282688, 139878215286783, -ERASE, 47754579877888, 47754579877888, -STORE, 94490669649920, 94490669785087, -STORE, 140737488347136, 140737488351231, -STORE, 140735382188032, 140737488351231, -ERASE, 140735382188032, 140735382188032, -STORE, 140735382188032, 140735382192127, -STORE, 94150181302272, 94150182014975, -ERASE, 94150181302272, 94150181302272, -STORE, 94150181302272, 94150181351423, -STORE, 94150181351424, 94150182014975, -ERASE, 94150181351424, 94150181351424, -STORE, 94150181351424, 94150181896191, -STORE, 94150181896192, 94150181994495, -STORE, 94150181994496, 94150182014975, -STORE, 139679752458240, 139679752630271, -ERASE, 139679752458240, 139679752458240, -STORE, 139679752458240, 139679752462335, -STORE, 139679752462336, 139679752630271, -ERASE, 139679752462336, 139679752462336, -STORE, 139679752462336, 139679752585215, -STORE, 139679752585216, 139679752617983, -STORE, 139679752617984, 139679752626175, -STORE, 139679752626176, 139679752630271, -STORE, 140735382536192, 140735382540287, -STORE, 140735382523904, 140735382536191, -STORE, 47953042538496, 47953042546687, -STORE, 47953042546688, 47953042554879, -STORE, 47953042554880, 47953042661375, -STORE, 47953042571264, 47953042661375, -STORE, 47953042554880, 47953042571263, -ERASE, 47953042571264, 47953042571264, -STORE, 47953042571264, 47953042644991, -STORE, 47953042644992, 47953042661375, -STORE, 47953042624512, 47953042644991, -STORE, 47953042571264, 47953042624511, -ERASE, 47953042571264, 47953042571264, -STORE, 47953042571264, 47953042624511, -STORE, 47953042640896, 47953042644991, -STORE, 47953042624512, 47953042640895, -ERASE, 47953042624512, 47953042624512, -STORE, 47953042624512, 47953042640895, -STORE, 47953042653184, 47953042661375, -STORE, 47953042644992, 47953042653183, -ERASE, 47953042644992, 47953042644992, -STORE, 47953042644992, 47953042653183, -ERASE, 47953042653184, 47953042653184, -STORE, 47953042653184, 47953042661375, -STORE, 47953042661376, 47953044500479, -STORE, 47953042800640, 47953044500479, -STORE, 47953042661376, 47953042800639, -ERASE, 47953042800640, 47953042800640, -STORE, 47953042800640, 47953044459519, -STORE, 47953044459520, 47953044500479, -STORE, 47953044144128, 47953044459519, -STORE, 47953042800640, 47953044144127, -ERASE, 47953042800640, 47953042800640, -STORE, 47953042800640, 47953044144127, -STORE, 47953044455424, 47953044459519, -STORE, 47953044144128, 47953044455423, -ERASE, 47953044144128, 47953044144128, -STORE, 47953044144128, 47953044455423, -STORE, 47953044484096, 47953044500479, -STORE, 47953044459520, 47953044484095, -ERASE, 47953044459520, 47953044459520, -STORE, 47953044459520, 47953044484095, -ERASE, 47953044484096, 47953044484096, -STORE, 47953044484096, 47953044500479, -STORE, 47953044484096, 47953044512767, -ERASE, 47953044459520, 47953044459520, -STORE, 47953044459520, 47953044475903, -STORE, 47953044475904, 47953044484095, -ERASE, 47953042644992, 47953042644992, -STORE, 47953042644992, 47953042649087, -STORE, 47953042649088, 47953042653183, -ERASE, 94150181994496, 94150181994496, -STORE, 94150181994496, 94150182010879, -STORE, 94150182010880, 94150182014975, -ERASE, 139679752617984, 139679752617984, -STORE, 139679752617984, 139679752622079, -STORE, 139679752622080, 139679752626175, -ERASE, 47953042538496, 47953042538496, -STORE, 140737488347136, 140737488351231, -STORE, 140737044123648, 140737488351231, -ERASE, 140737044123648, 140737044123648, -STORE, 140737044123648, 140737044127743, -STORE, 94425324294144, 94425325006847, -ERASE, 94425324294144, 94425324294144, -STORE, 94425324294144, 94425324343295, -STORE, 94425324343296, 94425325006847, -ERASE, 94425324343296, 94425324343296, -STORE, 94425324343296, 94425324888063, -STORE, 94425324888064, 94425324986367, -STORE, 94425324986368, 94425325006847, -STORE, 140382015016960, 140382015188991, -ERASE, 140382015016960, 140382015016960, -STORE, 140382015016960, 140382015021055, -STORE, 140382015021056, 140382015188991, -ERASE, 140382015021056, 140382015021056, -STORE, 140382015021056, 140382015143935, -STORE, 140382015143936, 140382015176703, -STORE, 140382015176704, 140382015184895, -STORE, 140382015184896, 140382015188991, -STORE, 140737045585920, 140737045590015, -STORE, 140737045573632, 140737045585919, -STORE, 47250779979776, 47250779987967, -STORE, 47250779987968, 47250779996159, -STORE, 47250779996160, 47250780102655, -STORE, 47250780012544, 47250780102655, -STORE, 47250779996160, 47250780012543, -ERASE, 47250780012544, 47250780012544, -STORE, 47250780012544, 47250780086271, -STORE, 47250780086272, 47250780102655, -STORE, 47250780065792, 47250780086271, -STORE, 47250780012544, 47250780065791, -ERASE, 47250780012544, 47250780012544, -STORE, 47250780012544, 47250780065791, -STORE, 47250780082176, 47250780086271, -STORE, 47250780065792, 47250780082175, -ERASE, 47250780065792, 47250780065792, -STORE, 47250780065792, 47250780082175, -STORE, 47250780094464, 47250780102655, -STORE, 47250780086272, 47250780094463, -ERASE, 47250780086272, 47250780086272, -STORE, 47250780086272, 47250780094463, -ERASE, 47250780094464, 47250780094464, -STORE, 47250780094464, 47250780102655, -STORE, 47250780102656, 47250781941759, -STORE, 47250780241920, 47250781941759, -STORE, 47250780102656, 47250780241919, -ERASE, 47250780241920, 47250780241920, -STORE, 47250780241920, 47250781900799, -STORE, 47250781900800, 47250781941759, -STORE, 47250781585408, 47250781900799, -STORE, 47250780241920, 47250781585407, -ERASE, 47250780241920, 47250780241920, -STORE, 47250780241920, 47250781585407, -STORE, 47250781896704, 47250781900799, -STORE, 47250781585408, 47250781896703, -ERASE, 47250781585408, 47250781585408, -STORE, 47250781585408, 47250781896703, -STORE, 47250781925376, 47250781941759, -STORE, 47250781900800, 47250781925375, -ERASE, 47250781900800, 47250781900800, -STORE, 47250781900800, 47250781925375, -ERASE, 47250781925376, 47250781925376, -STORE, 47250781925376, 47250781941759, -STORE, 47250781925376, 47250781954047, -ERASE, 47250781900800, 47250781900800, -STORE, 47250781900800, 47250781917183, -STORE, 47250781917184, 47250781925375, -ERASE, 47250780086272, 47250780086272, -STORE, 47250780086272, 47250780090367, -STORE, 47250780090368, 47250780094463, -ERASE, 94425324986368, 94425324986368, -STORE, 94425324986368, 94425325002751, -STORE, 94425325002752, 94425325006847, -ERASE, 140382015176704, 140382015176704, -STORE, 140382015176704, 140382015180799, -STORE, 140382015180800, 140382015184895, -ERASE, 47250779979776, 47250779979776, -STORE, 94425351438336, 94425351573503, -STORE, 140737488347136, 140737488351231, -STORE, 140736801144832, 140737488351231, -ERASE, 140736801144832, 140736801144832, -STORE, 140736801144832, 140736801148927, -STORE, 94629429358592, 94629430071295, -ERASE, 94629429358592, 94629429358592, -STORE, 94629429358592, 94629429407743, -STORE, 94629429407744, 94629430071295, -ERASE, 94629429407744, 94629429407744, -STORE, 94629429407744, 94629429952511, -STORE, 94629429952512, 94629430050815, -STORE, 94629430050816, 94629430071295, -STORE, 139801685483520, 139801685655551, -ERASE, 139801685483520, 139801685483520, -STORE, 139801685483520, 139801685487615, -STORE, 139801685487616, 139801685655551, -ERASE, 139801685487616, 139801685487616, -STORE, 139801685487616, 139801685610495, -STORE, 139801685610496, 139801685643263, -STORE, 139801685643264, 139801685651455, -STORE, 139801685651456, 139801685655551, -STORE, 140736801198080, 140736801202175, -STORE, 140736801185792, 140736801198079, -STORE, 47831109513216, 47831109521407, -STORE, 47831109521408, 47831109529599, -STORE, 47831109529600, 47831109636095, -STORE, 47831109545984, 47831109636095, -STORE, 47831109529600, 47831109545983, -ERASE, 47831109545984, 47831109545984, -STORE, 47831109545984, 47831109619711, -STORE, 47831109619712, 47831109636095, -STORE, 47831109599232, 47831109619711, -STORE, 47831109545984, 47831109599231, -ERASE, 47831109545984, 47831109545984, -STORE, 47831109545984, 47831109599231, -STORE, 47831109615616, 47831109619711, -STORE, 47831109599232, 47831109615615, -ERASE, 47831109599232, 47831109599232, -STORE, 47831109599232, 47831109615615, -STORE, 47831109627904, 47831109636095, -STORE, 47831109619712, 47831109627903, -ERASE, 47831109619712, 47831109619712, -STORE, 47831109619712, 47831109627903, -ERASE, 47831109627904, 47831109627904, -STORE, 47831109627904, 47831109636095, -STORE, 47831109636096, 47831111475199, -STORE, 47831109775360, 47831111475199, -STORE, 47831109636096, 47831109775359, -ERASE, 47831109775360, 47831109775360, -STORE, 47831109775360, 47831111434239, -STORE, 47831111434240, 47831111475199, -STORE, 47831111118848, 47831111434239, -STORE, 47831109775360, 47831111118847, -ERASE, 47831109775360, 47831109775360, -STORE, 47831109775360, 47831111118847, -STORE, 47831111430144, 47831111434239, -STORE, 47831111118848, 47831111430143, -ERASE, 47831111118848, 47831111118848, -STORE, 47831111118848, 47831111430143, -STORE, 47831111458816, 47831111475199, -STORE, 47831111434240, 47831111458815, -ERASE, 47831111434240, 47831111434240, -STORE, 47831111434240, 47831111458815, -ERASE, 47831111458816, 47831111458816, -STORE, 47831111458816, 47831111475199, -STORE, 47831111458816, 47831111487487, -ERASE, 47831111434240, 47831111434240, -STORE, 47831111434240, 47831111450623, -STORE, 47831111450624, 47831111458815, -ERASE, 47831109619712, 47831109619712, -STORE, 47831109619712, 47831109623807, -STORE, 47831109623808, 47831109627903, -ERASE, 94629430050816, 94629430050816, -STORE, 94629430050816, 94629430067199, -STORE, 94629430067200, 94629430071295, -ERASE, 139801685643264, 139801685643264, -STORE, 139801685643264, 139801685647359, -STORE, 139801685647360, 139801685651455, -ERASE, 47831109513216, 47831109513216, -STORE, 140737488347136, 140737488351231, -STORE, 140729419612160, 140737488351231, -ERASE, 140729419612160, 140729419612160, -STORE, 140729419612160, 140729419616255, -STORE, 94443354148864, 94443354861567, -ERASE, 94443354148864, 94443354148864, -STORE, 94443354148864, 94443354198015, -STORE, 94443354198016, 94443354861567, -ERASE, 94443354198016, 94443354198016, -STORE, 94443354198016, 94443354742783, -STORE, 94443354742784, 94443354841087, -STORE, 94443354841088, 94443354861567, -STORE, 139741700038656, 139741700210687, -ERASE, 139741700038656, 139741700038656, -STORE, 139741700038656, 139741700042751, -STORE, 139741700042752, 139741700210687, -ERASE, 139741700042752, 139741700042752, -STORE, 139741700042752, 139741700165631, -STORE, 139741700165632, 139741700198399, -STORE, 139741700198400, 139741700206591, -STORE, 139741700206592, 139741700210687, -STORE, 140729420574720, 140729420578815, -STORE, 140729420562432, 140729420574719, -STORE, 47891094958080, 47891094966271, -STORE, 47891094966272, 47891094974463, -STORE, 47891094974464, 47891095080959, -STORE, 47891094990848, 47891095080959, -STORE, 47891094974464, 47891094990847, -ERASE, 47891094990848, 47891094990848, -STORE, 47891094990848, 47891095064575, -STORE, 47891095064576, 47891095080959, -STORE, 47891095044096, 47891095064575, -STORE, 47891094990848, 47891095044095, -ERASE, 47891094990848, 47891094990848, -STORE, 47891094990848, 47891095044095, -STORE, 47891095060480, 47891095064575, -STORE, 47891095044096, 47891095060479, -ERASE, 47891095044096, 47891095044096, -STORE, 47891095044096, 47891095060479, -STORE, 47891095072768, 47891095080959, -STORE, 47891095064576, 47891095072767, -ERASE, 47891095064576, 47891095064576, -STORE, 47891095064576, 47891095072767, -ERASE, 47891095072768, 47891095072768, -STORE, 47891095072768, 47891095080959, -STORE, 47891095080960, 47891096920063, -STORE, 47891095220224, 47891096920063, -STORE, 47891095080960, 47891095220223, -ERASE, 47891095220224, 47891095220224, -STORE, 47891095220224, 47891096879103, -STORE, 47891096879104, 47891096920063, -STORE, 47891096563712, 47891096879103, -STORE, 47891095220224, 47891096563711, -ERASE, 47891095220224, 47891095220224, -STORE, 47891095220224, 47891096563711, -STORE, 47891096875008, 47891096879103, -STORE, 47891096563712, 47891096875007, -ERASE, 47891096563712, 47891096563712, -STORE, 47891096563712, 47891096875007, -STORE, 47891096903680, 47891096920063, -STORE, 47891096879104, 47891096903679, -ERASE, 47891096879104, 47891096879104, -STORE, 47891096879104, 47891096903679, -ERASE, 47891096903680, 47891096903680, -STORE, 47891096903680, 47891096920063, -STORE, 47891096903680, 47891096932351, -ERASE, 47891096879104, 47891096879104, -STORE, 47891096879104, 47891096895487, -STORE, 47891096895488, 47891096903679, -ERASE, 47891095064576, 47891095064576, -STORE, 47891095064576, 47891095068671, -STORE, 47891095068672, 47891095072767, -ERASE, 94443354841088, 94443354841088, -STORE, 94443354841088, 94443354857471, -STORE, 94443354857472, 94443354861567, -ERASE, 139741700198400, 139741700198400, -STORE, 139741700198400, 139741700202495, -STORE, 139741700202496, 139741700206591, -ERASE, 47891094958080, 47891094958080, -STORE, 94443360825344, 94443360960511, -STORE, 140737488347136, 140737488351231, -STORE, 140722961661952, 140737488351231, -ERASE, 140722961661952, 140722961661952, -STORE, 140722961661952, 140722961666047, -STORE, 94878388944896, 94878389657599, -ERASE, 94878388944896, 94878388944896, -STORE, 94878388944896, 94878388994047, -STORE, 94878388994048, 94878389657599, -ERASE, 94878388994048, 94878388994048, -STORE, 94878388994048, 94878389538815, -STORE, 94878389538816, 94878389637119, -STORE, 94878389637120, 94878389657599, -STORE, 140210690056192, 140210690228223, -ERASE, 140210690056192, 140210690056192, -STORE, 140210690056192, 140210690060287, -STORE, 140210690060288, 140210690228223, -ERASE, 140210690060288, 140210690060288, -STORE, 140210690060288, 140210690183167, -STORE, 140210690183168, 140210690215935, -STORE, 140210690215936, 140210690224127, -STORE, 140210690224128, 140210690228223, -STORE, 140722963148800, 140722963152895, -STORE, 140722963136512, 140722963148799, -STORE, 47422104940544, 47422104948735, -STORE, 47422104948736, 47422104956927, -STORE, 47422104956928, 47422105063423, -STORE, 47422104973312, 47422105063423, -STORE, 47422104956928, 47422104973311, -ERASE, 47422104973312, 47422104973312, -STORE, 47422104973312, 47422105047039, -STORE, 47422105047040, 47422105063423, -STORE, 47422105026560, 47422105047039, -STORE, 47422104973312, 47422105026559, -ERASE, 47422104973312, 47422104973312, -STORE, 47422104973312, 47422105026559, -STORE, 47422105042944, 47422105047039, -STORE, 47422105026560, 47422105042943, -ERASE, 47422105026560, 47422105026560, -STORE, 47422105026560, 47422105042943, -STORE, 47422105055232, 47422105063423, -STORE, 47422105047040, 47422105055231, -ERASE, 47422105047040, 47422105047040, -STORE, 47422105047040, 47422105055231, -ERASE, 47422105055232, 47422105055232, -STORE, 47422105055232, 47422105063423, -STORE, 47422105063424, 47422106902527, -STORE, 47422105202688, 47422106902527, -STORE, 47422105063424, 47422105202687, -ERASE, 47422105202688, 47422105202688, -STORE, 47422105202688, 47422106861567, -STORE, 47422106861568, 47422106902527, -STORE, 47422106546176, 47422106861567, -STORE, 47422105202688, 47422106546175, -ERASE, 47422105202688, 47422105202688, -STORE, 47422105202688, 47422106546175, -STORE, 47422106857472, 47422106861567, -STORE, 47422106546176, 47422106857471, -ERASE, 47422106546176, 47422106546176, -STORE, 47422106546176, 47422106857471, -STORE, 47422106886144, 47422106902527, -STORE, 47422106861568, 47422106886143, -ERASE, 47422106861568, 47422106861568, -STORE, 47422106861568, 47422106886143, -ERASE, 47422106886144, 47422106886144, -STORE, 47422106886144, 47422106902527, -STORE, 47422106886144, 47422106914815, -ERASE, 47422106861568, 47422106861568, -STORE, 47422106861568, 47422106877951, -STORE, 47422106877952, 47422106886143, -ERASE, 47422105047040, 47422105047040, -STORE, 47422105047040, 47422105051135, -STORE, 47422105051136, 47422105055231, -ERASE, 94878389637120, 94878389637120, -STORE, 94878389637120, 94878389653503, -STORE, 94878389653504, 94878389657599, -ERASE, 140210690215936, 140210690215936, -STORE, 140210690215936, 140210690220031, -STORE, 140210690220032, 140210690224127, -ERASE, 47422104940544, 47422104940544, -STORE, 140737488347136, 140737488351231, -STORE, 140727690309632, 140737488351231, -ERASE, 140727690309632, 140727690309632, -STORE, 140727690309632, 140727690313727, -STORE, 94121892208640, 94121892921343, -ERASE, 94121892208640, 94121892208640, -STORE, 94121892208640, 94121892257791, -STORE, 94121892257792, 94121892921343, -ERASE, 94121892257792, 94121892257792, -STORE, 94121892257792, 94121892802559, -STORE, 94121892802560, 94121892900863, -STORE, 94121892900864, 94121892921343, -STORE, 140662438326272, 140662438498303, -ERASE, 140662438326272, 140662438326272, -STORE, 140662438326272, 140662438330367, -STORE, 140662438330368, 140662438498303, -ERASE, 140662438330368, 140662438330368, -STORE, 140662438330368, 140662438453247, -STORE, 140662438453248, 140662438486015, -STORE, 140662438486016, 140662438494207, -STORE, 140662438494208, 140662438498303, -STORE, 140727690379264, 140727690383359, -STORE, 140727690366976, 140727690379263, -STORE, 46970356670464, 46970356678655, -STORE, 46970356678656, 46970356686847, -STORE, 46970356686848, 46970356793343, -STORE, 46970356703232, 46970356793343, -STORE, 46970356686848, 46970356703231, -ERASE, 46970356703232, 46970356703232, -STORE, 46970356703232, 46970356776959, -STORE, 46970356776960, 46970356793343, -STORE, 46970356756480, 46970356776959, -STORE, 46970356703232, 46970356756479, -ERASE, 46970356703232, 46970356703232, -STORE, 46970356703232, 46970356756479, -STORE, 46970356772864, 46970356776959, -STORE, 46970356756480, 46970356772863, -ERASE, 46970356756480, 46970356756480, -STORE, 46970356756480, 46970356772863, -STORE, 46970356785152, 46970356793343, -STORE, 46970356776960, 46970356785151, -ERASE, 46970356776960, 46970356776960, -STORE, 46970356776960, 46970356785151, -ERASE, 46970356785152, 46970356785152, -STORE, 46970356785152, 46970356793343, -STORE, 46970356793344, 46970358632447, -STORE, 46970356932608, 46970358632447, -STORE, 46970356793344, 46970356932607, -ERASE, 46970356932608, 46970356932608, -STORE, 46970356932608, 46970358591487, -STORE, 46970358591488, 46970358632447, -STORE, 46970358276096, 46970358591487, -STORE, 46970356932608, 46970358276095, -ERASE, 46970356932608, 46970356932608, -STORE, 46970356932608, 46970358276095, -STORE, 46970358587392, 46970358591487, -STORE, 46970358276096, 46970358587391, -ERASE, 46970358276096, 46970358276096, -STORE, 46970358276096, 46970358587391, -STORE, 46970358616064, 46970358632447, -STORE, 46970358591488, 46970358616063, -ERASE, 46970358591488, 46970358591488, -STORE, 46970358591488, 46970358616063, -ERASE, 46970358616064, 46970358616064, -STORE, 46970358616064, 46970358632447, -STORE, 46970358616064, 46970358644735, -ERASE, 46970358591488, 46970358591488, -STORE, 46970358591488, 46970358607871, -STORE, 46970358607872, 46970358616063, -ERASE, 46970356776960, 46970356776960, -STORE, 46970356776960, 46970356781055, -STORE, 46970356781056, 46970356785151, -ERASE, 94121892900864, 94121892900864, -STORE, 94121892900864, 94121892917247, -STORE, 94121892917248, 94121892921343, -ERASE, 140662438486016, 140662438486016, -STORE, 140662438486016, 140662438490111, -STORE, 140662438490112, 140662438494207, -ERASE, 46970356670464, 46970356670464, -STORE, 94121898610688, 94121898745855, -STORE, 140737488347136, 140737488351231, -STORE, 140737189351424, 140737488351231, -ERASE, 140737189351424, 140737189351424, -STORE, 140737189351424, 140737189355519, -STORE, 93847948832768, 93847949545471, -ERASE, 93847948832768, 93847948832768, -STORE, 93847948832768, 93847948881919, -STORE, 93847948881920, 93847949545471, -ERASE, 93847948881920, 93847948881920, -STORE, 93847948881920, 93847949426687, -STORE, 93847949426688, 93847949524991, -STORE, 93847949524992, 93847949545471, -STORE, 139698989985792, 139698990157823, -ERASE, 139698989985792, 139698989985792, -STORE, 139698989985792, 139698989989887, -STORE, 139698989989888, 139698990157823, -ERASE, 139698989989888, 139698989989888, -STORE, 139698989989888, 139698990112767, -STORE, 139698990112768, 139698990145535, -STORE, 139698990145536, 139698990153727, -STORE, 139698990153728, 139698990157823, -STORE, 140737189744640, 140737189748735, -STORE, 140737189732352, 140737189744639, -STORE, 47933805010944, 47933805019135, -STORE, 47933805019136, 47933805027327, -STORE, 47933805027328, 47933805133823, -STORE, 47933805043712, 47933805133823, -STORE, 47933805027328, 47933805043711, -ERASE, 47933805043712, 47933805043712, -STORE, 47933805043712, 47933805117439, -STORE, 47933805117440, 47933805133823, -STORE, 47933805096960, 47933805117439, -STORE, 47933805043712, 47933805096959, -ERASE, 47933805043712, 47933805043712, -STORE, 47933805043712, 47933805096959, -STORE, 47933805113344, 47933805117439, -STORE, 47933805096960, 47933805113343, -ERASE, 47933805096960, 47933805096960, -STORE, 47933805096960, 47933805113343, -STORE, 47933805125632, 47933805133823, -STORE, 47933805117440, 47933805125631, -ERASE, 47933805117440, 47933805117440, -STORE, 47933805117440, 47933805125631, -ERASE, 47933805125632, 47933805125632, -STORE, 47933805125632, 47933805133823, -STORE, 47933805133824, 47933806972927, -STORE, 47933805273088, 47933806972927, -STORE, 47933805133824, 47933805273087, -ERASE, 47933805273088, 47933805273088, -STORE, 47933805273088, 47933806931967, -STORE, 47933806931968, 47933806972927, -STORE, 47933806616576, 47933806931967, -STORE, 47933805273088, 47933806616575, -ERASE, 47933805273088, 47933805273088, -STORE, 47933805273088, 47933806616575, -STORE, 47933806927872, 47933806931967, -STORE, 47933806616576, 47933806927871, -ERASE, 47933806616576, 47933806616576, -STORE, 47933806616576, 47933806927871, -STORE, 47933806956544, 47933806972927, -STORE, 47933806931968, 47933806956543, -ERASE, 47933806931968, 47933806931968, -STORE, 47933806931968, 47933806956543, -ERASE, 47933806956544, 47933806956544, -STORE, 47933806956544, 47933806972927, -STORE, 47933806956544, 47933806985215, -ERASE, 47933806931968, 47933806931968, -STORE, 47933806931968, 47933806948351, -STORE, 47933806948352, 47933806956543, -ERASE, 47933805117440, 47933805117440, -STORE, 47933805117440, 47933805121535, -STORE, 47933805121536, 47933805125631, -ERASE, 93847949524992, 93847949524992, -STORE, 93847949524992, 93847949541375, -STORE, 93847949541376, 93847949545471, -ERASE, 139698990145536, 139698990145536, -STORE, 139698990145536, 139698990149631, -STORE, 139698990149632, 139698990153727, -ERASE, 47933805010944, 47933805010944, -STORE, 140737488347136, 140737488351231, -STORE, 140725553991680, 140737488351231, -ERASE, 140725553991680, 140725553991680, -STORE, 140725553991680, 140725553995775, -STORE, 93980056248320, 93980056961023, -ERASE, 93980056248320, 93980056248320, -STORE, 93980056248320, 93980056297471, -STORE, 93980056297472, 93980056961023, -ERASE, 93980056297472, 93980056297472, -STORE, 93980056297472, 93980056842239, -STORE, 93980056842240, 93980056940543, -STORE, 93980056940544, 93980056961023, -STORE, 140146588971008, 140146589143039, -ERASE, 140146588971008, 140146588971008, -STORE, 140146588971008, 140146588975103, -STORE, 140146588975104, 140146589143039, -ERASE, 140146588975104, 140146588975104, -STORE, 140146588975104, 140146589097983, -STORE, 140146589097984, 140146589130751, -STORE, 140146589130752, 140146589138943, -STORE, 140146589138944, 140146589143039, -STORE, 140725554860032, 140725554864127, -STORE, 140725554847744, 140725554860031, -STORE, 47486206025728, 47486206033919, -STORE, 47486206033920, 47486206042111, -STORE, 47486206042112, 47486206148607, -STORE, 47486206058496, 47486206148607, -STORE, 47486206042112, 47486206058495, -ERASE, 47486206058496, 47486206058496, -STORE, 47486206058496, 47486206132223, -STORE, 47486206132224, 47486206148607, -STORE, 47486206111744, 47486206132223, -STORE, 47486206058496, 47486206111743, -ERASE, 47486206058496, 47486206058496, -STORE, 47486206058496, 47486206111743, -STORE, 47486206128128, 47486206132223, -STORE, 47486206111744, 47486206128127, -ERASE, 47486206111744, 47486206111744, -STORE, 47486206111744, 47486206128127, -STORE, 47486206140416, 47486206148607, -STORE, 47486206132224, 47486206140415, -ERASE, 47486206132224, 47486206132224, -STORE, 47486206132224, 47486206140415, -ERASE, 47486206140416, 47486206140416, -STORE, 47486206140416, 47486206148607, -STORE, 47486206148608, 47486207987711, -STORE, 47486206287872, 47486207987711, -STORE, 47486206148608, 47486206287871, -ERASE, 47486206287872, 47486206287872, -STORE, 47486206287872, 47486207946751, -STORE, 47486207946752, 47486207987711, -STORE, 47486207631360, 47486207946751, -STORE, 47486206287872, 47486207631359, -ERASE, 47486206287872, 47486206287872, -STORE, 47486206287872, 47486207631359, -STORE, 47486207942656, 47486207946751, -STORE, 47486207631360, 47486207942655, -ERASE, 47486207631360, 47486207631360, -STORE, 47486207631360, 47486207942655, -STORE, 47486207971328, 47486207987711, -STORE, 47486207946752, 47486207971327, -ERASE, 47486207946752, 47486207946752, -STORE, 47486207946752, 47486207971327, -ERASE, 47486207971328, 47486207971328, -STORE, 47486207971328, 47486207987711, -STORE, 47486207971328, 47486207999999, -ERASE, 47486207946752, 47486207946752, -STORE, 47486207946752, 47486207963135, -STORE, 47486207963136, 47486207971327, -ERASE, 47486206132224, 47486206132224, -STORE, 47486206132224, 47486206136319, -STORE, 47486206136320, 47486206140415, -ERASE, 93980056940544, 93980056940544, -STORE, 93980056940544, 93980056956927, -STORE, 93980056956928, 93980056961023, -ERASE, 140146589130752, 140146589130752, -STORE, 140146589130752, 140146589134847, -STORE, 140146589134848, 140146589138943, -ERASE, 47486206025728, 47486206025728, -STORE, 93980070006784, 93980070141951, -STORE, 140737488347136, 140737488351231, -STORE, 140727334776832, 140737488351231, -ERASE, 140727334776832, 140727334776832, -STORE, 140727334776832, 140727334780927, -STORE, 94049747247104, 94049747959807, -ERASE, 94049747247104, 94049747247104, -STORE, 94049747247104, 94049747296255, -STORE, 94049747296256, 94049747959807, -ERASE, 94049747296256, 94049747296256, -STORE, 94049747296256, 94049747841023, -STORE, 94049747841024, 94049747939327, -STORE, 94049747939328, 94049747959807, -STORE, 140227307216896, 140227307388927, -ERASE, 140227307216896, 140227307216896, -STORE, 140227307216896, 140227307220991, -STORE, 140227307220992, 140227307388927, -ERASE, 140227307220992, 140227307220992, -STORE, 140227307220992, 140227307343871, -STORE, 140227307343872, 140227307376639, -STORE, 140227307376640, 140227307384831, -STORE, 140227307384832, 140227307388927, -STORE, 140727335337984, 140727335342079, -STORE, 140727335325696, 140727335337983, -STORE, 47405487779840, 47405487788031, -STORE, 47405487788032, 47405487796223, -STORE, 47405487796224, 47405487902719, -STORE, 47405487812608, 47405487902719, -STORE, 47405487796224, 47405487812607, -ERASE, 47405487812608, 47405487812608, -STORE, 47405487812608, 47405487886335, -STORE, 47405487886336, 47405487902719, -STORE, 47405487865856, 47405487886335, -STORE, 47405487812608, 47405487865855, -ERASE, 47405487812608, 47405487812608, -STORE, 47405487812608, 47405487865855, -STORE, 47405487882240, 47405487886335, -STORE, 47405487865856, 47405487882239, -ERASE, 47405487865856, 47405487865856, -STORE, 47405487865856, 47405487882239, -STORE, 47405487894528, 47405487902719, -STORE, 47405487886336, 47405487894527, -ERASE, 47405487886336, 47405487886336, -STORE, 47405487886336, 47405487894527, -ERASE, 47405487894528, 47405487894528, -STORE, 47405487894528, 47405487902719, -STORE, 47405487902720, 47405489741823, -STORE, 47405488041984, 47405489741823, -STORE, 47405487902720, 47405488041983, -ERASE, 47405488041984, 47405488041984, -STORE, 47405488041984, 47405489700863, -STORE, 47405489700864, 47405489741823, -STORE, 47405489385472, 47405489700863, -STORE, 47405488041984, 47405489385471, -ERASE, 47405488041984, 47405488041984, -STORE, 47405488041984, 47405489385471, -STORE, 47405489696768, 47405489700863, -STORE, 47405489385472, 47405489696767, -ERASE, 47405489385472, 47405489385472, -STORE, 47405489385472, 47405489696767, -STORE, 47405489725440, 47405489741823, -STORE, 47405489700864, 47405489725439, -ERASE, 47405489700864, 47405489700864, -STORE, 47405489700864, 47405489725439, -ERASE, 47405489725440, 47405489725440, -STORE, 47405489725440, 47405489741823, -STORE, 47405489725440, 47405489754111, -ERASE, 47405489700864, 47405489700864, -STORE, 47405489700864, 47405489717247, -STORE, 47405489717248, 47405489725439, -ERASE, 47405487886336, 47405487886336, -STORE, 47405487886336, 47405487890431, -STORE, 47405487890432, 47405487894527, -ERASE, 94049747939328, 94049747939328, -STORE, 94049747939328, 94049747955711, -STORE, 94049747955712, 94049747959807, -ERASE, 140227307376640, 140227307376640, -STORE, 140227307376640, 140227307380735, -STORE, 140227307380736, 140227307384831, -ERASE, 47405487779840, 47405487779840, -STORE, 94049758810112, 94049758945279, -STORE, 140737488347136, 140737488351231, -STORE, 140727079718912, 140737488351231, -ERASE, 140727079718912, 140727079718912, -STORE, 140727079718912, 140727079723007, -STORE, 94250996527104, 94250997239807, -ERASE, 94250996527104, 94250996527104, -STORE, 94250996527104, 94250996576255, -STORE, 94250996576256, 94250997239807, -ERASE, 94250996576256, 94250996576256, -STORE, 94250996576256, 94250997121023, -STORE, 94250997121024, 94250997219327, -STORE, 94250997219328, 94250997239807, -STORE, 140060022587392, 140060022759423, -ERASE, 140060022587392, 140060022587392, -STORE, 140060022587392, 140060022591487, -STORE, 140060022591488, 140060022759423, -ERASE, 140060022591488, 140060022591488, -STORE, 140060022591488, 140060022714367, -STORE, 140060022714368, 140060022747135, -STORE, 140060022747136, 140060022755327, -STORE, 140060022755328, 140060022759423, -STORE, 140727079788544, 140727079792639, -STORE, 140727079776256, 140727079788543, -/* this next one caused issues when lowering the efficiency */ -STORE, 47572772409344, 47572772417535, -STORE, 47572772417536, 47572772425727, -STORE, 47572772425728, 47572772532223, -STORE, 47572772442112, 47572772532223, -STORE, 47572772425728, 47572772442111, -ERASE, 47572772442112, 47572772442112, -STORE, 47572772442112, 47572772515839, -STORE, 47572772515840, 47572772532223, -STORE, 47572772495360, 47572772515839, -STORE, 47572772442112, 47572772495359, -ERASE, 47572772442112, 47572772442112, -STORE, 47572772442112, 47572772495359, -STORE, 47572772511744, 47572772515839, -STORE, 47572772495360, 47572772511743, -ERASE, 47572772495360, 47572772495360, -STORE, 47572772495360, 47572772511743, -STORE, 47572772524032, 47572772532223, -STORE, 47572772515840, 47572772524031, -ERASE, 47572772515840, 47572772515840, -STORE, 47572772515840, 47572772524031, -ERASE, 47572772524032, 47572772524032, -STORE, 47572772524032, 47572772532223, -STORE, 47572772532224, 47572774371327, -STORE, 47572772671488, 47572774371327, -STORE, 47572772532224, 47572772671487, -ERASE, 47572772671488, 47572772671488, -STORE, 47572772671488, 47572774330367, -STORE, 47572774330368, 47572774371327, -STORE, 47572774014976, 47572774330367, -STORE, 47572772671488, 47572774014975, -ERASE, 47572772671488, 47572772671488, -STORE, 47572772671488, 47572774014975, -STORE, 47572774326272, 47572774330367, -STORE, 47572774014976, 47572774326271, -ERASE, 47572774014976, 47572774014976, -STORE, 47572774014976, 47572774326271, -STORE, 47572774354944, 47572774371327, -STORE, 47572774330368, 47572774354943, -ERASE, 47572774330368, 47572774330368, -STORE, 47572774330368, 47572774354943, -ERASE, 47572774354944, 47572774354944, -STORE, 47572774354944, 47572774371327, -STORE, 47572774354944, 47572774383615, -ERASE, 47572774330368, 47572774330368, -STORE, 47572774330368, 47572774346751, -STORE, 47572774346752, 47572774354943, -ERASE, 47572772515840, 47572772515840, -STORE, 47572772515840, 47572772519935, -STORE, 47572772519936, 47572772524031, -ERASE, 94250997219328, 94250997219328, -STORE, 94250997219328, 94250997235711, -STORE, 94250997235712, 94250997239807, -ERASE, 140060022747136, 140060022747136, -STORE, 140060022747136, 140060022751231, -STORE, 140060022751232, 140060022755327, -ERASE, 47572772409344, 47572772409344, -STORE, 94251018305536, 94251018440703, -STORE, 140737488347136, 140737488351231, -STORE, 140730012389376, 140737488351231, -ERASE, 140730012389376, 140730012389376, -STORE, 140730012389376, 140730012393471, -STORE, 94382607675392, 94382607695871, -ERASE, 94382607675392, 94382607675392, -STORE, 94382607675392, 94382607679487, -STORE, 94382607679488, 94382607695871, -ERASE, 94382607679488, 94382607679488, -STORE, 94382607679488, 94382607683583, -STORE, 94382607683584, 94382607687679, -STORE, 94382607687680, 94382607695871, -STORE, 140252451454976, 140252451627007, -ERASE, 140252451454976, 140252451454976, -STORE, 140252451454976, 140252451459071, -STORE, 140252451459072, 140252451627007, -ERASE, 140252451459072, 140252451459072, -STORE, 140252451459072, 140252451581951, -STORE, 140252451581952, 140252451614719, -STORE, 140252451614720, 140252451622911, -STORE, 140252451622912, 140252451627007, -STORE, 140730013548544, 140730013552639, -STORE, 140730013536256, 140730013548543, -STORE, 47380343541760, 47380343549951, -STORE, 47380343549952, 47380343558143, -STORE, 47380343558144, 47380345397247, -STORE, 47380343697408, 47380345397247, -STORE, 47380343558144, 47380343697407, -ERASE, 47380343697408, 47380343697408, -STORE, 47380343697408, 47380345356287, -STORE, 47380345356288, 47380345397247, -STORE, 47380345040896, 47380345356287, -STORE, 47380343697408, 47380345040895, -ERASE, 47380343697408, 47380343697408, -STORE, 47380343697408, 47380345040895, -STORE, 47380345352192, 47380345356287, -STORE, 47380345040896, 47380345352191, -ERASE, 47380345040896, 47380345040896, -STORE, 47380345040896, 47380345352191, -STORE, 47380345380864, 47380345397247, -STORE, 47380345356288, 47380345380863, -ERASE, 47380345356288, 47380345356288, -STORE, 47380345356288, 47380345380863, -ERASE, 47380345380864, 47380345380864, -STORE, 47380345380864, 47380345397247, -ERASE, 47380345356288, 47380345356288, -STORE, 47380345356288, 47380345372671, -STORE, 47380345372672, 47380345380863, -ERASE, 94382607687680, 94382607687680, -STORE, 94382607687680, 94382607691775, -STORE, 94382607691776, 94382607695871, -ERASE, 140252451614720, 140252451614720, -STORE, 140252451614720, 140252451618815, -STORE, 140252451618816, 140252451622911, -ERASE, 47380343541760, 47380343541760, -STORE, 94382626803712, 94382626938879, -STORE, 140737488347136, 140737488351231, -STORE, 140730900271104, 140737488351231, -ERASE, 140730900271104, 140730900271104, -STORE, 140730900271104, 140730900275199, -STORE, 93855478120448, 93855478337535, -ERASE, 93855478120448, 93855478120448, -STORE, 93855478120448, 93855478198271, -STORE, 93855478198272, 93855478337535, -ERASE, 93855478198272, 93855478198272, -STORE, 93855478198272, 93855478243327, -STORE, 93855478243328, 93855478288383, -STORE, 93855478288384, 93855478337535, -STORE, 140092686573568, 140092686745599, -ERASE, 140092686573568, 140092686573568, -STORE, 140092686573568, 140092686577663, -STORE, 140092686577664, 140092686745599, -ERASE, 140092686577664, 140092686577664, -STORE, 140092686577664, 140092686700543, -STORE, 140092686700544, 140092686733311, -STORE, 140092686733312, 140092686741503, -STORE, 140092686741504, 140092686745599, -STORE, 140730900537344, 140730900541439, -STORE, 140730900525056, 140730900537343, -STORE, 47540108423168, 47540108431359, -STORE, 47540108431360, 47540108439551, -STORE, 47540108439552, 47540110278655, -STORE, 47540108578816, 47540110278655, -STORE, 47540108439552, 47540108578815, -ERASE, 47540108578816, 47540108578816, -STORE, 47540108578816, 47540110237695, -STORE, 47540110237696, 47540110278655, -STORE, 47540109922304, 47540110237695, -STORE, 47540108578816, 47540109922303, -ERASE, 47540108578816, 47540108578816, -STORE, 47540108578816, 47540109922303, -STORE, 47540110233600, 47540110237695, -STORE, 47540109922304, 47540110233599, -ERASE, 47540109922304, 47540109922304, -STORE, 47540109922304, 47540110233599, -STORE, 47540110262272, 47540110278655, -STORE, 47540110237696, 47540110262271, -ERASE, 47540110237696, 47540110237696, -STORE, 47540110237696, 47540110262271, -ERASE, 47540110262272, 47540110262272, -STORE, 47540110262272, 47540110278655, -ERASE, 47540110237696, 47540110237696, -STORE, 47540110237696, 47540110254079, -STORE, 47540110254080, 47540110262271, -ERASE, 93855478288384, 93855478288384, -STORE, 93855478288384, 93855478333439, -STORE, 93855478333440, 93855478337535, -ERASE, 140092686733312, 140092686733312, -STORE, 140092686733312, 140092686737407, -STORE, 140092686737408, 140092686741503, -ERASE, 47540108423168, 47540108423168, -STORE, 93855492222976, 93855492358143, -STORE, 93855492222976, 93855492493311, -STORE, 140737488347136, 140737488351231, -STORE, 140733498146816, 140737488351231, -ERASE, 140733498146816, 140733498146816, -STORE, 140733498146816, 140733498150911, -STORE, 94170739654656, 94170740367359, -ERASE, 94170739654656, 94170739654656, -STORE, 94170739654656, 94170739703807, -STORE, 94170739703808, 94170740367359, -ERASE, 94170739703808, 94170739703808, -STORE, 94170739703808, 94170740248575, -STORE, 94170740248576, 94170740346879, -STORE, 94170740346880, 94170740367359, -STORE, 140024788877312, 140024789049343, -ERASE, 140024788877312, 140024788877312, -STORE, 140024788877312, 140024788881407, -STORE, 140024788881408, 140024789049343, -ERASE, 140024788881408, 140024788881408, -STORE, 140024788881408, 140024789004287, -STORE, 140024789004288, 140024789037055, -STORE, 140024789037056, 140024789045247, -STORE, 140024789045248, 140024789049343, -STORE, 140733499023360, 140733499027455, -STORE, 140733499011072, 140733499023359, -STORE, 47608006119424, 47608006127615, -STORE, 47608006127616, 47608006135807, -STORE, 47608006135808, 47608006242303, -STORE, 47608006152192, 47608006242303, -STORE, 47608006135808, 47608006152191, -ERASE, 47608006152192, 47608006152192, -STORE, 47608006152192, 47608006225919, -STORE, 47608006225920, 47608006242303, -STORE, 47608006205440, 47608006225919, -STORE, 47608006152192, 47608006205439, -ERASE, 47608006152192, 47608006152192, -STORE, 47608006152192, 47608006205439, -STORE, 47608006221824, 47608006225919, -STORE, 47608006205440, 47608006221823, -ERASE, 47608006205440, 47608006205440, -STORE, 47608006205440, 47608006221823, -STORE, 47608006234112, 47608006242303, -STORE, 47608006225920, 47608006234111, -ERASE, 47608006225920, 47608006225920, -STORE, 47608006225920, 47608006234111, -ERASE, 47608006234112, 47608006234112, -STORE, 47608006234112, 47608006242303, -STORE, 47608006242304, 47608008081407, -STORE, 47608006381568, 47608008081407, -STORE, 47608006242304, 47608006381567, -ERASE, 47608006381568, 47608006381568, -STORE, 47608006381568, 47608008040447, -STORE, 47608008040448, 47608008081407, -STORE, 47608007725056, 47608008040447, -STORE, 47608006381568, 47608007725055, -ERASE, 47608006381568, 47608006381568, -STORE, 47608006381568, 47608007725055, -STORE, 47608008036352, 47608008040447, -STORE, 47608007725056, 47608008036351, -ERASE, 47608007725056, 47608007725056, -STORE, 47608007725056, 47608008036351, -STORE, 47608008065024, 47608008081407, -STORE, 47608008040448, 47608008065023, -ERASE, 47608008040448, 47608008040448, -STORE, 47608008040448, 47608008065023, -ERASE, 47608008065024, 47608008065024, -STORE, 47608008065024, 47608008081407, -STORE, 47608008065024, 47608008093695, -ERASE, 47608008040448, 47608008040448, -STORE, 47608008040448, 47608008056831, -STORE, 47608008056832, 47608008065023, -ERASE, 47608006225920, 47608006225920, -STORE, 47608006225920, 47608006230015, -STORE, 47608006230016, 47608006234111, -ERASE, 94170740346880, 94170740346880, -STORE, 94170740346880, 94170740363263, -STORE, 94170740363264, 94170740367359, -ERASE, 140024789037056, 140024789037056, -STORE, 140024789037056, 140024789041151, -STORE, 140024789041152, 140024789045247, -ERASE, 47608006119424, 47608006119424, -STORE, 140737488347136, 140737488351231, -STORE, 140730264326144, 140737488351231, -ERASE, 140730264326144, 140730264326144, -STORE, 140730264326144, 140730264330239, -STORE, 94653216407552, 94653217120255, -ERASE, 94653216407552, 94653216407552, -STORE, 94653216407552, 94653216456703, -STORE, 94653216456704, 94653217120255, -ERASE, 94653216456704, 94653216456704, -STORE, 94653216456704, 94653217001471, -STORE, 94653217001472, 94653217099775, -STORE, 94653217099776, 94653217120255, -STORE, 140103617011712, 140103617183743, -ERASE, 140103617011712, 140103617011712, -STORE, 140103617011712, 140103617015807, -STORE, 140103617015808, 140103617183743, -ERASE, 140103617015808, 140103617015808, -STORE, 140103617015808, 140103617138687, -STORE, 140103617138688, 140103617171455, -STORE, 140103617171456, 140103617179647, -STORE, 140103617179648, 140103617183743, -STORE, 140730265427968, 140730265432063, -STORE, 140730265415680, 140730265427967, -STORE, 47529177985024, 47529177993215, -STORE, 47529177993216, 47529178001407, -STORE, 47529178001408, 47529178107903, -STORE, 47529178017792, 47529178107903, -STORE, 47529178001408, 47529178017791, -ERASE, 47529178017792, 47529178017792, -STORE, 47529178017792, 47529178091519, -STORE, 47529178091520, 47529178107903, -STORE, 47529178071040, 47529178091519, -STORE, 47529178017792, 47529178071039, -ERASE, 47529178017792, 47529178017792, -STORE, 47529178017792, 47529178071039, -STORE, 47529178087424, 47529178091519, -STORE, 47529178071040, 47529178087423, -ERASE, 47529178071040, 47529178071040, -STORE, 47529178071040, 47529178087423, -STORE, 47529178099712, 47529178107903, -STORE, 47529178091520, 47529178099711, -ERASE, 47529178091520, 47529178091520, -STORE, 47529178091520, 47529178099711, -ERASE, 47529178099712, 47529178099712, -STORE, 47529178099712, 47529178107903, -STORE, 47529178107904, 47529179947007, -STORE, 47529178247168, 47529179947007, -STORE, 47529178107904, 47529178247167, -ERASE, 47529178247168, 47529178247168, -STORE, 47529178247168, 47529179906047, -STORE, 47529179906048, 47529179947007, -STORE, 47529179590656, 47529179906047, -STORE, 47529178247168, 47529179590655, -ERASE, 47529178247168, 47529178247168, -STORE, 47529178247168, 47529179590655, -STORE, 47529179901952, 47529179906047, -STORE, 47529179590656, 47529179901951, -ERASE, 47529179590656, 47529179590656, -STORE, 47529179590656, 47529179901951, -STORE, 47529179930624, 47529179947007, -STORE, 47529179906048, 47529179930623, -ERASE, 47529179906048, 47529179906048, -STORE, 47529179906048, 47529179930623, -ERASE, 47529179930624, 47529179930624, -STORE, 47529179930624, 47529179947007, -STORE, 47529179930624, 47529179959295, -ERASE, 47529179906048, 47529179906048, -STORE, 47529179906048, 47529179922431, -STORE, 47529179922432, 47529179930623, -ERASE, 47529178091520, 47529178091520, -STORE, 47529178091520, 47529178095615, -STORE, 47529178095616, 47529178099711, -ERASE, 94653217099776, 94653217099776, -STORE, 94653217099776, 94653217116159, -STORE, 94653217116160, 94653217120255, -ERASE, 140103617171456, 140103617171456, -STORE, 140103617171456, 140103617175551, -STORE, 140103617175552, 140103617179647, -ERASE, 47529177985024, 47529177985024, -STORE, 94653241135104, 94653241270271, -STORE, 140737488347136, 140737488351231, -STORE, 140736284549120, 140737488351231, -ERASE, 140736284549120, 140736284549120, -STORE, 140736284549120, 140736284553215, -STORE, 93963663822848, 93963664506879, -ERASE, 93963663822848, 93963663822848, -STORE, 93963663822848, 93963663884287, -STORE, 93963663884288, 93963664506879, -ERASE, 93963663884288, 93963663884288, -STORE, 93963663884288, 93963664240639, -STORE, 93963664240640, 93963664379903, -STORE, 93963664379904, 93963664506879, -STORE, 140450188439552, 140450188611583, -ERASE, 140450188439552, 140450188439552, -STORE, 140450188439552, 140450188443647, -STORE, 140450188443648, 140450188611583, -ERASE, 140450188443648, 140450188443648, -STORE, 140450188443648, 140450188566527, -STORE, 140450188566528, 140450188599295, -STORE, 140450188599296, 140450188607487, -STORE, 140450188607488, 140450188611583, -STORE, 140736284577792, 140736284581887, -STORE, 140736284565504, 140736284577791, -STORE, 47182606557184, 47182606565375, -STORE, 47182606565376, 47182606573567, -STORE, 47182606573568, 47182608412671, -STORE, 47182606712832, 47182608412671, -STORE, 47182606573568, 47182606712831, -ERASE, 47182606712832, 47182606712832, -STORE, 47182606712832, 47182608371711, -STORE, 47182608371712, 47182608412671, -STORE, 47182608056320, 47182608371711, -STORE, 47182606712832, 47182608056319, -ERASE, 47182606712832, 47182606712832, -STORE, 47182606712832, 47182608056319, -STORE, 47182608367616, 47182608371711, -STORE, 47182608056320, 47182608367615, -ERASE, 47182608056320, 47182608056320, -STORE, 47182608056320, 47182608367615, -STORE, 47182608396288, 47182608412671, -STORE, 47182608371712, 47182608396287, -ERASE, 47182608371712, 47182608371712, -STORE, 47182608371712, 47182608396287, -ERASE, 47182608396288, 47182608396288, -STORE, 47182608396288, 47182608412671, -STORE, 47182608412672, 47182608523263, -STORE, 47182608429056, 47182608523263, -STORE, 47182608412672, 47182608429055, -ERASE, 47182608429056, 47182608429056, -STORE, 47182608429056, 47182608515071, -STORE, 47182608515072, 47182608523263, -STORE, 47182608490496, 47182608515071, -STORE, 47182608429056, 47182608490495, -ERASE, 47182608429056, 47182608429056, -STORE, 47182608429056, 47182608490495, -STORE, 47182608510976, 47182608515071, -STORE, 47182608490496, 47182608510975, -ERASE, 47182608490496, 47182608490496, -STORE, 47182608490496, 47182608510975, -ERASE, 47182608515072, 47182608515072, -STORE, 47182608515072, 47182608523263, -STORE, 47182608523264, 47182608568319, -ERASE, 47182608523264, 47182608523264, -STORE, 47182608523264, 47182608531455, -STORE, 47182608531456, 47182608568319, -STORE, 47182608551936, 47182608568319, -STORE, 47182608531456, 47182608551935, -ERASE, 47182608531456, 47182608531456, -STORE, 47182608531456, 47182608551935, -STORE, 47182608560128, 47182608568319, -STORE, 47182608551936, 47182608560127, -ERASE, 47182608551936, 47182608551936, -STORE, 47182608551936, 47182608568319, -ERASE, 47182608551936, 47182608551936, -STORE, 47182608551936, 47182608560127, -STORE, 47182608560128, 47182608568319, -ERASE, 47182608560128, 47182608560128, -STORE, 47182608560128, 47182608568319, -STORE, 47182608568320, 47182608916479, -STORE, 47182608609280, 47182608916479, -STORE, 47182608568320, 47182608609279, -ERASE, 47182608609280, 47182608609280, -STORE, 47182608609280, 47182608891903, -STORE, 47182608891904, 47182608916479, -STORE, 47182608822272, 47182608891903, -STORE, 47182608609280, 47182608822271, -ERASE, 47182608609280, 47182608609280, -STORE, 47182608609280, 47182608822271, -STORE, 47182608887808, 47182608891903, -STORE, 47182608822272, 47182608887807, -ERASE, 47182608822272, 47182608822272, -STORE, 47182608822272, 47182608887807, -ERASE, 47182608891904, 47182608891904, -STORE, 47182608891904, 47182608916479, -STORE, 47182608916480, 47182611177471, -STORE, 47182609068032, 47182611177471, -STORE, 47182608916480, 47182609068031, -ERASE, 47182609068032, 47182609068032, -STORE, 47182609068032, 47182611161087, -STORE, 47182611161088, 47182611177471, -STORE, 47182611169280, 47182611177471, -STORE, 47182611161088, 47182611169279, -ERASE, 47182611161088, 47182611161088, -STORE, 47182611161088, 47182611169279, -ERASE, 47182611169280, 47182611169280, -STORE, 47182611169280, 47182611177471, -STORE, 47182611177472, 47182611312639, -ERASE, 47182611177472, 47182611177472, -STORE, 47182611177472, 47182611202047, -STORE, 47182611202048, 47182611312639, -STORE, 47182611263488, 47182611312639, -STORE, 47182611202048, 47182611263487, -ERASE, 47182611202048, 47182611202048, -STORE, 47182611202048, 47182611263487, -STORE, 47182611288064, 47182611312639, -STORE, 47182611263488, 47182611288063, -ERASE, 47182611263488, 47182611263488, -STORE, 47182611263488, 47182611312639, -ERASE, 47182611263488, 47182611263488, -STORE, 47182611263488, 47182611288063, -STORE, 47182611288064, 47182611312639, -STORE, 47182611296256, 47182611312639, -STORE, 47182611288064, 47182611296255, -ERASE, 47182611288064, 47182611288064, -STORE, 47182611288064, 47182611296255, -ERASE, 47182611296256, 47182611296256, -STORE, 47182611296256, 47182611312639, -STORE, 47182611296256, 47182611320831, -STORE, 47182611320832, 47182611484671, -ERASE, 47182611320832, 47182611320832, -STORE, 47182611320832, 47182611333119, -STORE, 47182611333120, 47182611484671, -STORE, 47182611431424, 47182611484671, -STORE, 47182611333120, 47182611431423, -ERASE, 47182611333120, 47182611333120, -STORE, 47182611333120, 47182611431423, -STORE, 47182611476480, 47182611484671, -STORE, 47182611431424, 47182611476479, -ERASE, 47182611431424, 47182611431424, -STORE, 47182611431424, 47182611484671, -ERASE, 47182611431424, 47182611431424, -STORE, 47182611431424, 47182611476479, -STORE, 47182611476480, 47182611484671, -ERASE, 47182611476480, 47182611476480, -STORE, 47182611476480, 47182611484671, -STORE, 47182611484672, 47182612082687, -STORE, 47182611603456, 47182612082687, -STORE, 47182611484672, 47182611603455, -ERASE, 47182611603456, 47182611603456, -STORE, 47182611603456, 47182612029439, -STORE, 47182612029440, 47182612082687, -STORE, 47182611918848, 47182612029439, -STORE, 47182611603456, 47182611918847, -ERASE, 47182611603456, 47182611603456, -STORE, 47182611603456, 47182611918847, -STORE, 47182612025344, 47182612029439, -STORE, 47182611918848, 47182612025343, -ERASE, 47182611918848, 47182611918848, -STORE, 47182611918848, 47182612025343, -ERASE, 47182612029440, 47182612029440, -STORE, 47182612029440, 47182612082687, -STORE, 47182612082688, 47182615134207, -STORE, 47182612627456, 47182615134207, -STORE, 47182612082688, 47182612627455, -ERASE, 47182612627456, 47182612627456, -STORE, 47182612627456, 47182614913023, -STORE, 47182614913024, 47182615134207, -STORE, 47182614323200, 47182614913023, -STORE, 47182612627456, 47182614323199, -ERASE, 47182612627456, 47182612627456, -STORE, 47182612627456, 47182614323199, -STORE, 47182614908928, 47182614913023, -STORE, 47182614323200, 47182614908927, -ERASE, 47182614323200, 47182614323200, -STORE, 47182614323200, 47182614908927, -STORE, 47182615117824, 47182615134207, -STORE, 47182614913024, 47182615117823, -ERASE, 47182614913024, 47182614913024, -STORE, 47182614913024, 47182615117823, -ERASE, 47182615117824, 47182615117824, -STORE, 47182615117824, 47182615134207, -STORE, 47182615134208, 47182615166975, -ERASE, 47182615134208, 47182615134208, -STORE, 47182615134208, 47182615142399, -STORE, 47182615142400, 47182615166975, -STORE, 47182615154688, 47182615166975, -STORE, 47182615142400, 47182615154687, -ERASE, 47182615142400, 47182615142400, -STORE, 47182615142400, 47182615154687, -STORE, 47182615158784, 47182615166975, -STORE, 47182615154688, 47182615158783, -ERASE, 47182615154688, 47182615154688, -STORE, 47182615154688, 47182615166975, -ERASE, 47182615154688, 47182615154688, -STORE, 47182615154688, 47182615158783, -STORE, 47182615158784, 47182615166975, -ERASE, 47182615158784, 47182615158784, -STORE, 47182615158784, 47182615166975, -STORE, 47182615166976, 47182615203839, -ERASE, 47182615166976, 47182615166976, -STORE, 47182615166976, 47182615175167, -STORE, 47182615175168, 47182615203839, -STORE, 47182615191552, 47182615203839, -STORE, 47182615175168, 47182615191551, -ERASE, 47182615175168, 47182615175168, -STORE, 47182615175168, 47182615191551, -STORE, 47182615195648, 47182615203839, -STORE, 47182615191552, 47182615195647, -ERASE, 47182615191552, 47182615191552, -STORE, 47182615191552, 47182615203839, -ERASE, 47182615191552, 47182615191552, -STORE, 47182615191552, 47182615195647, -STORE, 47182615195648, 47182615203839, -ERASE, 47182615195648, 47182615195648, -STORE, 47182615195648, 47182615203839, -STORE, 47182615203840, 47182615678975, -ERASE, 47182615203840, 47182615203840, -STORE, 47182615203840, 47182615212031, -STORE, 47182615212032, 47182615678975, -STORE, 47182615547904, 47182615678975, -STORE, 47182615212032, 47182615547903, -ERASE, 47182615212032, 47182615212032, -STORE, 47182615212032, 47182615547903, -STORE, 47182615670784, 47182615678975, -STORE, 47182615547904, 47182615670783, -ERASE, 47182615547904, 47182615547904, -STORE, 47182615547904, 47182615678975, -ERASE, 47182615547904, 47182615547904, -STORE, 47182615547904, 47182615670783, -STORE, 47182615670784, 47182615678975, -ERASE, 47182615670784, 47182615670784, -STORE, 47182615670784, 47182615678975, -STORE, 47182615678976, 47182615687167, -STORE, 47182615687168, 47182615707647, -ERASE, 47182615687168, 47182615687168, -STORE, 47182615687168, 47182615691263, -STORE, 47182615691264, 47182615707647, -STORE, 47182615695360, 47182615707647, -STORE, 47182615691264, 47182615695359, -ERASE, 47182615691264, 47182615691264, -STORE, 47182615691264, 47182615695359, -STORE, 47182615699456, 47182615707647, -STORE, 47182615695360, 47182615699455, -ERASE, 47182615695360, 47182615695360, -STORE, 47182615695360, 47182615707647, -ERASE, 47182615695360, 47182615695360, -STORE, 47182615695360, 47182615699455, -STORE, 47182615699456, 47182615707647, -ERASE, 47182615699456, 47182615699456, -STORE, 47182615699456, 47182615707647, -STORE, 47182615707648, 47182615715839, -ERASE, 47182608371712, 47182608371712, -STORE, 47182608371712, 47182608388095, -STORE, 47182608388096, 47182608396287, -ERASE, 47182615699456, 47182615699456, -STORE, 47182615699456, 47182615703551, -STORE, 47182615703552, 47182615707647, -ERASE, 47182611288064, 47182611288064, -STORE, 47182611288064, 47182611292159, -STORE, 47182611292160, 47182611296255, -ERASE, 47182615670784, 47182615670784, -STORE, 47182615670784, 47182615674879, -STORE, 47182615674880, 47182615678975, -ERASE, 47182615195648, 47182615195648, -STORE, 47182615195648, 47182615199743, -STORE, 47182615199744, 47182615203839, -ERASE, 47182615158784, 47182615158784, -STORE, 47182615158784, 47182615162879, -STORE, 47182615162880, 47182615166975, -ERASE, 47182614913024, 47182614913024, -STORE, 47182614913024, 47182615109631, -STORE, 47182615109632, 47182615117823, -ERASE, 47182612029440, 47182612029440, -STORE, 47182612029440, 47182612066303, -STORE, 47182612066304, 47182612082687, -ERASE, 47182611476480, 47182611476480, -STORE, 47182611476480, 47182611480575, -STORE, 47182611480576, 47182611484671, -ERASE, 47182611161088, 47182611161088, -STORE, 47182611161088, 47182611165183, -STORE, 47182611165184, 47182611169279, -ERASE, 47182608891904, 47182608891904, -STORE, 47182608891904, 47182608912383, -STORE, 47182608912384, 47182608916479, -ERASE, 47182608560128, 47182608560128, -STORE, 47182608560128, 47182608564223, -STORE, 47182608564224, 47182608568319, -ERASE, 47182608515072, 47182608515072, -STORE, 47182608515072, 47182608519167, -STORE, 47182608519168, 47182608523263, -ERASE, 93963664379904, 93963664379904, -STORE, 93963664379904, 93963664502783, -STORE, 93963664502784, 93963664506879, -ERASE, 140450188599296, 140450188599296, -STORE, 140450188599296, 140450188603391, -STORE, 140450188603392, 140450188607487, -ERASE, 47182606557184, 47182606557184, -STORE, 93963694723072, 93963694858239, -STORE, 140737488347136, 140737488351231, -STORE, 140730313261056, 140737488351231, -ERASE, 140730313261056, 140730313261056, -STORE, 140730313261056, 140730313265151, -STORE, 94386579017728, 94386579697663, -ERASE, 94386579017728, 94386579017728, -STORE, 94386579017728, 94386579083263, -STORE, 94386579083264, 94386579697663, -ERASE, 94386579083264, 94386579083264, -STORE, 94386579083264, 94386579431423, -STORE, 94386579431424, 94386579570687, -STORE, 94386579570688, 94386579697663, -STORE, 140124810838016, 140124811010047, -ERASE, 140124810838016, 140124810838016, -STORE, 140124810838016, 140124810842111, -STORE, 140124810842112, 140124811010047, -ERASE, 140124810842112, 140124810842112, -STORE, 140124810842112, 140124810964991, -STORE, 140124810964992, 140124810997759, -STORE, 140124810997760, 140124811005951, -STORE, 140124811005952, 140124811010047, -STORE, 140730313601024, 140730313605119, -STORE, 140730313588736, 140730313601023, -STORE, 47507984158720, 47507984166911, -STORE, 47507984166912, 47507984175103, -STORE, 47507984175104, 47507986014207, -STORE, 47507984314368, 47507986014207, -STORE, 47507984175104, 47507984314367, -ERASE, 47507984314368, 47507984314368, -STORE, 47507984314368, 47507985973247, -STORE, 47507985973248, 47507986014207, -STORE, 47507985657856, 47507985973247, -STORE, 47507984314368, 47507985657855, -ERASE, 47507984314368, 47507984314368, -STORE, 47507984314368, 47507985657855, -STORE, 47507985969152, 47507985973247, -STORE, 47507985657856, 47507985969151, -ERASE, 47507985657856, 47507985657856, -STORE, 47507985657856, 47507985969151, -STORE, 47507985997824, 47507986014207, -STORE, 47507985973248, 47507985997823, -ERASE, 47507985973248, 47507985973248, -STORE, 47507985973248, 47507985997823, -ERASE, 47507985997824, 47507985997824, -STORE, 47507985997824, 47507986014207, -STORE, 47507986014208, 47507986124799, -STORE, 47507986030592, 47507986124799, -STORE, 47507986014208, 47507986030591, -ERASE, 47507986030592, 47507986030592, -STORE, 47507986030592, 47507986116607, -STORE, 47507986116608, 47507986124799, -STORE, 47507986092032, 47507986116607, -STORE, 47507986030592, 47507986092031, -ERASE, 47507986030592, 47507986030592, -STORE, 47507986030592, 47507986092031, -STORE, 47507986112512, 47507986116607, -STORE, 47507986092032, 47507986112511, -ERASE, 47507986092032, 47507986092032, -STORE, 47507986092032, 47507986112511, -ERASE, 47507986116608, 47507986116608, -STORE, 47507986116608, 47507986124799, -STORE, 47507986124800, 47507986169855, -ERASE, 47507986124800, 47507986124800, -STORE, 47507986124800, 47507986132991, -STORE, 47507986132992, 47507986169855, -STORE, 47507986153472, 47507986169855, -STORE, 47507986132992, 47507986153471, -ERASE, 47507986132992, 47507986132992, -STORE, 47507986132992, 47507986153471, -STORE, 47507986161664, 47507986169855, -STORE, 47507986153472, 47507986161663, -ERASE, 47507986153472, 47507986153472, -STORE, 47507986153472, 47507986169855, -ERASE, 47507986153472, 47507986153472, -STORE, 47507986153472, 47507986161663, -STORE, 47507986161664, 47507986169855, -ERASE, 47507986161664, 47507986161664, -STORE, 47507986161664, 47507986169855, -STORE, 47507986169856, 47507986518015, -STORE, 47507986210816, 47507986518015, -STORE, 47507986169856, 47507986210815, -ERASE, 47507986210816, 47507986210816, -STORE, 47507986210816, 47507986493439, -STORE, 47507986493440, 47507986518015, -STORE, 47507986423808, 47507986493439, -STORE, 47507986210816, 47507986423807, -ERASE, 47507986210816, 47507986210816, -STORE, 47507986210816, 47507986423807, -STORE, 47507986489344, 47507986493439, -STORE, 47507986423808, 47507986489343, -ERASE, 47507986423808, 47507986423808, -STORE, 47507986423808, 47507986489343, -ERASE, 47507986493440, 47507986493440, -STORE, 47507986493440, 47507986518015, -STORE, 47507986518016, 47507988779007, -STORE, 47507986669568, 47507988779007, -STORE, 47507986518016, 47507986669567, -ERASE, 47507986669568, 47507986669568, -STORE, 47507986669568, 47507988762623, -STORE, 47507988762624, 47507988779007, -STORE, 47507988770816, 47507988779007, -STORE, 47507988762624, 47507988770815, -ERASE, 47507988762624, 47507988762624, -STORE, 47507988762624, 47507988770815, -ERASE, 47507988770816, 47507988770816, -STORE, 47507988770816, 47507988779007, -STORE, 47507988779008, 47507988914175, -ERASE, 47507988779008, 47507988779008, -STORE, 47507988779008, 47507988803583, -STORE, 47507988803584, 47507988914175, -STORE, 47507988865024, 47507988914175, -STORE, 47507988803584, 47507988865023, -ERASE, 47507988803584, 47507988803584, -STORE, 47507988803584, 47507988865023, -STORE, 47507988889600, 47507988914175, -STORE, 47507988865024, 47507988889599, -ERASE, 47507988865024, 47507988865024, -STORE, 47507988865024, 47507988914175, -ERASE, 47507988865024, 47507988865024, -STORE, 47507988865024, 47507988889599, -STORE, 47507988889600, 47507988914175, -STORE, 47507988897792, 47507988914175, -STORE, 47507988889600, 47507988897791, -ERASE, 47507988889600, 47507988889600, -STORE, 47507988889600, 47507988897791, -ERASE, 47507988897792, 47507988897792, -STORE, 47507988897792, 47507988914175, -STORE, 47507988897792, 47507988922367, -STORE, 47507988922368, 47507989086207, -ERASE, 47507988922368, 47507988922368, -STORE, 47507988922368, 47507988934655, -STORE, 47507988934656, 47507989086207, -STORE, 47507989032960, 47507989086207, -STORE, 47507988934656, 47507989032959, -ERASE, 47507988934656, 47507988934656, -STORE, 47507988934656, 47507989032959, -STORE, 47507989078016, 47507989086207, -STORE, 47507989032960, 47507989078015, -ERASE, 47507989032960, 47507989032960, -STORE, 47507989032960, 47507989086207, -ERASE, 47507989032960, 47507989032960, -STORE, 47507989032960, 47507989078015, -STORE, 47507989078016, 47507989086207, -ERASE, 47507989078016, 47507989078016, -STORE, 47507989078016, 47507989086207, -STORE, 47507989086208, 47507989684223, -STORE, 47507989204992, 47507989684223, -STORE, 47507989086208, 47507989204991, -ERASE, 47507989204992, 47507989204992, -STORE, 47507989204992, 47507989630975, -STORE, 47507989630976, 47507989684223, -STORE, 47507989520384, 47507989630975, -STORE, 47507989204992, 47507989520383, -ERASE, 47507989204992, 47507989204992, -STORE, 47507989204992, 47507989520383, -STORE, 47507989626880, 47507989630975, -STORE, 47507989520384, 47507989626879, -ERASE, 47507989520384, 47507989520384, -STORE, 47507989520384, 47507989626879, -ERASE, 47507989630976, 47507989630976, -STORE, 47507989630976, 47507989684223, -STORE, 47507989684224, 47507992735743, -STORE, 47507990228992, 47507992735743, -STORE, 47507989684224, 47507990228991, -ERASE, 47507990228992, 47507990228992, -STORE, 47507990228992, 47507992514559, -STORE, 47507992514560, 47507992735743, -STORE, 47507991924736, 47507992514559, -STORE, 47507990228992, 47507991924735, -ERASE, 47507990228992, 47507990228992, -STORE, 47507990228992, 47507991924735, -STORE, 47507992510464, 47507992514559, -STORE, 47507991924736, 47507992510463, -ERASE, 47507991924736, 47507991924736, -STORE, 47507991924736, 47507992510463, -STORE, 47507992719360, 47507992735743, -STORE, 47507992514560, 47507992719359, -ERASE, 47507992514560, 47507992514560, -STORE, 47507992514560, 47507992719359, -ERASE, 47507992719360, 47507992719360, -STORE, 47507992719360, 47507992735743, -STORE, 47507992735744, 47507992768511, -ERASE, 47507992735744, 47507992735744, -STORE, 47507992735744, 47507992743935, -STORE, 47507992743936, 47507992768511, -STORE, 47507992756224, 47507992768511, -STORE, 47507992743936, 47507992756223, -ERASE, 47507992743936, 47507992743936, -STORE, 47507992743936, 47507992756223, -STORE, 47507992760320, 47507992768511, -STORE, 47507992756224, 47507992760319, -ERASE, 47507992756224, 47507992756224, -STORE, 47507992756224, 47507992768511, -ERASE, 47507992756224, 47507992756224, -STORE, 47507992756224, 47507992760319, -STORE, 47507992760320, 47507992768511, -ERASE, 47507992760320, 47507992760320, -STORE, 47507992760320, 47507992768511, -STORE, 47507992768512, 47507992805375, -ERASE, 47507992768512, 47507992768512, -STORE, 47507992768512, 47507992776703, -STORE, 47507992776704, 47507992805375, -STORE, 47507992793088, 47507992805375, -STORE, 47507992776704, 47507992793087, -ERASE, 47507992776704, 47507992776704, -STORE, 47507992776704, 47507992793087, -STORE, 47507992797184, 47507992805375, -STORE, 47507992793088, 47507992797183, -ERASE, 47507992793088, 47507992793088, -STORE, 47507992793088, 47507992805375, -ERASE, 47507992793088, 47507992793088, -STORE, 47507992793088, 47507992797183, -STORE, 47507992797184, 47507992805375, -ERASE, 47507992797184, 47507992797184, -STORE, 47507992797184, 47507992805375, -STORE, 47507992805376, 47507993280511, -ERASE, 47507992805376, 47507992805376, -STORE, 47507992805376, 47507992813567, -STORE, 47507992813568, 47507993280511, -STORE, 47507993149440, 47507993280511, -STORE, 47507992813568, 47507993149439, -ERASE, 47507992813568, 47507992813568, -STORE, 47507992813568, 47507993149439, -STORE, 47507993272320, 47507993280511, -STORE, 47507993149440, 47507993272319, -ERASE, 47507993149440, 47507993149440, -STORE, 47507993149440, 47507993280511, -ERASE, 47507993149440, 47507993149440, -STORE, 47507993149440, 47507993272319, -STORE, 47507993272320, 47507993280511, -ERASE, 47507993272320, 47507993272320, -STORE, 47507993272320, 47507993280511, -STORE, 47507993280512, 47507993288703, -STORE, 47507993288704, 47507993309183, -ERASE, 47507993288704, 47507993288704, -STORE, 47507993288704, 47507993292799, -STORE, 47507993292800, 47507993309183, -STORE, 47507993296896, 47507993309183, -STORE, 47507993292800, 47507993296895, -ERASE, 47507993292800, 47507993292800, -STORE, 47507993292800, 47507993296895, -STORE, 47507993300992, 47507993309183, -STORE, 47507993296896, 47507993300991, -ERASE, 47507993296896, 47507993296896, -STORE, 47507993296896, 47507993309183, -ERASE, 47507993296896, 47507993296896, -STORE, 47507993296896, 47507993300991, -STORE, 47507993300992, 47507993309183, -ERASE, 47507993300992, 47507993300992, -STORE, 47507993300992, 47507993309183, -STORE, 47507993309184, 47507993317375, -ERASE, 47507985973248, 47507985973248, -STORE, 47507985973248, 47507985989631, -STORE, 47507985989632, 47507985997823, -ERASE, 47507993300992, 47507993300992, -STORE, 47507993300992, 47507993305087, -STORE, 47507993305088, 47507993309183, -ERASE, 47507988889600, 47507988889600, -STORE, 47507988889600, 47507988893695, -STORE, 47507988893696, 47507988897791, -ERASE, 47507993272320, 47507993272320, -STORE, 47507993272320, 47507993276415, -STORE, 47507993276416, 47507993280511, -ERASE, 47507992797184, 47507992797184, -STORE, 47507992797184, 47507992801279, -STORE, 47507992801280, 47507992805375, -ERASE, 47507992760320, 47507992760320, -STORE, 47507992760320, 47507992764415, -STORE, 47507992764416, 47507992768511, -ERASE, 47507992514560, 47507992514560, -STORE, 47507992514560, 47507992711167, -STORE, 47507992711168, 47507992719359, -ERASE, 47507989630976, 47507989630976, -STORE, 47507989630976, 47507989667839, -STORE, 47507989667840, 47507989684223, -ERASE, 47507989078016, 47507989078016, -STORE, 47507989078016, 47507989082111, -STORE, 47507989082112, 47507989086207, -ERASE, 47507988762624, 47507988762624, -STORE, 47507988762624, 47507988766719, -STORE, 47507988766720, 47507988770815, -ERASE, 47507986493440, 47507986493440, -STORE, 47507986493440, 47507986513919, -STORE, 47507986513920, 47507986518015, -ERASE, 47507986161664, 47507986161664, -STORE, 47507986161664, 47507986165759, -STORE, 47507986165760, 47507986169855, -ERASE, 47507986116608, 47507986116608, -STORE, 47507986116608, 47507986120703, -STORE, 47507986120704, 47507986124799, -ERASE, 94386579570688, 94386579570688, -STORE, 94386579570688, 94386579693567, -STORE, 94386579693568, 94386579697663, -ERASE, 140124810997760, 140124810997760, -STORE, 140124810997760, 140124811001855, -STORE, 140124811001856, 140124811005951, -ERASE, 47507984158720, 47507984158720, -STORE, 94386583982080, 94386584117247, -STORE, 94386583982080, 94386584256511, -ERASE, 94386583982080, 94386583982080, -STORE, 94386583982080, 94386584223743, -STORE, 94386584223744, 94386584256511, -ERASE, 94386584223744, 94386584223744, -STORE, 140737488347136, 140737488351231, -STORE, 140733763395584, 140737488351231, -ERASE, 140733763395584, 140733763395584, -STORE, 140733763395584, 140733763399679, -STORE, 94011546472448, 94011547152383, -ERASE, 94011546472448, 94011546472448, -STORE, 94011546472448, 94011546537983, -STORE, 94011546537984, 94011547152383, -ERASE, 94011546537984, 94011546537984, -STORE, 94011546537984, 94011546886143, -STORE, 94011546886144, 94011547025407, -STORE, 94011547025408, 94011547152383, -STORE, 139757597949952, 139757598121983, -ERASE, 139757597949952, 139757597949952, -STORE, 139757597949952, 139757597954047, -STORE, 139757597954048, 139757598121983, -ERASE, 139757597954048, 139757597954048, -STORE, 139757597954048, 139757598076927, -STORE, 139757598076928, 139757598109695, -STORE, 139757598109696, 139757598117887, -STORE, 139757598117888, 139757598121983, -STORE, 140733763596288, 140733763600383, -STORE, 140733763584000, 140733763596287, -STORE, 47875197046784, 47875197054975, -STORE, 47875197054976, 47875197063167, -STORE, 47875197063168, 47875198902271, -STORE, 47875197202432, 47875198902271, -STORE, 47875197063168, 47875197202431, -ERASE, 47875197202432, 47875197202432, -STORE, 47875197202432, 47875198861311, -STORE, 47875198861312, 47875198902271, -STORE, 47875198545920, 47875198861311, -STORE, 47875197202432, 47875198545919, -ERASE, 47875197202432, 47875197202432, -STORE, 47875197202432, 47875198545919, -STORE, 47875198857216, 47875198861311, -STORE, 47875198545920, 47875198857215, -ERASE, 47875198545920, 47875198545920, -STORE, 47875198545920, 47875198857215, -STORE, 47875198885888, 47875198902271, -STORE, 47875198861312, 47875198885887, -ERASE, 47875198861312, 47875198861312, -STORE, 47875198861312, 47875198885887, -ERASE, 47875198885888, 47875198885888, -STORE, 47875198885888, 47875198902271, -STORE, 47875198902272, 47875199012863, -STORE, 47875198918656, 47875199012863, -STORE, 47875198902272, 47875198918655, -ERASE, 47875198918656, 47875198918656, -STORE, 47875198918656, 47875199004671, -STORE, 47875199004672, 47875199012863, -STORE, 47875198980096, 47875199004671, -STORE, 47875198918656, 47875198980095, -ERASE, 47875198918656, 47875198918656, -STORE, 47875198918656, 47875198980095, -STORE, 47875199000576, 47875199004671, -STORE, 47875198980096, 47875199000575, -ERASE, 47875198980096, 47875198980096, -STORE, 47875198980096, 47875199000575, -ERASE, 47875199004672, 47875199004672, -STORE, 47875199004672, 47875199012863, -STORE, 47875199012864, 47875199057919, -ERASE, 47875199012864, 47875199012864, -STORE, 47875199012864, 47875199021055, -STORE, 47875199021056, 47875199057919, -STORE, 47875199041536, 47875199057919, -STORE, 47875199021056, 47875199041535, -ERASE, 47875199021056, 47875199021056, -STORE, 47875199021056, 47875199041535, -STORE, 47875199049728, 47875199057919, -STORE, 47875199041536, 47875199049727, -ERASE, 47875199041536, 47875199041536, -STORE, 47875199041536, 47875199057919, -ERASE, 47875199041536, 47875199041536, -STORE, 47875199041536, 47875199049727, -STORE, 47875199049728, 47875199057919, -ERASE, 47875199049728, 47875199049728, -STORE, 47875199049728, 47875199057919, -STORE, 47875199057920, 47875199406079, -STORE, 47875199098880, 47875199406079, -STORE, 47875199057920, 47875199098879, -ERASE, 47875199098880, 47875199098880, -STORE, 47875199098880, 47875199381503, -STORE, 47875199381504, 47875199406079, -STORE, 47875199311872, 47875199381503, -STORE, 47875199098880, 47875199311871, -ERASE, 47875199098880, 47875199098880, -STORE, 47875199098880, 47875199311871, -STORE, 47875199377408, 47875199381503, -STORE, 47875199311872, 47875199377407, -ERASE, 47875199311872, 47875199311872, -STORE, 47875199311872, 47875199377407, -ERASE, 47875199381504, 47875199381504, -STORE, 47875199381504, 47875199406079, -STORE, 47875199406080, 47875201667071, -STORE, 47875199557632, 47875201667071, -STORE, 47875199406080, 47875199557631, -ERASE, 47875199557632, 47875199557632, -STORE, 47875199557632, 47875201650687, -STORE, 47875201650688, 47875201667071, -STORE, 47875201658880, 47875201667071, -STORE, 47875201650688, 47875201658879, -ERASE, 47875201650688, 47875201650688, -STORE, 47875201650688, 47875201658879, -ERASE, 47875201658880, 47875201658880, -STORE, 47875201658880, 47875201667071, -STORE, 47875201667072, 47875201802239, -ERASE, 47875201667072, 47875201667072, -STORE, 47875201667072, 47875201691647, -STORE, 47875201691648, 47875201802239, -STORE, 47875201753088, 47875201802239, -STORE, 47875201691648, 47875201753087, -ERASE, 47875201691648, 47875201691648, -STORE, 47875201691648, 47875201753087, -STORE, 47875201777664, 47875201802239, -STORE, 47875201753088, 47875201777663, -ERASE, 47875201753088, 47875201753088, -STORE, 47875201753088, 47875201802239, -ERASE, 47875201753088, 47875201753088, -STORE, 47875201753088, 47875201777663, -STORE, 47875201777664, 47875201802239, -STORE, 47875201785856, 47875201802239, -STORE, 47875201777664, 47875201785855, -ERASE, 47875201777664, 47875201777664, -STORE, 47875201777664, 47875201785855, -ERASE, 47875201785856, 47875201785856, -STORE, 47875201785856, 47875201802239, -STORE, 47875201785856, 47875201810431, -STORE, 47875201810432, 47875201974271, -ERASE, 47875201810432, 47875201810432, -STORE, 47875201810432, 47875201822719, -STORE, 47875201822720, 47875201974271, -STORE, 47875201921024, 47875201974271, -STORE, 47875201822720, 47875201921023, -ERASE, 47875201822720, 47875201822720, -STORE, 47875201822720, 47875201921023, -STORE, 47875201966080, 47875201974271, -STORE, 47875201921024, 47875201966079, -ERASE, 47875201921024, 47875201921024, -STORE, 47875201921024, 47875201974271, -ERASE, 47875201921024, 47875201921024, -STORE, 47875201921024, 47875201966079, -STORE, 47875201966080, 47875201974271, -ERASE, 47875201966080, 47875201966080, -STORE, 47875201966080, 47875201974271, -STORE, 47875201974272, 47875202572287, -STORE, 47875202093056, 47875202572287, -STORE, 47875201974272, 47875202093055, -ERASE, 47875202093056, 47875202093056, -STORE, 47875202093056, 47875202519039, -STORE, 47875202519040, 47875202572287, -STORE, 47875202408448, 47875202519039, -STORE, 47875202093056, 47875202408447, -ERASE, 47875202093056, 47875202093056, -STORE, 47875202093056, 47875202408447, -STORE, 47875202514944, 47875202519039, -STORE, 47875202408448, 47875202514943, -ERASE, 47875202408448, 47875202408448, -STORE, 47875202408448, 47875202514943, -ERASE, 47875202519040, 47875202519040, -STORE, 47875202519040, 47875202572287, -STORE, 47875202572288, 47875205623807, -STORE, 47875203117056, 47875205623807, -STORE, 47875202572288, 47875203117055, -ERASE, 47875203117056, 47875203117056, -STORE, 47875203117056, 47875205402623, -STORE, 47875205402624, 47875205623807, -STORE, 47875204812800, 47875205402623, -STORE, 47875203117056, 47875204812799, -ERASE, 47875203117056, 47875203117056, -STORE, 47875203117056, 47875204812799, -STORE, 47875205398528, 47875205402623, -STORE, 47875204812800, 47875205398527, -ERASE, 47875204812800, 47875204812800, -STORE, 47875204812800, 47875205398527, -STORE, 47875205607424, 47875205623807, -STORE, 47875205402624, 47875205607423, -ERASE, 47875205402624, 47875205402624, -STORE, 47875205402624, 47875205607423, -ERASE, 47875205607424, 47875205607424, -STORE, 47875205607424, 47875205623807, -STORE, 47875205623808, 47875205656575, -ERASE, 47875205623808, 47875205623808, -STORE, 47875205623808, 47875205631999, -STORE, 47875205632000, 47875205656575, -STORE, 47875205644288, 47875205656575, -STORE, 47875205632000, 47875205644287, -ERASE, 47875205632000, 47875205632000, -STORE, 47875205632000, 47875205644287, -STORE, 47875205648384, 47875205656575, -STORE, 47875205644288, 47875205648383, -ERASE, 47875205644288, 47875205644288, -STORE, 47875205644288, 47875205656575, -ERASE, 47875205644288, 47875205644288, -STORE, 47875205644288, 47875205648383, -STORE, 47875205648384, 47875205656575, -ERASE, 47875205648384, 47875205648384, -STORE, 47875205648384, 47875205656575, -STORE, 47875205656576, 47875205693439, -ERASE, 47875205656576, 47875205656576, -STORE, 47875205656576, 47875205664767, -STORE, 47875205664768, 47875205693439, -STORE, 47875205681152, 47875205693439, -STORE, 47875205664768, 47875205681151, -ERASE, 47875205664768, 47875205664768, -STORE, 47875205664768, 47875205681151, -STORE, 47875205685248, 47875205693439, -STORE, 47875205681152, 47875205685247, -ERASE, 47875205681152, 47875205681152, -STORE, 47875205681152, 47875205693439, -ERASE, 47875205681152, 47875205681152, -STORE, 47875205681152, 47875205685247, -STORE, 47875205685248, 47875205693439, -ERASE, 47875205685248, 47875205685248, -STORE, 47875205685248, 47875205693439, -STORE, 47875205693440, 47875206168575, -ERASE, 47875205693440, 47875205693440, -STORE, 47875205693440, 47875205701631, -STORE, 47875205701632, 47875206168575, -STORE, 47875206037504, 47875206168575, -STORE, 47875205701632, 47875206037503, -ERASE, 47875205701632, 47875205701632, -STORE, 47875205701632, 47875206037503, -STORE, 47875206160384, 47875206168575, -STORE, 47875206037504, 47875206160383, -ERASE, 47875206037504, 47875206037504, -STORE, 47875206037504, 47875206168575, -ERASE, 47875206037504, 47875206037504, -STORE, 47875206037504, 47875206160383, -STORE, 47875206160384, 47875206168575, -ERASE, 47875206160384, 47875206160384, -STORE, 47875206160384, 47875206168575, -STORE, 47875206168576, 47875206176767, -STORE, 47875206176768, 47875206197247, -ERASE, 47875206176768, 47875206176768, -STORE, 47875206176768, 47875206180863, -STORE, 47875206180864, 47875206197247, -STORE, 47875206184960, 47875206197247, -STORE, 47875206180864, 47875206184959, -ERASE, 47875206180864, 47875206180864, -STORE, 47875206180864, 47875206184959, -STORE, 47875206189056, 47875206197247, -STORE, 47875206184960, 47875206189055, -ERASE, 47875206184960, 47875206184960, -STORE, 47875206184960, 47875206197247, -ERASE, 47875206184960, 47875206184960, -STORE, 47875206184960, 47875206189055, -STORE, 47875206189056, 47875206197247, -ERASE, 47875206189056, 47875206189056, -STORE, 47875206189056, 47875206197247, -STORE, 47875206197248, 47875206205439, -ERASE, 47875198861312, 47875198861312, -STORE, 47875198861312, 47875198877695, -STORE, 47875198877696, 47875198885887, -ERASE, 47875206189056, 47875206189056, -STORE, 47875206189056, 47875206193151, -STORE, 47875206193152, 47875206197247, -ERASE, 47875201777664, 47875201777664, -STORE, 47875201777664, 47875201781759, -STORE, 47875201781760, 47875201785855, -ERASE, 47875206160384, 47875206160384, -STORE, 47875206160384, 47875206164479, -STORE, 47875206164480, 47875206168575, -ERASE, 47875205685248, 47875205685248, -STORE, 47875205685248, 47875205689343, -STORE, 47875205689344, 47875205693439, -ERASE, 47875205648384, 47875205648384, -STORE, 47875205648384, 47875205652479, -STORE, 47875205652480, 47875205656575, -ERASE, 47875205402624, 47875205402624, -STORE, 47875205402624, 47875205599231, -STORE, 47875205599232, 47875205607423, -ERASE, 47875202519040, 47875202519040, -STORE, 47875202519040, 47875202555903, -STORE, 47875202555904, 47875202572287, -ERASE, 47875201966080, 47875201966080, -STORE, 47875201966080, 47875201970175, -STORE, 47875201970176, 47875201974271, -ERASE, 47875201650688, 47875201650688, -STORE, 47875201650688, 47875201654783, -STORE, 47875201654784, 47875201658879, -ERASE, 47875199381504, 47875199381504, -STORE, 47875199381504, 47875199401983, -STORE, 47875199401984, 47875199406079, -ERASE, 47875199049728, 47875199049728, -STORE, 47875199049728, 47875199053823, -STORE, 47875199053824, 47875199057919, -ERASE, 47875199004672, 47875199004672, -STORE, 47875199004672, 47875199008767, -STORE, 47875199008768, 47875199012863, -ERASE, 94011547025408, 94011547025408, -STORE, 94011547025408, 94011547148287, -STORE, 94011547148288, 94011547152383, -ERASE, 139757598109696, 139757598109696, -STORE, 139757598109696, 139757598113791, -STORE, 139757598113792, 139757598117887, -ERASE, 47875197046784, 47875197046784, -STORE, 94011557584896, 94011557720063, -STORE, 94011557584896, 94011557855231, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557851135, -STORE, 94011557851136, 94011557855231, -ERASE, 94011557851136, 94011557851136, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557847039, -STORE, 94011557847040, 94011557851135, -ERASE, 94011557847040, 94011557847040, -STORE, 94011557584896, 94011557982207, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557978111, -STORE, 94011557978112, 94011557982207, -ERASE, 94011557978112, 94011557978112, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557974015, -STORE, 94011557974016, 94011557978111, -ERASE, 94011557974016, 94011557974016, -STORE, 140737488347136, 140737488351231, -STORE, 140734130360320, 140737488351231, -ERASE, 140734130360320, 140734130360320, -STORE, 140734130360320, 140734130364415, -STORE, 94641232105472, 94641232785407, -ERASE, 94641232105472, 94641232105472, -STORE, 94641232105472, 94641232171007, -STORE, 94641232171008, 94641232785407, -ERASE, 94641232171008, 94641232171008, -STORE, 94641232171008, 94641232519167, -STORE, 94641232519168, 94641232658431, -STORE, 94641232658432, 94641232785407, -STORE, 139726599516160, 139726599688191, -ERASE, 139726599516160, 139726599516160, -STORE, 139726599516160, 139726599520255, -STORE, 139726599520256, 139726599688191, -ERASE, 139726599520256, 139726599520256, -STORE, 139726599520256, 139726599643135, -STORE, 139726599643136, 139726599675903, -STORE, 139726599675904, 139726599684095, -STORE, 139726599684096, 139726599688191, -STORE, 140734130446336, 140734130450431, -STORE, 140734130434048, 140734130446335, -STORE, 47906195480576, 47906195488767, -STORE, 47906195488768, 47906195496959, -STORE, 47906195496960, 47906197336063, -STORE, 47906195636224, 47906197336063, -STORE, 47906195496960, 47906195636223, -ERASE, 47906195636224, 47906195636224, -STORE, 47906195636224, 47906197295103, -STORE, 47906197295104, 47906197336063, -STORE, 47906196979712, 47906197295103, -STORE, 47906195636224, 47906196979711, -ERASE, 47906195636224, 47906195636224, -STORE, 47906195636224, 47906196979711, -STORE, 47906197291008, 47906197295103, -STORE, 47906196979712, 47906197291007, -ERASE, 47906196979712, 47906196979712, -STORE, 47906196979712, 47906197291007, -STORE, 47906197319680, 47906197336063, -STORE, 47906197295104, 47906197319679, -ERASE, 47906197295104, 47906197295104, -STORE, 47906197295104, 47906197319679, -ERASE, 47906197319680, 47906197319680, -STORE, 47906197319680, 47906197336063, -STORE, 47906197336064, 47906197446655, -STORE, 47906197352448, 47906197446655, -STORE, 47906197336064, 47906197352447, -ERASE, 47906197352448, 47906197352448, -STORE, 47906197352448, 47906197438463, -STORE, 47906197438464, 47906197446655, -STORE, 47906197413888, 47906197438463, -STORE, 47906197352448, 47906197413887, -ERASE, 47906197352448, 47906197352448, -STORE, 47906197352448, 47906197413887, -STORE, 47906197434368, 47906197438463, -STORE, 47906197413888, 47906197434367, -ERASE, 47906197413888, 47906197413888, -STORE, 47906197413888, 47906197434367, -ERASE, 47906197438464, 47906197438464, -STORE, 47906197438464, 47906197446655, -STORE, 47906197446656, 47906197491711, -ERASE, 47906197446656, 47906197446656, -STORE, 47906197446656, 47906197454847, -STORE, 47906197454848, 47906197491711, -STORE, 47906197475328, 47906197491711, -STORE, 47906197454848, 47906197475327, -ERASE, 47906197454848, 47906197454848, -STORE, 47906197454848, 47906197475327, -STORE, 47906197483520, 47906197491711, -STORE, 47906197475328, 47906197483519, -ERASE, 47906197475328, 47906197475328, -STORE, 47906197475328, 47906197491711, -ERASE, 47906197475328, 47906197475328, -STORE, 47906197475328, 47906197483519, -STORE, 47906197483520, 47906197491711, -ERASE, 47906197483520, 47906197483520, -STORE, 47906197483520, 47906197491711, -STORE, 47906197491712, 47906197839871, -STORE, 47906197532672, 47906197839871, -STORE, 47906197491712, 47906197532671, -ERASE, 47906197532672, 47906197532672, -STORE, 47906197532672, 47906197815295, -STORE, 47906197815296, 47906197839871, -STORE, 47906197745664, 47906197815295, -STORE, 47906197532672, 47906197745663, -ERASE, 47906197532672, 47906197532672, -STORE, 47906197532672, 47906197745663, -STORE, 47906197811200, 47906197815295, -STORE, 47906197745664, 47906197811199, -ERASE, 47906197745664, 47906197745664, -STORE, 47906197745664, 47906197811199, -ERASE, 47906197815296, 47906197815296, -STORE, 47906197815296, 47906197839871, -STORE, 47906197839872, 47906200100863, -STORE, 47906197991424, 47906200100863, -STORE, 47906197839872, 47906197991423, -ERASE, 47906197991424, 47906197991424, -STORE, 47906197991424, 47906200084479, -STORE, 47906200084480, 47906200100863, -STORE, 47906200092672, 47906200100863, -STORE, 47906200084480, 47906200092671, -ERASE, 47906200084480, 47906200084480, -STORE, 47906200084480, 47906200092671, -ERASE, 47906200092672, 47906200092672, -STORE, 47906200092672, 47906200100863, -STORE, 47906200100864, 47906200236031, -ERASE, 47906200100864, 47906200100864, -STORE, 47906200100864, 47906200125439, -STORE, 47906200125440, 47906200236031, -STORE, 47906200186880, 47906200236031, -STORE, 47906200125440, 47906200186879, -ERASE, 47906200125440, 47906200125440, -STORE, 47906200125440, 47906200186879, -STORE, 47906200211456, 47906200236031, -STORE, 47906200186880, 47906200211455, -ERASE, 47906200186880, 47906200186880, -STORE, 47906200186880, 47906200236031, -ERASE, 47906200186880, 47906200186880, -STORE, 47906200186880, 47906200211455, -STORE, 47906200211456, 47906200236031, -STORE, 47906200219648, 47906200236031, -STORE, 47906200211456, 47906200219647, -ERASE, 47906200211456, 47906200211456, -STORE, 47906200211456, 47906200219647, -ERASE, 47906200219648, 47906200219648, -STORE, 47906200219648, 47906200236031, -STORE, 47906200219648, 47906200244223, -STORE, 47906200244224, 47906200408063, -ERASE, 47906200244224, 47906200244224, -STORE, 47906200244224, 47906200256511, -STORE, 47906200256512, 47906200408063, -STORE, 47906200354816, 47906200408063, -STORE, 47906200256512, 47906200354815, -ERASE, 47906200256512, 47906200256512, -STORE, 47906200256512, 47906200354815, -STORE, 47906200399872, 47906200408063, -STORE, 47906200354816, 47906200399871, -ERASE, 47906200354816, 47906200354816, -STORE, 47906200354816, 47906200408063, -ERASE, 47906200354816, 47906200354816, -STORE, 47906200354816, 47906200399871, -STORE, 47906200399872, 47906200408063, -ERASE, 47906200399872, 47906200399872, -STORE, 47906200399872, 47906200408063, -STORE, 47906200408064, 47906201006079, -STORE, 47906200526848, 47906201006079, -STORE, 47906200408064, 47906200526847, -ERASE, 47906200526848, 47906200526848, -STORE, 47906200526848, 47906200952831, -STORE, 47906200952832, 47906201006079, -STORE, 47906200842240, 47906200952831, -STORE, 47906200526848, 47906200842239, -ERASE, 47906200526848, 47906200526848, -STORE, 47906200526848, 47906200842239, -STORE, 47906200948736, 47906200952831, -STORE, 47906200842240, 47906200948735, -ERASE, 47906200842240, 47906200842240, -STORE, 47906200842240, 47906200948735, -ERASE, 47906200952832, 47906200952832, -STORE, 47906200952832, 47906201006079, -STORE, 47906201006080, 47906204057599, -STORE, 47906201550848, 47906204057599, -STORE, 47906201006080, 47906201550847, -ERASE, 47906201550848, 47906201550848, -STORE, 47906201550848, 47906203836415, -STORE, 47906203836416, 47906204057599, -STORE, 47906203246592, 47906203836415, -STORE, 47906201550848, 47906203246591, -ERASE, 47906201550848, 47906201550848, -STORE, 47906201550848, 47906203246591, -STORE, 47906203832320, 47906203836415, -STORE, 47906203246592, 47906203832319, -ERASE, 47906203246592, 47906203246592, -STORE, 47906203246592, 47906203832319, -STORE, 47906204041216, 47906204057599, -STORE, 47906203836416, 47906204041215, -ERASE, 47906203836416, 47906203836416, -STORE, 47906203836416, 47906204041215, -ERASE, 47906204041216, 47906204041216, -STORE, 47906204041216, 47906204057599, -STORE, 47906204057600, 47906204090367, -ERASE, 47906204057600, 47906204057600, -STORE, 47906204057600, 47906204065791, -STORE, 47906204065792, 47906204090367, -STORE, 47906204078080, 47906204090367, -STORE, 47906204065792, 47906204078079, -ERASE, 47906204065792, 47906204065792, -STORE, 47906204065792, 47906204078079, -STORE, 47906204082176, 47906204090367, -STORE, 47906204078080, 47906204082175, -ERASE, 47906204078080, 47906204078080, -STORE, 47906204078080, 47906204090367, -ERASE, 47906204078080, 47906204078080, -STORE, 47906204078080, 47906204082175, -STORE, 47906204082176, 47906204090367, -ERASE, 47906204082176, 47906204082176, -STORE, 47906204082176, 47906204090367, -STORE, 47906204090368, 47906204127231, -ERASE, 47906204090368, 47906204090368, -STORE, 47906204090368, 47906204098559, -STORE, 47906204098560, 47906204127231, -STORE, 47906204114944, 47906204127231, -STORE, 47906204098560, 47906204114943, -ERASE, 47906204098560, 47906204098560, -STORE, 47906204098560, 47906204114943, -STORE, 47906204119040, 47906204127231, -STORE, 47906204114944, 47906204119039, -ERASE, 47906204114944, 47906204114944, -STORE, 47906204114944, 47906204127231, -ERASE, 47906204114944, 47906204114944, -STORE, 47906204114944, 47906204119039, -STORE, 47906204119040, 47906204127231, -ERASE, 47906204119040, 47906204119040, -STORE, 47906204119040, 47906204127231, -STORE, 47906204127232, 47906204602367, -ERASE, 47906204127232, 47906204127232, -STORE, 47906204127232, 47906204135423, -STORE, 47906204135424, 47906204602367, -STORE, 47906204471296, 47906204602367, -STORE, 47906204135424, 47906204471295, -ERASE, 47906204135424, 47906204135424, -STORE, 47906204135424, 47906204471295, -STORE, 47906204594176, 47906204602367, -STORE, 47906204471296, 47906204594175, -ERASE, 47906204471296, 47906204471296, -STORE, 47906204471296, 47906204602367, -ERASE, 47906204471296, 47906204471296, -STORE, 47906204471296, 47906204594175, -STORE, 47906204594176, 47906204602367, -ERASE, 47906204594176, 47906204594176, -STORE, 47906204594176, 47906204602367, -STORE, 47906204602368, 47906204610559, -STORE, 47906204610560, 47906204631039, -ERASE, 47906204610560, 47906204610560, -STORE, 47906204610560, 47906204614655, -STORE, 47906204614656, 47906204631039, -STORE, 47906204618752, 47906204631039, -STORE, 47906204614656, 47906204618751, -ERASE, 47906204614656, 47906204614656, -STORE, 47906204614656, 47906204618751, -STORE, 47906204622848, 47906204631039, -STORE, 47906204618752, 47906204622847, -ERASE, 47906204618752, 47906204618752, -STORE, 47906204618752, 47906204631039, -ERASE, 47906204618752, 47906204618752, -STORE, 47906204618752, 47906204622847, -STORE, 47906204622848, 47906204631039, -ERASE, 47906204622848, 47906204622848, -STORE, 47906204622848, 47906204631039, -STORE, 47906204631040, 47906204639231, -ERASE, 47906197295104, 47906197295104, -STORE, 47906197295104, 47906197311487, -STORE, 47906197311488, 47906197319679, -ERASE, 47906204622848, 47906204622848, -STORE, 47906204622848, 47906204626943, -STORE, 47906204626944, 47906204631039, -ERASE, 47906200211456, 47906200211456, -STORE, 47906200211456, 47906200215551, -STORE, 47906200215552, 47906200219647, -ERASE, 47906204594176, 47906204594176, -STORE, 47906204594176, 47906204598271, -STORE, 47906204598272, 47906204602367, -ERASE, 47906204119040, 47906204119040, -STORE, 47906204119040, 47906204123135, -STORE, 47906204123136, 47906204127231, -ERASE, 47906204082176, 47906204082176, -STORE, 47906204082176, 47906204086271, -STORE, 47906204086272, 47906204090367, -ERASE, 47906203836416, 47906203836416, -STORE, 47906203836416, 47906204033023, -STORE, 47906204033024, 47906204041215, -ERASE, 47906200952832, 47906200952832, -STORE, 47906200952832, 47906200989695, -STORE, 47906200989696, 47906201006079, -ERASE, 47906200399872, 47906200399872, -STORE, 47906200399872, 47906200403967, -STORE, 47906200403968, 47906200408063, -ERASE, 47906200084480, 47906200084480, -STORE, 47906200084480, 47906200088575, -STORE, 47906200088576, 47906200092671, -ERASE, 47906197815296, 47906197815296, -STORE, 47906197815296, 47906197835775, -STORE, 47906197835776, 47906197839871, -ERASE, 47906197483520, 47906197483520, -STORE, 47906197483520, 47906197487615, -STORE, 47906197487616, 47906197491711, -ERASE, 47906197438464, 47906197438464, -STORE, 47906197438464, 47906197442559, -STORE, 47906197442560, 47906197446655, -ERASE, 94641232658432, 94641232658432, -STORE, 94641232658432, 94641232781311, -STORE, 94641232781312, 94641232785407, -ERASE, 139726599675904, 139726599675904, -STORE, 139726599675904, 139726599679999, -STORE, 139726599680000, 139726599684095, -ERASE, 47906195480576, 47906195480576, -STORE, 94641242615808, 94641242750975, - }; + check_seq(mt, 50, false); + mt_set_non_kernel(4); + check_store_range(mt, 5, 47, xa_mk_value(47), 0); + MT_BUG_ON(mt, !mt_height(mt)); + mtree_destroy(mt); - unsigned long set10[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140736427839488, 140737488351231, -ERASE, 140736427839488, 140736427839488, -STORE, 140736427839488, 140736427843583, -STORE, 94071213395968, 94071213567999, -ERASE, 94071213395968, 94071213395968, -STORE, 94071213395968, 94071213412351, -STORE, 94071213412352, 94071213567999, -ERASE, 94071213412352, 94071213412352, -STORE, 94071213412352, 94071213514751, -STORE, 94071213514752, 94071213555711, -STORE, 94071213555712, 94071213567999, -STORE, 139968410644480, 139968410816511, -ERASE, 139968410644480, 139968410644480, -STORE, 139968410644480, 139968410648575, -STORE, 139968410648576, 139968410816511, -ERASE, 139968410648576, 139968410648576, -STORE, 139968410648576, 139968410771455, -STORE, 139968410771456, 139968410804223, -STORE, 139968410804224, 139968410812415, -STORE, 139968410812416, 139968410816511, -STORE, 140736429277184, 140736429281279, -STORE, 140736429264896, 140736429277183, -STORE, 47664384352256, 47664384360447, -STORE, 47664384360448, 47664384368639, -STORE, 47664384368640, 47664384532479, -ERASE, 47664384368640, 47664384368640, -STORE, 47664384368640, 47664384380927, -STORE, 47664384380928, 47664384532479, -STORE, 47664384479232, 47664384532479, -STORE, 47664384380928, 47664384479231, -ERASE, 47664384380928, 47664384380928, -STORE, 47664384380928, 47664384479231, -STORE, 47664384524288, 47664384532479, -STORE, 47664384479232, 47664384524287, -ERASE, 47664384479232, 47664384479232, -STORE, 47664384479232, 47664384532479, -ERASE, 47664384479232, 47664384479232, -STORE, 47664384479232, 47664384524287, -STORE, 47664384524288, 47664384532479, -ERASE, 47664384524288, 47664384524288, -STORE, 47664384524288, 47664384532479, -STORE, 47664384532480, 47664387583999, -STORE, 47664385077248, 47664387583999, -STORE, 47664384532480, 47664385077247, -ERASE, 47664385077248, 47664385077248, -STORE, 47664385077248, 47664387362815, -STORE, 47664387362816, 47664387583999, -STORE, 47664386772992, 47664387362815, -STORE, 47664385077248, 47664386772991, -ERASE, 47664385077248, 47664385077248, -STORE, 47664385077248, 47664386772991, -STORE, 47664387358720, 47664387362815, -STORE, 47664386772992, 47664387358719, -ERASE, 47664386772992, 47664386772992, -STORE, 47664386772992, 47664387358719, -STORE, 47664387567616, 47664387583999, -STORE, 47664387362816, 47664387567615, -ERASE, 47664387362816, 47664387362816, -STORE, 47664387362816, 47664387567615, -ERASE, 47664387567616, 47664387567616, -STORE, 47664387567616, 47664387583999, -STORE, 47664387584000, 47664389423103, -STORE, 47664387723264, 47664389423103, -STORE, 47664387584000, 47664387723263, -ERASE, 47664387723264, 47664387723264, -STORE, 47664387723264, 47664389382143, -STORE, 47664389382144, 47664389423103, -STORE, 47664389066752, 47664389382143, -STORE, 47664387723264, 47664389066751, -ERASE, 47664387723264, 47664387723264, -STORE, 47664387723264, 47664389066751, -STORE, 47664389378048, 47664389382143, -STORE, 47664389066752, 47664389378047, -ERASE, 47664389066752, 47664389066752, -STORE, 47664389066752, 47664389378047, -STORE, 47664389406720, 47664389423103, -STORE, 47664389382144, 47664389406719, -ERASE, 47664389382144, 47664389382144, -STORE, 47664389382144, 47664389406719, -ERASE, 47664389406720, 47664389406720, -STORE, 47664389406720, 47664389423103, -STORE, 47664389423104, 47664389558271, -ERASE, 47664389423104, 47664389423104, -STORE, 47664389423104, 47664389447679, -STORE, 47664389447680, 47664389558271, -STORE, 47664389509120, 47664389558271, -STORE, 47664389447680, 47664389509119, -ERASE, 47664389447680, 47664389447680, -STORE, 47664389447680, 47664389509119, -STORE, 47664389533696, 47664389558271, -STORE, 47664389509120, 47664389533695, -ERASE, 47664389509120, 47664389509120, -STORE, 47664389509120, 47664389558271, -ERASE, 47664389509120, 47664389509120, -STORE, 47664389509120, 47664389533695, -STORE, 47664389533696, 47664389558271, -STORE, 47664389541888, 47664389558271, -STORE, 47664389533696, 47664389541887, -ERASE, 47664389533696, 47664389533696, -STORE, 47664389533696, 47664389541887, -ERASE, 47664389541888, 47664389541888, -STORE, 47664389541888, 47664389558271, -STORE, 47664389558272, 47664389578751, -ERASE, 47664389558272, 47664389558272, -STORE, 47664389558272, 47664389562367, -STORE, 47664389562368, 47664389578751, -STORE, 47664389566464, 47664389578751, -STORE, 47664389562368, 47664389566463, -ERASE, 47664389562368, 47664389562368, -STORE, 47664389562368, 47664389566463, -STORE, 47664389570560, 47664389578751, -STORE, 47664389566464, 47664389570559, -ERASE, 47664389566464, 47664389566464, -STORE, 47664389566464, 47664389578751, -ERASE, 47664389566464, 47664389566464, -STORE, 47664389566464, 47664389570559, -STORE, 47664389570560, 47664389578751, -ERASE, 47664389570560, 47664389570560, -STORE, 47664389570560, 47664389578751, -STORE, 47664389578752, 47664389586943, -ERASE, 47664389382144, 47664389382144, -STORE, 47664389382144, 47664389398527, -STORE, 47664389398528, 47664389406719, -ERASE, 47664389570560, 47664389570560, -STORE, 47664389570560, 47664389574655, -STORE, 47664389574656, 47664389578751, -ERASE, 47664389533696, 47664389533696, -STORE, 47664389533696, 47664389537791, -STORE, 47664389537792, 47664389541887, -ERASE, 47664387362816, 47664387362816, -STORE, 47664387362816, 47664387559423, -STORE, 47664387559424, 47664387567615, -ERASE, 47664384524288, 47664384524288, -STORE, 47664384524288, 47664384528383, -STORE, 47664384528384, 47664384532479, -ERASE, 94071213555712, 94071213555712, -STORE, 94071213555712, 94071213563903, -STORE, 94071213563904, 94071213567999, -ERASE, 139968410804224, 139968410804224, -STORE, 139968410804224, 139968410808319, -STORE, 139968410808320, 139968410812415, -ERASE, 47664384352256, 47664384352256, -STORE, 94071244402688, 94071244537855, -STORE, 140737488347136, 140737488351231, -STORE, 140728271503360, 140737488351231, -ERASE, 140728271503360, 140728271503360, -STORE, 140728271503360, 140728271507455, -STORE, 94410361982976, 94410362155007, -ERASE, 94410361982976, 94410361982976, -STORE, 94410361982976, 94410361999359, -STORE, 94410361999360, 94410362155007, -ERASE, 94410361999360, 94410361999360, -STORE, 94410361999360, 94410362101759, -STORE, 94410362101760, 94410362142719, -STORE, 94410362142720, 94410362155007, -STORE, 140351953997824, 140351954169855, -ERASE, 140351953997824, 140351953997824, -STORE, 140351953997824, 140351954001919, -STORE, 140351954001920, 140351954169855, -ERASE, 140351954001920, 140351954001920, -STORE, 140351954001920, 140351954124799, -STORE, 140351954124800, 140351954157567, -STORE, 140351954157568, 140351954165759, -STORE, 140351954165760, 140351954169855, -STORE, 140728272429056, 140728272433151, -STORE, 140728272416768, 140728272429055, -STORE, 47280840998912, 47280841007103, -STORE, 47280841007104, 47280841015295, -STORE, 47280841015296, 47280841179135, -ERASE, 47280841015296, 47280841015296, -STORE, 47280841015296, 47280841027583, -STORE, 47280841027584, 47280841179135, -STORE, 47280841125888, 47280841179135, -STORE, 47280841027584, 47280841125887, -ERASE, 47280841027584, 47280841027584, -STORE, 47280841027584, 47280841125887, -STORE, 47280841170944, 47280841179135, -STORE, 47280841125888, 47280841170943, -ERASE, 47280841125888, 47280841125888, -STORE, 47280841125888, 47280841179135, -ERASE, 47280841125888, 47280841125888, -STORE, 47280841125888, 47280841170943, -STORE, 47280841170944, 47280841179135, -ERASE, 47280841170944, 47280841170944, -STORE, 47280841170944, 47280841179135, -STORE, 47280841179136, 47280844230655, -STORE, 47280841723904, 47280844230655, -STORE, 47280841179136, 47280841723903, -ERASE, 47280841723904, 47280841723904, -STORE, 47280841723904, 47280844009471, -STORE, 47280844009472, 47280844230655, -STORE, 47280843419648, 47280844009471, -STORE, 47280841723904, 47280843419647, -ERASE, 47280841723904, 47280841723904, -STORE, 47280841723904, 47280843419647, -STORE, 47280844005376, 47280844009471, -STORE, 47280843419648, 47280844005375, -ERASE, 47280843419648, 47280843419648, -STORE, 47280843419648, 47280844005375, -STORE, 47280844214272, 47280844230655, -STORE, 47280844009472, 47280844214271, -ERASE, 47280844009472, 47280844009472, -STORE, 47280844009472, 47280844214271, -ERASE, 47280844214272, 47280844214272, -STORE, 47280844214272, 47280844230655, -STORE, 47280844230656, 47280846069759, -STORE, 47280844369920, 47280846069759, -STORE, 47280844230656, 47280844369919, -ERASE, 47280844369920, 47280844369920, -STORE, 47280844369920, 47280846028799, -STORE, 47280846028800, 47280846069759, -STORE, 47280845713408, 47280846028799, -STORE, 47280844369920, 47280845713407, -ERASE, 47280844369920, 47280844369920, -STORE, 47280844369920, 47280845713407, -STORE, 47280846024704, 47280846028799, -STORE, 47280845713408, 47280846024703, -ERASE, 47280845713408, 47280845713408, -STORE, 47280845713408, 47280846024703, -STORE, 47280846053376, 47280846069759, -STORE, 47280846028800, 47280846053375, -ERASE, 47280846028800, 47280846028800, -STORE, 47280846028800, 47280846053375, -ERASE, 47280846053376, 47280846053376, -STORE, 47280846053376, 47280846069759, -STORE, 47280846069760, 47280846204927, -ERASE, 47280846069760, 47280846069760, -STORE, 47280846069760, 47280846094335, -STORE, 47280846094336, 47280846204927, -STORE, 47280846155776, 47280846204927, -STORE, 47280846094336, 47280846155775, -ERASE, 47280846094336, 47280846094336, -STORE, 47280846094336, 47280846155775, -STORE, 47280846180352, 47280846204927, -STORE, 47280846155776, 47280846180351, -ERASE, 47280846155776, 47280846155776, -STORE, 47280846155776, 47280846204927, -ERASE, 47280846155776, 47280846155776, -STORE, 47280846155776, 47280846180351, -STORE, 47280846180352, 47280846204927, -STORE, 47280846188544, 47280846204927, -STORE, 47280846180352, 47280846188543, -ERASE, 47280846180352, 47280846180352, -STORE, 47280846180352, 47280846188543, -ERASE, 47280846188544, 47280846188544, -STORE, 47280846188544, 47280846204927, -STORE, 47280846204928, 47280846225407, -ERASE, 47280846204928, 47280846204928, -STORE, 47280846204928, 47280846209023, -STORE, 47280846209024, 47280846225407, -STORE, 47280846213120, 47280846225407, -STORE, 47280846209024, 47280846213119, -ERASE, 47280846209024, 47280846209024, -STORE, 47280846209024, 47280846213119, -STORE, 47280846217216, 47280846225407, -STORE, 47280846213120, 47280846217215, -ERASE, 47280846213120, 47280846213120, -STORE, 47280846213120, 47280846225407, -ERASE, 47280846213120, 47280846213120, -STORE, 47280846213120, 47280846217215, -STORE, 47280846217216, 47280846225407, -ERASE, 47280846217216, 47280846217216, -STORE, 47280846217216, 47280846225407, -STORE, 47280846225408, 47280846233599, -ERASE, 47280846028800, 47280846028800, -STORE, 47280846028800, 47280846045183, -STORE, 47280846045184, 47280846053375, -ERASE, 47280846217216, 47280846217216, -STORE, 47280846217216, 47280846221311, -STORE, 47280846221312, 47280846225407, -ERASE, 47280846180352, 47280846180352, -STORE, 47280846180352, 47280846184447, -STORE, 47280846184448, 47280846188543, -ERASE, 47280844009472, 47280844009472, -STORE, 47280844009472, 47280844206079, -STORE, 47280844206080, 47280844214271, -ERASE, 47280841170944, 47280841170944, -STORE, 47280841170944, 47280841175039, -STORE, 47280841175040, 47280841179135, -ERASE, 94410362142720, 94410362142720, -STORE, 94410362142720, 94410362150911, -STORE, 94410362150912, 94410362155007, -ERASE, 140351954157568, 140351954157568, -STORE, 140351954157568, 140351954161663, -STORE, 140351954161664, 140351954165759, -ERASE, 47280840998912, 47280840998912, -STORE, 94410379456512, 94410379591679, -STORE, 140737488347136, 140737488351231, -STORE, 140732946362368, 140737488351231, -ERASE, 140732946362368, 140732946362368, -STORE, 140732946362368, 140732946366463, -STORE, 94352937934848, 94352938106879, -ERASE, 94352937934848, 94352937934848, -STORE, 94352937934848, 94352937951231, -STORE, 94352937951232, 94352938106879, -ERASE, 94352937951232, 94352937951232, -STORE, 94352937951232, 94352938053631, -STORE, 94352938053632, 94352938094591, -STORE, 94352938094592, 94352938106879, -STORE, 140595518742528, 140595518914559, -ERASE, 140595518742528, 140595518742528, -STORE, 140595518742528, 140595518746623, -STORE, 140595518746624, 140595518914559, -ERASE, 140595518746624, 140595518746624, -STORE, 140595518746624, 140595518869503, -STORE, 140595518869504, 140595518902271, -STORE, 140595518902272, 140595518910463, -STORE, 140595518910464, 140595518914559, -STORE, 140732947468288, 140732947472383, -STORE, 140732947456000, 140732947468287, -STORE, 47037276254208, 47037276262399, -STORE, 47037276262400, 47037276270591, -STORE, 47037276270592, 47037276434431, -ERASE, 47037276270592, 47037276270592, -STORE, 47037276270592, 47037276282879, -STORE, 47037276282880, 47037276434431, -STORE, 47037276381184, 47037276434431, -STORE, 47037276282880, 47037276381183, -ERASE, 47037276282880, 47037276282880, -STORE, 47037276282880, 47037276381183, -STORE, 47037276426240, 47037276434431, -STORE, 47037276381184, 47037276426239, -ERASE, 47037276381184, 47037276381184, -STORE, 47037276381184, 47037276434431, -ERASE, 47037276381184, 47037276381184, -STORE, 47037276381184, 47037276426239, -STORE, 47037276426240, 47037276434431, -ERASE, 47037276426240, 47037276426240, -STORE, 47037276426240, 47037276434431, -STORE, 47037276434432, 47037279485951, -STORE, 47037276979200, 47037279485951, -STORE, 47037276434432, 47037276979199, -ERASE, 47037276979200, 47037276979200, -STORE, 47037276979200, 47037279264767, -STORE, 47037279264768, 47037279485951, -STORE, 47037278674944, 47037279264767, -STORE, 47037276979200, 47037278674943, -ERASE, 47037276979200, 47037276979200, -STORE, 47037276979200, 47037278674943, -STORE, 47037279260672, 47037279264767, -STORE, 47037278674944, 47037279260671, -ERASE, 47037278674944, 47037278674944, -STORE, 47037278674944, 47037279260671, -STORE, 47037279469568, 47037279485951, -STORE, 47037279264768, 47037279469567, -ERASE, 47037279264768, 47037279264768, -STORE, 47037279264768, 47037279469567, -ERASE, 47037279469568, 47037279469568, -STORE, 47037279469568, 47037279485951, -STORE, 47037279485952, 47037281325055, -STORE, 47037279625216, 47037281325055, -STORE, 47037279485952, 47037279625215, -ERASE, 47037279625216, 47037279625216, -STORE, 47037279625216, 47037281284095, -STORE, 47037281284096, 47037281325055, -STORE, 47037280968704, 47037281284095, -STORE, 47037279625216, 47037280968703, -ERASE, 47037279625216, 47037279625216, -STORE, 47037279625216, 47037280968703, -STORE, 47037281280000, 47037281284095, -STORE, 47037280968704, 47037281279999, -ERASE, 47037280968704, 47037280968704, -STORE, 47037280968704, 47037281279999, -STORE, 47037281308672, 47037281325055, -STORE, 47037281284096, 47037281308671, -ERASE, 47037281284096, 47037281284096, -STORE, 47037281284096, 47037281308671, -ERASE, 47037281308672, 47037281308672, -STORE, 47037281308672, 47037281325055, -STORE, 47037281325056, 47037281460223, -ERASE, 47037281325056, 47037281325056, -STORE, 47037281325056, 47037281349631, -STORE, 47037281349632, 47037281460223, -STORE, 47037281411072, 47037281460223, -STORE, 47037281349632, 47037281411071, -ERASE, 47037281349632, 47037281349632, -STORE, 47037281349632, 47037281411071, -STORE, 47037281435648, 47037281460223, -STORE, 47037281411072, 47037281435647, -ERASE, 47037281411072, 47037281411072, -STORE, 47037281411072, 47037281460223, -ERASE, 47037281411072, 47037281411072, -STORE, 47037281411072, 47037281435647, -STORE, 47037281435648, 47037281460223, -STORE, 47037281443840, 47037281460223, -STORE, 47037281435648, 47037281443839, -ERASE, 47037281435648, 47037281435648, -STORE, 47037281435648, 47037281443839, -ERASE, 47037281443840, 47037281443840, -STORE, 47037281443840, 47037281460223, -STORE, 47037281460224, 47037281480703, -ERASE, 47037281460224, 47037281460224, -STORE, 47037281460224, 47037281464319, -STORE, 47037281464320, 47037281480703, -STORE, 47037281468416, 47037281480703, -STORE, 47037281464320, 47037281468415, -ERASE, 47037281464320, 47037281464320, -STORE, 47037281464320, 47037281468415, -STORE, 47037281472512, 47037281480703, -STORE, 47037281468416, 47037281472511, -ERASE, 47037281468416, 47037281468416, -STORE, 47037281468416, 47037281480703, -ERASE, 47037281468416, 47037281468416, -STORE, 47037281468416, 47037281472511, -STORE, 47037281472512, 47037281480703, -ERASE, 47037281472512, 47037281472512, -STORE, 47037281472512, 47037281480703, -STORE, 47037281480704, 47037281488895, -ERASE, 47037281284096, 47037281284096, -STORE, 47037281284096, 47037281300479, -STORE, 47037281300480, 47037281308671, -ERASE, 47037281472512, 47037281472512, -STORE, 47037281472512, 47037281476607, -STORE, 47037281476608, 47037281480703, -ERASE, 47037281435648, 47037281435648, -STORE, 47037281435648, 47037281439743, -STORE, 47037281439744, 47037281443839, -ERASE, 47037279264768, 47037279264768, -STORE, 47037279264768, 47037279461375, -STORE, 47037279461376, 47037279469567, -ERASE, 47037276426240, 47037276426240, -STORE, 47037276426240, 47037276430335, -STORE, 47037276430336, 47037276434431, -ERASE, 94352938094592, 94352938094592, -STORE, 94352938094592, 94352938102783, -STORE, 94352938102784, 94352938106879, -ERASE, 140595518902272, 140595518902272, -STORE, 140595518902272, 140595518906367, -STORE, 140595518906368, 140595518910463, -ERASE, 47037276254208, 47037276254208, -STORE, 94352938438656, 94352938573823, -STORE, 140737488347136, 140737488351231, -STORE, 140733506027520, 140737488351231, -ERASE, 140733506027520, 140733506027520, -STORE, 140733506027520, 140733506031615, -STORE, 94150123073536, 94150123245567, -ERASE, 94150123073536, 94150123073536, -STORE, 94150123073536, 94150123089919, -STORE, 94150123089920, 94150123245567, -ERASE, 94150123089920, 94150123089920, -STORE, 94150123089920, 94150123192319, -STORE, 94150123192320, 94150123233279, -STORE, 94150123233280, 94150123245567, -STORE, 140081290375168, 140081290547199, -ERASE, 140081290375168, 140081290375168, -STORE, 140081290375168, 140081290379263, -STORE, 140081290379264, 140081290547199, -ERASE, 140081290379264, 140081290379264, -STORE, 140081290379264, 140081290502143, -STORE, 140081290502144, 140081290534911, -STORE, 140081290534912, 140081290543103, -STORE, 140081290543104, 140081290547199, -STORE, 140733506707456, 140733506711551, -STORE, 140733506695168, 140733506707455, -STORE, 47551504621568, 47551504629759, -STORE, 47551504629760, 47551504637951, -STORE, 47551504637952, 47551504801791, -ERASE, 47551504637952, 47551504637952, -STORE, 47551504637952, 47551504650239, -STORE, 47551504650240, 47551504801791, -STORE, 47551504748544, 47551504801791, -STORE, 47551504650240, 47551504748543, -ERASE, 47551504650240, 47551504650240, -STORE, 47551504650240, 47551504748543, -STORE, 47551504793600, 47551504801791, -STORE, 47551504748544, 47551504793599, -ERASE, 47551504748544, 47551504748544, -STORE, 47551504748544, 47551504801791, -ERASE, 47551504748544, 47551504748544, -STORE, 47551504748544, 47551504793599, -STORE, 47551504793600, 47551504801791, -ERASE, 47551504793600, 47551504793600, -STORE, 47551504793600, 47551504801791, -STORE, 47551504801792, 47551507853311, -STORE, 47551505346560, 47551507853311, -STORE, 47551504801792, 47551505346559, -ERASE, 47551505346560, 47551505346560, -STORE, 47551505346560, 47551507632127, -STORE, 47551507632128, 47551507853311, -STORE, 47551507042304, 47551507632127, -STORE, 47551505346560, 47551507042303, -ERASE, 47551505346560, 47551505346560, -STORE, 47551505346560, 47551507042303, -STORE, 47551507628032, 47551507632127, -STORE, 47551507042304, 47551507628031, -ERASE, 47551507042304, 47551507042304, -STORE, 47551507042304, 47551507628031, -STORE, 47551507836928, 47551507853311, -STORE, 47551507632128, 47551507836927, -ERASE, 47551507632128, 47551507632128, -STORE, 47551507632128, 47551507836927, -ERASE, 47551507836928, 47551507836928, -STORE, 47551507836928, 47551507853311, -STORE, 47551507853312, 47551509692415, -STORE, 47551507992576, 47551509692415, -STORE, 47551507853312, 47551507992575, -ERASE, 47551507992576, 47551507992576, -STORE, 47551507992576, 47551509651455, -STORE, 47551509651456, 47551509692415, -STORE, 47551509336064, 47551509651455, -STORE, 47551507992576, 47551509336063, -ERASE, 47551507992576, 47551507992576, -STORE, 47551507992576, 47551509336063, -STORE, 47551509647360, 47551509651455, -STORE, 47551509336064, 47551509647359, -ERASE, 47551509336064, 47551509336064, -STORE, 47551509336064, 47551509647359, -STORE, 47551509676032, 47551509692415, -STORE, 47551509651456, 47551509676031, -ERASE, 47551509651456, 47551509651456, -STORE, 47551509651456, 47551509676031, -ERASE, 47551509676032, 47551509676032, -STORE, 47551509676032, 47551509692415, -STORE, 47551509692416, 47551509827583, -ERASE, 47551509692416, 47551509692416, -STORE, 47551509692416, 47551509716991, -STORE, 47551509716992, 47551509827583, -STORE, 47551509778432, 47551509827583, -STORE, 47551509716992, 47551509778431, -ERASE, 47551509716992, 47551509716992, -STORE, 47551509716992, 47551509778431, -STORE, 47551509803008, 47551509827583, -STORE, 47551509778432, 47551509803007, -ERASE, 47551509778432, 47551509778432, -STORE, 47551509778432, 47551509827583, -ERASE, 47551509778432, 47551509778432, -STORE, 47551509778432, 47551509803007, -STORE, 47551509803008, 47551509827583, -STORE, 47551509811200, 47551509827583, -STORE, 47551509803008, 47551509811199, -ERASE, 47551509803008, 47551509803008, -STORE, 47551509803008, 47551509811199, -ERASE, 47551509811200, 47551509811200, -STORE, 47551509811200, 47551509827583, -STORE, 47551509827584, 47551509848063, -ERASE, 47551509827584, 47551509827584, -STORE, 47551509827584, 47551509831679, -STORE, 47551509831680, 47551509848063, -STORE, 47551509835776, 47551509848063, -STORE, 47551509831680, 47551509835775, -ERASE, 47551509831680, 47551509831680, -STORE, 47551509831680, 47551509835775, -STORE, 47551509839872, 47551509848063, -STORE, 47551509835776, 47551509839871, -ERASE, 47551509835776, 47551509835776, -STORE, 47551509835776, 47551509848063, -ERASE, 47551509835776, 47551509835776, -STORE, 47551509835776, 47551509839871, -STORE, 47551509839872, 47551509848063, -ERASE, 47551509839872, 47551509839872, -STORE, 47551509839872, 47551509848063, -STORE, 47551509848064, 47551509856255, -ERASE, 47551509651456, 47551509651456, -STORE, 47551509651456, 47551509667839, -STORE, 47551509667840, 47551509676031, -ERASE, 47551509839872, 47551509839872, -STORE, 47551509839872, 47551509843967, -STORE, 47551509843968, 47551509848063, -ERASE, 47551509803008, 47551509803008, -STORE, 47551509803008, 47551509807103, -STORE, 47551509807104, 47551509811199, -ERASE, 47551507632128, 47551507632128, -STORE, 47551507632128, 47551507828735, -STORE, 47551507828736, 47551507836927, -ERASE, 47551504793600, 47551504793600, -STORE, 47551504793600, 47551504797695, -STORE, 47551504797696, 47551504801791, -ERASE, 94150123233280, 94150123233280, -STORE, 94150123233280, 94150123241471, -STORE, 94150123241472, 94150123245567, -ERASE, 140081290534912, 140081290534912, -STORE, 140081290534912, 140081290539007, -STORE, 140081290539008, 140081290543103, -ERASE, 47551504621568, 47551504621568, -STORE, 94150148112384, 94150148247551, -STORE, 140737488347136, 140737488351231, -STORE, 140734389334016, 140737488351231, -ERASE, 140734389334016, 140734389334016, -STORE, 140734389334016, 140734389338111, -STORE, 94844636606464, 94844636778495, -ERASE, 94844636606464, 94844636606464, -STORE, 94844636606464, 94844636622847, -STORE, 94844636622848, 94844636778495, -ERASE, 94844636622848, 94844636622848, -STORE, 94844636622848, 94844636725247, -STORE, 94844636725248, 94844636766207, -STORE, 94844636766208, 94844636778495, -STORE, 139922765217792, 139922765389823, -ERASE, 139922765217792, 139922765217792, -STORE, 139922765217792, 139922765221887, -STORE, 139922765221888, 139922765389823, -ERASE, 139922765221888, 139922765221888, -STORE, 139922765221888, 139922765344767, -STORE, 139922765344768, 139922765377535, -STORE, 139922765377536, 139922765385727, -STORE, 139922765385728, 139922765389823, -STORE, 140734389678080, 140734389682175, -STORE, 140734389665792, 140734389678079, -STORE, 47710029778944, 47710029787135, -STORE, 47710029787136, 47710029795327, -STORE, 47710029795328, 47710029959167, -ERASE, 47710029795328, 47710029795328, -STORE, 47710029795328, 47710029807615, -STORE, 47710029807616, 47710029959167, -STORE, 47710029905920, 47710029959167, -STORE, 47710029807616, 47710029905919, -ERASE, 47710029807616, 47710029807616, -STORE, 47710029807616, 47710029905919, -STORE, 47710029950976, 47710029959167, -STORE, 47710029905920, 47710029950975, -ERASE, 47710029905920, 47710029905920, -STORE, 47710029905920, 47710029959167, -ERASE, 47710029905920, 47710029905920, -STORE, 47710029905920, 47710029950975, -STORE, 47710029950976, 47710029959167, -ERASE, 47710029950976, 47710029950976, -STORE, 47710029950976, 47710029959167, -STORE, 47710029959168, 47710033010687, -STORE, 47710030503936, 47710033010687, -STORE, 47710029959168, 47710030503935, -ERASE, 47710030503936, 47710030503936, -STORE, 47710030503936, 47710032789503, -STORE, 47710032789504, 47710033010687, -STORE, 47710032199680, 47710032789503, -STORE, 47710030503936, 47710032199679, -ERASE, 47710030503936, 47710030503936, -STORE, 47710030503936, 47710032199679, -STORE, 47710032785408, 47710032789503, -STORE, 47710032199680, 47710032785407, -ERASE, 47710032199680, 47710032199680, -STORE, 47710032199680, 47710032785407, -STORE, 47710032994304, 47710033010687, -STORE, 47710032789504, 47710032994303, -ERASE, 47710032789504, 47710032789504, -STORE, 47710032789504, 47710032994303, -ERASE, 47710032994304, 47710032994304, -STORE, 47710032994304, 47710033010687, -STORE, 47710033010688, 47710034849791, -STORE, 47710033149952, 47710034849791, -STORE, 47710033010688, 47710033149951, -ERASE, 47710033149952, 47710033149952, -STORE, 47710033149952, 47710034808831, -STORE, 47710034808832, 47710034849791, -STORE, 47710034493440, 47710034808831, -STORE, 47710033149952, 47710034493439, -ERASE, 47710033149952, 47710033149952, -STORE, 47710033149952, 47710034493439, -STORE, 47710034804736, 47710034808831, -STORE, 47710034493440, 47710034804735, -ERASE, 47710034493440, 47710034493440, -STORE, 47710034493440, 47710034804735, -STORE, 47710034833408, 47710034849791, -STORE, 47710034808832, 47710034833407, -ERASE, 47710034808832, 47710034808832, -STORE, 47710034808832, 47710034833407, -ERASE, 47710034833408, 47710034833408, -STORE, 47710034833408, 47710034849791, -STORE, 47710034849792, 47710034984959, -ERASE, 47710034849792, 47710034849792, -STORE, 47710034849792, 47710034874367, -STORE, 47710034874368, 47710034984959, -STORE, 47710034935808, 47710034984959, -STORE, 47710034874368, 47710034935807, -ERASE, 47710034874368, 47710034874368, -STORE, 47710034874368, 47710034935807, -STORE, 47710034960384, 47710034984959, -STORE, 47710034935808, 47710034960383, -ERASE, 47710034935808, 47710034935808, -STORE, 47710034935808, 47710034984959, -ERASE, 47710034935808, 47710034935808, -STORE, 47710034935808, 47710034960383, -STORE, 47710034960384, 47710034984959, -STORE, 47710034968576, 47710034984959, -STORE, 47710034960384, 47710034968575, -ERASE, 47710034960384, 47710034960384, -STORE, 47710034960384, 47710034968575, -ERASE, 47710034968576, 47710034968576, -STORE, 47710034968576, 47710034984959, -STORE, 47710034984960, 47710035005439, -ERASE, 47710034984960, 47710034984960, -STORE, 47710034984960, 47710034989055, -STORE, 47710034989056, 47710035005439, -STORE, 47710034993152, 47710035005439, -STORE, 47710034989056, 47710034993151, -ERASE, 47710034989056, 47710034989056, -STORE, 47710034989056, 47710034993151, -STORE, 47710034997248, 47710035005439, -STORE, 47710034993152, 47710034997247, -ERASE, 47710034993152, 47710034993152, -STORE, 47710034993152, 47710035005439, -ERASE, 47710034993152, 47710034993152, -STORE, 47710034993152, 47710034997247, -STORE, 47710034997248, 47710035005439, -ERASE, 47710034997248, 47710034997248, -STORE, 47710034997248, 47710035005439, -STORE, 47710035005440, 47710035013631, -ERASE, 47710034808832, 47710034808832, -STORE, 47710034808832, 47710034825215, -STORE, 47710034825216, 47710034833407, -ERASE, 47710034997248, 47710034997248, -STORE, 47710034997248, 47710035001343, -STORE, 47710035001344, 47710035005439, -ERASE, 47710034960384, 47710034960384, -STORE, 47710034960384, 47710034964479, -STORE, 47710034964480, 47710034968575, -ERASE, 47710032789504, 47710032789504, -STORE, 47710032789504, 47710032986111, -STORE, 47710032986112, 47710032994303, -ERASE, 47710029950976, 47710029950976, -STORE, 47710029950976, 47710029955071, -STORE, 47710029955072, 47710029959167, -ERASE, 94844636766208, 94844636766208, -STORE, 94844636766208, 94844636774399, -STORE, 94844636774400, 94844636778495, -ERASE, 139922765377536, 139922765377536, -STORE, 139922765377536, 139922765381631, -STORE, 139922765381632, 139922765385727, -ERASE, 47710029778944, 47710029778944, -STORE, 94844641775616, 94844641910783, -STORE, 140737488347136, 140737488351231, -STORE, 140732213886976, 140737488351231, -ERASE, 140732213886976, 140732213886976, -STORE, 140732213886976, 140732213891071, -STORE, 94240508887040, 94240509059071, -ERASE, 94240508887040, 94240508887040, -STORE, 94240508887040, 94240508903423, -STORE, 94240508903424, 94240509059071, -ERASE, 94240508903424, 94240508903424, -STORE, 94240508903424, 94240509005823, -STORE, 94240509005824, 94240509046783, -STORE, 94240509046784, 94240509059071, -STORE, 140275106516992, 140275106689023, -ERASE, 140275106516992, 140275106516992, -STORE, 140275106516992, 140275106521087, -STORE, 140275106521088, 140275106689023, -ERASE, 140275106521088, 140275106521088, -STORE, 140275106521088, 140275106643967, -STORE, 140275106643968, 140275106676735, -STORE, 140275106676736, 140275106684927, -STORE, 140275106684928, 140275106689023, -STORE, 140732213977088, 140732213981183, -STORE, 140732213964800, 140732213977087, -STORE, 47357688479744, 47357688487935, -STORE, 47357688487936, 47357688496127, -STORE, 47357688496128, 47357688659967, -ERASE, 47357688496128, 47357688496128, -STORE, 47357688496128, 47357688508415, -STORE, 47357688508416, 47357688659967, -STORE, 47357688606720, 47357688659967, -STORE, 47357688508416, 47357688606719, -ERASE, 47357688508416, 47357688508416, -STORE, 47357688508416, 47357688606719, -STORE, 47357688651776, 47357688659967, -STORE, 47357688606720, 47357688651775, -ERASE, 47357688606720, 47357688606720, -STORE, 47357688606720, 47357688659967, -ERASE, 47357688606720, 47357688606720, -STORE, 47357688606720, 47357688651775, -STORE, 47357688651776, 47357688659967, -ERASE, 47357688651776, 47357688651776, -STORE, 47357688651776, 47357688659967, -STORE, 47357688659968, 47357691711487, -STORE, 47357689204736, 47357691711487, -STORE, 47357688659968, 47357689204735, -ERASE, 47357689204736, 47357689204736, -STORE, 47357689204736, 47357691490303, -STORE, 47357691490304, 47357691711487, -STORE, 47357690900480, 47357691490303, -STORE, 47357689204736, 47357690900479, -ERASE, 47357689204736, 47357689204736, -STORE, 47357689204736, 47357690900479, -STORE, 47357691486208, 47357691490303, -STORE, 47357690900480, 47357691486207, -ERASE, 47357690900480, 47357690900480, -STORE, 47357690900480, 47357691486207, -STORE, 47357691695104, 47357691711487, -STORE, 47357691490304, 47357691695103, -ERASE, 47357691490304, 47357691490304, -STORE, 47357691490304, 47357691695103, -ERASE, 47357691695104, 47357691695104, -STORE, 47357691695104, 47357691711487, -STORE, 47357691711488, 47357693550591, -STORE, 47357691850752, 47357693550591, -STORE, 47357691711488, 47357691850751, -ERASE, 47357691850752, 47357691850752, -STORE, 47357691850752, 47357693509631, -STORE, 47357693509632, 47357693550591, -STORE, 47357693194240, 47357693509631, -STORE, 47357691850752, 47357693194239, -ERASE, 47357691850752, 47357691850752, -STORE, 47357691850752, 47357693194239, -STORE, 47357693505536, 47357693509631, -STORE, 47357693194240, 47357693505535, -ERASE, 47357693194240, 47357693194240, -STORE, 47357693194240, 47357693505535, -STORE, 47357693534208, 47357693550591, -STORE, 47357693509632, 47357693534207, -ERASE, 47357693509632, 47357693509632, -STORE, 47357693509632, 47357693534207, -ERASE, 47357693534208, 47357693534208, -STORE, 47357693534208, 47357693550591, -STORE, 47357693550592, 47357693685759, -ERASE, 47357693550592, 47357693550592, -STORE, 47357693550592, 47357693575167, -STORE, 47357693575168, 47357693685759, -STORE, 47357693636608, 47357693685759, -STORE, 47357693575168, 47357693636607, -ERASE, 47357693575168, 47357693575168, -STORE, 47357693575168, 47357693636607, -STORE, 47357693661184, 47357693685759, -STORE, 47357693636608, 47357693661183, -ERASE, 47357693636608, 47357693636608, -STORE, 47357693636608, 47357693685759, -ERASE, 47357693636608, 47357693636608, -STORE, 47357693636608, 47357693661183, -STORE, 47357693661184, 47357693685759, -STORE, 47357693669376, 47357693685759, -STORE, 47357693661184, 47357693669375, -ERASE, 47357693661184, 47357693661184, -STORE, 47357693661184, 47357693669375, -ERASE, 47357693669376, 47357693669376, -STORE, 47357693669376, 47357693685759, -STORE, 47357693685760, 47357693706239, -ERASE, 47357693685760, 47357693685760, -STORE, 47357693685760, 47357693689855, -STORE, 47357693689856, 47357693706239, -STORE, 47357693693952, 47357693706239, -STORE, 47357693689856, 47357693693951, -ERASE, 47357693689856, 47357693689856, -STORE, 47357693689856, 47357693693951, -STORE, 47357693698048, 47357693706239, -STORE, 47357693693952, 47357693698047, -ERASE, 47357693693952, 47357693693952, -STORE, 47357693693952, 47357693706239, -ERASE, 47357693693952, 47357693693952, -STORE, 47357693693952, 47357693698047, -STORE, 47357693698048, 47357693706239, -ERASE, 47357693698048, 47357693698048, -STORE, 47357693698048, 47357693706239, -STORE, 47357693706240, 47357693714431, -ERASE, 47357693509632, 47357693509632, -STORE, 47357693509632, 47357693526015, -STORE, 47357693526016, 47357693534207, -ERASE, 47357693698048, 47357693698048, -STORE, 47357693698048, 47357693702143, -STORE, 47357693702144, 47357693706239, -ERASE, 47357693661184, 47357693661184, -STORE, 47357693661184, 47357693665279, -STORE, 47357693665280, 47357693669375, -ERASE, 47357691490304, 47357691490304, -STORE, 47357691490304, 47357691686911, -STORE, 47357691686912, 47357691695103, -ERASE, 47357688651776, 47357688651776, -STORE, 47357688651776, 47357688655871, -STORE, 47357688655872, 47357688659967, -ERASE, 94240509046784, 94240509046784, -STORE, 94240509046784, 94240509054975, -STORE, 94240509054976, 94240509059071, -ERASE, 140275106676736, 140275106676736, -STORE, 140275106676736, 140275106680831, -STORE, 140275106680832, 140275106684927, -ERASE, 47357688479744, 47357688479744, -STORE, 94240518361088, 94240518496255, -STORE, 140737488347136, 140737488351231, -STORE, 140732688277504, 140737488351231, -ERASE, 140732688277504, 140732688277504, -STORE, 140732688277504, 140732688281599, -STORE, 94629171351552, 94629172064255, -ERASE, 94629171351552, 94629171351552, -STORE, 94629171351552, 94629171400703, -STORE, 94629171400704, 94629172064255, -ERASE, 94629171400704, 94629171400704, -STORE, 94629171400704, 94629171945471, -STORE, 94629171945472, 94629172043775, -STORE, 94629172043776, 94629172064255, -STORE, 139770707644416, 139770707816447, -ERASE, 139770707644416, 139770707644416, -STORE, 139770707644416, 139770707648511, -STORE, 139770707648512, 139770707816447, -ERASE, 139770707648512, 139770707648512, -STORE, 139770707648512, 139770707771391, -STORE, 139770707771392, 139770707804159, -STORE, 139770707804160, 139770707812351, -STORE, 139770707812352, 139770707816447, -STORE, 140732689121280, 140732689125375, -STORE, 140732689108992, 140732689121279, -STORE, 47862087352320, 47862087360511, -STORE, 47862087360512, 47862087368703, -STORE, 47862087368704, 47862087475199, -STORE, 47862087385088, 47862087475199, -STORE, 47862087368704, 47862087385087, -ERASE, 47862087385088, 47862087385088, -STORE, 47862087385088, 47862087458815, -STORE, 47862087458816, 47862087475199, -STORE, 47862087438336, 47862087458815, -STORE, 47862087385088, 47862087438335, -ERASE, 47862087385088, 47862087385088, -STORE, 47862087385088, 47862087438335, -STORE, 47862087454720, 47862087458815, -STORE, 47862087438336, 47862087454719, -ERASE, 47862087438336, 47862087438336, -STORE, 47862087438336, 47862087454719, -STORE, 47862087467008, 47862087475199, -STORE, 47862087458816, 47862087467007, -ERASE, 47862087458816, 47862087458816, -STORE, 47862087458816, 47862087467007, -ERASE, 47862087467008, 47862087467008, -STORE, 47862087467008, 47862087475199, -STORE, 47862087475200, 47862089314303, -STORE, 47862087614464, 47862089314303, -STORE, 47862087475200, 47862087614463, -ERASE, 47862087614464, 47862087614464, -STORE, 47862087614464, 47862089273343, -STORE, 47862089273344, 47862089314303, -STORE, 47862088957952, 47862089273343, -STORE, 47862087614464, 47862088957951, -ERASE, 47862087614464, 47862087614464, -STORE, 47862087614464, 47862088957951, -STORE, 47862089269248, 47862089273343, -STORE, 47862088957952, 47862089269247, -ERASE, 47862088957952, 47862088957952, -STORE, 47862088957952, 47862089269247, -STORE, 47862089297920, 47862089314303, -STORE, 47862089273344, 47862089297919, -ERASE, 47862089273344, 47862089273344, -STORE, 47862089273344, 47862089297919, -ERASE, 47862089297920, 47862089297920, -STORE, 47862089297920, 47862089314303, -STORE, 47862089297920, 47862089326591, -ERASE, 47862089273344, 47862089273344, -STORE, 47862089273344, 47862089289727, -STORE, 47862089289728, 47862089297919, -ERASE, 47862087458816, 47862087458816, -STORE, 47862087458816, 47862087462911, -STORE, 47862087462912, 47862087467007, -ERASE, 94629172043776, 94629172043776, -STORE, 94629172043776, 94629172060159, -STORE, 94629172060160, 94629172064255, -ERASE, 139770707804160, 139770707804160, -STORE, 139770707804160, 139770707808255, -STORE, 139770707808256, 139770707812351, -ERASE, 47862087352320, 47862087352320, -STORE, 94629197533184, 94629197668351, -STORE, 140737488347136, 140737488351231, -STORE, 140727540711424, 140737488351231, -ERASE, 140727540711424, 140727540711424, -STORE, 140727540711424, 140727540715519, -STORE, 94299865313280, 94299866025983, -ERASE, 94299865313280, 94299865313280, -STORE, 94299865313280, 94299865362431, -STORE, 94299865362432, 94299866025983, -ERASE, 94299865362432, 94299865362432, -STORE, 94299865362432, 94299865907199, -STORE, 94299865907200, 94299866005503, -STORE, 94299866005504, 94299866025983, -STORE, 140680268763136, 140680268935167, -ERASE, 140680268763136, 140680268763136, -STORE, 140680268763136, 140680268767231, -STORE, 140680268767232, 140680268935167, -ERASE, 140680268767232, 140680268767232, -STORE, 140680268767232, 140680268890111, -STORE, 140680268890112, 140680268922879, -STORE, 140680268922880, 140680268931071, -STORE, 140680268931072, 140680268935167, -STORE, 140727541424128, 140727541428223, -STORE, 140727541411840, 140727541424127, -STORE, 46952526233600, 46952526241791, -STORE, 46952526241792, 46952526249983, -STORE, 46952526249984, 46952526356479, -STORE, 46952526266368, 46952526356479, -STORE, 46952526249984, 46952526266367, -ERASE, 46952526266368, 46952526266368, -STORE, 46952526266368, 46952526340095, -STORE, 46952526340096, 46952526356479, -STORE, 46952526319616, 46952526340095, -STORE, 46952526266368, 46952526319615, -ERASE, 46952526266368, 46952526266368, -STORE, 46952526266368, 46952526319615, -STORE, 46952526336000, 46952526340095, -STORE, 46952526319616, 46952526335999, -ERASE, 46952526319616, 46952526319616, -STORE, 46952526319616, 46952526335999, -STORE, 46952526348288, 46952526356479, -STORE, 46952526340096, 46952526348287, -ERASE, 46952526340096, 46952526340096, -STORE, 46952526340096, 46952526348287, -ERASE, 46952526348288, 46952526348288, -STORE, 46952526348288, 46952526356479, -STORE, 46952526356480, 46952528195583, -STORE, 46952526495744, 46952528195583, -STORE, 46952526356480, 46952526495743, -ERASE, 46952526495744, 46952526495744, -STORE, 46952526495744, 46952528154623, -STORE, 46952528154624, 46952528195583, -STORE, 46952527839232, 46952528154623, -STORE, 46952526495744, 46952527839231, -ERASE, 46952526495744, 46952526495744, -STORE, 46952526495744, 46952527839231, -STORE, 46952528150528, 46952528154623, -STORE, 46952527839232, 46952528150527, -ERASE, 46952527839232, 46952527839232, -STORE, 46952527839232, 46952528150527, -STORE, 46952528179200, 46952528195583, -STORE, 46952528154624, 46952528179199, -ERASE, 46952528154624, 46952528154624, -STORE, 46952528154624, 46952528179199, -ERASE, 46952528179200, 46952528179200, -STORE, 46952528179200, 46952528195583, -STORE, 46952528179200, 46952528207871, -ERASE, 46952528154624, 46952528154624, -STORE, 46952528154624, 46952528171007, -STORE, 46952528171008, 46952528179199, -ERASE, 46952526340096, 46952526340096, -STORE, 46952526340096, 46952526344191, -STORE, 46952526344192, 46952526348287, -ERASE, 94299866005504, 94299866005504, -STORE, 94299866005504, 94299866021887, -STORE, 94299866021888, 94299866025983, -ERASE, 140680268922880, 140680268922880, -STORE, 140680268922880, 140680268926975, -STORE, 140680268926976, 140680268931071, -ERASE, 46952526233600, 46952526233600, -STORE, 140737488347136, 140737488351231, -STORE, 140722874793984, 140737488351231, -ERASE, 140722874793984, 140722874793984, -STORE, 140722874793984, 140722874798079, -STORE, 94448916213760, 94448916926463, -ERASE, 94448916213760, 94448916213760, -STORE, 94448916213760, 94448916262911, -STORE, 94448916262912, 94448916926463, -ERASE, 94448916262912, 94448916262912, -STORE, 94448916262912, 94448916807679, -STORE, 94448916807680, 94448916905983, -STORE, 94448916905984, 94448916926463, -STORE, 140389117046784, 140389117218815, -ERASE, 140389117046784, 140389117046784, -STORE, 140389117046784, 140389117050879, -STORE, 140389117050880, 140389117218815, -ERASE, 140389117050880, 140389117050880, -STORE, 140389117050880, 140389117173759, -STORE, 140389117173760, 140389117206527, -STORE, 140389117206528, 140389117214719, -STORE, 140389117214720, 140389117218815, -STORE, 140722875297792, 140722875301887, -STORE, 140722875285504, 140722875297791, -STORE, 47243677949952, 47243677958143, -STORE, 47243677958144, 47243677966335, -STORE, 47243677966336, 47243678072831, -STORE, 47243677982720, 47243678072831, -STORE, 47243677966336, 47243677982719, -ERASE, 47243677982720, 47243677982720, -STORE, 47243677982720, 47243678056447, -STORE, 47243678056448, 47243678072831, -STORE, 47243678035968, 47243678056447, -STORE, 47243677982720, 47243678035967, -ERASE, 47243677982720, 47243677982720, -STORE, 47243677982720, 47243678035967, -STORE, 47243678052352, 47243678056447, -STORE, 47243678035968, 47243678052351, -ERASE, 47243678035968, 47243678035968, -STORE, 47243678035968, 47243678052351, -STORE, 47243678064640, 47243678072831, -STORE, 47243678056448, 47243678064639, -ERASE, 47243678056448, 47243678056448, -STORE, 47243678056448, 47243678064639, -ERASE, 47243678064640, 47243678064640, -STORE, 47243678064640, 47243678072831, -STORE, 47243678072832, 47243679911935, -STORE, 47243678212096, 47243679911935, -STORE, 47243678072832, 47243678212095, -ERASE, 47243678212096, 47243678212096, -STORE, 47243678212096, 47243679870975, -STORE, 47243679870976, 47243679911935, -STORE, 47243679555584, 47243679870975, -STORE, 47243678212096, 47243679555583, -ERASE, 47243678212096, 47243678212096, -STORE, 47243678212096, 47243679555583, -STORE, 47243679866880, 47243679870975, -STORE, 47243679555584, 47243679866879, -ERASE, 47243679555584, 47243679555584, -STORE, 47243679555584, 47243679866879, -STORE, 47243679895552, 47243679911935, -STORE, 47243679870976, 47243679895551, -ERASE, 47243679870976, 47243679870976, -STORE, 47243679870976, 47243679895551, -ERASE, 47243679895552, 47243679895552, -STORE, 47243679895552, 47243679911935, -STORE, 47243679895552, 47243679924223, -ERASE, 47243679870976, 47243679870976, -STORE, 47243679870976, 47243679887359, -STORE, 47243679887360, 47243679895551, -ERASE, 47243678056448, 47243678056448, -STORE, 47243678056448, 47243678060543, -STORE, 47243678060544, 47243678064639, -ERASE, 94448916905984, 94448916905984, -STORE, 94448916905984, 94448916922367, -STORE, 94448916922368, 94448916926463, -ERASE, 140389117206528, 140389117206528, -STORE, 140389117206528, 140389117210623, -STORE, 140389117210624, 140389117214719, -ERASE, 47243677949952, 47243677949952, -STORE, 140737488347136, 140737488351231, -STORE, 140733068505088, 140737488351231, -ERASE, 140733068505088, 140733068505088, -STORE, 140733068505088, 140733068509183, -STORE, 94207145750528, 94207146463231, -ERASE, 94207145750528, 94207145750528, -STORE, 94207145750528, 94207145799679, -STORE, 94207145799680, 94207146463231, -ERASE, 94207145799680, 94207145799680, -STORE, 94207145799680, 94207146344447, -STORE, 94207146344448, 94207146442751, -STORE, 94207146442752, 94207146463231, -STORE, 140684504911872, 140684505083903, -ERASE, 140684504911872, 140684504911872, -STORE, 140684504911872, 140684504915967, -STORE, 140684504915968, 140684505083903, -ERASE, 140684504915968, 140684504915968, -STORE, 140684504915968, 140684505038847, -STORE, 140684505038848, 140684505071615, -STORE, 140684505071616, 140684505079807, -STORE, 140684505079808, 140684505083903, -STORE, 140733068607488, 140733068611583, -STORE, 140733068595200, 140733068607487, -STORE, 46948290084864, 46948290093055, -STORE, 46948290093056, 46948290101247, -STORE, 46948290101248, 46948290207743, -STORE, 46948290117632, 46948290207743, -STORE, 46948290101248, 46948290117631, -ERASE, 46948290117632, 46948290117632, -STORE, 46948290117632, 46948290191359, -STORE, 46948290191360, 46948290207743, -STORE, 46948290170880, 46948290191359, -STORE, 46948290117632, 46948290170879, -ERASE, 46948290117632, 46948290117632, -STORE, 46948290117632, 46948290170879, -STORE, 46948290187264, 46948290191359, -STORE, 46948290170880, 46948290187263, -ERASE, 46948290170880, 46948290170880, -STORE, 46948290170880, 46948290187263, -STORE, 46948290199552, 46948290207743, -STORE, 46948290191360, 46948290199551, -ERASE, 46948290191360, 46948290191360, -STORE, 46948290191360, 46948290199551, -ERASE, 46948290199552, 46948290199552, -STORE, 46948290199552, 46948290207743, -STORE, 46948290207744, 46948292046847, -STORE, 46948290347008, 46948292046847, -STORE, 46948290207744, 46948290347007, -ERASE, 46948290347008, 46948290347008, -STORE, 46948290347008, 46948292005887, -STORE, 46948292005888, 46948292046847, -STORE, 46948291690496, 46948292005887, -STORE, 46948290347008, 46948291690495, -ERASE, 46948290347008, 46948290347008, -STORE, 46948290347008, 46948291690495, -STORE, 46948292001792, 46948292005887, -STORE, 46948291690496, 46948292001791, -ERASE, 46948291690496, 46948291690496, -STORE, 46948291690496, 46948292001791, -STORE, 46948292030464, 46948292046847, -STORE, 46948292005888, 46948292030463, -ERASE, 46948292005888, 46948292005888, -STORE, 46948292005888, 46948292030463, -ERASE, 46948292030464, 46948292030464, -STORE, 46948292030464, 46948292046847, -STORE, 46948292030464, 46948292059135, -ERASE, 46948292005888, 46948292005888, -STORE, 46948292005888, 46948292022271, -STORE, 46948292022272, 46948292030463, -ERASE, 46948290191360, 46948290191360, -STORE, 46948290191360, 46948290195455, -STORE, 46948290195456, 46948290199551, -ERASE, 94207146442752, 94207146442752, -STORE, 94207146442752, 94207146459135, -STORE, 94207146459136, 94207146463231, -ERASE, 140684505071616, 140684505071616, -STORE, 140684505071616, 140684505075711, -STORE, 140684505075712, 140684505079807, -ERASE, 46948290084864, 46948290084864, -STORE, 140737488347136, 140737488351231, -STORE, 140726367158272, 140737488351231, -ERASE, 140726367158272, 140726367158272, -STORE, 140726367158272, 140726367162367, -STORE, 94436124106752, 94436124819455, -ERASE, 94436124106752, 94436124106752, -STORE, 94436124106752, 94436124155903, -STORE, 94436124155904, 94436124819455, -ERASE, 94436124155904, 94436124155904, -STORE, 94436124155904, 94436124700671, -STORE, 94436124700672, 94436124798975, -STORE, 94436124798976, 94436124819455, -STORE, 140049025044480, 140049025216511, -ERASE, 140049025044480, 140049025044480, -STORE, 140049025044480, 140049025048575, -STORE, 140049025048576, 140049025216511, -ERASE, 140049025048576, 140049025048576, -STORE, 140049025048576, 140049025171455, -STORE, 140049025171456, 140049025204223, -STORE, 140049025204224, 140049025212415, -STORE, 140049025212416, 140049025216511, -STORE, 140726367256576, 140726367260671, -STORE, 140726367244288, 140726367256575, -STORE, 47583769952256, 47583769960447, -STORE, 47583769960448, 47583769968639, -STORE, 47583769968640, 47583770075135, -STORE, 47583769985024, 47583770075135, -STORE, 47583769968640, 47583769985023, -ERASE, 47583769985024, 47583769985024, -STORE, 47583769985024, 47583770058751, -STORE, 47583770058752, 47583770075135, -STORE, 47583770038272, 47583770058751, -STORE, 47583769985024, 47583770038271, -ERASE, 47583769985024, 47583769985024, -STORE, 47583769985024, 47583770038271, -STORE, 47583770054656, 47583770058751, -STORE, 47583770038272, 47583770054655, -ERASE, 47583770038272, 47583770038272, -STORE, 47583770038272, 47583770054655, -STORE, 47583770066944, 47583770075135, -STORE, 47583770058752, 47583770066943, -ERASE, 47583770058752, 47583770058752, -STORE, 47583770058752, 47583770066943, -ERASE, 47583770066944, 47583770066944, -STORE, 47583770066944, 47583770075135, -STORE, 47583770075136, 47583771914239, -STORE, 47583770214400, 47583771914239, -STORE, 47583770075136, 47583770214399, -ERASE, 47583770214400, 47583770214400, -STORE, 47583770214400, 47583771873279, -STORE, 47583771873280, 47583771914239, -STORE, 47583771557888, 47583771873279, -STORE, 47583770214400, 47583771557887, -ERASE, 47583770214400, 47583770214400, -STORE, 47583770214400, 47583771557887, -STORE, 47583771869184, 47583771873279, -STORE, 47583771557888, 47583771869183, -ERASE, 47583771557888, 47583771557888, -STORE, 47583771557888, 47583771869183, -STORE, 47583771897856, 47583771914239, -STORE, 47583771873280, 47583771897855, -ERASE, 47583771873280, 47583771873280, -STORE, 47583771873280, 47583771897855, -ERASE, 47583771897856, 47583771897856, -STORE, 47583771897856, 47583771914239, -STORE, 47583771897856, 47583771926527, -ERASE, 47583771873280, 47583771873280, -STORE, 47583771873280, 47583771889663, -STORE, 47583771889664, 47583771897855, -ERASE, 47583770058752, 47583770058752, -STORE, 47583770058752, 47583770062847, -STORE, 47583770062848, 47583770066943, -ERASE, 94436124798976, 94436124798976, -STORE, 94436124798976, 94436124815359, -STORE, 94436124815360, 94436124819455, -ERASE, 140049025204224, 140049025204224, -STORE, 140049025204224, 140049025208319, -STORE, 140049025208320, 140049025212415, -ERASE, 47583769952256, 47583769952256, -STORE, 140737488347136, 140737488351231, -STORE, 140727116099584, 140737488351231, -ERASE, 140727116099584, 140727116099584, -STORE, 140727116099584, 140727116103679, -STORE, 94166319734784, 94166320447487, -ERASE, 94166319734784, 94166319734784, -STORE, 94166319734784, 94166319783935, -STORE, 94166319783936, 94166320447487, -ERASE, 94166319783936, 94166319783936, -STORE, 94166319783936, 94166320328703, -STORE, 94166320328704, 94166320427007, -STORE, 94166320427008, 94166320447487, -STORE, 139976559542272, 139976559714303, -ERASE, 139976559542272, 139976559542272, -STORE, 139976559542272, 139976559546367, -STORE, 139976559546368, 139976559714303, -ERASE, 139976559546368, 139976559546368, -STORE, 139976559546368, 139976559669247, -STORE, 139976559669248, 139976559702015, -STORE, 139976559702016, 139976559710207, -STORE, 139976559710208, 139976559714303, -STORE, 140727116222464, 140727116226559, -STORE, 140727116210176, 140727116222463, -STORE, 47656235454464, 47656235462655, -STORE, 47656235462656, 47656235470847, -STORE, 47656235470848, 47656235577343, -STORE, 47656235487232, 47656235577343, -STORE, 47656235470848, 47656235487231, -ERASE, 47656235487232, 47656235487232, -STORE, 47656235487232, 47656235560959, -STORE, 47656235560960, 47656235577343, -STORE, 47656235540480, 47656235560959, -STORE, 47656235487232, 47656235540479, -ERASE, 47656235487232, 47656235487232, -STORE, 47656235487232, 47656235540479, -STORE, 47656235556864, 47656235560959, -STORE, 47656235540480, 47656235556863, -ERASE, 47656235540480, 47656235540480, -STORE, 47656235540480, 47656235556863, -STORE, 47656235569152, 47656235577343, -STORE, 47656235560960, 47656235569151, -ERASE, 47656235560960, 47656235560960, -STORE, 47656235560960, 47656235569151, -ERASE, 47656235569152, 47656235569152, -STORE, 47656235569152, 47656235577343, -STORE, 47656235577344, 47656237416447, -STORE, 47656235716608, 47656237416447, -STORE, 47656235577344, 47656235716607, -ERASE, 47656235716608, 47656235716608, -STORE, 47656235716608, 47656237375487, -STORE, 47656237375488, 47656237416447, -STORE, 47656237060096, 47656237375487, -STORE, 47656235716608, 47656237060095, -ERASE, 47656235716608, 47656235716608, -STORE, 47656235716608, 47656237060095, -STORE, 47656237371392, 47656237375487, -STORE, 47656237060096, 47656237371391, -ERASE, 47656237060096, 47656237060096, -STORE, 47656237060096, 47656237371391, -STORE, 47656237400064, 47656237416447, -STORE, 47656237375488, 47656237400063, -ERASE, 47656237375488, 47656237375488, -STORE, 47656237375488, 47656237400063, -ERASE, 47656237400064, 47656237400064, -STORE, 47656237400064, 47656237416447, -STORE, 47656237400064, 47656237428735, -ERASE, 47656237375488, 47656237375488, -STORE, 47656237375488, 47656237391871, -STORE, 47656237391872, 47656237400063, -ERASE, 47656235560960, 47656235560960, -STORE, 47656235560960, 47656235565055, -STORE, 47656235565056, 47656235569151, -ERASE, 94166320427008, 94166320427008, -STORE, 94166320427008, 94166320443391, -STORE, 94166320443392, 94166320447487, -ERASE, 139976559702016, 139976559702016, -STORE, 139976559702016, 139976559706111, -STORE, 139976559706112, 139976559710207, -ERASE, 47656235454464, 47656235454464, -STORE, 94166332153856, 94166332289023, -STORE, 140737488347136, 140737488351231, -STORE, 140726412816384, 140737488351231, -ERASE, 140726412816384, 140726412816384, -STORE, 140726412816384, 140726412820479, -STORE, 94094884507648, 94094885220351, -ERASE, 94094884507648, 94094884507648, -STORE, 94094884507648, 94094884556799, -STORE, 94094884556800, 94094885220351, -ERASE, 94094884556800, 94094884556800, -STORE, 94094884556800, 94094885101567, -STORE, 94094885101568, 94094885199871, -STORE, 94094885199872, 94094885220351, -STORE, 139773773938688, 139773774110719, -ERASE, 139773773938688, 139773773938688, -STORE, 139773773938688, 139773773942783, -STORE, 139773773942784, 139773774110719, -ERASE, 139773773942784, 139773773942784, -STORE, 139773773942784, 139773774065663, -STORE, 139773774065664, 139773774098431, -STORE, 139773774098432, 139773774106623, -STORE, 139773774106624, 139773774110719, -STORE, 140726412963840, 140726412967935, -STORE, 140726412951552, 140726412963839, -STORE, 47859021058048, 47859021066239, -STORE, 47859021066240, 47859021074431, -STORE, 47859021074432, 47859021180927, -STORE, 47859021090816, 47859021180927, -STORE, 47859021074432, 47859021090815, -ERASE, 47859021090816, 47859021090816, -STORE, 47859021090816, 47859021164543, -STORE, 47859021164544, 47859021180927, -STORE, 47859021144064, 47859021164543, -STORE, 47859021090816, 47859021144063, -ERASE, 47859021090816, 47859021090816, -STORE, 47859021090816, 47859021144063, -STORE, 47859021160448, 47859021164543, -STORE, 47859021144064, 47859021160447, -ERASE, 47859021144064, 47859021144064, -STORE, 47859021144064, 47859021160447, -STORE, 47859021172736, 47859021180927, -STORE, 47859021164544, 47859021172735, -ERASE, 47859021164544, 47859021164544, -STORE, 47859021164544, 47859021172735, -ERASE, 47859021172736, 47859021172736, -STORE, 47859021172736, 47859021180927, -STORE, 47859021180928, 47859023020031, -STORE, 47859021320192, 47859023020031, -STORE, 47859021180928, 47859021320191, -ERASE, 47859021320192, 47859021320192, -STORE, 47859021320192, 47859022979071, -STORE, 47859022979072, 47859023020031, -STORE, 47859022663680, 47859022979071, -STORE, 47859021320192, 47859022663679, -ERASE, 47859021320192, 47859021320192, -STORE, 47859021320192, 47859022663679, -STORE, 47859022974976, 47859022979071, -STORE, 47859022663680, 47859022974975, -ERASE, 47859022663680, 47859022663680, -STORE, 47859022663680, 47859022974975, -STORE, 47859023003648, 47859023020031, -STORE, 47859022979072, 47859023003647, -ERASE, 47859022979072, 47859022979072, -STORE, 47859022979072, 47859023003647, -ERASE, 47859023003648, 47859023003648, -STORE, 47859023003648, 47859023020031, -STORE, 47859023003648, 47859023032319, -ERASE, 47859022979072, 47859022979072, -STORE, 47859022979072, 47859022995455, -STORE, 47859022995456, 47859023003647, -ERASE, 47859021164544, 47859021164544, -STORE, 47859021164544, 47859021168639, -STORE, 47859021168640, 47859021172735, -ERASE, 94094885199872, 94094885199872, -STORE, 94094885199872, 94094885216255, -STORE, 94094885216256, 94094885220351, -ERASE, 139773774098432, 139773774098432, -STORE, 139773774098432, 139773774102527, -STORE, 139773774102528, 139773774106623, -ERASE, 47859021058048, 47859021058048, -STORE, 94094901108736, 94094901243903, -STORE, 140737488347136, 140737488351231, -STORE, 140736567963648, 140737488351231, -ERASE, 140736567963648, 140736567963648, -STORE, 140736567963648, 140736567967743, -STORE, 94924425748480, 94924426461183, -ERASE, 94924425748480, 94924425748480, -STORE, 94924425748480, 94924425797631, -STORE, 94924425797632, 94924426461183, -ERASE, 94924425797632, 94924425797632, -STORE, 94924425797632, 94924426342399, -STORE, 94924426342400, 94924426440703, -STORE, 94924426440704, 94924426461183, -STORE, 140042126319616, 140042126491647, -ERASE, 140042126319616, 140042126319616, -STORE, 140042126319616, 140042126323711, -STORE, 140042126323712, 140042126491647, -ERASE, 140042126323712, 140042126323712, -STORE, 140042126323712, 140042126446591, -STORE, 140042126446592, 140042126479359, -STORE, 140042126479360, 140042126487551, -STORE, 140042126487552, 140042126491647, -STORE, 140736568672256, 140736568676351, -STORE, 140736568659968, 140736568672255, -STORE, 47590668677120, 47590668685311, -STORE, 47590668685312, 47590668693503, -STORE, 47590668693504, 47590668799999, -STORE, 47590668709888, 47590668799999, -STORE, 47590668693504, 47590668709887, -ERASE, 47590668709888, 47590668709888, -STORE, 47590668709888, 47590668783615, -STORE, 47590668783616, 47590668799999, -STORE, 47590668763136, 47590668783615, -STORE, 47590668709888, 47590668763135, -ERASE, 47590668709888, 47590668709888, -STORE, 47590668709888, 47590668763135, -STORE, 47590668779520, 47590668783615, -STORE, 47590668763136, 47590668779519, -ERASE, 47590668763136, 47590668763136, -STORE, 47590668763136, 47590668779519, -STORE, 47590668791808, 47590668799999, -STORE, 47590668783616, 47590668791807, -ERASE, 47590668783616, 47590668783616, -STORE, 47590668783616, 47590668791807, -ERASE, 47590668791808, 47590668791808, -STORE, 47590668791808, 47590668799999, -STORE, 47590668800000, 47590670639103, -STORE, 47590668939264, 47590670639103, -STORE, 47590668800000, 47590668939263, -ERASE, 47590668939264, 47590668939264, -STORE, 47590668939264, 47590670598143, -STORE, 47590670598144, 47590670639103, -STORE, 47590670282752, 47590670598143, -STORE, 47590668939264, 47590670282751, -ERASE, 47590668939264, 47590668939264, -STORE, 47590668939264, 47590670282751, -STORE, 47590670594048, 47590670598143, -STORE, 47590670282752, 47590670594047, -ERASE, 47590670282752, 47590670282752, -STORE, 47590670282752, 47590670594047, -STORE, 47590670622720, 47590670639103, -STORE, 47590670598144, 47590670622719, -ERASE, 47590670598144, 47590670598144, -STORE, 47590670598144, 47590670622719, -ERASE, 47590670622720, 47590670622720, -STORE, 47590670622720, 47590670639103, -STORE, 47590670622720, 47590670651391, -ERASE, 47590670598144, 47590670598144, -STORE, 47590670598144, 47590670614527, -STORE, 47590670614528, 47590670622719, -ERASE, 47590668783616, 47590668783616, -STORE, 47590668783616, 47590668787711, -STORE, 47590668787712, 47590668791807, -ERASE, 94924426440704, 94924426440704, -STORE, 94924426440704, 94924426457087, -STORE, 94924426457088, 94924426461183, -ERASE, 140042126479360, 140042126479360, -STORE, 140042126479360, 140042126483455, -STORE, 140042126483456, 140042126487551, -ERASE, 47590668677120, 47590668677120, -STORE, 140737488347136, 140737488351231, -STORE, 140733281439744, 140737488351231, -ERASE, 140733281439744, 140733281439744, -STORE, 140733281439744, 140733281443839, -STORE, 94490667069440, 94490667782143, -ERASE, 94490667069440, 94490667069440, -STORE, 94490667069440, 94490667118591, -STORE, 94490667118592, 94490667782143, -ERASE, 94490667118592, 94490667118592, -STORE, 94490667118592, 94490667663359, -STORE, 94490667663360, 94490667761663, -STORE, 94490667761664, 94490667782143, -STORE, 139878215118848, 139878215290879, -ERASE, 139878215118848, 139878215118848, -STORE, 139878215118848, 139878215122943, -STORE, 139878215122944, 139878215290879, -ERASE, 139878215122944, 139878215122944, -STORE, 139878215122944, 139878215245823, -STORE, 139878215245824, 139878215278591, -STORE, 139878215278592, 139878215286783, -STORE, 139878215286784, 139878215290879, -STORE, 140733281464320, 140733281468415, -STORE, 140733281452032, 140733281464319, -STORE, 47754579877888, 47754579886079, -STORE, 47754579886080, 47754579894271, -STORE, 47754579894272, 47754580000767, -STORE, 47754579910656, 47754580000767, -STORE, 47754579894272, 47754579910655, -ERASE, 47754579910656, 47754579910656, -STORE, 47754579910656, 47754579984383, -STORE, 47754579984384, 47754580000767, -STORE, 47754579963904, 47754579984383, -STORE, 47754579910656, 47754579963903, -ERASE, 47754579910656, 47754579910656, -STORE, 47754579910656, 47754579963903, -STORE, 47754579980288, 47754579984383, -STORE, 47754579963904, 47754579980287, -ERASE, 47754579963904, 47754579963904, -STORE, 47754579963904, 47754579980287, -STORE, 47754579992576, 47754580000767, -STORE, 47754579984384, 47754579992575, -ERASE, 47754579984384, 47754579984384, -STORE, 47754579984384, 47754579992575, -ERASE, 47754579992576, 47754579992576, -STORE, 47754579992576, 47754580000767, -STORE, 47754580000768, 47754581839871, -STORE, 47754580140032, 47754581839871, -STORE, 47754580000768, 47754580140031, -ERASE, 47754580140032, 47754580140032, -STORE, 47754580140032, 47754581798911, -STORE, 47754581798912, 47754581839871, -STORE, 47754581483520, 47754581798911, -STORE, 47754580140032, 47754581483519, -ERASE, 47754580140032, 47754580140032, -STORE, 47754580140032, 47754581483519, -STORE, 47754581794816, 47754581798911, -STORE, 47754581483520, 47754581794815, -ERASE, 47754581483520, 47754581483520, -STORE, 47754581483520, 47754581794815, -STORE, 47754581823488, 47754581839871, -STORE, 47754581798912, 47754581823487, -ERASE, 47754581798912, 47754581798912, -STORE, 47754581798912, 47754581823487, -ERASE, 47754581823488, 47754581823488, -STORE, 47754581823488, 47754581839871, -STORE, 47754581823488, 47754581852159, -ERASE, 47754581798912, 47754581798912, -STORE, 47754581798912, 47754581815295, -STORE, 47754581815296, 47754581823487, -ERASE, 47754579984384, 47754579984384, -STORE, 47754579984384, 47754579988479, -STORE, 47754579988480, 47754579992575, -ERASE, 94490667761664, 94490667761664, -STORE, 94490667761664, 94490667778047, -STORE, 94490667778048, 94490667782143, -ERASE, 139878215278592, 139878215278592, -STORE, 139878215278592, 139878215282687, -STORE, 139878215282688, 139878215286783, -ERASE, 47754579877888, 47754579877888, -STORE, 94490669649920, 94490669785087, -STORE, 140737488347136, 140737488351231, -STORE, 140735382188032, 140737488351231, -ERASE, 140735382188032, 140735382188032, -STORE, 140735382188032, 140735382192127, -STORE, 94150181302272, 94150182014975, -ERASE, 94150181302272, 94150181302272, -STORE, 94150181302272, 94150181351423, -STORE, 94150181351424, 94150182014975, -ERASE, 94150181351424, 94150181351424, -STORE, 94150181351424, 94150181896191, -STORE, 94150181896192, 94150181994495, -STORE, 94150181994496, 94150182014975, -STORE, 139679752458240, 139679752630271, -ERASE, 139679752458240, 139679752458240, -STORE, 139679752458240, 139679752462335, -STORE, 139679752462336, 139679752630271, -ERASE, 139679752462336, 139679752462336, -STORE, 139679752462336, 139679752585215, -STORE, 139679752585216, 139679752617983, -STORE, 139679752617984, 139679752626175, -STORE, 139679752626176, 139679752630271, -STORE, 140735382536192, 140735382540287, -STORE, 140735382523904, 140735382536191, -STORE, 47953042538496, 47953042546687, -STORE, 47953042546688, 47953042554879, -STORE, 47953042554880, 47953042661375, -STORE, 47953042571264, 47953042661375, -STORE, 47953042554880, 47953042571263, -ERASE, 47953042571264, 47953042571264, -STORE, 47953042571264, 47953042644991, -STORE, 47953042644992, 47953042661375, -STORE, 47953042624512, 47953042644991, -STORE, 47953042571264, 47953042624511, -ERASE, 47953042571264, 47953042571264, -STORE, 47953042571264, 47953042624511, -STORE, 47953042640896, 47953042644991, -STORE, 47953042624512, 47953042640895, -ERASE, 47953042624512, 47953042624512, -STORE, 47953042624512, 47953042640895, -STORE, 47953042653184, 47953042661375, -STORE, 47953042644992, 47953042653183, -ERASE, 47953042644992, 47953042644992, -STORE, 47953042644992, 47953042653183, -ERASE, 47953042653184, 47953042653184, -STORE, 47953042653184, 47953042661375, -STORE, 47953042661376, 47953044500479, -STORE, 47953042800640, 47953044500479, -STORE, 47953042661376, 47953042800639, -ERASE, 47953042800640, 47953042800640, -STORE, 47953042800640, 47953044459519, -STORE, 47953044459520, 47953044500479, -STORE, 47953044144128, 47953044459519, -STORE, 47953042800640, 47953044144127, -ERASE, 47953042800640, 47953042800640, -STORE, 47953042800640, 47953044144127, -STORE, 47953044455424, 47953044459519, -STORE, 47953044144128, 47953044455423, -ERASE, 47953044144128, 47953044144128, -STORE, 47953044144128, 47953044455423, -STORE, 47953044484096, 47953044500479, -STORE, 47953044459520, 47953044484095, -ERASE, 47953044459520, 47953044459520, -STORE, 47953044459520, 47953044484095, -ERASE, 47953044484096, 47953044484096, -STORE, 47953044484096, 47953044500479, -STORE, 47953044484096, 47953044512767, -ERASE, 47953044459520, 47953044459520, -STORE, 47953044459520, 47953044475903, -STORE, 47953044475904, 47953044484095, -ERASE, 47953042644992, 47953042644992, -STORE, 47953042644992, 47953042649087, -STORE, 47953042649088, 47953042653183, -ERASE, 94150181994496, 94150181994496, -STORE, 94150181994496, 94150182010879, -STORE, 94150182010880, 94150182014975, -ERASE, 139679752617984, 139679752617984, -STORE, 139679752617984, 139679752622079, -STORE, 139679752622080, 139679752626175, -ERASE, 47953042538496, 47953042538496, -STORE, 140737488347136, 140737488351231, -STORE, 140737044123648, 140737488351231, -ERASE, 140737044123648, 140737044123648, -STORE, 140737044123648, 140737044127743, -STORE, 94425324294144, 94425325006847, -ERASE, 94425324294144, 94425324294144, -STORE, 94425324294144, 94425324343295, -STORE, 94425324343296, 94425325006847, -ERASE, 94425324343296, 94425324343296, -STORE, 94425324343296, 94425324888063, -STORE, 94425324888064, 94425324986367, -STORE, 94425324986368, 94425325006847, -STORE, 140382015016960, 140382015188991, -ERASE, 140382015016960, 140382015016960, -STORE, 140382015016960, 140382015021055, -STORE, 140382015021056, 140382015188991, -ERASE, 140382015021056, 140382015021056, -STORE, 140382015021056, 140382015143935, -STORE, 140382015143936, 140382015176703, -STORE, 140382015176704, 140382015184895, -STORE, 140382015184896, 140382015188991, -STORE, 140737045585920, 140737045590015, -STORE, 140737045573632, 140737045585919, -STORE, 47250779979776, 47250779987967, -STORE, 47250779987968, 47250779996159, -STORE, 47250779996160, 47250780102655, -STORE, 47250780012544, 47250780102655, -STORE, 47250779996160, 47250780012543, -ERASE, 47250780012544, 47250780012544, -STORE, 47250780012544, 47250780086271, -STORE, 47250780086272, 47250780102655, -STORE, 47250780065792, 47250780086271, -STORE, 47250780012544, 47250780065791, -ERASE, 47250780012544, 47250780012544, -STORE, 47250780012544, 47250780065791, -STORE, 47250780082176, 47250780086271, -STORE, 47250780065792, 47250780082175, -ERASE, 47250780065792, 47250780065792, -STORE, 47250780065792, 47250780082175, -STORE, 47250780094464, 47250780102655, -STORE, 47250780086272, 47250780094463, -ERASE, 47250780086272, 47250780086272, -STORE, 47250780086272, 47250780094463, -ERASE, 47250780094464, 47250780094464, -STORE, 47250780094464, 47250780102655, -STORE, 47250780102656, 47250781941759, -STORE, 47250780241920, 47250781941759, -STORE, 47250780102656, 47250780241919, -ERASE, 47250780241920, 47250780241920, -STORE, 47250780241920, 47250781900799, -STORE, 47250781900800, 47250781941759, -STORE, 47250781585408, 47250781900799, -STORE, 47250780241920, 47250781585407, -ERASE, 47250780241920, 47250780241920, -STORE, 47250780241920, 47250781585407, -STORE, 47250781896704, 47250781900799, -STORE, 47250781585408, 47250781896703, -ERASE, 47250781585408, 47250781585408, -STORE, 47250781585408, 47250781896703, -STORE, 47250781925376, 47250781941759, -STORE, 47250781900800, 47250781925375, -ERASE, 47250781900800, 47250781900800, -STORE, 47250781900800, 47250781925375, -ERASE, 47250781925376, 47250781925376, -STORE, 47250781925376, 47250781941759, -STORE, 47250781925376, 47250781954047, -ERASE, 47250781900800, 47250781900800, -STORE, 47250781900800, 47250781917183, -STORE, 47250781917184, 47250781925375, -ERASE, 47250780086272, 47250780086272, -STORE, 47250780086272, 47250780090367, -STORE, 47250780090368, 47250780094463, -ERASE, 94425324986368, 94425324986368, -STORE, 94425324986368, 94425325002751, -STORE, 94425325002752, 94425325006847, -ERASE, 140382015176704, 140382015176704, -STORE, 140382015176704, 140382015180799, -STORE, 140382015180800, 140382015184895, -ERASE, 47250779979776, 47250779979776, -STORE, 94425351438336, 94425351573503, -STORE, 140737488347136, 140737488351231, -STORE, 140736801144832, 140737488351231, -ERASE, 140736801144832, 140736801144832, -STORE, 140736801144832, 140736801148927, -STORE, 94629429358592, 94629430071295, -ERASE, 94629429358592, 94629429358592, -STORE, 94629429358592, 94629429407743, -STORE, 94629429407744, 94629430071295, -ERASE, 94629429407744, 94629429407744, -STORE, 94629429407744, 94629429952511, -STORE, 94629429952512, 94629430050815, -STORE, 94629430050816, 94629430071295, -STORE, 139801685483520, 139801685655551, -ERASE, 139801685483520, 139801685483520, -STORE, 139801685483520, 139801685487615, -STORE, 139801685487616, 139801685655551, -ERASE, 139801685487616, 139801685487616, -STORE, 139801685487616, 139801685610495, -STORE, 139801685610496, 139801685643263, -STORE, 139801685643264, 139801685651455, -STORE, 139801685651456, 139801685655551, -STORE, 140736801198080, 140736801202175, -STORE, 140736801185792, 140736801198079, -STORE, 47831109513216, 47831109521407, -STORE, 47831109521408, 47831109529599, -STORE, 47831109529600, 47831109636095, -STORE, 47831109545984, 47831109636095, -STORE, 47831109529600, 47831109545983, -ERASE, 47831109545984, 47831109545984, -STORE, 47831109545984, 47831109619711, -STORE, 47831109619712, 47831109636095, -STORE, 47831109599232, 47831109619711, -STORE, 47831109545984, 47831109599231, -ERASE, 47831109545984, 47831109545984, -STORE, 47831109545984, 47831109599231, -STORE, 47831109615616, 47831109619711, -STORE, 47831109599232, 47831109615615, -ERASE, 47831109599232, 47831109599232, -STORE, 47831109599232, 47831109615615, -STORE, 47831109627904, 47831109636095, -STORE, 47831109619712, 47831109627903, -ERASE, 47831109619712, 47831109619712, -STORE, 47831109619712, 47831109627903, -ERASE, 47831109627904, 47831109627904, -STORE, 47831109627904, 47831109636095, -STORE, 47831109636096, 47831111475199, -STORE, 47831109775360, 47831111475199, -STORE, 47831109636096, 47831109775359, -ERASE, 47831109775360, 47831109775360, -STORE, 47831109775360, 47831111434239, -STORE, 47831111434240, 47831111475199, -STORE, 47831111118848, 47831111434239, -STORE, 47831109775360, 47831111118847, -ERASE, 47831109775360, 47831109775360, -STORE, 47831109775360, 47831111118847, -STORE, 47831111430144, 47831111434239, -STORE, 47831111118848, 47831111430143, -ERASE, 47831111118848, 47831111118848, -STORE, 47831111118848, 47831111430143, -STORE, 47831111458816, 47831111475199, -STORE, 47831111434240, 47831111458815, -ERASE, 47831111434240, 47831111434240, -STORE, 47831111434240, 47831111458815, -ERASE, 47831111458816, 47831111458816, -STORE, 47831111458816, 47831111475199, -STORE, 47831111458816, 47831111487487, -ERASE, 47831111434240, 47831111434240, -STORE, 47831111434240, 47831111450623, -STORE, 47831111450624, 47831111458815, -ERASE, 47831109619712, 47831109619712, -STORE, 47831109619712, 47831109623807, -STORE, 47831109623808, 47831109627903, -ERASE, 94629430050816, 94629430050816, -STORE, 94629430050816, 94629430067199, -STORE, 94629430067200, 94629430071295, -ERASE, 139801685643264, 139801685643264, -STORE, 139801685643264, 139801685647359, -STORE, 139801685647360, 139801685651455, -ERASE, 47831109513216, 47831109513216, -STORE, 140737488347136, 140737488351231, -STORE, 140729419612160, 140737488351231, -ERASE, 140729419612160, 140729419612160, -STORE, 140729419612160, 140729419616255, -STORE, 94443354148864, 94443354861567, -ERASE, 94443354148864, 94443354148864, -STORE, 94443354148864, 94443354198015, -STORE, 94443354198016, 94443354861567, -ERASE, 94443354198016, 94443354198016, -STORE, 94443354198016, 94443354742783, -STORE, 94443354742784, 94443354841087, -STORE, 94443354841088, 94443354861567, -STORE, 139741700038656, 139741700210687, -ERASE, 139741700038656, 139741700038656, -STORE, 139741700038656, 139741700042751, -STORE, 139741700042752, 139741700210687, -ERASE, 139741700042752, 139741700042752, -STORE, 139741700042752, 139741700165631, -STORE, 139741700165632, 139741700198399, -STORE, 139741700198400, 139741700206591, -STORE, 139741700206592, 139741700210687, -STORE, 140729420574720, 140729420578815, -STORE, 140729420562432, 140729420574719, -STORE, 47891094958080, 47891094966271, -STORE, 47891094966272, 47891094974463, -STORE, 47891094974464, 47891095080959, -STORE, 47891094990848, 47891095080959, -STORE, 47891094974464, 47891094990847, -ERASE, 47891094990848, 47891094990848, -STORE, 47891094990848, 47891095064575, -STORE, 47891095064576, 47891095080959, -STORE, 47891095044096, 47891095064575, -STORE, 47891094990848, 47891095044095, -ERASE, 47891094990848, 47891094990848, -STORE, 47891094990848, 47891095044095, -STORE, 47891095060480, 47891095064575, -STORE, 47891095044096, 47891095060479, -ERASE, 47891095044096, 47891095044096, -STORE, 47891095044096, 47891095060479, -STORE, 47891095072768, 47891095080959, -STORE, 47891095064576, 47891095072767, -ERASE, 47891095064576, 47891095064576, -STORE, 47891095064576, 47891095072767, -ERASE, 47891095072768, 47891095072768, -STORE, 47891095072768, 47891095080959, -STORE, 47891095080960, 47891096920063, -STORE, 47891095220224, 47891096920063, -STORE, 47891095080960, 47891095220223, -ERASE, 47891095220224, 47891095220224, -STORE, 47891095220224, 47891096879103, -STORE, 47891096879104, 47891096920063, -STORE, 47891096563712, 47891096879103, -STORE, 47891095220224, 47891096563711, -ERASE, 47891095220224, 47891095220224, -STORE, 47891095220224, 47891096563711, -STORE, 47891096875008, 47891096879103, -STORE, 47891096563712, 47891096875007, -ERASE, 47891096563712, 47891096563712, -STORE, 47891096563712, 47891096875007, -STORE, 47891096903680, 47891096920063, -STORE, 47891096879104, 47891096903679, -ERASE, 47891096879104, 47891096879104, -STORE, 47891096879104, 47891096903679, -ERASE, 47891096903680, 47891096903680, -STORE, 47891096903680, 47891096920063, -STORE, 47891096903680, 47891096932351, -ERASE, 47891096879104, 47891096879104, -STORE, 47891096879104, 47891096895487, -STORE, 47891096895488, 47891096903679, -ERASE, 47891095064576, 47891095064576, -STORE, 47891095064576, 47891095068671, -STORE, 47891095068672, 47891095072767, -ERASE, 94443354841088, 94443354841088, -STORE, 94443354841088, 94443354857471, -STORE, 94443354857472, 94443354861567, -ERASE, 139741700198400, 139741700198400, -STORE, 139741700198400, 139741700202495, -STORE, 139741700202496, 139741700206591, -ERASE, 47891094958080, 47891094958080, -STORE, 94443360825344, 94443360960511, -STORE, 140737488347136, 140737488351231, -STORE, 140722961661952, 140737488351231, -ERASE, 140722961661952, 140722961661952, -STORE, 140722961661952, 140722961666047, -STORE, 94878388944896, 94878389657599, -ERASE, 94878388944896, 94878388944896, -STORE, 94878388944896, 94878388994047, -STORE, 94878388994048, 94878389657599, -ERASE, 94878388994048, 94878388994048, -STORE, 94878388994048, 94878389538815, -STORE, 94878389538816, 94878389637119, -STORE, 94878389637120, 94878389657599, -STORE, 140210690056192, 140210690228223, -ERASE, 140210690056192, 140210690056192, -STORE, 140210690056192, 140210690060287, -STORE, 140210690060288, 140210690228223, -ERASE, 140210690060288, 140210690060288, -STORE, 140210690060288, 140210690183167, -STORE, 140210690183168, 140210690215935, -STORE, 140210690215936, 140210690224127, -STORE, 140210690224128, 140210690228223, -STORE, 140722963148800, 140722963152895, -STORE, 140722963136512, 140722963148799, -STORE, 47422104940544, 47422104948735, -STORE, 47422104948736, 47422104956927, -STORE, 47422104956928, 47422105063423, -STORE, 47422104973312, 47422105063423, -STORE, 47422104956928, 47422104973311, -ERASE, 47422104973312, 47422104973312, -STORE, 47422104973312, 47422105047039, -STORE, 47422105047040, 47422105063423, -STORE, 47422105026560, 47422105047039, -STORE, 47422104973312, 47422105026559, -ERASE, 47422104973312, 47422104973312, -STORE, 47422104973312, 47422105026559, -STORE, 47422105042944, 47422105047039, -STORE, 47422105026560, 47422105042943, -ERASE, 47422105026560, 47422105026560, -STORE, 47422105026560, 47422105042943, -STORE, 47422105055232, 47422105063423, -STORE, 47422105047040, 47422105055231, -ERASE, 47422105047040, 47422105047040, -STORE, 47422105047040, 47422105055231, -ERASE, 47422105055232, 47422105055232, -STORE, 47422105055232, 47422105063423, -STORE, 47422105063424, 47422106902527, -STORE, 47422105202688, 47422106902527, -STORE, 47422105063424, 47422105202687, -ERASE, 47422105202688, 47422105202688, -STORE, 47422105202688, 47422106861567, -STORE, 47422106861568, 47422106902527, -STORE, 47422106546176, 47422106861567, -STORE, 47422105202688, 47422106546175, -ERASE, 47422105202688, 47422105202688, -STORE, 47422105202688, 47422106546175, -STORE, 47422106857472, 47422106861567, -STORE, 47422106546176, 47422106857471, -ERASE, 47422106546176, 47422106546176, -STORE, 47422106546176, 47422106857471, -STORE, 47422106886144, 47422106902527, -STORE, 47422106861568, 47422106886143, -ERASE, 47422106861568, 47422106861568, -STORE, 47422106861568, 47422106886143, -ERASE, 47422106886144, 47422106886144, -STORE, 47422106886144, 47422106902527, -STORE, 47422106886144, 47422106914815, -ERASE, 47422106861568, 47422106861568, -STORE, 47422106861568, 47422106877951, -STORE, 47422106877952, 47422106886143, -ERASE, 47422105047040, 47422105047040, -STORE, 47422105047040, 47422105051135, -STORE, 47422105051136, 47422105055231, -ERASE, 94878389637120, 94878389637120, -STORE, 94878389637120, 94878389653503, -STORE, 94878389653504, 94878389657599, -ERASE, 140210690215936, 140210690215936, -STORE, 140210690215936, 140210690220031, -STORE, 140210690220032, 140210690224127, -ERASE, 47422104940544, 47422104940544, -STORE, 140737488347136, 140737488351231, -STORE, 140727690309632, 140737488351231, -ERASE, 140727690309632, 140727690309632, -STORE, 140727690309632, 140727690313727, -STORE, 94121892208640, 94121892921343, -ERASE, 94121892208640, 94121892208640, -STORE, 94121892208640, 94121892257791, -STORE, 94121892257792, 94121892921343, -ERASE, 94121892257792, 94121892257792, -STORE, 94121892257792, 94121892802559, -STORE, 94121892802560, 94121892900863, -STORE, 94121892900864, 94121892921343, -STORE, 140662438326272, 140662438498303, -ERASE, 140662438326272, 140662438326272, -STORE, 140662438326272, 140662438330367, -STORE, 140662438330368, 140662438498303, -ERASE, 140662438330368, 140662438330368, -STORE, 140662438330368, 140662438453247, -STORE, 140662438453248, 140662438486015, -STORE, 140662438486016, 140662438494207, -STORE, 140662438494208, 140662438498303, -STORE, 140727690379264, 140727690383359, -STORE, 140727690366976, 140727690379263, -STORE, 46970356670464, 46970356678655, -STORE, 46970356678656, 46970356686847, -STORE, 46970356686848, 46970356793343, -STORE, 46970356703232, 46970356793343, -STORE, 46970356686848, 46970356703231, -ERASE, 46970356703232, 46970356703232, -STORE, 46970356703232, 46970356776959, -STORE, 46970356776960, 46970356793343, -STORE, 46970356756480, 46970356776959, -STORE, 46970356703232, 46970356756479, -ERASE, 46970356703232, 46970356703232, -STORE, 46970356703232, 46970356756479, -STORE, 46970356772864, 46970356776959, -STORE, 46970356756480, 46970356772863, -ERASE, 46970356756480, 46970356756480, -STORE, 46970356756480, 46970356772863, -STORE, 46970356785152, 46970356793343, -STORE, 46970356776960, 46970356785151, -ERASE, 46970356776960, 46970356776960, -STORE, 46970356776960, 46970356785151, -ERASE, 46970356785152, 46970356785152, -STORE, 46970356785152, 46970356793343, -STORE, 46970356793344, 46970358632447, -STORE, 46970356932608, 46970358632447, -STORE, 46970356793344, 46970356932607, -ERASE, 46970356932608, 46970356932608, -STORE, 46970356932608, 46970358591487, -STORE, 46970358591488, 46970358632447, -STORE, 46970358276096, 46970358591487, -STORE, 46970356932608, 46970358276095, -ERASE, 46970356932608, 46970356932608, -STORE, 46970356932608, 46970358276095, -STORE, 46970358587392, 46970358591487, -STORE, 46970358276096, 46970358587391, -ERASE, 46970358276096, 46970358276096, -STORE, 46970358276096, 46970358587391, -STORE, 46970358616064, 46970358632447, -STORE, 46970358591488, 46970358616063, -ERASE, 46970358591488, 46970358591488, -STORE, 46970358591488, 46970358616063, -ERASE, 46970358616064, 46970358616064, -STORE, 46970358616064, 46970358632447, -STORE, 46970358616064, 46970358644735, -ERASE, 46970358591488, 46970358591488, -STORE, 46970358591488, 46970358607871, -STORE, 46970358607872, 46970358616063, -ERASE, 46970356776960, 46970356776960, -STORE, 46970356776960, 46970356781055, -STORE, 46970356781056, 46970356785151, -ERASE, 94121892900864, 94121892900864, -STORE, 94121892900864, 94121892917247, -STORE, 94121892917248, 94121892921343, -ERASE, 140662438486016, 140662438486016, -STORE, 140662438486016, 140662438490111, -STORE, 140662438490112, 140662438494207, -ERASE, 46970356670464, 46970356670464, -STORE, 94121898610688, 94121898745855, -STORE, 140737488347136, 140737488351231, -STORE, 140737189351424, 140737488351231, -ERASE, 140737189351424, 140737189351424, -STORE, 140737189351424, 140737189355519, -STORE, 93847948832768, 93847949545471, -ERASE, 93847948832768, 93847948832768, -STORE, 93847948832768, 93847948881919, -STORE, 93847948881920, 93847949545471, -ERASE, 93847948881920, 93847948881920, -STORE, 93847948881920, 93847949426687, -STORE, 93847949426688, 93847949524991, -STORE, 93847949524992, 93847949545471, -STORE, 139698989985792, 139698990157823, -ERASE, 139698989985792, 139698989985792, -STORE, 139698989985792, 139698989989887, -STORE, 139698989989888, 139698990157823, -ERASE, 139698989989888, 139698989989888, -STORE, 139698989989888, 139698990112767, -STORE, 139698990112768, 139698990145535, -STORE, 139698990145536, 139698990153727, -STORE, 139698990153728, 139698990157823, -STORE, 140737189744640, 140737189748735, -STORE, 140737189732352, 140737189744639, -STORE, 47933805010944, 47933805019135, -STORE, 47933805019136, 47933805027327, -STORE, 47933805027328, 47933805133823, -STORE, 47933805043712, 47933805133823, -STORE, 47933805027328, 47933805043711, -ERASE, 47933805043712, 47933805043712, -STORE, 47933805043712, 47933805117439, -STORE, 47933805117440, 47933805133823, -STORE, 47933805096960, 47933805117439, -STORE, 47933805043712, 47933805096959, -ERASE, 47933805043712, 47933805043712, -STORE, 47933805043712, 47933805096959, -STORE, 47933805113344, 47933805117439, -STORE, 47933805096960, 47933805113343, -ERASE, 47933805096960, 47933805096960, -STORE, 47933805096960, 47933805113343, -STORE, 47933805125632, 47933805133823, -STORE, 47933805117440, 47933805125631, -ERASE, 47933805117440, 47933805117440, -STORE, 47933805117440, 47933805125631, -ERASE, 47933805125632, 47933805125632, -STORE, 47933805125632, 47933805133823, -STORE, 47933805133824, 47933806972927, -STORE, 47933805273088, 47933806972927, -STORE, 47933805133824, 47933805273087, -ERASE, 47933805273088, 47933805273088, -STORE, 47933805273088, 47933806931967, -STORE, 47933806931968, 47933806972927, -STORE, 47933806616576, 47933806931967, -STORE, 47933805273088, 47933806616575, -ERASE, 47933805273088, 47933805273088, -STORE, 47933805273088, 47933806616575, -STORE, 47933806927872, 47933806931967, -STORE, 47933806616576, 47933806927871, -ERASE, 47933806616576, 47933806616576, -STORE, 47933806616576, 47933806927871, -STORE, 47933806956544, 47933806972927, -STORE, 47933806931968, 47933806956543, -ERASE, 47933806931968, 47933806931968, -STORE, 47933806931968, 47933806956543, -ERASE, 47933806956544, 47933806956544, -STORE, 47933806956544, 47933806972927, -STORE, 47933806956544, 47933806985215, -ERASE, 47933806931968, 47933806931968, -STORE, 47933806931968, 47933806948351, -STORE, 47933806948352, 47933806956543, -ERASE, 47933805117440, 47933805117440, -STORE, 47933805117440, 47933805121535, -STORE, 47933805121536, 47933805125631, -ERASE, 93847949524992, 93847949524992, -STORE, 93847949524992, 93847949541375, -STORE, 93847949541376, 93847949545471, -ERASE, 139698990145536, 139698990145536, -STORE, 139698990145536, 139698990149631, -STORE, 139698990149632, 139698990153727, -ERASE, 47933805010944, 47933805010944, -STORE, 140737488347136, 140737488351231, -STORE, 140725553991680, 140737488351231, -ERASE, 140725553991680, 140725553991680, -STORE, 140725553991680, 140725553995775, -STORE, 93980056248320, 93980056961023, -ERASE, 93980056248320, 93980056248320, -STORE, 93980056248320, 93980056297471, -STORE, 93980056297472, 93980056961023, -ERASE, 93980056297472, 93980056297472, -STORE, 93980056297472, 93980056842239, -STORE, 93980056842240, 93980056940543, -STORE, 93980056940544, 93980056961023, -STORE, 140146588971008, 140146589143039, -ERASE, 140146588971008, 140146588971008, -STORE, 140146588971008, 140146588975103, -STORE, 140146588975104, 140146589143039, -ERASE, 140146588975104, 140146588975104, -STORE, 140146588975104, 140146589097983, -STORE, 140146589097984, 140146589130751, -STORE, 140146589130752, 140146589138943, -STORE, 140146589138944, 140146589143039, -STORE, 140725554860032, 140725554864127, -STORE, 140725554847744, 140725554860031, -STORE, 47486206025728, 47486206033919, -STORE, 47486206033920, 47486206042111, -STORE, 47486206042112, 47486206148607, -STORE, 47486206058496, 47486206148607, -STORE, 47486206042112, 47486206058495, -ERASE, 47486206058496, 47486206058496, -STORE, 47486206058496, 47486206132223, -STORE, 47486206132224, 47486206148607, -STORE, 47486206111744, 47486206132223, -STORE, 47486206058496, 47486206111743, -ERASE, 47486206058496, 47486206058496, -STORE, 47486206058496, 47486206111743, -STORE, 47486206128128, 47486206132223, -STORE, 47486206111744, 47486206128127, -ERASE, 47486206111744, 47486206111744, -STORE, 47486206111744, 47486206128127, -STORE, 47486206140416, 47486206148607, -STORE, 47486206132224, 47486206140415, -ERASE, 47486206132224, 47486206132224, -STORE, 47486206132224, 47486206140415, -ERASE, 47486206140416, 47486206140416, -STORE, 47486206140416, 47486206148607, -STORE, 47486206148608, 47486207987711, -STORE, 47486206287872, 47486207987711, -STORE, 47486206148608, 47486206287871, -ERASE, 47486206287872, 47486206287872, -STORE, 47486206287872, 47486207946751, -STORE, 47486207946752, 47486207987711, -STORE, 47486207631360, 47486207946751, -STORE, 47486206287872, 47486207631359, -ERASE, 47486206287872, 47486206287872, -STORE, 47486206287872, 47486207631359, -STORE, 47486207942656, 47486207946751, -STORE, 47486207631360, 47486207942655, -ERASE, 47486207631360, 47486207631360, -STORE, 47486207631360, 47486207942655, -STORE, 47486207971328, 47486207987711, -STORE, 47486207946752, 47486207971327, -ERASE, 47486207946752, 47486207946752, -STORE, 47486207946752, 47486207971327, -ERASE, 47486207971328, 47486207971328, -STORE, 47486207971328, 47486207987711, -STORE, 47486207971328, 47486207999999, -ERASE, 47486207946752, 47486207946752, -STORE, 47486207946752, 47486207963135, -STORE, 47486207963136, 47486207971327, -ERASE, 47486206132224, 47486206132224, -STORE, 47486206132224, 47486206136319, -STORE, 47486206136320, 47486206140415, -ERASE, 93980056940544, 93980056940544, -STORE, 93980056940544, 93980056956927, -STORE, 93980056956928, 93980056961023, -ERASE, 140146589130752, 140146589130752, -STORE, 140146589130752, 140146589134847, -STORE, 140146589134848, 140146589138943, -ERASE, 47486206025728, 47486206025728, -STORE, 93980070006784, 93980070141951, -STORE, 140737488347136, 140737488351231, -STORE, 140727334776832, 140737488351231, -ERASE, 140727334776832, 140727334776832, -STORE, 140727334776832, 140727334780927, -STORE, 94049747247104, 94049747959807, -ERASE, 94049747247104, 94049747247104, -STORE, 94049747247104, 94049747296255, -STORE, 94049747296256, 94049747959807, -ERASE, 94049747296256, 94049747296256, -STORE, 94049747296256, 94049747841023, -STORE, 94049747841024, 94049747939327, -STORE, 94049747939328, 94049747959807, -STORE, 140227307216896, 140227307388927, -ERASE, 140227307216896, 140227307216896, -STORE, 140227307216896, 140227307220991, -STORE, 140227307220992, 140227307388927, -ERASE, 140227307220992, 140227307220992, -STORE, 140227307220992, 140227307343871, -STORE, 140227307343872, 140227307376639, -STORE, 140227307376640, 140227307384831, -STORE, 140227307384832, 140227307388927, -STORE, 140727335337984, 140727335342079, -STORE, 140727335325696, 140727335337983, -STORE, 47405487779840, 47405487788031, -STORE, 47405487788032, 47405487796223, -STORE, 47405487796224, 47405487902719, -STORE, 47405487812608, 47405487902719, -STORE, 47405487796224, 47405487812607, -ERASE, 47405487812608, 47405487812608, -STORE, 47405487812608, 47405487886335, -STORE, 47405487886336, 47405487902719, -STORE, 47405487865856, 47405487886335, -STORE, 47405487812608, 47405487865855, -ERASE, 47405487812608, 47405487812608, -STORE, 47405487812608, 47405487865855, -STORE, 47405487882240, 47405487886335, -STORE, 47405487865856, 47405487882239, -ERASE, 47405487865856, 47405487865856, -STORE, 47405487865856, 47405487882239, -STORE, 47405487894528, 47405487902719, -STORE, 47405487886336, 47405487894527, -ERASE, 47405487886336, 47405487886336, -STORE, 47405487886336, 47405487894527, -ERASE, 47405487894528, 47405487894528, -STORE, 47405487894528, 47405487902719, -STORE, 47405487902720, 47405489741823, -STORE, 47405488041984, 47405489741823, -STORE, 47405487902720, 47405488041983, -ERASE, 47405488041984, 47405488041984, -STORE, 47405488041984, 47405489700863, -STORE, 47405489700864, 47405489741823, -STORE, 47405489385472, 47405489700863, -STORE, 47405488041984, 47405489385471, -ERASE, 47405488041984, 47405488041984, -STORE, 47405488041984, 47405489385471, -STORE, 47405489696768, 47405489700863, -STORE, 47405489385472, 47405489696767, -ERASE, 47405489385472, 47405489385472, -STORE, 47405489385472, 47405489696767, -STORE, 47405489725440, 47405489741823, -STORE, 47405489700864, 47405489725439, -ERASE, 47405489700864, 47405489700864, -STORE, 47405489700864, 47405489725439, -ERASE, 47405489725440, 47405489725440, -STORE, 47405489725440, 47405489741823, -STORE, 47405489725440, 47405489754111, -ERASE, 47405489700864, 47405489700864, -STORE, 47405489700864, 47405489717247, -STORE, 47405489717248, 47405489725439, -ERASE, 47405487886336, 47405487886336, -STORE, 47405487886336, 47405487890431, -STORE, 47405487890432, 47405487894527, -ERASE, 94049747939328, 94049747939328, -STORE, 94049747939328, 94049747955711, -STORE, 94049747955712, 94049747959807, -ERASE, 140227307376640, 140227307376640, -STORE, 140227307376640, 140227307380735, -STORE, 140227307380736, 140227307384831, -ERASE, 47405487779840, 47405487779840, -STORE, 94049758810112, 94049758945279, -STORE, 140737488347136, 140737488351231, -STORE, 140727079718912, 140737488351231, -ERASE, 140727079718912, 140727079718912, -STORE, 140727079718912, 140727079723007, -STORE, 94250996527104, 94250997239807, -ERASE, 94250996527104, 94250996527104, -STORE, 94250996527104, 94250996576255, -STORE, 94250996576256, 94250997239807, -ERASE, 94250996576256, 94250996576256, -STORE, 94250996576256, 94250997121023, -STORE, 94250997121024, 94250997219327, -STORE, 94250997219328, 94250997239807, -STORE, 140060022587392, 140060022759423, -ERASE, 140060022587392, 140060022587392, -STORE, 140060022587392, 140060022591487, -STORE, 140060022591488, 140060022759423, -ERASE, 140060022591488, 140060022591488, -STORE, 140060022591488, 140060022714367, -STORE, 140060022714368, 140060022747135, -STORE, 140060022747136, 140060022755327, -STORE, 140060022755328, 140060022759423, -STORE, 140727079788544, 140727079792639, -STORE, 140727079776256, 140727079788543, -STORE, 47572772409344, 47572772417535, -STORE, 47572772417536, 47572772425727, -STORE, 47572772425728, 47572772532223, -STORE, 47572772442112, 47572772532223, -STORE, 47572772425728, 47572772442111, -ERASE, 47572772442112, 47572772442112, -STORE, 47572772442112, 47572772515839, -STORE, 47572772515840, 47572772532223, -STORE, 47572772495360, 47572772515839, -STORE, 47572772442112, 47572772495359, -ERASE, 47572772442112, 47572772442112, -STORE, 47572772442112, 47572772495359, -STORE, 47572772511744, 47572772515839, -STORE, 47572772495360, 47572772511743, -ERASE, 47572772495360, 47572772495360, -STORE, 47572772495360, 47572772511743, -STORE, 47572772524032, 47572772532223, -STORE, 47572772515840, 47572772524031, -ERASE, 47572772515840, 47572772515840, -STORE, 47572772515840, 47572772524031, -ERASE, 47572772524032, 47572772524032, -STORE, 47572772524032, 47572772532223, -STORE, 47572772532224, 47572774371327, -STORE, 47572772671488, 47572774371327, -STORE, 47572772532224, 47572772671487, -ERASE, 47572772671488, 47572772671488, -STORE, 47572772671488, 47572774330367, -STORE, 47572774330368, 47572774371327, -STORE, 47572774014976, 47572774330367, -STORE, 47572772671488, 47572774014975, -ERASE, 47572772671488, 47572772671488, -STORE, 47572772671488, 47572774014975, -STORE, 47572774326272, 47572774330367, -STORE, 47572774014976, 47572774326271, -ERASE, 47572774014976, 47572774014976, -STORE, 47572774014976, 47572774326271, -STORE, 47572774354944, 47572774371327, -STORE, 47572774330368, 47572774354943, -ERASE, 47572774330368, 47572774330368, -STORE, 47572774330368, 47572774354943, -ERASE, 47572774354944, 47572774354944, -STORE, 47572774354944, 47572774371327, -STORE, 47572774354944, 47572774383615, -ERASE, 47572774330368, 47572774330368, -STORE, 47572774330368, 47572774346751, -STORE, 47572774346752, 47572774354943, -ERASE, 47572772515840, 47572772515840, -STORE, 47572772515840, 47572772519935, -STORE, 47572772519936, 47572772524031, -ERASE, 94250997219328, 94250997219328, -STORE, 94250997219328, 94250997235711, -STORE, 94250997235712, 94250997239807, -ERASE, 140060022747136, 140060022747136, -STORE, 140060022747136, 140060022751231, -STORE, 140060022751232, 140060022755327, -ERASE, 47572772409344, 47572772409344, -STORE, 94251018305536, 94251018440703, -STORE, 140737488347136, 140737488351231, -STORE, 140730012389376, 140737488351231, -ERASE, 140730012389376, 140730012389376, -STORE, 140730012389376, 140730012393471, -STORE, 94382607675392, 94382607695871, -ERASE, 94382607675392, 94382607675392, -STORE, 94382607675392, 94382607679487, -STORE, 94382607679488, 94382607695871, -ERASE, 94382607679488, 94382607679488, -STORE, 94382607679488, 94382607683583, -STORE, 94382607683584, 94382607687679, -STORE, 94382607687680, 94382607695871, -STORE, 140252451454976, 140252451627007, -ERASE, 140252451454976, 140252451454976, -STORE, 140252451454976, 140252451459071, -STORE, 140252451459072, 140252451627007, -ERASE, 140252451459072, 140252451459072, -STORE, 140252451459072, 140252451581951, -STORE, 140252451581952, 140252451614719, -STORE, 140252451614720, 140252451622911, -STORE, 140252451622912, 140252451627007, -STORE, 140730013548544, 140730013552639, -STORE, 140730013536256, 140730013548543, -STORE, 47380343541760, 47380343549951, -STORE, 47380343549952, 47380343558143, -STORE, 47380343558144, 47380345397247, -STORE, 47380343697408, 47380345397247, -STORE, 47380343558144, 47380343697407, -ERASE, 47380343697408, 47380343697408, -STORE, 47380343697408, 47380345356287, -STORE, 47380345356288, 47380345397247, -STORE, 47380345040896, 47380345356287, -STORE, 47380343697408, 47380345040895, -ERASE, 47380343697408, 47380343697408, -STORE, 47380343697408, 47380345040895, -STORE, 47380345352192, 47380345356287, -STORE, 47380345040896, 47380345352191, -ERASE, 47380345040896, 47380345040896, -STORE, 47380345040896, 47380345352191, -STORE, 47380345380864, 47380345397247, -STORE, 47380345356288, 47380345380863, -ERASE, 47380345356288, 47380345356288, -STORE, 47380345356288, 47380345380863, -ERASE, 47380345380864, 47380345380864, -STORE, 47380345380864, 47380345397247, -ERASE, 47380345356288, 47380345356288, -STORE, 47380345356288, 47380345372671, -STORE, 47380345372672, 47380345380863, -ERASE, 94382607687680, 94382607687680, -STORE, 94382607687680, 94382607691775, -STORE, 94382607691776, 94382607695871, -ERASE, 140252451614720, 140252451614720, -STORE, 140252451614720, 140252451618815, -STORE, 140252451618816, 140252451622911, -ERASE, 47380343541760, 47380343541760, -STORE, 94382626803712, 94382626938879, -STORE, 140737488347136, 140737488351231, -STORE, 140730900271104, 140737488351231, -ERASE, 140730900271104, 140730900271104, -STORE, 140730900271104, 140730900275199, -STORE, 93855478120448, 93855478337535, -ERASE, 93855478120448, 93855478120448, -STORE, 93855478120448, 93855478198271, -STORE, 93855478198272, 93855478337535, -ERASE, 93855478198272, 93855478198272, -STORE, 93855478198272, 93855478243327, -STORE, 93855478243328, 93855478288383, -STORE, 93855478288384, 93855478337535, -STORE, 140092686573568, 140092686745599, -ERASE, 140092686573568, 140092686573568, -STORE, 140092686573568, 140092686577663, -STORE, 140092686577664, 140092686745599, -ERASE, 140092686577664, 140092686577664, -STORE, 140092686577664, 140092686700543, -STORE, 140092686700544, 140092686733311, -STORE, 140092686733312, 140092686741503, -STORE, 140092686741504, 140092686745599, -STORE, 140730900537344, 140730900541439, -STORE, 140730900525056, 140730900537343, -STORE, 47540108423168, 47540108431359, -STORE, 47540108431360, 47540108439551, -STORE, 47540108439552, 47540110278655, -STORE, 47540108578816, 47540110278655, -STORE, 47540108439552, 47540108578815, -ERASE, 47540108578816, 47540108578816, -STORE, 47540108578816, 47540110237695, -STORE, 47540110237696, 47540110278655, -STORE, 47540109922304, 47540110237695, -STORE, 47540108578816, 47540109922303, -ERASE, 47540108578816, 47540108578816, -STORE, 47540108578816, 47540109922303, -STORE, 47540110233600, 47540110237695, -STORE, 47540109922304, 47540110233599, -ERASE, 47540109922304, 47540109922304, -STORE, 47540109922304, 47540110233599, -STORE, 47540110262272, 47540110278655, -STORE, 47540110237696, 47540110262271, -ERASE, 47540110237696, 47540110237696, -STORE, 47540110237696, 47540110262271, -ERASE, 47540110262272, 47540110262272, -STORE, 47540110262272, 47540110278655, -ERASE, 47540110237696, 47540110237696, -STORE, 47540110237696, 47540110254079, -STORE, 47540110254080, 47540110262271, -ERASE, 93855478288384, 93855478288384, -STORE, 93855478288384, 93855478333439, -STORE, 93855478333440, 93855478337535, -ERASE, 140092686733312, 140092686733312, -STORE, 140092686733312, 140092686737407, -STORE, 140092686737408, 140092686741503, -ERASE, 47540108423168, 47540108423168, -STORE, 93855492222976, 93855492358143, -STORE, 93855492222976, 93855492493311, -STORE, 140737488347136, 140737488351231, -STORE, 140733498146816, 140737488351231, -ERASE, 140733498146816, 140733498146816, -STORE, 140733498146816, 140733498150911, -STORE, 94170739654656, 94170740367359, -ERASE, 94170739654656, 94170739654656, -STORE, 94170739654656, 94170739703807, -STORE, 94170739703808, 94170740367359, -ERASE, 94170739703808, 94170739703808, -STORE, 94170739703808, 94170740248575, -STORE, 94170740248576, 94170740346879, -STORE, 94170740346880, 94170740367359, -STORE, 140024788877312, 140024789049343, -ERASE, 140024788877312, 140024788877312, -STORE, 140024788877312, 140024788881407, -STORE, 140024788881408, 140024789049343, -ERASE, 140024788881408, 140024788881408, -STORE, 140024788881408, 140024789004287, -STORE, 140024789004288, 140024789037055, -STORE, 140024789037056, 140024789045247, -STORE, 140024789045248, 140024789049343, -STORE, 140733499023360, 140733499027455, -STORE, 140733499011072, 140733499023359, -STORE, 47608006119424, 47608006127615, -STORE, 47608006127616, 47608006135807, -STORE, 47608006135808, 47608006242303, -STORE, 47608006152192, 47608006242303, -STORE, 47608006135808, 47608006152191, -ERASE, 47608006152192, 47608006152192, -STORE, 47608006152192, 47608006225919, -STORE, 47608006225920, 47608006242303, -STORE, 47608006205440, 47608006225919, -STORE, 47608006152192, 47608006205439, -ERASE, 47608006152192, 47608006152192, -STORE, 47608006152192, 47608006205439, -STORE, 47608006221824, 47608006225919, -STORE, 47608006205440, 47608006221823, -ERASE, 47608006205440, 47608006205440, -STORE, 47608006205440, 47608006221823, -STORE, 47608006234112, 47608006242303, -STORE, 47608006225920, 47608006234111, -ERASE, 47608006225920, 47608006225920, -STORE, 47608006225920, 47608006234111, -ERASE, 47608006234112, 47608006234112, -STORE, 47608006234112, 47608006242303, -STORE, 47608006242304, 47608008081407, -STORE, 47608006381568, 47608008081407, -STORE, 47608006242304, 47608006381567, -ERASE, 47608006381568, 47608006381568, -STORE, 47608006381568, 47608008040447, -STORE, 47608008040448, 47608008081407, -STORE, 47608007725056, 47608008040447, -STORE, 47608006381568, 47608007725055, -ERASE, 47608006381568, 47608006381568, -STORE, 47608006381568, 47608007725055, -STORE, 47608008036352, 47608008040447, -STORE, 47608007725056, 47608008036351, -ERASE, 47608007725056, 47608007725056, -STORE, 47608007725056, 47608008036351, -STORE, 47608008065024, 47608008081407, -STORE, 47608008040448, 47608008065023, -ERASE, 47608008040448, 47608008040448, -STORE, 47608008040448, 47608008065023, -ERASE, 47608008065024, 47608008065024, -STORE, 47608008065024, 47608008081407, -STORE, 47608008065024, 47608008093695, -ERASE, 47608008040448, 47608008040448, -STORE, 47608008040448, 47608008056831, -STORE, 47608008056832, 47608008065023, -ERASE, 47608006225920, 47608006225920, -STORE, 47608006225920, 47608006230015, -STORE, 47608006230016, 47608006234111, -ERASE, 94170740346880, 94170740346880, -STORE, 94170740346880, 94170740363263, -STORE, 94170740363264, 94170740367359, -ERASE, 140024789037056, 140024789037056, -STORE, 140024789037056, 140024789041151, -STORE, 140024789041152, 140024789045247, -ERASE, 47608006119424, 47608006119424, -STORE, 140737488347136, 140737488351231, -STORE, 140730264326144, 140737488351231, -ERASE, 140730264326144, 140730264326144, -STORE, 140730264326144, 140730264330239, -STORE, 94653216407552, 94653217120255, -ERASE, 94653216407552, 94653216407552, -STORE, 94653216407552, 94653216456703, -STORE, 94653216456704, 94653217120255, -ERASE, 94653216456704, 94653216456704, -STORE, 94653216456704, 94653217001471, -STORE, 94653217001472, 94653217099775, -STORE, 94653217099776, 94653217120255, -STORE, 140103617011712, 140103617183743, -ERASE, 140103617011712, 140103617011712, -STORE, 140103617011712, 140103617015807, -STORE, 140103617015808, 140103617183743, -ERASE, 140103617015808, 140103617015808, -STORE, 140103617015808, 140103617138687, -STORE, 140103617138688, 140103617171455, -STORE, 140103617171456, 140103617179647, -STORE, 140103617179648, 140103617183743, -STORE, 140730265427968, 140730265432063, -STORE, 140730265415680, 140730265427967, -STORE, 47529177985024, 47529177993215, -STORE, 47529177993216, 47529178001407, -STORE, 47529178001408, 47529178107903, -STORE, 47529178017792, 47529178107903, -STORE, 47529178001408, 47529178017791, -ERASE, 47529178017792, 47529178017792, -STORE, 47529178017792, 47529178091519, -STORE, 47529178091520, 47529178107903, -STORE, 47529178071040, 47529178091519, -STORE, 47529178017792, 47529178071039, -ERASE, 47529178017792, 47529178017792, -STORE, 47529178017792, 47529178071039, -STORE, 47529178087424, 47529178091519, -STORE, 47529178071040, 47529178087423, -ERASE, 47529178071040, 47529178071040, -STORE, 47529178071040, 47529178087423, -STORE, 47529178099712, 47529178107903, -STORE, 47529178091520, 47529178099711, -ERASE, 47529178091520, 47529178091520, -STORE, 47529178091520, 47529178099711, -ERASE, 47529178099712, 47529178099712, -STORE, 47529178099712, 47529178107903, -STORE, 47529178107904, 47529179947007, -STORE, 47529178247168, 47529179947007, -STORE, 47529178107904, 47529178247167, -ERASE, 47529178247168, 47529178247168, -STORE, 47529178247168, 47529179906047, -STORE, 47529179906048, 47529179947007, -STORE, 47529179590656, 47529179906047, -STORE, 47529178247168, 47529179590655, -ERASE, 47529178247168, 47529178247168, -STORE, 47529178247168, 47529179590655, -STORE, 47529179901952, 47529179906047, -STORE, 47529179590656, 47529179901951, -ERASE, 47529179590656, 47529179590656, -STORE, 47529179590656, 47529179901951, -STORE, 47529179930624, 47529179947007, -STORE, 47529179906048, 47529179930623, -ERASE, 47529179906048, 47529179906048, -STORE, 47529179906048, 47529179930623, -ERASE, 47529179930624, 47529179930624, -STORE, 47529179930624, 47529179947007, -STORE, 47529179930624, 47529179959295, -ERASE, 47529179906048, 47529179906048, -STORE, 47529179906048, 47529179922431, -STORE, 47529179922432, 47529179930623, -ERASE, 47529178091520, 47529178091520, -STORE, 47529178091520, 47529178095615, -STORE, 47529178095616, 47529178099711, -ERASE, 94653217099776, 94653217099776, -STORE, 94653217099776, 94653217116159, -STORE, 94653217116160, 94653217120255, -ERASE, 140103617171456, 140103617171456, -STORE, 140103617171456, 140103617175551, -STORE, 140103617175552, 140103617179647, -ERASE, 47529177985024, 47529177985024, -STORE, 94653241135104, 94653241270271, -STORE, 140737488347136, 140737488351231, -STORE, 140736284549120, 140737488351231, -ERASE, 140736284549120, 140736284549120, -STORE, 140736284549120, 140736284553215, -STORE, 93963663822848, 93963664506879, -ERASE, 93963663822848, 93963663822848, -STORE, 93963663822848, 93963663884287, -STORE, 93963663884288, 93963664506879, -ERASE, 93963663884288, 93963663884288, -STORE, 93963663884288, 93963664240639, -STORE, 93963664240640, 93963664379903, -STORE, 93963664379904, 93963664506879, -STORE, 140450188439552, 140450188611583, -ERASE, 140450188439552, 140450188439552, -STORE, 140450188439552, 140450188443647, -STORE, 140450188443648, 140450188611583, -ERASE, 140450188443648, 140450188443648, -STORE, 140450188443648, 140450188566527, -STORE, 140450188566528, 140450188599295, -STORE, 140450188599296, 140450188607487, -STORE, 140450188607488, 140450188611583, -STORE, 140736284577792, 140736284581887, -STORE, 140736284565504, 140736284577791, -STORE, 47182606557184, 47182606565375, -STORE, 47182606565376, 47182606573567, -STORE, 47182606573568, 47182608412671, -STORE, 47182606712832, 47182608412671, -STORE, 47182606573568, 47182606712831, -ERASE, 47182606712832, 47182606712832, -STORE, 47182606712832, 47182608371711, -STORE, 47182608371712, 47182608412671, -STORE, 47182608056320, 47182608371711, -STORE, 47182606712832, 47182608056319, -ERASE, 47182606712832, 47182606712832, -STORE, 47182606712832, 47182608056319, -STORE, 47182608367616, 47182608371711, -STORE, 47182608056320, 47182608367615, -ERASE, 47182608056320, 47182608056320, -STORE, 47182608056320, 47182608367615, -STORE, 47182608396288, 47182608412671, -STORE, 47182608371712, 47182608396287, -ERASE, 47182608371712, 47182608371712, -STORE, 47182608371712, 47182608396287, -ERASE, 47182608396288, 47182608396288, -STORE, 47182608396288, 47182608412671, -STORE, 47182608412672, 47182608523263, -STORE, 47182608429056, 47182608523263, -STORE, 47182608412672, 47182608429055, -ERASE, 47182608429056, 47182608429056, -STORE, 47182608429056, 47182608515071, -STORE, 47182608515072, 47182608523263, -STORE, 47182608490496, 47182608515071, -STORE, 47182608429056, 47182608490495, -ERASE, 47182608429056, 47182608429056, -STORE, 47182608429056, 47182608490495, -STORE, 47182608510976, 47182608515071, -STORE, 47182608490496, 47182608510975, -ERASE, 47182608490496, 47182608490496, -STORE, 47182608490496, 47182608510975, -ERASE, 47182608515072, 47182608515072, -STORE, 47182608515072, 47182608523263, -STORE, 47182608523264, 47182608568319, -ERASE, 47182608523264, 47182608523264, -STORE, 47182608523264, 47182608531455, -STORE, 47182608531456, 47182608568319, -STORE, 47182608551936, 47182608568319, -STORE, 47182608531456, 47182608551935, -ERASE, 47182608531456, 47182608531456, -STORE, 47182608531456, 47182608551935, -STORE, 47182608560128, 47182608568319, -STORE, 47182608551936, 47182608560127, -ERASE, 47182608551936, 47182608551936, -STORE, 47182608551936, 47182608568319, -ERASE, 47182608551936, 47182608551936, -STORE, 47182608551936, 47182608560127, -STORE, 47182608560128, 47182608568319, -ERASE, 47182608560128, 47182608560128, -STORE, 47182608560128, 47182608568319, -STORE, 47182608568320, 47182608916479, -STORE, 47182608609280, 47182608916479, -STORE, 47182608568320, 47182608609279, -ERASE, 47182608609280, 47182608609280, -STORE, 47182608609280, 47182608891903, -STORE, 47182608891904, 47182608916479, -STORE, 47182608822272, 47182608891903, -STORE, 47182608609280, 47182608822271, -ERASE, 47182608609280, 47182608609280, -STORE, 47182608609280, 47182608822271, -STORE, 47182608887808, 47182608891903, -STORE, 47182608822272, 47182608887807, -ERASE, 47182608822272, 47182608822272, -STORE, 47182608822272, 47182608887807, -ERASE, 47182608891904, 47182608891904, -STORE, 47182608891904, 47182608916479, -STORE, 47182608916480, 47182611177471, -STORE, 47182609068032, 47182611177471, -STORE, 47182608916480, 47182609068031, -ERASE, 47182609068032, 47182609068032, -STORE, 47182609068032, 47182611161087, -STORE, 47182611161088, 47182611177471, -STORE, 47182611169280, 47182611177471, -STORE, 47182611161088, 47182611169279, -ERASE, 47182611161088, 47182611161088, -STORE, 47182611161088, 47182611169279, -ERASE, 47182611169280, 47182611169280, -STORE, 47182611169280, 47182611177471, -STORE, 47182611177472, 47182611312639, -ERASE, 47182611177472, 47182611177472, -STORE, 47182611177472, 47182611202047, -STORE, 47182611202048, 47182611312639, -STORE, 47182611263488, 47182611312639, -STORE, 47182611202048, 47182611263487, -ERASE, 47182611202048, 47182611202048, -STORE, 47182611202048, 47182611263487, -STORE, 47182611288064, 47182611312639, -STORE, 47182611263488, 47182611288063, -ERASE, 47182611263488, 47182611263488, -STORE, 47182611263488, 47182611312639, -ERASE, 47182611263488, 47182611263488, -STORE, 47182611263488, 47182611288063, -STORE, 47182611288064, 47182611312639, -STORE, 47182611296256, 47182611312639, -STORE, 47182611288064, 47182611296255, -ERASE, 47182611288064, 47182611288064, -STORE, 47182611288064, 47182611296255, -ERASE, 47182611296256, 47182611296256, -STORE, 47182611296256, 47182611312639, -STORE, 47182611296256, 47182611320831, -STORE, 47182611320832, 47182611484671, -ERASE, 47182611320832, 47182611320832, -STORE, 47182611320832, 47182611333119, -STORE, 47182611333120, 47182611484671, -STORE, 47182611431424, 47182611484671, -STORE, 47182611333120, 47182611431423, -ERASE, 47182611333120, 47182611333120, -STORE, 47182611333120, 47182611431423, -STORE, 47182611476480, 47182611484671, -STORE, 47182611431424, 47182611476479, -ERASE, 47182611431424, 47182611431424, -STORE, 47182611431424, 47182611484671, -ERASE, 47182611431424, 47182611431424, -STORE, 47182611431424, 47182611476479, -STORE, 47182611476480, 47182611484671, -ERASE, 47182611476480, 47182611476480, -STORE, 47182611476480, 47182611484671, -STORE, 47182611484672, 47182612082687, -STORE, 47182611603456, 47182612082687, -STORE, 47182611484672, 47182611603455, -ERASE, 47182611603456, 47182611603456, -STORE, 47182611603456, 47182612029439, -STORE, 47182612029440, 47182612082687, -STORE, 47182611918848, 47182612029439, -STORE, 47182611603456, 47182611918847, -ERASE, 47182611603456, 47182611603456, -STORE, 47182611603456, 47182611918847, -STORE, 47182612025344, 47182612029439, -STORE, 47182611918848, 47182612025343, -ERASE, 47182611918848, 47182611918848, -STORE, 47182611918848, 47182612025343, -ERASE, 47182612029440, 47182612029440, -STORE, 47182612029440, 47182612082687, -STORE, 47182612082688, 47182615134207, -STORE, 47182612627456, 47182615134207, -STORE, 47182612082688, 47182612627455, -ERASE, 47182612627456, 47182612627456, -STORE, 47182612627456, 47182614913023, -STORE, 47182614913024, 47182615134207, -STORE, 47182614323200, 47182614913023, -STORE, 47182612627456, 47182614323199, -ERASE, 47182612627456, 47182612627456, -STORE, 47182612627456, 47182614323199, -STORE, 47182614908928, 47182614913023, -STORE, 47182614323200, 47182614908927, -ERASE, 47182614323200, 47182614323200, -STORE, 47182614323200, 47182614908927, -STORE, 47182615117824, 47182615134207, -STORE, 47182614913024, 47182615117823, -ERASE, 47182614913024, 47182614913024, -STORE, 47182614913024, 47182615117823, -ERASE, 47182615117824, 47182615117824, -STORE, 47182615117824, 47182615134207, -STORE, 47182615134208, 47182615166975, -ERASE, 47182615134208, 47182615134208, -STORE, 47182615134208, 47182615142399, -STORE, 47182615142400, 47182615166975, -STORE, 47182615154688, 47182615166975, -STORE, 47182615142400, 47182615154687, -ERASE, 47182615142400, 47182615142400, -STORE, 47182615142400, 47182615154687, -STORE, 47182615158784, 47182615166975, -STORE, 47182615154688, 47182615158783, -ERASE, 47182615154688, 47182615154688, -STORE, 47182615154688, 47182615166975, -ERASE, 47182615154688, 47182615154688, -STORE, 47182615154688, 47182615158783, -STORE, 47182615158784, 47182615166975, -ERASE, 47182615158784, 47182615158784, -STORE, 47182615158784, 47182615166975, -STORE, 47182615166976, 47182615203839, -ERASE, 47182615166976, 47182615166976, -STORE, 47182615166976, 47182615175167, -STORE, 47182615175168, 47182615203839, -STORE, 47182615191552, 47182615203839, -STORE, 47182615175168, 47182615191551, -ERASE, 47182615175168, 47182615175168, -STORE, 47182615175168, 47182615191551, -STORE, 47182615195648, 47182615203839, -STORE, 47182615191552, 47182615195647, -ERASE, 47182615191552, 47182615191552, -STORE, 47182615191552, 47182615203839, -ERASE, 47182615191552, 47182615191552, -STORE, 47182615191552, 47182615195647, -STORE, 47182615195648, 47182615203839, -ERASE, 47182615195648, 47182615195648, -STORE, 47182615195648, 47182615203839, -STORE, 47182615203840, 47182615678975, -ERASE, 47182615203840, 47182615203840, -STORE, 47182615203840, 47182615212031, -STORE, 47182615212032, 47182615678975, -STORE, 47182615547904, 47182615678975, -STORE, 47182615212032, 47182615547903, -ERASE, 47182615212032, 47182615212032, -STORE, 47182615212032, 47182615547903, -STORE, 47182615670784, 47182615678975, -STORE, 47182615547904, 47182615670783, -ERASE, 47182615547904, 47182615547904, -STORE, 47182615547904, 47182615678975, -ERASE, 47182615547904, 47182615547904, -STORE, 47182615547904, 47182615670783, -STORE, 47182615670784, 47182615678975, -ERASE, 47182615670784, 47182615670784, -STORE, 47182615670784, 47182615678975, -STORE, 47182615678976, 47182615687167, -STORE, 47182615687168, 47182615707647, -ERASE, 47182615687168, 47182615687168, -STORE, 47182615687168, 47182615691263, -STORE, 47182615691264, 47182615707647, -STORE, 47182615695360, 47182615707647, -STORE, 47182615691264, 47182615695359, -ERASE, 47182615691264, 47182615691264, -STORE, 47182615691264, 47182615695359, -STORE, 47182615699456, 47182615707647, -STORE, 47182615695360, 47182615699455, -ERASE, 47182615695360, 47182615695360, -STORE, 47182615695360, 47182615707647, -ERASE, 47182615695360, 47182615695360, -STORE, 47182615695360, 47182615699455, -STORE, 47182615699456, 47182615707647, -ERASE, 47182615699456, 47182615699456, -STORE, 47182615699456, 47182615707647, -STORE, 47182615707648, 47182615715839, -ERASE, 47182608371712, 47182608371712, -STORE, 47182608371712, 47182608388095, -STORE, 47182608388096, 47182608396287, -ERASE, 47182615699456, 47182615699456, -STORE, 47182615699456, 47182615703551, -STORE, 47182615703552, 47182615707647, -ERASE, 47182611288064, 47182611288064, -STORE, 47182611288064, 47182611292159, -STORE, 47182611292160, 47182611296255, -ERASE, 47182615670784, 47182615670784, -STORE, 47182615670784, 47182615674879, -STORE, 47182615674880, 47182615678975, -ERASE, 47182615195648, 47182615195648, -STORE, 47182615195648, 47182615199743, -STORE, 47182615199744, 47182615203839, -ERASE, 47182615158784, 47182615158784, -STORE, 47182615158784, 47182615162879, -STORE, 47182615162880, 47182615166975, -ERASE, 47182614913024, 47182614913024, -STORE, 47182614913024, 47182615109631, -STORE, 47182615109632, 47182615117823, -ERASE, 47182612029440, 47182612029440, -STORE, 47182612029440, 47182612066303, -STORE, 47182612066304, 47182612082687, -ERASE, 47182611476480, 47182611476480, -STORE, 47182611476480, 47182611480575, -STORE, 47182611480576, 47182611484671, -ERASE, 47182611161088, 47182611161088, -STORE, 47182611161088, 47182611165183, -STORE, 47182611165184, 47182611169279, -ERASE, 47182608891904, 47182608891904, -STORE, 47182608891904, 47182608912383, -STORE, 47182608912384, 47182608916479, -ERASE, 47182608560128, 47182608560128, -STORE, 47182608560128, 47182608564223, -STORE, 47182608564224, 47182608568319, -ERASE, 47182608515072, 47182608515072, -STORE, 47182608515072, 47182608519167, -STORE, 47182608519168, 47182608523263, -ERASE, 93963664379904, 93963664379904, -STORE, 93963664379904, 93963664502783, -STORE, 93963664502784, 93963664506879, -ERASE, 140450188599296, 140450188599296, -STORE, 140450188599296, 140450188603391, -STORE, 140450188603392, 140450188607487, -ERASE, 47182606557184, 47182606557184, -STORE, 93963694723072, 93963694858239, -STORE, 140737488347136, 140737488351231, -STORE, 140730313261056, 140737488351231, -ERASE, 140730313261056, 140730313261056, -STORE, 140730313261056, 140730313265151, -STORE, 94386579017728, 94386579697663, -ERASE, 94386579017728, 94386579017728, -STORE, 94386579017728, 94386579083263, -STORE, 94386579083264, 94386579697663, -ERASE, 94386579083264, 94386579083264, -STORE, 94386579083264, 94386579431423, -STORE, 94386579431424, 94386579570687, -STORE, 94386579570688, 94386579697663, -STORE, 140124810838016, 140124811010047, -ERASE, 140124810838016, 140124810838016, -STORE, 140124810838016, 140124810842111, -STORE, 140124810842112, 140124811010047, -ERASE, 140124810842112, 140124810842112, -STORE, 140124810842112, 140124810964991, -STORE, 140124810964992, 140124810997759, -STORE, 140124810997760, 140124811005951, -STORE, 140124811005952, 140124811010047, -STORE, 140730313601024, 140730313605119, -STORE, 140730313588736, 140730313601023, -STORE, 47507984158720, 47507984166911, -STORE, 47507984166912, 47507984175103, -STORE, 47507984175104, 47507986014207, -STORE, 47507984314368, 47507986014207, -STORE, 47507984175104, 47507984314367, -ERASE, 47507984314368, 47507984314368, -STORE, 47507984314368, 47507985973247, -STORE, 47507985973248, 47507986014207, -STORE, 47507985657856, 47507985973247, -STORE, 47507984314368, 47507985657855, -ERASE, 47507984314368, 47507984314368, -STORE, 47507984314368, 47507985657855, -STORE, 47507985969152, 47507985973247, -STORE, 47507985657856, 47507985969151, -ERASE, 47507985657856, 47507985657856, -STORE, 47507985657856, 47507985969151, -STORE, 47507985997824, 47507986014207, -STORE, 47507985973248, 47507985997823, -ERASE, 47507985973248, 47507985973248, -STORE, 47507985973248, 47507985997823, -ERASE, 47507985997824, 47507985997824, -STORE, 47507985997824, 47507986014207, -STORE, 47507986014208, 47507986124799, -STORE, 47507986030592, 47507986124799, -STORE, 47507986014208, 47507986030591, -ERASE, 47507986030592, 47507986030592, -STORE, 47507986030592, 47507986116607, -STORE, 47507986116608, 47507986124799, -STORE, 47507986092032, 47507986116607, -STORE, 47507986030592, 47507986092031, -ERASE, 47507986030592, 47507986030592, -STORE, 47507986030592, 47507986092031, -STORE, 47507986112512, 47507986116607, -STORE, 47507986092032, 47507986112511, -ERASE, 47507986092032, 47507986092032, -STORE, 47507986092032, 47507986112511, -ERASE, 47507986116608, 47507986116608, -STORE, 47507986116608, 47507986124799, -STORE, 47507986124800, 47507986169855, -ERASE, 47507986124800, 47507986124800, -STORE, 47507986124800, 47507986132991, -STORE, 47507986132992, 47507986169855, -STORE, 47507986153472, 47507986169855, -STORE, 47507986132992, 47507986153471, -ERASE, 47507986132992, 47507986132992, -STORE, 47507986132992, 47507986153471, -STORE, 47507986161664, 47507986169855, -STORE, 47507986153472, 47507986161663, -ERASE, 47507986153472, 47507986153472, -STORE, 47507986153472, 47507986169855, -ERASE, 47507986153472, 47507986153472, -STORE, 47507986153472, 47507986161663, -STORE, 47507986161664, 47507986169855, -ERASE, 47507986161664, 47507986161664, -STORE, 47507986161664, 47507986169855, -STORE, 47507986169856, 47507986518015, -STORE, 47507986210816, 47507986518015, -STORE, 47507986169856, 47507986210815, -ERASE, 47507986210816, 47507986210816, -STORE, 47507986210816, 47507986493439, -STORE, 47507986493440, 47507986518015, -STORE, 47507986423808, 47507986493439, -STORE, 47507986210816, 47507986423807, -ERASE, 47507986210816, 47507986210816, -STORE, 47507986210816, 47507986423807, -STORE, 47507986489344, 47507986493439, -STORE, 47507986423808, 47507986489343, -ERASE, 47507986423808, 47507986423808, -STORE, 47507986423808, 47507986489343, -ERASE, 47507986493440, 47507986493440, -STORE, 47507986493440, 47507986518015, -STORE, 47507986518016, 47507988779007, -STORE, 47507986669568, 47507988779007, -STORE, 47507986518016, 47507986669567, -ERASE, 47507986669568, 47507986669568, -STORE, 47507986669568, 47507988762623, -STORE, 47507988762624, 47507988779007, -STORE, 47507988770816, 47507988779007, -STORE, 47507988762624, 47507988770815, -ERASE, 47507988762624, 47507988762624, -STORE, 47507988762624, 47507988770815, -ERASE, 47507988770816, 47507988770816, -STORE, 47507988770816, 47507988779007, -STORE, 47507988779008, 47507988914175, -ERASE, 47507988779008, 47507988779008, -STORE, 47507988779008, 47507988803583, -STORE, 47507988803584, 47507988914175, -STORE, 47507988865024, 47507988914175, -STORE, 47507988803584, 47507988865023, -ERASE, 47507988803584, 47507988803584, -STORE, 47507988803584, 47507988865023, -STORE, 47507988889600, 47507988914175, -STORE, 47507988865024, 47507988889599, -ERASE, 47507988865024, 47507988865024, -STORE, 47507988865024, 47507988914175, -ERASE, 47507988865024, 47507988865024, -STORE, 47507988865024, 47507988889599, -STORE, 47507988889600, 47507988914175, -STORE, 47507988897792, 47507988914175, -STORE, 47507988889600, 47507988897791, -ERASE, 47507988889600, 47507988889600, -STORE, 47507988889600, 47507988897791, -ERASE, 47507988897792, 47507988897792, -STORE, 47507988897792, 47507988914175, -STORE, 47507988897792, 47507988922367, -STORE, 47507988922368, 47507989086207, -ERASE, 47507988922368, 47507988922368, -STORE, 47507988922368, 47507988934655, -STORE, 47507988934656, 47507989086207, -STORE, 47507989032960, 47507989086207, -STORE, 47507988934656, 47507989032959, -ERASE, 47507988934656, 47507988934656, -STORE, 47507988934656, 47507989032959, -STORE, 47507989078016, 47507989086207, -STORE, 47507989032960, 47507989078015, -ERASE, 47507989032960, 47507989032960, -STORE, 47507989032960, 47507989086207, -ERASE, 47507989032960, 47507989032960, -STORE, 47507989032960, 47507989078015, -STORE, 47507989078016, 47507989086207, -ERASE, 47507989078016, 47507989078016, -STORE, 47507989078016, 47507989086207, -STORE, 47507989086208, 47507989684223, -STORE, 47507989204992, 47507989684223, -STORE, 47507989086208, 47507989204991, -ERASE, 47507989204992, 47507989204992, -STORE, 47507989204992, 47507989630975, -STORE, 47507989630976, 47507989684223, -STORE, 47507989520384, 47507989630975, -STORE, 47507989204992, 47507989520383, -ERASE, 47507989204992, 47507989204992, -STORE, 47507989204992, 47507989520383, -STORE, 47507989626880, 47507989630975, -STORE, 47507989520384, 47507989626879, -ERASE, 47507989520384, 47507989520384, -STORE, 47507989520384, 47507989626879, -ERASE, 47507989630976, 47507989630976, -STORE, 47507989630976, 47507989684223, -STORE, 47507989684224, 47507992735743, -STORE, 47507990228992, 47507992735743, -STORE, 47507989684224, 47507990228991, -ERASE, 47507990228992, 47507990228992, -STORE, 47507990228992, 47507992514559, -STORE, 47507992514560, 47507992735743, -STORE, 47507991924736, 47507992514559, -STORE, 47507990228992, 47507991924735, -ERASE, 47507990228992, 47507990228992, -STORE, 47507990228992, 47507991924735, -STORE, 47507992510464, 47507992514559, -STORE, 47507991924736, 47507992510463, -ERASE, 47507991924736, 47507991924736, -STORE, 47507991924736, 47507992510463, -STORE, 47507992719360, 47507992735743, -STORE, 47507992514560, 47507992719359, -ERASE, 47507992514560, 47507992514560, -STORE, 47507992514560, 47507992719359, -ERASE, 47507992719360, 47507992719360, -STORE, 47507992719360, 47507992735743, -STORE, 47507992735744, 47507992768511, -ERASE, 47507992735744, 47507992735744, -STORE, 47507992735744, 47507992743935, -STORE, 47507992743936, 47507992768511, -STORE, 47507992756224, 47507992768511, -STORE, 47507992743936, 47507992756223, -ERASE, 47507992743936, 47507992743936, -STORE, 47507992743936, 47507992756223, -STORE, 47507992760320, 47507992768511, -STORE, 47507992756224, 47507992760319, -ERASE, 47507992756224, 47507992756224, -STORE, 47507992756224, 47507992768511, -ERASE, 47507992756224, 47507992756224, -STORE, 47507992756224, 47507992760319, -STORE, 47507992760320, 47507992768511, -ERASE, 47507992760320, 47507992760320, -STORE, 47507992760320, 47507992768511, -STORE, 47507992768512, 47507992805375, -ERASE, 47507992768512, 47507992768512, -STORE, 47507992768512, 47507992776703, -STORE, 47507992776704, 47507992805375, -STORE, 47507992793088, 47507992805375, -STORE, 47507992776704, 47507992793087, -ERASE, 47507992776704, 47507992776704, -STORE, 47507992776704, 47507992793087, -STORE, 47507992797184, 47507992805375, -STORE, 47507992793088, 47507992797183, -ERASE, 47507992793088, 47507992793088, -STORE, 47507992793088, 47507992805375, -ERASE, 47507992793088, 47507992793088, -STORE, 47507992793088, 47507992797183, -STORE, 47507992797184, 47507992805375, -ERASE, 47507992797184, 47507992797184, -STORE, 47507992797184, 47507992805375, -STORE, 47507992805376, 47507993280511, -ERASE, 47507992805376, 47507992805376, -STORE, 47507992805376, 47507992813567, -STORE, 47507992813568, 47507993280511, -STORE, 47507993149440, 47507993280511, -STORE, 47507992813568, 47507993149439, -ERASE, 47507992813568, 47507992813568, -STORE, 47507992813568, 47507993149439, -STORE, 47507993272320, 47507993280511, -STORE, 47507993149440, 47507993272319, -ERASE, 47507993149440, 47507993149440, -STORE, 47507993149440, 47507993280511, -ERASE, 47507993149440, 47507993149440, -STORE, 47507993149440, 47507993272319, -STORE, 47507993272320, 47507993280511, -ERASE, 47507993272320, 47507993272320, -STORE, 47507993272320, 47507993280511, -STORE, 47507993280512, 47507993288703, -STORE, 47507993288704, 47507993309183, -ERASE, 47507993288704, 47507993288704, -STORE, 47507993288704, 47507993292799, -STORE, 47507993292800, 47507993309183, -STORE, 47507993296896, 47507993309183, -STORE, 47507993292800, 47507993296895, -ERASE, 47507993292800, 47507993292800, -STORE, 47507993292800, 47507993296895, -STORE, 47507993300992, 47507993309183, -STORE, 47507993296896, 47507993300991, -ERASE, 47507993296896, 47507993296896, -STORE, 47507993296896, 47507993309183, -ERASE, 47507993296896, 47507993296896, -STORE, 47507993296896, 47507993300991, -STORE, 47507993300992, 47507993309183, -ERASE, 47507993300992, 47507993300992, -STORE, 47507993300992, 47507993309183, -STORE, 47507993309184, 47507993317375, -ERASE, 47507985973248, 47507985973248, -STORE, 47507985973248, 47507985989631, -STORE, 47507985989632, 47507985997823, -ERASE, 47507993300992, 47507993300992, -STORE, 47507993300992, 47507993305087, -STORE, 47507993305088, 47507993309183, -ERASE, 47507988889600, 47507988889600, -STORE, 47507988889600, 47507988893695, -STORE, 47507988893696, 47507988897791, -ERASE, 47507993272320, 47507993272320, -STORE, 47507993272320, 47507993276415, -STORE, 47507993276416, 47507993280511, -ERASE, 47507992797184, 47507992797184, -STORE, 47507992797184, 47507992801279, -STORE, 47507992801280, 47507992805375, -ERASE, 47507992760320, 47507992760320, -STORE, 47507992760320, 47507992764415, -STORE, 47507992764416, 47507992768511, -ERASE, 47507992514560, 47507992514560, -STORE, 47507992514560, 47507992711167, -STORE, 47507992711168, 47507992719359, -ERASE, 47507989630976, 47507989630976, -STORE, 47507989630976, 47507989667839, -STORE, 47507989667840, 47507989684223, -ERASE, 47507989078016, 47507989078016, -STORE, 47507989078016, 47507989082111, -STORE, 47507989082112, 47507989086207, -ERASE, 47507988762624, 47507988762624, -STORE, 47507988762624, 47507988766719, -STORE, 47507988766720, 47507988770815, -ERASE, 47507986493440, 47507986493440, -STORE, 47507986493440, 47507986513919, -STORE, 47507986513920, 47507986518015, -ERASE, 47507986161664, 47507986161664, -STORE, 47507986161664, 47507986165759, -STORE, 47507986165760, 47507986169855, -ERASE, 47507986116608, 47507986116608, -STORE, 47507986116608, 47507986120703, -STORE, 47507986120704, 47507986124799, -ERASE, 94386579570688, 94386579570688, -STORE, 94386579570688, 94386579693567, -STORE, 94386579693568, 94386579697663, -ERASE, 140124810997760, 140124810997760, -STORE, 140124810997760, 140124811001855, -STORE, 140124811001856, 140124811005951, -ERASE, 47507984158720, 47507984158720, -STORE, 94386583982080, 94386584117247, -STORE, 94386583982080, 94386584256511, -ERASE, 94386583982080, 94386583982080, -STORE, 94386583982080, 94386584223743, -STORE, 94386584223744, 94386584256511, -ERASE, 94386584223744, 94386584223744, -STORE, 140737488347136, 140737488351231, -STORE, 140733763395584, 140737488351231, -ERASE, 140733763395584, 140733763395584, -STORE, 140733763395584, 140733763399679, -STORE, 94011546472448, 94011547152383, -ERASE, 94011546472448, 94011546472448, -STORE, 94011546472448, 94011546537983, -STORE, 94011546537984, 94011547152383, -ERASE, 94011546537984, 94011546537984, -STORE, 94011546537984, 94011546886143, -STORE, 94011546886144, 94011547025407, -STORE, 94011547025408, 94011547152383, -STORE, 139757597949952, 139757598121983, -ERASE, 139757597949952, 139757597949952, -STORE, 139757597949952, 139757597954047, -STORE, 139757597954048, 139757598121983, -ERASE, 139757597954048, 139757597954048, -STORE, 139757597954048, 139757598076927, -STORE, 139757598076928, 139757598109695, -STORE, 139757598109696, 139757598117887, -STORE, 139757598117888, 139757598121983, -STORE, 140733763596288, 140733763600383, -STORE, 140733763584000, 140733763596287, -STORE, 47875197046784, 47875197054975, -STORE, 47875197054976, 47875197063167, -STORE, 47875197063168, 47875198902271, -STORE, 47875197202432, 47875198902271, -STORE, 47875197063168, 47875197202431, -ERASE, 47875197202432, 47875197202432, -STORE, 47875197202432, 47875198861311, -STORE, 47875198861312, 47875198902271, -STORE, 47875198545920, 47875198861311, -STORE, 47875197202432, 47875198545919, -ERASE, 47875197202432, 47875197202432, -STORE, 47875197202432, 47875198545919, -STORE, 47875198857216, 47875198861311, -STORE, 47875198545920, 47875198857215, -ERASE, 47875198545920, 47875198545920, -STORE, 47875198545920, 47875198857215, -STORE, 47875198885888, 47875198902271, -STORE, 47875198861312, 47875198885887, -ERASE, 47875198861312, 47875198861312, -STORE, 47875198861312, 47875198885887, -ERASE, 47875198885888, 47875198885888, -STORE, 47875198885888, 47875198902271, -STORE, 47875198902272, 47875199012863, -STORE, 47875198918656, 47875199012863, -STORE, 47875198902272, 47875198918655, -ERASE, 47875198918656, 47875198918656, -STORE, 47875198918656, 47875199004671, -STORE, 47875199004672, 47875199012863, -STORE, 47875198980096, 47875199004671, -STORE, 47875198918656, 47875198980095, -ERASE, 47875198918656, 47875198918656, -STORE, 47875198918656, 47875198980095, -STORE, 47875199000576, 47875199004671, -STORE, 47875198980096, 47875199000575, -ERASE, 47875198980096, 47875198980096, -STORE, 47875198980096, 47875199000575, -ERASE, 47875199004672, 47875199004672, -STORE, 47875199004672, 47875199012863, -STORE, 47875199012864, 47875199057919, -ERASE, 47875199012864, 47875199012864, -STORE, 47875199012864, 47875199021055, -STORE, 47875199021056, 47875199057919, -STORE, 47875199041536, 47875199057919, -STORE, 47875199021056, 47875199041535, -ERASE, 47875199021056, 47875199021056, -STORE, 47875199021056, 47875199041535, -STORE, 47875199049728, 47875199057919, -STORE, 47875199041536, 47875199049727, -ERASE, 47875199041536, 47875199041536, -STORE, 47875199041536, 47875199057919, -ERASE, 47875199041536, 47875199041536, -STORE, 47875199041536, 47875199049727, -STORE, 47875199049728, 47875199057919, -ERASE, 47875199049728, 47875199049728, -STORE, 47875199049728, 47875199057919, -STORE, 47875199057920, 47875199406079, -STORE, 47875199098880, 47875199406079, -STORE, 47875199057920, 47875199098879, -ERASE, 47875199098880, 47875199098880, -STORE, 47875199098880, 47875199381503, -STORE, 47875199381504, 47875199406079, -STORE, 47875199311872, 47875199381503, -STORE, 47875199098880, 47875199311871, -ERASE, 47875199098880, 47875199098880, -STORE, 47875199098880, 47875199311871, -STORE, 47875199377408, 47875199381503, -STORE, 47875199311872, 47875199377407, -ERASE, 47875199311872, 47875199311872, -STORE, 47875199311872, 47875199377407, -ERASE, 47875199381504, 47875199381504, -STORE, 47875199381504, 47875199406079, -STORE, 47875199406080, 47875201667071, -STORE, 47875199557632, 47875201667071, -STORE, 47875199406080, 47875199557631, -ERASE, 47875199557632, 47875199557632, -STORE, 47875199557632, 47875201650687, -STORE, 47875201650688, 47875201667071, -STORE, 47875201658880, 47875201667071, -STORE, 47875201650688, 47875201658879, -ERASE, 47875201650688, 47875201650688, -STORE, 47875201650688, 47875201658879, -ERASE, 47875201658880, 47875201658880, -STORE, 47875201658880, 47875201667071, -STORE, 47875201667072, 47875201802239, -ERASE, 47875201667072, 47875201667072, -STORE, 47875201667072, 47875201691647, -STORE, 47875201691648, 47875201802239, -STORE, 47875201753088, 47875201802239, -STORE, 47875201691648, 47875201753087, -ERASE, 47875201691648, 47875201691648, -STORE, 47875201691648, 47875201753087, -STORE, 47875201777664, 47875201802239, -STORE, 47875201753088, 47875201777663, -ERASE, 47875201753088, 47875201753088, -STORE, 47875201753088, 47875201802239, -ERASE, 47875201753088, 47875201753088, -STORE, 47875201753088, 47875201777663, -STORE, 47875201777664, 47875201802239, -STORE, 47875201785856, 47875201802239, -STORE, 47875201777664, 47875201785855, -ERASE, 47875201777664, 47875201777664, -STORE, 47875201777664, 47875201785855, -ERASE, 47875201785856, 47875201785856, -STORE, 47875201785856, 47875201802239, -STORE, 47875201785856, 47875201810431, -STORE, 47875201810432, 47875201974271, -ERASE, 47875201810432, 47875201810432, -STORE, 47875201810432, 47875201822719, -STORE, 47875201822720, 47875201974271, -STORE, 47875201921024, 47875201974271, -STORE, 47875201822720, 47875201921023, -ERASE, 47875201822720, 47875201822720, -STORE, 47875201822720, 47875201921023, -STORE, 47875201966080, 47875201974271, -STORE, 47875201921024, 47875201966079, -ERASE, 47875201921024, 47875201921024, -STORE, 47875201921024, 47875201974271, -ERASE, 47875201921024, 47875201921024, -STORE, 47875201921024, 47875201966079, -STORE, 47875201966080, 47875201974271, -ERASE, 47875201966080, 47875201966080, -STORE, 47875201966080, 47875201974271, -STORE, 47875201974272, 47875202572287, -STORE, 47875202093056, 47875202572287, -STORE, 47875201974272, 47875202093055, -ERASE, 47875202093056, 47875202093056, -STORE, 47875202093056, 47875202519039, -STORE, 47875202519040, 47875202572287, -STORE, 47875202408448, 47875202519039, -STORE, 47875202093056, 47875202408447, -ERASE, 47875202093056, 47875202093056, -STORE, 47875202093056, 47875202408447, -STORE, 47875202514944, 47875202519039, -STORE, 47875202408448, 47875202514943, -ERASE, 47875202408448, 47875202408448, -STORE, 47875202408448, 47875202514943, -ERASE, 47875202519040, 47875202519040, -STORE, 47875202519040, 47875202572287, -STORE, 47875202572288, 47875205623807, -STORE, 47875203117056, 47875205623807, -STORE, 47875202572288, 47875203117055, -ERASE, 47875203117056, 47875203117056, -STORE, 47875203117056, 47875205402623, -STORE, 47875205402624, 47875205623807, -STORE, 47875204812800, 47875205402623, -STORE, 47875203117056, 47875204812799, -ERASE, 47875203117056, 47875203117056, -STORE, 47875203117056, 47875204812799, -STORE, 47875205398528, 47875205402623, -STORE, 47875204812800, 47875205398527, -ERASE, 47875204812800, 47875204812800, -STORE, 47875204812800, 47875205398527, -STORE, 47875205607424, 47875205623807, -STORE, 47875205402624, 47875205607423, -ERASE, 47875205402624, 47875205402624, -STORE, 47875205402624, 47875205607423, -ERASE, 47875205607424, 47875205607424, -STORE, 47875205607424, 47875205623807, -STORE, 47875205623808, 47875205656575, -ERASE, 47875205623808, 47875205623808, -STORE, 47875205623808, 47875205631999, -STORE, 47875205632000, 47875205656575, -STORE, 47875205644288, 47875205656575, -STORE, 47875205632000, 47875205644287, -ERASE, 47875205632000, 47875205632000, -STORE, 47875205632000, 47875205644287, -STORE, 47875205648384, 47875205656575, -STORE, 47875205644288, 47875205648383, -ERASE, 47875205644288, 47875205644288, -STORE, 47875205644288, 47875205656575, -ERASE, 47875205644288, 47875205644288, -STORE, 47875205644288, 47875205648383, -STORE, 47875205648384, 47875205656575, -ERASE, 47875205648384, 47875205648384, -STORE, 47875205648384, 47875205656575, -STORE, 47875205656576, 47875205693439, -ERASE, 47875205656576, 47875205656576, -STORE, 47875205656576, 47875205664767, -STORE, 47875205664768, 47875205693439, -STORE, 47875205681152, 47875205693439, -STORE, 47875205664768, 47875205681151, -ERASE, 47875205664768, 47875205664768, -STORE, 47875205664768, 47875205681151, -STORE, 47875205685248, 47875205693439, -STORE, 47875205681152, 47875205685247, -ERASE, 47875205681152, 47875205681152, -STORE, 47875205681152, 47875205693439, -ERASE, 47875205681152, 47875205681152, -STORE, 47875205681152, 47875205685247, -STORE, 47875205685248, 47875205693439, -ERASE, 47875205685248, 47875205685248, -STORE, 47875205685248, 47875205693439, -STORE, 47875205693440, 47875206168575, -ERASE, 47875205693440, 47875205693440, -STORE, 47875205693440, 47875205701631, -STORE, 47875205701632, 47875206168575, -STORE, 47875206037504, 47875206168575, -STORE, 47875205701632, 47875206037503, -ERASE, 47875205701632, 47875205701632, -STORE, 47875205701632, 47875206037503, -STORE, 47875206160384, 47875206168575, -STORE, 47875206037504, 47875206160383, -ERASE, 47875206037504, 47875206037504, -STORE, 47875206037504, 47875206168575, -ERASE, 47875206037504, 47875206037504, -STORE, 47875206037504, 47875206160383, -STORE, 47875206160384, 47875206168575, -ERASE, 47875206160384, 47875206160384, -STORE, 47875206160384, 47875206168575, -STORE, 47875206168576, 47875206176767, -STORE, 47875206176768, 47875206197247, -ERASE, 47875206176768, 47875206176768, -STORE, 47875206176768, 47875206180863, -STORE, 47875206180864, 47875206197247, -STORE, 47875206184960, 47875206197247, -STORE, 47875206180864, 47875206184959, -ERASE, 47875206180864, 47875206180864, -STORE, 47875206180864, 47875206184959, -STORE, 47875206189056, 47875206197247, -STORE, 47875206184960, 47875206189055, -ERASE, 47875206184960, 47875206184960, -STORE, 47875206184960, 47875206197247, -ERASE, 47875206184960, 47875206184960, -STORE, 47875206184960, 47875206189055, -STORE, 47875206189056, 47875206197247, -ERASE, 47875206189056, 47875206189056, -STORE, 47875206189056, 47875206197247, -STORE, 47875206197248, 47875206205439, -ERASE, 47875198861312, 47875198861312, -STORE, 47875198861312, 47875198877695, -STORE, 47875198877696, 47875198885887, -ERASE, 47875206189056, 47875206189056, -STORE, 47875206189056, 47875206193151, -STORE, 47875206193152, 47875206197247, -ERASE, 47875201777664, 47875201777664, -STORE, 47875201777664, 47875201781759, -STORE, 47875201781760, 47875201785855, -ERASE, 47875206160384, 47875206160384, -STORE, 47875206160384, 47875206164479, -STORE, 47875206164480, 47875206168575, -ERASE, 47875205685248, 47875205685248, -STORE, 47875205685248, 47875205689343, -STORE, 47875205689344, 47875205693439, -ERASE, 47875205648384, 47875205648384, -STORE, 47875205648384, 47875205652479, -STORE, 47875205652480, 47875205656575, -ERASE, 47875205402624, 47875205402624, -STORE, 47875205402624, 47875205599231, -STORE, 47875205599232, 47875205607423, -ERASE, 47875202519040, 47875202519040, -STORE, 47875202519040, 47875202555903, -STORE, 47875202555904, 47875202572287, -ERASE, 47875201966080, 47875201966080, -STORE, 47875201966080, 47875201970175, -STORE, 47875201970176, 47875201974271, -ERASE, 47875201650688, 47875201650688, -STORE, 47875201650688, 47875201654783, -STORE, 47875201654784, 47875201658879, -ERASE, 47875199381504, 47875199381504, -STORE, 47875199381504, 47875199401983, -STORE, 47875199401984, 47875199406079, -ERASE, 47875199049728, 47875199049728, -STORE, 47875199049728, 47875199053823, -STORE, 47875199053824, 47875199057919, -ERASE, 47875199004672, 47875199004672, -STORE, 47875199004672, 47875199008767, -STORE, 47875199008768, 47875199012863, -ERASE, 94011547025408, 94011547025408, -STORE, 94011547025408, 94011547148287, -STORE, 94011547148288, 94011547152383, -ERASE, 139757598109696, 139757598109696, -STORE, 139757598109696, 139757598113791, -STORE, 139757598113792, 139757598117887, -ERASE, 47875197046784, 47875197046784, -STORE, 94011557584896, 94011557720063, -STORE, 94011557584896, 94011557855231, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557851135, -STORE, 94011557851136, 94011557855231, -ERASE, 94011557851136, 94011557851136, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557847039, -STORE, 94011557847040, 94011557851135, -ERASE, 94011557847040, 94011557847040, -STORE, 94011557584896, 94011557982207, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557978111, -STORE, 94011557978112, 94011557982207, -ERASE, 94011557978112, 94011557978112, -ERASE, 94011557584896, 94011557584896, -STORE, 94011557584896, 94011557974015, -STORE, 94011557974016, 94011557978111, -ERASE, 94011557974016, 94011557974016, -STORE, 140737488347136, 140737488351231, -STORE, 140734130360320, 140737488351231, -ERASE, 140734130360320, 140734130360320, -STORE, 140734130360320, 140734130364415, -STORE, 94641232105472, 94641232785407, -ERASE, 94641232105472, 94641232105472, -STORE, 94641232105472, 94641232171007, -STORE, 94641232171008, 94641232785407, -ERASE, 94641232171008, 94641232171008, -STORE, 94641232171008, 94641232519167, -STORE, 94641232519168, 94641232658431, -STORE, 94641232658432, 94641232785407, -STORE, 139726599516160, 139726599688191, -ERASE, 139726599516160, 139726599516160, -STORE, 139726599516160, 139726599520255, -STORE, 139726599520256, 139726599688191, -ERASE, 139726599520256, 139726599520256, -STORE, 139726599520256, 139726599643135, -STORE, 139726599643136, 139726599675903, -STORE, 139726599675904, 139726599684095, -STORE, 139726599684096, 139726599688191, -STORE, 140734130446336, 140734130450431, -STORE, 140734130434048, 140734130446335, -STORE, 47906195480576, 47906195488767, -STORE, 47906195488768, 47906195496959, -STORE, 47906195496960, 47906197336063, -STORE, 47906195636224, 47906197336063, -STORE, 47906195496960, 47906195636223, -ERASE, 47906195636224, 47906195636224, -STORE, 47906195636224, 47906197295103, -STORE, 47906197295104, 47906197336063, -STORE, 47906196979712, 47906197295103, -STORE, 47906195636224, 47906196979711, -ERASE, 47906195636224, 47906195636224, -STORE, 47906195636224, 47906196979711, -STORE, 47906197291008, 47906197295103, -STORE, 47906196979712, 47906197291007, -ERASE, 47906196979712, 47906196979712, -STORE, 47906196979712, 47906197291007, -STORE, 47906197319680, 47906197336063, -STORE, 47906197295104, 47906197319679, -ERASE, 47906197295104, 47906197295104, -STORE, 47906197295104, 47906197319679, -ERASE, 47906197319680, 47906197319680, -STORE, 47906197319680, 47906197336063, -STORE, 47906197336064, 47906197446655, -STORE, 47906197352448, 47906197446655, -STORE, 47906197336064, 47906197352447, -ERASE, 47906197352448, 47906197352448, -STORE, 47906197352448, 47906197438463, -STORE, 47906197438464, 47906197446655, -STORE, 47906197413888, 47906197438463, -STORE, 47906197352448, 47906197413887, -ERASE, 47906197352448, 47906197352448, -STORE, 47906197352448, 47906197413887, -STORE, 47906197434368, 47906197438463, -STORE, 47906197413888, 47906197434367, -ERASE, 47906197413888, 47906197413888, -STORE, 47906197413888, 47906197434367, -ERASE, 47906197438464, 47906197438464, -STORE, 47906197438464, 47906197446655, -STORE, 47906197446656, 47906197491711, -ERASE, 47906197446656, 47906197446656, -STORE, 47906197446656, 47906197454847, -STORE, 47906197454848, 47906197491711, -STORE, 47906197475328, 47906197491711, -STORE, 47906197454848, 47906197475327, -ERASE, 47906197454848, 47906197454848, -STORE, 47906197454848, 47906197475327, -STORE, 47906197483520, 47906197491711, -STORE, 47906197475328, 47906197483519, -ERASE, 47906197475328, 47906197475328, -STORE, 47906197475328, 47906197491711, -ERASE, 47906197475328, 47906197475328, -STORE, 47906197475328, 47906197483519, -STORE, 47906197483520, 47906197491711, -ERASE, 47906197483520, 47906197483520, -STORE, 47906197483520, 47906197491711, -STORE, 47906197491712, 47906197839871, -STORE, 47906197532672, 47906197839871, -STORE, 47906197491712, 47906197532671, -ERASE, 47906197532672, 47906197532672, -STORE, 47906197532672, 47906197815295, -STORE, 47906197815296, 47906197839871, -STORE, 47906197745664, 47906197815295, -STORE, 47906197532672, 47906197745663, -ERASE, 47906197532672, 47906197532672, -STORE, 47906197532672, 47906197745663, -STORE, 47906197811200, 47906197815295, -STORE, 47906197745664, 47906197811199, -ERASE, 47906197745664, 47906197745664, -STORE, 47906197745664, 47906197811199, -ERASE, 47906197815296, 47906197815296, -STORE, 47906197815296, 47906197839871, -STORE, 47906197839872, 47906200100863, -STORE, 47906197991424, 47906200100863, -STORE, 47906197839872, 47906197991423, -ERASE, 47906197991424, 47906197991424, -STORE, 47906197991424, 47906200084479, -STORE, 47906200084480, 47906200100863, -STORE, 47906200092672, 47906200100863, -STORE, 47906200084480, 47906200092671, -ERASE, 47906200084480, 47906200084480, -STORE, 47906200084480, 47906200092671, -ERASE, 47906200092672, 47906200092672, -STORE, 47906200092672, 47906200100863, -STORE, 47906200100864, 47906200236031, -ERASE, 47906200100864, 47906200100864, -STORE, 47906200100864, 47906200125439, -STORE, 47906200125440, 47906200236031, -STORE, 47906200186880, 47906200236031, -STORE, 47906200125440, 47906200186879, -ERASE, 47906200125440, 47906200125440, -STORE, 47906200125440, 47906200186879, -STORE, 47906200211456, 47906200236031, -STORE, 47906200186880, 47906200211455, -ERASE, 47906200186880, 47906200186880, -STORE, 47906200186880, 47906200236031, -ERASE, 47906200186880, 47906200186880, -STORE, 47906200186880, 47906200211455, -STORE, 47906200211456, 47906200236031, -STORE, 47906200219648, 47906200236031, -STORE, 47906200211456, 47906200219647, -ERASE, 47906200211456, 47906200211456, -STORE, 47906200211456, 47906200219647, -ERASE, 47906200219648, 47906200219648, -STORE, 47906200219648, 47906200236031, -STORE, 47906200219648, 47906200244223, -STORE, 47906200244224, 47906200408063, -ERASE, 47906200244224, 47906200244224, -STORE, 47906200244224, 47906200256511, -STORE, 47906200256512, 47906200408063, -STORE, 47906200354816, 47906200408063, -STORE, 47906200256512, 47906200354815, -ERASE, 47906200256512, 47906200256512, -STORE, 47906200256512, 47906200354815, -STORE, 47906200399872, 47906200408063, -STORE, 47906200354816, 47906200399871, -ERASE, 47906200354816, 47906200354816, -STORE, 47906200354816, 47906200408063, -ERASE, 47906200354816, 47906200354816, -STORE, 47906200354816, 47906200399871, -STORE, 47906200399872, 47906200408063, -ERASE, 47906200399872, 47906200399872, -STORE, 47906200399872, 47906200408063, -STORE, 47906200408064, 47906201006079, -STORE, 47906200526848, 47906201006079, -STORE, 47906200408064, 47906200526847, -ERASE, 47906200526848, 47906200526848, -STORE, 47906200526848, 47906200952831, -STORE, 47906200952832, 47906201006079, -STORE, 47906200842240, 47906200952831, -STORE, 47906200526848, 47906200842239, -ERASE, 47906200526848, 47906200526848, -STORE, 47906200526848, 47906200842239, -STORE, 47906200948736, 47906200952831, -STORE, 47906200842240, 47906200948735, -ERASE, 47906200842240, 47906200842240, -STORE, 47906200842240, 47906200948735, -ERASE, 47906200952832, 47906200952832, -STORE, 47906200952832, 47906201006079, -STORE, 47906201006080, 47906204057599, -STORE, 47906201550848, 47906204057599, -STORE, 47906201006080, 47906201550847, -ERASE, 47906201550848, 47906201550848, -STORE, 47906201550848, 47906203836415, -STORE, 47906203836416, 47906204057599, -STORE, 47906203246592, 47906203836415, -STORE, 47906201550848, 47906203246591, -ERASE, 47906201550848, 47906201550848, -STORE, 47906201550848, 47906203246591, -STORE, 47906203832320, 47906203836415, -STORE, 47906203246592, 47906203832319, -ERASE, 47906203246592, 47906203246592, -STORE, 47906203246592, 47906203832319, -STORE, 47906204041216, 47906204057599, -STORE, 47906203836416, 47906204041215, -ERASE, 47906203836416, 47906203836416, -STORE, 47906203836416, 47906204041215, -ERASE, 47906204041216, 47906204041216, -STORE, 47906204041216, 47906204057599, -STORE, 47906204057600, 47906204090367, -ERASE, 47906204057600, 47906204057600, -STORE, 47906204057600, 47906204065791, -STORE, 47906204065792, 47906204090367, -STORE, 47906204078080, 47906204090367, -STORE, 47906204065792, 47906204078079, -ERASE, 47906204065792, 47906204065792, -STORE, 47906204065792, 47906204078079, -STORE, 47906204082176, 47906204090367, -STORE, 47906204078080, 47906204082175, -ERASE, 47906204078080, 47906204078080, -STORE, 47906204078080, 47906204090367, -ERASE, 47906204078080, 47906204078080, -STORE, 47906204078080, 47906204082175, -STORE, 47906204082176, 47906204090367, -ERASE, 47906204082176, 47906204082176, -STORE, 47906204082176, 47906204090367, -STORE, 47906204090368, 47906204127231, -ERASE, 47906204090368, 47906204090368, -STORE, 47906204090368, 47906204098559, -STORE, 47906204098560, 47906204127231, -STORE, 47906204114944, 47906204127231, -STORE, 47906204098560, 47906204114943, -ERASE, 47906204098560, 47906204098560, -STORE, 47906204098560, 47906204114943, -STORE, 47906204119040, 47906204127231, -STORE, 47906204114944, 47906204119039, -ERASE, 47906204114944, 47906204114944, -STORE, 47906204114944, 47906204127231, -ERASE, 47906204114944, 47906204114944, -STORE, 47906204114944, 47906204119039, -STORE, 47906204119040, 47906204127231, -ERASE, 47906204119040, 47906204119040, -STORE, 47906204119040, 47906204127231, -STORE, 47906204127232, 47906204602367, -ERASE, 47906204127232, 47906204127232, -STORE, 47906204127232, 47906204135423, -STORE, 47906204135424, 47906204602367, -STORE, 47906204471296, 47906204602367, -STORE, 47906204135424, 47906204471295, -ERASE, 47906204135424, 47906204135424, -STORE, 47906204135424, 47906204471295, -STORE, 47906204594176, 47906204602367, -STORE, 47906204471296, 47906204594175, -ERASE, 47906204471296, 47906204471296, -STORE, 47906204471296, 47906204602367, -ERASE, 47906204471296, 47906204471296, -STORE, 47906204471296, 47906204594175, -STORE, 47906204594176, 47906204602367, -ERASE, 47906204594176, 47906204594176, -STORE, 47906204594176, 47906204602367, -STORE, 47906204602368, 47906204610559, -STORE, 47906204610560, 47906204631039, -ERASE, 47906204610560, 47906204610560, -STORE, 47906204610560, 47906204614655, -STORE, 47906204614656, 47906204631039, -STORE, 47906204618752, 47906204631039, -STORE, 47906204614656, 47906204618751, -ERASE, 47906204614656, 47906204614656, -STORE, 47906204614656, 47906204618751, -STORE, 47906204622848, 47906204631039, -STORE, 47906204618752, 47906204622847, -ERASE, 47906204618752, 47906204618752, -STORE, 47906204618752, 47906204631039, -ERASE, 47906204618752, 47906204618752, -STORE, 47906204618752, 47906204622847, -STORE, 47906204622848, 47906204631039, -ERASE, 47906204622848, 47906204622848, -STORE, 47906204622848, 47906204631039, -STORE, 47906204631040, 47906204639231, -ERASE, 47906197295104, 47906197295104, -STORE, 47906197295104, 47906197311487, -STORE, 47906197311488, 47906197319679, -ERASE, 47906204622848, 47906204622848, -STORE, 47906204622848, 47906204626943, -STORE, 47906204626944, 47906204631039, -ERASE, 47906200211456, 47906200211456, -STORE, 47906200211456, 47906200215551, -STORE, 47906200215552, 47906200219647, -ERASE, 47906204594176, 47906204594176, -STORE, 47906204594176, 47906204598271, -STORE, 47906204598272, 47906204602367, -ERASE, 47906204119040, 47906204119040, -STORE, 47906204119040, 47906204123135, -STORE, 47906204123136, 47906204127231, -ERASE, 47906204082176, 47906204082176, -STORE, 47906204082176, 47906204086271, -STORE, 47906204086272, 47906204090367, -ERASE, 47906203836416, 47906203836416, -STORE, 47906203836416, 47906204033023, -STORE, 47906204033024, 47906204041215, -ERASE, 47906200952832, 47906200952832, -STORE, 47906200952832, 47906200989695, -STORE, 47906200989696, 47906201006079, -ERASE, 47906200399872, 47906200399872, -STORE, 47906200399872, 47906200403967, -STORE, 47906200403968, 47906200408063, -ERASE, 47906200084480, 47906200084480, -STORE, 47906200084480, 47906200088575, -STORE, 47906200088576, 47906200092671, -ERASE, 47906197815296, 47906197815296, -STORE, 47906197815296, 47906197835775, -STORE, 47906197835776, 47906197839871, -ERASE, 47906197483520, 47906197483520, -STORE, 47906197483520, 47906197487615, -STORE, 47906197487616, 47906197491711, -ERASE, 47906197438464, 47906197438464, -STORE, 47906197438464, 47906197442559, -STORE, 47906197442560, 47906197446655, -ERASE, 94641232658432, 94641232658432, -STORE, 94641232658432, 94641232781311, -STORE, 94641232781312, 94641232785407, -ERASE, 139726599675904, 139726599675904, -STORE, 139726599675904, 139726599679999, -STORE, 139726599680000, 139726599684095, -ERASE, 47906195480576, 47906195480576, -STORE, 94641242615808, 94641242750975, - }; - unsigned long set11[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140732658499584, 140737488351231, -ERASE, 140732658499584, 140732658499584, -STORE, 140732658499584, 140732658503679, -STORE, 94029856579584, 94029856751615, -ERASE, 94029856579584, 94029856579584, -STORE, 94029856579584, 94029856595967, -STORE, 94029856595968, 94029856751615, -ERASE, 94029856595968, 94029856595968, -STORE, 94029856595968, 94029856698367, -STORE, 94029856698368, 94029856739327, -STORE, 94029856739328, 94029856751615, -STORE, 140014592573440, 140014592745471, -ERASE, 140014592573440, 140014592573440, -STORE, 140014592573440, 140014592577535, -STORE, 140014592577536, 140014592745471, -ERASE, 140014592577536, 140014592577536, -STORE, 140014592577536, 140014592700415, -STORE, 140014592700416, 140014592733183, -STORE, 140014592733184, 140014592741375, -STORE, 140014592741376, 140014592745471, -STORE, 140732658565120, 140732658569215, -STORE, 140732658552832, 140732658565119, - }; - - unsigned long set12[] = { /* contains 12 values. */ -STORE, 140737488347136, 140737488351231, -STORE, 140732658499584, 140737488351231, -ERASE, 140732658499584, 140732658499584, -STORE, 140732658499584, 140732658503679, -STORE, 94029856579584, 94029856751615, -ERASE, 94029856579584, 94029856579584, -STORE, 94029856579584, 94029856595967, -STORE, 94029856595968, 94029856751615, -ERASE, 94029856595968, 94029856595968, -STORE, 94029856595968, 94029856698367, -STORE, 94029856698368, 94029856739327, -STORE, 94029856739328, 94029856751615, -STORE, 140014592573440, 140014592745471, -ERASE, 140014592573440, 140014592573440, -STORE, 140014592573440, 140014592577535, -STORE, 140014592577536, 140014592745471, -ERASE, 140014592577536, 140014592577536, -STORE, 140014592577536, 140014592700415, -STORE, 140014592700416, 140014592733183, -STORE, 140014592733184, 140014592741375, -STORE, 140014592741376, 140014592745471, -STORE, 140732658565120, 140732658569215, -STORE, 140732658552832, 140732658565119, -STORE, 140014592741375, 140014592741375, /* contrived */ -STORE, 140014592733184, 140014592741376, /* creates first entry retry. */ - }; - unsigned long set13[] = { -STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */ -STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */ -STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */ -STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */ -STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */ -STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */ -STORE, 140373518663680, 140373518684159,/*: deleted (257) */ -STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */ -STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */ -STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */ -STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */ - }; - unsigned long set14[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140731667996672, 140737488351231, -SNULL, 140731668000767, 140737488351231, -STORE, 140731667996672, 140731668000767, -STORE, 140731667865600, 140731668000767, -STORE, 94077521272832, 94077521313791, -SNULL, 94077521301503, 94077521313791, -STORE, 94077521272832, 94077521301503, -STORE, 94077521301504, 94077521313791, -ERASE, 94077521301504, 94077521313791, -STORE, 94077521305600, 94077521313791, -STORE, 139826134630400, 139826136883199, -SNULL, 139826134773759, 139826136883199, -STORE, 139826134630400, 139826134773759, -STORE, 139826134773760, 139826136883199, -ERASE, 139826134773760, 139826136883199, -STORE, 139826136870912, 139826136879103, -STORE, 139826136879104, 139826136883199, -STORE, 140731668013056, 140731668017151, -STORE, 140731668000768, 140731668013055, -STORE, 139826136862720, 139826136870911, -STORE, 139826132406272, 139826134630399, -SNULL, 139826134056959, 139826134630399, -STORE, 139826132406272, 139826134056959, -STORE, 139826134056960, 139826134630399, -SNULL, 139826134056960, 139826134626303, -STORE, 139826134626304, 139826134630399, -STORE, 139826134056960, 139826134626303, -ERASE, 139826134056960, 139826134626303, -STORE, 139826134056960, 139826134626303, -ERASE, 139826134626304, 139826134630399, -STORE, 139826134626304, 139826134630399, -STORE, 139826136842240, 139826136862719, -STORE, 139826130022400, 139826132406271, -SNULL, 139826130022400, 139826130288639, -STORE, 139826130288640, 139826132406271, -STORE, 139826130022400, 139826130288639, -SNULL, 139826132381695, 139826132406271, -STORE, 139826130288640, 139826132381695, -STORE, 139826132381696, 139826132406271, -SNULL, 139826132381696, 139826132402175, -STORE, 139826132402176, 139826132406271, -STORE, 139826132381696, 139826132402175, -ERASE, 139826132381696, 139826132402175, -STORE, 139826132381696, 139826132402175, -ERASE, 139826132402176, 139826132406271, -STORE, 139826132402176, 139826132406271, -STORE, 139826127806464, 139826130022399, -SNULL, 139826127806464, 139826127904767, -STORE, 139826127904768, 139826130022399, -STORE, 139826127806464, 139826127904767, -SNULL, 139826129997823, 139826130022399, -STORE, 139826127904768, 139826129997823, -STORE, 139826129997824, 139826130022399, -SNULL, 139826129997824, 139826130006015, -STORE, 139826130006016, 139826130022399, -STORE, 139826129997824, 139826130006015, -ERASE, 139826129997824, 139826130006015, -STORE, 139826129997824, 139826130006015, -ERASE, 139826130006016, 139826130022399, -STORE, 139826130006016, 139826130022399, -STORE, 139826124009472, 139826127806463, -SNULL, 139826124009472, 139826125668351, -STORE, 139826125668352, 139826127806463, -STORE, 139826124009472, 139826125668351, -SNULL, 139826127765503, 139826127806463, -STORE, 139826125668352, 139826127765503, -STORE, 139826127765504, 139826127806463, -SNULL, 139826127765504, 139826127790079, -STORE, 139826127790080, 139826127806463, -STORE, 139826127765504, 139826127790079, -ERASE, 139826127765504, 139826127790079, -STORE, 139826127765504, 139826127790079, -ERASE, 139826127790080, 139826127806463, -STORE, 139826127790080, 139826127806463, -STORE, 139826121748480, 139826124009471, -SNULL, 139826121748480, 139826121900031, -STORE, 139826121900032, 139826124009471, -STORE, 139826121748480, 139826121900031, -SNULL, 139826123993087, 139826124009471, -STORE, 139826121900032, 139826123993087, -STORE, 139826123993088, 139826124009471, -SNULL, 139826123993088, 139826124001279, -STORE, 139826124001280, 139826124009471, -STORE, 139826123993088, 139826124001279, -ERASE, 139826123993088, 139826124001279, -STORE, 139826123993088, 139826124001279, -ERASE, 139826124001280, 139826124009471, -STORE, 139826124001280, 139826124009471, -STORE, 139826119626752, 139826121748479, -SNULL, 139826119626752, 139826119643135, -STORE, 139826119643136, 139826121748479, -STORE, 139826119626752, 139826119643135, -SNULL, 139826121740287, 139826121748479, -STORE, 139826119643136, 139826121740287, -STORE, 139826121740288, 139826121748479, -ERASE, 139826121740288, 139826121748479, -STORE, 139826121740288, 139826121748479, -STORE, 139826136834048, 139826136842239, -STORE, 139826117496832, 139826119626751, -SNULL, 139826117496832, 139826117525503, -STORE, 139826117525504, 139826119626751, -STORE, 139826117496832, 139826117525503, -SNULL, 139826119618559, 139826119626751, -STORE, 139826117525504, 139826119618559, -STORE, 139826119618560, 139826119626751, -ERASE, 139826119618560, 139826119626751, -STORE, 139826119618560, 139826119626751, -STORE, 139826115244032, 139826117496831, -SNULL, 139826115244032, 139826115395583, -STORE, 139826115395584, 139826117496831, -STORE, 139826115244032, 139826115395583, -SNULL, 139826117488639, 139826117496831, -STORE, 139826115395584, 139826117488639, -STORE, 139826117488640, 139826117496831, -ERASE, 139826117488640, 139826117496831, -STORE, 139826117488640, 139826117496831, -STORE, 139826113073152, 139826115244031, -SNULL, 139826113073152, 139826113142783, -STORE, 139826113142784, 139826115244031, -STORE, 139826113073152, 139826113142783, -SNULL, 139826115235839, 139826115244031, -STORE, 139826113142784, 139826115235839, -STORE, 139826115235840, 139826115244031, -ERASE, 139826115235840, 139826115244031, -STORE, 139826115235840, 139826115244031, -STORE, 139826109861888, 139826113073151, -SNULL, 139826109861888, 139826110939135, -STORE, 139826110939136, 139826113073151, -STORE, 139826109861888, 139826110939135, -SNULL, 139826113036287, 139826113073151, -STORE, 139826110939136, 139826113036287, -STORE, 139826113036288, 139826113073151, -ERASE, 139826113036288, 139826113073151, -STORE, 139826113036288, 139826113073151, -STORE, 139826107727872, 139826109861887, -SNULL, 139826107727872, 139826107756543, -STORE, 139826107756544, 139826109861887, -STORE, 139826107727872, 139826107756543, -SNULL, 139826109853695, 139826109861887, -STORE, 139826107756544, 139826109853695, -STORE, 139826109853696, 139826109861887, -ERASE, 139826109853696, 139826109861887, -STORE, 139826109853696, 139826109861887, -STORE, 139826105417728, 139826107727871, -SNULL, 139826105417728, 139826105622527, -STORE, 139826105622528, 139826107727871, -STORE, 139826105417728, 139826105622527, -SNULL, 139826107719679, 139826107727871, -STORE, 139826105622528, 139826107719679, -STORE, 139826107719680, 139826107727871, -ERASE, 139826107719680, 139826107727871, -STORE, 139826107719680, 139826107727871, -STORE, 139826136825856, 139826136842239, -STORE, 139826103033856, 139826105417727, -SNULL, 139826103033856, 139826103226367, -STORE, 139826103226368, 139826105417727, -STORE, 139826103033856, 139826103226367, -SNULL, 139826105319423, 139826105417727, -STORE, 139826103226368, 139826105319423, -STORE, 139826105319424, 139826105417727, -ERASE, 139826105319424, 139826105417727, -STORE, 139826105319424, 139826105417727, -STORE, 139826100916224, 139826103033855, -SNULL, 139826100916224, 139826100932607, -STORE, 139826100932608, 139826103033855, -STORE, 139826100916224, 139826100932607, -SNULL, 139826103025663, 139826103033855, -STORE, 139826100932608, 139826103025663, -STORE, 139826103025664, 139826103033855, -ERASE, 139826103025664, 139826103033855, -STORE, 139826103025664, 139826103033855, -STORE, 139826098348032, 139826100916223, -SNULL, 139826098348032, 139826098814975, -STORE, 139826098814976, 139826100916223, -STORE, 139826098348032, 139826098814975, -SNULL, 139826100908031, 139826100916223, -STORE, 139826098814976, 139826100908031, -STORE, 139826100908032, 139826100916223, -ERASE, 139826100908032, 139826100916223, -STORE, 139826100908032, 139826100916223, -STORE, 139826096234496, 139826098348031, -SNULL, 139826096234496, 139826096246783, -STORE, 139826096246784, 139826098348031, -STORE, 139826096234496, 139826096246783, -SNULL, 139826098339839, 139826098348031, -STORE, 139826096246784, 139826098339839, -STORE, 139826098339840, 139826098348031, -ERASE, 139826098339840, 139826098348031, -STORE, 139826098339840, 139826098348031, -STORE, 139826094055424, 139826096234495, -SNULL, 139826094055424, 139826094133247, -STORE, 139826094133248, 139826096234495, -STORE, 139826094055424, 139826094133247, -SNULL, 139826096226303, 139826096234495, -STORE, 139826094133248, 139826096226303, -STORE, 139826096226304, 139826096234495, -ERASE, 139826096226304, 139826096234495, -STORE, 139826096226304, 139826096234495, -STORE, 139826136817664, 139826136842239, -STORE, 139826091937792, 139826094055423, -SNULL, 139826091937792, 139826091954175, -STORE, 139826091954176, 139826094055423, -STORE, 139826091937792, 139826091954175, -SNULL, 139826094047231, 139826094055423, -STORE, 139826091954176, 139826094047231, -STORE, 139826094047232, 139826094055423, -ERASE, 139826094047232, 139826094055423, -STORE, 139826094047232, 139826094055423, -STORE, 139826136809472, 139826136842239, -SNULL, 139826127781887, 139826127790079, -STORE, 139826127765504, 139826127781887, -STORE, 139826127781888, 139826127790079, -SNULL, 139826094051327, 139826094055423, -STORE, 139826094047232, 139826094051327, -STORE, 139826094051328, 139826094055423, -SNULL, 139826096230399, 139826096234495, -STORE, 139826096226304, 139826096230399, -STORE, 139826096230400, 139826096234495, -SNULL, 139826098343935, 139826098348031, -STORE, 139826098339840, 139826098343935, -STORE, 139826098343936, 139826098348031, -SNULL, 139826130001919, 139826130006015, -STORE, 139826129997824, 139826130001919, -STORE, 139826130001920, 139826130006015, -SNULL, 139826100912127, 139826100916223, -STORE, 139826100908032, 139826100912127, -STORE, 139826100912128, 139826100916223, -SNULL, 139826103029759, 139826103033855, -STORE, 139826103025664, 139826103029759, -STORE, 139826103029760, 139826103033855, -SNULL, 139826105413631, 139826105417727, -STORE, 139826105319424, 139826105413631, -STORE, 139826105413632, 139826105417727, -SNULL, 139826107723775, 139826107727871, -STORE, 139826107719680, 139826107723775, -STORE, 139826107723776, 139826107727871, -SNULL, 139826109857791, 139826109861887, -STORE, 139826109853696, 139826109857791, -STORE, 139826109857792, 139826109861887, -SNULL, 139826113044479, 139826113073151, -STORE, 139826113036288, 139826113044479, -STORE, 139826113044480, 139826113073151, -SNULL, 139826115239935, 139826115244031, -STORE, 139826115235840, 139826115239935, -STORE, 139826115239936, 139826115244031, -SNULL, 139826117492735, 139826117496831, -STORE, 139826117488640, 139826117492735, -STORE, 139826117492736, 139826117496831, -SNULL, 139826119622655, 139826119626751, -STORE, 139826119618560, 139826119622655, -STORE, 139826119622656, 139826119626751, -SNULL, 139826121744383, 139826121748479, -STORE, 139826121740288, 139826121744383, -STORE, 139826121744384, 139826121748479, -SNULL, 139826123997183, 139826124001279, -STORE, 139826123993088, 139826123997183, -STORE, 139826123997184, 139826124001279, -SNULL, 139826132398079, 139826132402175, -STORE, 139826132381696, 139826132398079, -STORE, 139826132398080, 139826132402175, -SNULL, 139826134622207, 139826134626303, -STORE, 139826134056960, 139826134622207, -STORE, 139826134622208, 139826134626303, -SNULL, 94077521309695, 94077521313791, -STORE, 94077521305600, 94077521309695, -STORE, 94077521309696, 94077521313791, -SNULL, 139826136875007, 139826136879103, -STORE, 139826136870912, 139826136875007, -STORE, 139826136875008, 139826136879103, -ERASE, 139826136842240, 139826136862719, -STORE, 94077554049024, 94077554184191, -STORE, 139826136543232, 139826136842239, -STORE, 139826136276992, 139826136842239, -STORE, 139826136010752, 139826136842239, -STORE, 139826135744512, 139826136842239, -SNULL, 139826136543231, 139826136842239, -STORE, 139826135744512, 139826136543231, -STORE, 139826136543232, 139826136842239, -SNULL, 139826136543232, 139826136809471, -STORE, 139826136809472, 139826136842239, -STORE, 139826136543232, 139826136809471, - }; - unsigned long set15[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140722061451264, 140737488351231, -SNULL, 140722061455359, 140737488351231, -STORE, 140722061451264, 140722061455359, -STORE, 140722061320192, 140722061455359, -STORE, 94728600248320, 94728600289279, -SNULL, 94728600276991, 94728600289279, -STORE, 94728600248320, 94728600276991, -STORE, 94728600276992, 94728600289279, -ERASE, 94728600276992, 94728600289279, -STORE, 94728600281088, 94728600289279, -STORE, 139906806779904, 139906809032703, -SNULL, 139906806923263, 139906809032703, -STORE, 139906806779904, 139906806923263, -STORE, 139906806923264, 139906809032703, -ERASE, 139906806923264, 139906809032703, -STORE, 139906809020416, 139906809028607, -STORE, 139906809028608, 139906809032703, -STORE, 140722061692928, 140722061697023, -STORE, 140722061680640, 140722061692927, -STORE, 139906809012224, 139906809020415, -STORE, 139906804555776, 139906806779903, -SNULL, 139906806206463, 139906806779903, -STORE, 139906804555776, 139906806206463, -STORE, 139906806206464, 139906806779903, -SNULL, 139906806206464, 139906806775807, -STORE, 139906806775808, 139906806779903, -STORE, 139906806206464, 139906806775807, -ERASE, 139906806206464, 139906806775807, -STORE, 139906806206464, 139906806775807, -ERASE, 139906806775808, 139906806779903, -STORE, 139906806775808, 139906806779903, -STORE, 139906808991744, 139906809012223, -STORE, 139906802171904, 139906804555775, -SNULL, 139906802171904, 139906802438143, -STORE, 139906802438144, 139906804555775, -STORE, 139906802171904, 139906802438143, -SNULL, 139906804531199, 139906804555775, -STORE, 139906802438144, 139906804531199, -STORE, 139906804531200, 139906804555775, -SNULL, 139906804531200, 139906804551679, -STORE, 139906804551680, 139906804555775, -STORE, 139906804531200, 139906804551679, -ERASE, 139906804531200, 139906804551679, -STORE, 139906804531200, 139906804551679, -ERASE, 139906804551680, 139906804555775, -STORE, 139906804551680, 139906804555775, -STORE, 139906799955968, 139906802171903, -SNULL, 139906799955968, 139906800054271, -STORE, 139906800054272, 139906802171903, -STORE, 139906799955968, 139906800054271, -SNULL, 139906802147327, 139906802171903, -STORE, 139906800054272, 139906802147327, -STORE, 139906802147328, 139906802171903, -SNULL, 139906802147328, 139906802155519, -STORE, 139906802155520, 139906802171903, -STORE, 139906802147328, 139906802155519, -ERASE, 139906802147328, 139906802155519, -STORE, 139906802147328, 139906802155519, -ERASE, 139906802155520, 139906802171903, -STORE, 139906802155520, 139906802171903, -STORE, 139906796158976, 139906799955967, -SNULL, 139906796158976, 139906797817855, -STORE, 139906797817856, 139906799955967, -STORE, 139906796158976, 139906797817855, -SNULL, 139906799915007, 139906799955967, -STORE, 139906797817856, 139906799915007, -STORE, 139906799915008, 139906799955967, -SNULL, 139906799915008, 139906799939583, -STORE, 139906799939584, 139906799955967, -STORE, 139906799915008, 139906799939583, -ERASE, 139906799915008, 139906799939583, -STORE, 139906799915008, 139906799939583, -ERASE, 139906799939584, 139906799955967, -STORE, 139906799939584, 139906799955967, -STORE, 139906793897984, 139906796158975, -SNULL, 139906793897984, 139906794049535, -STORE, 139906794049536, 139906796158975, -STORE, 139906793897984, 139906794049535, -SNULL, 139906796142591, 139906796158975, -STORE, 139906794049536, 139906796142591, -STORE, 139906796142592, 139906796158975, -SNULL, 139906796142592, 139906796150783, -STORE, 139906796150784, 139906796158975, -STORE, 139906796142592, 139906796150783, -ERASE, 139906796142592, 139906796150783, -STORE, 139906796142592, 139906796150783, -ERASE, 139906796150784, 139906796158975, -STORE, 139906796150784, 139906796158975, -STORE, 139906791776256, 139906793897983, -SNULL, 139906791776256, 139906791792639, -STORE, 139906791792640, 139906793897983, -STORE, 139906791776256, 139906791792639, -SNULL, 139906793889791, 139906793897983, -STORE, 139906791792640, 139906793889791, -STORE, 139906793889792, 139906793897983, -ERASE, 139906793889792, 139906793897983, -STORE, 139906793889792, 139906793897983, -STORE, 139906808983552, 139906808991743, -STORE, 139906789646336, 139906791776255, -SNULL, 139906789646336, 139906789675007, -STORE, 139906789675008, 139906791776255, -STORE, 139906789646336, 139906789675007, -SNULL, 139906791768063, 139906791776255, -STORE, 139906789675008, 139906791768063, -STORE, 139906791768064, 139906791776255, -ERASE, 139906791768064, 139906791776255, -STORE, 139906791768064, 139906791776255, -STORE, 139906787393536, 139906789646335, -SNULL, 139906787393536, 139906787545087, -STORE, 139906787545088, 139906789646335, -STORE, 139906787393536, 139906787545087, -SNULL, 139906789638143, 139906789646335, -STORE, 139906787545088, 139906789638143, -STORE, 139906789638144, 139906789646335, -ERASE, 139906789638144, 139906789646335, -STORE, 139906789638144, 139906789646335, -STORE, 139906785222656, 139906787393535, -SNULL, 139906785222656, 139906785292287, -STORE, 139906785292288, 139906787393535, -STORE, 139906785222656, 139906785292287, -SNULL, 139906787385343, 139906787393535, -STORE, 139906785292288, 139906787385343, -STORE, 139906787385344, 139906787393535, -ERASE, 139906787385344, 139906787393535, -STORE, 139906787385344, 139906787393535, -STORE, 139906782011392, 139906785222655, -SNULL, 139906782011392, 139906783088639, -STORE, 139906783088640, 139906785222655, -STORE, 139906782011392, 139906783088639, -SNULL, 139906785185791, 139906785222655, -STORE, 139906783088640, 139906785185791, -STORE, 139906785185792, 139906785222655, -ERASE, 139906785185792, 139906785222655, -STORE, 139906785185792, 139906785222655, -STORE, 139906779877376, 139906782011391, -SNULL, 139906779877376, 139906779906047, -STORE, 139906779906048, 139906782011391, -STORE, 139906779877376, 139906779906047, -SNULL, 139906782003199, 139906782011391, -STORE, 139906779906048, 139906782003199, -STORE, 139906782003200, 139906782011391, -ERASE, 139906782003200, 139906782011391, -STORE, 139906782003200, 139906782011391, -STORE, 139906777567232, 139906779877375, -SNULL, 139906777567232, 139906777772031, -STORE, 139906777772032, 139906779877375, -STORE, 139906777567232, 139906777772031, -SNULL, 139906779869183, 139906779877375, -STORE, 139906777772032, 139906779869183, -STORE, 139906779869184, 139906779877375, -ERASE, 139906779869184, 139906779877375, -STORE, 139906779869184, 139906779877375, -STORE, 139906808975360, 139906808991743, -STORE, 139906775183360, 139906777567231, -SNULL, 139906775183360, 139906775375871, -STORE, 139906775375872, 139906777567231, -STORE, 139906775183360, 139906775375871, -SNULL, 139906777468927, 139906777567231, -STORE, 139906775375872, 139906777468927, -STORE, 139906777468928, 139906777567231, -ERASE, 139906777468928, 139906777567231, -STORE, 139906777468928, 139906777567231, -STORE, 139906773065728, 139906775183359, -SNULL, 139906773065728, 139906773082111, -STORE, 139906773082112, 139906775183359, -STORE, 139906773065728, 139906773082111, -SNULL, 139906775175167, 139906775183359, -STORE, 139906773082112, 139906775175167, -STORE, 139906775175168, 139906775183359, -ERASE, 139906775175168, 139906775183359, -STORE, 139906775175168, 139906775183359, -STORE, 139906770497536, 139906773065727, -SNULL, 139906770497536, 139906770964479, -STORE, 139906770964480, 139906773065727, -STORE, 139906770497536, 139906770964479, -SNULL, 139906773057535, 139906773065727, -STORE, 139906770964480, 139906773057535, -STORE, 139906773057536, 139906773065727, -ERASE, 139906773057536, 139906773065727, -STORE, 139906773057536, 139906773065727, -STORE, 139906768384000, 139906770497535, -SNULL, 139906768384000, 139906768396287, -STORE, 139906768396288, 139906770497535, -STORE, 139906768384000, 139906768396287, -SNULL, 139906770489343, 139906770497535, -STORE, 139906768396288, 139906770489343, -STORE, 139906770489344, 139906770497535, -ERASE, 139906770489344, 139906770497535, -STORE, 139906770489344, 139906770497535, -STORE, 139906766204928, 139906768383999, -SNULL, 139906766204928, 139906766282751, -STORE, 139906766282752, 139906768383999, -STORE, 139906766204928, 139906766282751, -SNULL, 139906768375807, 139906768383999, -STORE, 139906766282752, 139906768375807, -STORE, 139906768375808, 139906768383999, -ERASE, 139906768375808, 139906768383999, -STORE, 139906768375808, 139906768383999, -STORE, 139906808967168, 139906808991743, -STORE, 139906764087296, 139906766204927, -SNULL, 139906764087296, 139906764103679, -STORE, 139906764103680, 139906766204927, -STORE, 139906764087296, 139906764103679, -SNULL, 139906766196735, 139906766204927, -STORE, 139906764103680, 139906766196735, -STORE, 139906766196736, 139906766204927, -ERASE, 139906766196736, 139906766204927, -STORE, 139906766196736, 139906766204927, -STORE, 139906808958976, 139906808991743, -SNULL, 139906799931391, 139906799939583, -STORE, 139906799915008, 139906799931391, -STORE, 139906799931392, 139906799939583, -SNULL, 139906766200831, 139906766204927, -STORE, 139906766196736, 139906766200831, -STORE, 139906766200832, 139906766204927, -SNULL, 139906768379903, 139906768383999, -STORE, 139906768375808, 139906768379903, -STORE, 139906768379904, 139906768383999, -SNULL, 139906770493439, 139906770497535, -STORE, 139906770489344, 139906770493439, -STORE, 139906770493440, 139906770497535, -SNULL, 139906802151423, 139906802155519, -STORE, 139906802147328, 139906802151423, -STORE, 139906802151424, 139906802155519, -SNULL, 139906773061631, 139906773065727, -STORE, 139906773057536, 139906773061631, -STORE, 139906773061632, 139906773065727, -SNULL, 139906775179263, 139906775183359, -STORE, 139906775175168, 139906775179263, -STORE, 139906775179264, 139906775183359, -SNULL, 139906777563135, 139906777567231, -STORE, 139906777468928, 139906777563135, -STORE, 139906777563136, 139906777567231, -SNULL, 139906779873279, 139906779877375, -STORE, 139906779869184, 139906779873279, -STORE, 139906779873280, 139906779877375, -SNULL, 139906782007295, 139906782011391, -STORE, 139906782003200, 139906782007295, -STORE, 139906782007296, 139906782011391, -SNULL, 139906785193983, 139906785222655, -STORE, 139906785185792, 139906785193983, -STORE, 139906785193984, 139906785222655, -SNULL, 139906787389439, 139906787393535, -STORE, 139906787385344, 139906787389439, -STORE, 139906787389440, 139906787393535, -SNULL, 139906789642239, 139906789646335, -STORE, 139906789638144, 139906789642239, -STORE, 139906789642240, 139906789646335, -SNULL, 139906791772159, 139906791776255, -STORE, 139906791768064, 139906791772159, -STORE, 139906791772160, 139906791776255, -SNULL, 139906793893887, 139906793897983, -STORE, 139906793889792, 139906793893887, -STORE, 139906793893888, 139906793897983, -SNULL, 139906796146687, 139906796150783, -STORE, 139906796142592, 139906796146687, -STORE, 139906796146688, 139906796150783, -SNULL, 139906804547583, 139906804551679, -STORE, 139906804531200, 139906804547583, -STORE, 139906804547584, 139906804551679, -SNULL, 139906806771711, 139906806775807, -STORE, 139906806206464, 139906806771711, -STORE, 139906806771712, 139906806775807, -SNULL, 94728600285183, 94728600289279, -STORE, 94728600281088, 94728600285183, -STORE, 94728600285184, 94728600289279, -SNULL, 139906809024511, 139906809028607, -STORE, 139906809020416, 139906809024511, -STORE, 139906809024512, 139906809028607, -ERASE, 139906808991744, 139906809012223, -STORE, 94728620138496, 94728620273663, -STORE, 139906808692736, 139906808991743, -STORE, 139906808426496, 139906808991743, -STORE, 139906808160256, 139906808991743, -STORE, 139906807894016, 139906808991743, -SNULL, 139906808692735, 139906808991743, -STORE, 139906807894016, 139906808692735, -STORE, 139906808692736, 139906808991743, -SNULL, 139906808692736, 139906808958975, -STORE, 139906808958976, 139906808991743, -STORE, 139906808692736, 139906808958975, - }; - - unsigned long set16[] = { -STORE, 94174808662016, 94174809321471, -STORE, 94174811414528, 94174811426815, -STORE, 94174811426816, 94174811430911, -STORE, 94174811430912, 94174811443199, -STORE, 94174841700352, 94174841835519, -STORE, 140173257838592, 140173259497471, -STORE, 140173259497472, 140173261594623, -STORE, 140173261594624, 140173261611007, -STORE, 140173261611008, 140173261619199, -STORE, 140173261619200, 140173261635583, -STORE, 140173261635584, 140173261778943, -STORE, 140173263863808, 140173263871999, -STORE, 140173263876096, 140173263880191, -STORE, 140173263880192, 140173263884287, -STORE, 140173263884288, 140173263888383, -STORE, 140729801007104, 140729801142271, -STORE, 140729801617408, 140729801629695, -STORE, 140729801629696, 140729801633791, -STORE, 140737488347136, 140737488351231, -STORE, 140728166858752, 140737488351231, -SNULL, 140728166862847, 140737488351231, -STORE, 140728166858752, 140728166862847, -STORE, 140728166727680, 140728166862847, -STORE, 93912949866496, 93912950337535, -SNULL, 93912950288383, 93912950337535, -STORE, 93912949866496, 93912950288383, -STORE, 93912950288384, 93912950337535, -ERASE, 93912950288384, 93912950337535, -STORE, 93912950292480, 93912950337535, -STORE, 139921863385088, 139921865637887, -SNULL, 139921863528447, 139921865637887, -STORE, 139921863385088, 139921863528447, -STORE, 139921863528448, 139921865637887, -ERASE, 139921863528448, 139921865637887, -STORE, 139921865625600, 139921865633791, -STORE, 139921865633792, 139921865637887, -STORE, 140728167899136, 140728167903231, -STORE, 140728167886848, 140728167899135, -STORE, 139921865601024, 139921865625599, -STORE, 139921865592832, 139921865601023, -STORE, 139921861251072, 139921863385087, -SNULL, 139921861251072, 139921861279743, -STORE, 139921861279744, 139921863385087, -STORE, 139921861251072, 139921861279743, -SNULL, 139921863376895, 139921863385087, -STORE, 139921861279744, 139921863376895, -STORE, 139921863376896, 139921863385087, -ERASE, 139921863376896, 139921863385087, -STORE, 139921863376896, 139921863385087, -STORE, 139921858867200, 139921861251071, -SNULL, 139921858867200, 139921859133439, -STORE, 139921859133440, 139921861251071, -STORE, 139921858867200, 139921859133439, -SNULL, 139921861226495, 139921861251071, -STORE, 139921859133440, 139921861226495, -STORE, 139921861226496, 139921861251071, -SNULL, 139921861226496, 139921861246975, -STORE, 139921861246976, 139921861251071, -STORE, 139921861226496, 139921861246975, -ERASE, 139921861226496, 139921861246975, -STORE, 139921861226496, 139921861246975, -ERASE, 139921861246976, 139921861251071, -STORE, 139921861246976, 139921861251071, -STORE, 139921856675840, 139921858867199, -SNULL, 139921856675840, 139921856765951, -STORE, 139921856765952, 139921858867199, -STORE, 139921856675840, 139921856765951, -SNULL, 139921858859007, 139921858867199, -STORE, 139921856765952, 139921858859007, -STORE, 139921858859008, 139921858867199, -ERASE, 139921858859008, 139921858867199, -STORE, 139921858859008, 139921858867199, -STORE, 139921854414848, 139921856675839, -SNULL, 139921854414848, 139921854566399, -STORE, 139921854566400, 139921856675839, -STORE, 139921854414848, 139921854566399, -SNULL, 139921856659455, 139921856675839, -STORE, 139921854566400, 139921856659455, -STORE, 139921856659456, 139921856675839, -SNULL, 139921856659456, 139921856667647, -STORE, 139921856667648, 139921856675839, -STORE, 139921856659456, 139921856667647, -ERASE, 139921856659456, 139921856667647, -STORE, 139921856659456, 139921856667647, -ERASE, 139921856667648, 139921856675839, -STORE, 139921856667648, 139921856675839, -STORE, 139921852284928, 139921854414847, -SNULL, 139921852284928, 139921852313599, -STORE, 139921852313600, 139921854414847, -STORE, 139921852284928, 139921852313599, -SNULL, 139921854406655, 139921854414847, -STORE, 139921852313600, 139921854406655, -STORE, 139921854406656, 139921854414847, -ERASE, 139921854406656, 139921854414847, -STORE, 139921854406656, 139921854414847, -STORE, 139921850068992, 139921852284927, -SNULL, 139921850068992, 139921850167295, -STORE, 139921850167296, 139921852284927, -STORE, 139921850068992, 139921850167295, -SNULL, 139921852260351, 139921852284927, -STORE, 139921850167296, 139921852260351, -STORE, 139921852260352, 139921852284927, -SNULL, 139921852260352, 139921852268543, -STORE, 139921852268544, 139921852284927, -STORE, 139921852260352, 139921852268543, -ERASE, 139921852260352, 139921852268543, -STORE, 139921852260352, 139921852268543, -ERASE, 139921852268544, 139921852284927, -STORE, 139921852268544, 139921852284927, -STORE, 139921865584640, 139921865601023, -STORE, 139921846272000, 139921850068991, -SNULL, 139921846272000, 139921847930879, -STORE, 139921847930880, 139921850068991, -STORE, 139921846272000, 139921847930879, -SNULL, 139921850028031, 139921850068991, -STORE, 139921847930880, 139921850028031, -STORE, 139921850028032, 139921850068991, -SNULL, 139921850028032, 139921850052607, -STORE, 139921850052608, 139921850068991, -STORE, 139921850028032, 139921850052607, -ERASE, 139921850028032, 139921850052607, -STORE, 139921850028032, 139921850052607, -ERASE, 139921850052608, 139921850068991, -STORE, 139921850052608, 139921850068991, -STORE, 139921844154368, 139921846271999, -SNULL, 139921844154368, 139921844170751, -STORE, 139921844170752, 139921846271999, -STORE, 139921844154368, 139921844170751, -SNULL, 139921846263807, 139921846271999, -STORE, 139921844170752, 139921846263807, -STORE, 139921846263808, 139921846271999, -ERASE, 139921846263808, 139921846271999, -STORE, 139921846263808, 139921846271999, -STORE, 139921842036736, 139921844154367, -SNULL, 139921842036736, 139921842053119, -STORE, 139921842053120, 139921844154367, -STORE, 139921842036736, 139921842053119, -SNULL, 139921844146175, 139921844154367, -STORE, 139921842053120, 139921844146175, -STORE, 139921844146176, 139921844154367, -ERASE, 139921844146176, 139921844154367, -STORE, 139921844146176, 139921844154367, -STORE, 139921839468544, 139921842036735, -SNULL, 139921839468544, 139921839935487, -STORE, 139921839935488, 139921842036735, -STORE, 139921839468544, 139921839935487, -SNULL, 139921842028543, 139921842036735, -STORE, 139921839935488, 139921842028543, -STORE, 139921842028544, 139921842036735, -ERASE, 139921842028544, 139921842036735, -STORE, 139921842028544, 139921842036735, -STORE, 139921837355008, 139921839468543, -SNULL, 139921837355008, 139921837367295, -STORE, 139921837367296, 139921839468543, -STORE, 139921837355008, 139921837367295, -SNULL, 139921839460351, 139921839468543, -STORE, 139921837367296, 139921839460351, -STORE, 139921839460352, 139921839468543, -ERASE, 139921839460352, 139921839468543, -STORE, 139921839460352, 139921839468543, -STORE, 139921865576448, 139921865601023, -STORE, 139921865564160, 139921865601023, -SNULL, 139921850044415, 139921850052607, -STORE, 139921850028032, 139921850044415, -STORE, 139921850044416, 139921850052607, -SNULL, 139921839464447, 139921839468543, -STORE, 139921839460352, 139921839464447, -STORE, 139921839464448, 139921839468543, -SNULL, 139921852264447, 139921852268543, -STORE, 139921852260352, 139921852264447, -STORE, 139921852264448, 139921852268543, -SNULL, 139921842032639, 139921842036735, -STORE, 139921842028544, 139921842032639, -STORE, 139921842032640, 139921842036735, -SNULL, 139921844150271, 139921844154367, -STORE, 139921844146176, 139921844150271, -STORE, 139921844150272, 139921844154367, -SNULL, 139921846267903, 139921846271999, -STORE, 139921846263808, 139921846267903, -STORE, 139921846267904, 139921846271999, -SNULL, 139921854410751, 139921854414847, -STORE, 139921854406656, 139921854410751, -STORE, 139921854410752, 139921854414847, -SNULL, 139921856663551, 139921856667647, -STORE, 139921856659456, 139921856663551, -STORE, 139921856663552, 139921856667647, -SNULL, 139921858863103, 139921858867199, -STORE, 139921858859008, 139921858863103, -STORE, 139921858863104, 139921858867199, -SNULL, 139921861242879, 139921861246975, -STORE, 139921861226496, 139921861242879, -STORE, 139921861242880, 139921861246975, -SNULL, 139921863380991, 139921863385087, -STORE, 139921863376896, 139921863380991, -STORE, 139921863380992, 139921863385087, -SNULL, 93912950333439, 93912950337535, -STORE, 93912950292480, 93912950333439, -STORE, 93912950333440, 93912950337535, -SNULL, 139921865629695, 139921865633791, -STORE, 139921865625600, 139921865629695, -STORE, 139921865629696, 139921865633791, -ERASE, 139921865601024, 139921865625599, -STORE, 93912968110080, 93912968245247, -STORE, 139921828913152, 139921837355007, -STORE, 139921865621504, 139921865625599, -STORE, 139921865617408, 139921865621503, -STORE, 139921865613312, 139921865617407, -STORE, 139921865547776, 139921865564159, - }; - - unsigned long set17[] = { -STORE, 94397057224704, 94397057646591, -STORE, 94397057650688, 94397057691647, -STORE, 94397057691648, 94397057695743, -STORE, 94397075271680, 94397075406847, -STORE, 139953169051648, 139953169063935, -STORE, 139953169063936, 139953171156991, -STORE, 139953171156992, 139953171161087, -STORE, 139953171161088, 139953171165183, -STORE, 139953171165184, 139953171632127, -STORE, 139953171632128, 139953173725183, -STORE, 139953173725184, 139953173729279, -STORE, 139953173729280, 139953173733375, -STORE, 139953173733376, 139953173749759, -STORE, 139953173749760, 139953175842815, -STORE, 139953175842816, 139953175846911, -STORE, 139953175846912, 139953175851007, -STORE, 139953175851008, 139953175867391, -STORE, 139953175867392, 139953177960447, -STORE, 139953177960448, 139953177964543, -STORE, 139953177964544, 139953177968639, -STORE, 139953177968640, 139953179627519, -STORE, 139953179627520, 139953181724671, -STORE, 139953181724672, 139953181741055, -STORE, 139953181741056, 139953181749247, -STORE, 139953181749248, 139953181765631, -STORE, 139953181765632, 139953181863935, -STORE, 139953181863936, 139953183956991, -STORE, 139953183956992, 139953183961087, -STORE, 139953183961088, 139953183965183, -STORE, 139953183965184, 139953183981567, -STORE, 139953183981568, 139953184010239, -STORE, 139953184010240, 139953186103295, -STORE, 139953186103296, 139953186107391, -STORE, 139953186107392, 139953186111487, -STORE, 139953186111488, 139953186263039, -STORE, 139953186263040, 139953188356095, -STORE, 139953188356096, 139953188360191, -STORE, 139953188360192, 139953188364287, -STORE, 139953188364288, 139953188372479, -STORE, 139953188372480, 139953188462591, -STORE, 139953188462592, 139953190555647, -STORE, 139953190555648, 139953190559743, -STORE, 139953190559744, 139953190563839, -STORE, 139953190563840, 139953190830079, -STORE, 139953190830080, 139953192923135, -STORE, 139953192923136, 139953192939519, -STORE, 139953192939520, 139953192943615, -STORE, 139953192943616, 139953192947711, -STORE, 139953192947712, 139953192976383, -STORE, 139953192976384, 139953195073535, -STORE, 139953195073536, 139953195077631, -STORE, 139953195077632, 139953195081727, -STORE, 139953195081728, 139953195225087, -STORE, 139953197281280, 139953197318143, -STORE, 139953197322240, 139953197326335, -STORE, 139953197326336, 139953197330431, -STORE, 139953197330432, 139953197334527, -STORE, 140720477511680, 140720477646847, -STORE, 140720478302208, 140720478314495, -STORE, 140720478314496, 140720478318591, - }; - unsigned long set18[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140724953673728, 140737488351231, -SNULL, 140724953677823, 140737488351231, -STORE, 140724953673728, 140724953677823, -STORE, 140724953542656, 140724953677823, -STORE, 94675199266816, 94675199311871, -SNULL, 94675199303679, 94675199311871, -STORE, 94675199266816, 94675199303679, -STORE, 94675199303680, 94675199311871, -ERASE, 94675199303680, 94675199311871, -STORE, 94675199303680, 94675199311871, -STORE, 140222970605568, 140222972858367, -SNULL, 140222970748927, 140222972858367, -STORE, 140222970605568, 140222970748927, -STORE, 140222970748928, 140222972858367, -ERASE, 140222970748928, 140222972858367, -STORE, 140222972846080, 140222972854271, -STORE, 140222972854272, 140222972858367, -STORE, 140724954365952, 140724954370047, -STORE, 140724954353664, 140724954365951, -STORE, 140222972841984, 140222972846079, -STORE, 140222972833792, 140222972841983, -STORE, 140222968475648, 140222970605567, -SNULL, 140222968475648, 140222968504319, -STORE, 140222968504320, 140222970605567, -STORE, 140222968475648, 140222968504319, -SNULL, 140222970597375, 140222970605567, -STORE, 140222968504320, 140222970597375, -STORE, 140222970597376, 140222970605567, -ERASE, 140222970597376, 140222970605567, -STORE, 140222970597376, 140222970605567, - }; - unsigned long set19[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140725182459904, 140737488351231, -SNULL, 140725182463999, 140737488351231, -STORE, 140725182459904, 140725182463999, -STORE, 140725182328832, 140725182463999, -STORE, 94730166636544, 94730166763519, -SNULL, 94730166747135, 94730166763519, -STORE, 94730166636544, 94730166747135, -STORE, 94730166747136, 94730166763519, -ERASE, 94730166747136, 94730166763519, -STORE, 94730166751232, 94730166763519, -STORE, 140656834555904, 140656836808703, -SNULL, 140656834699263, 140656836808703, -STORE, 140656834555904, 140656834699263, -STORE, 140656834699264, 140656836808703, -ERASE, 140656834699264, 140656836808703, -STORE, 140656836796416, 140656836804607, -STORE, 140656836804608, 140656836808703, -STORE, 140725183389696, 140725183393791, -STORE, 140725183377408, 140725183389695, -STORE, 140656836788224, 140656836796415, -STORE, 140656832331776, 140656834555903, -SNULL, 140656833982463, 140656834555903, -STORE, 140656832331776, 140656833982463, -STORE, 140656833982464, 140656834555903, -SNULL, 140656833982464, 140656834551807, -STORE, 140656834551808, 140656834555903, -STORE, 140656833982464, 140656834551807, -ERASE, 140656833982464, 140656834551807, -STORE, 140656833982464, 140656834551807, -ERASE, 140656834551808, 140656834555903, -STORE, 140656834551808, 140656834555903, -STORE, 140656836763648, 140656836788223, -STORE, 140656830070784, 140656832331775, -SNULL, 140656830070784, 140656830222335, -STORE, 140656830222336, 140656832331775, -STORE, 140656830070784, 140656830222335, -SNULL, 140656832315391, 140656832331775, -STORE, 140656830222336, 140656832315391, -STORE, 140656832315392, 140656832331775, -SNULL, 140656832315392, 140656832323583, -STORE, 140656832323584, 140656832331775, -STORE, 140656832315392, 140656832323583, -ERASE, 140656832315392, 140656832323583, -STORE, 140656832315392, 140656832323583, -ERASE, 140656832323584, 140656832331775, -STORE, 140656832323584, 140656832331775, -STORE, 140656827940864, 140656830070783, -SNULL, 140656827940864, 140656827969535, -STORE, 140656827969536, 140656830070783, -STORE, 140656827940864, 140656827969535, -SNULL, 140656830062591, 140656830070783, -STORE, 140656827969536, 140656830062591, -STORE, 140656830062592, 140656830070783, -ERASE, 140656830062592, 140656830070783, -STORE, 140656830062592, 140656830070783, -STORE, 140656825724928, 140656827940863, -SNULL, 140656825724928, 140656825823231, -STORE, 140656825823232, 140656827940863, -STORE, 140656825724928, 140656825823231, -SNULL, 140656827916287, 140656827940863, -STORE, 140656825823232, 140656827916287, -STORE, 140656827916288, 140656827940863, -SNULL, 140656827916288, 140656827924479, -STORE, 140656827924480, 140656827940863, -STORE, 140656827916288, 140656827924479, -ERASE, 140656827916288, 140656827924479, -STORE, 140656827916288, 140656827924479, -ERASE, 140656827924480, 140656827940863, -STORE, 140656827924480, 140656827940863, -STORE, 140656821927936, 140656825724927, -SNULL, 140656821927936, 140656823586815, -STORE, 140656823586816, 140656825724927, -STORE, 140656821927936, 140656823586815, -SNULL, 140656825683967, 140656825724927, -STORE, 140656823586816, 140656825683967, -STORE, 140656825683968, 140656825724927, -SNULL, 140656825683968, 140656825708543, -STORE, 140656825708544, 140656825724927, -STORE, 140656825683968, 140656825708543, -ERASE, 140656825683968, 140656825708543, -STORE, 140656825683968, 140656825708543, -ERASE, 140656825708544, 140656825724927, -STORE, 140656825708544, 140656825724927, -STORE, 140656819806208, 140656821927935, -SNULL, 140656819806208, 140656819822591, -STORE, 140656819822592, 140656821927935, -STORE, 140656819806208, 140656819822591, -SNULL, 140656821919743, 140656821927935, -STORE, 140656819822592, 140656821919743, -STORE, 140656821919744, 140656821927935, -ERASE, 140656821919744, 140656821927935, -STORE, 140656821919744, 140656821927935, -STORE, 140656836755456, 140656836763647, -STORE, 140656817553408, 140656819806207, -SNULL, 140656817553408, 140656817704959, -STORE, 140656817704960, 140656819806207, -STORE, 140656817553408, 140656817704959, -SNULL, 140656819798015, 140656819806207, -STORE, 140656817704960, 140656819798015, -STORE, 140656819798016, 140656819806207, -ERASE, 140656819798016, 140656819806207, -STORE, 140656819798016, 140656819806207, -STORE, 140656815382528, 140656817553407, -SNULL, 140656815382528, 140656815452159, -STORE, 140656815452160, 140656817553407, -STORE, 140656815382528, 140656815452159, -SNULL, 140656817545215, 140656817553407, -STORE, 140656815452160, 140656817545215, -STORE, 140656817545216, 140656817553407, -ERASE, 140656817545216, 140656817553407, -STORE, 140656817545216, 140656817553407, -STORE, 140656812171264, 140656815382527, -SNULL, 140656812171264, 140656813248511, -STORE, 140656813248512, 140656815382527, -STORE, 140656812171264, 140656813248511, -SNULL, 140656815345663, 140656815382527, -STORE, 140656813248512, 140656815345663, -STORE, 140656815345664, 140656815382527, -ERASE, 140656815345664, 140656815382527, -STORE, 140656815345664, 140656815382527, -STORE, 140656810037248, 140656812171263, -SNULL, 140656810037248, 140656810065919, -STORE, 140656810065920, 140656812171263, -STORE, 140656810037248, 140656810065919, -SNULL, 140656812163071, 140656812171263, -STORE, 140656810065920, 140656812163071, -STORE, 140656812163072, 140656812171263, -ERASE, 140656812163072, 140656812171263, -STORE, 140656812163072, 140656812171263, -STORE, 140656807727104, 140656810037247, -SNULL, 140656807727104, 140656807931903, -STORE, 140656807931904, 140656810037247, -STORE, 140656807727104, 140656807931903, -SNULL, 140656810029055, 140656810037247, -STORE, 140656807931904, 140656810029055, -STORE, 140656810029056, 140656810037247, -ERASE, 140656810029056, 140656810037247, -STORE, 140656810029056, 140656810037247, -STORE, 140656805343232, 140656807727103, -SNULL, 140656805343232, 140656805535743, -STORE, 140656805535744, 140656807727103, -STORE, 140656805343232, 140656805535743, -SNULL, 140656807628799, 140656807727103, -STORE, 140656805535744, 140656807628799, -STORE, 140656807628800, 140656807727103, -ERASE, 140656807628800, 140656807727103, -STORE, 140656807628800, 140656807727103, -STORE, 140656836747264, 140656836763647, -STORE, 140656802775040, 140656805343231, -SNULL, 140656802775040, 140656803241983, -STORE, 140656803241984, 140656805343231, -STORE, 140656802775040, 140656803241983, -SNULL, 140656805335039, 140656805343231, -STORE, 140656803241984, 140656805335039, -STORE, 140656805335040, 140656805343231, -ERASE, 140656805335040, 140656805343231, -STORE, 140656805335040, 140656805343231, -STORE, 140656800661504, 140656802775039, -SNULL, 140656800661504, 140656800673791, -STORE, 140656800673792, 140656802775039, -STORE, 140656800661504, 140656800673791, -SNULL, 140656802766847, 140656802775039, -STORE, 140656800673792, 140656802766847, -STORE, 140656802766848, 140656802775039, -ERASE, 140656802766848, 140656802775039, -STORE, 140656802766848, 140656802775039, -STORE, 140656798482432, 140656800661503, -SNULL, 140656798482432, 140656798560255, -STORE, 140656798560256, 140656800661503, -STORE, 140656798482432, 140656798560255, -SNULL, 140656800653311, 140656800661503, -STORE, 140656798560256, 140656800653311, -STORE, 140656800653312, 140656800661503, -ERASE, 140656800653312, 140656800661503, -STORE, 140656800653312, 140656800661503, -STORE, 140656796364800, 140656798482431, -SNULL, 140656796364800, 140656796381183, -STORE, 140656796381184, 140656798482431, -STORE, 140656796364800, 140656796381183, -SNULL, 140656798474239, 140656798482431, -STORE, 140656796381184, 140656798474239, -STORE, 140656798474240, 140656798482431, -ERASE, 140656798474240, 140656798482431, -STORE, 140656798474240, 140656798482431, -STORE, 140656836739072, 140656836763647, -STORE, 140656836726784, 140656836763647, -SNULL, 140656825700351, 140656825708543, -STORE, 140656825683968, 140656825700351, -STORE, 140656825700352, 140656825708543, -SNULL, 140656798478335, 140656798482431, -STORE, 140656798474240, 140656798478335, -STORE, 140656798478336, 140656798482431, -SNULL, 140656800657407, 140656800661503, -STORE, 140656800653312, 140656800657407, -STORE, 140656800657408, 140656800661503, -SNULL, 140656802770943, 140656802775039, -STORE, 140656802766848, 140656802770943, -STORE, 140656802770944, 140656802775039, -SNULL, 140656827920383, 140656827924479, -STORE, 140656827916288, 140656827920383, -STORE, 140656827920384, 140656827924479, -SNULL, 140656805339135, 140656805343231, -STORE, 140656805335040, 140656805339135, -STORE, 140656805339136, 140656805343231, -SNULL, 140656807723007, 140656807727103, -STORE, 140656807628800, 140656807723007, -STORE, 140656807723008, 140656807727103, -SNULL, 140656810033151, 140656810037247, -STORE, 140656810029056, 140656810033151, -STORE, 140656810033152, 140656810037247, -SNULL, 140656812167167, 140656812171263, -STORE, 140656812163072, 140656812167167, -STORE, 140656812167168, 140656812171263, -SNULL, 140656815353855, 140656815382527, -STORE, 140656815345664, 140656815353855, -STORE, 140656815353856, 140656815382527, -SNULL, 140656817549311, 140656817553407, -STORE, 140656817545216, 140656817549311, -STORE, 140656817549312, 140656817553407, -SNULL, 140656819802111, 140656819806207, -STORE, 140656819798016, 140656819802111, -STORE, 140656819802112, 140656819806207, -SNULL, 140656821923839, 140656821927935, -STORE, 140656821919744, 140656821923839, -STORE, 140656821923840, 140656821927935, -SNULL, 140656830066687, 140656830070783, -STORE, 140656830062592, 140656830066687, -STORE, 140656830066688, 140656830070783, -SNULL, 140656832319487, 140656832323583, -STORE, 140656832315392, 140656832319487, -STORE, 140656832319488, 140656832323583, -SNULL, 140656834547711, 140656834551807, -STORE, 140656833982464, 140656834547711, -STORE, 140656834547712, 140656834551807, -SNULL, 94730166759423, 94730166763519, -STORE, 94730166751232, 94730166759423, -STORE, 94730166759424, 94730166763519, -SNULL, 140656836800511, 140656836804607, -STORE, 140656836796416, 140656836800511, -STORE, 140656836800512, 140656836804607, -ERASE, 140656836763648, 140656836788223, -STORE, 94730171318272, 94730171453439, -STORE, 140656836784128, 140656836788223, -STORE, 140656836780032, 140656836784127, -STORE, 140656791920640, 140656796364799, -STORE, 140656836775936, 140656836780031, -STORE, 140656787476480, 140656791920639, -STORE, 140656779083776, 140656787476479, -SNULL, 140656779087871, 140656787476479, -STORE, 140656779083776, 140656779087871, -STORE, 140656779087872, 140656787476479, -STORE, 140656836771840, 140656836775935, -STORE, 140656774639616, 140656779083775, -STORE, 140656766246912, 140656774639615, -SNULL, 140656766251007, 140656774639615, -STORE, 140656766246912, 140656766251007, -STORE, 140656766251008, 140656774639615, -ERASE, 140656791920640, 140656796364799, -ERASE, 140656836780032, 140656836784127, -ERASE, 140656787476480, 140656791920639, -ERASE, 140656836775936, 140656836780031, -STORE, 140656836780032, 140656836784127, -STORE, 140656791920640, 140656796364799, -STORE, 140656836775936, 140656836780031, -STORE, 140656787476480, 140656791920639, -ERASE, 140656774639616, 140656779083775, - }; - unsigned long set20[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140735952392192, 140737488351231, -SNULL, 140735952396287, 140737488351231, -STORE, 140735952392192, 140735952396287, -STORE, 140735952261120, 140735952396287, -STORE, 94849008947200, 94849009414143, -SNULL, 94849009364991, 94849009414143, -STORE, 94849008947200, 94849009364991, -STORE, 94849009364992, 94849009414143, -ERASE, 94849009364992, 94849009414143, -STORE, 94849009364992, 94849009414143, -STORE, 140590397943808, 140590400196607, -SNULL, 140590398087167, 140590400196607, -STORE, 140590397943808, 140590398087167, -STORE, 140590398087168, 140590400196607, -ERASE, 140590398087168, 140590400196607, -STORE, 140590400184320, 140590400192511, -STORE, 140590400192512, 140590400196607, -STORE, 140735952850944, 140735952855039, -STORE, 140735952838656, 140735952850943, -STORE, 140590400180224, 140590400184319, -STORE, 140590400172032, 140590400180223, -STORE, 140590395809792, 140590397943807, -SNULL, 140590395809792, 140590395838463, -STORE, 140590395838464, 140590397943807, -STORE, 140590395809792, 140590395838463, -SNULL, 140590397935615, 140590397943807, -STORE, 140590395838464, 140590397935615, -STORE, 140590397935616, 140590397943807, -ERASE, 140590397935616, 140590397943807, -STORE, 140590397935616, 140590397943807, -STORE, 140590393425920, 140590395809791, -SNULL, 140590393425920, 140590393692159, -STORE, 140590393692160, 140590395809791, -STORE, 140590393425920, 140590393692159, -SNULL, 140590395785215, 140590395809791, -STORE, 140590393692160, 140590395785215, -STORE, 140590395785216, 140590395809791, -SNULL, 140590395785216, 140590395805695, -STORE, 140590395805696, 140590395809791, -STORE, 140590395785216, 140590395805695, -ERASE, 140590395785216, 140590395805695, -STORE, 140590395785216, 140590395805695, -ERASE, 140590395805696, 140590395809791, -STORE, 140590395805696, 140590395809791, -STORE, 140590391234560, 140590393425919, -SNULL, 140590391234560, 140590391324671, -STORE, 140590391324672, 140590393425919, -STORE, 140590391234560, 140590391324671, -SNULL, 140590393417727, 140590393425919, -STORE, 140590391324672, 140590393417727, -STORE, 140590393417728, 140590393425919, -ERASE, 140590393417728, 140590393425919, -STORE, 140590393417728, 140590393425919, -STORE, 140590388973568, 140590391234559, -SNULL, 140590388973568, 140590389125119, -STORE, 140590389125120, 140590391234559, -STORE, 140590388973568, 140590389125119, -SNULL, 140590391218175, 140590391234559, -STORE, 140590389125120, 140590391218175, -STORE, 140590391218176, 140590391234559, -SNULL, 140590391218176, 140590391226367, -STORE, 140590391226368, 140590391234559, -STORE, 140590391218176, 140590391226367, -ERASE, 140590391218176, 140590391226367, -STORE, 140590391218176, 140590391226367, -ERASE, 140590391226368, 140590391234559, -STORE, 140590391226368, 140590391234559, -STORE, 140590386843648, 140590388973567, -SNULL, 140590386843648, 140590386872319, -STORE, 140590386872320, 140590388973567, -STORE, 140590386843648, 140590386872319, -SNULL, 140590388965375, 140590388973567, -STORE, 140590386872320, 140590388965375, -STORE, 140590388965376, 140590388973567, -ERASE, 140590388965376, 140590388973567, -STORE, 140590388965376, 140590388973567, -STORE, 140590384627712, 140590386843647, -SNULL, 140590384627712, 140590384726015, -STORE, 140590384726016, 140590386843647, -STORE, 140590384627712, 140590384726015, -SNULL, 140590386819071, 140590386843647, -STORE, 140590384726016, 140590386819071, -STORE, 140590386819072, 140590386843647, -SNULL, 140590386819072, 140590386827263, -STORE, 140590386827264, 140590386843647, -STORE, 140590386819072, 140590386827263, -ERASE, 140590386819072, 140590386827263, -STORE, 140590386819072, 140590386827263, -ERASE, 140590386827264, 140590386843647, -STORE, 140590386827264, 140590386843647, -STORE, 140590400163840, 140590400180223, -STORE, 140590380830720, 140590384627711, -SNULL, 140590380830720, 140590382489599, -STORE, 140590382489600, 140590384627711, -STORE, 140590380830720, 140590382489599, -SNULL, 140590384586751, 140590384627711, -STORE, 140590382489600, 140590384586751, -STORE, 140590384586752, 140590384627711, -SNULL, 140590384586752, 140590384611327, -STORE, 140590384611328, 140590384627711, -STORE, 140590384586752, 140590384611327, -ERASE, 140590384586752, 140590384611327, -STORE, 140590384586752, 140590384611327, -ERASE, 140590384611328, 140590384627711, -STORE, 140590384611328, 140590384627711, -STORE, 140590378713088, 140590380830719, -SNULL, 140590378713088, 140590378729471, -STORE, 140590378729472, 140590380830719, -STORE, 140590378713088, 140590378729471, -SNULL, 140590380822527, 140590380830719, -STORE, 140590378729472, 140590380822527, -STORE, 140590380822528, 140590380830719, -ERASE, 140590380822528, 140590380830719, -STORE, 140590380822528, 140590380830719, -STORE, 140590376595456, 140590378713087, -SNULL, 140590376595456, 140590376611839, -STORE, 140590376611840, 140590378713087, -STORE, 140590376595456, 140590376611839, -SNULL, 140590378704895, 140590378713087, -STORE, 140590376611840, 140590378704895, -STORE, 140590378704896, 140590378713087, -ERASE, 140590378704896, 140590378713087, -STORE, 140590378704896, 140590378713087, -STORE, 140590374027264, 140590376595455, -SNULL, 140590374027264, 140590374494207, -STORE, 140590374494208, 140590376595455, -STORE, 140590374027264, 140590374494207, -SNULL, 140590376587263, 140590376595455, -STORE, 140590374494208, 140590376587263, -STORE, 140590376587264, 140590376595455, -ERASE, 140590376587264, 140590376595455, -STORE, 140590376587264, 140590376595455, -STORE, 140590371913728, 140590374027263, -SNULL, 140590371913728, 140590371926015, -STORE, 140590371926016, 140590374027263, -STORE, 140590371913728, 140590371926015, -SNULL, 140590374019071, 140590374027263, -STORE, 140590371926016, 140590374019071, -STORE, 140590374019072, 140590374027263, -ERASE, 140590374019072, 140590374027263, -STORE, 140590374019072, 140590374027263, -STORE, 140590400155648, 140590400180223, -STORE, 140590400143360, 140590400180223, -SNULL, 140590384603135, 140590384611327, -STORE, 140590384586752, 140590384603135, -STORE, 140590384603136, 140590384611327, -SNULL, 140590374023167, 140590374027263, -STORE, 140590374019072, 140590374023167, -STORE, 140590374023168, 140590374027263, -SNULL, 140590386823167, 140590386827263, -STORE, 140590386819072, 140590386823167, -STORE, 140590386823168, 140590386827263, -SNULL, 140590376591359, 140590376595455, - }; - unsigned long set21[] = { -STORE, 93874710941696, 93874711363583, -STORE, 93874711367680, 93874711408639, -STORE, 93874711408640, 93874711412735, -STORE, 93874720989184, 93874721124351, -STORE, 140708365086720, 140708365099007, -STORE, 140708365099008, 140708367192063, -STORE, 140708367192064, 140708367196159, -STORE, 140708367196160, 140708367200255, -STORE, 140708367200256, 140708367667199, -STORE, 140708367667200, 140708369760255, -STORE, 140708369760256, 140708369764351, -STORE, 140708369764352, 140708369768447, -STORE, 140708369768448, 140708369784831, -STORE, 140708369784832, 140708371877887, -STORE, 140708371877888, 140708371881983, -STORE, 140708371881984, 140708371886079, -STORE, 140708371886080, 140708371902463, -STORE, 140708371902464, 140708373995519, -STORE, 140708373995520, 140708373999615, -STORE, 140708373999616, 140708374003711, -STORE, 140708374003712, 140708375662591, -STORE, 140708375662592, 140708377759743, -STORE, 140708377759744, 140708377776127, -STORE, 140708377776128, 140708377784319, -STORE, 140708377784320, 140708377800703, -STORE, 140708377800704, 140708377899007, -STORE, 140708377899008, 140708379992063, -STORE, 140708379992064, 140708379996159, -STORE, 140708379996160, 140708380000255, -STORE, 140708380000256, 140708380016639, -STORE, 140708380016640, 140708380045311, -STORE, 140708380045312, 140708382138367, -STORE, 140708382138368, 140708382142463, -STORE, 140708382142464, 140708382146559, -STORE, 140708382146560, 140708382298111, -STORE, 140708382298112, 140708384391167, -STORE, 140708384391168, 140708384395263, -STORE, 140708384395264, 140708384399359, -STORE, 140708384399360, 140708384407551, -STORE, 140708384407552, 140708384497663, -STORE, 140708384497664, 140708386590719, -STORE, 140708386590720, 140708386594815, -STORE, 140708386594816, 140708386598911, -STORE, 140708386598912, 140708386865151, -STORE, 140708386865152, 140708388958207, -STORE, 140708388958208, 140708388974591, -STORE, 140708388974592, 140708388978687, -STORE, 140708388978688, 140708388982783, -STORE, 140708388982784, 140708389011455, -STORE, 140708389011456, 140708391108607, -STORE, 140708391108608, 140708391112703, -STORE, 140708391112704, 140708391116799, -STORE, 140708391116800, 140708391260159, -STORE, 140708393291776, 140708393308159, -STORE, 140708393308160, 140708393312255, -STORE, 140708393312256, 140708393316351, -STORE, 140708393316352, 140708393353215, -STORE, 140708393353216, 140708393357311, -STORE, 140708393357312, 140708393361407, -STORE, 140708393361408, 140708393365503, -STORE, 140708393365504, 140708393369599, -STORE, 140730557042688, 140730557177855, -STORE, 140730557235200, 140730557247487, -STORE, 140730557247488, 140730557251583, -ERASE, 140708393353216, 140708393357311, -ERASE, 140708393312256, 140708393316351, -ERASE, 140708393308160, 140708393312255, -ERASE, 140708393291776, 140708393308159, - }; - unsigned long set22[] = { -STORE, 93951397134336, 93951397183487, -STORE, 93951397183488, 93951397728255, -STORE, 93951397728256, 93951397826559, -STORE, 93951397826560, 93951397842943, -STORE, 93951397842944, 93951397847039, -STORE, 93951425974272, 93951426109439, -STORE, 140685152665600, 140685152677887, -STORE, 140685152677888, 140685152829439, -STORE, 140685152829440, 140685154181119, -STORE, 140685154181120, 140685154484223, -STORE, 140685154484224, 140685154496511, -STORE, 140685154496512, 140685154508799, -STORE, 140685154508800, 140685154525183, -STORE, 140685154525184, 140685154541567, -STORE, 140685154541568, 140685154590719, -STORE, 140685154590720, 140685154603007, -STORE, 140685154603008, 140685154607103, -STORE, 140685154607104, 140685154611199, -STORE, 140685154611200, 140685154615295, -STORE, 140685154615296, 140685154631679, -STORE, 140685154639872, 140685154643967, -STORE, 140685154643968, 140685154766847, -STORE, 140685154766848, 140685154799615, -STORE, 140685154803712, 140685154807807, -STORE, 140685154807808, 140685154811903, -STORE, 140685154811904, 140685154815999, -STORE, 140722188902400, 140722189037567, -STORE, 140722189512704, 140722189524991, -STORE, 140722189524992, 140722189529087, -STORE, 140737488347136, 140737488351231, -STORE, 140733429354496, 140737488351231, -SNULL, 140733429358591, 140737488351231, -STORE, 140733429354496, 140733429358591, -STORE, 140733429223424, 140733429358591, -STORE, 94526683537408, 94526683660287, -SNULL, 94526683553791, 94526683660287, -STORE, 94526683537408, 94526683553791, -STORE, 94526683553792, 94526683660287, -ERASE, 94526683553792, 94526683660287, -STORE, 94526683553792, 94526683623423, -STORE, 94526683623424, 94526683647999, -STORE, 94526683652096, 94526683660287, -STORE, 140551363747840, 140551363923967, -SNULL, 140551363751935, 140551363923967, -STORE, 140551363747840, 140551363751935, -STORE, 140551363751936, 140551363923967, -ERASE, 140551363751936, 140551363923967, -STORE, 140551363751936, 140551363874815, -STORE, 140551363874816, 140551363907583, -STORE, 140551363911680, 140551363919871, -STORE, 140551363919872, 140551363923967, -STORE, 140733429690368, 140733429694463, -STORE, 140733429678080, 140733429690367, -STORE, 140551363739648, 140551363747839, -STORE, 140551363731456, 140551363739647, -STORE, 140551363379200, 140551363731455, -SNULL, 140551363379200, 140551363420159, -STORE, 140551363420160, 140551363731455, -STORE, 140551363379200, 140551363420159, -SNULL, 140551363706879, 140551363731455, -STORE, 140551363420160, 140551363706879, -STORE, 140551363706880, 140551363731455, -SNULL, 140551363420160, 140551363637247, -STORE, 140551363637248, 140551363706879, -STORE, 140551363420160, 140551363637247, -ERASE, 140551363420160, 140551363637247, -STORE, 140551363420160, 140551363637247, -SNULL, 140551363637248, 140551363702783, -STORE, 140551363702784, 140551363706879, -STORE, 140551363637248, 140551363702783, -ERASE, 140551363637248, 140551363702783, -STORE, 140551363637248, 140551363702783, -ERASE, 140551363706880, 140551363731455, -STORE, 140551363706880, 140551363731455, -STORE, 140551361531904, 140551363379199, -SNULL, 140551361683455, 140551363379199, -STORE, 140551361531904, 140551361683455, -STORE, 140551361683456, 140551363379199, -SNULL, 140551361683456, 140551363035135, -STORE, 140551363035136, 140551363379199, -STORE, 140551361683456, 140551363035135, -ERASE, 140551361683456, 140551363035135, -STORE, 140551361683456, 140551363035135, -SNULL, 140551363035136, 140551363338239, -STORE, 140551363338240, 140551363379199, -STORE, 140551363035136, 140551363338239, -ERASE, 140551363035136, 140551363338239, -STORE, 140551363035136, 140551363379199, -SNULL, 140551363338239, 140551363379199, -STORE, 140551363035136, 140551363338239, -STORE, 140551363338240, 140551363379199, -SNULL, 140551363338240, 140551363362815, -STORE, 140551363362816, 140551363379199, -STORE, 140551363338240, 140551363362815, -ERASE, 140551363338240, 140551363362815, -STORE, 140551363338240, 140551363362815, -ERASE, 140551363362816, 140551363379199, -STORE, 140551363362816, 140551363379199, -STORE, 140551361519616, 140551361531903, -SNULL, 140551363350527, 140551363362815, -STORE, 140551363338240, 140551363350527, -STORE, 140551363350528, 140551363362815, -SNULL, 140551363727359, 140551363731455, -STORE, 140551363706880, 140551363727359, -STORE, 140551363727360, 140551363731455, -SNULL, 94526683656191, 94526683660287, -STORE, 94526683652096, 94526683656191, -STORE, 94526683656192, 94526683660287, -SNULL, 140551363915775, 140551363919871, -STORE, 140551363911680, 140551363915775, -STORE, 140551363915776, 140551363919871, -ERASE, 140551363739648, 140551363747839, -STORE, 94526715490304, 94526715625471, -STORE, 140551361253376, 140551361531903, -STORE, 140551360987136, 140551361531903, -STORE, 140551360720896, 140551361531903, -STORE, 140551360454656, 140551361531903, -SNULL, 140551361253375, 140551361531903, -STORE, 140551360454656, 140551361253375, -STORE, 140551361253376, 140551361531903, -SNULL, 140551361253376, 140551361519615, -STORE, 140551361519616, 140551361531903, -STORE, 140551361253376, 140551361519615, -ERASE, 140551361253376, 140551361519615, - }; - - unsigned long set23[] = { -STORE, 94014447943680, 94014448156671, -STORE, 94014450253824, 94014450257919, -STORE, 94014450257920, 94014450266111, -STORE, 94014450266112, 94014450278399, -STORE, 94014464225280, 94014464630783, -STORE, 139761764306944, 139761765965823, -STORE, 139761765965824, 139761768062975, -STORE, 139761768062976, 139761768079359, -STORE, 139761768079360, 139761768087551, -STORE, 139761768087552, 139761768103935, -STORE, 139761768103936, 139761768116223, -STORE, 139761768116224, 139761770209279, -STORE, 139761770209280, 139761770213375, -STORE, 139761770213376, 139761770217471, -STORE, 139761770217472, 139761770360831, -STORE, 139761770729472, 139761772412927, -STORE, 139761772412928, 139761772429311, -STORE, 139761772457984, 139761772462079, -STORE, 139761772462080, 139761772466175, -STORE, 139761772466176, 139761772470271, -STORE, 140724336517120, 140724336652287, -STORE, 140724336955392, 140724336967679, -STORE, 140724336967680, 140724336971775, -STORE, 140737488347136, 140737488351231, -STORE, 140721840295936, 140737488351231, -SNULL, 140721840300031, 140737488351231, -STORE, 140721840295936, 140721840300031, -STORE, 140721840164864, 140721840300031, -STORE, 93937913667584, 93937915830271, -SNULL, 93937913729023, 93937915830271, -STORE, 93937913667584, 93937913729023, -STORE, 93937913729024, 93937915830271, -ERASE, 93937913729024, 93937915830271, -STORE, 93937915822080, 93937915830271, -STORE, 140598835335168, 140598837587967, -SNULL, 140598835478527, 140598837587967, -STORE, 140598835335168, 140598835478527, -STORE, 140598835478528, 140598837587967, -ERASE, 140598835478528, 140598837587967, -STORE, 140598837575680, 140598837583871, -STORE, 140598837583872, 140598837587967, -STORE, 140721841086464, 140721841090559, -STORE, 140721841074176, 140721841086463, -STORE, 140598837547008, 140598837575679, -STORE, 140598837538816, 140598837547007, -STORE, 140598831538176, 140598835335167, -SNULL, 140598831538176, 140598833197055, -STORE, 140598833197056, 140598835335167, -STORE, 140598831538176, 140598833197055, -SNULL, 140598835294207, 140598835335167, -STORE, 140598833197056, 140598835294207, -STORE, 140598835294208, 140598835335167, -SNULL, 140598835294208, 140598835318783, -STORE, 140598835318784, 140598835335167, -STORE, 140598835294208, 140598835318783, -ERASE, 140598835294208, 140598835318783, -STORE, 140598835294208, 140598835318783, -ERASE, 140598835318784, 140598835335167, -STORE, 140598835318784, 140598835335167, -SNULL, 140598835310591, 140598835318783, -STORE, 140598835294208, 140598835310591, -STORE, 140598835310592, 140598835318783, -SNULL, 93937915826175, 93937915830271, -STORE, 93937915822080, 93937915826175, -STORE, 93937915826176, 93937915830271, -SNULL, 140598837579775, 140598837583871, -STORE, 140598837575680, 140598837579775, -STORE, 140598837579776, 140598837583871, -ERASE, 140598837547008, 140598837575679, -STORE, 93937929179136, 93937929314303, -STORE, 140598835855360, 140598837538815, -STORE, 140737488347136, 140737488351231, -STORE, 140728187723776, 140737488351231, -SNULL, 140728187727871, 140737488351231, -STORE, 140728187723776, 140728187727871, -STORE, 140728187592704, 140728187727871, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 140583951437824, 140583953690623, -SNULL, 140583951581183, 140583953690623, -STORE, 140583951437824, 140583951581183, -STORE, 140583951581184, 140583953690623, -ERASE, 140583951581184, 140583953690623, -STORE, 140583953678336, 140583953686527, -STORE, 140583953686528, 140583953690623, -STORE, 140728189116416, 140728189120511, -STORE, 140728189104128, 140728189116415, -STORE, 140583953649664, 140583953678335, -STORE, 140583953641472, 140583953649663, -STORE, 140583948275712, 140583951437823, -SNULL, 140583948275712, 140583949336575, -STORE, 140583949336576, 140583951437823, -STORE, 140583948275712, 140583949336575, -SNULL, 140583951429631, 140583951437823, -STORE, 140583949336576, 140583951429631, -STORE, 140583951429632, 140583951437823, -ERASE, 140583951429632, 140583951437823, -STORE, 140583951429632, 140583951437823, -STORE, 140583944478720, 140583948275711, -SNULL, 140583944478720, 140583946137599, -STORE, 140583946137600, 140583948275711, -STORE, 140583944478720, 140583946137599, -SNULL, 140583948234751, 140583948275711, -STORE, 140583946137600, 140583948234751, -STORE, 140583948234752, 140583948275711, -SNULL, 140583948234752, 140583948259327, -STORE, 140583948259328, 140583948275711, -STORE, 140583948234752, 140583948259327, -ERASE, 140583948234752, 140583948259327, -STORE, 140583948234752, 140583948259327, -ERASE, 140583948259328, 140583948275711, -STORE, 140583948259328, 140583948275711, -STORE, 140583953629184, 140583953649663, -SNULL, 140583948251135, 140583948259327, -STORE, 140583948234752, 140583948251135, -STORE, 140583948251136, 140583948259327, -SNULL, 140583951433727, 140583951437823, -STORE, 140583951429632, 140583951433727, -STORE, 140583951433728, 140583951437823, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 140583953682431, 140583953686527, -STORE, 140583953678336, 140583953682431, -STORE, 140583953682432, 140583953686527, -ERASE, 140583953649664, 140583953678335, -STORE, 17821696, 17956863, -STORE, 17821696, 18104319, -STORE, 140583951945728, 140583953629183, -STORE, 94014447943680, 94014448156671, -STORE, 94014450253824, 94014450257919, -STORE, 94014450257920, 94014450266111, -STORE, 94014450266112, 94014450278399, -STORE, 94014464225280, 94014465196031, -STORE, 139761764306944, 139761765965823, -STORE, 139761765965824, 139761768062975, -STORE, 139761768062976, 139761768079359, -STORE, 139761768079360, 139761768087551, -STORE, 139761768087552, 139761768103935, -STORE, 139761768103936, 139761768116223, -STORE, 139761768116224, 139761770209279, -STORE, 139761770209280, 139761770213375, -STORE, 139761770213376, 139761770217471, -STORE, 139761770217472, 139761770360831, -STORE, 139761770729472, 139761772412927, -STORE, 139761772412928, 139761772429311, -STORE, 139761772457984, 139761772462079, -STORE, 139761772462080, 139761772466175, -STORE, 139761772466176, 139761772470271, -STORE, 140724336517120, 140724336652287, -STORE, 140724336955392, 140724336967679, -STORE, 140724336967680, 140724336971775, -STORE, 140737488347136, 140737488351231, -STORE, 140726063296512, 140737488351231, -SNULL, 140726063300607, 140737488351231, -STORE, 140726063296512, 140726063300607, -STORE, 140726063165440, 140726063300607, -STORE, 94016795934720, 94016798158847, -SNULL, 94016796045311, 94016798158847, -STORE, 94016795934720, 94016796045311, -STORE, 94016796045312, 94016798158847, -ERASE, 94016796045312, 94016798158847, -STORE, 94016798138368, 94016798150655, -STORE, 94016798150656, 94016798158847, -STORE, 139975915966464, 139975918219263, -SNULL, 139975916109823, 139975918219263, -STORE, 139975915966464, 139975916109823, -STORE, 139975916109824, 139975918219263, -ERASE, 139975916109824, 139975918219263, -STORE, 139975918206976, 139975918215167, -STORE, 139975918215168, 139975918219263, -STORE, 140726064541696, 140726064545791, -STORE, 140726064529408, 140726064541695, -STORE, 139975918178304, 139975918206975, -STORE, 139975918170112, 139975918178303, -STORE, 139975912169472, 139975915966463, -SNULL, 139975912169472, 139975913828351, -STORE, 139975913828352, 139975915966463, -STORE, 139975912169472, 139975913828351, -SNULL, 139975915925503, 139975915966463, -STORE, 139975913828352, 139975915925503, -STORE, 139975915925504, 139975915966463, -SNULL, 139975915925504, 139975915950079, -STORE, 139975915950080, 139975915966463, -STORE, 139975915925504, 139975915950079, -ERASE, 139975915925504, 139975915950079, -STORE, 139975915925504, 139975915950079, -ERASE, 139975915950080, 139975915966463, -STORE, 139975915950080, 139975915966463, -SNULL, 139975915941887, 139975915950079, -STORE, 139975915925504, 139975915941887, -STORE, 139975915941888, 139975915950079, -SNULL, 94016798146559, 94016798150655, -STORE, 94016798138368, 94016798146559, -STORE, 94016798146560, 94016798150655, -SNULL, 139975918211071, 139975918215167, -STORE, 139975918206976, 139975918211071, -STORE, 139975918211072, 139975918215167, -ERASE, 139975918178304, 139975918206975, -STORE, 94016804925440, 94016805060607, -STORE, 94596177661952, 94596177772543, -STORE, 94596179865600, 94596179873791, -STORE, 94596179873792, 94596179877887, -STORE, 94596179877888, 94596179886079, -STORE, 94596211597312, 94596211863551, -STORE, 140127351840768, 140127353499647, -STORE, 140127353499648, 140127355596799, -STORE, 140127355596800, 140127355613183, -STORE, 140127355613184, 140127355621375, -STORE, 140127355621376, 140127355637759, -STORE, 140127355637760, 140127355781119, -STORE, 140127357841408, 140127357849599, -STORE, 140127357878272, 140127357882367, -STORE, 140127357882368, 140127357886463, -STORE, 140127357886464, 140127357890559, -STORE, 140726167252992, 140726167392255, -STORE, 140726167838720, 140726167851007, -STORE, 140726167851008, 140726167855103, -STORE, 140737488347136, 140737488351231, -STORE, 140731874017280, 140737488351231, -SNULL, 140731874021375, 140737488351231, -STORE, 140731874017280, 140731874021375, -STORE, 140731873886208, 140731874021375, -STORE, 94178682265600, 94178684489727, -SNULL, 94178682376191, 94178684489727, -STORE, 94178682265600, 94178682376191, -STORE, 94178682376192, 94178684489727, -ERASE, 94178682376192, 94178684489727, -STORE, 94178684469248, 94178684481535, -STORE, 94178684481536, 94178684489727, -STORE, 140460853403648, 140460855656447, -SNULL, 140460853547007, 140460855656447, -STORE, 140460853403648, 140460853547007, -STORE, 140460853547008, 140460855656447, -ERASE, 140460853547008, 140460855656447, -STORE, 140460855644160, 140460855652351, -STORE, 140460855652352, 140460855656447, -STORE, 140731874103296, 140731874107391, -STORE, 140731874091008, 140731874103295, -STORE, 140460855615488, 140460855644159, -STORE, 140460855607296, 140460855615487, -STORE, 140460849606656, 140460853403647, -SNULL, 140460849606656, 140460851265535, -STORE, 140460851265536, 140460853403647, -STORE, 140460849606656, 140460851265535, -SNULL, 140460853362687, 140460853403647, -STORE, 140460851265536, 140460853362687, -STORE, 140460853362688, 140460853403647, -SNULL, 140460853362688, 140460853387263, -STORE, 140460853387264, 140460853403647, -STORE, 140460853362688, 140460853387263, -ERASE, 140460853362688, 140460853387263, -STORE, 140460853362688, 140460853387263, -ERASE, 140460853387264, 140460853403647, -STORE, 140460853387264, 140460853403647, -SNULL, 140460853379071, 140460853387263, -STORE, 140460853362688, 140460853379071, -STORE, 140460853379072, 140460853387263, -SNULL, 94178684477439, 94178684481535, -STORE, 94178684469248, 94178684477439, -STORE, 94178684477440, 94178684481535, -SNULL, 140460855648255, 140460855652351, -STORE, 140460855644160, 140460855648255, -STORE, 140460855648256, 140460855652351, -ERASE, 140460855615488, 140460855644159, -STORE, 94178692063232, 94178692198399, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733096603648, 140737488351231, -SNULL, 140733096611839, 140737488351231, -STORE, 140733096603648, 140733096611839, -STORE, 140733096472576, 140733096611839, -STORE, 94796716122112, 94796718325759, -SNULL, 94796716224511, 94796718325759, -STORE, 94796716122112, 94796716224511, -STORE, 94796716224512, 94796718325759, -ERASE, 94796716224512, 94796718325759, -STORE, 94796718317568, 94796718325759, -STORE, 139667892793344, 139667895046143, -SNULL, 139667892936703, 139667895046143, -STORE, 139667892793344, 139667892936703, -STORE, 139667892936704, 139667895046143, -ERASE, 139667892936704, 139667895046143, -STORE, 139667895033856, 139667895042047, -STORE, 139667895042048, 139667895046143, -STORE, 140733096857600, 140733096861695, -STORE, 140733096845312, 140733096857599, -STORE, 139667895005184, 139667895033855, -STORE, 139667894996992, 139667895005183, -STORE, 139667890532352, 139667892793343, -SNULL, 139667890532352, 139667890683903, -STORE, 139667890683904, 139667892793343, -STORE, 139667890532352, 139667890683903, -SNULL, 139667892776959, 139667892793343, -STORE, 139667890683904, 139667892776959, -STORE, 139667892776960, 139667892793343, -SNULL, 139667892776960, 139667892785151, -STORE, 139667892785152, 139667892793343, -STORE, 139667892776960, 139667892785151, -ERASE, 139667892776960, 139667892785151, -STORE, 139667892776960, 139667892785151, -ERASE, 139667892785152, 139667892793343, -STORE, 139667892785152, 139667892793343, -STORE, 139667886735360, 139667890532351, -SNULL, 139667886735360, 139667888394239, -STORE, 139667888394240, 139667890532351, -STORE, 139667886735360, 139667888394239, -SNULL, 139667890491391, 139667890532351, -STORE, 139667888394240, 139667890491391, -STORE, 139667890491392, 139667890532351, -SNULL, 139667890491392, 139667890515967, -STORE, 139667890515968, 139667890532351, -STORE, 139667890491392, 139667890515967, -ERASE, 139667890491392, 139667890515967, -STORE, 139667890491392, 139667890515967, -ERASE, 139667890515968, 139667890532351, -STORE, 139667890515968, 139667890532351, -STORE, 139667884167168, 139667886735359, -SNULL, 139667884167168, 139667884634111, -STORE, 139667884634112, 139667886735359, -STORE, 139667884167168, 139667884634111, -SNULL, 139667886727167, 139667886735359, -STORE, 139667884634112, 139667886727167, -STORE, 139667886727168, 139667886735359, -ERASE, 139667886727168, 139667886735359, -STORE, 139667886727168, 139667886735359, -STORE, 139667882053632, 139667884167167, -SNULL, 139667882053632, 139667882065919, -STORE, 139667882065920, 139667884167167, -STORE, 139667882053632, 139667882065919, -SNULL, 139667884158975, 139667884167167, -STORE, 139667882065920, 139667884158975, -STORE, 139667884158976, 139667884167167, -ERASE, 139667884158976, 139667884167167, -STORE, 139667884158976, 139667884167167, -STORE, 139667879837696, 139667882053631, -SNULL, 139667879837696, 139667879935999, -STORE, 139667879936000, 139667882053631, -STORE, 139667879837696, 139667879935999, -SNULL, 139667882029055, 139667882053631, -STORE, 139667879936000, 139667882029055, -STORE, 139667882029056, 139667882053631, -SNULL, 139667882029056, 139667882037247, -STORE, 139667882037248, 139667882053631, -STORE, 139667882029056, 139667882037247, -ERASE, 139667882029056, 139667882037247, -STORE, 139667882029056, 139667882037247, -ERASE, 139667882037248, 139667882053631, -STORE, 139667882037248, 139667882053631, -STORE, 139667894988800, 139667895005183, -SNULL, 139667890507775, 139667890515967, -STORE, 139667890491392, 139667890507775, -STORE, 139667890507776, 139667890515967, -SNULL, 139667882033151, 139667882037247, -STORE, 139667882029056, 139667882033151, -STORE, 139667882033152, 139667882037247, -SNULL, 139667884163071, 139667884167167, -STORE, 139667884158976, 139667884163071, -STORE, 139667884163072, 139667884167167, -SNULL, 139667886731263, 139667886735359, -STORE, 139667886727168, 139667886731263, -STORE, 139667886731264, 139667886735359, -SNULL, 139667892781055, 139667892785151, -STORE, 139667892776960, 139667892781055, -STORE, 139667892781056, 139667892785151, -SNULL, 94796718321663, 94796718325759, -STORE, 94796718317568, 94796718321663, -STORE, 94796718321664, 94796718325759, -SNULL, 139667895037951, 139667895042047, -STORE, 139667895033856, 139667895037951, -STORE, 139667895037952, 139667895042047, -ERASE, 139667895005184, 139667895033855, -STORE, 94796726063104, 94796726198271, -STORE, 139667893305344, 139667894988799, -STORE, 139667895005184, 139667895033855, -STORE, 94796726063104, 94796726333439, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722489507840, 140737488351231, -SNULL, 140722489516031, 140737488351231, -STORE, 140722489507840, 140722489516031, -STORE, 140722489376768, 140722489516031, -STORE, 93980993265664, 93980995489791, -SNULL, 93980993376255, 93980995489791, -STORE, 93980993265664, 93980993376255, -STORE, 93980993376256, 93980995489791, -ERASE, 93980993376256, 93980995489791, -STORE, 93980995469312, 93980995481599, -STORE, 93980995481600, 93980995489791, -STORE, 140261313593344, 140261315846143, -SNULL, 140261313736703, 140261315846143, -STORE, 140261313593344, 140261313736703, -STORE, 140261313736704, 140261315846143, -ERASE, 140261313736704, 140261315846143, -STORE, 140261315833856, 140261315842047, -STORE, 140261315842048, 140261315846143, -STORE, 140722489675776, 140722489679871, -STORE, 140722489663488, 140722489675775, -STORE, 140261315805184, 140261315833855, -STORE, 140261315796992, 140261315805183, -STORE, 140261309796352, 140261313593343, -SNULL, 140261309796352, 140261311455231, -STORE, 140261311455232, 140261313593343, -STORE, 140261309796352, 140261311455231, -SNULL, 140261313552383, 140261313593343, -STORE, 140261311455232, 140261313552383, -STORE, 140261313552384, 140261313593343, -SNULL, 140261313552384, 140261313576959, -STORE, 140261313576960, 140261313593343, -STORE, 140261313552384, 140261313576959, -ERASE, 140261313552384, 140261313576959, -STORE, 140261313552384, 140261313576959, -ERASE, 140261313576960, 140261313593343, -STORE, 140261313576960, 140261313593343, -SNULL, 140261313568767, 140261313576959, -STORE, 140261313552384, 140261313568767, -STORE, 140261313568768, 140261313576959, -SNULL, 93980995477503, 93980995481599, -STORE, 93980995469312, 93980995477503, -STORE, 93980995477504, 93980995481599, -SNULL, 140261315837951, 140261315842047, -STORE, 140261315833856, 140261315837951, -STORE, 140261315837952, 140261315842047, -ERASE, 140261315805184, 140261315833855, -STORE, 93980997443584, 93980997578751, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140737488338944, 140737488351231, -STORE, 140734059450368, 140737488351231, -SNULL, 140734059462655, 140737488351231, -STORE, 140734059450368, 140734059462655, -STORE, 140734059319296, 140734059462655, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 140307554983936, 140307557236735, -SNULL, 140307555127295, 140307557236735, -STORE, 140307554983936, 140307555127295, -STORE, 140307555127296, 140307557236735, -ERASE, 140307555127296, 140307557236735, -STORE, 140307557224448, 140307557232639, -STORE, 140307557232640, 140307557236735, -STORE, 140734059483136, 140734059487231, -STORE, 140734059470848, 140734059483135, -STORE, 140307557195776, 140307557224447, -STORE, 140307557187584, 140307557195775, -STORE, 140307551821824, 140307554983935, -SNULL, 140307551821824, 140307552882687, -STORE, 140307552882688, 140307554983935, -STORE, 140307551821824, 140307552882687, -SNULL, 140307554975743, 140307554983935, -STORE, 140307552882688, 140307554975743, -STORE, 140307554975744, 140307554983935, -ERASE, 140307554975744, 140307554983935, -STORE, 140307554975744, 140307554983935, -STORE, 140307548024832, 140307551821823, -SNULL, 140307548024832, 140307549683711, -STORE, 140307549683712, 140307551821823, -STORE, 140307548024832, 140307549683711, -SNULL, 140307551780863, 140307551821823, -STORE, 140307549683712, 140307551780863, -STORE, 140307551780864, 140307551821823, -SNULL, 140307551780864, 140307551805439, -STORE, 140307551805440, 140307551821823, -STORE, 140307551780864, 140307551805439, -ERASE, 140307551780864, 140307551805439, -STORE, 140307551780864, 140307551805439, -ERASE, 140307551805440, 140307551821823, -STORE, 140307551805440, 140307551821823, -STORE, 140307557175296, 140307557195775, -SNULL, 140307551797247, 140307551805439, -STORE, 140307551780864, 140307551797247, -STORE, 140307551797248, 140307551805439, -SNULL, 140307554979839, 140307554983935, -STORE, 140307554975744, 140307554979839, -STORE, 140307554979840, 140307554983935, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 140307557228543, 140307557232639, -STORE, 140307557224448, 140307557228543, -STORE, 140307557228544, 140307557232639, -ERASE, 140307557195776, 140307557224447, -STORE, 39698432, 39833599, -STORE, 39698432, 39981055, -STORE, 94306485321728, 94306485432319, -STORE, 94306487525376, 94306487533567, -STORE, 94306487533568, 94306487537663, -STORE, 94306487537664, 94306487545855, -STORE, 94306488868864, 94306489004031, -STORE, 140497673998336, 140497675657215, -STORE, 140497675657216, 140497677754367, -STORE, 140497677754368, 140497677770751, -STORE, 140497677770752, 140497677778943, -STORE, 140497677778944, 140497677795327, -STORE, 140497677795328, 140497677938687, -STORE, 140497679998976, 140497680007167, -STORE, 140497680035840, 140497680039935, -STORE, 140497680039936, 140497680044031, -STORE, 140497680044032, 140497680048127, -STORE, 140732780462080, 140732780601343, -STORE, 140732782239744, 140732782252031, -STORE, 140732782252032, 140732782256127, -STORE, 94236915900416, 94236916011007, -STORE, 94236918104064, 94236918112255, -STORE, 94236918112256, 94236918116351, -STORE, 94236918116352, 94236918124543, -STORE, 94236939489280, 94236939624447, -STORE, 140046091743232, 140046093402111, -STORE, 140046093402112, 140046095499263, -STORE, 140046095499264, 140046095515647, -STORE, 140046095515648, 140046095523839, -STORE, 140046095523840, 140046095540223, -STORE, 140046095540224, 140046095683583, -STORE, 140046097743872, 140046097752063, -STORE, 140046097780736, 140046097784831, -STORE, 140046097784832, 140046097788927, -STORE, 140046097788928, 140046097793023, -STORE, 140726694449152, 140726694588415, -STORE, 140726695313408, 140726695325695, -STORE, 140726695325696, 140726695329791, -STORE, 94894582779904, 94894582992895, -STORE, 94894585090048, 94894585094143, -STORE, 94894585094144, 94894585102335, -STORE, 94894585102336, 94894585114623, -STORE, 94894592868352, 94894594293759, -STORE, 139733563842560, 139733565501439, -STORE, 139733565501440, 139733567598591, -STORE, 139733567598592, 139733567614975, -STORE, 139733567614976, 139733567623167, -STORE, 139733567623168, 139733567639551, -STORE, 139733567639552, 139733567651839, -STORE, 139733567651840, 139733569744895, -STORE, 139733569744896, 139733569748991, -STORE, 139733569748992, 139733569753087, -STORE, 139733569753088, 139733569896447, -STORE, 139733570265088, 139733571948543, -STORE, 139733571948544, 139733571964927, -STORE, 139733571993600, 139733571997695, -STORE, 139733571997696, 139733572001791, -STORE, 139733572001792, 139733572005887, -STORE, 140726369255424, 140726369394687, -STORE, 140726370402304, 140726370414591, -STORE, 140726370414592, 140726370418687, -STORE, 94899236483072, 94899236696063, -STORE, 94899238793216, 94899238797311, -STORE, 94899238797312, 94899238805503, -STORE, 94899238805504, 94899238817791, -STORE, 94899263045632, 94899263979519, -STORE, 140040959893504, 140040961552383, -STORE, 140040961552384, 140040963649535, -STORE, 140040963649536, 140040963665919, -STORE, 140040963665920, 140040963674111, -STORE, 140040963674112, 140040963690495, -STORE, 140040963690496, 140040963702783, -STORE, 140040963702784, 140040965795839, -STORE, 140040965795840, 140040965799935, -STORE, 140040965799936, 140040965804031, -STORE, 140040965804032, 140040965947391, -STORE, 140040966316032, 140040967999487, -STORE, 140040967999488, 140040968015871, -STORE, 140040968044544, 140040968048639, -STORE, 140040968048640, 140040968052735, -STORE, 140040968052736, 140040968056831, -STORE, 140729921359872, 140729921499135, -STORE, 140729921613824, 140729921626111, -STORE, 140729921626112, 140729921630207, -STORE, 94818265190400, 94818265403391, -STORE, 94818267500544, 94818267504639, -STORE, 94818267504640, 94818267512831, -STORE, 94818267512832, 94818267525119, -STORE, 94818283372544, 94818285858815, -STORE, 139818425675776, 139818427334655, -STORE, 139818427334656, 139818429431807, -STORE, 139818429431808, 139818429448191, -STORE, 139818429448192, 139818429456383, -STORE, 139818429456384, 139818429472767, -STORE, 139818429472768, 139818429485055, -STORE, 139818429485056, 139818431578111, -STORE, 139818431578112, 139818431582207, -STORE, 139818431582208, 139818431586303, -STORE, 139818431586304, 139818431729663, -STORE, 139818432098304, 139818433781759, -STORE, 139818433781760, 139818433798143, -STORE, 139818433826816, 139818433830911, -STORE, 139818433830912, 139818433835007, -STORE, 139818433835008, 139818433839103, -STORE, 140726170509312, 140726170648575, -STORE, 140726171824128, 140726171836415, -STORE, 140726171836416, 140726171840511, -STORE, 94611513188352, 94611513401343, -STORE, 94611515498496, 94611515502591, -STORE, 94611515502592, 94611515510783, -STORE, 94611515510784, 94611515523071, -STORE, 94611516502016, 94611516907519, -STORE, 140596246388736, 140596248047615, -STORE, 140596248047616, 140596250144767, -STORE, 140596250144768, 140596250161151, -STORE, 140596250161152, 140596250169343, -STORE, 140596250169344, 140596250185727, -STORE, 140596250185728, 140596250198015, -STORE, 140596250198016, 140596252291071, -STORE, 140596252291072, 140596252295167, -STORE, 140596252295168, 140596252299263, -STORE, 140596252299264, 140596252442623, -STORE, 140596252811264, 140596254494719, -STORE, 140596254494720, 140596254511103, -STORE, 140596254539776, 140596254543871, -STORE, 140596254543872, 140596254547967, -STORE, 140596254547968, 140596254552063, -STORE, 140731551338496, 140731551477759, -STORE, 140731551780864, 140731551793151, -STORE, 140731551793152, 140731551797247, -STORE, 94313835851776, 94313836064767, -STORE, 94313838161920, 94313838166015, -STORE, 94313838166016, 94313838174207, -STORE, 94313838174208, 94313838186495, -STORE, 94313858416640, 94313861906431, -STORE, 140693503918080, 140693505576959, -STORE, 140693505576960, 140693507674111, -STORE, 140693507674112, 140693507690495, -STORE, 140693507690496, 140693507698687, -STORE, 140693507698688, 140693507715071, -STORE, 140693507715072, 140693507727359, -STORE, 140693507727360, 140693509820415, -STORE, 140693509820416, 140693509824511, -STORE, 140693509824512, 140693509828607, -STORE, 140693509828608, 140693509971967, -STORE, 140693510340608, 140693512024063, -STORE, 140693512024064, 140693512040447, -STORE, 140693512069120, 140693512073215, -STORE, 140693512073216, 140693512077311, -STORE, 140693512077312, 140693512081407, -STORE, 140721116065792, 140721116205055, -STORE, 140721117831168, 140721117843455, -STORE, 140721117843456, 140721117847551, -STORE, 94843650150400, 94843650363391, -STORE, 94843652460544, 94843652464639, -STORE, 94843652464640, 94843652472831, -STORE, 94843652472832, 94843652485119, -STORE, 94843685388288, 94843686281215, -STORE, 140484193681408, 140484195340287, -STORE, 140484195340288, 140484197437439, -STORE, 140484197437440, 140484197453823, -STORE, 140484197453824, 140484197462015, -STORE, 140484197462016, 140484197478399, -STORE, 140484197478400, 140484197490687, -STORE, 140484197490688, 140484199583743, -STORE, 140484199583744, 140484199587839, -STORE, 140484199587840, 140484199591935, -STORE, 140484199591936, 140484199735295, -STORE, 140484200103936, 140484201787391, -STORE, 140484201787392, 140484201803775, -STORE, 140484201832448, 140484201836543, -STORE, 140484201836544, 140484201840639, -STORE, 140484201840640, 140484201844735, -STORE, 140726294315008, 140726294454271, -STORE, 140726295646208, 140726295658495, -STORE, 140726295658496, 140726295662591, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140720422371328, 140737488351231, -SNULL, 140720422379519, 140737488351231, -STORE, 140720422371328, 140720422379519, -STORE, 140720422240256, 140720422379519, -STORE, 94417967845376, 94417970180095, -SNULL, 94417968058367, 94417970180095, -STORE, 94417967845376, 94417968058367, -STORE, 94417968058368, 94417970180095, -ERASE, 94417968058368, 94417970180095, -STORE, 94417970155520, 94417970167807, -STORE, 94417970167808, 94417970180095, -STORE, 140252450045952, 140252452298751, -SNULL, 140252450189311, 140252452298751, -STORE, 140252450045952, 140252450189311, -STORE, 140252450189312, 140252452298751, -ERASE, 140252450189312, 140252452298751, -STORE, 140252452286464, 140252452294655, -STORE, 140252452294656, 140252452298751, -STORE, 140720422416384, 140720422420479, -STORE, 140720422404096, 140720422416383, -STORE, 140252452257792, 140252452286463, -STORE, 140252452249600, 140252452257791, -STORE, 140252447932416, 140252450045951, -SNULL, 140252447932416, 140252447944703, -STORE, 140252447944704, 140252450045951, -STORE, 140252447932416, 140252447944703, -SNULL, 140252450037759, 140252450045951, -STORE, 140252447944704, 140252450037759, -STORE, 140252450037760, 140252450045951, -ERASE, 140252450037760, 140252450045951, -STORE, 140252450037760, 140252450045951, -STORE, 140252444135424, 140252447932415, -SNULL, 140252444135424, 140252445794303, -STORE, 140252445794304, 140252447932415, -STORE, 140252444135424, 140252445794303, -SNULL, 140252447891455, 140252447932415, -STORE, 140252445794304, 140252447891455, -STORE, 140252447891456, 140252447932415, -SNULL, 140252447891456, 140252447916031, -STORE, 140252447916032, 140252447932415, -STORE, 140252447891456, 140252447916031, -ERASE, 140252447891456, 140252447916031, -STORE, 140252447891456, 140252447916031, -ERASE, 140252447916032, 140252447932415, -STORE, 140252447916032, 140252447932415, -STORE, 140252452241408, 140252452257791, -SNULL, 140252447907839, 140252447916031, -STORE, 140252447891456, 140252447907839, -STORE, 140252447907840, 140252447916031, -SNULL, 140252450041855, 140252450045951, -STORE, 140252450037760, 140252450041855, -STORE, 140252450041856, 140252450045951, -SNULL, 94417970159615, 94417970167807, -STORE, 94417970155520, 94417970159615, -STORE, 94417970159616, 94417970167807, -SNULL, 140252452290559, 140252452294655, -STORE, 140252452286464, 140252452290559, -STORE, 140252452290560, 140252452294655, -ERASE, 140252452257792, 140252452286463, -STORE, 94417996333056, 94417996468223, -STORE, 140252450557952, 140252452241407, -STORE, 94417996333056, 94417996603391, -STORE, 94417996333056, 94417996738559, -STORE, 94417996333056, 94417996910591, -SNULL, 94417996881919, 94417996910591, -STORE, 94417996333056, 94417996881919, -STORE, 94417996881920, 94417996910591, -ERASE, 94417996881920, 94417996910591, -STORE, 94417996333056, 94417997017087, -STORE, 94417996333056, 94417997152255, -SNULL, 94417997135871, 94417997152255, -STORE, 94417996333056, 94417997135871, -STORE, 94417997135872, 94417997152255, -ERASE, 94417997135872, 94417997152255, -STORE, 94417996333056, 94417997291519, -SNULL, 94417997271039, 94417997291519, -STORE, 94417996333056, 94417997271039, -STORE, 94417997271040, 94417997291519, -ERASE, 94417997271040, 94417997291519, -STORE, 94417996333056, 94417997406207, -SNULL, 94417997381631, 94417997406207, -STORE, 94417996333056, 94417997381631, -STORE, 94417997381632, 94417997406207, -ERASE, 94417997381632, 94417997406207, -STORE, 94417996333056, 94417997516799, -SNULL, 94417997488127, 94417997516799, -STORE, 94417996333056, 94417997488127, -STORE, 94417997488128, 94417997516799, -ERASE, 94417997488128, 94417997516799, -STORE, 94417996333056, 94417997643775, -SNULL, 94417997631487, 94417997643775, -STORE, 94417996333056, 94417997631487, -STORE, 94417997631488, 94417997643775, -ERASE, 94417997631488, 94417997643775, -SNULL, 94417997590527, 94417997631487, -STORE, 94417996333056, 94417997590527, -STORE, 94417997590528, 94417997631487, -ERASE, 94417997590528, 94417997631487, -STORE, 94417996333056, 94417997733887, -STORE, 94417996333056, 94417997869055, -STORE, 94417996333056, 94417998004223, -SNULL, 94417998000127, 94417998004223, -STORE, 94417996333056, 94417998000127, -STORE, 94417998000128, 94417998004223, -ERASE, 94417998000128, 94417998004223, -STORE, 94049170993152, 94049171206143, -STORE, 94049173303296, 94049173307391, -STORE, 94049173307392, 94049173315583, -STORE, 94049173315584, 94049173327871, -STORE, 94049176236032, 94049183645695, -STORE, 139807795544064, 139807797202943, -STORE, 139807797202944, 139807799300095, -STORE, 139807799300096, 139807799316479, -STORE, 139807799316480, 139807799324671, -STORE, 139807799324672, 139807799341055, -STORE, 139807799341056, 139807799353343, -STORE, 139807799353344, 139807801446399, -STORE, 139807801446400, 139807801450495, -STORE, 139807801450496, 139807801454591, -STORE, 139807801454592, 139807801597951, -STORE, 139807801966592, 139807803650047, -STORE, 139807803650048, 139807803666431, -STORE, 139807803695104, 139807803699199, -STORE, 139807803699200, 139807803703295, -STORE, 139807803703296, 139807803707391, -STORE, 140727555538944, 140727555678207, -STORE, 140727555940352, 140727555952639, -STORE, 140727555952640, 140727555956735, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722483441664, 140737488351231, -SNULL, 140722483449855, 140737488351231, -STORE, 140722483441664, 140722483449855, -STORE, 140722483310592, 140722483449855, -STORE, 94416704921600, 94416707145727, -SNULL, 94416705032191, 94416707145727, -STORE, 94416704921600, 94416705032191, -STORE, 94416705032192, 94416707145727, -ERASE, 94416705032192, 94416707145727, -STORE, 94416707125248, 94416707137535, -STORE, 94416707137536, 94416707145727, -STORE, 140555439296512, 140555441549311, -SNULL, 140555439439871, 140555441549311, -STORE, 140555439296512, 140555439439871, -STORE, 140555439439872, 140555441549311, -ERASE, 140555439439872, 140555441549311, -STORE, 140555441537024, 140555441545215, -STORE, 140555441545216, 140555441549311, -STORE, 140722484781056, 140722484785151, -STORE, 140722484768768, 140722484781055, -STORE, 140555441508352, 140555441537023, -STORE, 140555441500160, 140555441508351, -STORE, 140555435499520, 140555439296511, -SNULL, 140555435499520, 140555437158399, -STORE, 140555437158400, 140555439296511, -STORE, 140555435499520, 140555437158399, -SNULL, 140555439255551, 140555439296511, -STORE, 140555437158400, 140555439255551, -STORE, 140555439255552, 140555439296511, -SNULL, 140555439255552, 140555439280127, -STORE, 140555439280128, 140555439296511, -STORE, 140555439255552, 140555439280127, -ERASE, 140555439255552, 140555439280127, -STORE, 140555439255552, 140555439280127, -ERASE, 140555439280128, 140555439296511, -STORE, 140555439280128, 140555439296511, -SNULL, 140555439271935, 140555439280127, -STORE, 140555439255552, 140555439271935, -STORE, 140555439271936, 140555439280127, -SNULL, 94416707133439, 94416707137535, -STORE, 94416707125248, 94416707133439, -STORE, 94416707133440, 94416707137535, -SNULL, 140555441541119, 140555441545215, -STORE, 140555441537024, 140555441541119, -STORE, 140555441541120, 140555441545215, -ERASE, 140555441508352, 140555441537023, -STORE, 94416724672512, 94416724807679, -STORE, 94686636953600, 94686637166591, -STORE, 94686639263744, 94686639267839, -STORE, 94686639267840, 94686639276031, -STORE, 94686639276032, 94686639288319, -STORE, 94686662193152, 94686663163903, -STORE, 140312944431104, 140312946089983, -STORE, 140312946089984, 140312948187135, -STORE, 140312948187136, 140312948203519, -STORE, 140312948203520, 140312948211711, -STORE, 140312948211712, 140312948228095, -STORE, 140312948228096, 140312948240383, -STORE, 140312948240384, 140312950333439, -STORE, 140312950333440, 140312950337535, -STORE, 140312950337536, 140312950341631, -STORE, 140312950341632, 140312950484991, -STORE, 140312950853632, 140312952537087, -STORE, 140312952537088, 140312952553471, -STORE, 140312952582144, 140312952586239, -STORE, 140312952586240, 140312952590335, -STORE, 140312952590336, 140312952594431, -STORE, 140730598920192, 140730599059455, -STORE, 140730599108608, 140730599120895, -STORE, 140730599120896, 140730599124991, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140726234079232, 140737488351231, -SNULL, 140726234087423, 140737488351231, -STORE, 140726234079232, 140726234087423, -STORE, 140726233948160, 140726234087423, -STORE, 94589467578368, 94589469802495, -SNULL, 94589467688959, 94589469802495, -STORE, 94589467578368, 94589467688959, -STORE, 94589467688960, 94589469802495, -ERASE, 94589467688960, 94589469802495, -STORE, 94589469782016, 94589469794303, -STORE, 94589469794304, 94589469802495, -STORE, 140587082842112, 140587085094911, -SNULL, 140587082985471, 140587085094911, -STORE, 140587082842112, 140587082985471, -STORE, 140587082985472, 140587085094911, -ERASE, 140587082985472, 140587085094911, -STORE, 140587085082624, 140587085090815, -STORE, 140587085090816, 140587085094911, -STORE, 140726234103808, 140726234107903, -STORE, 140726234091520, 140726234103807, -STORE, 140587085053952, 140587085082623, -STORE, 140587085045760, 140587085053951, -STORE, 140587079045120, 140587082842111, -SNULL, 140587079045120, 140587080703999, -STORE, 140587080704000, 140587082842111, -STORE, 140587079045120, 140587080703999, -SNULL, 140587082801151, 140587082842111, -STORE, 140587080704000, 140587082801151, -STORE, 140587082801152, 140587082842111, -SNULL, 140587082801152, 140587082825727, -STORE, 140587082825728, 140587082842111, -STORE, 140587082801152, 140587082825727, -ERASE, 140587082801152, 140587082825727, -STORE, 140587082801152, 140587082825727, -ERASE, 140587082825728, 140587082842111, -STORE, 140587082825728, 140587082842111, -SNULL, 140587082817535, 140587082825727, -STORE, 140587082801152, 140587082817535, -STORE, 140587082817536, 140587082825727, -SNULL, 94589469790207, 94589469794303, -STORE, 94589469782016, 94589469790207, -STORE, 94589469790208, 94589469794303, -SNULL, 140587085086719, 140587085090815, -STORE, 140587085082624, 140587085086719, -STORE, 140587085086720, 140587085090815, -ERASE, 140587085053952, 140587085082623, -STORE, 94589477507072, 94589477642239, -STORE, 94225448325120, 94225448538111, -STORE, 94225450635264, 94225450639359, -STORE, 94225450639360, 94225450647551, -STORE, 94225450647552, 94225450659839, -STORE, 94225470246912, 94225473548287, -STORE, 140199245496320, 140199247155199, -STORE, 140199247155200, 140199249252351, -STORE, 140199249252352, 140199249268735, -STORE, 140199249268736, 140199249276927, -STORE, 140199249276928, 140199249293311, -STORE, 140199249293312, 140199249305599, -STORE, 140199249305600, 140199251398655, -STORE, 140199251398656, 140199251402751, -STORE, 140199251402752, 140199251406847, -STORE, 140199251406848, 140199251550207, -STORE, 140199251918848, 140199253602303, -STORE, 140199253602304, 140199253618687, -STORE, 140199253647360, 140199253651455, -STORE, 140199253651456, 140199253655551, -STORE, 140199253655552, 140199253659647, -STORE, 140726264414208, 140726264553471, -STORE, 140726265843712, 140726265855999, -STORE, 140726265856000, 140726265860095, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733508358144, 140737488351231, -SNULL, 140733508366335, 140737488351231, -STORE, 140733508358144, 140733508366335, -STORE, 140733508227072, 140733508366335, -STORE, 94766263947264, 94766266171391, -SNULL, 94766264057855, 94766266171391, -STORE, 94766263947264, 94766264057855, -STORE, 94766264057856, 94766266171391, -ERASE, 94766264057856, 94766266171391, -STORE, 94766266150912, 94766266163199, -STORE, 94766266163200, 94766266171391, -STORE, 140693985132544, 140693987385343, -SNULL, 140693985275903, 140693987385343, -STORE, 140693985132544, 140693985275903, -STORE, 140693985275904, 140693987385343, -ERASE, 140693985275904, 140693987385343, -STORE, 140693987373056, 140693987381247, -STORE, 140693987381248, 140693987385343, -STORE, 140733509939200, 140733509943295, -STORE, 140733509926912, 140733509939199, -STORE, 140693987344384, 140693987373055, -STORE, 140693987336192, 140693987344383, -STORE, 140693981335552, 140693985132543, -SNULL, 140693981335552, 140693982994431, -STORE, 140693982994432, 140693985132543, -STORE, 140693981335552, 140693982994431, -SNULL, 140693985091583, 140693985132543, -STORE, 140693982994432, 140693985091583, -STORE, 140693985091584, 140693985132543, -SNULL, 140693985091584, 140693985116159, -STORE, 140693985116160, 140693985132543, -STORE, 140693985091584, 140693985116159, -ERASE, 140693985091584, 140693985116159, -STORE, 140693985091584, 140693985116159, -ERASE, 140693985116160, 140693985132543, -STORE, 140693985116160, 140693985132543, -SNULL, 140693985107967, 140693985116159, -STORE, 140693985091584, 140693985107967, -STORE, 140693985107968, 140693985116159, -SNULL, 94766266159103, 94766266163199, -STORE, 94766266150912, 94766266159103, -STORE, 94766266159104, 94766266163199, -SNULL, 140693987377151, 140693987381247, -STORE, 140693987373056, 140693987377151, -STORE, 140693987377152, 140693987381247, -ERASE, 140693987344384, 140693987373055, -STORE, 94766282035200, 94766282170367, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724769353728, 140737488351231, -SNULL, 140724769361919, 140737488351231, -STORE, 140724769353728, 140724769361919, -STORE, 140724769222656, 140724769361919, -STORE, 94710460526592, 94710462750719, -SNULL, 94710460637183, 94710462750719, -STORE, 94710460526592, 94710460637183, -STORE, 94710460637184, 94710462750719, -ERASE, 94710460637184, 94710462750719, -STORE, 94710462730240, 94710462742527, -STORE, 94710462742528, 94710462750719, -STORE, 140469764395008, 140469766647807, -SNULL, 140469764538367, 140469766647807, -STORE, 140469764395008, 140469764538367, -STORE, 140469764538368, 140469766647807, -ERASE, 140469764538368, 140469766647807, -STORE, 140469766635520, 140469766643711, -STORE, 140469766643712, 140469766647807, -STORE, 140724770877440, 140724770881535, -STORE, 140724770865152, 140724770877439, -STORE, 140469766606848, 140469766635519, -STORE, 140469766598656, 140469766606847, -STORE, 140469760598016, 140469764395007, -SNULL, 140469760598016, 140469762256895, -STORE, 140469762256896, 140469764395007, -STORE, 140469760598016, 140469762256895, -SNULL, 140469764354047, 140469764395007, -STORE, 140469762256896, 140469764354047, -STORE, 140469764354048, 140469764395007, -SNULL, 140469764354048, 140469764378623, -STORE, 140469764378624, 140469764395007, -STORE, 140469764354048, 140469764378623, -ERASE, 140469764354048, 140469764378623, -STORE, 140469764354048, 140469764378623, -ERASE, 140469764378624, 140469764395007, -STORE, 140469764378624, 140469764395007, -SNULL, 140469764370431, 140469764378623, -STORE, 140469764354048, 140469764370431, -STORE, 140469764370432, 140469764378623, -SNULL, 94710462738431, 94710462742527, -STORE, 94710462730240, 94710462738431, -STORE, 94710462738432, 94710462742527, -SNULL, 140469766639615, 140469766643711, -STORE, 140469766635520, 140469766639615, -STORE, 140469766639616, 140469766643711, -ERASE, 140469766606848, 140469766635519, -STORE, 94710485581824, 94710485716991, -STORE, 94105755795456, 94105756008447, -STORE, 94105758105600, 94105758109695, -STORE, 94105758109696, 94105758117887, -STORE, 94105758117888, 94105758130175, -STORE, 94105788981248, 94105794871295, -STORE, 140641190031360, 140641191690239, -STORE, 140641191690240, 140641193787391, -STORE, 140641193787392, 140641193803775, -STORE, 140641193803776, 140641193811967, -STORE, 140641193811968, 140641193828351, -STORE, 140641193828352, 140641193840639, -STORE, 140641193840640, 140641195933695, -STORE, 140641195933696, 140641195937791, -STORE, 140641195937792, 140641195941887, -STORE, 140641195941888, 140641196085247, -STORE, 140641196453888, 140641198137343, -STORE, 140641198137344, 140641198153727, -STORE, 140641198182400, 140641198186495, -STORE, 140641198186496, 140641198190591, -STORE, 140641198190592, 140641198194687, -STORE, 140731980034048, 140731980173311, -STORE, 140731981078528, 140731981090815, -STORE, 140731981090816, 140731981094911, -STORE, 93828086431744, 93828086644735, -STORE, 93828088741888, 93828088745983, -STORE, 93828088745984, 93828088754175, -STORE, 93828088754176, 93828088766463, -STORE, 93828094193664, 93828096831487, -STORE, 139844717334528, 139844718993407, -STORE, 139844718993408, 139844721090559, -STORE, 139844721090560, 139844721106943, -STORE, 139844721106944, 139844721115135, -STORE, 139844721115136, 139844721131519, -STORE, 139844721131520, 139844721143807, -STORE, 139844721143808, 139844723236863, -STORE, 139844723236864, 139844723240959, -STORE, 139844723240960, 139844723245055, -STORE, 139844723245056, 139844723388415, -STORE, 139844723757056, 139844725440511, -STORE, 139844725440512, 139844725456895, -STORE, 139844725485568, 139844725489663, -STORE, 139844725489664, 139844725493759, -STORE, 139844725493760, 139844725497855, -STORE, 140729996185600, 140729996324863, -STORE, 140729996828672, 140729996840959, -STORE, 140729996840960, 140729996845055, -STORE, 140737488347136, 140737488351231, -STORE, 140722494771200, 140737488351231, -SNULL, 140722494775295, 140737488351231, -STORE, 140722494771200, 140722494775295, -STORE, 140722494640128, 140722494775295, -STORE, 94324011311104, 94324013535231, -SNULL, 94324011421695, 94324013535231, -STORE, 94324011311104, 94324011421695, -STORE, 94324011421696, 94324013535231, -ERASE, 94324011421696, 94324013535231, -STORE, 94324013514752, 94324013527039, -STORE, 94324013527040, 94324013535231, -STORE, 140151462309888, 140151464562687, -SNULL, 140151462453247, 140151464562687, -STORE, 140151462309888, 140151462453247, -STORE, 140151462453248, 140151464562687, -ERASE, 140151462453248, 140151464562687, -STORE, 140151464550400, 140151464558591, -STORE, 140151464558592, 140151464562687, -STORE, 140722495467520, 140722495471615, -STORE, 140722495455232, 140722495467519, -STORE, 140151464521728, 140151464550399, -STORE, 140151464513536, 140151464521727, -STORE, 140151458512896, 140151462309887, -SNULL, 140151458512896, 140151460171775, -STORE, 140151460171776, 140151462309887, -STORE, 140151458512896, 140151460171775, -SNULL, 140151462268927, 140151462309887, -STORE, 140151460171776, 140151462268927, -STORE, 140151462268928, 140151462309887, -SNULL, 140151462268928, 140151462293503, -STORE, 140151462293504, 140151462309887, -STORE, 140151462268928, 140151462293503, -ERASE, 140151462268928, 140151462293503, -STORE, 140151462268928, 140151462293503, -ERASE, 140151462293504, 140151462309887, -STORE, 140151462293504, 140151462309887, -SNULL, 140151462285311, 140151462293503, -STORE, 140151462268928, 140151462285311, -STORE, 140151462285312, 140151462293503, -SNULL, 94324013522943, 94324013527039, -STORE, 94324013514752, 94324013522943, -STORE, 94324013522944, 94324013527039, -SNULL, 140151464554495, 140151464558591, -STORE, 140151464550400, 140151464554495, -STORE, 140151464554496, 140151464558591, -ERASE, 140151464521728, 140151464550399, -STORE, 94324024778752, 94324024913919, -STORE, 94899262967808, 94899263180799, -STORE, 94899265277952, 94899265282047, -STORE, 94899265282048, 94899265290239, -STORE, 94899265290240, 94899265302527, -STORE, 94899295469568, 94899298689023, -STORE, 140434388418560, 140434390077439, -STORE, 140434390077440, 140434392174591, -STORE, 140434392174592, 140434392190975, -STORE, 140434392190976, 140434392199167, -STORE, 140434392199168, 140434392215551, -STORE, 140434392215552, 140434392227839, -STORE, 140434392227840, 140434394320895, -STORE, 140434394320896, 140434394324991, -STORE, 140434394324992, 140434394329087, -STORE, 140434394329088, 140434394472447, -STORE, 140434394841088, 140434396524543, -STORE, 140434396524544, 140434396540927, -STORE, 140434396569600, 140434396573695, -STORE, 140434396573696, 140434396577791, -STORE, 140434396577792, 140434396581887, -STORE, 140720618135552, 140720618274815, -STORE, 140720618418176, 140720618430463, -STORE, 140720618430464, 140720618434559, -STORE, 94425529798656, 94425530011647, -STORE, 94425532108800, 94425532112895, -STORE, 94425532112896, 94425532121087, -STORE, 94425532121088, 94425532133375, -STORE, 94425557753856, 94425566576639, -STORE, 140600528470016, 140600530128895, -STORE, 140600530128896, 140600532226047, -STORE, 140600532226048, 140600532242431, -STORE, 140600532242432, 140600532250623, -STORE, 140600532250624, 140600532267007, -STORE, 140600532267008, 140600532279295, -STORE, 140600532279296, 140600534372351, -STORE, 140600534372352, 140600534376447, -STORE, 140600534376448, 140600534380543, -STORE, 140600534380544, 140600534523903, -STORE, 140600534892544, 140600536575999, -STORE, 140600536576000, 140600536592383, -STORE, 140600536621056, 140600536625151, -STORE, 140600536625152, 140600536629247, -STORE, 140600536629248, 140600536633343, -STORE, 140721857785856, 140721857925119, -STORE, 140721858068480, 140721858080767, -STORE, 140721858080768, 140721858084863, -STORE, 94425529798656, 94425530011647, -STORE, 94425532108800, 94425532112895, -STORE, 94425532112896, 94425532121087, -STORE, 94425532121088, 94425532133375, -STORE, 94425557753856, 94425568772095, -STORE, 140600528470016, 140600530128895, -STORE, 140600530128896, 140600532226047, -STORE, 140600532226048, 140600532242431, -STORE, 140600532242432, 140600532250623, -STORE, 140600532250624, 140600532267007, -STORE, 140600532267008, 140600532279295, -STORE, 140600532279296, 140600534372351, -STORE, 140600534372352, 140600534376447, -STORE, 140600534376448, 140600534380543, -STORE, 140600534380544, 140600534523903, -STORE, 140600534892544, 140600536575999, -STORE, 140600536576000, 140600536592383, -STORE, 140600536621056, 140600536625151, -STORE, 140600536625152, 140600536629247, -STORE, 140600536629248, 140600536633343, -STORE, 140721857785856, 140721857925119, -STORE, 140721858068480, 140721858080767, -STORE, 140721858080768, 140721858084863, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735611645952, 140737488351231, -SNULL, 140735611654143, 140737488351231, -STORE, 140735611645952, 140735611654143, -STORE, 140735611514880, 140735611654143, -STORE, 94592137641984, 94592139866111, -SNULL, 94592137752575, 94592139866111, -STORE, 94592137641984, 94592137752575, -STORE, 94592137752576, 94592139866111, -ERASE, 94592137752576, 94592139866111, -STORE, 94592139845632, 94592139857919, -STORE, 94592139857920, 94592139866111, -STORE, 140350425030656, 140350427283455, -SNULL, 140350425174015, 140350427283455, -STORE, 140350425030656, 140350425174015, -STORE, 140350425174016, 140350427283455, -ERASE, 140350425174016, 140350427283455, -STORE, 140350427271168, 140350427279359, -STORE, 140350427279360, 140350427283455, -STORE, 140735612043264, 140735612047359, -STORE, 140735612030976, 140735612043263, -STORE, 140350427242496, 140350427271167, -STORE, 140350427234304, 140350427242495, -STORE, 140350421233664, 140350425030655, -SNULL, 140350421233664, 140350422892543, -STORE, 140350422892544, 140350425030655, -STORE, 140350421233664, 140350422892543, -SNULL, 140350424989695, 140350425030655, -STORE, 140350422892544, 140350424989695, -STORE, 140350424989696, 140350425030655, -SNULL, 140350424989696, 140350425014271, -STORE, 140350425014272, 140350425030655, -STORE, 140350424989696, 140350425014271, -ERASE, 140350424989696, 140350425014271, -STORE, 140350424989696, 140350425014271, -ERASE, 140350425014272, 140350425030655, -STORE, 140350425014272, 140350425030655, -SNULL, 140350425006079, 140350425014271, -STORE, 140350424989696, 140350425006079, -STORE, 140350425006080, 140350425014271, -SNULL, 94592139853823, 94592139857919, -STORE, 94592139845632, 94592139853823, -STORE, 94592139853824, 94592139857919, -SNULL, 140350427275263, 140350427279359, -STORE, 140350427271168, 140350427275263, -STORE, 140350427275264, 140350427279359, -ERASE, 140350427242496, 140350427271167, -STORE, 94592164823040, 94592164958207, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723500535808, 140737488351231, -SNULL, 140723500543999, 140737488351231, -STORE, 140723500535808, 140723500543999, -STORE, 140723500404736, 140723500543999, -STORE, 94458379010048, 94458381234175, -SNULL, 94458379120639, 94458381234175, -STORE, 94458379010048, 94458379120639, -STORE, 94458379120640, 94458381234175, -ERASE, 94458379120640, 94458381234175, -STORE, 94458381213696, 94458381225983, -STORE, 94458381225984, 94458381234175, -STORE, 139771674230784, 139771676483583, -SNULL, 139771674374143, 139771676483583, -STORE, 139771674230784, 139771674374143, -STORE, 139771674374144, 139771676483583, -ERASE, 139771674374144, 139771676483583, -STORE, 139771676471296, 139771676479487, -STORE, 139771676479488, 139771676483583, -STORE, 140723500769280, 140723500773375, -STORE, 140723500756992, 140723500769279, -STORE, 139771676442624, 139771676471295, -STORE, 139771676434432, 139771676442623, -STORE, 139771670433792, 139771674230783, -SNULL, 139771670433792, 139771672092671, -STORE, 139771672092672, 139771674230783, -STORE, 139771670433792, 139771672092671, -SNULL, 139771674189823, 139771674230783, -STORE, 139771672092672, 139771674189823, -STORE, 139771674189824, 139771674230783, -SNULL, 139771674189824, 139771674214399, -STORE, 139771674214400, 139771674230783, -STORE, 139771674189824, 139771674214399, -ERASE, 139771674189824, 139771674214399, -STORE, 139771674189824, 139771674214399, -ERASE, 139771674214400, 139771674230783, -STORE, 139771674214400, 139771674230783, -SNULL, 139771674206207, 139771674214399, -STORE, 139771674189824, 139771674206207, -STORE, 139771674206208, 139771674214399, -SNULL, 94458381221887, 94458381225983, -STORE, 94458381213696, 94458381221887, -STORE, 94458381221888, 94458381225983, -SNULL, 139771676475391, 139771676479487, -STORE, 139771676471296, 139771676475391, -STORE, 139771676475392, 139771676479487, -ERASE, 139771676442624, 139771676471295, -STORE, 94458401873920, 94458402009087, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731316264960, 140737488351231, -SNULL, 140731316273151, 140737488351231, -STORE, 140731316264960, 140731316273151, -STORE, 140731316133888, 140731316273151, -STORE, 94437830881280, 94437833215999, -SNULL, 94437831094271, 94437833215999, -STORE, 94437830881280, 94437831094271, -STORE, 94437831094272, 94437833215999, -ERASE, 94437831094272, 94437833215999, -STORE, 94437833191424, 94437833203711, -STORE, 94437833203712, 94437833215999, -STORE, 140265986031616, 140265988284415, -SNULL, 140265986174975, 140265988284415, -STORE, 140265986031616, 140265986174975, -STORE, 140265986174976, 140265988284415, -ERASE, 140265986174976, 140265988284415, -STORE, 140265988272128, 140265988280319, -STORE, 140265988280320, 140265988284415, -STORE, 140731316318208, 140731316322303, -STORE, 140731316305920, 140731316318207, -STORE, 140265988243456, 140265988272127, -STORE, 140265988235264, 140265988243455, -STORE, 140265983918080, 140265986031615, -SNULL, 140265983918080, 140265983930367, -STORE, 140265983930368, 140265986031615, -STORE, 140265983918080, 140265983930367, -SNULL, 140265986023423, 140265986031615, -STORE, 140265983930368, 140265986023423, -STORE, 140265986023424, 140265986031615, -ERASE, 140265986023424, 140265986031615, -STORE, 140265986023424, 140265986031615, -STORE, 140265980121088, 140265983918079, -SNULL, 140265980121088, 140265981779967, -STORE, 140265981779968, 140265983918079, -STORE, 140265980121088, 140265981779967, -SNULL, 140265983877119, 140265983918079, -STORE, 140265981779968, 140265983877119, -STORE, 140265983877120, 140265983918079, -SNULL, 140265983877120, 140265983901695, -STORE, 140265983901696, 140265983918079, -STORE, 140265983877120, 140265983901695, -ERASE, 140265983877120, 140265983901695, -STORE, 140265983877120, 140265983901695, -ERASE, 140265983901696, 140265983918079, -STORE, 140265983901696, 140265983918079, -STORE, 140265988227072, 140265988243455, -SNULL, 140265983893503, 140265983901695, -STORE, 140265983877120, 140265983893503, -STORE, 140265983893504, 140265983901695, -SNULL, 140265986027519, 140265986031615, -STORE, 140265986023424, 140265986027519, -STORE, 140265986027520, 140265986031615, -SNULL, 94437833195519, 94437833203711, -STORE, 94437833191424, 94437833195519, -STORE, 94437833195520, 94437833203711, -SNULL, 140265988276223, 140265988280319, -STORE, 140265988272128, 140265988276223, -STORE, 140265988276224, 140265988280319, -ERASE, 140265988243456, 140265988272127, -STORE, 94437847638016, 94437847773183, -STORE, 140265986543616, 140265988227071, -STORE, 94437847638016, 94437847908351, -STORE, 94437847638016, 94437848043519, -STORE, 94437847638016, 94437848190975, -SNULL, 94437848178687, 94437848190975, -STORE, 94437847638016, 94437848178687, -STORE, 94437848178688, 94437848190975, -ERASE, 94437848178688, 94437848190975, -STORE, 94437847638016, 94437848330239, -STORE, 94437847638016, 94437848465407, -SNULL, 94437848444927, 94437848465407, -STORE, 94437847638016, 94437848444927, -STORE, 94437848444928, 94437848465407, -ERASE, 94437848444928, 94437848465407, -STORE, 94437847638016, 94437848584191, -STORE, 94437847638016, 94437848719359, -SNULL, 94437848678399, 94437848719359, -STORE, 94437847638016, 94437848678399, -STORE, 94437848678400, 94437848719359, -ERASE, 94437848678400, 94437848719359, -STORE, 94437847638016, 94437848842239, -SNULL, 94437848825855, 94437848842239, -STORE, 94437847638016, 94437848825855, -STORE, 94437848825856, 94437848842239, -ERASE, 94437848825856, 94437848842239, -STORE, 94437847638016, 94437848961023, -STORE, 94437847638016, 94437849096191, -STORE, 94661814710272, 94661814923263, -STORE, 94661817020416, 94661817024511, -STORE, 94661817024512, 94661817032703, -STORE, 94661817032704, 94661817044991, -STORE, 94661840424960, 94661841240063, -STORE, 140582259814400, 140582261473279, -STORE, 140582261473280, 140582263570431, -STORE, 140582263570432, 140582263586815, -STORE, 140582263586816, 140582263595007, -STORE, 140582263595008, 140582263611391, -STORE, 140582263611392, 140582263623679, -STORE, 140582263623680, 140582265716735, -STORE, 140582265716736, 140582265720831, -STORE, 140582265720832, 140582265724927, -STORE, 140582265724928, 140582265868287, -STORE, 140582266236928, 140582267920383, -STORE, 140582267920384, 140582267936767, -STORE, 140582267965440, 140582267969535, -STORE, 140582267969536, 140582267973631, -STORE, 140582267973632, 140582267977727, -STORE, 140735472508928, 140735472648191, -STORE, 140735472672768, 140735472685055, -STORE, 140735472685056, 140735472689151, -STORE, 94440069140480, 94440069353471, -STORE, 94440071450624, 94440071454719, -STORE, 94440071454720, 94440071462911, -STORE, 94440071462912, 94440071475199, -STORE, 94440072122368, 94440079048703, -STORE, 140112218095616, 140112219754495, -STORE, 140112219754496, 140112221851647, -STORE, 140112221851648, 140112221868031, -STORE, 140112221868032, 140112221876223, -STORE, 140112221876224, 140112221892607, -STORE, 140112221892608, 140112221904895, -STORE, 140112221904896, 140112223997951, -STORE, 140112223997952, 140112224002047, -STORE, 140112224002048, 140112224006143, -STORE, 140112224006144, 140112224149503, -STORE, 140112224518144, 140112226201599, -STORE, 140112226201600, 140112226217983, -STORE, 140112226246656, 140112226250751, -STORE, 140112226250752, 140112226254847, -STORE, 140112226254848, 140112226258943, -STORE, 140737460969472, 140737461108735, -STORE, 140737462083584, 140737462095871, -STORE, 140737462095872, 140737462099967, -STORE, 94257654345728, 94257654390783, -STORE, 94257656483840, 94257656487935, -STORE, 94257656487936, 94257656492031, -STORE, 94257656492032, 94257656496127, -STORE, 94257665859584, 94257665994751, -STORE, 140507070345216, 140507070386175, -STORE, 140507070386176, 140507072483327, -STORE, 140507072483328, 140507072487423, -STORE, 140507072487424, 140507072491519, -STORE, 140507072491520, 140507072516095, -STORE, 140507072516096, 140507072561151, -STORE, 140507072561152, 140507074654207, -STORE, 140507074654208, 140507074658303, -STORE, 140507074658304, 140507074662399, -STORE, 140507074662400, 140507074744319, -STORE, 140507074744320, 140507076841471, -STORE, 140507076841472, 140507076845567, -STORE, 140507076845568, 140507076849663, -STORE, 140507076849664, 140507076857855, -STORE, 140507076857856, 140507076886527, -STORE, 140507076886528, 140507078979583, -STORE, 140507078979584, 140507078983679, -STORE, 140507078983680, 140507078987775, -STORE, 140507078987776, 140507079086079, -STORE, 140507079086080, 140507081179135, -STORE, 140507081179136, 140507081183231, -STORE, 140507081183232, 140507081187327, -STORE, 140507081187328, 140507081203711, -STORE, 140507081203712, 140507081220095, -STORE, 140507081220096, 140507083317247, -STORE, 140507083317248, 140507083321343, -STORE, 140507083321344, 140507083325439, -STORE, 140507083325440, 140507083792383, -STORE, 140507083792384, 140507085885439, -STORE, 140507085885440, 140507085889535, -STORE, 140507085889536, 140507085893631, -STORE, 140507085893632, 140507085905919, -STORE, 140507085905920, 140507087998975, -STORE, 140507087998976, 140507088003071, -STORE, 140507088003072, 140507088007167, -STORE, 140507088007168, 140507088125951, -STORE, 140507088125952, 140507090219007, -STORE, 140507090219008, 140507090223103, -STORE, 140507090223104, 140507090227199, -STORE, 140507090227200, 140507090268159, -STORE, 140507090268160, 140507091927039, -STORE, 140507091927040, 140507094024191, -STORE, 140507094024192, 140507094040575, -STORE, 140507094040576, 140507094048767, -STORE, 140507094048768, 140507094065151, -STORE, 140507094065152, 140507094216703, -STORE, 140507094216704, 140507096309759, -STORE, 140507096309760, 140507096313855, -STORE, 140507096313856, 140507096317951, -STORE, 140507096317952, 140507096326143, -STORE, 140507096326144, 140507096379391, -STORE, 140507096379392, 140507098472447, -STORE, 140507098472448, 140507098476543, -STORE, 140507098476544, 140507098480639, -STORE, 140507098480640, 140507098623999, -STORE, 140507098980352, 140507100663807, -STORE, 140507100663808, 140507100692479, -STORE, 140507100721152, 140507100725247, -STORE, 140507100725248, 140507100729343, -STORE, 140507100729344, 140507100733439, -STORE, 140728152780800, 140728152915967, -STORE, 140728153698304, 140728153710591, -STORE, 140728153710592, 140728153714687, -STORE, 140507068137472, 140507070345215, -SNULL, 140507068137472, 140507068190719, -STORE, 140507068190720, 140507070345215, -STORE, 140507068137472, 140507068190719, -SNULL, 140507070287871, 140507070345215, -STORE, 140507068190720, 140507070287871, -STORE, 140507070287872, 140507070345215, -SNULL, 140507070287872, 140507070296063, -STORE, 140507070296064, 140507070345215, -STORE, 140507070287872, 140507070296063, -ERASE, 140507070287872, 140507070296063, -STORE, 140507070287872, 140507070296063, -ERASE, 140507070296064, 140507070345215, -STORE, 140507070296064, 140507070345215, -STORE, 140507100692480, 140507100721151, -STORE, 140507065810944, 140507068137471, -SNULL, 140507065810944, 140507065843711, -STORE, 140507065843712, 140507068137471, -STORE, 140507065810944, 140507065843711, -SNULL, 140507067940863, 140507068137471, -STORE, 140507065843712, 140507067940863, -STORE, 140507067940864, 140507068137471, -SNULL, 140507067940864, 140507067949055, -STORE, 140507067949056, 140507068137471, -STORE, 140507067940864, 140507067949055, -ERASE, 140507067940864, 140507067949055, -STORE, 140507067940864, 140507067949055, -ERASE, 140507067949056, 140507068137471, -STORE, 140507067949056, 140507068137471, -SNULL, 140507067944959, 140507067949055, -STORE, 140507067940864, 140507067944959, -STORE, 140507067944960, 140507067949055, -SNULL, 140507070291967, 140507070296063, -STORE, 140507070287872, 140507070291967, -STORE, 140507070291968, 140507070296063, -ERASE, 140507100692480, 140507100721151, -STORE, 140507063705600, 140507065810943, -SNULL, 140507063705600, 140507063709695, -STORE, 140507063709696, 140507065810943, -STORE, 140507063705600, 140507063709695, -SNULL, 140507065802751, 140507065810943, -STORE, 140507063709696, 140507065802751, -STORE, 140507065802752, 140507065810943, -ERASE, 140507065802752, 140507065810943, -STORE, 140507065802752, 140507065810943, -SNULL, 140507065806847, 140507065810943, -STORE, 140507065802752, 140507065806847, -STORE, 140507065806848, 140507065810943, -STORE, 140507061600256, 140507063705599, -SNULL, 140507061600256, 140507061604351, -STORE, 140507061604352, 140507063705599, -STORE, 140507061600256, 140507061604351, -SNULL, 140507063697407, 140507063705599, -STORE, 140507061604352, 140507063697407, -STORE, 140507063697408, 140507063705599, -ERASE, 140507063697408, 140507063705599, -STORE, 140507063697408, 140507063705599, -SNULL, 140507063701503, 140507063705599, -STORE, 140507063697408, 140507063701503, -STORE, 140507063701504, 140507063705599, -STORE, 140507059490816, 140507061600255, -SNULL, 140507059490816, 140507059499007, -STORE, 140507059499008, 140507061600255, -STORE, 140507059490816, 140507059499007, -SNULL, 140507061592063, 140507061600255, -STORE, 140507059499008, 140507061592063, -STORE, 140507061592064, 140507061600255, -ERASE, 140507061592064, 140507061600255, -STORE, 140507061592064, 140507061600255, -SNULL, 140507061596159, 140507061600255, -STORE, 140507061592064, 140507061596159, -STORE, 140507061596160, 140507061600255, -STORE, 140507057377280, 140507059490815, -SNULL, 140507057377280, 140507057389567, -STORE, 140507057389568, 140507059490815, -STORE, 140507057377280, 140507057389567, -SNULL, 140507059482623, 140507059490815, -STORE, 140507057389568, 140507059482623, -STORE, 140507059482624, 140507059490815, -ERASE, 140507059482624, 140507059490815, -STORE, 140507059482624, 140507059490815, -SNULL, 140507059486719, 140507059490815, -STORE, 140507059482624, 140507059486719, -STORE, 140507059486720, 140507059490815, -STORE, 140507055255552, 140507057377279, -SNULL, 140507055255552, 140507055276031, -STORE, 140507055276032, 140507057377279, -STORE, 140507055255552, 140507055276031, -SNULL, 140507057369087, 140507057377279, -STORE, 140507055276032, 140507057369087, -STORE, 140507057369088, 140507057377279, -ERASE, 140507057369088, 140507057377279, -STORE, 140507057369088, 140507057377279, -SNULL, 140507057373183, 140507057377279, -STORE, 140507057369088, 140507057373183, -STORE, 140507057373184, 140507057377279, -STORE, 140507098693632, 140507098980351, -SNULL, 140507098959871, 140507098980351, -STORE, 140507098693632, 140507098959871, -STORE, 140507098959872, 140507098980351, -SNULL, 140507098959872, 140507098976255, -STORE, 140507098976256, 140507098980351, -STORE, 140507098959872, 140507098976255, -ERASE, 140507098959872, 140507098976255, -STORE, 140507098959872, 140507098976255, -ERASE, 140507098976256, 140507098980351, -STORE, 140507098976256, 140507098980351, -STORE, 140507100692480, 140507100721151, -STORE, 140507053125632, 140507055255551, -SNULL, 140507053125632, 140507053154303, -STORE, 140507053154304, 140507055255551, -STORE, 140507053125632, 140507053154303, -SNULL, 140507055247359, 140507055255551, -STORE, 140507053154304, 140507055247359, -STORE, 140507055247360, 140507055255551, -ERASE, 140507055247360, 140507055255551, -STORE, 140507055247360, 140507055255551, -STORE, 140507051012096, 140507053125631, -SNULL, 140507051012096, 140507051024383, -STORE, 140507051024384, 140507053125631, -STORE, 140507051012096, 140507051024383, -SNULL, 140507053117439, 140507053125631, -STORE, 140507051024384, 140507053117439, -STORE, 140507053117440, 140507053125631, -ERASE, 140507053117440, 140507053125631, -STORE, 140507053117440, 140507053125631, -SNULL, 140507053121535, 140507053125631, -STORE, 140507053117440, 140507053121535, -STORE, 140507053121536, 140507053125631, -SNULL, 140507055251455, 140507055255551, -STORE, 140507055247360, 140507055251455, -STORE, 140507055251456, 140507055255551, -SNULL, 140507098972159, 140507098976255, -STORE, 140507098959872, 140507098972159, -STORE, 140507098972160, 140507098976255, -ERASE, 140507100692480, 140507100721151, -STORE, 140507100717056, 140507100721151, -ERASE, 140507100717056, 140507100721151, -STORE, 140507100717056, 140507100721151, -ERASE, 140507100717056, 140507100721151, -STORE, 140507100717056, 140507100721151, -ERASE, 140507100717056, 140507100721151, -STORE, 140507100717056, 140507100721151, -ERASE, 140507100717056, 140507100721151, -STORE, 140507100692480, 140507100721151, -ERASE, 140507068137472, 140507068190719, -ERASE, 140507068190720, 140507070287871, -ERASE, 140507070287872, 140507070291967, -ERASE, 140507070291968, 140507070296063, -ERASE, 140507070296064, 140507070345215, -ERASE, 140507065810944, 140507065843711, -ERASE, 140507065843712, 140507067940863, -ERASE, 140507067940864, 140507067944959, -ERASE, 140507067944960, 140507067949055, -ERASE, 140507067949056, 140507068137471, -ERASE, 140507063705600, 140507063709695, -ERASE, 140507063709696, 140507065802751, -ERASE, 140507065802752, 140507065806847, -ERASE, 140507065806848, 140507065810943, -ERASE, 140507061600256, 140507061604351, -ERASE, 140507061604352, 140507063697407, -ERASE, 140507063697408, 140507063701503, -ERASE, 140507063701504, 140507063705599, -ERASE, 140507059490816, 140507059499007, -ERASE, 140507059499008, 140507061592063, -ERASE, 140507061592064, 140507061596159, -ERASE, 140507061596160, 140507061600255, -ERASE, 140507057377280, 140507057389567, -ERASE, 140507057389568, 140507059482623, -ERASE, 140507059482624, 140507059486719, -ERASE, 140507059486720, 140507059490815, -ERASE, 140507055255552, 140507055276031, -ERASE, 140507055276032, 140507057369087, -ERASE, 140507057369088, 140507057373183, -ERASE, 140507057373184, 140507057377279, -ERASE, 140507098693632, 140507098959871, -ERASE, 140507098959872, 140507098972159, -ERASE, 140507098972160, 140507098976255, -ERASE, 140507098976256, 140507098980351, -ERASE, 140507051012096, 140507051024383, -ERASE, 140507051024384, 140507053117439, -ERASE, 140507053117440, 140507053121535, -ERASE, 140507053121536, 140507053125631, -STORE, 94036448296960, 94036448509951, -STORE, 94036450607104, 94036450611199, -STORE, 94036450611200, 94036450619391, -STORE, 94036450619392, 94036450631679, -STORE, 94036482445312, 94036502376447, -STORE, 140469487013888, 140469488672767, -STORE, 140469488672768, 140469490769919, -STORE, 140469490769920, 140469490786303, -STORE, 140469490786304, 140469490794495, -STORE, 140469490794496, 140469490810879, -STORE, 140469490810880, 140469490823167, -STORE, 140469490823168, 140469492916223, -STORE, 140469492916224, 140469492920319, -STORE, 140469492920320, 140469492924415, -STORE, 140469492924416, 140469493067775, -STORE, 140469493436416, 140469495119871, -STORE, 140469495119872, 140469495136255, -STORE, 140469495164928, 140469495169023, -STORE, 140469495169024, 140469495173119, -STORE, 140469495173120, 140469495177215, -STORE, 140732281446400, 140732281585663, -STORE, 140732282736640, 140732282748927, -STORE, 140732282748928, 140732282753023, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723411931136, 140737488351231, -SNULL, 140723411939327, 140737488351231, -STORE, 140723411931136, 140723411939327, -STORE, 140723411800064, 140723411939327, -STORE, 93993768685568, 93993770909695, -SNULL, 93993768796159, 93993770909695, -STORE, 93993768685568, 93993768796159, -STORE, 93993768796160, 93993770909695, -ERASE, 93993768796160, 93993770909695, -STORE, 93993770889216, 93993770901503, -STORE, 93993770901504, 93993770909695, -STORE, 140508681740288, 140508683993087, -SNULL, 140508681883647, 140508683993087, -STORE, 140508681740288, 140508681883647, -STORE, 140508681883648, 140508683993087, -ERASE, 140508681883648, 140508683993087, -STORE, 140508683980800, 140508683988991, -STORE, 140508683988992, 140508683993087, -STORE, 140723412070400, 140723412074495, -STORE, 140723412058112, 140723412070399, -STORE, 140508683952128, 140508683980799, -STORE, 140508683943936, 140508683952127, -STORE, 140508677943296, 140508681740287, -SNULL, 140508677943296, 140508679602175, -STORE, 140508679602176, 140508681740287, -STORE, 140508677943296, 140508679602175, -SNULL, 140508681699327, 140508681740287, -STORE, 140508679602176, 140508681699327, -STORE, 140508681699328, 140508681740287, -SNULL, 140508681699328, 140508681723903, -STORE, 140508681723904, 140508681740287, -STORE, 140508681699328, 140508681723903, -ERASE, 140508681699328, 140508681723903, -STORE, 140508681699328, 140508681723903, -ERASE, 140508681723904, 140508681740287, -STORE, 140508681723904, 140508681740287, -SNULL, 140508681715711, 140508681723903, -STORE, 140508681699328, 140508681715711, -STORE, 140508681715712, 140508681723903, -SNULL, 93993770897407, 93993770901503, -STORE, 93993770889216, 93993770897407, -STORE, 93993770897408, 93993770901503, -SNULL, 140508683984895, 140508683988991, -STORE, 140508683980800, 140508683984895, -STORE, 140508683984896, 140508683988991, -ERASE, 140508683952128, 140508683980799, -STORE, 93993791582208, 93993791717375, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140734685458432, 140737488351231, -SNULL, 140734685466623, 140737488351231, -STORE, 140734685458432, 140734685466623, -STORE, 140734685327360, 140734685466623, -STORE, 93832321548288, 93832323772415, -SNULL, 93832321658879, 93832323772415, -STORE, 93832321548288, 93832321658879, -STORE, 93832321658880, 93832323772415, -ERASE, 93832321658880, 93832323772415, -STORE, 93832323751936, 93832323764223, -STORE, 93832323764224, 93832323772415, -STORE, 140650945118208, 140650947371007, -SNULL, 140650945261567, 140650947371007, -STORE, 140650945118208, 140650945261567, -STORE, 140650945261568, 140650947371007, -ERASE, 140650945261568, 140650947371007, -STORE, 140650947358720, 140650947366911, -STORE, 140650947366912, 140650947371007, -STORE, 140734686081024, 140734686085119, -STORE, 140734686068736, 140734686081023, -STORE, 140650947330048, 140650947358719, -STORE, 140650947321856, 140650947330047, -STORE, 140650941321216, 140650945118207, -SNULL, 140650941321216, 140650942980095, -STORE, 140650942980096, 140650945118207, -STORE, 140650941321216, 140650942980095, -SNULL, 140650945077247, 140650945118207, -STORE, 140650942980096, 140650945077247, -STORE, 140650945077248, 140650945118207, -SNULL, 140650945077248, 140650945101823, -STORE, 140650945101824, 140650945118207, -STORE, 140650945077248, 140650945101823, -ERASE, 140650945077248, 140650945101823, -STORE, 140650945077248, 140650945101823, -ERASE, 140650945101824, 140650945118207, -STORE, 140650945101824, 140650945118207, -SNULL, 140650945093631, 140650945101823, -STORE, 140650945077248, 140650945093631, -STORE, 140650945093632, 140650945101823, -SNULL, 93832323760127, 93832323764223, -STORE, 93832323751936, 93832323760127, -STORE, 93832323760128, 93832323764223, -SNULL, 140650947362815, 140650947366911, -STORE, 140650947358720, 140650947362815, -STORE, 140650947362816, 140650947366911, -ERASE, 140650947330048, 140650947358719, -STORE, 93832331890688, 93832332025855, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728333520896, 140737488351231, -SNULL, 140728333529087, 140737488351231, -STORE, 140728333520896, 140728333529087, -STORE, 140728333389824, 140728333529087, -STORE, 94872734732288, 94872736956415, -SNULL, 94872734842879, 94872736956415, -STORE, 94872734732288, 94872734842879, -STORE, 94872734842880, 94872736956415, -ERASE, 94872734842880, 94872736956415, -STORE, 94872736935936, 94872736948223, -STORE, 94872736948224, 94872736956415, -STORE, 139755193257984, 139755195510783, -SNULL, 139755193401343, 139755195510783, -STORE, 139755193257984, 139755193401343, -STORE, 139755193401344, 139755195510783, -ERASE, 139755193401344, 139755195510783, -STORE, 139755195498496, 139755195506687, -STORE, 139755195506688, 139755195510783, -STORE, 140728333926400, 140728333930495, -STORE, 140728333914112, 140728333926399, -STORE, 139755195469824, 139755195498495, -STORE, 139755195461632, 139755195469823, -STORE, 139755189460992, 139755193257983, -SNULL, 139755189460992, 139755191119871, -STORE, 139755191119872, 139755193257983, -STORE, 139755189460992, 139755191119871, -SNULL, 139755193217023, 139755193257983, -STORE, 139755191119872, 139755193217023, -STORE, 139755193217024, 139755193257983, -SNULL, 139755193217024, 139755193241599, -STORE, 139755193241600, 139755193257983, -STORE, 139755193217024, 139755193241599, -ERASE, 139755193217024, 139755193241599, -STORE, 139755193217024, 139755193241599, -ERASE, 139755193241600, 139755193257983, -STORE, 139755193241600, 139755193257983, -SNULL, 139755193233407, 139755193241599, -STORE, 139755193217024, 139755193233407, -STORE, 139755193233408, 139755193241599, -SNULL, 94872736944127, 94872736948223, -STORE, 94872736935936, 94872736944127, -STORE, 94872736944128, 94872736948223, -SNULL, 139755195502591, 139755195506687, -STORE, 139755195498496, 139755195502591, -STORE, 139755195502592, 139755195506687, -ERASE, 139755195469824, 139755195498495, -STORE, 94872749744128, 94872749879295, -STORE, 94720243642368, 94720243855359, -STORE, 94720245952512, 94720245956607, -STORE, 94720245956608, 94720245964799, -STORE, 94720245964800, 94720245977087, -STORE, 94720277745664, 94720278151167, -STORE, 140453174497280, 140453176156159, -STORE, 140453176156160, 140453178253311, -STORE, 140453178253312, 140453178269695, -STORE, 140453178269696, 140453178277887, -STORE, 140453178277888, 140453178294271, -STORE, 140453178294272, 140453178306559, -STORE, 140453178306560, 140453180399615, -STORE, 140453180399616, 140453180403711, -STORE, 140453180403712, 140453180407807, -STORE, 140453180407808, 140453180551167, -STORE, 140453180919808, 140453182603263, -STORE, 140453182603264, 140453182619647, -STORE, 140453182648320, 140453182652415, -STORE, 140453182652416, 140453182656511, -STORE, 140453182656512, 140453182660607, -STORE, 140733223923712, 140733224062975, -STORE, 140733224808448, 140733224820735, -STORE, 140733224820736, 140733224824831, -STORE, 94321091141632, 94321091354623, -STORE, 94321093451776, 94321093455871, -STORE, 94321093455872, 94321093464063, -STORE, 94321093464064, 94321093476351, -STORE, 94321115873280, 94321117229055, -STORE, 139695978840064, 139695980498943, -STORE, 139695980498944, 139695982596095, -STORE, 139695982596096, 139695982612479, -STORE, 139695982612480, 139695982620671, -STORE, 139695982620672, 139695982637055, -STORE, 139695982637056, 139695982649343, -STORE, 139695982649344, 139695984742399, -STORE, 139695984742400, 139695984746495, -STORE, 139695984746496, 139695984750591, -STORE, 139695984750592, 139695984893951, -STORE, 139695985262592, 139695986946047, -STORE, 139695986946048, 139695986962431, -STORE, 139695986991104, 139695986995199, -STORE, 139695986995200, 139695986999295, -STORE, 139695986999296, 139695987003391, -STORE, 140734650564608, 140734650703871, -STORE, 140734650785792, 140734650798079, -STORE, 140734650798080, 140734650802175, -STORE, 94523438456832, 94523438669823, -STORE, 94523440766976, 94523440771071, -STORE, 94523440771072, 94523440779263, -STORE, 94523440779264, 94523440791551, -STORE, 94523464544256, 94523465842687, -STORE, 140453231493120, 140453233151999, -STORE, 140453233152000, 140453235249151, -STORE, 140453235249152, 140453235265535, -STORE, 140453235265536, 140453235273727, -STORE, 140453235273728, 140453235290111, -STORE, 140453235290112, 140453235302399, -STORE, 140453235302400, 140453237395455, -STORE, 140453237395456, 140453237399551, -STORE, 140453237399552, 140453237403647, -STORE, 140453237403648, 140453237547007, -STORE, 140453237915648, 140453239599103, -STORE, 140453239599104, 140453239615487, -STORE, 140453239644160, 140453239648255, -STORE, 140453239648256, 140453239652351, -STORE, 140453239652352, 140453239656447, -STORE, 140734679445504, 140734679584767, -STORE, 140734680018944, 140734680031231, -STORE, 140734680031232, 140734680035327, -STORE, 94614776987648, 94614777200639, -STORE, 94614779297792, 94614779301887, -STORE, 94614779301888, 94614779310079, -STORE, 94614779310080, 94614779322367, -STORE, 94614798467072, 94614800699391, -STORE, 139677037182976, 139677038841855, -STORE, 139677038841856, 139677040939007, -STORE, 139677040939008, 139677040955391, -STORE, 139677040955392, 139677040963583, -STORE, 139677040963584, 139677040979967, -STORE, 139677040979968, 139677040992255, -STORE, 139677040992256, 139677043085311, -STORE, 139677043085312, 139677043089407, -STORE, 139677043089408, 139677043093503, -STORE, 139677043093504, 139677043236863, -STORE, 139677043605504, 139677045288959, -STORE, 139677045288960, 139677045305343, -STORE, 139677045334016, 139677045338111, -STORE, 139677045338112, 139677045342207, -STORE, 139677045342208, 139677045346303, -STORE, 140721604411392, 140721604550655, -STORE, 140721606135808, 140721606148095, -STORE, 140721606148096, 140721606152191, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140729280544768, 140737488351231, -SNULL, 140729280552959, 140737488351231, -STORE, 140729280544768, 140729280552959, -STORE, 140729280413696, 140729280552959, -STORE, 94863939334144, 94863941558271, -SNULL, 94863939444735, 94863941558271, -STORE, 94863939334144, 94863939444735, -STORE, 94863939444736, 94863941558271, -ERASE, 94863939444736, 94863941558271, -STORE, 94863941537792, 94863941550079, -STORE, 94863941550080, 94863941558271, -STORE, 139691047276544, 139691049529343, -SNULL, 139691047419903, 139691049529343, -STORE, 139691047276544, 139691047419903, -STORE, 139691047419904, 139691049529343, -ERASE, 139691047419904, 139691049529343, -STORE, 139691049517056, 139691049525247, -STORE, 139691049525248, 139691049529343, -STORE, 140729281679360, 140729281683455, -STORE, 140729281667072, 140729281679359, -STORE, 139691049488384, 139691049517055, -STORE, 139691049480192, 139691049488383, -STORE, 139691043479552, 139691047276543, -SNULL, 139691043479552, 139691045138431, -STORE, 139691045138432, 139691047276543, -STORE, 139691043479552, 139691045138431, -SNULL, 139691047235583, 139691047276543, -STORE, 139691045138432, 139691047235583, -STORE, 139691047235584, 139691047276543, -SNULL, 139691047235584, 139691047260159, -STORE, 139691047260160, 139691047276543, -STORE, 139691047235584, 139691047260159, -ERASE, 139691047235584, 139691047260159, -STORE, 139691047235584, 139691047260159, -ERASE, 139691047260160, 139691047276543, -STORE, 139691047260160, 139691047276543, -SNULL, 139691047251967, 139691047260159, -STORE, 139691047235584, 139691047251967, -STORE, 139691047251968, 139691047260159, -SNULL, 94863941545983, 94863941550079, -STORE, 94863941537792, 94863941545983, -STORE, 94863941545984, 94863941550079, -SNULL, 139691049521151, 139691049525247, -STORE, 139691049517056, 139691049521151, -STORE, 139691049521152, 139691049525247, -ERASE, 139691049488384, 139691049517055, -STORE, 94863951294464, 94863951429631, -STORE, 93998209294336, 93998209507327, -STORE, 93998211604480, 93998211608575, -STORE, 93998211608576, 93998211616767, -STORE, 93998211616768, 93998211629055, -STORE, 93998227210240, 93998227615743, -STORE, 140243029913600, 140243031572479, -STORE, 140243031572480, 140243033669631, -STORE, 140243033669632, 140243033686015, -STORE, 140243033686016, 140243033694207, -STORE, 140243033694208, 140243033710591, -STORE, 140243033710592, 140243033722879, -STORE, 140243033722880, 140243035815935, -STORE, 140243035815936, 140243035820031, -STORE, 140243035820032, 140243035824127, -STORE, 140243035824128, 140243035967487, -STORE, 140243036336128, 140243038019583, -STORE, 140243038019584, 140243038035967, -STORE, 140243038064640, 140243038068735, -STORE, 140243038068736, 140243038072831, -STORE, 140243038072832, 140243038076927, -STORE, 140734976479232, 140734976618495, -STORE, 140734977978368, 140734977990655, -STORE, 140734977990656, 140734977994751, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722742775808, 140737488351231, -SNULL, 140722742783999, 140737488351231, -STORE, 140722742775808, 140722742783999, -STORE, 140722742644736, 140722742783999, -STORE, 93857673662464, 93857675997183, -SNULL, 93857673875455, 93857675997183, -STORE, 93857673662464, 93857673875455, -STORE, 93857673875456, 93857675997183, -ERASE, 93857673875456, 93857675997183, -STORE, 93857675972608, 93857675984895, -STORE, 93857675984896, 93857675997183, -STORE, 140629677498368, 140629679751167, -SNULL, 140629677641727, 140629679751167, -STORE, 140629677498368, 140629677641727, -STORE, 140629677641728, 140629679751167, -ERASE, 140629677641728, 140629679751167, -STORE, 140629679738880, 140629679747071, -STORE, 140629679747072, 140629679751167, -STORE, 140722743222272, 140722743226367, -STORE, 140722743209984, 140722743222271, -STORE, 140629679710208, 140629679738879, -STORE, 140629679702016, 140629679710207, -STORE, 140629675384832, 140629677498367, -SNULL, 140629675384832, 140629675397119, -STORE, 140629675397120, 140629677498367, -STORE, 140629675384832, 140629675397119, -SNULL, 140629677490175, 140629677498367, -STORE, 140629675397120, 140629677490175, -STORE, 140629677490176, 140629677498367, -ERASE, 140629677490176, 140629677498367, -STORE, 140629677490176, 140629677498367, -STORE, 140629671587840, 140629675384831, -SNULL, 140629671587840, 140629673246719, -STORE, 140629673246720, 140629675384831, -STORE, 140629671587840, 140629673246719, -SNULL, 140629675343871, 140629675384831, -STORE, 140629673246720, 140629675343871, -STORE, 140629675343872, 140629675384831, -SNULL, 140629675343872, 140629675368447, -STORE, 140629675368448, 140629675384831, -STORE, 140629675343872, 140629675368447, -ERASE, 140629675343872, 140629675368447, -STORE, 140629675343872, 140629675368447, -ERASE, 140629675368448, 140629675384831, -STORE, 140629675368448, 140629675384831, -STORE, 140629679693824, 140629679710207, -SNULL, 140629675360255, 140629675368447, -STORE, 140629675343872, 140629675360255, -STORE, 140629675360256, 140629675368447, -SNULL, 140629677494271, 140629677498367, -STORE, 140629677490176, 140629677494271, -STORE, 140629677494272, 140629677498367, -SNULL, 93857675976703, 93857675984895, -STORE, 93857675972608, 93857675976703, -STORE, 93857675976704, 93857675984895, -SNULL, 140629679742975, 140629679747071, -STORE, 140629679738880, 140629679742975, -STORE, 140629679742976, 140629679747071, -ERASE, 140629679710208, 140629679738879, -STORE, 93857705832448, 93857705967615, -STORE, 140629678010368, 140629679693823, -STORE, 93857705832448, 93857706102783, -STORE, 93857705832448, 93857706237951, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735922421760, 140737488351231, -SNULL, 140735922429951, 140737488351231, -STORE, 140735922421760, 140735922429951, -STORE, 140735922290688, 140735922429951, -STORE, 94651136139264, 94651138363391, -SNULL, 94651136249855, 94651138363391, -STORE, 94651136139264, 94651136249855, -STORE, 94651136249856, 94651138363391, -ERASE, 94651136249856, 94651138363391, -STORE, 94651138342912, 94651138355199, -STORE, 94651138355200, 94651138363391, -STORE, 140325788266496, 140325790519295, -SNULL, 140325788409855, 140325790519295, -STORE, 140325788266496, 140325788409855, -STORE, 140325788409856, 140325790519295, -ERASE, 140325788409856, 140325790519295, -STORE, 140325790507008, 140325790515199, -STORE, 140325790515200, 140325790519295, -STORE, 140735923572736, 140735923576831, -STORE, 140735923560448, 140735923572735, -STORE, 140325790478336, 140325790507007, -STORE, 140325790470144, 140325790478335, -STORE, 140325784469504, 140325788266495, -SNULL, 140325784469504, 140325786128383, -STORE, 140325786128384, 140325788266495, -STORE, 140325784469504, 140325786128383, -SNULL, 140325788225535, 140325788266495, -STORE, 140325786128384, 140325788225535, -STORE, 140325788225536, 140325788266495, -SNULL, 140325788225536, 140325788250111, -STORE, 140325788250112, 140325788266495, -STORE, 140325788225536, 140325788250111, -ERASE, 140325788225536, 140325788250111, -STORE, 140325788225536, 140325788250111, -ERASE, 140325788250112, 140325788266495, -STORE, 140325788250112, 140325788266495, -SNULL, 140325788241919, 140325788250111, -STORE, 140325788225536, 140325788241919, -STORE, 140325788241920, 140325788250111, -SNULL, 94651138351103, 94651138355199, -STORE, 94651138342912, 94651138351103, -STORE, 94651138351104, 94651138355199, -SNULL, 140325790511103, 140325790515199, -STORE, 140325790507008, 140325790511103, -STORE, 140325790511104, 140325790515199, -ERASE, 140325790478336, 140325790507007, -STORE, 94651146297344, 94651146432511, -STORE, 94212330168320, 94212330381311, -STORE, 94212332478464, 94212332482559, -STORE, 94212332482560, 94212332490751, -STORE, 94212332490752, 94212332503039, -STORE, 94212348891136, 94212349825023, -STORE, 140611630604288, 140611632263167, -STORE, 140611632263168, 140611634360319, -STORE, 140611634360320, 140611634376703, -STORE, 140611634376704, 140611634384895, -STORE, 140611634384896, 140611634401279, -STORE, 140611634401280, 140611634413567, -STORE, 140611634413568, 140611636506623, -STORE, 140611636506624, 140611636510719, -STORE, 140611636510720, 140611636514815, -STORE, 140611636514816, 140611636658175, -STORE, 140611637026816, 140611638710271, -STORE, 140611638710272, 140611638726655, -STORE, 140611638755328, 140611638759423, -STORE, 140611638759424, 140611638763519, -STORE, 140611638763520, 140611638767615, -STORE, 140726974533632, 140726974672895, -STORE, 140726974943232, 140726974955519, -STORE, 140726974955520, 140726974959615, -STORE, 94572463521792, 94572463734783, -STORE, 94572465831936, 94572465836031, -STORE, 94572465836032, 94572465844223, -STORE, 94572465844224, 94572465856511, -STORE, 94572491534336, 94572492865535, -STORE, 140644351492096, 140644353150975, -STORE, 140644353150976, 140644355248127, -STORE, 140644355248128, 140644355264511, -STORE, 140644355264512, 140644355272703, -STORE, 140644355272704, 140644355289087, -STORE, 140644355289088, 140644355301375, -STORE, 140644355301376, 140644357394431, -STORE, 140644357394432, 140644357398527, -STORE, 140644357398528, 140644357402623, -STORE, 140644357402624, 140644357545983, -STORE, 140644357914624, 140644359598079, -STORE, 140644359598080, 140644359614463, -STORE, 140644359643136, 140644359647231, -STORE, 140644359647232, 140644359651327, -STORE, 140644359651328, 140644359655423, -STORE, 140727841824768, 140727841964031, -STORE, 140727843188736, 140727843201023, -STORE, 140727843201024, 140727843205119, -STORE, 94144315457536, 94144315670527, -STORE, 94144317767680, 94144317771775, -STORE, 94144317771776, 94144317779967, -STORE, 94144317779968, 94144317792255, -STORE, 94144318369792, 94144320815103, -STORE, 140316717645824, 140316719304703, -STORE, 140316719304704, 140316721401855, -STORE, 140316721401856, 140316721418239, -STORE, 140316721418240, 140316721426431, -STORE, 140316721426432, 140316721442815, -STORE, 140316721442816, 140316721455103, -STORE, 140316721455104, 140316723548159, -STORE, 140316723548160, 140316723552255, -STORE, 140316723552256, 140316723556351, -STORE, 140316723556352, 140316723699711, -STORE, 140316724068352, 140316725751807, -STORE, 140316725751808, 140316725768191, -STORE, 140316725796864, 140316725800959, -STORE, 140316725800960, 140316725805055, -STORE, 140316725805056, 140316725809151, -STORE, 140725744283648, 140725744422911, -STORE, 140725745852416, 140725745864703, -STORE, 140725745864704, 140725745868799, -STORE, 94646858846208, 94646859059199, -STORE, 94646861156352, 94646861160447, -STORE, 94646861160448, 94646861168639, -STORE, 94646861168640, 94646861180927, -STORE, 94646879805440, 94646881894399, -STORE, 140435449745408, 140435451404287, -STORE, 140435451404288, 140435453501439, -STORE, 140435453501440, 140435453517823, -STORE, 140435453517824, 140435453526015, -STORE, 140435453526016, 140435453542399, -STORE, 140435453542400, 140435453554687, -STORE, 140435453554688, 140435455647743, -STORE, 140435455647744, 140435455651839, -STORE, 140435455651840, 140435455655935, -STORE, 140435455655936, 140435455799295, -STORE, 140435456167936, 140435457851391, -STORE, 140435457851392, 140435457867775, -STORE, 140435457896448, 140435457900543, -STORE, 140435457900544, 140435457904639, -STORE, 140435457904640, 140435457908735, -STORE, 140721033818112, 140721033957375, -STORE, 140721034018816, 140721034031103, -STORE, 140721034031104, 140721034035199, -STORE, 94872903438336, 94872903651327, -STORE, 94872905748480, 94872905752575, -STORE, 94872905752576, 94872905760767, -STORE, 94872905760768, 94872905773055, -STORE, 94872931246080, 94872931651583, -STORE, 139771607810048, 139771609468927, -STORE, 139771609468928, 139771611566079, -STORE, 139771611566080, 139771611582463, -STORE, 139771611582464, 139771611590655, -STORE, 139771611590656, 139771611607039, -STORE, 139771611607040, 139771611619327, -STORE, 139771611619328, 139771613712383, -STORE, 139771613712384, 139771613716479, -STORE, 139771613716480, 139771613720575, -STORE, 139771613720576, 139771613863935, -STORE, 139771614232576, 139771615916031, -STORE, 139771615916032, 139771615932415, -STORE, 139771615961088, 139771615965183, -STORE, 139771615965184, 139771615969279, -STORE, 139771615969280, 139771615973375, -STORE, 140725402931200, 140725403070463, -STORE, 140725403852800, 140725403865087, -STORE, 140725403865088, 140725403869183, -STORE, 94740737736704, 94740737949695, -STORE, 94740740046848, 94740740050943, -STORE, 94740740050944, 94740740059135, -STORE, 94740740059136, 94740740071423, -STORE, 94740743249920, 94740744724479, -STORE, 140640287010816, 140640288669695, -STORE, 140640288669696, 140640290766847, -STORE, 140640290766848, 140640290783231, -STORE, 140640290783232, 140640290791423, -STORE, 140640290791424, 140640290807807, -STORE, 140640290807808, 140640290820095, -STORE, 140640290820096, 140640292913151, -STORE, 140640292913152, 140640292917247, -STORE, 140640292917248, 140640292921343, -STORE, 140640292921344, 140640293064703, -STORE, 140640293433344, 140640295116799, -STORE, 140640295116800, 140640295133183, -STORE, 140640295161856, 140640295165951, -STORE, 140640295165952, 140640295170047, -STORE, 140640295170048, 140640295174143, -STORE, 140725133303808, 140725133443071, -STORE, 140725133684736, 140725133697023, -STORE, 140725133697024, 140725133701119, -STORE, 140737488347136, 140737488351231, -STORE, 140722826371072, 140737488351231, -SNULL, 140722826375167, 140737488351231, -STORE, 140722826371072, 140722826375167, -STORE, 140722826240000, 140722826375167, -STORE, 94113818611712, 94113820835839, -SNULL, 94113818722303, 94113820835839, -STORE, 94113818611712, 94113818722303, -STORE, 94113818722304, 94113820835839, -ERASE, 94113818722304, 94113820835839, -STORE, 94113820815360, 94113820827647, -STORE, 94113820827648, 94113820835839, -STORE, 139628194508800, 139628196761599, -SNULL, 139628194652159, 139628196761599, -STORE, 139628194508800, 139628194652159, -STORE, 139628194652160, 139628196761599, -ERASE, 139628194652160, 139628196761599, -STORE, 139628196749312, 139628196757503, -STORE, 139628196757504, 139628196761599, -STORE, 140722826727424, 140722826731519, -STORE, 140722826715136, 140722826727423, -STORE, 139628196720640, 139628196749311, -STORE, 139628196712448, 139628196720639, -STORE, 139628190711808, 139628194508799, -SNULL, 139628190711808, 139628192370687, -STORE, 139628192370688, 139628194508799, -STORE, 139628190711808, 139628192370687, -SNULL, 139628194467839, 139628194508799, -STORE, 139628192370688, 139628194467839, -STORE, 139628194467840, 139628194508799, -SNULL, 139628194467840, 139628194492415, -STORE, 139628194492416, 139628194508799, -STORE, 139628194467840, 139628194492415, -ERASE, 139628194467840, 139628194492415, -STORE, 139628194467840, 139628194492415, -ERASE, 139628194492416, 139628194508799, -STORE, 139628194492416, 139628194508799, -SNULL, 139628194484223, 139628194492415, -STORE, 139628194467840, 139628194484223, -STORE, 139628194484224, 139628194492415, -SNULL, 94113820823551, 94113820827647, -STORE, 94113820815360, 94113820823551, -STORE, 94113820823552, 94113820827647, -SNULL, 139628196753407, 139628196757503, -STORE, 139628196749312, 139628196753407, -STORE, 139628196753408, 139628196757503, -ERASE, 139628196720640, 139628196749311, -STORE, 94113830850560, 94113830985727, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731865833472, 140737488351231, -SNULL, 140731865841663, 140737488351231, -STORE, 140731865833472, 140731865841663, -STORE, 140731865702400, 140731865841663, -STORE, 94763339386880, 94763341611007, -SNULL, 94763339497471, 94763341611007, -STORE, 94763339386880, 94763339497471, -STORE, 94763339497472, 94763341611007, -ERASE, 94763339497472, 94763341611007, -STORE, 94763341590528, 94763341602815, -STORE, 94763341602816, 94763341611007, -STORE, 139778398486528, 139778400739327, -SNULL, 139778398629887, 139778400739327, -STORE, 139778398486528, 139778398629887, -STORE, 139778398629888, 139778400739327, -ERASE, 139778398629888, 139778400739327, -STORE, 139778400727040, 139778400735231, -STORE, 139778400735232, 139778400739327, -STORE, 140731865858048, 140731865862143, -STORE, 140731865845760, 140731865858047, -STORE, 139778400698368, 139778400727039, -STORE, 139778400690176, 139778400698367, -STORE, 139778394689536, 139778398486527, -SNULL, 139778394689536, 139778396348415, -STORE, 139778396348416, 139778398486527, -STORE, 139778394689536, 139778396348415, -SNULL, 139778398445567, 139778398486527, -STORE, 139778396348416, 139778398445567, -STORE, 139778398445568, 139778398486527, -SNULL, 139778398445568, 139778398470143, -STORE, 139778398470144, 139778398486527, -STORE, 139778398445568, 139778398470143, -ERASE, 139778398445568, 139778398470143, -STORE, 139778398445568, 139778398470143, -ERASE, 139778398470144, 139778398486527, -STORE, 139778398470144, 139778398486527, -SNULL, 139778398461951, 139778398470143, -STORE, 139778398445568, 139778398461951, -STORE, 139778398461952, 139778398470143, -SNULL, 94763341598719, 94763341602815, -STORE, 94763341590528, 94763341598719, -STORE, 94763341598720, 94763341602815, -SNULL, 139778400731135, 139778400735231, -STORE, 139778400727040, 139778400731135, -STORE, 139778400731136, 139778400735231, -ERASE, 139778400698368, 139778400727039, -STORE, 94763362197504, 94763362332671, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140737488338944, 140737488351231, -STORE, 140732053192704, 140737488351231, -SNULL, 140732053204991, 140737488351231, -STORE, 140732053192704, 140732053204991, -STORE, 140732053061632, 140732053204991, -STORE, 4194304, 26279935, -STORE, 28372992, 28454911, -STORE, 28454912, 29806591, -STORE, 140176018599936, 140176020852735, -SNULL, 140176018743295, 140176020852735, -STORE, 140176018599936, 140176018743295, -STORE, 140176018743296, 140176020852735, -ERASE, 140176018743296, 140176020852735, -STORE, 140176020840448, 140176020848639, -STORE, 140176020848640, 140176020852735, -STORE, 140732053381120, 140732053385215, -STORE, 140732053368832, 140732053381119, -STORE, 140176020811776, 140176020840447, -STORE, 140176020803584, 140176020811775, -STORE, 140176014766080, 140176018599935, -SNULL, 140176014766080, 140176016474111, -STORE, 140176016474112, 140176018599935, -STORE, 140176014766080, 140176016474111, -SNULL, 140176018567167, 140176018599935, -STORE, 140176016474112, 140176018567167, -STORE, 140176018567168, 140176018599935, -ERASE, 140176018567168, 140176018599935, -STORE, 140176018567168, 140176018599935, -STORE, 140176012570624, 140176014766079, -SNULL, 140176012570624, 140176012664831, -STORE, 140176012664832, 140176014766079, -STORE, 140176012570624, 140176012664831, -SNULL, 140176014757887, 140176014766079, -STORE, 140176012664832, 140176014757887, -STORE, 140176014757888, 140176014766079, -ERASE, 140176014757888, 140176014766079, -STORE, 140176014757888, 140176014766079, -STORE, 140176010051584, 140176012570623, -SNULL, 140176010051584, 140176010465279, -STORE, 140176010465280, 140176012570623, -STORE, 140176010051584, 140176010465279, -SNULL, 140176012558335, 140176012570623, -STORE, 140176010465280, 140176012558335, -STORE, 140176012558336, 140176012570623, -ERASE, 140176012558336, 140176012570623, -STORE, 140176012558336, 140176012570623, -STORE, 140176007417856, 140176010051583, -SNULL, 140176007417856, 140176007946239, -STORE, 140176007946240, 140176010051583, -STORE, 140176007417856, 140176007946239, -SNULL, 140176010043391, 140176010051583, -STORE, 140176007946240, 140176010043391, -STORE, 140176010043392, 140176010051583, -ERASE, 140176010043392, 140176010051583, -STORE, 140176010043392, 140176010051583, -STORE, 140176005304320, 140176007417855, -SNULL, 140176005304320, 140176005316607, -STORE, 140176005316608, 140176007417855, -STORE, 140176005304320, 140176005316607, -SNULL, 140176007409663, 140176007417855, -STORE, 140176005316608, 140176007409663, -STORE, 140176007409664, 140176007417855, -ERASE, 140176007409664, 140176007417855, -STORE, 140176007409664, 140176007417855, -STORE, 140176003100672, 140176005304319, -SNULL, 140176003100672, 140176003203071, -STORE, 140176003203072, 140176005304319, -STORE, 140176003100672, 140176003203071, -SNULL, 140176005296127, 140176005304319, -STORE, 140176003203072, 140176005296127, -STORE, 140176005296128, 140176005304319, -ERASE, 140176005296128, 140176005304319, -STORE, 140176005296128, 140176005304319, -STORE, 140176020795392, 140176020811775, -STORE, 140175999938560, 140176003100671, -SNULL, 140175999938560, 140176000999423, -STORE, 140176000999424, 140176003100671, -STORE, 140175999938560, 140176000999423, -SNULL, 140176003092479, 140176003100671, -STORE, 140176000999424, 140176003092479, -STORE, 140176003092480, 140176003100671, -ERASE, 140176003092480, 140176003100671, -STORE, 140176003092480, 140176003100671, -STORE, 140175996141568, 140175999938559, -SNULL, 140175996141568, 140175997800447, -STORE, 140175997800448, 140175999938559, -STORE, 140175996141568, 140175997800447, -SNULL, 140175999897599, 140175999938559, -STORE, 140175997800448, 140175999897599, -STORE, 140175999897600, 140175999938559, -SNULL, 140175999897600, 140175999922175, -STORE, 140175999922176, 140175999938559, -STORE, 140175999897600, 140175999922175, -ERASE, 140175999897600, 140175999922175, -STORE, 140175999897600, 140175999922175, -ERASE, 140175999922176, 140175999938559, -STORE, 140175999922176, 140175999938559, -STORE, 140176020783104, 140176020811775, -SNULL, 140175999913983, 140175999922175, -STORE, 140175999897600, 140175999913983, -STORE, 140175999913984, 140175999922175, -SNULL, 140176003096575, 140176003100671, -STORE, 140176003092480, 140176003096575, -STORE, 140176003096576, 140176003100671, -SNULL, 140176005300223, 140176005304319, -STORE, 140176005296128, 140176005300223, -STORE, 140176005300224, 140176005304319, -SNULL, 140176007413759, 140176007417855, -STORE, 140176007409664, 140176007413759, -STORE, 140176007413760, 140176007417855, -SNULL, 140176010047487, 140176010051583, -STORE, 140176010043392, 140176010047487, -STORE, 140176010047488, 140176010051583, -SNULL, 140176012566527, 140176012570623, -STORE, 140176012558336, 140176012566527, -STORE, 140176012566528, 140176012570623, -SNULL, 140176014761983, 140176014766079, -STORE, 140176014757888, 140176014761983, -STORE, 140176014761984, 140176014766079, -SNULL, 140176018571263, 140176018599935, -STORE, 140176018567168, 140176018571263, -STORE, 140176018571264, 140176018599935, -SNULL, 28405759, 28454911, -STORE, 28372992, 28405759, -STORE, 28405760, 28454911, -SNULL, 140176020844543, 140176020848639, -STORE, 140176020840448, 140176020844543, -STORE, 140176020844544, 140176020848639, -ERASE, 140176020811776, 140176020840447, -STORE, 53080064, 53215231, -STORE, 140176019099648, 140176020783103, -STORE, 140176020836352, 140176020840447, -STORE, 140176018964480, 140176019099647, -STORE, 53080064, 53358591, -STORE, 140175994044416, 140175996141567, -STORE, 140176020828160, 140176020840447, -STORE, 140176020819968, 140176020840447, -STORE, 140176020783104, 140176020819967, -STORE, 140176018948096, 140176019099647, -STORE, 53080064, 53493759, -STORE, 53080064, 53649407, -STORE, 140176018939904, 140176019099647, -STORE, 140176018931712, 140176019099647, -STORE, 53080064, 53784575, -STORE, 53080064, 53919743, -STORE, 140176018915328, 140176019099647, -STORE, 140176018907136, 140176019099647, -STORE, 53080064, 54059007, -STORE, 140175993769984, 140175996141567, -STORE, 140176018747392, 140176019099647, -STORE, 53080064, 54198271, -SNULL, 54190079, 54198271, -STORE, 53080064, 54190079, -STORE, 54190080, 54198271, -ERASE, 54190080, 54198271, -SNULL, 54181887, 54190079, -STORE, 53080064, 54181887, -STORE, 54181888, 54190079, -ERASE, 54181888, 54190079, -SNULL, 54173695, 54181887, -STORE, 53080064, 54173695, -STORE, 54173696, 54181887, -ERASE, 54173696, 54181887, -SNULL, 54165503, 54173695, -STORE, 53080064, 54165503, -STORE, 54165504, 54173695, -ERASE, 54165504, 54173695, -STORE, 140175993753600, 140175996141567, -STORE, 140175993688064, 140175996141567, -STORE, 140175993655296, 140175996141567, -STORE, 140175991558144, 140175996141567, -STORE, 140175991492608, 140175996141567, -STORE, 53080064, 54312959, -STORE, 140175991361536, 140175996141567, -STORE, 140175991099392, 140175996141567, -STORE, 140175991091200, 140175996141567, -STORE, 140175991074816, 140175996141567, -STORE, 140175991066624, 140175996141567, -STORE, 140175991058432, 140175996141567, -STORE, 53080064, 54448127, -SNULL, 54439935, 54448127, -STORE, 53080064, 54439935, -STORE, 54439936, 54448127, -ERASE, 54439936, 54448127, -SNULL, 54431743, 54439935, -STORE, 53080064, 54431743, -STORE, 54431744, 54439935, -ERASE, 54431744, 54439935, -SNULL, 54419455, 54431743, -STORE, 53080064, 54419455, -STORE, 54419456, 54431743, -ERASE, 54419456, 54431743, -SNULL, 54403071, 54419455, -STORE, 53080064, 54403071, -STORE, 54403072, 54419455, -ERASE, 54403072, 54419455, -STORE, 140175991042048, 140175996141567, -STORE, 53080064, 54538239, -SNULL, 54534143, 54538239, -STORE, 53080064, 54534143, -STORE, 54534144, 54538239, -ERASE, 54534144, 54538239, -SNULL, 54530047, 54534143, -STORE, 53080064, 54530047, -STORE, 54530048, 54534143, -ERASE, 54530048, 54534143, -SNULL, 54525951, 54530047, -STORE, 53080064, 54525951, -STORE, 54525952, 54530047, -ERASE, 54525952, 54530047, -SNULL, 54521855, 54525951, -STORE, 53080064, 54521855, -STORE, 54521856, 54525951, -ERASE, 54521856, 54525951, -SNULL, 54517759, 54521855, -STORE, 53080064, 54517759, -STORE, 54517760, 54521855, -ERASE, 54517760, 54521855, -SNULL, 54513663, 54517759, -STORE, 53080064, 54513663, -STORE, 54513664, 54517759, -ERASE, 54513664, 54517759, -SNULL, 54509567, 54513663, -STORE, 53080064, 54509567, -STORE, 54509568, 54513663, -ERASE, 54509568, 54513663, -STORE, 140175991025664, 140175996141567, -STORE, 140175990992896, 140175996141567, -STORE, 53080064, 54644735, -SNULL, 54628351, 54644735, -STORE, 53080064, 54628351, -STORE, 54628352, 54644735, -ERASE, 54628352, 54644735, -SNULL, 54616063, 54628351, -STORE, 53080064, 54616063, -STORE, 54616064, 54628351, -ERASE, 54616064, 54628351, -STORE, 140175988895744, 140175996141567, -STORE, 53080064, 54767615, -STORE, 140175988879360, 140175996141567, -STORE, 140175988617216, 140175996141567, -STORE, 140175988609024, 140175996141567, -STORE, 140175988600832, 140175996141567, -STORE, 53080064, 54906879, -SNULL, 54898687, 54906879, -STORE, 53080064, 54898687, -STORE, 54898688, 54906879, -ERASE, 54898688, 54906879, -SNULL, 54853631, 54898687, -STORE, 53080064, 54853631, -STORE, 54853632, 54898687, -ERASE, 54853632, 54898687, -STORE, 140175986503680, 140175996141567, -STORE, 53080064, 54996991, -STORE, 140175986495488, 140175996141567, -STORE, 140175986487296, 140175996141567, -STORE, 140175985438720, 140175996141567, -STORE, 53080064, 55136255, -STORE, 140175985405952, 140175996141567, -STORE, 140175985139712, 140175996141567, -SNULL, 140176018964479, 140176019099647, -STORE, 140176018747392, 140176018964479, -STORE, 140176018964480, 140176019099647, -ERASE, 140176018964480, 140176019099647, -STORE, 140175983042560, 140175996141567, -STORE, 140175982518272, 140175996141567, -STORE, 140175980421120, 140175996141567, -STORE, 53080064, 55287807, -STORE, 53080064, 55427071, -STORE, 140176019091456, 140176019099647, -STORE, 140176019083264, 140176019099647, -STORE, 140176019075072, 140176019099647, -STORE, 140176019066880, 140176019099647, -STORE, 140176019058688, 140176019099647, -STORE, 140175980158976, 140175996141567, -STORE, 140176019050496, 140176019099647, -STORE, 140176019042304, 140176019099647, -STORE, 140176019034112, 140176019099647, -STORE, 140176019025920, 140176019099647, -STORE, 140176019017728, 140176019099647, -STORE, 140176019009536, 140176019099647, -STORE, 140176019001344, 140176019099647, -STORE, 140176018993152, 140176019099647, -STORE, 140176018984960, 140176019099647, -STORE, 140176018976768, 140176019099647, -STORE, 140176018968576, 140176019099647, -STORE, 140175978061824, 140175996141567, -STORE, 53080064, 55603199, -STORE, 140175978029056, 140175996141567, -STORE, 140175977996288, 140175996141567, -STORE, 53080064, 55738367, -STORE, 53080064, 55881727, -STORE, 140175977963520, 140175996141567, -STORE, 140175977930752, 140175996141567, -STORE, 53080064, 56041471, -STORE, 140175977897984, 140175996141567, -STORE, 140175977865216, 140175996141567, -SNULL, 55881727, 56041471, -STORE, 53080064, 55881727, -STORE, 55881728, 56041471, -ERASE, 55881728, 56041471, -SNULL, 55721983, 55881727, -STORE, 53080064, 55721983, -STORE, 55721984, 55881727, -ERASE, 55721984, 55881727, -SNULL, 55570431, 55721983, -STORE, 53080064, 55570431, -STORE, 55570432, 55721983, -ERASE, 55570432, 55721983, -STORE, 140175977857024, 140175996141567, -STORE, 140175975759872, 140175996141567, -STORE, 53080064, 55754751, -STORE, 53080064, 55943167, -STORE, 140175975751680, 140175996141567, -STORE, 140175975743488, 140175996141567, -STORE, 140175975735296, 140175996141567, -STORE, 140175975727104, 140175996141567, -STORE, 140175975718912, 140175996141567, -STORE, 140175975710720, 140175996141567, -STORE, 140175975702528, 140175996141567, -STORE, 140175975694336, 140175996141567, -STORE, 140175975686144, 140175996141567, -STORE, 140175975677952, 140175996141567, -STORE, 140175975669760, 140175996141567, -STORE, 140175974621184, 140175996141567, -STORE, 140175974612992, 140175996141567, -STORE, 53080064, 56139775, -STORE, 140175972515840, 140175996141567, -STORE, 53080064, 56401919, -STORE, 140175970418688, 140175996141567, -STORE, 140175970410496, 140175996141567, -STORE, 140175970402304, 140175996141567, -STORE, 140175970394112, 140175996141567, -STORE, 53080064, 56569855, -STORE, 140175969865728, 140175996141567, -SNULL, 140175985139711, 140175996141567, -STORE, 140175969865728, 140175985139711, -STORE, 140175985139712, 140175996141567, -SNULL, 140175985139712, 140175985405951, -STORE, 140175985405952, 140175996141567, -STORE, 140175985139712, 140175985405951, -ERASE, 140175985139712, 140175985405951, -STORE, 140175965671424, 140175985139711, -STORE, 140175985397760, 140175996141567, -STORE, 140175985389568, 140175996141567, -STORE, 140175985381376, 140175996141567, -STORE, 140175985373184, 140175996141567, -STORE, 140175985364992, 140175996141567, -STORE, 140175985356800, 140175996141567, -STORE, 140175985348608, 140175996141567, -STORE, 140175985340416, 140175996141567, -STORE, 140175985332224, 140175996141567, -STORE, 140175985324032, 140175996141567, -STORE, 140175985315840, 140175996141567, -STORE, 140175985307648, 140175996141567, -STORE, 140175985299456, 140175996141567, -STORE, 140175985291264, 140175996141567, -STORE, 140175985283072, 140175996141567, -STORE, 140175985274880, 140175996141567, -STORE, 140175963574272, 140175985139711, -STORE, 140175985266688, 140175996141567, -STORE, 140175961477120, 140175985139711, -STORE, 53080064, 56831999, -STORE, 140175959379968, 140175985139711, -STORE, 140175985258496, 140175996141567, -STORE, 140175957282816, 140175985139711, -STORE, 140175985250304, 140175996141567, -STORE, 140175985242112, 140175996141567, -STORE, 140175985233920, 140175996141567, -STORE, 140175985225728, 140175996141567, -STORE, 140175985217536, 140175996141567, -STORE, 140175957151744, 140175985139711, -STORE, 140175956627456, 140175985139711, -SNULL, 140175980158975, 140175985139711, -STORE, 140175956627456, 140175980158975, -STORE, 140175980158976, 140175985139711, -SNULL, 140175980158976, 140175980421119, -STORE, 140175980421120, 140175985139711, -STORE, 140175980158976, 140175980421119, -ERASE, 140175980158976, 140175980421119, -STORE, 140175954530304, 140175980158975, -STORE, 140175985209344, 140175996141567, -STORE, 53080064, 57094143, -STORE, 140175952433152, 140175980158975, -STORE, 140175985192960, 140175996141567, -STORE, 140175985184768, 140175996141567, -STORE, 140175985176576, 140175996141567, -STORE, 140175985168384, 140175996141567, -STORE, 140175985160192, 140175996141567, -STORE, 140175985152000, 140175996141567, -STORE, 140175985143808, 140175996141567, -STORE, 140175980412928, 140175985139711, -STORE, 140175980404736, 140175985139711, -STORE, 140175980396544, 140175985139711, -STORE, 140175980388352, 140175985139711, -STORE, 140175980380160, 140175985139711, -STORE, 140175980371968, 140175985139711, -STORE, 140175980363776, 140175985139711, -STORE, 140175980355584, 140175985139711, -STORE, 140175980347392, 140175985139711, -STORE, 140175980339200, 140175985139711, -STORE, 53080064, 57356287, -SNULL, 140176018747392, 140176018907135, -STORE, 140176018907136, 140176018964479, -STORE, 140176018747392, 140176018907135, -ERASE, 140176018747392, 140176018907135, -STORE, 140175952146432, 140175980158975, -STORE, 140175950049280, 140175980158975, -SNULL, 140175952146431, 140175980158975, -STORE, 140175950049280, 140175952146431, -STORE, 140175952146432, 140175980158975, -SNULL, 140175952146432, 140175952433151, -STORE, 140175952433152, 140175980158975, -STORE, 140175952146432, 140175952433151, -ERASE, 140175952146432, 140175952433151, -STORE, 140176018898944, 140176018964479, -STORE, 53080064, 57749503, -STORE, 140175949520896, 140175952146431, -STORE, 140175947423744, 140175952146431, -SNULL, 140175993769983, 140175996141567, -STORE, 140175985143808, 140175993769983, -STORE, 140175993769984, 140175996141567, -SNULL, 140175993769984, 140175994044415, -STORE, 140175994044416, 140175996141567, -STORE, 140175993769984, 140175994044415, -ERASE, 140175993769984, 140175994044415, -STORE, 140176018890752, 140176018964479, -STORE, 140176018882560, 140176018964479, -STORE, 140176018874368, 140176018964479, -STORE, 140176018866176, 140176018964479, -STORE, 140176018849792, 140176018964479, -STORE, 140176018841600, 140176018964479, -STORE, 140176018825216, 140176018964479, -STORE, 140176018817024, 140176018964479, -STORE, 140176018800640, 140176018964479, -STORE, 140176018792448, 140176018964479, -STORE, 140176018759680, 140176018964479, -STORE, 140176018751488, 140176018964479, -STORE, 140175994028032, 140175996141567, -STORE, 140176018743296, 140176018964479, -STORE, 140175994011648, 140175996141567, -STORE, 140175994003456, 140175996141567, -STORE, 140175993987072, 140175996141567, -STORE, 140175993978880, 140175996141567, -STORE, 140175993946112, 140175996141567, -STORE, 140175993937920, 140175996141567, -STORE, 140175993921536, 140175996141567, -STORE, 140175993913344, 140175996141567, -STORE, 140175993896960, 140175996141567, -STORE, 140175993888768, 140175996141567, -STORE, 140175993872384, 140175996141567, -STORE, 140175993864192, 140175996141567, -STORE, 140175993831424, 140175996141567, -STORE, 140175993823232, 140175996141567, -STORE, 140175993806848, 140175996141567, -STORE, 140175993798656, 140175996141567, -STORE, 140175993782272, 140175996141567, -STORE, 140175993774080, 140175996141567, -STORE, 140175980322816, 140175985139711, -STORE, 140175980314624, 140175985139711, -STORE, 140175980281856, 140175985139711, -STORE, 140175980273664, 140175985139711, -STORE, 140175980257280, 140175985139711, -STORE, 140175945326592, 140175952146431, -STORE, 140175980249088, 140175985139711, -STORE, 140175980232704, 140175985139711, -STORE, 140175980224512, 140175985139711, -STORE, 140175980208128, 140175985139711, -STORE, 140175980199936, 140175985139711, -STORE, 140175980167168, 140175985139711, -STORE, 140175952433152, 140175985139711, -STORE, 140175952416768, 140175985139711, -STORE, 140175952408576, 140175985139711, -STORE, 140175952392192, 140175985139711, -STORE, 140175952384000, 140175985139711, -STORE, 140175952367616, 140175985139711, -STORE, 140175943229440, 140175952146431, -STORE, 140175952359424, 140175985139711, -STORE, 140175952326656, 140175985139711, -STORE, 140175952318464, 140175985139711, -STORE, 140175952302080, 140175985139711, -STORE, 140175952293888, 140175985139711, -STORE, 140175952277504, 140175985139711, -STORE, 140175952269312, 140175985139711, -STORE, 140175952252928, 140175985139711, -STORE, 140175952244736, 140175985139711, -STORE, 140175952211968, 140175985139711, -STORE, 140175952203776, 140175985139711, -STORE, 140175952187392, 140175985139711, -STORE, 140175952179200, 140175985139711, -STORE, 140175952162816, 140175985139711, -STORE, 140175952154624, 140175985139711, -STORE, 140175943213056, 140175952146431, -STORE, 140175943213056, 140175985139711, -STORE, 140175943180288, 140175985139711, -STORE, 140175943172096, 140175985139711, -STORE, 140175943155712, 140175985139711, -STORE, 140175943147520, 140175985139711, -STORE, 140175943131136, 140175985139711, -STORE, 140175943122944, 140175985139711, -STORE, 140175943106560, 140175985139711, -STORE, 140175943098368, 140175985139711, -STORE, 140175943065600, 140175985139711, -STORE, 140175943057408, 140175985139711, -STORE, 140175943041024, 140175985139711, -STORE, 140175943032832, 140175985139711, -STORE, 140175943016448, 140175985139711, -STORE, 140175943008256, 140175985139711, -STORE, 140175942991872, 140175985139711, -STORE, 140175942983680, 140175985139711, -STORE, 140175942950912, 140175985139711, -STORE, 140175942942720, 140175985139711, -STORE, 140175942926336, 140175985139711, -STORE, 140175942918144, 140175985139711, -STORE, 140175942901760, 140175985139711, -STORE, 140175942893568, 140175985139711, -STORE, 140175942877184, 140175985139711, -STORE, 140175942868992, 140175985139711, -STORE, 140175942836224, 140175985139711, -STORE, 140175942828032, 140175985139711, -STORE, 140175942811648, 140175985139711, -STORE, 140175942803456, 140175985139711, -STORE, 140175942787072, 140175985139711, -STORE, 140175942778880, 140175985139711, -STORE, 140175942762496, 140175985139711, -STORE, 140175942754304, 140175985139711, -STORE, 140175942721536, 140175985139711, -STORE, 140175942713344, 140175985139711, -STORE, 140175942696960, 140175985139711, -STORE, 140175942688768, 140175985139711, -STORE, 140175942672384, 140175985139711, -STORE, 140175942664192, 140175985139711, -STORE, 140175942647808, 140175985139711, -STORE, 140175942639616, 140175985139711, -STORE, 140175942606848, 140175985139711, -STORE, 140175942598656, 140175985139711, -STORE, 140175942582272, 140175985139711, -STORE, 140175942574080, 140175985139711, -STORE, 140175942557696, 140175985139711, -STORE, 140175942549504, 140175985139711, -STORE, 140175942533120, 140175985139711, -STORE, 140175942524928, 140175985139711, -STORE, 140175942492160, 140175985139711, -STORE, 140175942483968, 140175985139711, -STORE, 140175942467584, 140175985139711, -STORE, 140175942459392, 140175985139711, -STORE, 140175942443008, 140175985139711, -STORE, 140175942434816, 140175985139711, -STORE, 140175942418432, 140175985139711, -STORE, 140175942410240, 140175985139711, -STORE, 140175942377472, 140175985139711, -STORE, 140175942369280, 140175985139711, -STORE, 140175942352896, 140175985139711, -STORE, 140175942344704, 140175985139711, -STORE, 140175942328320, 140175985139711, -STORE, 140175942320128, 140175985139711, -STORE, 140175942303744, 140175985139711, -STORE, 140175942295552, 140175985139711, -STORE, 140175942262784, 140175985139711, -STORE, 140175942254592, 140175985139711, -STORE, 140175942238208, 140175985139711, -STORE, 140175942230016, 140175985139711, -STORE, 140175942213632, 140175985139711, -STORE, 140175942205440, 140175985139711, -STORE, 140175942189056, 140175985139711, -STORE, 140175942180864, 140175985139711, -STORE, 140175942148096, 140175985139711, -STORE, 140175942139904, 140175985139711, -STORE, 140175942123520, 140175985139711, -STORE, 140175942115328, 140175985139711, -STORE, 140175942098944, 140175985139711, -STORE, 140175942090752, 140175985139711, -STORE, 140175942074368, 140175985139711, -STORE, 140175942066176, 140175985139711, -STORE, 140175942033408, 140175985139711, -STORE, 140175942025216, 140175985139711, -STORE, 140175942008832, 140175985139711, -STORE, 140175942000640, 140175985139711, -STORE, 140175941984256, 140175985139711, -STORE, 140175941976064, 140175985139711, -STORE, 140175941959680, 140175985139711, -STORE, 140175939862528, 140175985139711, -STORE, 140175939854336, 140175985139711, -STORE, 140175939821568, 140175985139711, -STORE, 140175939813376, 140175985139711, -STORE, 140175939796992, 140175985139711, -STORE, 140175939788800, 140175985139711, -STORE, 140175939772416, 140175985139711, -STORE, 140175939764224, 140175985139711, -STORE, 140175939747840, 140175985139711, -STORE, 140175939739648, 140175985139711, -STORE, 140175939706880, 140175985139711, -STORE, 140175939698688, 140175985139711, -STORE, 140175939682304, 140175985139711, -STORE, 140175939674112, 140175985139711, -STORE, 140175939657728, 140175985139711, -STORE, 140175939649536, 140175985139711, -STORE, 140175939633152, 140175985139711, -STORE, 140175939624960, 140175985139711, -STORE, 140175939592192, 140175985139711, -STORE, 140175939584000, 140175985139711, -STORE, 140175939567616, 140175985139711, -STORE, 140175939559424, 140175985139711, -STORE, 140175939543040, 140175985139711, -STORE, 140175939534848, 140175985139711, -STORE, 140175939518464, 140175985139711, -STORE, 140175939510272, 140175985139711, -STORE, 140175939477504, 140175985139711, -STORE, 140175939469312, 140175985139711, -STORE, 140175939452928, 140175985139711, -STORE, 140175939444736, 140175985139711, -STORE, 140175939428352, 140175985139711, -STORE, 140175939420160, 140175985139711, -STORE, 140175939403776, 140175985139711, -STORE, 140175939395584, 140175985139711, -STORE, 140175939362816, 140175985139711, -STORE, 140175939354624, 140175985139711, -STORE, 140175939338240, 140175985139711, -STORE, 140175939330048, 140175985139711, -STORE, 140175939313664, 140175985139711, -STORE, 140175939305472, 140175985139711, -STORE, 140175939289088, 140175985139711, -STORE, 140175939280896, 140175985139711, -STORE, 140175939248128, 140175985139711, -STORE, 140175939239936, 140175985139711, -STORE, 140175939223552, 140175985139711, -STORE, 140175939215360, 140175985139711, -STORE, 140175939198976, 140175985139711, -STORE, 140175939190784, 140175985139711, -STORE, 140175939174400, 140175985139711, -STORE, 140175939166208, 140175985139711, -STORE, 140175939133440, 140175985139711, -STORE, 140175939125248, 140175985139711, -STORE, 140175939108864, 140175985139711, -STORE, 140175939100672, 140175985139711, -STORE, 140175939084288, 140175985139711, -STORE, 140175939076096, 140175985139711, -STORE, 140175939059712, 140175985139711, -STORE, 140175939051520, 140175985139711, -STORE, 140175939018752, 140175985139711, -STORE, 140175939010560, 140175985139711, -STORE, 140175938994176, 140175985139711, -STORE, 140175938985984, 140175985139711, -STORE, 140175938969600, 140175985139711, -STORE, 140175938961408, 140175985139711, -STORE, 140175938945024, 140175985139711, -STORE, 140175938936832, 140175985139711, -STORE, 140175938904064, 140175985139711, -STORE, 140175938895872, 140175985139711, -STORE, 140175938879488, 140175985139711, -STORE, 140175938871296, 140175985139711, -STORE, 140175938854912, 140175985139711, -STORE, 140175938846720, 140175985139711, -STORE, 140175938830336, 140175985139711, -STORE, 140175938822144, 140175985139711, -STORE, 140175938789376, 140175985139711, -STORE, 140175938781184, 140175985139711, -STORE, 140175938764800, 140175985139711, -STORE, 140175938756608, 140175985139711, -STORE, 140175938740224, 140175985139711, -STORE, 140175938732032, 140175985139711, -STORE, 140175938715648, 140175985139711, -STORE, 140175938707456, 140175985139711, -STORE, 140175938674688, 140175985139711, -STORE, 140175938666496, 140175985139711, -STORE, 140175938650112, 140175985139711, -STORE, 140175938641920, 140175985139711, -STORE, 140175938625536, 140175985139711, -STORE, 140175938617344, 140175985139711, -STORE, 140175938600960, 140175985139711, -STORE, 140175938592768, 140175985139711, -STORE, 140175938560000, 140175985139711, -STORE, 140175938551808, 140175985139711, -STORE, 140175938535424, 140175985139711, -STORE, 140175938527232, 140175985139711, -STORE, 140175938510848, 140175985139711, -STORE, 140175938502656, 140175985139711, -STORE, 140175938486272, 140175985139711, -STORE, 140175938478080, 140175985139711, -STORE, 140175938445312, 140175985139711, -STORE, 140175938437120, 140175985139711, -STORE, 140175938420736, 140175985139711, -STORE, 140175938412544, 140175985139711, -STORE, 140175938396160, 140175985139711, -STORE, 140175938387968, 140175985139711, -STORE, 140175938371584, 140175985139711, -STORE, 140175938363392, 140175985139711, -STORE, 140175938330624, 140175985139711, -STORE, 140175938322432, 140175985139711, -STORE, 140175938306048, 140175985139711, -STORE, 140175938297856, 140175985139711, -STORE, 140175938281472, 140175985139711, -STORE, 140175938273280, 140175985139711, -STORE, 140175938256896, 140175985139711, -STORE, 140175938248704, 140175985139711, -STORE, 140175938215936, 140175985139711, -STORE, 140175938207744, 140175985139711, -STORE, 140175938191360, 140175985139711, -STORE, 140175938183168, 140175985139711, -STORE, 140175938166784, 140175985139711, -STORE, 140175938158592, 140175985139711, -STORE, 140175938142208, 140175985139711, -STORE, 140175936045056, 140175985139711, -STORE, 140175936036864, 140175985139711, -STORE, 140175936004096, 140175985139711, -STORE, 140175935995904, 140175985139711, -STORE, 140175935979520, 140175985139711, -STORE, 140175935971328, 140175985139711, -STORE, 140175935954944, 140175985139711, -STORE, 140175935946752, 140175985139711, -STORE, 140175935930368, 140175985139711, -STORE, 140175935922176, 140175985139711, -STORE, 140175935889408, 140175985139711, -STORE, 140175935881216, 140175985139711, -STORE, 140175935864832, 140175985139711, -STORE, 140175935856640, 140175985139711, -STORE, 140175935840256, 140175985139711, -STORE, 140175935832064, 140175985139711, -STORE, 140175935815680, 140175985139711, -STORE, 140175935807488, 140175985139711, -STORE, 140175935774720, 140175985139711, -STORE, 140175935766528, 140175985139711, -STORE, 140175935750144, 140175985139711, -STORE, 140175935741952, 140175985139711, -STORE, 140175935725568, 140175985139711, -STORE, 140175935717376, 140175985139711, -STORE, 140175935700992, 140175985139711, -STORE, 140175935692800, 140175985139711, -STORE, 140175935660032, 140175985139711, -STORE, 140175935651840, 140175985139711, -STORE, 140175935635456, 140175985139711, -STORE, 140175935627264, 140175985139711, -STORE, 140175935610880, 140175985139711, -STORE, 140175935602688, 140175985139711, -STORE, 140175935586304, 140175985139711, -STORE, 140175935578112, 140175985139711, -STORE, 140175935545344, 140175985139711, -STORE, 140175935537152, 140175985139711, -STORE, 140175935520768, 140175985139711, -STORE, 140175935512576, 140175985139711, -STORE, 140175935496192, 140175985139711, -STORE, 140175935488000, 140175985139711, -STORE, 140175935471616, 140175985139711, -STORE, 140175935463424, 140175985139711, -STORE, 140175935430656, 140175985139711, -STORE, 140175935422464, 140175985139711, -STORE, 140175935406080, 140175985139711, -STORE, 140175935397888, 140175985139711, -STORE, 140175935381504, 140175985139711, -STORE, 140175935373312, 140175985139711, -STORE, 140175935356928, 140175985139711, -STORE, 140175935348736, 140175985139711, -STORE, 140175935315968, 140175985139711, -STORE, 140175935307776, 140175985139711, -STORE, 140175935291392, 140175985139711, -STORE, 140175935283200, 140175985139711, -STORE, 140175935266816, 140175985139711, -STORE, 140175935258624, 140175985139711, -STORE, 140175935242240, 140175985139711, -STORE, 140175935234048, 140175985139711, -STORE, 140175935201280, 140175985139711, -STORE, 140175935193088, 140175985139711, -STORE, 140175935176704, 140175985139711, -STORE, 140175935168512, 140175985139711, -STORE, 140175935152128, 140175985139711, -STORE, 140175935143936, 140175985139711, -STORE, 140175935127552, 140175985139711, -STORE, 140175935119360, 140175985139711, -STORE, 140175935086592, 140175985139711, -STORE, 140175935078400, 140175985139711, -STORE, 140175935062016, 140175985139711, -STORE, 140175935053824, 140175985139711, -STORE, 140175935037440, 140175985139711, -STORE, 140175935029248, 140175985139711, -STORE, 140175935012864, 140175985139711, -STORE, 140175935004672, 140175985139711, -STORE, 140175934971904, 140175985139711, -STORE, 140175934963712, 140175985139711, -STORE, 140175934947328, 140175985139711, -STORE, 140175934939136, 140175985139711, -STORE, 140175934922752, 140175985139711, -STORE, 140175934914560, 140175985139711, -STORE, 140175934898176, 140175985139711, -STORE, 140175934889984, 140175985139711, -STORE, 140175934857216, 140175985139711, -STORE, 140175934849024, 140175985139711, -STORE, 140175934832640, 140175985139711, -STORE, 140175934824448, 140175985139711, -STORE, 140175934808064, 140175985139711, -STORE, 140175934799872, 140175985139711, -STORE, 140175934783488, 140175985139711, -STORE, 140175934775296, 140175985139711, -STORE, 140175934742528, 140175985139711, -STORE, 140175934734336, 140175985139711, -STORE, 140175934717952, 140175985139711, -STORE, 140175934709760, 140175985139711, -STORE, 140175934693376, 140175985139711, -STORE, 140175934685184, 140175985139711, -STORE, 140175934668800, 140175985139711, -STORE, 140175934660608, 140175985139711, -STORE, 140175934627840, 140175985139711, -STORE, 140175934619648, 140175985139711, -STORE, 140175934603264, 140175985139711, -STORE, 140175934595072, 140175985139711, -STORE, 140175934578688, 140175985139711, -STORE, 140175934570496, 140175985139711, -STORE, 140175934554112, 140175985139711, -STORE, 140175934545920, 140175985139711, -STORE, 140175934513152, 140175985139711, -STORE, 140175934504960, 140175985139711, -STORE, 140175934488576, 140175985139711, -STORE, 140175934480384, 140175985139711, -STORE, 140175934464000, 140175985139711, -STORE, 140175934455808, 140175985139711, -STORE, 140175934439424, 140175985139711, -STORE, 140175934431232, 140175985139711, -STORE, 140175934398464, 140175985139711, -STORE, 140175934390272, 140175985139711, -STORE, 140175934373888, 140175985139711, -STORE, 140175934365696, 140175985139711, -STORE, 140175934349312, 140175985139711, -STORE, 140175934341120, 140175985139711, -STORE, 140175934324736, 140175985139711, -STORE, 140175932227584, 140175985139711, -STORE, 140175932219392, 140175985139711, -STORE, 140175932186624, 140175985139711, -STORE, 140175932178432, 140175985139711, -STORE, 140175932162048, 140175985139711, -STORE, 140175932153856, 140175985139711, -STORE, 140175932137472, 140175985139711, -STORE, 53080064, 57884671, -STORE, 140175932129280, 140175985139711, -STORE, 140175932112896, 140175985139711, -STORE, 140175932104704, 140175985139711, -STORE, 140175932071936, 140175985139711, -STORE, 140175932063744, 140175985139711, -STORE, 140175932047360, 140175985139711, -STORE, 140175932039168, 140175985139711, -STORE, 140175932022784, 140175985139711, -STORE, 140175932014592, 140175985139711, -STORE, 140175931998208, 140175985139711, -STORE, 140175931990016, 140175985139711, -STORE, 140175931957248, 140175985139711, -STORE, 140175931949056, 140175985139711, -STORE, 140175931932672, 140175985139711, -STORE, 140175931924480, 140175985139711, -STORE, 140175931908096, 140175985139711, -STORE, 140175931899904, 140175985139711, -STORE, 140175931883520, 140175985139711, -STORE, 140175931875328, 140175985139711, -STORE, 140175931842560, 140175985139711, -STORE, 140175931834368, 140175985139711, -STORE, 140175931817984, 140175985139711, -STORE, 140175931809792, 140175985139711, -STORE, 140175931793408, 140175985139711, -STORE, 140175931785216, 140175985139711, -STORE, 140175931768832, 140175985139711, -STORE, 140175931760640, 140175985139711, -STORE, 140175931727872, 140175985139711, -STORE, 140175931719680, 140175985139711, -STORE, 140175931703296, 140175985139711, -STORE, 140175931695104, 140175985139711, -STORE, 140175931678720, 140175985139711, -STORE, 140175931670528, 140175985139711, -STORE, 140175931654144, 140175985139711, -STORE, 140175931645952, 140175985139711, -STORE, 140175931613184, 140175985139711, -STORE, 140175931604992, 140175985139711, -STORE, 140175931588608, 140175985139711, -STORE, 140175931580416, 140175985139711, -STORE, 140175931564032, 140175985139711, -STORE, 140175931555840, 140175985139711, -STORE, 140175931539456, 140175985139711, -STORE, 140175931531264, 140175985139711, -STORE, 140175931498496, 140175985139711, -STORE, 140175931490304, 140175985139711, -STORE, 140175931473920, 140175985139711, -STORE, 140175931465728, 140175985139711, -STORE, 140175931449344, 140175985139711, -STORE, 140175931441152, 140175985139711, -STORE, 140175931424768, 140175985139711, -STORE, 140175931416576, 140175985139711, -STORE, 140175931383808, 140175985139711, -STORE, 140175931375616, 140175985139711, -STORE, 140175931359232, 140175985139711, -STORE, 140175931351040, 140175985139711, -STORE, 140175931334656, 140175985139711, -STORE, 140175931326464, 140175985139711, -STORE, 140175931310080, 140175985139711, -STORE, 140175931301888, 140175985139711, -STORE, 140175931269120, 140175985139711, -STORE, 140175931260928, 140175985139711, -STORE, 140175931244544, 140175985139711, -STORE, 140175931236352, 140175985139711, -STORE, 140175931219968, 140175985139711, -STORE, 140175931211776, 140175985139711, -STORE, 140175931195392, 140175985139711, -STORE, 140175931187200, 140175985139711, -STORE, 140175931154432, 140175985139711, -STORE, 140175931146240, 140175985139711, -STORE, 140175931129856, 140175985139711, -STORE, 140175931121664, 140175985139711, -STORE, 140175931105280, 140175985139711, -STORE, 140175931097088, 140175985139711, -STORE, 140175931080704, 140175985139711, -STORE, 140175931072512, 140175985139711, -STORE, 140175931039744, 140175985139711, -STORE, 140175931031552, 140175985139711, -STORE, 140175931015168, 140175985139711, -STORE, 140175931006976, 140175985139711, -STORE, 140175930990592, 140175985139711, -STORE, 140175930982400, 140175985139711, -STORE, 140175930966016, 140175985139711, -STORE, 140175930957824, 140175985139711, -STORE, 140175930925056, 140175985139711, -STORE, 140175930916864, 140175985139711, -STORE, 140175930900480, 140175985139711, -STORE, 140175930892288, 140175985139711, -STORE, 140175930875904, 140175985139711, -STORE, 140175930867712, 140175985139711, -STORE, 140175930851328, 140175985139711, -STORE, 140175930843136, 140175985139711, -STORE, 140175930810368, 140175985139711, -STORE, 140175930802176, 140175985139711, -STORE, 140175930785792, 140175985139711, -STORE, 140175930777600, 140175985139711, -STORE, 140175930761216, 140175985139711, -STORE, 140175930753024, 140175985139711, -STORE, 140175930736640, 140175985139711, -STORE, 140175930728448, 140175985139711, -STORE, 140175930695680, 140175985139711, -STORE, 140175930687488, 140175985139711, -STORE, 140175930671104, 140175985139711, -STORE, 140175930662912, 140175985139711, -STORE, 140175930646528, 140175985139711, -STORE, 140175930638336, 140175985139711, -STORE, 140175930621952, 140175985139711, -STORE, 140175930613760, 140175985139711, -STORE, 140175930580992, 140175985139711, -STORE, 140175930572800, 140175985139711, -STORE, 140175930556416, 140175985139711, -STORE, 140175930548224, 140175985139711, -STORE, 140175930531840, 140175985139711, -STORE, 140175930523648, 140175985139711, -STORE, 140175930507264, 140175985139711, -STORE, 140175928410112, 140175985139711, -STORE, 140175928401920, 140175985139711, -STORE, 140175928369152, 140175985139711, -STORE, 140175928360960, 140175985139711, -STORE, 140175928344576, 140175985139711, -STORE, 140175928336384, 140175985139711, -STORE, 140175928320000, 140175985139711, -STORE, 140175928311808, 140175985139711, -STORE, 140175928295424, 140175985139711, -STORE, 140175927242752, 140175985139711, -SNULL, 140175956627455, 140175985139711, -STORE, 140175927242752, 140175956627455, -STORE, 140175956627456, 140175985139711, - }; - unsigned long set24[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140735281639424, 140737488351231, -SNULL, 140735281643519, 140737488351231, -STORE, 140735281639424, 140735281643519, -STORE, 140735281508352, 140735281643519, -STORE, 94717834911744, 94717834928127, -SNULL, 94717834915839, 94717834928127, -STORE, 94717834911744, 94717834915839, -STORE, 94717834915840, 94717834928127, -ERASE, 94717834915840, 94717834928127, -STORE, 94717834919936, 94717834928127, -STORE, 140428246065152, 140428248317951, -SNULL, 140428246208511, 140428248317951, -STORE, 140428246065152, 140428246208511, -STORE, 140428246208512, 140428248317951, -ERASE, 140428246208512, 140428248317951, -STORE, 140428248305664, 140428248313855, -STORE, 140428248313856, 140428248317951, -STORE, 140735281811456, 140735281815551, -STORE, 140735281799168, 140735281811455, -STORE, 140428248297472, 140428248305663, -STORE, 140428243841024, 140428246065151, -SNULL, 140428245491711, 140428246065151, -STORE, 140428243841024, 140428245491711, -STORE, 140428245491712, 140428246065151, -SNULL, 140428245491712, 140428246061055, -STORE, 140428246061056, 140428246065151, -STORE, 140428245491712, 140428246061055, -ERASE, 140428245491712, 140428246061055, -STORE, 140428245491712, 140428246061055, -ERASE, 140428246061056, 140428246065151, -STORE, 140428246061056, 140428246065151, -STORE, 140428248268800, 140428248297471, -STORE, 140428241625088, 140428243841023, -SNULL, 140428241625088, 140428241723391, -STORE, 140428241723392, 140428243841023, -STORE, 140428241625088, 140428241723391, -SNULL, 140428243816447, 140428243841023, -STORE, 140428241723392, 140428243816447, -STORE, 140428243816448, 140428243841023, -SNULL, 140428243816448, 140428243824639, -STORE, 140428243824640, 140428243841023, -STORE, 140428243816448, 140428243824639, -ERASE, 140428243816448, 140428243824639, -STORE, 140428243816448, 140428243824639, -ERASE, 140428243824640, 140428243841023, -STORE, 140428243824640, 140428243841023, -STORE, 140428237828096, 140428241625087, -SNULL, 140428237828096, 140428239486975, -STORE, 140428239486976, 140428241625087, -STORE, 140428237828096, 140428239486975, -SNULL, 140428241584127, 140428241625087, -STORE, 140428239486976, 140428241584127, -STORE, 140428241584128, 140428241625087, -SNULL, 140428241584128, 140428241608703, -STORE, 140428241608704, 140428241625087, -STORE, 140428241584128, 140428241608703, -ERASE, 140428241584128, 140428241608703, -STORE, 140428241584128, 140428241608703, -ERASE, 140428241608704, 140428241625087, -STORE, 140428241608704, 140428241625087, -STORE, 140428235567104, 140428237828095, -SNULL, 140428235567104, 140428235718655, -STORE, 140428235718656, 140428237828095, -STORE, 140428235567104, 140428235718655, -SNULL, 140428237811711, 140428237828095, -STORE, 140428235718656, 140428237811711, -STORE, 140428237811712, 140428237828095, -SNULL, 140428237811712, 140428237819903, -STORE, 140428237819904, 140428237828095, -STORE, 140428237811712, 140428237819903, -ERASE, 140428237811712, 140428237819903, -STORE, 140428237811712, 140428237819903, -ERASE, 140428237819904, 140428237828095, -STORE, 140428237819904, 140428237828095, -STORE, 140428233445376, 140428235567103, -SNULL, 140428233445376, 140428233461759, -STORE, 140428233461760, 140428235567103, -STORE, 140428233445376, 140428233461759, -SNULL, 140428235558911, 140428235567103, -STORE, 140428233461760, 140428235558911, -STORE, 140428235558912, 140428235567103, -ERASE, 140428235558912, 140428235567103, -STORE, 140428235558912, 140428235567103, -STORE, 140428231315456, 140428233445375, -SNULL, 140428231315456, 140428231344127, -STORE, 140428231344128, 140428233445375, -STORE, 140428231315456, 140428231344127, -SNULL, 140428233437183, 140428233445375, -STORE, 140428231344128, 140428233437183, -STORE, 140428233437184, 140428233445375, -ERASE, 140428233437184, 140428233445375, -STORE, 140428233437184, 140428233445375, -STORE, 140428248260608, 140428248268799, -STORE, 140428229062656, 140428231315455, -SNULL, 140428229062656, 140428229214207, -STORE, 140428229214208, 140428231315455, -STORE, 140428229062656, 140428229214207, -SNULL, 140428231307263, 140428231315455, -STORE, 140428229214208, 140428231307263, -STORE, 140428231307264, 140428231315455, -ERASE, 140428231307264, 140428231315455, -STORE, 140428231307264, 140428231315455, -STORE, 140428226891776, 140428229062655, -SNULL, 140428226891776, 140428226961407, -STORE, 140428226961408, 140428229062655, -STORE, 140428226891776, 140428226961407, -SNULL, 140428229054463, 140428229062655, -STORE, 140428226961408, 140428229054463, -STORE, 140428229054464, 140428229062655, -ERASE, 140428229054464, 140428229062655, -STORE, 140428229054464, 140428229062655, -STORE, 140428223680512, 140428226891775, -SNULL, 140428223680512, 140428224757759, -STORE, 140428224757760, 140428226891775, -STORE, 140428223680512, 140428224757759, -SNULL, 140428226854911, 140428226891775, -STORE, 140428224757760, 140428226854911, -STORE, 140428226854912, 140428226891775, -ERASE, 140428226854912, 140428226891775, -STORE, 140428226854912, 140428226891775, -STORE, 140428221546496, 140428223680511, -SNULL, 140428221546496, 140428221575167, -STORE, 140428221575168, 140428223680511, -STORE, 140428221546496, 140428221575167, -SNULL, 140428223672319, 140428223680511, -STORE, 140428221575168, 140428223672319, -STORE, 140428223672320, 140428223680511, -ERASE, 140428223672320, 140428223680511, -STORE, 140428223672320, 140428223680511, -STORE, 140428219236352, 140428221546495, -SNULL, 140428219236352, 140428219441151, -STORE, 140428219441152, 140428221546495, -STORE, 140428219236352, 140428219441151, -SNULL, 140428221538303, 140428221546495, -STORE, 140428219441152, 140428221538303, -STORE, 140428221538304, 140428221546495, -ERASE, 140428221538304, 140428221546495, -STORE, 140428221538304, 140428221546495, -STORE, 140428216852480, 140428219236351, -SNULL, 140428216852480, 140428217044991, -STORE, 140428217044992, 140428219236351, -STORE, 140428216852480, 140428217044991, -SNULL, 140428219138047, 140428219236351, -STORE, 140428217044992, 140428219138047, -STORE, 140428219138048, 140428219236351, -ERASE, 140428219138048, 140428219236351, -STORE, 140428219138048, 140428219236351, -STORE, 140428248252416, 140428248268799, -STORE, 140428214284288, 140428216852479, -SNULL, 140428214284288, 140428214751231, -STORE, 140428214751232, 140428216852479, -STORE, 140428214284288, 140428214751231, -SNULL, 140428216844287, 140428216852479, -STORE, 140428214751232, 140428216844287, -STORE, 140428216844288, 140428216852479, -ERASE, 140428216844288, 140428216852479, -STORE, 140428216844288, 140428216852479, -STORE, 140428212170752, 140428214284287, -SNULL, 140428212170752, 140428212183039, -STORE, 140428212183040, 140428214284287, -STORE, 140428212170752, 140428212183039, -SNULL, 140428214276095, 140428214284287, -STORE, 140428212183040, 140428214276095, -STORE, 140428214276096, 140428214284287, -ERASE, 140428214276096, 140428214284287, -STORE, 140428214276096, 140428214284287, -STORE, 140428209991680, 140428212170751, -SNULL, 140428209991680, 140428210069503, -STORE, 140428210069504, 140428212170751, -STORE, 140428209991680, 140428210069503, -SNULL, 140428212162559, 140428212170751, -STORE, 140428210069504, 140428212162559, -STORE, 140428212162560, 140428212170751, -ERASE, 140428212162560, 140428212170751, -STORE, 140428212162560, 140428212170751, -STORE, 140428207874048, 140428209991679, -SNULL, 140428207874048, 140428207890431, -STORE, 140428207890432, 140428209991679, -STORE, 140428207874048, 140428207890431, -SNULL, 140428209983487, 140428209991679, -STORE, 140428207890432, 140428209983487, -STORE, 140428209983488, 140428209991679, -ERASE, 140428209983488, 140428209991679, -STORE, 140428209983488, 140428209991679, -STORE, 140428248244224, 140428248268799, -STORE, 140428248231936, 140428248268799, -SNULL, 140428241600511, 140428241608703, -STORE, 140428241584128, 140428241600511, -STORE, 140428241600512, 140428241608703, -SNULL, 140428209987583, 140428209991679, -STORE, 140428209983488, 140428209987583, -STORE, 140428209987584, 140428209991679, -SNULL, 140428212166655, 140428212170751, -STORE, 140428212162560, 140428212166655, -STORE, 140428212166656, 140428212170751, -SNULL, 140428214280191, 140428214284287, -STORE, 140428214276096, 140428214280191, -STORE, 140428214280192, 140428214284287, -SNULL, 140428243820543, 140428243824639, -STORE, 140428243816448, 140428243820543, -STORE, 140428243820544, 140428243824639, -SNULL, 140428216848383, 140428216852479, -STORE, 140428216844288, 140428216848383, -STORE, 140428216848384, 140428216852479, -SNULL, 140428219232255, 140428219236351, -STORE, 140428219138048, 140428219232255, -STORE, 140428219232256, 140428219236351, -SNULL, 140428221542399, 140428221546495, -STORE, 140428221538304, 140428221542399, -STORE, 140428221542400, 140428221546495, -SNULL, 140428223676415, 140428223680511, -STORE, 140428223672320, 140428223676415, -STORE, 140428223676416, 140428223680511, -SNULL, 140428226863103, 140428226891775, -STORE, 140428226854912, 140428226863103, -STORE, 140428226863104, 140428226891775, -SNULL, 140428229058559, 140428229062655, -STORE, 140428229054464, 140428229058559, -STORE, 140428229058560, 140428229062655, -SNULL, 140428231311359, 140428231315455, -STORE, 140428231307264, 140428231311359, -STORE, 140428231311360, 140428231315455, -SNULL, 140428233441279, 140428233445375, -STORE, 140428233437184, 140428233441279, -STORE, 140428233441280, 140428233445375, -SNULL, 140428235563007, 140428235567103, -STORE, 140428235558912, 140428235563007, -STORE, 140428235563008, 140428235567103, -SNULL, 140428237815807, 140428237819903, -STORE, 140428237811712, 140428237815807, -STORE, 140428237815808, 140428237819903, -SNULL, 140428246056959, 140428246061055, -STORE, 140428245491712, 140428246056959, -STORE, 140428246056960, 140428246061055, -SNULL, 94717834924031, 94717834928127, -STORE, 94717834919936, 94717834924031, -STORE, 94717834924032, 94717834928127, -SNULL, 140428248309759, 140428248313855, -STORE, 140428248305664, 140428248309759, -STORE, 140428248309760, 140428248313855, -ERASE, 140428248268800, 140428248297471, -STORE, 94717843058688, 94717843193855, -STORE, 94749677137920, 94749677559807, -STORE, 94749677563904, 94749677604863, -STORE, 94749677604864, 94749677608959, -STORE, 94749710970880, 94749711241215, -STORE, 140490884894720, 140490884935679, -STORE, 140490884935680, 140490887032831, -STORE, 140490887032832, 140490887036927, -STORE, 140490887036928, 140490887041023, -STORE, 140490887041024, 140490887065599, -STORE, 140490887065600, 140490887110655, -STORE, 140490887110656, 140490889203711, -STORE, 140490889203712, 140490889207807, -STORE, 140490889207808, 140490889211903, -STORE, 140490889211904, 140490889293823, -STORE, 140490889293824, 140490891390975, -STORE, 140490891390976, 140490891395071, -STORE, 140490891395072, 140490891399167, -STORE, 140490891399168, 140490891407359, -STORE, 140490891407360, 140490891436031, -STORE, 140490891436032, 140490893529087, -STORE, 140490893529088, 140490893533183, -STORE, 140490893533184, 140490893537279, -STORE, 140490893537280, 140490901979135, -STORE, 140490901979136, 140490901991423, -STORE, 140490901991424, 140490904084479, -STORE, 140490904084480, 140490904088575, -STORE, 140490904088576, 140490904092671, -STORE, 140490904092672, 140490904559615, -STORE, 140490904559616, 140490906652671, -STORE, 140490906652672, 140490906656767, -STORE, 140490906656768, 140490906660863, -STORE, 140490906660864, 140490906677247, -STORE, 140490906677248, 140490908770303, -STORE, 140490908770304, 140490908774399, -STORE, 140490908774400, 140490908778495, -STORE, 140490908778496, 140490908794879, -STORE, 140490908794880, 140490910887935, -STORE, 140490910887936, 140490910892031, -STORE, 140490910892032, 140490910896127, -STORE, 140490910896128, 140490912555007, -STORE, 140490912555008, 140490914652159, -STORE, 140490914652160, 140490914668543, -STORE, 140490914668544, 140490914676735, -STORE, 140490914676736, 140490914693119, -STORE, 140490914693120, 140490914791423, -STORE, 140490914791424, 140490916884479, -STORE, 140490916884480, 140490916888575, -STORE, 140490916888576, 140490916892671, -STORE, 140490916892672, 140490916909055, -STORE, 140490916909056, 140490916937727, -STORE, 140490916937728, 140490919030783, -STORE, 140490919030784, 140490919034879, -STORE, 140490919034880, 140490919038975, -STORE, 140490919038976, 140490919190527, -STORE, 140490919190528, 140490921283583, -STORE, 140490921283584, 140490921287679, -STORE, 140490921287680, 140490921291775, -STORE, 140490921291776, 140490921299967, -STORE, 140490921299968, 140490921390079, -STORE, 140490921390080, 140490923483135, -STORE, 140490923483136, 140490923487231, -STORE, 140490923487232, 140490923491327, -STORE, 140490923491328, 140490923757567, -STORE, 140490923757568, 140490925850623, -STORE, 140490925850624, 140490925867007, -STORE, 140490925867008, 140490925871103, -STORE, 140490925871104, 140490925875199, -STORE, 140490925875200, 140490925903871, -STORE, 140490925903872, 140490928001023, -STORE, 140490928001024, 140490928005119, -STORE, 140490928005120, 140490928009215, -STORE, 140490928009216, 140490928152575, -STORE, 140490930184192, 140490930221055, -STORE, 140490930221056, 140490930237439, -STORE, 140490930237440, 140490930241535, -STORE, 140490930241536, 140490930245631, -STORE, 140490930245632, 140490930249727, -STORE, 140490930249728, 140490930253823, -STORE, 140490930253824, 140490930257919, -STORE, 140490930257920, 140490930262015, -STORE, 140724611694592, 140724611829759, -STORE, 140724612427776, 140724612440063, -STORE, 140724612440064, 140724612444159, -STORE, 94103163662336, 94103163772927, -STORE, 94103165865984, 94103165874175, -STORE, 94103165874176, 94103165878271, -STORE, 94103165878272, 94103165886463, -STORE, 94103182548992, 94103182684159, -STORE, 140092694708224, 140092696367103, -STORE, 140092696367104, 140092698464255, -STORE, 140092698464256, 140092698480639, -STORE, 140092698480640, 140092698488831, -STORE, 140092698488832, 140092698505215, -STORE, 140092698505216, 140092698648575, -STORE, 140092700708864, 140092700717055, -STORE, 140092700745728, 140092700749823, -STORE, 140092700749824, 140092700753919, -STORE, 140092700753920, 140092700758015, -STORE, 140736800911360, 140736801046527, -STORE, 140736802308096, 140736802320383, -STORE, 140736802320384, 140736802324479, -STORE, 93948802064384, 93948802174975, -STORE, 93948804268032, 93948804276223, -STORE, 93948804276224, 93948804280319, -STORE, 93948804280320, 93948804288511, -STORE, 93948806266880, 93948806402047, -STORE, 140222999113728, 140223000772607, -STORE, 140223000772608, 140223002869759, -STORE, 140223002869760, 140223002886143, -STORE, 140223002886144, 140223002894335, -STORE, 140223002894336, 140223002910719, -STORE, 140223002910720, 140223003054079, -STORE, 140223005114368, 140223005122559, -STORE, 140223005151232, 140223005155327, -STORE, 140223005155328, 140223005159423, -STORE, 140223005159424, 140223005163519, -STORE, 140720877506560, 140720877641727, -STORE, 140720878231552, 140720878243839, -STORE, 140720878243840, 140720878247935, -STORE, 140737488347136, 140737488351231, -STORE, 140733232087040, 140737488351231, -SNULL, 140733232091135, 140737488351231, -STORE, 140733232087040, 140733232091135, -STORE, 140733231955968, 140733232091135, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 140161681321984, 140161683574783, -SNULL, 140161681465343, 140161683574783, -STORE, 140161681321984, 140161681465343, -STORE, 140161681465344, 140161683574783, -ERASE, 140161681465344, 140161683574783, -STORE, 140161683562496, 140161683570687, -STORE, 140161683570688, 140161683574783, -STORE, 140733232214016, 140733232218111, -STORE, 140733232201728, 140733232214015, -STORE, 140161683533824, 140161683562495, -STORE, 140161683525632, 140161683533823, -STORE, 140161678159872, 140161681321983, -SNULL, 140161678159872, 140161679220735, -STORE, 140161679220736, 140161681321983, -STORE, 140161678159872, 140161679220735, -SNULL, 140161681313791, 140161681321983, -STORE, 140161679220736, 140161681313791, -STORE, 140161681313792, 140161681321983, -ERASE, 140161681313792, 140161681321983, -STORE, 140161681313792, 140161681321983, -STORE, 140161674362880, 140161678159871, -SNULL, 140161674362880, 140161676021759, -STORE, 140161676021760, 140161678159871, -STORE, 140161674362880, 140161676021759, -SNULL, 140161678118911, 140161678159871, -STORE, 140161676021760, 140161678118911, -STORE, 140161678118912, 140161678159871, -SNULL, 140161678118912, 140161678143487, -STORE, 140161678143488, 140161678159871, -STORE, 140161678118912, 140161678143487, -ERASE, 140161678118912, 140161678143487, -STORE, 140161678118912, 140161678143487, -ERASE, 140161678143488, 140161678159871, -STORE, 140161678143488, 140161678159871, -STORE, 140161683513344, 140161683533823, -SNULL, 140161678135295, 140161678143487, -STORE, 140161678118912, 140161678135295, -STORE, 140161678135296, 140161678143487, -SNULL, 140161681317887, 140161681321983, -STORE, 140161681313792, 140161681317887, -STORE, 140161681317888, 140161681321983, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 140161683566591, 140161683570687, -STORE, 140161683562496, 140161683566591, -STORE, 140161683566592, 140161683570687, -ERASE, 140161683533824, 140161683562495, -STORE, 25477120, 25612287, -STORE, 25477120, 25759743, -STORE, 140161681829888, 140161683513343, -STORE, 25477120, 25915391, -STORE, 25477120, 26054655, -SNULL, 25800703, 26054655, -STORE, 25477120, 25800703, -STORE, 25800704, 26054655, -ERASE, 25800704, 26054655, -STORE, 140737488347136, 140737488351231, -STORE, 140723218452480, 140737488351231, -SNULL, 140723218456575, 140737488351231, -STORE, 140723218452480, 140723218456575, -STORE, 140723218321408, 140723218456575, -STORE, 4194304, 26279935, -STORE, 28372992, 28454911, -STORE, 28454912, 29806591, -STORE, 140398872264704, 140398874517503, -SNULL, 140398872408063, 140398874517503, -STORE, 140398872264704, 140398872408063, -STORE, 140398872408064, 140398874517503, -ERASE, 140398872408064, 140398874517503, -STORE, 140398874505216, 140398874513407, -STORE, 140398874513408, 140398874517503, -STORE, 140723219247104, 140723219251199, -STORE, 140723219234816, 140723219247103, -STORE, 140398874476544, 140398874505215, -STORE, 140398874468352, 140398874476543, -STORE, 140398868430848, 140398872264703, -SNULL, 140398868430848, 140398870138879, -STORE, 140398870138880, 140398872264703, -STORE, 140398868430848, 140398870138879, -SNULL, 140398872231935, 140398872264703, -STORE, 140398870138880, 140398872231935, -STORE, 140398872231936, 140398872264703, -ERASE, 140398872231936, 140398872264703, -STORE, 140398872231936, 140398872264703, -STORE, 140398866235392, 140398868430847, -SNULL, 140398866235392, 140398866329599, -STORE, 140398866329600, 140398868430847, -STORE, 140398866235392, 140398866329599, -SNULL, 140398868422655, 140398868430847, -STORE, 140398866329600, 140398868422655, -STORE, 140398868422656, 140398868430847, -ERASE, 140398868422656, 140398868430847, -STORE, 140398868422656, 140398868430847, -STORE, 140398863716352, 140398866235391, -SNULL, 140398863716352, 140398864130047, -STORE, 140398864130048, 140398866235391, -STORE, 140398863716352, 140398864130047, -SNULL, 140398866223103, 140398866235391, -STORE, 140398864130048, 140398866223103, -STORE, 140398866223104, 140398866235391, -ERASE, 140398866223104, 140398866235391, -STORE, 140398866223104, 140398866235391, -STORE, 140398861082624, 140398863716351, -SNULL, 140398861082624, 140398861611007, -STORE, 140398861611008, 140398863716351, -STORE, 140398861082624, 140398861611007, -SNULL, 140398863708159, 140398863716351, -STORE, 140398861611008, 140398863708159, -STORE, 140398863708160, 140398863716351, -ERASE, 140398863708160, 140398863716351, -STORE, 140398863708160, 140398863716351, -STORE, 140398858969088, 140398861082623, -SNULL, 140398858969088, 140398858981375, -STORE, 140398858981376, 140398861082623, -STORE, 140398858969088, 140398858981375, -SNULL, 140398861074431, 140398861082623, -STORE, 140398858981376, 140398861074431, -STORE, 140398861074432, 140398861082623, -ERASE, 140398861074432, 140398861082623, -STORE, 140398861074432, 140398861082623, -STORE, 140398856765440, 140398858969087, -SNULL, 140398856765440, 140398856867839, -STORE, 140398856867840, 140398858969087, -STORE, 140398856765440, 140398856867839, -SNULL, 140398858960895, 140398858969087, -STORE, 140398856867840, 140398858960895, -STORE, 140398858960896, 140398858969087, -ERASE, 140398858960896, 140398858969087, -STORE, 140398858960896, 140398858969087, -STORE, 140398874460160, 140398874476543, -STORE, 140398853603328, 140398856765439, -SNULL, 140398853603328, 140398854664191, -STORE, 140398854664192, 140398856765439, -STORE, 140398853603328, 140398854664191, -SNULL, 140398856757247, 140398856765439, -STORE, 140398854664192, 140398856757247, -STORE, 140398856757248, 140398856765439, -ERASE, 140398856757248, 140398856765439, -STORE, 140398856757248, 140398856765439, -STORE, 140398849806336, 140398853603327, -SNULL, 140398849806336, 140398851465215, -STORE, 140398851465216, 140398853603327, -STORE, 140398849806336, 140398851465215, -SNULL, 140398853562367, 140398853603327, -STORE, 140398851465216, 140398853562367, -STORE, 140398853562368, 140398853603327, -SNULL, 140398853562368, 140398853586943, -STORE, 140398853586944, 140398853603327, -STORE, 140398853562368, 140398853586943, -ERASE, 140398853562368, 140398853586943, -STORE, 140398853562368, 140398853586943, -ERASE, 140398853586944, 140398853603327, -STORE, 140398853586944, 140398853603327, -STORE, 140398874447872, 140398874476543, -SNULL, 140398853578751, 140398853586943, -STORE, 140398853562368, 140398853578751, -STORE, 140398853578752, 140398853586943, -SNULL, 140398856761343, 140398856765439, -STORE, 140398856757248, 140398856761343, -STORE, 140398856761344, 140398856765439, -SNULL, 140398858964991, 140398858969087, -STORE, 140398858960896, 140398858964991, -STORE, 140398858964992, 140398858969087, -SNULL, 140398861078527, 140398861082623, -STORE, 140398861074432, 140398861078527, -STORE, 140398861078528, 140398861082623, -SNULL, 140398863712255, 140398863716351, -STORE, 140398863708160, 140398863712255, -STORE, 140398863712256, 140398863716351, -SNULL, 140398866231295, 140398866235391, -STORE, 140398866223104, 140398866231295, -STORE, 140398866231296, 140398866235391, -SNULL, 140398868426751, 140398868430847, -STORE, 140398868422656, 140398868426751, -STORE, 140398868426752, 140398868430847, -SNULL, 140398872236031, 140398872264703, -STORE, 140398872231936, 140398872236031, -STORE, 140398872236032, 140398872264703, -SNULL, 28405759, 28454911, -STORE, 28372992, 28405759, -STORE, 28405760, 28454911, -SNULL, 140398874509311, 140398874513407, -STORE, 140398874505216, 140398874509311, -STORE, 140398874509312, 140398874513407, -ERASE, 140398874476544, 140398874505215, -STORE, 43278336, 43413503, -STORE, 140398872764416, 140398874447871, -STORE, 140398874501120, 140398874505215, -STORE, 140398872629248, 140398872764415, -STORE, 43278336, 43556863, -STORE, 140398847709184, 140398849806335, -STORE, 140398874492928, 140398874505215, -STORE, 140398874484736, 140398874505215, -STORE, 140398874447872, 140398874484735, -STORE, 140398872612864, 140398872764415, -STORE, 43278336, 43692031, -STORE, 43278336, 43880447, -STORE, 140398872604672, 140398872764415, -STORE, 140398872596480, 140398872764415, -STORE, 43278336, 44044287, -STORE, 140398872580096, 140398872764415, -STORE, 140737488347136, 140737488351231, -STORE, 140734403092480, 140737488351231, -SNULL, 140734403096575, 140737488351231, -STORE, 140734403092480, 140734403096575, -STORE, 140734402961408, 140734403096575, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 140240662380544, 140240664633343, -SNULL, 140240662523903, 140240664633343, -STORE, 140240662380544, 140240662523903, -STORE, 140240662523904, 140240664633343, -ERASE, 140240662523904, 140240664633343, -STORE, 140240664621056, 140240664629247, -STORE, 140240664629248, 140240664633343, -STORE, 140734403145728, 140734403149823, -STORE, 140734403133440, 140734403145727, -STORE, 140240664592384, 140240664621055, -STORE, 140240664584192, 140240664592383, -STORE, 140240659218432, 140240662380543, -SNULL, 140240659218432, 140240660279295, -STORE, 140240660279296, 140240662380543, -STORE, 140240659218432, 140240660279295, -SNULL, 140240662372351, 140240662380543, -STORE, 140240660279296, 140240662372351, -STORE, 140240662372352, 140240662380543, -ERASE, 140240662372352, 140240662380543, -STORE, 140240662372352, 140240662380543, -STORE, 140240655421440, 140240659218431, -SNULL, 140240655421440, 140240657080319, -STORE, 140240657080320, 140240659218431, -STORE, 140240655421440, 140240657080319, -SNULL, 140240659177471, 140240659218431, -STORE, 140240657080320, 140240659177471, -STORE, 140240659177472, 140240659218431, -SNULL, 140240659177472, 140240659202047, -STORE, 140240659202048, 140240659218431, -STORE, 140240659177472, 140240659202047, -ERASE, 140240659177472, 140240659202047, -STORE, 140240659177472, 140240659202047, -ERASE, 140240659202048, 140240659218431, -STORE, 140240659202048, 140240659218431, -STORE, 140240664571904, 140240664592383, -SNULL, 140240659193855, 140240659202047, -STORE, 140240659177472, 140240659193855, -STORE, 140240659193856, 140240659202047, -SNULL, 140240662376447, 140240662380543, -STORE, 140240662372352, 140240662376447, -STORE, 140240662376448, 140240662380543, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 140240664625151, 140240664629247, -STORE, 140240664621056, 140240664625151, -STORE, 140240664625152, 140240664629247, -ERASE, 140240664592384, 140240664621055, -STORE, 30646272, 30781439, -STORE, 30646272, 30928895, -STORE, 140240662888448, 140240664571903, -STORE, 94256659468288, 94256659578879, -STORE, 94256661671936, 94256661680127, -STORE, 94256661680128, 94256661684223, -STORE, 94256661684224, 94256661692415, -STORE, 94256687980544, 94256688115711, -STORE, 139801712504832, 139801714163711, -STORE, 139801714163712, 139801716260863, -STORE, 139801716260864, 139801716277247, -STORE, 139801716277248, 139801716285439, -STORE, 139801716285440, 139801716301823, -STORE, 139801716301824, 139801716445183, -STORE, 139801718505472, 139801718513663, -STORE, 139801718542336, 139801718546431, -STORE, 139801718546432, 139801718550527, -STORE, 139801718550528, 139801718554623, -STORE, 140721575538688, 140721575673855, -STORE, 140721577013248, 140721577025535, -STORE, 140721577025536, 140721577029631, -STORE, 140737488347136, 140737488351231, -STORE, 140729259393024, 140737488351231, -SNULL, 140729259397119, 140737488351231, -STORE, 140729259393024, 140729259397119, -STORE, 140729259261952, 140729259397119, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 139682376638464, 139682378891263, -SNULL, 139682376781823, 139682378891263, -STORE, 139682376638464, 139682376781823, -STORE, 139682376781824, 139682378891263, -ERASE, 139682376781824, 139682378891263, -STORE, 139682378878976, 139682378887167, -STORE, 139682378887168, 139682378891263, -STORE, 140729260462080, 140729260466175, -STORE, 140729260449792, 140729260462079, -STORE, 139682378850304, 139682378878975, -STORE, 139682378842112, 139682378850303, -STORE, 139682373476352, 139682376638463, -SNULL, 139682373476352, 139682374537215, -STORE, 139682374537216, 139682376638463, -STORE, 139682373476352, 139682374537215, -SNULL, 139682376630271, 139682376638463, -STORE, 139682374537216, 139682376630271, -STORE, 139682376630272, 139682376638463, -ERASE, 139682376630272, 139682376638463, -STORE, 139682376630272, 139682376638463, -STORE, 139682369679360, 139682373476351, -SNULL, 139682369679360, 139682371338239, -STORE, 139682371338240, 139682373476351, -STORE, 139682369679360, 139682371338239, -SNULL, 139682373435391, 139682373476351, -STORE, 139682371338240, 139682373435391, -STORE, 139682373435392, 139682373476351, -SNULL, 139682373435392, 139682373459967, -STORE, 139682373459968, 139682373476351, -STORE, 139682373435392, 139682373459967, -ERASE, 139682373435392, 139682373459967, -STORE, 139682373435392, 139682373459967, -ERASE, 139682373459968, 139682373476351, -STORE, 139682373459968, 139682373476351, -STORE, 139682378829824, 139682378850303, -SNULL, 139682373451775, 139682373459967, -STORE, 139682373435392, 139682373451775, -STORE, 139682373451776, 139682373459967, -SNULL, 139682376634367, 139682376638463, -STORE, 139682376630272, 139682376634367, -STORE, 139682376634368, 139682376638463, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 139682378883071, 139682378887167, -STORE, 139682378878976, 139682378883071, -STORE, 139682378883072, 139682378887167, -ERASE, 139682378850304, 139682378878975, -STORE, 10022912, 10158079, -STORE, 10022912, 10305535, -STORE, 139682377146368, 139682378829823, -STORE, 140737488347136, 140737488351231, -STORE, 140731831926784, 140737488351231, -SNULL, 140731831930879, 140737488351231, -STORE, 140731831926784, 140731831930879, -STORE, 140731831795712, 140731831930879, -STORE, 94615305261056, 94615307485183, -SNULL, 94615305371647, 94615307485183, -STORE, 94615305261056, 94615305371647, -STORE, 94615305371648, 94615307485183, -ERASE, 94615305371648, 94615307485183, -STORE, 94615307464704, 94615307476991, -STORE, 94615307476992, 94615307485183, -STORE, 140163912994816, 140163915247615, -SNULL, 140163913138175, 140163915247615, -STORE, 140163912994816, 140163913138175, -STORE, 140163913138176, 140163915247615, -ERASE, 140163913138176, 140163915247615, -STORE, 140163915235328, 140163915243519, -STORE, 140163915243520, 140163915247615, -STORE, 140731832217600, 140731832221695, -STORE, 140731832205312, 140731832217599, -STORE, 140163915206656, 140163915235327, -STORE, 140163915198464, 140163915206655, -STORE, 140163909197824, 140163912994815, -SNULL, 140163909197824, 140163910856703, -STORE, 140163910856704, 140163912994815, -STORE, 140163909197824, 140163910856703, -SNULL, 140163912953855, 140163912994815, -STORE, 140163910856704, 140163912953855, -STORE, 140163912953856, 140163912994815, -SNULL, 140163912953856, 140163912978431, -STORE, 140163912978432, 140163912994815, -STORE, 140163912953856, 140163912978431, -ERASE, 140163912953856, 140163912978431, -STORE, 140163912953856, 140163912978431, -ERASE, 140163912978432, 140163912994815, -STORE, 140163912978432, 140163912994815, -SNULL, 140163912970239, 140163912978431, -STORE, 140163912953856, 140163912970239, -STORE, 140163912970240, 140163912978431, -SNULL, 94615307472895, 94615307476991, -STORE, 94615307464704, 94615307472895, -STORE, 94615307472896, 94615307476991, -SNULL, 140163915239423, 140163915243519, -STORE, 140163915235328, 140163915239423, -STORE, 140163915239424, 140163915243519, -ERASE, 140163915206656, 140163915235327, -STORE, 94615330672640, 94615330807807, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140725254479872, 140737488351231, -SNULL, 140725254488063, 140737488351231, -STORE, 140725254479872, 140725254488063, -STORE, 140725254348800, 140725254488063, -STORE, 94572781277184, 94572785741823, -SNULL, 94572783312895, 94572785741823, -STORE, 94572781277184, 94572783312895, -STORE, 94572783312896, 94572785741823, -ERASE, 94572783312896, 94572785741823, -STORE, 94572785405952, 94572785455103, -STORE, 94572785455104, 94572785741823, -STORE, 139636001341440, 139636003594239, -SNULL, 139636001484799, 139636003594239, -STORE, 139636001341440, 139636001484799, -STORE, 139636001484800, 139636003594239, -ERASE, 139636001484800, 139636003594239, -STORE, 139636003581952, 139636003590143, -STORE, 139636003590144, 139636003594239, -STORE, 140725255557120, 140725255561215, -STORE, 140725255544832, 140725255557119, -STORE, 139636003553280, 139636003581951, -STORE, 139636003545088, 139636003553279, -STORE, 139635998773248, 139636001341439, -SNULL, 139635998773248, 139635999240191, -STORE, 139635999240192, 139636001341439, -STORE, 139635998773248, 139635999240191, -SNULL, 139636001333247, 139636001341439, -STORE, 139635999240192, 139636001333247, -STORE, 139636001333248, 139636001341439, -ERASE, 139636001333248, 139636001341439, -STORE, 139636001333248, 139636001341439, -STORE, 139635996569600, 139635998773247, -SNULL, 139635996569600, 139635996671999, -STORE, 139635996672000, 139635998773247, -STORE, 139635996569600, 139635996671999, -SNULL, 139635998765055, 139635998773247, -STORE, 139635996672000, 139635998765055, -STORE, 139635998765056, 139635998773247, -ERASE, 139635998765056, 139635998773247, -STORE, 139635998765056, 139635998773247, -STORE, 139635994353664, 139635996569599, -SNULL, 139635994353664, 139635994451967, -STORE, 139635994451968, 139635996569599, -STORE, 139635994353664, 139635994451967, -SNULL, 139635996545023, 139635996569599, -STORE, 139635994451968, 139635996545023, -STORE, 139635996545024, 139635996569599, -SNULL, 139635996545024, 139635996553215, -STORE, 139635996553216, 139635996569599, -STORE, 139635996545024, 139635996553215, -ERASE, 139635996545024, 139635996553215, -STORE, 139635996545024, 139635996553215, -ERASE, 139635996553216, 139635996569599, -STORE, 139635996553216, 139635996569599, -STORE, 139635992223744, 139635994353663, -SNULL, 139635992223744, 139635992252415, -STORE, 139635992252416, 139635994353663, -STORE, 139635992223744, 139635992252415, -SNULL, 139635994345471, 139635994353663, -STORE, 139635992252416, 139635994345471, -STORE, 139635994345472, 139635994353663, -ERASE, 139635994345472, 139635994353663, -STORE, 139635994345472, 139635994353663, -STORE, 139635988426752, 139635992223743, -SNULL, 139635988426752, 139635990085631, -STORE, 139635990085632, 139635992223743, -STORE, 139635988426752, 139635990085631, -SNULL, 139635992182783, 139635992223743, -STORE, 139635990085632, 139635992182783, -STORE, 139635992182784, 139635992223743, -SNULL, 139635992182784, 139635992207359, -STORE, 139635992207360, 139635992223743, -STORE, 139635992182784, 139635992207359, -ERASE, 139635992182784, 139635992207359, -STORE, 139635992182784, 139635992207359, -ERASE, 139635992207360, 139635992223743, -STORE, 139635992207360, 139635992223743, -STORE, 139636003536896, 139636003553279, -SNULL, 139635992199167, 139635992207359, -STORE, 139635992182784, 139635992199167, -STORE, 139635992199168, 139635992207359, -SNULL, 139635996549119, 139635996553215, -STORE, 139635996545024, 139635996549119, -STORE, 139635996549120, 139635996553215, -SNULL, 139635994349567, 139635994353663, -STORE, 139635994345472, 139635994349567, -STORE, 139635994349568, 139635994353663, -SNULL, 139635998769151, 139635998773247, -STORE, 139635998765056, 139635998769151, -STORE, 139635998769152, 139635998773247, -SNULL, 139636001337343, 139636001341439, -STORE, 139636001333248, 139636001337343, -STORE, 139636001337344, 139636001341439, -SNULL, 94572785418239, 94572785455103, -STORE, 94572785405952, 94572785418239, -STORE, 94572785418240, 94572785455103, -SNULL, 139636003586047, 139636003590143, -STORE, 139636003581952, 139636003586047, -STORE, 139636003586048, 139636003590143, -ERASE, 139636003553280, 139636003581951, -STORE, 94572798435328, 94572798570495, -STORE, 139636001853440, 139636003536895, -STORE, 139635981426688, 139635988426751, -STORE, 139635980615680, 139635981426687, -STORE, 94572798435328, 94572798705663, -STORE, 94572798435328, 94572798840831, -STORE, 94572798435328, 94572798975999, -STORE, 94572798435328, 94572799111167, -STORE, 94572798435328, 94572799246335, -STORE, 94572798435328, 94572799381503, -STORE, 94572798435328, 94572799516671, -STORE, 94572798435328, 94572799651839, -STORE, 94572798435328, 94572799787007, -STORE, 94572798435328, 94572799922175, -STORE, 94572798435328, 94572800057343, -STORE, 94572798435328, 94572800192511, -STORE, 94572798435328, 94572800327679, -STORE, 94572798435328, 94572800462847, -STORE, 94572798435328, 94572800598015, -STORE, 94572798435328, 94572800733183, -STORE, 94572798435328, 94572800868351, -STORE, 94572798435328, 94572801003519, -STORE, 94572798435328, 94572801138687, -STORE, 94572798435328, 94572801273855, -STORE, 94572798435328, 94572801409023, -STORE, 94572798435328, 94572801544191, -STORE, 94572798435328, 94572801679359, -STORE, 94572798435328, 94572801814527, -STORE, 94572798435328, 94572801949695, -STORE, 94572798435328, 94572802084863, -STORE, 94572798435328, 94572802220031, -STORE, 94572798435328, 94572802355199, -STORE, 94572798435328, 94572802490367, -STORE, 94572798435328, 94572802625535, -STORE, 94572798435328, 94572802760703, -STORE, 94572798435328, 94572802895871, -STORE, 94572798435328, 94572803031039, -STORE, 94572798435328, 94572803166207, -STORE, 94572798435328, 94572803301375, -STORE, 94572798435328, 94572803436543, -STORE, 94572798435328, 94572803571711, -STORE, 94572798435328, 94572803706879, -STORE, 94572798435328, 94572803842047, -STORE, 94572798435328, 94572803977215, -STORE, 94572798435328, 94572804112383, -STORE, 94572798435328, 94572804247551, -STORE, 94572798435328, 94572804382719, -STORE, 94572798435328, 94572804517887, -STORE, 94572798435328, 94572804653055, -STORE, 94572798435328, 94572804788223, -STORE, 94572798435328, 94572804923391, -STORE, 94572798435328, 94572805058559, -STORE, 94572798435328, 94572805193727, -STORE, 94572798435328, 94572805328895, -STORE, 94572798435328, 94572805464063, -STORE, 94572798435328, 94572805599231, -STORE, 94572798435328, 94572805734399, -STORE, 94572798435328, 94572805869567, -STORE, 94572798435328, 94572806004735, -STORE, 94572798435328, 94572806139903, -STORE, 94572798435328, 94572806275071, -STORE, 94572798435328, 94572806410239, -STORE, 94572798435328, 94572806545407, -STORE, 94572798435328, 94572806680575, -STORE, 94572798435328, 94572806815743, -STORE, 94572798435328, 94572806950911, -STORE, 94572798435328, 94572807086079, -STORE, 94572798435328, 94572807221247, -STORE, 94572798435328, 94572807356415, -STORE, 94572798435328, 94572807491583, -STORE, 94572798435328, 94572807626751, -STORE, 94572798435328, 94572807761919, -STORE, 94572798435328, 94572807897087, -STORE, 94572798435328, 94572808032255, -STORE, 94572798435328, 94572808167423, -STORE, 94572798435328, 94572808302591, -STORE, 94572798435328, 94572808437759, -STORE, 94572798435328, 94572808572927, -ERASE, 139635981426688, 139635988426751, -STORE, 139635985088512, 139635988426751, -STORE, 139635778273280, 139635980615679, -STORE, 139635567632384, 139635778273279, -STORE, 94572798435328, 94572808716287, -STORE, 139635984564224, 139635985088511, -STORE, 139635559239680, 139635567632383, -SNULL, 139635559243775, 139635567632383, -STORE, 139635559239680, 139635559243775, -STORE, 139635559243776, 139635567632383, -STORE, 139635550846976, 139635559239679, -SNULL, 139635550851071, 139635559239679, -STORE, 139635550846976, 139635550851071, -STORE, 139635550851072, 139635559239679, -STORE, 139635542454272, 139635550846975, -STORE, 139635408236544, 139635542454271, -SNULL, 139635408236544, 139635426590719, -STORE, 139635426590720, 139635542454271, -STORE, 139635408236544, 139635426590719, -ERASE, 139635408236544, 139635426590719, -STORE, 139635292372992, 139635542454271, -SNULL, 139635359481855, 139635542454271, -STORE, 139635292372992, 139635359481855, -STORE, 139635359481856, 139635542454271, -SNULL, 139635359481856, 139635426590719, -STORE, 139635426590720, 139635542454271, -STORE, 139635359481856, 139635426590719, -ERASE, 139635359481856, 139635426590719, -SNULL, 139635542458367, 139635550846975, -STORE, 139635542454272, 139635542458367, -STORE, 139635542458368, 139635550846975, -STORE, 139635418198016, 139635426590719, -SNULL, 139635493699583, 139635542454271, -STORE, 139635426590720, 139635493699583, -STORE, 139635493699584, 139635542454271, -ERASE, 139635493699584, 139635542454271, -SNULL, 139635426725887, 139635493699583, -STORE, 139635426590720, 139635426725887, -STORE, 139635426725888, 139635493699583, -SNULL, 139635292508159, 139635359481855, -STORE, 139635292372992, 139635292508159, -STORE, 139635292508160, 139635359481855, -SNULL, 139635418202111, 139635426590719, -STORE, 139635418198016, 139635418202111, -STORE, 139635418202112, 139635426590719, -STORE, 139635225264128, 139635292372991, -STORE, 139635534061568, 139635542454271, -SNULL, 139635534065663, 139635542454271, -STORE, 139635534061568, 139635534065663, -STORE, 139635534065664, 139635542454271, -STORE, 139635525668864, 139635534061567, -SNULL, 139635525672959, 139635534061567, -STORE, 139635525668864, 139635525672959, -STORE, 139635525672960, 139635534061567, -SNULL, 139635225399295, 139635292372991, -STORE, 139635225264128, 139635225399295, -STORE, 139635225399296, 139635292372991, -STORE, 139635091046400, 139635225264127, -SNULL, 139635158155263, 139635225264127, -STORE, 139635091046400, 139635158155263, -STORE, 139635158155264, 139635225264127, -ERASE, 139635158155264, 139635225264127, -STORE, 139634956828672, 139635158155263, -STORE, 139635517276160, 139635525668863, -SNULL, 139635517280255, 139635525668863, -STORE, 139635517276160, 139635517280255, -STORE, 139635517280256, 139635525668863, -SNULL, 139634956828672, 139635091046399, -STORE, 139635091046400, 139635158155263, -STORE, 139634956828672, 139635091046399, -SNULL, 139635091181567, 139635158155263, -STORE, 139635091046400, 139635091181567, -STORE, 139635091181568, 139635158155263, -SNULL, 139635023937535, 139635091046399, -STORE, 139634956828672, 139635023937535, -STORE, 139635023937536, 139635091046399, -ERASE, 139635023937536, 139635091046399, -STORE, 139634956828672, 139635091046399, -SNULL, 139634956828672, 139635023937535, -STORE, 139635023937536, 139635091046399, -STORE, 139634956828672, 139635023937535, -SNULL, 139635024072703, 139635091046399, -STORE, 139635023937536, 139635024072703, -STORE, 139635024072704, 139635091046399, -STORE, 139635508883456, 139635517276159, -SNULL, 139635508887551, 139635517276159, -STORE, 139635508883456, 139635508887551, -STORE, 139635508887552, 139635517276159, -STORE, 139634822610944, 139635023937535, -SNULL, 139634822610944, 139634956828671, -STORE, 139634956828672, 139635023937535, -STORE, 139634822610944, 139634956828671, -SNULL, 139634956963839, 139635023937535, -STORE, 139634956828672, 139634956963839, -STORE, 139634956963840, 139635023937535, -STORE, 139635500490752, 139635508883455, -SNULL, 139634889719807, 139634956828671, -STORE, 139634822610944, 139634889719807, -STORE, 139634889719808, 139634956828671, -ERASE, 139634889719808, 139634956828671, -SNULL, 139635500494847, 139635508883455, -STORE, 139635500490752, 139635500494847, -STORE, 139635500494848, 139635508883455, -SNULL, 139634822746111, 139634889719807, -STORE, 139634822610944, 139634822746111, -STORE, 139634822746112, 139634889719807, -STORE, 139635409805312, 139635418198015, -STORE, 139634822746112, 139634956828671, -SNULL, 139634822746112, 139634889719807, -STORE, 139634889719808, 139634956828671, -STORE, 139634822746112, 139634889719807, -SNULL, 139634889854975, 139634956828671, -STORE, 139634889719808, 139634889854975, -STORE, 139634889854976, 139634956828671, -SNULL, 139635409809407, 139635418198015, -STORE, 139635409805312, 139635409809407, -STORE, 139635409809408, 139635418198015, -STORE, 139635401412608, 139635409805311, -STORE, 139634688393216, 139634822610943, -SNULL, 139634755502079, 139634822610943, -STORE, 139634688393216, 139634755502079, -STORE, 139634755502080, 139634822610943, -ERASE, 139634755502080, 139634822610943, -SNULL, 139635401416703, 139635409805311, -STORE, 139635401412608, 139635401416703, -STORE, 139635401416704, 139635409805311, -STORE, 139634554175488, 139634755502079, -SNULL, 139634554175488, 139634688393215, -STORE, 139634688393216, 139634755502079, -STORE, 139634554175488, 139634688393215, -SNULL, 139634688528383, 139634755502079, -STORE, 139634688393216, 139634688528383, -STORE, 139634688528384, 139634755502079, -STORE, 139635393019904, 139635401412607, -SNULL, 139634621284351, 139634688393215, -STORE, 139634554175488, 139634621284351, -STORE, 139634621284352, 139634688393215, -ERASE, 139634621284352, 139634688393215, -SNULL, 139634554310655, 139634621284351, -STORE, 139634554175488, 139634554310655, -STORE, 139634554310656, 139634621284351, -STORE, 139634554310656, 139634688393215, -SNULL, 139635393023999, 139635401412607, -STORE, 139635393019904, 139635393023999, -STORE, 139635393024000, 139635401412607, -SNULL, 139634554310656, 139634621284351, -STORE, 139634621284352, 139634688393215, -STORE, 139634554310656, 139634621284351, -SNULL, 139634621419519, 139634688393215, -STORE, 139634621284352, 139634621419519, -STORE, 139634621419520, 139634688393215, -STORE, 139635384627200, 139635393019903, -SNULL, 139635384631295, 139635393019903, -STORE, 139635384627200, 139635384631295, -STORE, 139635384631296, 139635393019903, -STORE, 139635376234496, 139635384627199, -SNULL, 139635376238591, 139635384627199, -STORE, 139635376234496, 139635376238591, -STORE, 139635376238592, 139635384627199, -STORE, 139635367841792, 139635376234495, -SNULL, 139635367845887, 139635376234495, -STORE, 139635367841792, 139635367845887, -STORE, 139635367845888, 139635376234495, -STORE, 139634419957760, 139634554175487, -SNULL, 139634487066623, 139634554175487, -STORE, 139634419957760, 139634487066623, -STORE, 139634487066624, 139634554175487, -ERASE, 139634487066624, 139634554175487, -STORE, 139635216871424, 139635225264127, -SNULL, 139635216875519, 139635225264127, -STORE, 139635216871424, 139635216875519, -STORE, 139635216875520, 139635225264127, -SNULL, 139634420092927, 139634487066623, -STORE, 139634419957760, 139634420092927, -STORE, 139634420092928, 139634487066623, -STORE, 139635208478720, 139635216871423, -SNULL, 139635208482815, 139635216871423, -STORE, 139635208478720, 139635208482815, -STORE, 139635208482816, 139635216871423, -STORE, 139635200086016, 139635208478719, -SNULL, 139635200090111, 139635208478719, -STORE, 139635200086016, 139635200090111, -STORE, 139635200090112, 139635208478719, -STORE, 139635191693312, 139635200086015, -SNULL, 139635191697407, 139635200086015, -STORE, 139635191693312, 139635191697407, -STORE, 139635191697408, 139635200086015, -STORE, 139635183300608, 139635191693311, -SNULL, 139635183304703, 139635191693311, -STORE, 139635183300608, 139635183304703, -STORE, 139635183304704, 139635191693311, -STORE, 139634420092928, 139634554175487, -SNULL, 139634420092928, 139634487066623, -STORE, 139634487066624, 139634554175487, -STORE, 139634420092928, 139634487066623, -SNULL, 139634487201791, 139634554175487, -STORE, 139634487066624, 139634487201791, -STORE, 139634487201792, 139634554175487, -ERASE, 139635559239680, 139635559243775, -ERASE, 139635559243776, 139635567632383, -ERASE, 139635550846976, 139635550851071, -ERASE, 139635550851072, 139635559239679, -ERASE, 139635542454272, 139635542458367, -ERASE, 139635542458368, 139635550846975, -ERASE, 139635418198016, 139635418202111, -ERASE, 139635418202112, 139635426590719, -ERASE, 139635534061568, 139635534065663, -ERASE, 139635534065664, 139635542454271, -ERASE, 139635525668864, 139635525672959, -ERASE, 139635525672960, 139635534061567, -ERASE, 139635517276160, 139635517280255, -ERASE, 139635517280256, 139635525668863, -ERASE, 139635508883456, 139635508887551, -ERASE, 139635508887552, 139635517276159, -ERASE, 139635500490752, 139635500494847, -ERASE, 139635500494848, 139635508883455, -ERASE, 139635409805312, 139635409809407, -ERASE, 139635409809408, 139635418198015, -ERASE, 139635401412608, 139635401416703, -ERASE, 139635401416704, 139635409805311, -ERASE, 139635393019904, 139635393023999, -ERASE, 139635393024000, 139635401412607, -ERASE, 139635384627200, 139635384631295, -ERASE, 139635384631296, 139635393019903, - }; - unsigned long set25[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722547441664, 140737488351231, -SNULL, 140722547449855, 140737488351231, -STORE, 140722547441664, 140722547449855, -STORE, 140722547310592, 140722547449855, -STORE, 94827521732608, 94827523956735, -SNULL, 94827521843199, 94827523956735, -STORE, 94827521732608, 94827521843199, -STORE, 94827521843200, 94827523956735, -ERASE, 94827521843200, 94827523956735, -STORE, 94827523936256, 94827523948543, -STORE, 94827523948544, 94827523956735, -STORE, 139816136847360, 139816139100159, -SNULL, 139816136990719, 139816139100159, -STORE, 139816136847360, 139816136990719, -STORE, 139816136990720, 139816139100159, -ERASE, 139816136990720, 139816139100159, -STORE, 139816139087872, 139816139096063, -STORE, 139816139096064, 139816139100159, -STORE, 140722548142080, 140722548146175, -STORE, 140722548129792, 140722548142079, -STORE, 139816139059200, 139816139087871, -STORE, 139816139051008, 139816139059199, -STORE, 139816133050368, 139816136847359, -SNULL, 139816133050368, 139816134709247, -STORE, 139816134709248, 139816136847359, -STORE, 139816133050368, 139816134709247, -SNULL, 139816136806399, 139816136847359, -STORE, 139816134709248, 139816136806399, -STORE, 139816136806400, 139816136847359, -SNULL, 139816136806400, 139816136830975, -STORE, 139816136830976, 139816136847359, -STORE, 139816136806400, 139816136830975, -ERASE, 139816136806400, 139816136830975, -STORE, 139816136806400, 139816136830975, -ERASE, 139816136830976, 139816136847359, -STORE, 139816136830976, 139816136847359, -SNULL, 139816136822783, 139816136830975, -STORE, 139816136806400, 139816136822783, -STORE, 139816136822784, 139816136830975, -SNULL, 94827523944447, 94827523948543, -STORE, 94827523936256, 94827523944447, -STORE, 94827523944448, 94827523948543, -SNULL, 139816139091967, 139816139096063, -STORE, 139816139087872, 139816139091967, -STORE, 139816139091968, 139816139096063, -ERASE, 139816139059200, 139816139087871, -STORE, 94827534970880, 94827535106047, -STORE, 94114394132480, 94114394345471, -STORE, 94114396442624, 94114396446719, -STORE, 94114396446720, 94114396454911, -STORE, 94114396454912, 94114396467199, -STORE, 94114421575680, 94114427715583, -STORE, 139934313955328, 139934315614207, -STORE, 139934315614208, 139934317711359, -STORE, 139934317711360, 139934317727743, -STORE, 139934317727744, 139934317735935, -STORE, 139934317735936, 139934317752319, -STORE, 139934317752320, 139934317764607, -STORE, 139934317764608, 139934319857663, -STORE, 139934319857664, 139934319861759, -STORE, 139934319861760, 139934319865855, -STORE, 139934319865856, 139934320009215, -STORE, 139934320377856, 139934322061311, -STORE, 139934322061312, 139934322077695, -STORE, 139934322106368, 139934322110463, -STORE, 139934322110464, 139934322114559, -STORE, 139934322114560, 139934322118655, -STORE, 140731200376832, 140731200516095, -STORE, 140731200929792, 140731200942079, -STORE, 140731200942080, 140731200946175, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140734133174272, 140737488351231, -SNULL, 140734133182463, 140737488351231, -STORE, 140734133174272, 140734133182463, -STORE, 140734133043200, 140734133182463, -STORE, 94412675600384, 94412677824511, -SNULL, 94412675710975, 94412677824511, -STORE, 94412675600384, 94412675710975, -STORE, 94412675710976, 94412677824511, -ERASE, 94412675710976, 94412677824511, -STORE, 94412677804032, 94412677816319, -STORE, 94412677816320, 94412677824511, -STORE, 140320087945216, 140320090198015, -SNULL, 140320088088575, 140320090198015, -STORE, 140320087945216, 140320088088575, -STORE, 140320088088576, 140320090198015, -ERASE, 140320088088576, 140320090198015, -STORE, 140320090185728, 140320090193919, -STORE, 140320090193920, 140320090198015, -STORE, 140734134591488, 140734134595583, -STORE, 140734134579200, 140734134591487, -STORE, 140320090157056, 140320090185727, -STORE, 140320090148864, 140320090157055, -STORE, 140320084148224, 140320087945215, -SNULL, 140320084148224, 140320085807103, -STORE, 140320085807104, 140320087945215, -STORE, 140320084148224, 140320085807103, -SNULL, 140320087904255, 140320087945215, -STORE, 140320085807104, 140320087904255, -STORE, 140320087904256, 140320087945215, -SNULL, 140320087904256, 140320087928831, -STORE, 140320087928832, 140320087945215, -STORE, 140320087904256, 140320087928831, -ERASE, 140320087904256, 140320087928831, -STORE, 140320087904256, 140320087928831, -ERASE, 140320087928832, 140320087945215, -STORE, 140320087928832, 140320087945215, -SNULL, 140320087920639, 140320087928831, -STORE, 140320087904256, 140320087920639, -STORE, 140320087920640, 140320087928831, -SNULL, 94412677812223, 94412677816319, -STORE, 94412677804032, 94412677812223, -STORE, 94412677812224, 94412677816319, -SNULL, 140320090189823, 140320090193919, -STORE, 140320090185728, 140320090189823, -STORE, 140320090189824, 140320090193919, -ERASE, 140320090157056, 140320090185727, -STORE, 94412684546048, 94412684681215, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723005485056, 140737488351231, -SNULL, 140723005493247, 140737488351231, -STORE, 140723005485056, 140723005493247, -STORE, 140723005353984, 140723005493247, -STORE, 94387431936000, 94387434160127, -SNULL, 94387432046591, 94387434160127, -STORE, 94387431936000, 94387432046591, -STORE, 94387432046592, 94387434160127, -ERASE, 94387432046592, 94387434160127, -STORE, 94387434139648, 94387434151935, -STORE, 94387434151936, 94387434160127, -STORE, 140151675392000, 140151677644799, -SNULL, 140151675535359, 140151677644799, -STORE, 140151675392000, 140151675535359, -STORE, 140151675535360, 140151677644799, -ERASE, 140151675535360, 140151677644799, -STORE, 140151677632512, 140151677640703, -STORE, 140151677640704, 140151677644799, -STORE, 140723005784064, 140723005788159, -STORE, 140723005771776, 140723005784063, -STORE, 140151677603840, 140151677632511, -STORE, 140151677595648, 140151677603839, -STORE, 140151671595008, 140151675391999, -SNULL, 140151671595008, 140151673253887, -STORE, 140151673253888, 140151675391999, -STORE, 140151671595008, 140151673253887, -SNULL, 140151675351039, 140151675391999, -STORE, 140151673253888, 140151675351039, -STORE, 140151675351040, 140151675391999, -SNULL, 140151675351040, 140151675375615, -STORE, 140151675375616, 140151675391999, -STORE, 140151675351040, 140151675375615, -ERASE, 140151675351040, 140151675375615, -STORE, 140151675351040, 140151675375615, -ERASE, 140151675375616, 140151675391999, -STORE, 140151675375616, 140151675391999, -SNULL, 140151675367423, 140151675375615, -STORE, 140151675351040, 140151675367423, -STORE, 140151675367424, 140151675375615, -SNULL, 94387434147839, 94387434151935, -STORE, 94387434139648, 94387434147839, -STORE, 94387434147840, 94387434151935, -SNULL, 140151677636607, 140151677640703, -STORE, 140151677632512, 140151677636607, -STORE, 140151677636608, 140151677640703, -ERASE, 140151677603840, 140151677632511, -STORE, 94387458818048, 94387458953215, -STORE, 94909010997248, 94909011210239, -STORE, 94909013307392, 94909013311487, -STORE, 94909013311488, 94909013319679, -STORE, 94909013319680, 94909013331967, -STORE, 94909014827008, 94909023371263, -STORE, 140712411975680, 140712413634559, -STORE, 140712413634560, 140712415731711, -STORE, 140712415731712, 140712415748095, -STORE, 140712415748096, 140712415756287, -STORE, 140712415756288, 140712415772671, -STORE, 140712415772672, 140712415784959, -STORE, 140712415784960, 140712417878015, -STORE, 140712417878016, 140712417882111, -STORE, 140712417882112, 140712417886207, -STORE, 140712417886208, 140712418029567, -STORE, 140712418398208, 140712420081663, -STORE, 140712420081664, 140712420098047, -STORE, 140712420126720, 140712420130815, -STORE, 140712420130816, 140712420134911, -STORE, 140712420134912, 140712420139007, -STORE, 140729293111296, 140729293250559, -STORE, 140729293307904, 140729293320191, -STORE, 140729293320192, 140729293324287, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140720541691904, 140737488351231, -SNULL, 140720541700095, 140737488351231, -STORE, 140720541691904, 140720541700095, -STORE, 140720541560832, 140720541700095, -STORE, 94203603419136, 94203605643263, -SNULL, 94203603529727, 94203605643263, -STORE, 94203603419136, 94203603529727, -STORE, 94203603529728, 94203605643263, -ERASE, 94203603529728, 94203605643263, -STORE, 94203605622784, 94203605635071, -STORE, 94203605635072, 94203605643263, -STORE, 139847623081984, 139847625334783, -SNULL, 139847623225343, 139847625334783, -STORE, 139847623081984, 139847623225343, -STORE, 139847623225344, 139847625334783, -ERASE, 139847623225344, 139847625334783, -STORE, 139847625322496, 139847625330687, -STORE, 139847625330688, 139847625334783, -STORE, 140720542547968, 140720542552063, -STORE, 140720542535680, 140720542547967, -STORE, 139847625293824, 139847625322495, -STORE, 139847625285632, 139847625293823, -STORE, 139847619284992, 139847623081983, -SNULL, 139847619284992, 139847620943871, -STORE, 139847620943872, 139847623081983, -STORE, 139847619284992, 139847620943871, -SNULL, 139847623041023, 139847623081983, -STORE, 139847620943872, 139847623041023, -STORE, 139847623041024, 139847623081983, -SNULL, 139847623041024, 139847623065599, -STORE, 139847623065600, 139847623081983, -STORE, 139847623041024, 139847623065599, -ERASE, 139847623041024, 139847623065599, -STORE, 139847623041024, 139847623065599, -ERASE, 139847623065600, 139847623081983, -STORE, 139847623065600, 139847623081983, -SNULL, 139847623057407, 139847623065599, -STORE, 139847623041024, 139847623057407, -STORE, 139847623057408, 139847623065599, -SNULL, 94203605630975, 94203605635071, -STORE, 94203605622784, 94203605630975, -STORE, 94203605630976, 94203605635071, -SNULL, 139847625326591, 139847625330687, -STORE, 139847625322496, 139847625326591, -STORE, 139847625326592, 139847625330687, -ERASE, 139847625293824, 139847625322495, -STORE, 94203634880512, 94203635015679, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140721428738048, 140737488351231, -SNULL, 140721428746239, 140737488351231, -STORE, 140721428738048, 140721428746239, -STORE, 140721428606976, 140721428746239, -STORE, 93968808378368, 93968810602495, -SNULL, 93968808488959, 93968810602495, -STORE, 93968808378368, 93968808488959, -STORE, 93968808488960, 93968810602495, -ERASE, 93968808488960, 93968810602495, -STORE, 93968810582016, 93968810594303, -STORE, 93968810594304, 93968810602495, -STORE, 140397757026304, 140397759279103, -SNULL, 140397757169663, 140397759279103, -STORE, 140397757026304, 140397757169663, -STORE, 140397757169664, 140397759279103, -ERASE, 140397757169664, 140397759279103, -STORE, 140397759266816, 140397759275007, -STORE, 140397759275008, 140397759279103, -STORE, 140721430368256, 140721430372351, -STORE, 140721430355968, 140721430368255, -STORE, 140397759238144, 140397759266815, -STORE, 140397759229952, 140397759238143, -STORE, 140397753229312, 140397757026303, -SNULL, 140397753229312, 140397754888191, -STORE, 140397754888192, 140397757026303, -STORE, 140397753229312, 140397754888191, -SNULL, 140397756985343, 140397757026303, -STORE, 140397754888192, 140397756985343, -STORE, 140397756985344, 140397757026303, -SNULL, 140397756985344, 140397757009919, -STORE, 140397757009920, 140397757026303, -STORE, 140397756985344, 140397757009919, -ERASE, 140397756985344, 140397757009919, -STORE, 140397756985344, 140397757009919, -ERASE, 140397757009920, 140397757026303, -STORE, 140397757009920, 140397757026303, -SNULL, 140397757001727, 140397757009919, -STORE, 140397756985344, 140397757001727, -STORE, 140397757001728, 140397757009919, -SNULL, 93968810590207, 93968810594303, -STORE, 93968810582016, 93968810590207, -STORE, 93968810590208, 93968810594303, -SNULL, 140397759270911, 140397759275007, -STORE, 140397759266816, 140397759270911, -STORE, 140397759270912, 140397759275007, -ERASE, 140397759238144, 140397759266815, -STORE, 93968837025792, 93968837160959, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140721751044096, 140737488351231, -SNULL, 140721751052287, 140737488351231, -STORE, 140721751044096, 140721751052287, -STORE, 140721750913024, 140721751052287, -STORE, 94426051657728, 94426053881855, -SNULL, 94426051768319, 94426053881855, -STORE, 94426051657728, 94426051768319, -STORE, 94426051768320, 94426053881855, -ERASE, 94426051768320, 94426053881855, -STORE, 94426053861376, 94426053873663, -STORE, 94426053873664, 94426053881855, -STORE, 140228456181760, 140228458434559, -SNULL, 140228456325119, 140228458434559, -STORE, 140228456181760, 140228456325119, -STORE, 140228456325120, 140228458434559, -ERASE, 140228456325120, 140228458434559, -STORE, 140228458422272, 140228458430463, -STORE, 140228458430464, 140228458434559, -STORE, 140721751117824, 140721751121919, -STORE, 140721751105536, 140721751117823, -STORE, 140228458393600, 140228458422271, -STORE, 140228458385408, 140228458393599, -STORE, 140228452384768, 140228456181759, -SNULL, 140228452384768, 140228454043647, -STORE, 140228454043648, 140228456181759, -STORE, 140228452384768, 140228454043647, -SNULL, 140228456140799, 140228456181759, -STORE, 140228454043648, 140228456140799, -STORE, 140228456140800, 140228456181759, -SNULL, 140228456140800, 140228456165375, -STORE, 140228456165376, 140228456181759, -STORE, 140228456140800, 140228456165375, -ERASE, 140228456140800, 140228456165375, -STORE, 140228456140800, 140228456165375, -ERASE, 140228456165376, 140228456181759, -STORE, 140228456165376, 140228456181759, -SNULL, 140228456157183, 140228456165375, -STORE, 140228456140800, 140228456157183, -STORE, 140228456157184, 140228456165375, -SNULL, 94426053869567, 94426053873663, -STORE, 94426053861376, 94426053869567, -STORE, 94426053869568, 94426053873663, -SNULL, 140228458426367, 140228458430463, -STORE, 140228458422272, 140228458426367, -STORE, 140228458426368, 140228458430463, -ERASE, 140228458393600, 140228458422271, -STORE, 94426073681920, 94426073817087, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140732727623680, 140737488351231, -SNULL, 140732727631871, 140737488351231, -STORE, 140732727623680, 140732727631871, -STORE, 140732727492608, 140732727631871, -STORE, 94537485996032, 94537488220159, -SNULL, 94537486106623, 94537488220159, -STORE, 94537485996032, 94537486106623, -STORE, 94537486106624, 94537488220159, -ERASE, 94537486106624, 94537488220159, -STORE, 94537488199680, 94537488211967, -STORE, 94537488211968, 94537488220159, -STORE, 140446578036736, 140446580289535, -SNULL, 140446578180095, 140446580289535, -STORE, 140446578036736, 140446578180095, -STORE, 140446578180096, 140446580289535, -ERASE, 140446578180096, 140446580289535, -STORE, 140446580277248, 140446580285439, -STORE, 140446580285440, 140446580289535, -STORE, 140732727758848, 140732727762943, -STORE, 140732727746560, 140732727758847, -STORE, 140446580248576, 140446580277247, -STORE, 140446580240384, 140446580248575, -STORE, 140446574239744, 140446578036735, -SNULL, 140446574239744, 140446575898623, -STORE, 140446575898624, 140446578036735, -STORE, 140446574239744, 140446575898623, -SNULL, 140446577995775, 140446578036735, -STORE, 140446575898624, 140446577995775, -STORE, 140446577995776, 140446578036735, -SNULL, 140446577995776, 140446578020351, -STORE, 140446578020352, 140446578036735, -STORE, 140446577995776, 140446578020351, -ERASE, 140446577995776, 140446578020351, -STORE, 140446577995776, 140446578020351, -ERASE, 140446578020352, 140446578036735, -STORE, 140446578020352, 140446578036735, -SNULL, 140446578012159, 140446578020351, -STORE, 140446577995776, 140446578012159, -STORE, 140446578012160, 140446578020351, -SNULL, 94537488207871, 94537488211967, -STORE, 94537488199680, 94537488207871, -STORE, 94537488207872, 94537488211967, -SNULL, 140446580281343, 140446580285439, -STORE, 140446580277248, 140446580281343, -STORE, 140446580281344, 140446580285439, -ERASE, 140446580248576, 140446580277247, -STORE, 94537489014784, 94537489149951, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728766808064, 140737488351231, -SNULL, 140728766816255, 140737488351231, -STORE, 140728766808064, 140728766816255, -STORE, 140728766676992, 140728766816255, -STORE, 94418513866752, 94418516090879, -SNULL, 94418513977343, 94418516090879, -STORE, 94418513866752, 94418513977343, -STORE, 94418513977344, 94418516090879, -ERASE, 94418513977344, 94418516090879, -STORE, 94418516070400, 94418516082687, -STORE, 94418516082688, 94418516090879, -STORE, 140556479520768, 140556481773567, -SNULL, 140556479664127, 140556481773567, -STORE, 140556479520768, 140556479664127, -STORE, 140556479664128, 140556481773567, -ERASE, 140556479664128, 140556481773567, -STORE, 140556481761280, 140556481769471, -STORE, 140556481769472, 140556481773567, -STORE, 140728767148032, 140728767152127, -STORE, 140728767135744, 140728767148031, -STORE, 140556481732608, 140556481761279, -STORE, 140556481724416, 140556481732607, -STORE, 140556475723776, 140556479520767, -SNULL, 140556475723776, 140556477382655, -STORE, 140556477382656, 140556479520767, -STORE, 140556475723776, 140556477382655, -SNULL, 140556479479807, 140556479520767, -STORE, 140556477382656, 140556479479807, -STORE, 140556479479808, 140556479520767, -SNULL, 140556479479808, 140556479504383, -STORE, 140556479504384, 140556479520767, -STORE, 140556479479808, 140556479504383, -ERASE, 140556479479808, 140556479504383, -STORE, 140556479479808, 140556479504383, -ERASE, 140556479504384, 140556479520767, -STORE, 140556479504384, 140556479520767, -SNULL, 140556479496191, 140556479504383, -STORE, 140556479479808, 140556479496191, -STORE, 140556479496192, 140556479504383, -SNULL, 94418516078591, 94418516082687, -STORE, 94418516070400, 94418516078591, -STORE, 94418516078592, 94418516082687, -SNULL, 140556481765375, 140556481769471, -STORE, 140556481761280, 140556481765375, -STORE, 140556481765376, 140556481769471, -ERASE, 140556481732608, 140556481761279, -STORE, 94418541113344, 94418541248511, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723945873408, 140737488351231, -SNULL, 140723945881599, 140737488351231, -STORE, 140723945873408, 140723945881599, -STORE, 140723945742336, 140723945881599, -STORE, 94543169773568, 94543171997695, -SNULL, 94543169884159, 94543171997695, -STORE, 94543169773568, 94543169884159, -STORE, 94543169884160, 94543171997695, -ERASE, 94543169884160, 94543171997695, -STORE, 94543171977216, 94543171989503, -STORE, 94543171989504, 94543171997695, -STORE, 139890420883456, 139890423136255, -SNULL, 139890421026815, 139890423136255, -STORE, 139890420883456, 139890421026815, -STORE, 139890421026816, 139890423136255, -ERASE, 139890421026816, 139890423136255, -STORE, 139890423123968, 139890423132159, -STORE, 139890423132160, 139890423136255, -STORE, 140723946102784, 140723946106879, -STORE, 140723946090496, 140723946102783, -STORE, 139890423095296, 139890423123967, -STORE, 139890423087104, 139890423095295, -STORE, 139890417086464, 139890420883455, -SNULL, 139890417086464, 139890418745343, -STORE, 139890418745344, 139890420883455, -STORE, 139890417086464, 139890418745343, -SNULL, 139890420842495, 139890420883455, -STORE, 139890418745344, 139890420842495, -STORE, 139890420842496, 139890420883455, -SNULL, 139890420842496, 139890420867071, -STORE, 139890420867072, 139890420883455, -STORE, 139890420842496, 139890420867071, -ERASE, 139890420842496, 139890420867071, -STORE, 139890420842496, 139890420867071, -ERASE, 139890420867072, 139890420883455, -STORE, 139890420867072, 139890420883455, -SNULL, 139890420858879, 139890420867071, -STORE, 139890420842496, 139890420858879, -STORE, 139890420858880, 139890420867071, -SNULL, 94543171985407, 94543171989503, -STORE, 94543171977216, 94543171985407, -STORE, 94543171985408, 94543171989503, -SNULL, 139890423128063, 139890423132159, -STORE, 139890423123968, 139890423128063, -STORE, 139890423128064, 139890423132159, -ERASE, 139890423095296, 139890423123967, -STORE, 94543197097984, 94543197233151, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140736205979648, 140737488351231, -SNULL, 140736205987839, 140737488351231, -STORE, 140736205979648, 140736205987839, -STORE, 140736205848576, 140736205987839, -STORE, 94913209913344, 94913212137471, -SNULL, 94913210023935, 94913212137471, -STORE, 94913209913344, 94913210023935, -STORE, 94913210023936, 94913212137471, -ERASE, 94913210023936, 94913212137471, -STORE, 94913212116992, 94913212129279, -STORE, 94913212129280, 94913212137471, -STORE, 140006323052544, 140006325305343, -SNULL, 140006323195903, 140006325305343, -STORE, 140006323052544, 140006323195903, -STORE, 140006323195904, 140006325305343, -ERASE, 140006323195904, 140006325305343, -STORE, 140006325293056, 140006325301247, -STORE, 140006325301248, 140006325305343, -STORE, 140736206716928, 140736206721023, -STORE, 140736206704640, 140736206716927, -STORE, 140006325264384, 140006325293055, -STORE, 140006325256192, 140006325264383, -STORE, 140006319255552, 140006323052543, -SNULL, 140006319255552, 140006320914431, -STORE, 140006320914432, 140006323052543, -STORE, 140006319255552, 140006320914431, -SNULL, 140006323011583, 140006323052543, -STORE, 140006320914432, 140006323011583, -STORE, 140006323011584, 140006323052543, -SNULL, 140006323011584, 140006323036159, -STORE, 140006323036160, 140006323052543, -STORE, 140006323011584, 140006323036159, -ERASE, 140006323011584, 140006323036159, -STORE, 140006323011584, 140006323036159, -ERASE, 140006323036160, 140006323052543, -STORE, 140006323036160, 140006323052543, -SNULL, 140006323027967, 140006323036159, -STORE, 140006323011584, 140006323027967, -STORE, 140006323027968, 140006323036159, -SNULL, 94913212125183, 94913212129279, -STORE, 94913212116992, 94913212125183, -STORE, 94913212125184, 94913212129279, -SNULL, 140006325297151, 140006325301247, -STORE, 140006325293056, 140006325297151, -STORE, 140006325297152, 140006325301247, -ERASE, 140006325264384, 140006325293055, -STORE, 94913239932928, 94913240068095, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140726926897152, 140737488351231, -SNULL, 140726926905343, 140737488351231, -STORE, 140726926897152, 140726926905343, -STORE, 140726926766080, 140726926905343, -STORE, 94213246820352, 94213249044479, -SNULL, 94213246930943, 94213249044479, -STORE, 94213246820352, 94213246930943, -STORE, 94213246930944, 94213249044479, -ERASE, 94213246930944, 94213249044479, -STORE, 94213249024000, 94213249036287, -STORE, 94213249036288, 94213249044479, -STORE, 140368830242816, 140368832495615, -SNULL, 140368830386175, 140368832495615, -STORE, 140368830242816, 140368830386175, -STORE, 140368830386176, 140368832495615, -ERASE, 140368830386176, 140368832495615, -STORE, 140368832483328, 140368832491519, -STORE, 140368832491520, 140368832495615, -STORE, 140726926999552, 140726927003647, -STORE, 140726926987264, 140726926999551, -STORE, 140368832454656, 140368832483327, -STORE, 140368832446464, 140368832454655, -STORE, 140368826445824, 140368830242815, -SNULL, 140368826445824, 140368828104703, -STORE, 140368828104704, 140368830242815, -STORE, 140368826445824, 140368828104703, -SNULL, 140368830201855, 140368830242815, -STORE, 140368828104704, 140368830201855, -STORE, 140368830201856, 140368830242815, -SNULL, 140368830201856, 140368830226431, -STORE, 140368830226432, 140368830242815, -STORE, 140368830201856, 140368830226431, -ERASE, 140368830201856, 140368830226431, -STORE, 140368830201856, 140368830226431, -ERASE, 140368830226432, 140368830242815, -STORE, 140368830226432, 140368830242815, -SNULL, 140368830218239, 140368830226431, -STORE, 140368830201856, 140368830218239, -STORE, 140368830218240, 140368830226431, -SNULL, 94213249032191, 94213249036287, -STORE, 94213249024000, 94213249032191, -STORE, 94213249032192, 94213249036287, -SNULL, 140368832487423, 140368832491519, -STORE, 140368832483328, 140368832487423, -STORE, 140368832487424, 140368832491519, -ERASE, 140368832454656, 140368832483327, -STORE, 94213267435520, 94213267570687, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728954130432, 140737488351231, -SNULL, 140728954138623, 140737488351231, -STORE, 140728954130432, 140728954138623, -STORE, 140728953999360, 140728954138623, -STORE, 94672570966016, 94672573190143, -SNULL, 94672571076607, 94672573190143, -STORE, 94672570966016, 94672571076607, -STORE, 94672571076608, 94672573190143, -ERASE, 94672571076608, 94672573190143, -STORE, 94672573169664, 94672573181951, -STORE, 94672573181952, 94672573190143, -STORE, 140201696735232, 140201698988031, -SNULL, 140201696878591, 140201698988031, -STORE, 140201696735232, 140201696878591, -STORE, 140201696878592, 140201698988031, -ERASE, 140201696878592, 140201698988031, -STORE, 140201698975744, 140201698983935, -STORE, 140201698983936, 140201698988031, -STORE, 140728954163200, 140728954167295, -STORE, 140728954150912, 140728954163199, -STORE, 140201698947072, 140201698975743, -STORE, 140201698938880, 140201698947071, -STORE, 140201692938240, 140201696735231, -SNULL, 140201692938240, 140201694597119, -STORE, 140201694597120, 140201696735231, -STORE, 140201692938240, 140201694597119, -SNULL, 140201696694271, 140201696735231, -STORE, 140201694597120, 140201696694271, -STORE, 140201696694272, 140201696735231, -SNULL, 140201696694272, 140201696718847, -STORE, 140201696718848, 140201696735231, -STORE, 140201696694272, 140201696718847, -ERASE, 140201696694272, 140201696718847, -STORE, 140201696694272, 140201696718847, -ERASE, 140201696718848, 140201696735231, -STORE, 140201696718848, 140201696735231, -SNULL, 140201696710655, 140201696718847, -STORE, 140201696694272, 140201696710655, -STORE, 140201696710656, 140201696718847, -SNULL, 94672573177855, 94672573181951, -STORE, 94672573169664, 94672573177855, -STORE, 94672573177856, 94672573181951, -SNULL, 140201698979839, 140201698983935, -STORE, 140201698975744, 140201698979839, -STORE, 140201698979840, 140201698983935, -ERASE, 140201698947072, 140201698975743, -STORE, 94672595689472, 94672595824639, -STORE, 94114394132480, 94114394345471, -STORE, 94114396442624, 94114396446719, -STORE, 94114396446720, 94114396454911, -STORE, 94114396454912, 94114396467199, -STORE, 94114421575680, 94114428256255, -STORE, 139934313955328, 139934315614207, -STORE, 139934315614208, 139934317711359, -STORE, 139934317711360, 139934317727743, -STORE, 139934317727744, 139934317735935, -STORE, 139934317735936, 139934317752319, -STORE, 139934317752320, 139934317764607, -STORE, 139934317764608, 139934319857663, -STORE, 139934319857664, 139934319861759, -STORE, 139934319861760, 139934319865855, -STORE, 139934319865856, 139934320009215, -STORE, 139934320377856, 139934322061311, -STORE, 139934322061312, 139934322077695, -STORE, 139934322106368, 139934322110463, -STORE, 139934322110464, 139934322114559, -STORE, 139934322114560, 139934322118655, -STORE, 140731200376832, 140731200516095, -STORE, 140731200929792, 140731200942079, -STORE, 140731200942080, 140731200946175, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140721532362752, 140737488351231, -SNULL, 140721532370943, 140737488351231, -STORE, 140721532362752, 140721532370943, -STORE, 140721532231680, 140721532370943, -STORE, 94467222597632, 94467224821759, -SNULL, 94467222708223, 94467224821759, -STORE, 94467222597632, 94467222708223, -STORE, 94467222708224, 94467224821759, -ERASE, 94467222708224, 94467224821759, -STORE, 94467224801280, 94467224813567, -STORE, 94467224813568, 94467224821759, -STORE, 140191433543680, 140191435796479, -SNULL, 140191433687039, 140191435796479, -STORE, 140191433543680, 140191433687039, -STORE, 140191433687040, 140191435796479, -ERASE, 140191433687040, 140191435796479, -STORE, 140191435784192, 140191435792383, -STORE, 140191435792384, 140191435796479, -STORE, 140721533034496, 140721533038591, -STORE, 140721533022208, 140721533034495, -STORE, 140191435755520, 140191435784191, -STORE, 140191435747328, 140191435755519, -STORE, 140191429746688, 140191433543679, -SNULL, 140191429746688, 140191431405567, -STORE, 140191431405568, 140191433543679, -STORE, 140191429746688, 140191431405567, -SNULL, 140191433502719, 140191433543679, -STORE, 140191431405568, 140191433502719, -STORE, 140191433502720, 140191433543679, -SNULL, 140191433502720, 140191433527295, -STORE, 140191433527296, 140191433543679, -STORE, 140191433502720, 140191433527295, -ERASE, 140191433502720, 140191433527295, -STORE, 140191433502720, 140191433527295, -ERASE, 140191433527296, 140191433543679, -STORE, 140191433527296, 140191433543679, -SNULL, 140191433519103, 140191433527295, -STORE, 140191433502720, 140191433519103, -STORE, 140191433519104, 140191433527295, -SNULL, 94467224809471, 94467224813567, -STORE, 94467224801280, 94467224809471, -STORE, 94467224809472, 94467224813567, -SNULL, 140191435788287, 140191435792383, -STORE, 140191435784192, 140191435788287, -STORE, 140191435788288, 140191435792383, -ERASE, 140191435755520, 140191435784191, -STORE, 94467251847168, 94467251982335, -STORE, 94367895400448, 94367895613439, -STORE, 94367897710592, 94367897714687, -STORE, 94367897714688, 94367897722879, -STORE, 94367897722880, 94367897735167, -STORE, 94367925264384, 94367926861823, -STORE, 139801317548032, 139801319206911, -STORE, 139801319206912, 139801321304063, -STORE, 139801321304064, 139801321320447, -STORE, 139801321320448, 139801321328639, -STORE, 139801321328640, 139801321345023, -STORE, 139801321345024, 139801321357311, -STORE, 139801321357312, 139801323450367, -STORE, 139801323450368, 139801323454463, -STORE, 139801323454464, 139801323458559, -STORE, 139801323458560, 139801323601919, -STORE, 139801323970560, 139801325654015, -STORE, 139801325654016, 139801325670399, -STORE, 139801325699072, 139801325703167, -STORE, 139801325703168, 139801325707263, -STORE, 139801325707264, 139801325711359, -STORE, 140724442861568, 140724443000831, -STORE, 140724443611136, 140724443623423, -STORE, 140724443623424, 140724443627519, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731353149440, 140737488351231, -SNULL, 140731353157631, 140737488351231, -STORE, 140731353149440, 140731353157631, -STORE, 140731353018368, 140731353157631, -STORE, 94310379503616, 94310381838335, -SNULL, 94310379716607, 94310381838335, -STORE, 94310379503616, 94310379716607, -STORE, 94310379716608, 94310381838335, -ERASE, 94310379716608, 94310381838335, -STORE, 94310381813760, 94310381826047, -STORE, 94310381826048, 94310381838335, -STORE, 140515434659840, 140515436912639, -SNULL, 140515434803199, 140515436912639, -STORE, 140515434659840, 140515434803199, -STORE, 140515434803200, 140515436912639, -ERASE, 140515434803200, 140515436912639, -STORE, 140515436900352, 140515436908543, -STORE, 140515436908544, 140515436912639, -STORE, 140731353886720, 140731353890815, -STORE, 140731353874432, 140731353886719, -STORE, 140515436871680, 140515436900351, -STORE, 140515436863488, 140515436871679, -STORE, 140515432546304, 140515434659839, -SNULL, 140515432546304, 140515432558591, -STORE, 140515432558592, 140515434659839, -STORE, 140515432546304, 140515432558591, -SNULL, 140515434651647, 140515434659839, -STORE, 140515432558592, 140515434651647, -STORE, 140515434651648, 140515434659839, -ERASE, 140515434651648, 140515434659839, -STORE, 140515434651648, 140515434659839, -STORE, 140515428749312, 140515432546303, -SNULL, 140515428749312, 140515430408191, -STORE, 140515430408192, 140515432546303, -STORE, 140515428749312, 140515430408191, -SNULL, 140515432505343, 140515432546303, -STORE, 140515430408192, 140515432505343, -STORE, 140515432505344, 140515432546303, -SNULL, 140515432505344, 140515432529919, -STORE, 140515432529920, 140515432546303, -STORE, 140515432505344, 140515432529919, -ERASE, 140515432505344, 140515432529919, -STORE, 140515432505344, 140515432529919, -ERASE, 140515432529920, 140515432546303, -STORE, 140515432529920, 140515432546303, -STORE, 140515436855296, 140515436871679, -SNULL, 140515432521727, 140515432529919, -STORE, 140515432505344, 140515432521727, -STORE, 140515432521728, 140515432529919, -SNULL, 140515434655743, 140515434659839, -STORE, 140515434651648, 140515434655743, -STORE, 140515434655744, 140515434659839, -SNULL, 94310381817855, 94310381826047, -STORE, 94310381813760, 94310381817855, -STORE, 94310381817856, 94310381826047, -SNULL, 140515436904447, 140515436908543, -STORE, 140515436900352, 140515436904447, -STORE, 140515436904448, 140515436908543, -ERASE, 140515436871680, 140515436900351, -STORE, 94310395457536, 94310395592703, -STORE, 140515435171840, 140515436855295, -STORE, 94310395457536, 94310395727871, -STORE, 94310395457536, 94310395863039, -STORE, 94310395457536, 94310396047359, -SNULL, 94310396022783, 94310396047359, -STORE, 94310395457536, 94310396022783, -STORE, 94310396022784, 94310396047359, -ERASE, 94310396022784, 94310396047359, -STORE, 94310395457536, 94310396157951, -STORE, 94310395457536, 94310396293119, -SNULL, 94310396276735, 94310396293119, -STORE, 94310395457536, 94310396276735, -STORE, 94310396276736, 94310396293119, -ERASE, 94310396276736, 94310396293119, -STORE, 94310395457536, 94310396411903, -SNULL, 94310396383231, 94310396411903, -STORE, 94310395457536, 94310396383231, -STORE, 94310396383232, 94310396411903, -ERASE, 94310396383232, 94310396411903, -STORE, 94310395457536, 94310396522495, -STORE, 94310395457536, 94310396674047, -SNULL, 94310396657663, 94310396674047, -STORE, 94310395457536, 94310396657663, -STORE, 94310396657664, 94310396674047, -ERASE, 94310396657664, 94310396674047, -SNULL, 94310396624895, 94310396657663, -STORE, 94310395457536, 94310396624895, -STORE, 94310396624896, 94310396657663, -ERASE, 94310396624896, 94310396657663, -STORE, 94310395457536, 94310396776447, -SNULL, 94310396764159, 94310396776447, -STORE, 94310395457536, 94310396764159, -STORE, 94310396764160, 94310396776447, -ERASE, 94310396764160, 94310396776447, -SNULL, 94310396739583, 94310396764159, -STORE, 94310395457536, 94310396739583, -STORE, 94310396739584, 94310396764159, -ERASE, 94310396739584, 94310396764159, -STORE, 94310395457536, 94310396882943, -STORE, 94310395457536, 94310397018111, -STORE, 94310395457536, 94310397161471, -STORE, 94310395457536, 94310397300735, -SNULL, 94310397292543, 94310397300735, -STORE, 94310395457536, 94310397292543, -STORE, 94310397292544, 94310397300735, -ERASE, 94310397292544, 94310397300735, -STORE, 94359222210560, 94359222423551, -STORE, 94359224520704, 94359224524799, -STORE, 94359224524800, 94359224532991, -STORE, 94359224532992, 94359224545279, -STORE, 94359238348800, 94359239385087, -STORE, 140675699838976, 140675701497855, -STORE, 140675701497856, 140675703595007, -STORE, 140675703595008, 140675703611391, -STORE, 140675703611392, 140675703619583, -STORE, 140675703619584, 140675703635967, -STORE, 140675703635968, 140675703648255, -STORE, 140675703648256, 140675705741311, -STORE, 140675705741312, 140675705745407, -STORE, 140675705745408, 140675705749503, -STORE, 140675705749504, 140675705892863, -STORE, 140675706261504, 140675707944959, -STORE, 140675707944960, 140675707961343, -STORE, 140675707990016, 140675707994111, -STORE, 140675707994112, 140675707998207, -STORE, 140675707998208, 140675708002303, -STORE, 140721324634112, 140721324773375, -STORE, 140721324810240, 140721324822527, -STORE, 140721324822528, 140721324826623, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724099678208, 140737488351231, -SNULL, 140724099686399, 140737488351231, -STORE, 140724099678208, 140724099686399, -STORE, 140724099547136, 140724099686399, -STORE, 94586638516224, 94586640850943, -SNULL, 94586638729215, 94586640850943, -STORE, 94586638516224, 94586638729215, -STORE, 94586638729216, 94586640850943, -ERASE, 94586638729216, 94586640850943, -STORE, 94586640826368, 94586640838655, -STORE, 94586640838656, 94586640850943, -STORE, 140371033796608, 140371036049407, -SNULL, 140371033939967, 140371036049407, -STORE, 140371033796608, 140371033939967, -STORE, 140371033939968, 140371036049407, -ERASE, 140371033939968, 140371036049407, -STORE, 140371036037120, 140371036045311, -STORE, 140371036045312, 140371036049407, -STORE, 140724100001792, 140724100005887, -STORE, 140724099989504, 140724100001791, -STORE, 140371036008448, 140371036037119, -STORE, 140371036000256, 140371036008447, -STORE, 140371031683072, 140371033796607, -SNULL, 140371031683072, 140371031695359, -STORE, 140371031695360, 140371033796607, -STORE, 140371031683072, 140371031695359, -SNULL, 140371033788415, 140371033796607, -STORE, 140371031695360, 140371033788415, -STORE, 140371033788416, 140371033796607, -ERASE, 140371033788416, 140371033796607, -STORE, 140371033788416, 140371033796607, -STORE, 140371027886080, 140371031683071, -SNULL, 140371027886080, 140371029544959, -STORE, 140371029544960, 140371031683071, -STORE, 140371027886080, 140371029544959, -SNULL, 140371031642111, 140371031683071, -STORE, 140371029544960, 140371031642111, -STORE, 140371031642112, 140371031683071, -SNULL, 140371031642112, 140371031666687, -STORE, 140371031666688, 140371031683071, -STORE, 140371031642112, 140371031666687, -ERASE, 140371031642112, 140371031666687, -STORE, 140371031642112, 140371031666687, -ERASE, 140371031666688, 140371031683071, -STORE, 140371031666688, 140371031683071, -STORE, 140371035992064, 140371036008447, -SNULL, 140371031658495, 140371031666687, -STORE, 140371031642112, 140371031658495, -STORE, 140371031658496, 140371031666687, -SNULL, 140371033792511, 140371033796607, -STORE, 140371033788416, 140371033792511, -STORE, 140371033792512, 140371033796607, -SNULL, 94586640830463, 94586640838655, -STORE, 94586640826368, 94586640830463, -STORE, 94586640830464, 94586640838655, -SNULL, 140371036041215, 140371036045311, -STORE, 140371036037120, 140371036041215, -STORE, 140371036041216, 140371036045311, -ERASE, 140371036008448, 140371036037119, -STORE, 94586663849984, 94586663985151, -STORE, 140371034308608, 140371035992063, -STORE, 94586663849984, 94586664120319, -STORE, 94586663849984, 94586664255487, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140727532937216, 140737488351231, -SNULL, 140727532945407, 140737488351231, -STORE, 140727532937216, 140727532945407, -STORE, 140727532806144, 140727532945407, -STORE, 94849780191232, 94849782525951, -SNULL, 94849780404223, 94849782525951, -STORE, 94849780191232, 94849780404223, -STORE, 94849780404224, 94849782525951, -ERASE, 94849780404224, 94849782525951, -STORE, 94849782501376, 94849782513663, -STORE, 94849782513664, 94849782525951, -STORE, 140382070218752, 140382072471551, -SNULL, 140382070362111, 140382072471551, -STORE, 140382070218752, 140382070362111, -STORE, 140382070362112, 140382072471551, -ERASE, 140382070362112, 140382072471551, -STORE, 140382072459264, 140382072467455, -STORE, 140382072467456, 140382072471551, -STORE, 140727533092864, 140727533096959, -STORE, 140727533080576, 140727533092863, -STORE, 140382072430592, 140382072459263, -STORE, 140382072422400, 140382072430591, -STORE, 140382068105216, 140382070218751, -SNULL, 140382068105216, 140382068117503, -STORE, 140382068117504, 140382070218751, -STORE, 140382068105216, 140382068117503, -SNULL, 140382070210559, 140382070218751, -STORE, 140382068117504, 140382070210559, -STORE, 140382070210560, 140382070218751, -ERASE, 140382070210560, 140382070218751, -STORE, 140382070210560, 140382070218751, -STORE, 140382064308224, 140382068105215, -SNULL, 140382064308224, 140382065967103, -STORE, 140382065967104, 140382068105215, -STORE, 140382064308224, 140382065967103, -SNULL, 140382068064255, 140382068105215, -STORE, 140382065967104, 140382068064255, -STORE, 140382068064256, 140382068105215, -SNULL, 140382068064256, 140382068088831, -STORE, 140382068088832, 140382068105215, -STORE, 140382068064256, 140382068088831, -ERASE, 140382068064256, 140382068088831, -STORE, 140382068064256, 140382068088831, -ERASE, 140382068088832, 140382068105215, -STORE, 140382068088832, 140382068105215, -STORE, 140382072414208, 140382072430591, -SNULL, 140382068080639, 140382068088831, -STORE, 140382068064256, 140382068080639, -STORE, 140382068080640, 140382068088831, -SNULL, 140382070214655, 140382070218751, -STORE, 140382070210560, 140382070214655, -STORE, 140382070214656, 140382070218751, -SNULL, 94849782505471, 94849782513663, -STORE, 94849782501376, 94849782505471, -STORE, 94849782505472, 94849782513663, -SNULL, 140382072463359, 140382072467455, -STORE, 140382072459264, 140382072463359, -STORE, 140382072463360, 140382072467455, -ERASE, 140382072430592, 140382072459263, -STORE, 94849782845440, 94849782980607, -STORE, 140382070730752, 140382072414207, -STORE, 94849782845440, 94849783115775, -STORE, 94849782845440, 94849783250943, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722594377728, 140737488351231, -SNULL, 140722594385919, 140737488351231, -STORE, 140722594377728, 140722594385919, -STORE, 140722594246656, 140722594385919, -STORE, 94421466353664, 94421468577791, -SNULL, 94421466464255, 94421468577791, -STORE, 94421466353664, 94421466464255, -STORE, 94421466464256, 94421468577791, -ERASE, 94421466464256, 94421468577791, -STORE, 94421468557312, 94421468569599, -STORE, 94421468569600, 94421468577791, -STORE, 140345458057216, 140345460310015, -SNULL, 140345458200575, 140345460310015, -STORE, 140345458057216, 140345458200575, -STORE, 140345458200576, 140345460310015, -ERASE, 140345458200576, 140345460310015, -STORE, 140345460297728, 140345460305919, -STORE, 140345460305920, 140345460310015, -STORE, 140722595557376, 140722595561471, -STORE, 140722595545088, 140722595557375, -STORE, 140345460269056, 140345460297727, -STORE, 140345460260864, 140345460269055, -STORE, 140345454260224, 140345458057215, -SNULL, 140345454260224, 140345455919103, -STORE, 140345455919104, 140345458057215, -STORE, 140345454260224, 140345455919103, -SNULL, 140345458016255, 140345458057215, -STORE, 140345455919104, 140345458016255, -STORE, 140345458016256, 140345458057215, -SNULL, 140345458016256, 140345458040831, -STORE, 140345458040832, 140345458057215, -STORE, 140345458016256, 140345458040831, -ERASE, 140345458016256, 140345458040831, -STORE, 140345458016256, 140345458040831, -ERASE, 140345458040832, 140345458057215, -STORE, 140345458040832, 140345458057215, -SNULL, 140345458032639, 140345458040831, -STORE, 140345458016256, 140345458032639, -STORE, 140345458032640, 140345458040831, -SNULL, 94421468565503, 94421468569599, -STORE, 94421468557312, 94421468565503, -STORE, 94421468565504, 94421468569599, -SNULL, 140345460301823, 140345460305919, -STORE, 140345460297728, 140345460301823, -STORE, 140345460301824, 140345460305919, -ERASE, 140345460269056, 140345460297727, -STORE, 94421496004608, 94421496139775, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140726096302080, 140737488351231, -SNULL, 140726096310271, 140737488351231, -STORE, 140726096302080, 140726096310271, -STORE, 140726096171008, 140726096310271, -STORE, 94101992124416, 94101994459135, -SNULL, 94101992337407, 94101994459135, -STORE, 94101992124416, 94101992337407, -STORE, 94101992337408, 94101994459135, -ERASE, 94101992337408, 94101994459135, -STORE, 94101994434560, 94101994446847, -STORE, 94101994446848, 94101994459135, -STORE, 140192085594112, 140192087846911, -SNULL, 140192085737471, 140192087846911, -STORE, 140192085594112, 140192085737471, -STORE, 140192085737472, 140192087846911, -ERASE, 140192085737472, 140192087846911, -STORE, 140192087834624, 140192087842815, -STORE, 140192087842816, 140192087846911, -STORE, 140726096375808, 140726096379903, -STORE, 140726096363520, 140726096375807, -STORE, 140192087805952, 140192087834623, -STORE, 140192087797760, 140192087805951, -STORE, 140192083480576, 140192085594111, -SNULL, 140192083480576, 140192083492863, -STORE, 140192083492864, 140192085594111, -STORE, 140192083480576, 140192083492863, -SNULL, 140192085585919, 140192085594111, -STORE, 140192083492864, 140192085585919, -STORE, 140192085585920, 140192085594111, -ERASE, 140192085585920, 140192085594111, -STORE, 140192085585920, 140192085594111, -STORE, 140192079683584, 140192083480575, -SNULL, 140192079683584, 140192081342463, -STORE, 140192081342464, 140192083480575, -STORE, 140192079683584, 140192081342463, -SNULL, 140192083439615, 140192083480575, -STORE, 140192081342464, 140192083439615, -STORE, 140192083439616, 140192083480575, -SNULL, 140192083439616, 140192083464191, -STORE, 140192083464192, 140192083480575, -STORE, 140192083439616, 140192083464191, -ERASE, 140192083439616, 140192083464191, -STORE, 140192083439616, 140192083464191, -ERASE, 140192083464192, 140192083480575, -STORE, 140192083464192, 140192083480575, -STORE, 140192087789568, 140192087805951, -SNULL, 140192083455999, 140192083464191, -STORE, 140192083439616, 140192083455999, -STORE, 140192083456000, 140192083464191, -SNULL, 140192085590015, 140192085594111, -STORE, 140192085585920, 140192085590015, -STORE, 140192085590016, 140192085594111, -SNULL, 94101994438655, 94101994446847, -STORE, 94101994434560, 94101994438655, -STORE, 94101994438656, 94101994446847, -SNULL, 140192087838719, 140192087842815, -STORE, 140192087834624, 140192087838719, -STORE, 140192087838720, 140192087842815, -ERASE, 140192087805952, 140192087834623, -STORE, 94102011887616, 94102012022783, -STORE, 140192086106112, 140192087789567, -STORE, 94102011887616, 94102012157951, -STORE, 94102011887616, 94102012293119, -STORE, 94102011887616, 94102012440575, -SNULL, 94102012428287, 94102012440575, -STORE, 94102011887616, 94102012428287, -STORE, 94102012428288, 94102012440575, -ERASE, 94102012428288, 94102012440575, -STORE, 94102011887616, 94102012579839, -STORE, 94102011887616, 94102012715007, -SNULL, 94102012694527, 94102012715007, -STORE, 94102011887616, 94102012694527, -STORE, 94102012694528, 94102012715007, -ERASE, 94102012694528, 94102012715007, -STORE, 94102011887616, 94102012833791, -STORE, 94102011887616, 94102012968959, -SNULL, 94102012927999, 94102012968959, -STORE, 94102011887616, 94102012927999, -STORE, 94102012928000, 94102012968959, -ERASE, 94102012928000, 94102012968959, -STORE, 94102011887616, 94102013091839, -SNULL, 94102013075455, 94102013091839, -STORE, 94102011887616, 94102013075455, -STORE, 94102013075456, 94102013091839, -ERASE, 94102013075456, 94102013091839, -STORE, 94102011887616, 94102013210623, -STORE, 94102011887616, 94102013345791, -STORE, 93968727965696, 93968728178687, -STORE, 93968730275840, 93968730279935, -STORE, 93968730279936, 93968730288127, -STORE, 93968730288128, 93968730300415, -STORE, 93968731140096, 93968732704767, -STORE, 140588443168768, 140588444827647, -STORE, 140588444827648, 140588446924799, -STORE, 140588446924800, 140588446941183, -STORE, 140588446941184, 140588446949375, -STORE, 140588446949376, 140588446965759, -STORE, 140588446965760, 140588446978047, -STORE, 140588446978048, 140588449071103, -STORE, 140588449071104, 140588449075199, -STORE, 140588449075200, 140588449079295, -STORE, 140588449079296, 140588449222655, -STORE, 140588449591296, 140588451274751, -STORE, 140588451274752, 140588451291135, -STORE, 140588451319808, 140588451323903, -STORE, 140588451323904, 140588451327999, -STORE, 140588451328000, 140588451332095, -STORE, 140733877239808, 140733877379071, -STORE, 140733878702080, 140733878714367, -STORE, 140733878714368, 140733878718463, -STORE, 93968727965696, 93968728178687, -STORE, 93968730275840, 93968730279935, -STORE, 93968730279936, 93968730288127, -STORE, 93968730288128, 93968730300415, -STORE, 93968731140096, 93968732991487, -STORE, 140588443168768, 140588444827647, -STORE, 140588444827648, 140588446924799, -STORE, 140588446924800, 140588446941183, -STORE, 140588446941184, 140588446949375, -STORE, 140588446949376, 140588446965759, -STORE, 140588446965760, 140588446978047, -STORE, 140588446978048, 140588449071103, -STORE, 140588449071104, 140588449075199, -STORE, 140588449075200, 140588449079295, -STORE, 140588449079296, 140588449222655, -STORE, 140588449591296, 140588451274751, -STORE, 140588451274752, 140588451291135, -STORE, 140588451319808, 140588451323903, -STORE, 140588451323904, 140588451327999, -STORE, 140588451328000, 140588451332095, -STORE, 140733877239808, 140733877379071, -STORE, 140733878702080, 140733878714367, -STORE, 140733878714368, 140733878718463, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733054472192, 140737488351231, -SNULL, 140733054480383, 140737488351231, -STORE, 140733054472192, 140733054480383, -STORE, 140733054341120, 140733054480383, -STORE, 93992873623552, 93992875847679, -SNULL, 93992873734143, 93992875847679, -STORE, 93992873623552, 93992873734143, -STORE, 93992873734144, 93992875847679, -ERASE, 93992873734144, 93992875847679, -STORE, 93992875827200, 93992875839487, -STORE, 93992875839488, 93992875847679, -STORE, 139790881488896, 139790883741695, -SNULL, 139790881632255, 139790883741695, -STORE, 139790881488896, 139790881632255, -STORE, 139790881632256, 139790883741695, -ERASE, 139790881632256, 139790883741695, -STORE, 139790883729408, 139790883737599, -STORE, 139790883737600, 139790883741695, -STORE, 140733054754816, 140733054758911, -STORE, 140733054742528, 140733054754815, -STORE, 139790883700736, 139790883729407, -STORE, 139790883692544, 139790883700735, -STORE, 139790877691904, 139790881488895, -SNULL, 139790877691904, 139790879350783, -STORE, 139790879350784, 139790881488895, -STORE, 139790877691904, 139790879350783, -SNULL, 139790881447935, 139790881488895, -STORE, 139790879350784, 139790881447935, -STORE, 139790881447936, 139790881488895, -SNULL, 139790881447936, 139790881472511, -STORE, 139790881472512, 139790881488895, -STORE, 139790881447936, 139790881472511, -ERASE, 139790881447936, 139790881472511, -STORE, 139790881447936, 139790881472511, -ERASE, 139790881472512, 139790881488895, -STORE, 139790881472512, 139790881488895, -SNULL, 139790881464319, 139790881472511, -STORE, 139790881447936, 139790881464319, -STORE, 139790881464320, 139790881472511, -SNULL, 93992875835391, 93992875839487, -STORE, 93992875827200, 93992875835391, -STORE, 93992875835392, 93992875839487, -SNULL, 139790883733503, 139790883737599, -STORE, 139790883729408, 139790883733503, -STORE, 139790883733504, 139790883737599, -ERASE, 139790883700736, 139790883729407, -STORE, 93992877031424, 93992877166591, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728550887424, 140737488351231, -SNULL, 140728550895615, 140737488351231, -STORE, 140728550887424, 140728550895615, -STORE, 140728550756352, 140728550895615, -STORE, 94707634077696, 94707636301823, -SNULL, 94707634188287, 94707636301823, -STORE, 94707634077696, 94707634188287, -STORE, 94707634188288, 94707636301823, -ERASE, 94707634188288, 94707636301823, -STORE, 94707636281344, 94707636293631, -STORE, 94707636293632, 94707636301823, -STORE, 140553545666560, 140553547919359, -SNULL, 140553545809919, 140553547919359, -STORE, 140553545666560, 140553545809919, -STORE, 140553545809920, 140553547919359, -ERASE, 140553545809920, 140553547919359, -STORE, 140553547907072, 140553547915263, -STORE, 140553547915264, 140553547919359, -STORE, 140728552374272, 140728552378367, -STORE, 140728552361984, 140728552374271, -STORE, 140553547878400, 140553547907071, -STORE, 140553547870208, 140553547878399, -STORE, 140553541869568, 140553545666559, -SNULL, 140553541869568, 140553543528447, -STORE, 140553543528448, 140553545666559, -STORE, 140553541869568, 140553543528447, -SNULL, 140553545625599, 140553545666559, -STORE, 140553543528448, 140553545625599, -STORE, 140553545625600, 140553545666559, -SNULL, 140553545625600, 140553545650175, -STORE, 140553545650176, 140553545666559, -STORE, 140553545625600, 140553545650175, -ERASE, 140553545625600, 140553545650175, -STORE, 140553545625600, 140553545650175, -ERASE, 140553545650176, 140553545666559, -STORE, 140553545650176, 140553545666559, -SNULL, 140553545641983, 140553545650175, -STORE, 140553545625600, 140553545641983, -STORE, 140553545641984, 140553545650175, -SNULL, 94707636289535, 94707636293631, -STORE, 94707636281344, 94707636289535, -STORE, 94707636289536, 94707636293631, -SNULL, 140553547911167, 140553547915263, -STORE, 140553547907072, 140553547911167, -STORE, 140553547911168, 140553547915263, -ERASE, 140553547878400, 140553547907071, -STORE, 94707651411968, 94707651547135, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140732168695808, 140737488351231, -SNULL, 140732168703999, 140737488351231, -STORE, 140732168695808, 140732168703999, -STORE, 140732168564736, 140732168703999, -STORE, 94454287859712, 94454290083839, -SNULL, 94454287970303, 94454290083839, -STORE, 94454287859712, 94454287970303, -STORE, 94454287970304, 94454290083839, -ERASE, 94454287970304, 94454290083839, -STORE, 94454290063360, 94454290075647, -STORE, 94454290075648, 94454290083839, -STORE, 140564947107840, 140564949360639, -SNULL, 140564947251199, 140564949360639, -STORE, 140564947107840, 140564947251199, -STORE, 140564947251200, 140564949360639, -ERASE, 140564947251200, 140564949360639, -STORE, 140564949348352, 140564949356543, -STORE, 140564949356544, 140564949360639, -STORE, 140732168843264, 140732168847359, -STORE, 140732168830976, 140732168843263, -STORE, 140564949319680, 140564949348351, -STORE, 140564949311488, 140564949319679, -STORE, 140564943310848, 140564947107839, -SNULL, 140564943310848, 140564944969727, -STORE, 140564944969728, 140564947107839, -STORE, 140564943310848, 140564944969727, -SNULL, 140564947066879, 140564947107839, -STORE, 140564944969728, 140564947066879, -STORE, 140564947066880, 140564947107839, -SNULL, 140564947066880, 140564947091455, -STORE, 140564947091456, 140564947107839, -STORE, 140564947066880, 140564947091455, -ERASE, 140564947066880, 140564947091455, -STORE, 140564947066880, 140564947091455, -ERASE, 140564947091456, 140564947107839, -STORE, 140564947091456, 140564947107839, -SNULL, 140564947083263, 140564947091455, -STORE, 140564947066880, 140564947083263, -STORE, 140564947083264, 140564947091455, -SNULL, 94454290071551, 94454290075647, -STORE, 94454290063360, 94454290071551, -STORE, 94454290071552, 94454290075647, -SNULL, 140564949352447, 140564949356543, -STORE, 140564949348352, 140564949352447, -STORE, 140564949352448, 140564949356543, -ERASE, 140564949319680, 140564949348351, -STORE, 94454316236800, 94454316371967, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735155617792, 140737488351231, -SNULL, 140735155625983, 140737488351231, -STORE, 140735155617792, 140735155625983, -STORE, 140735155486720, 140735155625983, -STORE, 93915969556480, 93915971780607, -SNULL, 93915969667071, 93915971780607, -STORE, 93915969556480, 93915969667071, -STORE, 93915969667072, 93915971780607, -ERASE, 93915969667072, 93915971780607, -STORE, 93915971760128, 93915971772415, -STORE, 93915971772416, 93915971780607, -STORE, 140141164605440, 140141166858239, -SNULL, 140141164748799, 140141166858239, -STORE, 140141164605440, 140141164748799, -STORE, 140141164748800, 140141166858239, -ERASE, 140141164748800, 140141166858239, -STORE, 140141166845952, 140141166854143, -STORE, 140141166854144, 140141166858239, -STORE, 140735155691520, 140735155695615, -STORE, 140735155679232, 140735155691519, -STORE, 140141166817280, 140141166845951, -STORE, 140141166809088, 140141166817279, -STORE, 140141160808448, 140141164605439, -SNULL, 140141160808448, 140141162467327, -STORE, 140141162467328, 140141164605439, -STORE, 140141160808448, 140141162467327, -SNULL, 140141164564479, 140141164605439, -STORE, 140141162467328, 140141164564479, -STORE, 140141164564480, 140141164605439, -SNULL, 140141164564480, 140141164589055, -STORE, 140141164589056, 140141164605439, -STORE, 140141164564480, 140141164589055, -ERASE, 140141164564480, 140141164589055, -STORE, 140141164564480, 140141164589055, -ERASE, 140141164589056, 140141164605439, -STORE, 140141164589056, 140141164605439, -SNULL, 140141164580863, 140141164589055, -STORE, 140141164564480, 140141164580863, -STORE, 140141164580864, 140141164589055, -SNULL, 93915971768319, 93915971772415, -STORE, 93915971760128, 93915971768319, -STORE, 93915971768320, 93915971772415, -SNULL, 140141166850047, 140141166854143, -STORE, 140141166845952, 140141166850047, -STORE, 140141166850048, 140141166854143, -ERASE, 140141166817280, 140141166845951, -STORE, 93916002775040, 93916002910207, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728988409856, 140737488351231, -SNULL, 140728988418047, 140737488351231, -STORE, 140728988409856, 140728988418047, -STORE, 140728988278784, 140728988418047, -STORE, 94021634813952, 94021637038079, -SNULL, 94021634924543, 94021637038079, -STORE, 94021634813952, 94021634924543, -STORE, 94021634924544, 94021637038079, -ERASE, 94021634924544, 94021637038079, -STORE, 94021637017600, 94021637029887, -STORE, 94021637029888, 94021637038079, -STORE, 140638014038016, 140638016290815, -SNULL, 140638014181375, 140638016290815, -STORE, 140638014038016, 140638014181375, -STORE, 140638014181376, 140638016290815, -ERASE, 140638014181376, 140638016290815, -STORE, 140638016278528, 140638016286719, -STORE, 140638016286720, 140638016290815, -STORE, 140728988536832, 140728988540927, -STORE, 140728988524544, 140728988536831, -STORE, 140638016249856, 140638016278527, -STORE, 140638016241664, 140638016249855, -STORE, 140638010241024, 140638014038015, -SNULL, 140638010241024, 140638011899903, -STORE, 140638011899904, 140638014038015, -STORE, 140638010241024, 140638011899903, -SNULL, 140638013997055, 140638014038015, -STORE, 140638011899904, 140638013997055, -STORE, 140638013997056, 140638014038015, -SNULL, 140638013997056, 140638014021631, -STORE, 140638014021632, 140638014038015, -STORE, 140638013997056, 140638014021631, -ERASE, 140638013997056, 140638014021631, -STORE, 140638013997056, 140638014021631, -ERASE, 140638014021632, 140638014038015, -STORE, 140638014021632, 140638014038015, -SNULL, 140638014013439, 140638014021631, -STORE, 140638013997056, 140638014013439, -STORE, 140638014013440, 140638014021631, -SNULL, 94021637025791, 94021637029887, -STORE, 94021637017600, 94021637025791, -STORE, 94021637025792, 94021637029887, -SNULL, 140638016282623, 140638016286719, -STORE, 140638016278528, 140638016282623, -STORE, 140638016282624, 140638016286719, -ERASE, 140638016249856, 140638016278527, -STORE, 94021643124736, 94021643259903, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731219275776, 140737488351231, -SNULL, 140731219283967, 140737488351231, -STORE, 140731219275776, 140731219283967, -STORE, 140731219144704, 140731219283967, -STORE, 93888803647488, 93888805871615, -SNULL, 93888803758079, 93888805871615, -STORE, 93888803647488, 93888803758079, -STORE, 93888803758080, 93888805871615, -ERASE, 93888803758080, 93888805871615, -STORE, 93888805851136, 93888805863423, -STORE, 93888805863424, 93888805871615, -STORE, 139630576934912, 139630579187711, -SNULL, 139630577078271, 139630579187711, -STORE, 139630576934912, 139630577078271, -STORE, 139630577078272, 139630579187711, -ERASE, 139630577078272, 139630579187711, -STORE, 139630579175424, 139630579183615, -STORE, 139630579183616, 139630579187711, -STORE, 140731219718144, 140731219722239, -STORE, 140731219705856, 140731219718143, -STORE, 139630579146752, 139630579175423, -STORE, 139630579138560, 139630579146751, -STORE, 139630573137920, 139630576934911, -SNULL, 139630573137920, 139630574796799, -STORE, 139630574796800, 139630576934911, -STORE, 139630573137920, 139630574796799, -SNULL, 139630576893951, 139630576934911, -STORE, 139630574796800, 139630576893951, -STORE, 139630576893952, 139630576934911, -SNULL, 139630576893952, 139630576918527, -STORE, 139630576918528, 139630576934911, -STORE, 139630576893952, 139630576918527, -ERASE, 139630576893952, 139630576918527, -STORE, 139630576893952, 139630576918527, -ERASE, 139630576918528, 139630576934911, -STORE, 139630576918528, 139630576934911, -SNULL, 139630576910335, 139630576918527, -STORE, 139630576893952, 139630576910335, -STORE, 139630576910336, 139630576918527, -SNULL, 93888805859327, 93888805863423, -STORE, 93888805851136, 93888805859327, -STORE, 93888805859328, 93888805863423, -SNULL, 139630579179519, 139630579183615, -STORE, 139630579175424, 139630579179519, -STORE, 139630579179520, 139630579183615, -ERASE, 139630579146752, 139630579175423, -STORE, 93888822235136, 93888822370303, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733391151104, 140737488351231, -SNULL, 140733391159295, 140737488351231, -STORE, 140733391151104, 140733391159295, -STORE, 140733391020032, 140733391159295, -STORE, 94393875324928, 94393877549055, -SNULL, 94393875435519, 94393877549055, -STORE, 94393875324928, 94393875435519, -STORE, 94393875435520, 94393877549055, -ERASE, 94393875435520, 94393877549055, -STORE, 94393877528576, 94393877540863, -STORE, 94393877540864, 94393877549055, -STORE, 140292111740928, 140292113993727, -SNULL, 140292111884287, 140292113993727, -STORE, 140292111740928, 140292111884287, -STORE, 140292111884288, 140292113993727, -ERASE, 140292111884288, 140292113993727, -STORE, 140292113981440, 140292113989631, -STORE, 140292113989632, 140292113993727, -STORE, 140733391532032, 140733391536127, -STORE, 140733391519744, 140733391532031, -STORE, 140292113952768, 140292113981439, -STORE, 140292113944576, 140292113952767, -STORE, 140292107943936, 140292111740927, -SNULL, 140292107943936, 140292109602815, -STORE, 140292109602816, 140292111740927, -STORE, 140292107943936, 140292109602815, -SNULL, 140292111699967, 140292111740927, -STORE, 140292109602816, 140292111699967, -STORE, 140292111699968, 140292111740927, -SNULL, 140292111699968, 140292111724543, -STORE, 140292111724544, 140292111740927, -STORE, 140292111699968, 140292111724543, -ERASE, 140292111699968, 140292111724543, -STORE, 140292111699968, 140292111724543, -ERASE, 140292111724544, 140292111740927, -STORE, 140292111724544, 140292111740927, -SNULL, 140292111716351, 140292111724543, -STORE, 140292111699968, 140292111716351, -STORE, 140292111716352, 140292111724543, -SNULL, 94393877536767, 94393877540863, -STORE, 94393877528576, 94393877536767, -STORE, 94393877536768, 94393877540863, -SNULL, 140292113985535, 140292113989631, -STORE, 140292113981440, 140292113985535, -STORE, 140292113985536, 140292113989631, -ERASE, 140292113952768, 140292113981439, -STORE, 94393909342208, 94393909477375, -STORE, 94458367512576, 94458367725567, -STORE, 94458369822720, 94458369826815, -STORE, 94458369826816, 94458369835007, -STORE, 94458369835008, 94458369847295, -STORE, 94458393292800, 94458399666175, -STORE, 140619773841408, 140619775500287, -STORE, 140619775500288, 140619777597439, -STORE, 140619777597440, 140619777613823, -STORE, 140619777613824, 140619777622015, -STORE, 140619777622016, 140619777638399, -STORE, 140619777638400, 140619777650687, -STORE, 140619777650688, 140619779743743, -STORE, 140619779743744, 140619779747839, -STORE, 140619779747840, 140619779751935, -STORE, 140619779751936, 140619779895295, -STORE, 140619780263936, 140619781947391, -STORE, 140619781947392, 140619781963775, -STORE, 140619781992448, 140619781996543, -STORE, 140619781996544, 140619782000639, -STORE, 140619782000640, 140619782004735, -STORE, 140725811675136, 140725811814399, -STORE, 140725812813824, 140725812826111, -STORE, 140725812826112, 140725812830207, -STORE, 94458367512576, 94458367725567, -STORE, 94458369822720, 94458369826815, -STORE, 94458369826816, 94458369835007, -STORE, 94458369835008, 94458369847295, -STORE, 94458393292800, 94458400366591, -STORE, 140619773841408, 140619775500287, -STORE, 140619775500288, 140619777597439, -STORE, 140619777597440, 140619777613823, -STORE, 140619777613824, 140619777622015, -STORE, 140619777622016, 140619777638399, -STORE, 140619777638400, 140619777650687, -STORE, 140619777650688, 140619779743743, -STORE, 140619779743744, 140619779747839, -STORE, 140619779747840, 140619779751935, -STORE, 140619779751936, 140619779895295, -STORE, 140619780263936, 140619781947391, -STORE, 140619781947392, 140619781963775, -STORE, 140619781992448, 140619781996543, -STORE, 140619781996544, 140619782000639, -STORE, 140619782000640, 140619782004735, -STORE, 140725811675136, 140725811814399, -STORE, 140725812813824, 140725812826111, -STORE, 140725812826112, 140725812830207, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728740679680, 140737488351231, -SNULL, 140728740687871, 140737488351231, -STORE, 140728740679680, 140728740687871, -STORE, 140728740548608, 140728740687871, -STORE, 94764075249664, 94764077473791, -SNULL, 94764075360255, 94764077473791, -STORE, 94764075249664, 94764075360255, -STORE, 94764075360256, 94764077473791, -ERASE, 94764075360256, 94764077473791, -STORE, 94764077453312, 94764077465599, -STORE, 94764077465600, 94764077473791, -STORE, 139766406791168, 139766409043967, -SNULL, 139766406934527, 139766409043967, -STORE, 139766406791168, 139766406934527, -STORE, 139766406934528, 139766409043967, -ERASE, 139766406934528, 139766409043967, -STORE, 139766409031680, 139766409039871, -STORE, 139766409039872, 139766409043967, -STORE, 140728740913152, 140728740917247, -STORE, 140728740900864, 140728740913151, -STORE, 139766409003008, 139766409031679, -STORE, 139766408994816, 139766409003007, -STORE, 139766402994176, 139766406791167, -SNULL, 139766402994176, 139766404653055, -STORE, 139766404653056, 139766406791167, -STORE, 139766402994176, 139766404653055, -SNULL, 139766406750207, 139766406791167, -STORE, 139766404653056, 139766406750207, -STORE, 139766406750208, 139766406791167, -SNULL, 139766406750208, 139766406774783, -STORE, 139766406774784, 139766406791167, -STORE, 139766406750208, 139766406774783, -ERASE, 139766406750208, 139766406774783, -STORE, 139766406750208, 139766406774783, -ERASE, 139766406774784, 139766406791167, -STORE, 139766406774784, 139766406791167, -SNULL, 139766406766591, 139766406774783, -STORE, 139766406750208, 139766406766591, -STORE, 139766406766592, 139766406774783, -SNULL, 94764077461503, 94764077465599, -STORE, 94764077453312, 94764077461503, -STORE, 94764077461504, 94764077465599, -SNULL, 139766409035775, 139766409039871, -STORE, 139766409031680, 139766409035775, -STORE, 139766409035776, 139766409039871, -ERASE, 139766409003008, 139766409031679, -STORE, 94764090458112, 94764090593279, -STORE, 94758057480192, 94758057590783, -STORE, 94758059683840, 94758059692031, -STORE, 94758059692032, 94758059696127, -STORE, 94758059696128, 94758059704319, -STORE, 94758083215360, 94758083350527, -STORE, 139951456772096, 139951458430975, -STORE, 139951458430976, 139951460528127, -STORE, 139951460528128, 139951460544511, -STORE, 139951460544512, 139951460552703, -STORE, 139951460552704, 139951460569087, -STORE, 139951460569088, 139951460712447, -STORE, 139951462772736, 139951462780927, -STORE, 139951462809600, 139951462813695, -STORE, 139951462813696, 139951462817791, -STORE, 139951462817792, 139951462821887, -STORE, 140734098313216, 140734098452479, -STORE, 140734098911232, 140734098923519, -STORE, 140734098923520, 140734098927615, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724904095744, 140737488351231, -SNULL, 140724904103935, 140737488351231, -STORE, 140724904095744, 140724904103935, -STORE, 140724903964672, 140724904103935, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 140408497864704, 140408500117503, -SNULL, 140408498008063, 140408500117503, -STORE, 140408497864704, 140408498008063, -STORE, 140408498008064, 140408500117503, -ERASE, 140408498008064, 140408500117503, -STORE, 140408500105216, 140408500113407, -STORE, 140408500113408, 140408500117503, -STORE, 140724905369600, 140724905373695, -STORE, 140724905357312, 140724905369599, -STORE, 140408500076544, 140408500105215, -STORE, 140408500068352, 140408500076543, -STORE, 140408494702592, 140408497864703, -SNULL, 140408494702592, 140408495763455, -STORE, 140408495763456, 140408497864703, -STORE, 140408494702592, 140408495763455, -SNULL, 140408497856511, 140408497864703, -STORE, 140408495763456, 140408497856511, -STORE, 140408497856512, 140408497864703, -ERASE, 140408497856512, 140408497864703, -STORE, 140408497856512, 140408497864703, -STORE, 140408490905600, 140408494702591, -SNULL, 140408490905600, 140408492564479, -STORE, 140408492564480, 140408494702591, -STORE, 140408490905600, 140408492564479, -SNULL, 140408494661631, 140408494702591, -STORE, 140408492564480, 140408494661631, -STORE, 140408494661632, 140408494702591, -SNULL, 140408494661632, 140408494686207, -STORE, 140408494686208, 140408494702591, -STORE, 140408494661632, 140408494686207, -ERASE, 140408494661632, 140408494686207, -STORE, 140408494661632, 140408494686207, -ERASE, 140408494686208, 140408494702591, -STORE, 140408494686208, 140408494702591, -STORE, 140408500056064, 140408500076543, -SNULL, 140408494678015, 140408494686207, -STORE, 140408494661632, 140408494678015, -STORE, 140408494678016, 140408494686207, -SNULL, 140408497860607, 140408497864703, -STORE, 140408497856512, 140408497860607, -STORE, 140408497860608, 140408497864703, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 140408500109311, 140408500113407, -STORE, 140408500105216, 140408500109311, -STORE, 140408500109312, 140408500113407, -ERASE, 140408500076544, 140408500105215, -STORE, 25235456, 25370623, -STORE, 25235456, 25518079, -STORE, 140408498372608, 140408500056063, -STORE, 94543937388544, 94543937499135, -STORE, 94543939592192, 94543939600383, -STORE, 94543939600384, 94543939604479, -STORE, 94543939604480, 94543939612671, -STORE, 94543941447680, 94543941582847, -STORE, 140282621947904, 140282623606783, -STORE, 140282623606784, 140282625703935, -STORE, 140282625703936, 140282625720319, -STORE, 140282625720320, 140282625728511, -STORE, 140282625728512, 140282625744895, -STORE, 140282625744896, 140282625888255, -STORE, 140282627948544, 140282627956735, -STORE, 140282627985408, 140282627989503, -STORE, 140282627989504, 140282627993599, -STORE, 140282627993600, 140282627997695, -STORE, 140728295723008, 140728295862271, -STORE, 140728296476672, 140728296488959, -STORE, 140728296488960, 140728296493055, -STORE, 94431504838656, 94431505051647, -STORE, 94431507148800, 94431507152895, -STORE, 94431507152896, 94431507161087, -STORE, 94431507161088, 94431507173375, -STORE, 94431510286336, 94431510691839, -STORE, 139818797948928, 139818799607807, -STORE, 139818799607808, 139818801704959, -STORE, 139818801704960, 139818801721343, -STORE, 139818801721344, 139818801729535, -STORE, 139818801729536, 139818801745919, -STORE, 139818801745920, 139818801758207, -STORE, 139818801758208, 139818803851263, -STORE, 139818803851264, 139818803855359, -STORE, 139818803855360, 139818803859455, -STORE, 139818803859456, 139818804002815, -STORE, 139818804371456, 139818806054911, -STORE, 139818806054912, 139818806071295, -STORE, 139818806099968, 139818806104063, -STORE, 139818806104064, 139818806108159, -STORE, 139818806108160, 139818806112255, -STORE, 140731430457344, 140731430596607, -STORE, 140731431227392, 140731431239679, -STORE, 140731431239680, 140731431243775, -STORE, 94431504838656, 94431505051647, -STORE, 94431507148800, 94431507152895, -STORE, 94431507152896, 94431507161087, -STORE, 94431507161088, 94431507173375, -STORE, 94431510286336, 94431510691839, -STORE, 139818797948928, 139818799607807, -STORE, 139818799607808, 139818801704959, -STORE, 139818801704960, 139818801721343, -STORE, 139818801721344, 139818801729535, -STORE, 139818801729536, 139818801745919, -STORE, 139818801745920, 139818801758207, -STORE, 139818801758208, 139818803851263, -STORE, 139818803851264, 139818803855359, -STORE, 139818803855360, 139818803859455, -STORE, 139818803859456, 139818804002815, -STORE, 139818804371456, 139818806054911, -STORE, 139818806054912, 139818806071295, -STORE, 139818806099968, 139818806104063, -STORE, 139818806104064, 139818806108159, -STORE, 139818806108160, 139818806112255, -STORE, 140731430457344, 140731430596607, -STORE, 140731431227392, 140731431239679, -STORE, 140731431239680, 140731431243775, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140737488338944, 140737488351231, -STORE, 140736944451584, 140737488351231, -SNULL, 140736944463871, 140737488351231, -STORE, 140736944451584, 140736944463871, -STORE, 140736944320512, 140736944463871, -STORE, 4194304, 26279935, -STORE, 28372992, 28454911, -STORE, 28454912, 29806591, -STORE, 139693609893888, 139693612146687, -SNULL, 139693610037247, 139693612146687, -STORE, 139693609893888, 139693610037247, -STORE, 139693610037248, 139693612146687, -ERASE, 139693610037248, 139693612146687, -STORE, 139693612134400, 139693612142591, -STORE, 139693612142592, 139693612146687, -STORE, 140736945152000, 140736945156095, -STORE, 140736945139712, 140736945151999, -STORE, 139693612105728, 139693612134399, -STORE, 139693612097536, 139693612105727, -STORE, 139693606060032, 139693609893887, -SNULL, 139693606060032, 139693607768063, -STORE, 139693607768064, 139693609893887, -STORE, 139693606060032, 139693607768063, -SNULL, 139693609861119, 139693609893887, -STORE, 139693607768064, 139693609861119, -STORE, 139693609861120, 139693609893887, -ERASE, 139693609861120, 139693609893887, -STORE, 139693609861120, 139693609893887, -STORE, 139693603864576, 139693606060031, -SNULL, 139693603864576, 139693603958783, -STORE, 139693603958784, 139693606060031, -STORE, 139693603864576, 139693603958783, -SNULL, 139693606051839, 139693606060031, -STORE, 139693603958784, 139693606051839, -STORE, 139693606051840, 139693606060031, -ERASE, 139693606051840, 139693606060031, -STORE, 139693606051840, 139693606060031, -STORE, 139693601345536, 139693603864575, -SNULL, 139693601345536, 139693601759231, -STORE, 139693601759232, 139693603864575, -STORE, 139693601345536, 139693601759231, -SNULL, 139693603852287, 139693603864575, -STORE, 139693601759232, 139693603852287, -STORE, 139693603852288, 139693603864575, -ERASE, 139693603852288, 139693603864575, -STORE, 139693603852288, 139693603864575, -STORE, 139693598711808, 139693601345535, -SNULL, 139693598711808, 139693599240191, -STORE, 139693599240192, 139693601345535, -STORE, 139693598711808, 139693599240191, -SNULL, 139693601337343, 139693601345535, -STORE, 139693599240192, 139693601337343, -STORE, 139693601337344, 139693601345535, -ERASE, 139693601337344, 139693601345535, -STORE, 139693601337344, 139693601345535, -STORE, 139693596598272, 139693598711807, -SNULL, 139693596598272, 139693596610559, -STORE, 139693596610560, 139693598711807, -STORE, 139693596598272, 139693596610559, -SNULL, 139693598703615, 139693598711807, -STORE, 139693596610560, 139693598703615, -STORE, 139693598703616, 139693598711807, -ERASE, 139693598703616, 139693598711807, -STORE, 139693598703616, 139693598711807, -STORE, 139693594394624, 139693596598271, -SNULL, 139693594394624, 139693594497023, -STORE, 139693594497024, 139693596598271, -STORE, 139693594394624, 139693594497023, -SNULL, 139693596590079, 139693596598271, -STORE, 139693594497024, 139693596590079, -STORE, 139693596590080, 139693596598271, -ERASE, 139693596590080, 139693596598271, -STORE, 139693596590080, 139693596598271, -STORE, 139693612089344, 139693612105727, -STORE, 139693591232512, 139693594394623, -SNULL, 139693591232512, 139693592293375, -STORE, 139693592293376, 139693594394623, -STORE, 139693591232512, 139693592293375, -SNULL, 139693594386431, 139693594394623, -STORE, 139693592293376, 139693594386431, -STORE, 139693594386432, 139693594394623, -ERASE, 139693594386432, 139693594394623, -STORE, 139693594386432, 139693594394623, -STORE, 139693587435520, 139693591232511, -SNULL, 139693587435520, 139693589094399, -STORE, 139693589094400, 139693591232511, -STORE, 139693587435520, 139693589094399, -SNULL, 139693591191551, 139693591232511, -STORE, 139693589094400, 139693591191551, -STORE, 139693591191552, 139693591232511, -SNULL, 139693591191552, 139693591216127, -STORE, 139693591216128, 139693591232511, -STORE, 139693591191552, 139693591216127, -ERASE, 139693591191552, 139693591216127, -STORE, 139693591191552, 139693591216127, -ERASE, 139693591216128, 139693591232511, -STORE, 139693591216128, 139693591232511, -STORE, 139693612077056, 139693612105727, -SNULL, 139693591207935, 139693591216127, -STORE, 139693591191552, 139693591207935, -STORE, 139693591207936, 139693591216127, -SNULL, 139693594390527, 139693594394623, -STORE, 139693594386432, 139693594390527, -STORE, 139693594390528, 139693594394623, -SNULL, 139693596594175, 139693596598271, -STORE, 139693596590080, 139693596594175, -STORE, 139693596594176, 139693596598271, -SNULL, 139693598707711, 139693598711807, -STORE, 139693598703616, 139693598707711, -STORE, 139693598707712, 139693598711807, -SNULL, 139693601341439, 139693601345535, -STORE, 139693601337344, 139693601341439, -STORE, 139693601341440, 139693601345535, -SNULL, 139693603860479, 139693603864575, -STORE, 139693603852288, 139693603860479, -STORE, 139693603860480, 139693603864575, -SNULL, 139693606055935, 139693606060031, -STORE, 139693606051840, 139693606055935, -STORE, 139693606055936, 139693606060031, -SNULL, 139693609865215, 139693609893887, -STORE, 139693609861120, 139693609865215, -STORE, 139693609865216, 139693609893887, -SNULL, 28405759, 28454911, -STORE, 28372992, 28405759, -STORE, 28405760, 28454911, -SNULL, 139693612138495, 139693612142591, -STORE, 139693612134400, 139693612138495, -STORE, 139693612138496, 139693612142591, -ERASE, 139693612105728, 139693612134399, -STORE, 39976960, 40112127, -STORE, 139693610393600, 139693612077055, -STORE, 139693612130304, 139693612134399, -STORE, 139693610258432, 139693610393599, -STORE, 39976960, 40255487, -STORE, 139693585338368, 139693587435519, -STORE, 139693612122112, 139693612134399, -STORE, 139693612113920, 139693612134399, -STORE, 139693612077056, 139693612113919, -STORE, 139693610242048, 139693610393599, -STORE, 39976960, 40390655, -STORE, 39976960, 40546303, -STORE, 139693610233856, 139693610393599, -STORE, 139693610225664, 139693610393599, -STORE, 39976960, 40714239, -STORE, 139693610209280, 139693610393599, -STORE, 39976960, 40861695, -STORE, 94431504838656, 94431505051647, -STORE, 94431507148800, 94431507152895, -STORE, 94431507152896, 94431507161087, -STORE, 94431507161088, 94431507173375, -STORE, 94431510286336, 94431528759295, -STORE, 139818797948928, 139818799607807, -STORE, 139818799607808, 139818801704959, -STORE, 139818801704960, 139818801721343, -STORE, 139818801721344, 139818801729535, -STORE, 139818801729536, 139818801745919, -STORE, 139818801745920, 139818801758207, -STORE, 139818801758208, 139818803851263, -STORE, 139818803851264, 139818803855359, -STORE, 139818803855360, 139818803859455, -STORE, 139818803859456, 139818804002815, -STORE, 139818804371456, 139818806054911, -STORE, 139818806054912, 139818806071295, -STORE, 139818806099968, 139818806104063, -STORE, 139818806104064, 139818806108159, -STORE, 139818806108160, 139818806112255, -STORE, 140731430457344, 140731430596607, -STORE, 140731431227392, 140731431239679, -STORE, 140731431239680, 140731431243775, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140729993904128, 140737488351231, -SNULL, 140729993912319, 140737488351231, -STORE, 140729993904128, 140729993912319, -STORE, 140729993773056, 140729993912319, -STORE, 93926271991808, 93926274215935, -SNULL, 93926272102399, 93926274215935, -STORE, 93926271991808, 93926272102399, -STORE, 93926272102400, 93926274215935, -ERASE, 93926272102400, 93926274215935, -STORE, 93926274195456, 93926274207743, -STORE, 93926274207744, 93926274215935, -STORE, 139962167296000, 139962169548799, -SNULL, 139962167439359, 139962169548799, -STORE, 139962167296000, 139962167439359, -STORE, 139962167439360, 139962169548799, -ERASE, 139962167439360, 139962169548799, -STORE, 139962169536512, 139962169544703, -STORE, 139962169544704, 139962169548799, -STORE, 140729995096064, 140729995100159, -STORE, 140729995083776, 140729995096063, -STORE, 139962169507840, 139962169536511, -STORE, 139962169499648, 139962169507839, -STORE, 139962163499008, 139962167295999, -SNULL, 139962163499008, 139962165157887, -STORE, 139962165157888, 139962167295999, -STORE, 139962163499008, 139962165157887, -SNULL, 139962167255039, 139962167295999, -STORE, 139962165157888, 139962167255039, -STORE, 139962167255040, 139962167295999, -SNULL, 139962167255040, 139962167279615, -STORE, 139962167279616, 139962167295999, -STORE, 139962167255040, 139962167279615, -ERASE, 139962167255040, 139962167279615, -STORE, 139962167255040, 139962167279615, -ERASE, 139962167279616, 139962167295999, -STORE, 139962167279616, 139962167295999, -SNULL, 139962167271423, 139962167279615, -STORE, 139962167255040, 139962167271423, -STORE, 139962167271424, 139962167279615, -SNULL, 93926274203647, 93926274207743, -STORE, 93926274195456, 93926274203647, -STORE, 93926274203648, 93926274207743, -SNULL, 139962169540607, 139962169544703, -STORE, 139962169536512, 139962169540607, -STORE, 139962169540608, 139962169544703, -ERASE, 139962169507840, 139962169536511, -STORE, 93926291120128, 93926291255295, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724960579584, 140737488351231, -SNULL, 140724960587775, 140737488351231, -STORE, 140724960579584, 140724960587775, -STORE, 140724960448512, 140724960587775, -STORE, 94246489489408, 94246491713535, -SNULL, 94246489599999, 94246491713535, -STORE, 94246489489408, 94246489599999, -STORE, 94246489600000, 94246491713535, -ERASE, 94246489600000, 94246491713535, -STORE, 94246491693056, 94246491705343, -STORE, 94246491705344, 94246491713535, -STORE, 140098174926848, 140098177179647, -SNULL, 140098175070207, 140098177179647, -STORE, 140098174926848, 140098175070207, -STORE, 140098175070208, 140098177179647, -ERASE, 140098175070208, 140098177179647, -STORE, 140098177167360, 140098177175551, -STORE, 140098177175552, 140098177179647, -STORE, 140724961439744, 140724961443839, -STORE, 140724961427456, 140724961439743, -STORE, 140098177138688, 140098177167359, -STORE, 140098177130496, 140098177138687, -STORE, 140098171129856, 140098174926847, -SNULL, 140098171129856, 140098172788735, -STORE, 140098172788736, 140098174926847, -STORE, 140098171129856, 140098172788735, -SNULL, 140098174885887, 140098174926847, -STORE, 140098172788736, 140098174885887, -STORE, 140098174885888, 140098174926847, -SNULL, 140098174885888, 140098174910463, -STORE, 140098174910464, 140098174926847, -STORE, 140098174885888, 140098174910463, -ERASE, 140098174885888, 140098174910463, -STORE, 140098174885888, 140098174910463, -ERASE, 140098174910464, 140098174926847, -STORE, 140098174910464, 140098174926847, -SNULL, 140098174902271, 140098174910463, -STORE, 140098174885888, 140098174902271, -STORE, 140098174902272, 140098174910463, -SNULL, 94246491701247, 94246491705343, -STORE, 94246491693056, 94246491701247, -STORE, 94246491701248, 94246491705343, -SNULL, 140098177171455, 140098177175551, -STORE, 140098177167360, 140098177171455, -STORE, 140098177171456, 140098177175551, -ERASE, 140098177138688, 140098177167359, -STORE, 94246516998144, 94246517133311, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140730522918912, 140737488351231, -SNULL, 140730522927103, 140737488351231, -STORE, 140730522918912, 140730522927103, -STORE, 140730522787840, 140730522927103, -STORE, 94196043120640, 94196045344767, -SNULL, 94196043231231, 94196045344767, -STORE, 94196043120640, 94196043231231, -STORE, 94196043231232, 94196045344767, -ERASE, 94196043231232, 94196045344767, -STORE, 94196045324288, 94196045336575, -STORE, 94196045336576, 94196045344767, -STORE, 139815918940160, 139815921192959, -SNULL, 139815919083519, 139815921192959, -STORE, 139815918940160, 139815919083519, -STORE, 139815919083520, 139815921192959, -ERASE, 139815919083520, 139815921192959, -STORE, 139815921180672, 139815921188863, -STORE, 139815921188864, 139815921192959, -STORE, 140730523344896, 140730523348991, -STORE, 140730523332608, 140730523344895, -STORE, 139815921152000, 139815921180671, -STORE, 139815921143808, 139815921151999, -STORE, 139815915143168, 139815918940159, -SNULL, 139815915143168, 139815916802047, -STORE, 139815916802048, 139815918940159, -STORE, 139815915143168, 139815916802047, -SNULL, 139815918899199, 139815918940159, -STORE, 139815916802048, 139815918899199, -STORE, 139815918899200, 139815918940159, -SNULL, 139815918899200, 139815918923775, -STORE, 139815918923776, 139815918940159, -STORE, 139815918899200, 139815918923775, -ERASE, 139815918899200, 139815918923775, -STORE, 139815918899200, 139815918923775, -ERASE, 139815918923776, 139815918940159, -STORE, 139815918923776, 139815918940159, -SNULL, 139815918915583, 139815918923775, -STORE, 139815918899200, 139815918915583, -STORE, 139815918915584, 139815918923775, -SNULL, 94196045332479, 94196045336575, -STORE, 94196045324288, 94196045332479, -STORE, 94196045332480, 94196045336575, -SNULL, 139815921184767, 139815921188863, -STORE, 139815921180672, 139815921184767, -STORE, 139815921184768, 139815921188863, -ERASE, 139815921152000, 139815921180671, -STORE, 94196076183552, 94196076318719, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722460393472, 140737488351231, -SNULL, 140722460401663, 140737488351231, -STORE, 140722460393472, 140722460401663, -STORE, 140722460262400, 140722460401663, -STORE, 94569810399232, 94569812623359, -SNULL, 94569810509823, 94569812623359, -STORE, 94569810399232, 94569810509823, -STORE, 94569810509824, 94569812623359, -ERASE, 94569810509824, 94569812623359, -STORE, 94569812602880, 94569812615167, -STORE, 94569812615168, 94569812623359, -STORE, 139681565450240, 139681567703039, -SNULL, 139681565593599, 139681567703039, -STORE, 139681565450240, 139681565593599, -STORE, 139681565593600, 139681567703039, -ERASE, 139681565593600, 139681567703039, -STORE, 139681567690752, 139681567698943, -STORE, 139681567698944, 139681567703039, -STORE, 140722460569600, 140722460573695, -STORE, 140722460557312, 140722460569599, -STORE, 139681567662080, 139681567690751, -STORE, 139681567653888, 139681567662079, -STORE, 139681561653248, 139681565450239, -SNULL, 139681561653248, 139681563312127, -STORE, 139681563312128, 139681565450239, -STORE, 139681561653248, 139681563312127, -SNULL, 139681565409279, 139681565450239, -STORE, 139681563312128, 139681565409279, -STORE, 139681565409280, 139681565450239, -SNULL, 139681565409280, 139681565433855, -STORE, 139681565433856, 139681565450239, -STORE, 139681565409280, 139681565433855, -ERASE, 139681565409280, 139681565433855, -STORE, 139681565409280, 139681565433855, -ERASE, 139681565433856, 139681565450239, -STORE, 139681565433856, 139681565450239, -SNULL, 139681565425663, 139681565433855, -STORE, 139681565409280, 139681565425663, -STORE, 139681565425664, 139681565433855, -SNULL, 94569812611071, 94569812615167, -STORE, 94569812602880, 94569812611071, -STORE, 94569812611072, 94569812615167, -SNULL, 139681567694847, 139681567698943, -STORE, 139681567690752, 139681567694847, -STORE, 139681567694848, 139681567698943, -ERASE, 139681567662080, 139681567690751, -STORE, 94569818066944, 94569818202111, -STORE, 94431504838656, 94431505051647, -STORE, 94431507148800, 94431507152895, -STORE, 94431507152896, 94431507161087, -STORE, 94431507161088, 94431507173375, -STORE, 94431510286336, 94431534280703, -STORE, 139818797948928, 139818799607807, -STORE, 139818799607808, 139818801704959, -STORE, 139818801704960, 139818801721343, -STORE, 139818801721344, 139818801729535, -STORE, 139818801729536, 139818801745919, -STORE, 139818801745920, 139818801758207, -STORE, 139818801758208, 139818803851263, -STORE, 139818803851264, 139818803855359, -STORE, 139818803855360, 139818803859455, -STORE, 139818803859456, 139818804002815, -STORE, 139818804371456, 139818806054911, -STORE, 139818806054912, 139818806071295, -STORE, 139818806099968, 139818806104063, -STORE, 139818806104064, 139818806108159, -STORE, 139818806108160, 139818806112255, -STORE, 140731430457344, 140731430596607, -STORE, 140731431227392, 140731431239679, -STORE, 140731431239680, 140731431243775, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140725452365824, 140737488351231, -SNULL, 140725452374015, 140737488351231, -STORE, 140725452365824, 140725452374015, -STORE, 140725452234752, 140725452374015, -STORE, 94395067465728, 94395069689855, -SNULL, 94395067576319, 94395069689855, -STORE, 94395067465728, 94395067576319, -STORE, 94395067576320, 94395069689855, -ERASE, 94395067576320, 94395069689855, -STORE, 94395069669376, 94395069681663, -STORE, 94395069681664, 94395069689855, -STORE, 140269941211136, 140269943463935, -SNULL, 140269941354495, 140269943463935, -STORE, 140269941211136, 140269941354495, -STORE, 140269941354496, 140269943463935, -ERASE, 140269941354496, 140269943463935, -STORE, 140269943451648, 140269943459839, -STORE, 140269943459840, 140269943463935, -STORE, 140725452558336, 140725452562431, -STORE, 140725452546048, 140725452558335, -STORE, 140269943422976, 140269943451647, -STORE, 140269943414784, 140269943422975, -STORE, 140269937414144, 140269941211135, -SNULL, 140269937414144, 140269939073023, -STORE, 140269939073024, 140269941211135, -STORE, 140269937414144, 140269939073023, -SNULL, 140269941170175, 140269941211135, -STORE, 140269939073024, 140269941170175, -STORE, 140269941170176, 140269941211135, -SNULL, 140269941170176, 140269941194751, -STORE, 140269941194752, 140269941211135, -STORE, 140269941170176, 140269941194751, -ERASE, 140269941170176, 140269941194751, -STORE, 140269941170176, 140269941194751, -ERASE, 140269941194752, 140269941211135, -STORE, 140269941194752, 140269941211135, -SNULL, 140269941186559, 140269941194751, -STORE, 140269941170176, 140269941186559, -STORE, 140269941186560, 140269941194751, -SNULL, 94395069677567, 94395069681663, -STORE, 94395069669376, 94395069677567, -STORE, 94395069677568, 94395069681663, -SNULL, 140269943455743, 140269943459839, -STORE, 140269943451648, 140269943455743, -STORE, 140269943455744, 140269943459839, -ERASE, 140269943422976, 140269943451647, -STORE, 94395101691904, 94395101827071, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733860118528, 140737488351231, -SNULL, 140733860126719, 140737488351231, -STORE, 140733860118528, 140733860126719, -STORE, 140733859987456, 140733860126719, -STORE, 94484752990208, 94484755214335, -SNULL, 94484753100799, 94484755214335, -STORE, 94484752990208, 94484753100799, -STORE, 94484753100800, 94484755214335, -ERASE, 94484753100800, 94484755214335, -STORE, 94484755193856, 94484755206143, -STORE, 94484755206144, 94484755214335, -STORE, 139958922309632, 139958924562431, -SNULL, 139958922452991, 139958924562431, -STORE, 139958922309632, 139958922452991, -STORE, 139958922452992, 139958924562431, -ERASE, 139958922452992, 139958924562431, -STORE, 139958924550144, 139958924558335, -STORE, 139958924558336, 139958924562431, -STORE, 140733860253696, 140733860257791, -STORE, 140733860241408, 140733860253695, -STORE, 139958924521472, 139958924550143, -STORE, 139958924513280, 139958924521471, -STORE, 139958918512640, 139958922309631, -SNULL, 139958918512640, 139958920171519, -STORE, 139958920171520, 139958922309631, -STORE, 139958918512640, 139958920171519, -SNULL, 139958922268671, 139958922309631, -STORE, 139958920171520, 139958922268671, -STORE, 139958922268672, 139958922309631, -SNULL, 139958922268672, 139958922293247, -STORE, 139958922293248, 139958922309631, -STORE, 139958922268672, 139958922293247, -ERASE, 139958922268672, 139958922293247, -STORE, 139958922268672, 139958922293247, -ERASE, 139958922293248, 139958922309631, -STORE, 139958922293248, 139958922309631, -SNULL, 139958922285055, 139958922293247, -STORE, 139958922268672, 139958922285055, -STORE, 139958922285056, 139958922293247, -SNULL, 94484755202047, 94484755206143, -STORE, 94484755193856, 94484755202047, -STORE, 94484755202048, 94484755206143, -SNULL, 139958924554239, 139958924558335, -STORE, 139958924550144, 139958924554239, -STORE, 139958924554240, 139958924558335, -ERASE, 139958924521472, 139958924550143, -STORE, 94484777615360, 94484777750527, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731051036672, 140737488351231, -SNULL, 140731051044863, 140737488351231, -STORE, 140731051036672, 140731051044863, -STORE, 140731050905600, 140731051044863, -STORE, 93945822998528, 93945825222655, -SNULL, 93945823109119, 93945825222655, -STORE, 93945822998528, 93945823109119, -STORE, 93945823109120, 93945825222655, -ERASE, 93945823109120, 93945825222655, -STORE, 93945825202176, 93945825214463, -STORE, 93945825214464, 93945825222655, -STORE, 140153503997952, 140153506250751, -SNULL, 140153504141311, 140153506250751, -STORE, 140153503997952, 140153504141311, -STORE, 140153504141312, 140153506250751, -ERASE, 140153504141312, 140153506250751, -STORE, 140153506238464, 140153506246655, -STORE, 140153506246656, 140153506250751, -STORE, 140731051331584, 140731051335679, -STORE, 140731051319296, 140731051331583, -STORE, 140153506209792, 140153506238463, -STORE, 140153506201600, 140153506209791, -STORE, 140153500200960, 140153503997951, -SNULL, 140153500200960, 140153501859839, -STORE, 140153501859840, 140153503997951, -STORE, 140153500200960, 140153501859839, -SNULL, 140153503956991, 140153503997951, -STORE, 140153501859840, 140153503956991, -STORE, 140153503956992, 140153503997951, -SNULL, 140153503956992, 140153503981567, -STORE, 140153503981568, 140153503997951, -STORE, 140153503956992, 140153503981567, -ERASE, 140153503956992, 140153503981567, -STORE, 140153503956992, 140153503981567, -ERASE, 140153503981568, 140153503997951, -STORE, 140153503981568, 140153503997951, -SNULL, 140153503973375, 140153503981567, -STORE, 140153503956992, 140153503973375, -STORE, 140153503973376, 140153503981567, -SNULL, 93945825210367, 93945825214463, -STORE, 93945825202176, 93945825210367, -STORE, 93945825210368, 93945825214463, -SNULL, 140153506242559, 140153506246655, -STORE, 140153506238464, 140153506242559, -STORE, 140153506242560, 140153506246655, -ERASE, 140153506209792, 140153506238463, -STORE, 93945854537728, 93945854672895, -STORE, 94431504838656, 94431505051647, -STORE, 94431507148800, 94431507152895, -STORE, 94431507152896, 94431507161087, -STORE, 94431507161088, 94431507173375, -STORE, 94431510286336, 94431537885183, -STORE, 139818797948928, 139818799607807, -STORE, 139818799607808, 139818801704959, -STORE, 139818801704960, 139818801721343, -STORE, 139818801721344, 139818801729535, -STORE, 139818801729536, 139818801745919, -STORE, 139818801745920, 139818801758207, -STORE, 139818801758208, 139818803851263, -STORE, 139818803851264, 139818803855359, -STORE, 139818803855360, 139818803859455, -STORE, 139818803859456, 139818804002815, -STORE, 139818804371456, 139818806054911, -STORE, 139818806054912, 139818806071295, -STORE, 139818806099968, 139818806104063, -STORE, 139818806104064, 139818806108159, -STORE, 139818806108160, 139818806112255, -STORE, 140731430457344, 140731430596607, -STORE, 140731431227392, 140731431239679, -STORE, 140731431239680, 140731431243775, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140736025325568, 140737488351231, -SNULL, 140736025333759, 140737488351231, -STORE, 140736025325568, 140736025333759, -STORE, 140736025194496, 140736025333759, -STORE, 94809095172096, 94809097396223, -SNULL, 94809095282687, 94809097396223, -STORE, 94809095172096, 94809095282687, -STORE, 94809095282688, 94809097396223, -ERASE, 94809095282688, 94809097396223, -STORE, 94809097375744, 94809097388031, -STORE, 94809097388032, 94809097396223, -STORE, 140194992517120, 140194994769919, -SNULL, 140194992660479, 140194994769919, -STORE, 140194992517120, 140194992660479, -STORE, 140194992660480, 140194994769919, -ERASE, 140194992660480, 140194994769919, -STORE, 140194994757632, 140194994765823, -STORE, 140194994765824, 140194994769919, -STORE, 140736026173440, 140736026177535, -STORE, 140736026161152, 140736026173439, -STORE, 140194994728960, 140194994757631, -STORE, 140194994720768, 140194994728959, -STORE, 140194988720128, 140194992517119, -SNULL, 140194988720128, 140194990379007, -STORE, 140194990379008, 140194992517119, -STORE, 140194988720128, 140194990379007, -SNULL, 140194992476159, 140194992517119, -STORE, 140194990379008, 140194992476159, -STORE, 140194992476160, 140194992517119, -SNULL, 140194992476160, 140194992500735, -STORE, 140194992500736, 140194992517119, -STORE, 140194992476160, 140194992500735, -ERASE, 140194992476160, 140194992500735, -STORE, 140194992476160, 140194992500735, -ERASE, 140194992500736, 140194992517119, -STORE, 140194992500736, 140194992517119, -SNULL, 140194992492543, 140194992500735, -STORE, 140194992476160, 140194992492543, -STORE, 140194992492544, 140194992500735, -SNULL, 94809097383935, 94809097388031, -STORE, 94809097375744, 94809097383935, -STORE, 94809097383936, 94809097388031, -SNULL, 140194994761727, 140194994765823, -STORE, 140194994757632, 140194994761727, -STORE, 140194994761728, 140194994765823, -ERASE, 140194994728960, 140194994757631, -STORE, 94809124286464, 94809124421631, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140726342660096, 140737488351231, -SNULL, 140726342668287, 140737488351231, -STORE, 140726342660096, 140726342668287, -STORE, 140726342529024, 140726342668287, -STORE, 94140331462656, 94140333686783, -SNULL, 94140331573247, 94140333686783, -STORE, 94140331462656, 94140331573247, -STORE, 94140331573248, 94140333686783, -ERASE, 94140331573248, 94140333686783, -STORE, 94140333666304, 94140333678591, -STORE, 94140333678592, 94140333686783, -STORE, 140714077208576, 140714079461375, -SNULL, 140714077351935, 140714079461375, -STORE, 140714077208576, 140714077351935, -STORE, 140714077351936, 140714079461375, -ERASE, 140714077351936, 140714079461375, -STORE, 140714079449088, 140714079457279, -STORE, 140714079457280, 140714079461375, -STORE, 140726343933952, 140726343938047, -STORE, 140726343921664, 140726343933951, -STORE, 140714079420416, 140714079449087, -STORE, 140714079412224, 140714079420415, -STORE, 140714073411584, 140714077208575, -SNULL, 140714073411584, 140714075070463, -STORE, 140714075070464, 140714077208575, -STORE, 140714073411584, 140714075070463, -SNULL, 140714077167615, 140714077208575, -STORE, 140714075070464, 140714077167615, -STORE, 140714077167616, 140714077208575, -SNULL, 140714077167616, 140714077192191, -STORE, 140714077192192, 140714077208575, -STORE, 140714077167616, 140714077192191, -ERASE, 140714077167616, 140714077192191, -STORE, 140714077167616, 140714077192191, -ERASE, 140714077192192, 140714077208575, -STORE, 140714077192192, 140714077208575, -SNULL, 140714077183999, 140714077192191, -STORE, 140714077167616, 140714077183999, -STORE, 140714077184000, 140714077192191, -SNULL, 94140333674495, 94140333678591, -STORE, 94140333666304, 94140333674495, -STORE, 94140333674496, 94140333678591, -SNULL, 140714079453183, 140714079457279, -STORE, 140714079449088, 140714079453183, -STORE, 140714079453184, 140714079457279, -ERASE, 140714079420416, 140714079449087, -STORE, 94140341432320, 94140341567487, -STORE, 94431504838656, 94431505051647, -STORE, 94431507148800, 94431507152895, -STORE, 94431507152896, 94431507161087, -STORE, 94431507161088, 94431507173375, -STORE, 94431510286336, 94431539601407, -STORE, 139818797948928, 139818799607807, -STORE, 139818799607808, 139818801704959, -STORE, 139818801704960, 139818801721343, -STORE, 139818801721344, 139818801729535, -STORE, 139818801729536, 139818801745919, -STORE, 139818801745920, 139818801758207, -STORE, 139818801758208, 139818803851263, -STORE, 139818803851264, 139818803855359, -STORE, 139818803855360, 139818803859455, -STORE, 139818803859456, 139818804002815, -STORE, 139818804371456, 139818806054911, -STORE, 139818806054912, 139818806071295, -STORE, 139818806099968, 139818806104063, -STORE, 139818806104064, 139818806108159, -STORE, 139818806108160, 139818806112255, -STORE, 140731430457344, 140731430596607, -STORE, 140731431227392, 140731431239679, -STORE, 140731431239680, 140731431243775, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140725843607552, 140737488351231, -SNULL, 140725843615743, 140737488351231, -STORE, 140725843607552, 140725843615743, -STORE, 140725843476480, 140725843615743, -STORE, 94889043505152, 94889045839871, -SNULL, 94889043718143, 94889045839871, -STORE, 94889043505152, 94889043718143, -STORE, 94889043718144, 94889045839871, -ERASE, 94889043718144, 94889045839871, -STORE, 94889045815296, 94889045827583, -STORE, 94889045827584, 94889045839871, -STORE, 140250965946368, 140250968199167, -SNULL, 140250966089727, 140250968199167, -STORE, 140250965946368, 140250966089727, -STORE, 140250966089728, 140250968199167, -ERASE, 140250966089728, 140250968199167, -STORE, 140250968186880, 140250968195071, -STORE, 140250968195072, 140250968199167, -STORE, 140725844500480, 140725844504575, -STORE, 140725844488192, 140725844500479, -STORE, 140250968158208, 140250968186879, -STORE, 140250968150016, 140250968158207, -STORE, 140250963832832, 140250965946367, -SNULL, 140250963832832, 140250963845119, -STORE, 140250963845120, 140250965946367, -STORE, 140250963832832, 140250963845119, -SNULL, 140250965938175, 140250965946367, -STORE, 140250963845120, 140250965938175, -STORE, 140250965938176, 140250965946367, -ERASE, 140250965938176, 140250965946367, -STORE, 140250965938176, 140250965946367, -STORE, 140250960035840, 140250963832831, -SNULL, 140250960035840, 140250961694719, -STORE, 140250961694720, 140250963832831, -STORE, 140250960035840, 140250961694719, -SNULL, 140250963791871, 140250963832831, -STORE, 140250961694720, 140250963791871, -STORE, 140250963791872, 140250963832831, -SNULL, 140250963791872, 140250963816447, -STORE, 140250963816448, 140250963832831, -STORE, 140250963791872, 140250963816447, -ERASE, 140250963791872, 140250963816447, -STORE, 140250963791872, 140250963816447, -ERASE, 140250963816448, 140250963832831, -STORE, 140250963816448, 140250963832831, -STORE, 140250968141824, 140250968158207, -SNULL, 140250963808255, 140250963816447, -STORE, 140250963791872, 140250963808255, -STORE, 140250963808256, 140250963816447, -SNULL, 140250965942271, 140250965946367, -STORE, 140250965938176, 140250965942271, -STORE, 140250965942272, 140250965946367, -SNULL, 94889045819391, 94889045827583, -STORE, 94889045815296, 94889045819391, -STORE, 94889045819392, 94889045827583, -SNULL, 140250968190975, 140250968195071, -STORE, 140250968186880, 140250968190975, -STORE, 140250968190976, 140250968195071, -ERASE, 140250968158208, 140250968186879, -STORE, 94889052213248, 94889052348415, -STORE, 140250966458368, 140250968141823, -STORE, 94889052213248, 94889052483583, -STORE, 94889052213248, 94889052618751, -STORE, 94170851819520, 94170852032511, -STORE, 94170854129664, 94170854133759, -STORE, 94170854133760, 94170854141951, -STORE, 94170854141952, 94170854154239, -STORE, 94170866515968, 94170867740671, -STORE, 140062030422016, 140062032080895, -STORE, 140062032080896, 140062034178047, -STORE, 140062034178048, 140062034194431, -STORE, 140062034194432, 140062034202623, -STORE, 140062034202624, 140062034219007, -STORE, 140062034219008, 140062034231295, -STORE, 140062034231296, 140062036324351, -STORE, 140062036324352, 140062036328447, -STORE, 140062036328448, 140062036332543, -STORE, 140062036332544, 140062036475903, -STORE, 140062036844544, 140062038527999, -STORE, 140062038528000, 140062038544383, -STORE, 140062038573056, 140062038577151, -STORE, 140062038577152, 140062038581247, -STORE, 140062038581248, 140062038585343, -STORE, 140736210550784, 140736210690047, -STORE, 140736210759680, 140736210771967, -STORE, 140736210771968, 140736210776063, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724272365568, 140737488351231, -SNULL, 140724272373759, 140737488351231, -STORE, 140724272365568, 140724272373759, -STORE, 140724272234496, 140724272373759, -STORE, 94607711965184, 94607714189311, -SNULL, 94607712075775, 94607714189311, -STORE, 94607711965184, 94607712075775, -STORE, 94607712075776, 94607714189311, -ERASE, 94607712075776, 94607714189311, -STORE, 94607714168832, 94607714181119, -STORE, 94607714181120, 94607714189311, -STORE, 140054949253120, 140054951505919, -SNULL, 140054949396479, 140054951505919, -STORE, 140054949253120, 140054949396479, -STORE, 140054949396480, 140054951505919, -ERASE, 140054949396480, 140054951505919, -STORE, 140054951493632, 140054951501823, -STORE, 140054951501824, 140054951505919, -STORE, 140724272992256, 140724272996351, -STORE, 140724272979968, 140724272992255, -STORE, 140054951464960, 140054951493631, -STORE, 140054951456768, 140054951464959, -STORE, 140054945456128, 140054949253119, -SNULL, 140054945456128, 140054947115007, -STORE, 140054947115008, 140054949253119, -STORE, 140054945456128, 140054947115007, -SNULL, 140054949212159, 140054949253119, -STORE, 140054947115008, 140054949212159, -STORE, 140054949212160, 140054949253119, -SNULL, 140054949212160, 140054949236735, -STORE, 140054949236736, 140054949253119, -STORE, 140054949212160, 140054949236735, -ERASE, 140054949212160, 140054949236735, -STORE, 140054949212160, 140054949236735, -ERASE, 140054949236736, 140054949253119, -STORE, 140054949236736, 140054949253119, -SNULL, 140054949228543, 140054949236735, -STORE, 140054949212160, 140054949228543, -STORE, 140054949228544, 140054949236735, -SNULL, 94607714177023, 94607714181119, -STORE, 94607714168832, 94607714177023, -STORE, 94607714177024, 94607714181119, -SNULL, 140054951497727, 140054951501823, -STORE, 140054951493632, 140054951497727, -STORE, 140054951497728, 140054951501823, -ERASE, 140054951464960, 140054951493631, -STORE, 94607733374976, 94607733510143, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733586923520, 140737488351231, -SNULL, 140733586931711, 140737488351231, -STORE, 140733586923520, 140733586931711, -STORE, 140733586792448, 140733586931711, -STORE, 93901634904064, 93901637128191, -SNULL, 93901635014655, 93901637128191, -STORE, 93901634904064, 93901635014655, -STORE, 93901635014656, 93901637128191, -ERASE, 93901635014656, 93901637128191, -STORE, 93901637107712, 93901637119999, -STORE, 93901637120000, 93901637128191, -STORE, 140086104784896, 140086107037695, -SNULL, 140086104928255, 140086107037695, -STORE, 140086104784896, 140086104928255, -STORE, 140086104928256, 140086107037695, -ERASE, 140086104928256, 140086107037695, -STORE, 140086107025408, 140086107033599, -STORE, 140086107033600, 140086107037695, -STORE, 140733587263488, 140733587267583, -STORE, 140733587251200, 140733587263487, -STORE, 140086106996736, 140086107025407, -STORE, 140086106988544, 140086106996735, -STORE, 140086100987904, 140086104784895, -SNULL, 140086100987904, 140086102646783, -STORE, 140086102646784, 140086104784895, -STORE, 140086100987904, 140086102646783, -SNULL, 140086104743935, 140086104784895, -STORE, 140086102646784, 140086104743935, -STORE, 140086104743936, 140086104784895, -SNULL, 140086104743936, 140086104768511, -STORE, 140086104768512, 140086104784895, -STORE, 140086104743936, 140086104768511, -ERASE, 140086104743936, 140086104768511, -STORE, 140086104743936, 140086104768511, -ERASE, 140086104768512, 140086104784895, -STORE, 140086104768512, 140086104784895, -SNULL, 140086104760319, 140086104768511, -STORE, 140086104743936, 140086104760319, -STORE, 140086104760320, 140086104768511, -SNULL, 93901637115903, 93901637119999, -STORE, 93901637107712, 93901637115903, -STORE, 93901637115904, 93901637119999, -SNULL, 140086107029503, 140086107033599, -STORE, 140086107025408, 140086107029503, -STORE, 140086107029504, 140086107033599, -ERASE, 140086106996736, 140086107025407, -STORE, 93901662715904, 93901662851071, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723365613568, 140737488351231, -SNULL, 140723365621759, 140737488351231, -STORE, 140723365613568, 140723365621759, -STORE, 140723365482496, 140723365621759, -STORE, 94759193546752, 94759195770879, -SNULL, 94759193657343, 94759195770879, -STORE, 94759193546752, 94759193657343, -STORE, 94759193657344, 94759195770879, -ERASE, 94759193657344, 94759195770879, -STORE, 94759195750400, 94759195762687, -STORE, 94759195762688, 94759195770879, -STORE, 140607636246528, 140607638499327, -SNULL, 140607636389887, 140607638499327, -STORE, 140607636246528, 140607636389887, -STORE, 140607636389888, 140607638499327, -ERASE, 140607636389888, 140607638499327, -STORE, 140607638487040, 140607638495231, -STORE, 140607638495232, 140607638499327, -STORE, 140723365900288, 140723365904383, -STORE, 140723365888000, 140723365900287, -STORE, 140607638458368, 140607638487039, -STORE, 140607638450176, 140607638458367, -STORE, 140607632449536, 140607636246527, -SNULL, 140607632449536, 140607634108415, -STORE, 140607634108416, 140607636246527, -STORE, 140607632449536, 140607634108415, -SNULL, 140607636205567, 140607636246527, -STORE, 140607634108416, 140607636205567, -STORE, 140607636205568, 140607636246527, -SNULL, 140607636205568, 140607636230143, -STORE, 140607636230144, 140607636246527, -STORE, 140607636205568, 140607636230143, -ERASE, 140607636205568, 140607636230143, -STORE, 140607636205568, 140607636230143, -ERASE, 140607636230144, 140607636246527, -STORE, 140607636230144, 140607636246527, -SNULL, 140607636221951, 140607636230143, -STORE, 140607636205568, 140607636221951, -STORE, 140607636221952, 140607636230143, -SNULL, 94759195758591, 94759195762687, -STORE, 94759195750400, 94759195758591, -STORE, 94759195758592, 94759195762687, -SNULL, 140607638491135, 140607638495231, -STORE, 140607638487040, 140607638491135, -STORE, 140607638491136, 140607638495231, -ERASE, 140607638458368, 140607638487039, -STORE, 94759204995072, 94759205130239, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140732503789568, 140737488351231, -SNULL, 140732503797759, 140737488351231, -STORE, 140732503789568, 140732503797759, -STORE, 140732503658496, 140732503797759, -STORE, 94077792956416, 94077795180543, -SNULL, 94077793067007, 94077795180543, -STORE, 94077792956416, 94077793067007, -STORE, 94077793067008, 94077795180543, -ERASE, 94077793067008, 94077795180543, -STORE, 94077795160064, 94077795172351, -STORE, 94077795172352, 94077795180543, -STORE, 140359874252800, 140359876505599, -SNULL, 140359874396159, 140359876505599, -STORE, 140359874252800, 140359874396159, -STORE, 140359874396160, 140359876505599, -ERASE, 140359874396160, 140359876505599, -STORE, 140359876493312, 140359876501503, -STORE, 140359876501504, 140359876505599, -STORE, 140732504465408, 140732504469503, -STORE, 140732504453120, 140732504465407, -STORE, 140359876464640, 140359876493311, -STORE, 140359876456448, 140359876464639, -STORE, 140359870455808, 140359874252799, -SNULL, 140359870455808, 140359872114687, -STORE, 140359872114688, 140359874252799, -STORE, 140359870455808, 140359872114687, -SNULL, 140359874211839, 140359874252799, -STORE, 140359872114688, 140359874211839, -STORE, 140359874211840, 140359874252799, -SNULL, 140359874211840, 140359874236415, -STORE, 140359874236416, 140359874252799, -STORE, 140359874211840, 140359874236415, -ERASE, 140359874211840, 140359874236415, -STORE, 140359874211840, 140359874236415, -ERASE, 140359874236416, 140359874252799, -STORE, 140359874236416, 140359874252799, -SNULL, 140359874228223, 140359874236415, -STORE, 140359874211840, 140359874228223, -STORE, 140359874228224, 140359874236415, -SNULL, 94077795168255, 94077795172351, -STORE, 94077795160064, 94077795168255, -STORE, 94077795168256, 94077795172351, -SNULL, 140359876497407, 140359876501503, -STORE, 140359876493312, 140359876497407, -STORE, 140359876497408, 140359876501503, -ERASE, 140359876464640, 140359876493311, -STORE, 94077808717824, 94077808852991, -STORE, 94549486252032, 94549486465023, -STORE, 94549488562176, 94549488566271, -STORE, 94549488566272, 94549488574463, -STORE, 94549488574464, 94549488586751, -STORE, 94549503492096, 94549506121727, -STORE, 140085800894464, 140085802553343, -STORE, 140085802553344, 140085804650495, -STORE, 140085804650496, 140085804666879, -STORE, 140085804666880, 140085804675071, -STORE, 140085804675072, 140085804691455, -STORE, 140085804691456, 140085804703743, -STORE, 140085804703744, 140085806796799, -STORE, 140085806796800, 140085806800895, -STORE, 140085806800896, 140085806804991, -STORE, 140085806804992, 140085806948351, -STORE, 140085807316992, 140085809000447, -STORE, 140085809000448, 140085809016831, -STORE, 140085809045504, 140085809049599, -STORE, 140085809049600, 140085809053695, -STORE, 140085809053696, 140085809057791, -STORE, 140731810545664, 140731810684927, -STORE, 140731810967552, 140731810979839, -STORE, 140731810979840, 140731810983935, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724752330752, 140737488351231, -SNULL, 140724752338943, 140737488351231, -STORE, 140724752330752, 140724752338943, -STORE, 140724752199680, 140724752338943, -STORE, 94656357539840, 94656359874559, -SNULL, 94656357752831, 94656359874559, -STORE, 94656357539840, 94656357752831, -STORE, 94656357752832, 94656359874559, -ERASE, 94656357752832, 94656359874559, -STORE, 94656359849984, 94656359862271, -STORE, 94656359862272, 94656359874559, -STORE, 139632585203712, 139632587456511, -SNULL, 139632585347071, 139632587456511, -STORE, 139632585203712, 139632585347071, -STORE, 139632585347072, 139632587456511, -ERASE, 139632585347072, 139632587456511, -STORE, 139632587444224, 139632587452415, -STORE, 139632587452416, 139632587456511, -STORE, 139632587440128, 139632587444223, -STORE, 139632587427840, 139632587440127, -STORE, 139632587399168, 139632587427839, -STORE, 139632587390976, 139632587399167, -STORE, 139632583090176, 139632585203711, -SNULL, 139632583090176, 139632583102463, -STORE, 139632583102464, 139632585203711, -STORE, 139632583090176, 139632583102463, -SNULL, 139632585195519, 139632585203711, -STORE, 139632583102464, 139632585195519, -STORE, 139632585195520, 139632585203711, -ERASE, 139632585195520, 139632585203711, -STORE, 139632585195520, 139632585203711, -STORE, 139632579293184, 139632583090175, -SNULL, 139632579293184, 139632580952063, -STORE, 139632580952064, 139632583090175, -STORE, 139632579293184, 139632580952063, -SNULL, 139632583049215, 139632583090175, -STORE, 139632580952064, 139632583049215, -STORE, 139632583049216, 139632583090175, -SNULL, 139632583049216, 139632583073791, -STORE, 139632583073792, 139632583090175, -STORE, 139632583049216, 139632583073791, -ERASE, 139632583049216, 139632583073791, -STORE, 139632583049216, 139632583073791, -ERASE, 139632583073792, 139632583090175, -STORE, 139632583073792, 139632583090175, -STORE, 139632587382784, 139632587399167, -SNULL, 139632583065599, 139632583073791, -STORE, 139632583049216, 139632583065599, -STORE, 139632583065600, 139632583073791, -SNULL, 139632585199615, 139632585203711, -STORE, 139632585195520, 139632585199615, -STORE, 139632585199616, 139632585203711, -SNULL, 94656359854079, 94656359862271, -STORE, 94656359849984, 94656359854079, -STORE, 94656359854080, 94656359862271, -SNULL, 139632587448319, 139632587452415, -STORE, 139632587444224, 139632587448319, -STORE, 139632587448320, 139632587452415, -ERASE, 139632587399168, 139632587427839, -STORE, 94656378912768, 94656379047935, -STORE, 139632585699328, 139632587382783, -STORE, 94656378912768, 94656379183103, -STORE, 94656378912768, 94656379318271, -STORE, 94656378912768, 94656379494399, -SNULL, 94656379469823, 94656379494399, -STORE, 94656378912768, 94656379469823, -STORE, 94656379469824, 94656379494399, -ERASE, 94656379469824, 94656379494399, -STORE, 94656378912768, 94656379621375, -STORE, 94656378912768, 94656379756543, -STORE, 94656378912768, 94656379912191, -STORE, 94656378912768, 94656380055551, -STORE, 94656378912768, 94656380190719, -STORE, 94656378912768, 94656380338175, -SNULL, 94656380313599, 94656380338175, -STORE, 94656378912768, 94656380313599, -STORE, 94656380313600, 94656380338175, -ERASE, 94656380313600, 94656380338175, -STORE, 94656378912768, 94656380448767, -SNULL, 94656380432383, 94656380448767, -STORE, 94656378912768, 94656380432383, -STORE, 94656380432384, 94656380448767, -ERASE, 94656380432384, 94656380448767, -STORE, 94656378912768, 94656380567551, -STORE, 94656378912768, 94656380719103, -STORE, 94656378912768, 94656380858367, -STORE, 94656378912768, 94656380997631, -STORE, 94656378912768, 94656381132799, -SNULL, 94656381124607, 94656381132799, -STORE, 94656378912768, 94656381124607, -STORE, 94656381124608, 94656381132799, -ERASE, 94656381124608, 94656381132799, -STORE, 94656378912768, 94656381276159, -STORE, 94656378912768, 94656381427711, -STORE, 94604087611392, 94604087824383, -STORE, 94604089921536, 94604089925631, -STORE, 94604089925632, 94604089933823, -STORE, 94604089933824, 94604089946111, -STORE, 94604105125888, 94604106424319, -STORE, 140454937694208, 140454939353087, -STORE, 140454939353088, 140454941450239, -STORE, 140454941450240, 140454941466623, -STORE, 140454941466624, 140454941474815, -STORE, 140454941474816, 140454941491199, -STORE, 140454941491200, 140454941503487, -STORE, 140454941503488, 140454943596543, -STORE, 140454943596544, 140454943600639, -STORE, 140454943600640, 140454943604735, -STORE, 140454943604736, 140454943748095, -STORE, 140454944116736, 140454945800191, -STORE, 140454945800192, 140454945816575, -STORE, 140454945845248, 140454945849343, -STORE, 140454945849344, 140454945853439, -STORE, 140454945853440, 140454945857535, -STORE, 140728438214656, 140728438353919, -STORE, 140728439095296, 140728439107583, -STORE, 140728439107584, 140728439111679, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140727821099008, 140737488351231, -SNULL, 140727821107199, 140737488351231, -STORE, 140727821099008, 140727821107199, -STORE, 140727820967936, 140727821107199, -STORE, 94088457240576, 94088459575295, -SNULL, 94088457453567, 94088459575295, -STORE, 94088457240576, 94088457453567, -STORE, 94088457453568, 94088459575295, -ERASE, 94088457453568, 94088459575295, -STORE, 94088459550720, 94088459563007, -STORE, 94088459563008, 94088459575295, -STORE, 140234378989568, 140234381242367, -SNULL, 140234379132927, 140234381242367, -STORE, 140234378989568, 140234379132927, -STORE, 140234379132928, 140234381242367, -ERASE, 140234379132928, 140234381242367, -STORE, 140234381230080, 140234381238271, -STORE, 140234381238272, 140234381242367, -STORE, 140727822077952, 140727822082047, -STORE, 140727822065664, 140727822077951, -STORE, 140234381201408, 140234381230079, -STORE, 140234381193216, 140234381201407, -STORE, 140234376876032, 140234378989567, -SNULL, 140234376876032, 140234376888319, -STORE, 140234376888320, 140234378989567, -STORE, 140234376876032, 140234376888319, -SNULL, 140234378981375, 140234378989567, -STORE, 140234376888320, 140234378981375, -STORE, 140234378981376, 140234378989567, -ERASE, 140234378981376, 140234378989567, -STORE, 140234378981376, 140234378989567, -STORE, 140234373079040, 140234376876031, -SNULL, 140234373079040, 140234374737919, -STORE, 140234374737920, 140234376876031, -STORE, 140234373079040, 140234374737919, -SNULL, 140234376835071, 140234376876031, -STORE, 140234374737920, 140234376835071, -STORE, 140234376835072, 140234376876031, -SNULL, 140234376835072, 140234376859647, -STORE, 140234376859648, 140234376876031, -STORE, 140234376835072, 140234376859647, -ERASE, 140234376835072, 140234376859647, -STORE, 140234376835072, 140234376859647, -ERASE, 140234376859648, 140234376876031, -STORE, 140234376859648, 140234376876031, -STORE, 140234381185024, 140234381201407, -SNULL, 140234376851455, 140234376859647, -STORE, 140234376835072, 140234376851455, -STORE, 140234376851456, 140234376859647, -SNULL, 140234378985471, 140234378989567, -STORE, 140234378981376, 140234378985471, -STORE, 140234378985472, 140234378989567, -SNULL, 94088459554815, 94088459563007, -STORE, 94088459550720, 94088459554815, -STORE, 94088459554816, 94088459563007, -SNULL, 140234381234175, 140234381238271, -STORE, 140234381230080, 140234381234175, -STORE, 140234381234176, 140234381238271, -ERASE, 140234381201408, 140234381230079, -STORE, 94088468852736, 94088468987903, -STORE, 140234379501568, 140234381185023, -STORE, 94088468852736, 94088469123071, -STORE, 94088468852736, 94088469258239, -STORE, 94110050402304, 94110050615295, -STORE, 94110052712448, 94110052716543, -STORE, 94110052716544, 94110052724735, -STORE, 94110052724736, 94110052737023, -STORE, 94110061875200, 94110062415871, -STORE, 140139439357952, 140139441016831, -STORE, 140139441016832, 140139443113983, -STORE, 140139443113984, 140139443130367, -STORE, 140139443130368, 140139443138559, -STORE, 140139443138560, 140139443154943, -STORE, 140139443154944, 140139443167231, -STORE, 140139443167232, 140139445260287, -STORE, 140139445260288, 140139445264383, -STORE, 140139445264384, 140139445268479, -STORE, 140139445268480, 140139445411839, -STORE, 140139445780480, 140139447463935, -STORE, 140139447463936, 140139447480319, -STORE, 140139447508992, 140139447513087, -STORE, 140139447513088, 140139447517183, -STORE, 140139447517184, 140139447521279, -STORE, 140731901427712, 140731901566975, -STORE, 140731902259200, 140731902271487, -STORE, 140731902271488, 140731902275583, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140727282622464, 140737488351231, -SNULL, 140727282630655, 140737488351231, -STORE, 140727282622464, 140727282630655, -STORE, 140727282491392, 140727282630655, -STORE, 94266649866240, 94266652200959, -SNULL, 94266650079231, 94266652200959, -STORE, 94266649866240, 94266650079231, -STORE, 94266650079232, 94266652200959, -ERASE, 94266650079232, 94266652200959, -STORE, 94266652176384, 94266652188671, -STORE, 94266652188672, 94266652200959, -STORE, 139888497991680, 139888500244479, -SNULL, 139888498135039, 139888500244479, -STORE, 139888497991680, 139888498135039, -STORE, 139888498135040, 139888500244479, -ERASE, 139888498135040, 139888500244479, -STORE, 139888500232192, 139888500240383, -STORE, 139888500240384, 139888500244479, -STORE, 140727283113984, 140727283118079, -STORE, 140727283101696, 140727283113983, -STORE, 139888500203520, 139888500232191, -STORE, 139888500195328, 139888500203519, -STORE, 139888495878144, 139888497991679, -SNULL, 139888495878144, 139888495890431, -STORE, 139888495890432, 139888497991679, -STORE, 139888495878144, 139888495890431, -SNULL, 139888497983487, 139888497991679, -STORE, 139888495890432, 139888497983487, -STORE, 139888497983488, 139888497991679, -ERASE, 139888497983488, 139888497991679, -STORE, 139888497983488, 139888497991679, -STORE, 139888492081152, 139888495878143, -SNULL, 139888492081152, 139888493740031, -STORE, 139888493740032, 139888495878143, -STORE, 139888492081152, 139888493740031, -SNULL, 139888495837183, 139888495878143, -STORE, 139888493740032, 139888495837183, -STORE, 139888495837184, 139888495878143, -SNULL, 139888495837184, 139888495861759, -STORE, 139888495861760, 139888495878143, -STORE, 139888495837184, 139888495861759, -ERASE, 139888495837184, 139888495861759, -STORE, 139888495837184, 139888495861759, -ERASE, 139888495861760, 139888495878143, -STORE, 139888495861760, 139888495878143, -STORE, 139888500187136, 139888500203519, -SNULL, 139888495853567, 139888495861759, -STORE, 139888495837184, 139888495853567, -STORE, 139888495853568, 139888495861759, -SNULL, 139888497987583, 139888497991679, -STORE, 139888497983488, 139888497987583, -STORE, 139888497987584, 139888497991679, -SNULL, 94266652180479, 94266652188671, -STORE, 94266652176384, 94266652180479, -STORE, 94266652180480, 94266652188671, -SNULL, 139888500236287, 139888500240383, -STORE, 139888500232192, 139888500236287, -STORE, 139888500236288, 139888500240383, -ERASE, 139888500203520, 139888500232191, -STORE, 94266678542336, 94266678677503, -STORE, 139888498503680, 139888500187135, -STORE, 94266678542336, 94266678812671, -STORE, 94266678542336, 94266678947839, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722507702272, 140737488351231, -SNULL, 140722507710463, 140737488351231, -STORE, 140722507702272, 140722507710463, -STORE, 140722507571200, 140722507710463, -STORE, 94313981394944, 94313983729663, -SNULL, 94313981607935, 94313983729663, -STORE, 94313981394944, 94313981607935, -STORE, 94313981607936, 94313983729663, -ERASE, 94313981607936, 94313983729663, -STORE, 94313983705088, 94313983717375, -STORE, 94313983717376, 94313983729663, -STORE, 140456286076928, 140456288329727, -SNULL, 140456286220287, 140456288329727, -STORE, 140456286076928, 140456286220287, -STORE, 140456286220288, 140456288329727, -ERASE, 140456286220288, 140456288329727, -STORE, 140456288317440, 140456288325631, -STORE, 140456288325632, 140456288329727, -STORE, 140722507997184, 140722508001279, -STORE, 140722507984896, 140722507997183, -STORE, 140456288288768, 140456288317439, -STORE, 140456288280576, 140456288288767, -STORE, 140456283963392, 140456286076927, -SNULL, 140456283963392, 140456283975679, -STORE, 140456283975680, 140456286076927, -STORE, 140456283963392, 140456283975679, -SNULL, 140456286068735, 140456286076927, -STORE, 140456283975680, 140456286068735, -STORE, 140456286068736, 140456286076927, -ERASE, 140456286068736, 140456286076927, -STORE, 140456286068736, 140456286076927, -STORE, 140456280166400, 140456283963391, -SNULL, 140456280166400, 140456281825279, -STORE, 140456281825280, 140456283963391, -STORE, 140456280166400, 140456281825279, -SNULL, 140456283922431, 140456283963391, -STORE, 140456281825280, 140456283922431, -STORE, 140456283922432, 140456283963391, -SNULL, 140456283922432, 140456283947007, -STORE, 140456283947008, 140456283963391, -STORE, 140456283922432, 140456283947007, -ERASE, 140456283922432, 140456283947007, -STORE, 140456283922432, 140456283947007, -ERASE, 140456283947008, 140456283963391, -STORE, 140456283947008, 140456283963391, -STORE, 140456288272384, 140456288288767, -SNULL, 140456283938815, 140456283947007, -STORE, 140456283922432, 140456283938815, -STORE, 140456283938816, 140456283947007, -SNULL, 140456286072831, 140456286076927, -STORE, 140456286068736, 140456286072831, -STORE, 140456286072832, 140456286076927, -SNULL, 94313983709183, 94313983717375, -STORE, 94313983705088, 94313983709183, -STORE, 94313983709184, 94313983717375, -SNULL, 140456288321535, 140456288325631, -STORE, 140456288317440, 140456288321535, -STORE, 140456288321536, 140456288325631, -ERASE, 140456288288768, 140456288317439, -STORE, 94314006716416, 94314006851583, -STORE, 140456286588928, 140456288272383, -STORE, 94314006716416, 94314006986751, -STORE, 94314006716416, 94314007121919, -STORE, 93948644454400, 93948644667391, -STORE, 93948646764544, 93948646768639, -STORE, 93948646768640, 93948646776831, -STORE, 93948646776832, 93948646789119, -STORE, 93948664999936, 93948667142143, -STORE, 140187350659072, 140187352317951, -STORE, 140187352317952, 140187354415103, -STORE, 140187354415104, 140187354431487, -STORE, 140187354431488, 140187354439679, -STORE, 140187354439680, 140187354456063, -STORE, 140187354456064, 140187354468351, -STORE, 140187354468352, 140187356561407, -STORE, 140187356561408, 140187356565503, -STORE, 140187356565504, 140187356569599, -STORE, 140187356569600, 140187356712959, -STORE, 140187357081600, 140187358765055, -STORE, 140187358765056, 140187358781439, -STORE, 140187358810112, 140187358814207, -STORE, 140187358814208, 140187358818303, -STORE, 140187358818304, 140187358822399, -STORE, 140730484518912, 140730484658175, -STORE, 140730485690368, 140730485702655, -STORE, 140730485702656, 140730485706751, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140721211551744, 140737488351231, -SNULL, 140721211559935, 140737488351231, -STORE, 140721211551744, 140721211559935, -STORE, 140721211420672, 140721211559935, -STORE, 94105221423104, 94105223757823, -SNULL, 94105221636095, 94105223757823, -STORE, 94105221423104, 94105221636095, -STORE, 94105221636096, 94105223757823, -ERASE, 94105221636096, 94105223757823, -STORE, 94105223733248, 94105223745535, -STORE, 94105223745536, 94105223757823, -STORE, 140474453676032, 140474455928831, -SNULL, 140474453819391, 140474455928831, -STORE, 140474453676032, 140474453819391, -STORE, 140474453819392, 140474455928831, -ERASE, 140474453819392, 140474455928831, -STORE, 140474455916544, 140474455924735, -STORE, 140474455924736, 140474455928831, -STORE, 140721211703296, 140721211707391, -STORE, 140721211691008, 140721211703295, -STORE, 140474455887872, 140474455916543, -STORE, 140474455879680, 140474455887871, -STORE, 140474451562496, 140474453676031, -SNULL, 140474451562496, 140474451574783, -STORE, 140474451574784, 140474453676031, -STORE, 140474451562496, 140474451574783, -SNULL, 140474453667839, 140474453676031, -STORE, 140474451574784, 140474453667839, -STORE, 140474453667840, 140474453676031, -ERASE, 140474453667840, 140474453676031, -STORE, 140474453667840, 140474453676031, -STORE, 140474447765504, 140474451562495, -SNULL, 140474447765504, 140474449424383, -STORE, 140474449424384, 140474451562495, -STORE, 140474447765504, 140474449424383, -SNULL, 140474451521535, 140474451562495, -STORE, 140474449424384, 140474451521535, -STORE, 140474451521536, 140474451562495, -SNULL, 140474451521536, 140474451546111, -STORE, 140474451546112, 140474451562495, -STORE, 140474451521536, 140474451546111, -ERASE, 140474451521536, 140474451546111, -STORE, 140474451521536, 140474451546111, -ERASE, 140474451546112, 140474451562495, -STORE, 140474451546112, 140474451562495, -STORE, 140474455871488, 140474455887871, -SNULL, 140474451537919, 140474451546111, -STORE, 140474451521536, 140474451537919, -STORE, 140474451537920, 140474451546111, -SNULL, 140474453671935, 140474453676031, -STORE, 140474453667840, 140474453671935, -STORE, 140474453671936, 140474453676031, -SNULL, 94105223737343, 94105223745535, -STORE, 94105223733248, 94105223737343, -STORE, 94105223737344, 94105223745535, -SNULL, 140474455920639, 140474455924735, -STORE, 140474455916544, 140474455920639, -STORE, 140474455920640, 140474455924735, -ERASE, 140474455887872, 140474455916543, -STORE, 94105238712320, 94105238847487, -STORE, 140474454188032, 140474455871487, -STORE, 94105238712320, 94105238982655, -STORE, 94105238712320, 94105239117823, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140732356354048, 140737488351231, -SNULL, 140732356362239, 140737488351231, -STORE, 140732356354048, 140732356362239, -STORE, 140732356222976, 140732356362239, -STORE, 94461165989888, 94461168324607, -SNULL, 94461166202879, 94461168324607, -STORE, 94461165989888, 94461166202879, -STORE, 94461166202880, 94461168324607, -ERASE, 94461166202880, 94461168324607, -STORE, 94461168300032, 94461168312319, -STORE, 94461168312320, 94461168324607, -STORE, 140317255110656, 140317257363455, -SNULL, 140317255254015, 140317257363455, -STORE, 140317255110656, 140317255254015, -STORE, 140317255254016, 140317257363455, -ERASE, 140317255254016, 140317257363455, -STORE, 140317257351168, 140317257359359, -STORE, 140317257359360, 140317257363455, -STORE, 140732356583424, 140732356587519, -STORE, 140732356571136, 140732356583423, -STORE, 140317257322496, 140317257351167, -STORE, 140317257314304, 140317257322495, -STORE, 140317252997120, 140317255110655, -SNULL, 140317252997120, 140317253009407, -STORE, 140317253009408, 140317255110655, -STORE, 140317252997120, 140317253009407, -SNULL, 140317255102463, 140317255110655, -STORE, 140317253009408, 140317255102463, -STORE, 140317255102464, 140317255110655, -ERASE, 140317255102464, 140317255110655, -STORE, 140317255102464, 140317255110655, -STORE, 140317249200128, 140317252997119, -SNULL, 140317249200128, 140317250859007, -STORE, 140317250859008, 140317252997119, -STORE, 140317249200128, 140317250859007, -SNULL, 140317252956159, 140317252997119, -STORE, 140317250859008, 140317252956159, -STORE, 140317252956160, 140317252997119, -SNULL, 140317252956160, 140317252980735, -STORE, 140317252980736, 140317252997119, -STORE, 140317252956160, 140317252980735, -ERASE, 140317252956160, 140317252980735, -STORE, 140317252956160, 140317252980735, -ERASE, 140317252980736, 140317252997119, -STORE, 140317252980736, 140317252997119, -STORE, 140317257306112, 140317257322495, -SNULL, 140317252972543, 140317252980735, -STORE, 140317252956160, 140317252972543, -STORE, 140317252972544, 140317252980735, -SNULL, 140317255106559, 140317255110655, -STORE, 140317255102464, 140317255106559, -STORE, 140317255106560, 140317255110655, -SNULL, 94461168304127, 94461168312319, -STORE, 94461168300032, 94461168304127, -STORE, 94461168304128, 94461168312319, -SNULL, 140317257355263, 140317257359359, -STORE, 140317257351168, 140317257355263, -STORE, 140317257355264, 140317257359359, -ERASE, 140317257322496, 140317257351167, -STORE, 94461195268096, 94461195403263, -STORE, 140317255622656, 140317257306111, -STORE, 94461195268096, 94461195538431, -STORE, 94461195268096, 94461195673599, -STORE, 94110050402304, 94110050615295, -STORE, 94110052712448, 94110052716543, -STORE, 94110052716544, 94110052724735, -STORE, 94110052724736, 94110052737023, -STORE, 94110061875200, 94110062415871, -STORE, 140139439357952, 140139441016831, -STORE, 140139441016832, 140139443113983, -STORE, 140139443113984, 140139443130367, -STORE, 140139443130368, 140139443138559, -STORE, 140139443138560, 140139443154943, -STORE, 140139443154944, 140139443167231, -STORE, 140139443167232, 140139445260287, -STORE, 140139445260288, 140139445264383, -STORE, 140139445264384, 140139445268479, -STORE, 140139445268480, 140139445411839, -STORE, 140139445780480, 140139447463935, -STORE, 140139447463936, 140139447480319, -STORE, 140139447508992, 140139447513087, -STORE, 140139447513088, 140139447517183, -STORE, 140139447517184, 140139447521279, -STORE, 140731901427712, 140731901566975, -STORE, 140731902259200, 140731902271487, -STORE, 140731902271488, 140731902275583, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140720941613056, 140737488351231, -SNULL, 140720941621247, 140737488351231, -STORE, 140720941613056, 140720941621247, -STORE, 140720941481984, 140720941621247, -STORE, 93902377721856, 93902379945983, -SNULL, 93902377832447, 93902379945983, -STORE, 93902377721856, 93902377832447, -STORE, 93902377832448, 93902379945983, -ERASE, 93902377832448, 93902379945983, -STORE, 93902379925504, 93902379937791, -STORE, 93902379937792, 93902379945983, -STORE, 139836543635456, 139836545888255, -SNULL, 139836543778815, 139836545888255, -STORE, 139836543635456, 139836543778815, -STORE, 139836543778816, 139836545888255, -ERASE, 139836543778816, 139836545888255, -STORE, 139836545875968, 139836545884159, -STORE, 139836545884160, 139836545888255, -STORE, 140720941711360, 140720941715455, -STORE, 140720941699072, 140720941711359, -STORE, 139836545847296, 139836545875967, -STORE, 139836545839104, 139836545847295, -STORE, 139836539838464, 139836543635455, -SNULL, 139836539838464, 139836541497343, -STORE, 139836541497344, 139836543635455, -STORE, 139836539838464, 139836541497343, -SNULL, 139836543594495, 139836543635455, -STORE, 139836541497344, 139836543594495, -STORE, 139836543594496, 139836543635455, -SNULL, 139836543594496, 139836543619071, -STORE, 139836543619072, 139836543635455, -STORE, 139836543594496, 139836543619071, -ERASE, 139836543594496, 139836543619071, -STORE, 139836543594496, 139836543619071, -ERASE, 139836543619072, 139836543635455, -STORE, 139836543619072, 139836543635455, -SNULL, 139836543610879, 139836543619071, -STORE, 139836543594496, 139836543610879, -STORE, 139836543610880, 139836543619071, -SNULL, 93902379933695, 93902379937791, -STORE, 93902379925504, 93902379933695, -STORE, 93902379933696, 93902379937791, -SNULL, 139836545880063, 139836545884159, -STORE, 139836545875968, 139836545880063, -STORE, 139836545880064, 139836545884159, -ERASE, 139836545847296, 139836545875967, -STORE, 93902396891136, 93902397026303, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140736538206208, 140737488351231, -SNULL, 140736538214399, 140737488351231, -STORE, 140736538206208, 140736538214399, -STORE, 140736538075136, 140736538214399, -STORE, 94173471399936, 94173473734655, -SNULL, 94173471612927, 94173473734655, -STORE, 94173471399936, 94173471612927, -STORE, 94173471612928, 94173473734655, -ERASE, 94173471612928, 94173473734655, -STORE, 94173473710080, 94173473722367, -STORE, 94173473722368, 94173473734655, -STORE, 140035513556992, 140035515809791, -SNULL, 140035513700351, 140035515809791, -STORE, 140035513556992, 140035513700351, -STORE, 140035513700352, 140035515809791, -ERASE, 140035513700352, 140035515809791, -STORE, 140035515797504, 140035515805695, -STORE, 140035515805696, 140035515809791, -STORE, 140736538329088, 140736538333183, -STORE, 140736538316800, 140736538329087, -STORE, 140035515768832, 140035515797503, -STORE, 140035515760640, 140035515768831, -STORE, 140035511443456, 140035513556991, -SNULL, 140035511443456, 140035511455743, -STORE, 140035511455744, 140035513556991, -STORE, 140035511443456, 140035511455743, -SNULL, 140035513548799, 140035513556991, -STORE, 140035511455744, 140035513548799, -STORE, 140035513548800, 140035513556991, -ERASE, 140035513548800, 140035513556991, -STORE, 140035513548800, 140035513556991, -STORE, 140035507646464, 140035511443455, -SNULL, 140035507646464, 140035509305343, -STORE, 140035509305344, 140035511443455, -STORE, 140035507646464, 140035509305343, -SNULL, 140035511402495, 140035511443455, -STORE, 140035509305344, 140035511402495, -STORE, 140035511402496, 140035511443455, -SNULL, 140035511402496, 140035511427071, -STORE, 140035511427072, 140035511443455, -STORE, 140035511402496, 140035511427071, -ERASE, 140035511402496, 140035511427071, -STORE, 140035511402496, 140035511427071, -ERASE, 140035511427072, 140035511443455, -STORE, 140035511427072, 140035511443455, -STORE, 140035515752448, 140035515768831, -SNULL, 140035511418879, 140035511427071, -STORE, 140035511402496, 140035511418879, -STORE, 140035511418880, 140035511427071, -SNULL, 140035513552895, 140035513556991, -STORE, 140035513548800, 140035513552895, -STORE, 140035513552896, 140035513556991, -SNULL, 94173473714175, 94173473722367, -STORE, 94173473710080, 94173473714175, -STORE, 94173473714176, 94173473722367, -SNULL, 140035515801599, 140035515805695, -STORE, 140035515797504, 140035515801599, -STORE, 140035515801600, 140035515805695, -ERASE, 140035515768832, 140035515797503, -STORE, 94173478645760, 94173478780927, -STORE, 140035514068992, 140035515752447, -STORE, 94173478645760, 94173478916095, -STORE, 94173478645760, 94173479051263, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140724216176640, 140737488351231, -SNULL, 140724216184831, 140737488351231, -STORE, 140724216176640, 140724216184831, -STORE, 140724216045568, 140724216184831, -STORE, 94870930628608, 94870932963327, -SNULL, 94870930841599, 94870932963327, -STORE, 94870930628608, 94870930841599, -STORE, 94870930841600, 94870932963327, -ERASE, 94870930841600, 94870932963327, -STORE, 94870932938752, 94870932951039, -STORE, 94870932951040, 94870932963327, -STORE, 140453683736576, 140453685989375, -SNULL, 140453683879935, 140453685989375, -STORE, 140453683736576, 140453683879935, -STORE, 140453683879936, 140453685989375, -ERASE, 140453683879936, 140453685989375, -STORE, 140453685977088, 140453685985279, -STORE, 140453685985280, 140453685989375, -STORE, 140724216832000, 140724216836095, -STORE, 140724216819712, 140724216831999, -STORE, 140453685948416, 140453685977087, -STORE, 140453685940224, 140453685948415, -STORE, 140453681623040, 140453683736575, -SNULL, 140453681623040, 140453681635327, -STORE, 140453681635328, 140453683736575, -STORE, 140453681623040, 140453681635327, -SNULL, 140453683728383, 140453683736575, -STORE, 140453681635328, 140453683728383, -STORE, 140453683728384, 140453683736575, -ERASE, 140453683728384, 140453683736575, -STORE, 140453683728384, 140453683736575, -STORE, 140453677826048, 140453681623039, -SNULL, 140453677826048, 140453679484927, -STORE, 140453679484928, 140453681623039, -STORE, 140453677826048, 140453679484927, -SNULL, 140453681582079, 140453681623039, -STORE, 140453679484928, 140453681582079, -STORE, 140453681582080, 140453681623039, -SNULL, 140453681582080, 140453681606655, -STORE, 140453681606656, 140453681623039, -STORE, 140453681582080, 140453681606655, -ERASE, 140453681582080, 140453681606655, -STORE, 140453681582080, 140453681606655, -ERASE, 140453681606656, 140453681623039, -STORE, 140453681606656, 140453681623039, -STORE, 140453685932032, 140453685948415, -SNULL, 140453681598463, 140453681606655, -STORE, 140453681582080, 140453681598463, -STORE, 140453681598464, 140453681606655, -SNULL, 140453683732479, 140453683736575, -STORE, 140453683728384, 140453683732479, -STORE, 140453683732480, 140453683736575, -SNULL, 94870932942847, 94870932951039, -STORE, 94870932938752, 94870932942847, -STORE, 94870932942848, 94870932951039, -SNULL, 140453685981183, 140453685985279, -STORE, 140453685977088, 140453685981183, -STORE, 140453685981184, 140453685985279, -ERASE, 140453685948416, 140453685977087, -STORE, 94870940565504, 94870940700671, -STORE, 140453684248576, 140453685932031, -STORE, 94870940565504, 94870940835839, -STORE, 94870940565504, 94870940971007, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731275661312, 140737488351231, -SNULL, 140731275669503, 140737488351231, -STORE, 140731275661312, 140731275669503, -STORE, 140731275530240, 140731275669503, -STORE, 94642788548608, 94642790883327, -SNULL, 94642788761599, 94642790883327, -STORE, 94642788548608, 94642788761599, -STORE, 94642788761600, 94642790883327, -ERASE, 94642788761600, 94642790883327, -STORE, 94642790858752, 94642790871039, -STORE, 94642790871040, 94642790883327, -STORE, 140228458749952, 140228461002751, -SNULL, 140228458893311, 140228461002751, -STORE, 140228458749952, 140228458893311, -STORE, 140228458893312, 140228461002751, -ERASE, 140228458893312, 140228461002751, -STORE, 140228460990464, 140228460998655, -STORE, 140228460998656, 140228461002751, -STORE, 140731276349440, 140731276353535, -STORE, 140731276337152, 140731276349439, -STORE, 140228460961792, 140228460990463, -STORE, 140228460953600, 140228460961791, -STORE, 140228456636416, 140228458749951, -SNULL, 140228456636416, 140228456648703, -STORE, 140228456648704, 140228458749951, -STORE, 140228456636416, 140228456648703, -SNULL, 140228458741759, 140228458749951, -STORE, 140228456648704, 140228458741759, -STORE, 140228458741760, 140228458749951, -ERASE, 140228458741760, 140228458749951, -STORE, 140228458741760, 140228458749951, -STORE, 140228452839424, 140228456636415, -SNULL, 140228452839424, 140228454498303, -STORE, 140228454498304, 140228456636415, -STORE, 140228452839424, 140228454498303, -SNULL, 140228456595455, 140228456636415, -STORE, 140228454498304, 140228456595455, -STORE, 140228456595456, 140228456636415, -SNULL, 140228456595456, 140228456620031, -STORE, 140228456620032, 140228456636415, -STORE, 140228456595456, 140228456620031, -ERASE, 140228456595456, 140228456620031, -STORE, 140228456595456, 140228456620031, -ERASE, 140228456620032, 140228456636415, -STORE, 140228456620032, 140228456636415, -STORE, 140228460945408, 140228460961791, -SNULL, 140228456611839, 140228456620031, -STORE, 140228456595456, 140228456611839, -STORE, 140228456611840, 140228456620031, -SNULL, 140228458745855, 140228458749951, -STORE, 140228458741760, 140228458745855, -STORE, 140228458745856, 140228458749951, -SNULL, 94642790862847, 94642790871039, -STORE, 94642790858752, 94642790862847, -STORE, 94642790862848, 94642790871039, -SNULL, 140228460994559, 140228460998655, -STORE, 140228460990464, 140228460994559, -STORE, 140228460994560, 140228460998655, -ERASE, 140228460961792, 140228460990463, -STORE, 94642801549312, 94642801684479, -STORE, 140228459261952, 140228460945407, -STORE, 94642801549312, 94642801819647, -STORE, 94642801549312, 94642801954815, -STORE, 94604087611392, 94604087824383, -STORE, 94604089921536, 94604089925631, -STORE, 94604089925632, 94604089933823, -STORE, 94604089933824, 94604089946111, -STORE, 94604105125888, 94604106424319, -STORE, 140454937694208, 140454939353087, -STORE, 140454939353088, 140454941450239, -STORE, 140454941450240, 140454941466623, -STORE, 140454941466624, 140454941474815, -STORE, 140454941474816, 140454941491199, -STORE, 140454941491200, 140454941503487, -STORE, 140454941503488, 140454943596543, -STORE, 140454943596544, 140454943600639, -STORE, 140454943600640, 140454943604735, -STORE, 140454943604736, 140454943748095, -STORE, 140454944116736, 140454945800191, -STORE, 140454945800192, 140454945816575, -STORE, 140454945845248, 140454945849343, -STORE, 140454945849344, 140454945853439, -STORE, 140454945853440, 140454945857535, -STORE, 140728438214656, 140728438353919, -STORE, 140728439095296, 140728439107583, -STORE, 140728439107584, 140728439111679, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140721843453952, 140737488351231, -SNULL, 140721843462143, 140737488351231, -STORE, 140721843453952, 140721843462143, -STORE, 140721843322880, 140721843462143, -STORE, 94465962455040, 94465964789759, -SNULL, 94465962668031, 94465964789759, -STORE, 94465962455040, 94465962668031, -STORE, 94465962668032, 94465964789759, -ERASE, 94465962668032, 94465964789759, -STORE, 94465964765184, 94465964777471, -STORE, 94465964777472, 94465964789759, -STORE, 139913488314368, 139913490567167, -SNULL, 139913488457727, 139913490567167, -STORE, 139913488314368, 139913488457727, -STORE, 139913488457728, 139913490567167, -ERASE, 139913488457728, 139913490567167, -STORE, 139913490554880, 139913490563071, -STORE, 139913490563072, 139913490567167, -STORE, 140721843503104, 140721843507199, -STORE, 140721843490816, 140721843503103, -STORE, 139913490526208, 139913490554879, -STORE, 139913490518016, 139913490526207, -STORE, 139913486200832, 139913488314367, -SNULL, 139913486200832, 139913486213119, -STORE, 139913486213120, 139913488314367, -STORE, 139913486200832, 139913486213119, -SNULL, 139913488306175, 139913488314367, -STORE, 139913486213120, 139913488306175, -STORE, 139913488306176, 139913488314367, -ERASE, 139913488306176, 139913488314367, -STORE, 139913488306176, 139913488314367, -STORE, 139913482403840, 139913486200831, -SNULL, 139913482403840, 139913484062719, -STORE, 139913484062720, 139913486200831, -STORE, 139913482403840, 139913484062719, -SNULL, 139913486159871, 139913486200831, -STORE, 139913484062720, 139913486159871, -STORE, 139913486159872, 139913486200831, -SNULL, 139913486159872, 139913486184447, -STORE, 139913486184448, 139913486200831, -STORE, 139913486159872, 139913486184447, -ERASE, 139913486159872, 139913486184447, -STORE, 139913486159872, 139913486184447, -ERASE, 139913486184448, 139913486200831, -STORE, 139913486184448, 139913486200831, -STORE, 139913490509824, 139913490526207, -SNULL, 139913486176255, 139913486184447, -STORE, 139913486159872, 139913486176255, -STORE, 139913486176256, 139913486184447, -SNULL, 139913488310271, 139913488314367, -STORE, 139913488306176, 139913488310271, -STORE, 139913488310272, 139913488314367, -SNULL, 94465964769279, 94465964777471, -STORE, 94465964765184, 94465964769279, -STORE, 94465964769280, 94465964777471, -SNULL, 139913490558975, 139913490563071, -STORE, 139913490554880, 139913490558975, -STORE, 139913490558976, 139913490563071, -ERASE, 139913490526208, 139913490554879, -STORE, 94465970024448, 94465970159615, -STORE, 139913488826368, 139913490509823, -STORE, 94465970024448, 94465970294783, -STORE, 94465970024448, 94465970429951, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140720583307264, 140737488351231, -SNULL, 140720583315455, 140737488351231, -STORE, 140720583307264, 140720583315455, -STORE, 140720583176192, 140720583315455, -STORE, 94212322082816, 94212324417535, -SNULL, 94212322295807, 94212324417535, -STORE, 94212322082816, 94212322295807, -STORE, 94212322295808, 94212324417535, -ERASE, 94212322295808, 94212324417535, -STORE, 94212324392960, 94212324405247, -STORE, 94212324405248, 94212324417535, -STORE, 139659688538112, 139659690790911, -SNULL, 139659688681471, 139659690790911, -STORE, 139659688538112, 139659688681471, -STORE, 139659688681472, 139659690790911, -ERASE, 139659688681472, 139659690790911, -STORE, 139659690778624, 139659690786815, -STORE, 139659690786816, 139659690790911, -STORE, 140720584781824, 140720584785919, -STORE, 140720584769536, 140720584781823, -STORE, 139659690749952, 139659690778623, -STORE, 139659690741760, 139659690749951, -STORE, 139659686424576, 139659688538111, -SNULL, 139659686424576, 139659686436863, -STORE, 139659686436864, 139659688538111, -STORE, 139659686424576, 139659686436863, -SNULL, 139659688529919, 139659688538111, -STORE, 139659686436864, 139659688529919, -STORE, 139659688529920, 139659688538111, -ERASE, 139659688529920, 139659688538111, -STORE, 139659688529920, 139659688538111, -STORE, 139659682627584, 139659686424575, -SNULL, 139659682627584, 139659684286463, -STORE, 139659684286464, 139659686424575, -STORE, 139659682627584, 139659684286463, -SNULL, 139659686383615, 139659686424575, -STORE, 139659684286464, 139659686383615, -STORE, 139659686383616, 139659686424575, -SNULL, 139659686383616, 139659686408191, -STORE, 139659686408192, 139659686424575, -STORE, 139659686383616, 139659686408191, -ERASE, 139659686383616, 139659686408191, -STORE, 139659686383616, 139659686408191, -ERASE, 139659686408192, 139659686424575, -STORE, 139659686408192, 139659686424575, -STORE, 139659690733568, 139659690749951, -SNULL, 139659686399999, 139659686408191, -STORE, 139659686383616, 139659686399999, -STORE, 139659686400000, 139659686408191, -SNULL, 139659688534015, 139659688538111, -STORE, 139659688529920, 139659688534015, -STORE, 139659688534016, 139659688538111, -SNULL, 94212324397055, 94212324405247, -STORE, 94212324392960, 94212324397055, -STORE, 94212324397056, 94212324405247, -SNULL, 139659690782719, 139659690786815, -STORE, 139659690778624, 139659690782719, -STORE, 139659690782720, 139659690786815, -ERASE, 139659690749952, 139659690778623, -STORE, 94212355014656, 94212355149823, -STORE, 139659689050112, 139659690733567, -STORE, 94212355014656, 94212355284991, -STORE, 94212355014656, 94212355420159, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140727689830400, 140737488351231, -SNULL, 140727689838591, 140737488351231, -STORE, 140727689830400, 140727689838591, -STORE, 140727689699328, 140727689838591, -STORE, 94572390281216, 94572392615935, -SNULL, 94572390494207, 94572392615935, -STORE, 94572390281216, 94572390494207, -STORE, 94572390494208, 94572392615935, -ERASE, 94572390494208, 94572392615935, -STORE, 94572392591360, 94572392603647, -STORE, 94572392603648, 94572392615935, -STORE, 140575923769344, 140575926022143, -SNULL, 140575923912703, 140575926022143, -STORE, 140575923769344, 140575923912703, -STORE, 140575923912704, 140575926022143, -ERASE, 140575923912704, 140575926022143, -STORE, 140575926009856, 140575926018047, -STORE, 140575926018048, 140575926022143, -STORE, 140727689871360, 140727689875455, -STORE, 140727689859072, 140727689871359, -STORE, 140575925981184, 140575926009855, -STORE, 140575925972992, 140575925981183, -STORE, 140575921655808, 140575923769343, -SNULL, 140575921655808, 140575921668095, -STORE, 140575921668096, 140575923769343, -STORE, 140575921655808, 140575921668095, -SNULL, 140575923761151, 140575923769343, -STORE, 140575921668096, 140575923761151, -STORE, 140575923761152, 140575923769343, -ERASE, 140575923761152, 140575923769343, -STORE, 140575923761152, 140575923769343, -STORE, 140575917858816, 140575921655807, -SNULL, 140575917858816, 140575919517695, -STORE, 140575919517696, 140575921655807, -STORE, 140575917858816, 140575919517695, -SNULL, 140575921614847, 140575921655807, -STORE, 140575919517696, 140575921614847, -STORE, 140575921614848, 140575921655807, -SNULL, 140575921614848, 140575921639423, -STORE, 140575921639424, 140575921655807, -STORE, 140575921614848, 140575921639423, -ERASE, 140575921614848, 140575921639423, -STORE, 140575921614848, 140575921639423, -ERASE, 140575921639424, 140575921655807, -STORE, 140575921639424, 140575921655807, -STORE, 140575925964800, 140575925981183, -SNULL, 140575921631231, 140575921639423, -STORE, 140575921614848, 140575921631231, -STORE, 140575921631232, 140575921639423, -SNULL, 140575923765247, 140575923769343, -STORE, 140575923761152, 140575923765247, -STORE, 140575923765248, 140575923769343, -SNULL, 94572392595455, 94572392603647, -STORE, 94572392591360, 94572392595455, -STORE, 94572392595456, 94572392603647, -SNULL, 140575926013951, 140575926018047, -STORE, 140575926009856, 140575926013951, -STORE, 140575926013952, 140575926018047, -ERASE, 140575925981184, 140575926009855, -STORE, 94572402278400, 94572402413567, -STORE, 140575924281344, 140575925964799, -STORE, 94572402278400, 94572402548735, -STORE, 94572402278400, 94572402683903, -STORE, 94572402278400, 94572402851839, -SNULL, 94572402827263, 94572402851839, -STORE, 94572402278400, 94572402827263, -STORE, 94572402827264, 94572402851839, -ERASE, 94572402827264, 94572402851839, -STORE, 94572402278400, 94572402966527, -STORE, 94572402278400, 94572403109887, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140725520506880, 140737488351231, -SNULL, 140725520515071, 140737488351231, -STORE, 140725520506880, 140725520515071, -STORE, 140725520375808, 140725520515071, -STORE, 93829948788736, 93829951012863, -SNULL, 93829948899327, 93829951012863, -STORE, 93829948788736, 93829948899327, -STORE, 93829948899328, 93829951012863, -ERASE, 93829948899328, 93829951012863, -STORE, 93829950992384, 93829951004671, -STORE, 93829951004672, 93829951012863, -STORE, 140133696794624, 140133699047423, -SNULL, 140133696937983, 140133699047423, -STORE, 140133696794624, 140133696937983, -STORE, 140133696937984, 140133699047423, -ERASE, 140133696937984, 140133699047423, -STORE, 140133699035136, 140133699043327, -STORE, 140133699043328, 140133699047423, -STORE, 140725520875520, 140725520879615, -STORE, 140725520863232, 140725520875519, -STORE, 140133699006464, 140133699035135, -STORE, 140133698998272, 140133699006463, -STORE, 140133692997632, 140133696794623, -SNULL, 140133692997632, 140133694656511, -STORE, 140133694656512, 140133696794623, -STORE, 140133692997632, 140133694656511, -SNULL, 140133696753663, 140133696794623, -STORE, 140133694656512, 140133696753663, -STORE, 140133696753664, 140133696794623, -SNULL, 140133696753664, 140133696778239, -STORE, 140133696778240, 140133696794623, -STORE, 140133696753664, 140133696778239, -ERASE, 140133696753664, 140133696778239, -STORE, 140133696753664, 140133696778239, -ERASE, 140133696778240, 140133696794623, -STORE, 140133696778240, 140133696794623, -SNULL, 140133696770047, 140133696778239, -STORE, 140133696753664, 140133696770047, -STORE, 140133696770048, 140133696778239, -SNULL, 93829951000575, 93829951004671, -STORE, 93829950992384, 93829951000575, -STORE, 93829951000576, 93829951004671, -SNULL, 140133699039231, 140133699043327, -STORE, 140133699035136, 140133699039231, -STORE, 140133699039232, 140133699043327, -ERASE, 140133699006464, 140133699035135, -STORE, 93829978693632, 93829978828799, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140736118022144, 140737488351231, -SNULL, 140736118030335, 140737488351231, -STORE, 140736118022144, 140736118030335, -STORE, 140736117891072, 140736118030335, -STORE, 94467663982592, 94467666206719, -SNULL, 94467664093183, 94467666206719, -STORE, 94467663982592, 94467664093183, -STORE, 94467664093184, 94467666206719, -ERASE, 94467664093184, 94467666206719, -STORE, 94467666186240, 94467666198527, -STORE, 94467666198528, 94467666206719, -STORE, 140525377327104, 140525379579903, -SNULL, 140525377470463, 140525379579903, -STORE, 140525377327104, 140525377470463, -STORE, 140525377470464, 140525379579903, -ERASE, 140525377470464, 140525379579903, -STORE, 140525379567616, 140525379575807, -STORE, 140525379575808, 140525379579903, -STORE, 140736118771712, 140736118775807, -STORE, 140736118759424, 140736118771711, -STORE, 140525379538944, 140525379567615, -STORE, 140525379530752, 140525379538943, -STORE, 140525373530112, 140525377327103, -SNULL, 140525373530112, 140525375188991, -STORE, 140525375188992, 140525377327103, -STORE, 140525373530112, 140525375188991, -SNULL, 140525377286143, 140525377327103, -STORE, 140525375188992, 140525377286143, -STORE, 140525377286144, 140525377327103, -SNULL, 140525377286144, 140525377310719, -STORE, 140525377310720, 140525377327103, -STORE, 140525377286144, 140525377310719, -ERASE, 140525377286144, 140525377310719, -STORE, 140525377286144, 140525377310719, -ERASE, 140525377310720, 140525377327103, -STORE, 140525377310720, 140525377327103, -SNULL, 140525377302527, 140525377310719, -STORE, 140525377286144, 140525377302527, -STORE, 140525377302528, 140525377310719, -SNULL, 94467666194431, 94467666198527, -STORE, 94467666186240, 94467666194431, -STORE, 94467666194432, 94467666198527, -SNULL, 140525379571711, 140525379575807, -STORE, 140525379567616, 140525379571711, -STORE, 140525379571712, 140525379575807, -ERASE, 140525379538944, 140525379567615, -STORE, 94467693379584, 94467693514751, -STORE, 94200172744704, 94200172957695, -STORE, 94200175054848, 94200175058943, -STORE, 94200175058944, 94200175067135, -STORE, 94200175067136, 94200175079423, -STORE, 94200196673536, 94200198905855, -STORE, 140053867720704, 140053869379583, -STORE, 140053869379584, 140053871476735, -STORE, 140053871476736, 140053871493119, -STORE, 140053871493120, 140053871501311, -STORE, 140053871501312, 140053871517695, -STORE, 140053871517696, 140053871529983, -STORE, 140053871529984, 140053873623039, -STORE, 140053873623040, 140053873627135, -STORE, 140053873627136, 140053873631231, -STORE, 140053873631232, 140053873774591, -STORE, 140053874143232, 140053875826687, -STORE, 140053875826688, 140053875843071, -STORE, 140053875871744, 140053875875839, -STORE, 140053875875840, 140053875879935, -STORE, 140053875879936, 140053875884031, -STORE, 140728538484736, 140728538623999, -STORE, 140728538652672, 140728538664959, -STORE, 140728538664960, 140728538669055, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140732307775488, 140737488351231, -SNULL, 140732307783679, 140737488351231, -STORE, 140732307775488, 140732307783679, -STORE, 140732307644416, 140732307783679, -STORE, 93831417630720, 93831419965439, -SNULL, 93831417843711, 93831419965439, -STORE, 93831417630720, 93831417843711, -STORE, 93831417843712, 93831419965439, -ERASE, 93831417843712, 93831419965439, -STORE, 93831419940864, 93831419953151, -STORE, 93831419953152, 93831419965439, -STORE, 140241062088704, 140241064341503, -SNULL, 140241062232063, 140241064341503, -STORE, 140241062088704, 140241062232063, -STORE, 140241062232064, 140241064341503, -ERASE, 140241062232064, 140241064341503, -STORE, 140241064329216, 140241064337407, -STORE, 140241064337408, 140241064341503, -STORE, 140732308140032, 140732308144127, -STORE, 140732308127744, 140732308140031, -STORE, 140241064300544, 140241064329215, -STORE, 140241064292352, 140241064300543, -STORE, 140241059975168, 140241062088703, -SNULL, 140241059975168, 140241059987455, -STORE, 140241059987456, 140241062088703, -STORE, 140241059975168, 140241059987455, -SNULL, 140241062080511, 140241062088703, -STORE, 140241059987456, 140241062080511, -STORE, 140241062080512, 140241062088703, -ERASE, 140241062080512, 140241062088703, -STORE, 140241062080512, 140241062088703, -STORE, 140241056178176, 140241059975167, -SNULL, 140241056178176, 140241057837055, -STORE, 140241057837056, 140241059975167, -STORE, 140241056178176, 140241057837055, -SNULL, 140241059934207, 140241059975167, -STORE, 140241057837056, 140241059934207, -STORE, 140241059934208, 140241059975167, -SNULL, 140241059934208, 140241059958783, -STORE, 140241059958784, 140241059975167, -STORE, 140241059934208, 140241059958783, -ERASE, 140241059934208, 140241059958783, -STORE, 140241059934208, 140241059958783, -ERASE, 140241059958784, 140241059975167, -STORE, 140241059958784, 140241059975167, -STORE, 140241064284160, 140241064300543, -SNULL, 140241059950591, 140241059958783, -STORE, 140241059934208, 140241059950591, -STORE, 140241059950592, 140241059958783, -SNULL, 140241062084607, 140241062088703, -STORE, 140241062080512, 140241062084607, -STORE, 140241062084608, 140241062088703, -SNULL, 93831419944959, 93831419953151, -STORE, 93831419940864, 93831419944959, -STORE, 93831419944960, 93831419953151, -SNULL, 140241064333311, 140241064337407, -STORE, 140241064329216, 140241064333311, -STORE, 140241064333312, 140241064337407, -ERASE, 140241064300544, 140241064329215, -STORE, 93831435284480, 93831435419647, -STORE, 140241062600704, 140241064284159, -STORE, 93831435284480, 93831435554815, -STORE, 93831435284480, 93831435689983, -STORE, 93831435284480, 93831435862015, -SNULL, 93831435837439, 93831435862015, -STORE, 93831435284480, 93831435837439, -STORE, 93831435837440, 93831435862015, -ERASE, 93831435837440, 93831435862015, -STORE, 93831435284480, 93831435972607, -STORE, 93831435284480, 93831436107775, -SNULL, 93831436091391, 93831436107775, -STORE, 93831435284480, 93831436091391, -STORE, 93831436091392, 93831436107775, -ERASE, 93831436091392, 93831436107775, -STORE, 93831435284480, 93831436226559, -STORE, 93831435284480, 93831436361727, -STORE, 93831435284480, 93831436505087, -STORE, 93831435284480, 93831436652543, -STORE, 93831435284480, 93831436787711, -STORE, 93831435284480, 93831436926975, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140728546775040, 140737488351231, -SNULL, 140728546783231, 140737488351231, -STORE, 140728546775040, 140728546783231, -STORE, 140728546643968, 140728546783231, -STORE, 94456178786304, 94456181010431, -SNULL, 94456178896895, 94456181010431, -STORE, 94456178786304, 94456178896895, -STORE, 94456178896896, 94456181010431, -ERASE, 94456178896896, 94456181010431, -STORE, 94456180989952, 94456181002239, -STORE, 94456181002240, 94456181010431, -STORE, 140221893091328, 140221895344127, -SNULL, 140221893234687, 140221895344127, -STORE, 140221893091328, 140221893234687, -STORE, 140221893234688, 140221895344127, -ERASE, 140221893234688, 140221895344127, -STORE, 140221895331840, 140221895340031, -STORE, 140221895340032, 140221895344127, -STORE, 140728547803136, 140728547807231, -STORE, 140728547790848, 140728547803135, -STORE, 140221895303168, 140221895331839, -STORE, 140221895294976, 140221895303167, -STORE, 140221889294336, 140221893091327, -SNULL, 140221889294336, 140221890953215, -STORE, 140221890953216, 140221893091327, -STORE, 140221889294336, 140221890953215, -SNULL, 140221893050367, 140221893091327, -STORE, 140221890953216, 140221893050367, -STORE, 140221893050368, 140221893091327, -SNULL, 140221893050368, 140221893074943, -STORE, 140221893074944, 140221893091327, -STORE, 140221893050368, 140221893074943, -ERASE, 140221893050368, 140221893074943, -STORE, 140221893050368, 140221893074943, -ERASE, 140221893074944, 140221893091327, -STORE, 140221893074944, 140221893091327, -SNULL, 140221893066751, 140221893074943, -STORE, 140221893050368, 140221893066751, -STORE, 140221893066752, 140221893074943, -SNULL, 94456180998143, 94456181002239, -STORE, 94456180989952, 94456180998143, -STORE, 94456180998144, 94456181002239, -SNULL, 140221895335935, 140221895340031, -STORE, 140221895331840, 140221895335935, -STORE, 140221895335936, 140221895340031, -ERASE, 140221895303168, 140221895331839, -STORE, 94456203730944, 94456203866111, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140734438637568, 140737488351231, -SNULL, 140734438645759, 140737488351231, -STORE, 140734438637568, 140734438645759, -STORE, 140734438506496, 140734438645759, -STORE, 94652233351168, 94652235575295, -SNULL, 94652233461759, 94652235575295, -STORE, 94652233351168, 94652233461759, -STORE, 94652233461760, 94652235575295, -ERASE, 94652233461760, 94652235575295, -STORE, 94652235554816, 94652235567103, -STORE, 94652235567104, 94652235575295, -STORE, 140536493195264, 140536495448063, -SNULL, 140536493338623, 140536495448063, -STORE, 140536493195264, 140536493338623, -STORE, 140536493338624, 140536495448063, -ERASE, 140536493338624, 140536495448063, -STORE, 140536495435776, 140536495443967, -STORE, 140536495443968, 140536495448063, -STORE, 140734439002112, 140734439006207, -STORE, 140734438989824, 140734439002111, -STORE, 140536495407104, 140536495435775, -STORE, 140536495398912, 140536495407103, -STORE, 140536489398272, 140536493195263, -SNULL, 140536489398272, 140536491057151, -STORE, 140536491057152, 140536493195263, -STORE, 140536489398272, 140536491057151, -SNULL, 140536493154303, 140536493195263, -STORE, 140536491057152, 140536493154303, -STORE, 140536493154304, 140536493195263, -SNULL, 140536493154304, 140536493178879, -STORE, 140536493178880, 140536493195263, -STORE, 140536493154304, 140536493178879, -ERASE, 140536493154304, 140536493178879, -STORE, 140536493154304, 140536493178879, -ERASE, 140536493178880, 140536493195263, -STORE, 140536493178880, 140536493195263, -SNULL, 140536493170687, 140536493178879, -STORE, 140536493154304, 140536493170687, -STORE, 140536493170688, 140536493178879, -SNULL, 94652235563007, 94652235567103, -STORE, 94652235554816, 94652235563007, -STORE, 94652235563008, 94652235567103, -SNULL, 140536495439871, 140536495443967, -STORE, 140536495435776, 140536495439871, -STORE, 140536495439872, 140536495443967, -ERASE, 140536495407104, 140536495435775, -STORE, 94652265619456, 94652265754623, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140721814200320, 140737488351231, -SNULL, 140721814208511, 140737488351231, -STORE, 140721814200320, 140721814208511, -STORE, 140721814069248, 140721814208511, -STORE, 94062800691200, 94062802915327, -SNULL, 94062800801791, 94062802915327, -STORE, 94062800691200, 94062800801791, -STORE, 94062800801792, 94062802915327, -ERASE, 94062800801792, 94062802915327, -STORE, 94062802894848, 94062802907135, -STORE, 94062802907136, 94062802915327, -STORE, 139717739700224, 139717741953023, -SNULL, 139717739843583, 139717741953023, -STORE, 139717739700224, 139717739843583, -STORE, 139717739843584, 139717741953023, -ERASE, 139717739843584, 139717741953023, -STORE, 139717741940736, 139717741948927, -STORE, 139717741948928, 139717741953023, -STORE, 140721814224896, 140721814228991, -STORE, 140721814212608, 140721814224895, -STORE, 139717741912064, 139717741940735, -STORE, 139717741903872, 139717741912063, -STORE, 139717735903232, 139717739700223, -SNULL, 139717735903232, 139717737562111, -STORE, 139717737562112, 139717739700223, -STORE, 139717735903232, 139717737562111, -SNULL, 139717739659263, 139717739700223, -STORE, 139717737562112, 139717739659263, -STORE, 139717739659264, 139717739700223, -SNULL, 139717739659264, 139717739683839, -STORE, 139717739683840, 139717739700223, -STORE, 139717739659264, 139717739683839, -ERASE, 139717739659264, 139717739683839, -STORE, 139717739659264, 139717739683839, -ERASE, 139717739683840, 139717739700223, -STORE, 139717739683840, 139717739700223, -SNULL, 139717739675647, 139717739683839, -STORE, 139717739659264, 139717739675647, -STORE, 139717739675648, 139717739683839, -SNULL, 94062802903039, 94062802907135, -STORE, 94062802894848, 94062802903039, -STORE, 94062802903040, 94062802907135, -SNULL, 139717741944831, 139717741948927, -STORE, 139717741940736, 139717741944831, -STORE, 139717741944832, 139717741948927, -ERASE, 139717741912064, 139717741940735, -STORE, 94062814060544, 94062814195711, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723945754624, 140737488351231, -SNULL, 140723945762815, 140737488351231, -STORE, 140723945754624, 140723945762815, -STORE, 140723945623552, 140723945762815, -STORE, 94886119305216, 94886121639935, -SNULL, 94886119518207, 94886121639935, -STORE, 94886119305216, 94886119518207, -STORE, 94886119518208, 94886121639935, -ERASE, 94886119518208, 94886121639935, -STORE, 94886121615360, 94886121627647, -STORE, 94886121627648, 94886121639935, -STORE, 140152532131840, 140152534384639, -SNULL, 140152532275199, 140152534384639, -STORE, 140152532131840, 140152532275199, -STORE, 140152532275200, 140152534384639, -ERASE, 140152532275200, 140152534384639, -STORE, 140152534372352, 140152534380543, -STORE, 140152534380544, 140152534384639, -STORE, 140723946213376, 140723946217471, -STORE, 140723946201088, 140723946213375, -STORE, 140152534343680, 140152534372351, -STORE, 140152534335488, 140152534343679, -STORE, 140152530018304, 140152532131839, -SNULL, 140152530018304, 140152530030591, -STORE, 140152530030592, 140152532131839, -STORE, 140152530018304, 140152530030591, -SNULL, 140152532123647, 140152532131839, -STORE, 140152530030592, 140152532123647, -STORE, 140152532123648, 140152532131839, -ERASE, 140152532123648, 140152532131839, -STORE, 140152532123648, 140152532131839, -STORE, 140152526221312, 140152530018303, -SNULL, 140152526221312, 140152527880191, -STORE, 140152527880192, 140152530018303, -STORE, 140152526221312, 140152527880191, -SNULL, 140152529977343, 140152530018303, -STORE, 140152527880192, 140152529977343, -STORE, 140152529977344, 140152530018303, -SNULL, 140152529977344, 140152530001919, -STORE, 140152530001920, 140152530018303, -STORE, 140152529977344, 140152530001919, -ERASE, 140152529977344, 140152530001919, -STORE, 140152529977344, 140152530001919, -ERASE, 140152530001920, 140152530018303, -STORE, 140152530001920, 140152530018303, -STORE, 140152534327296, 140152534343679, -SNULL, 140152529993727, 140152530001919, -STORE, 140152529977344, 140152529993727, -STORE, 140152529993728, 140152530001919, -SNULL, 140152532127743, 140152532131839, -STORE, 140152532123648, 140152532127743, -STORE, 140152532127744, 140152532131839, -SNULL, 94886121619455, 94886121627647, -STORE, 94886121615360, 94886121619455, -STORE, 94886121619456, 94886121627647, -SNULL, 140152534376447, 140152534380543, -STORE, 140152534372352, 140152534376447, -STORE, 140152534376448, 140152534380543, -ERASE, 140152534343680, 140152534372351, -STORE, 94886129770496, 94886129905663, -STORE, 140152532643840, 140152534327295, -STORE, 94886129770496, 94886130040831, -STORE, 94886129770496, 94886130175999, -STORE, 94886129770496, 94886130348031, -SNULL, 94886130323455, 94886130348031, -STORE, 94886129770496, 94886130323455, -STORE, 94886130323456, 94886130348031, -ERASE, 94886130323456, 94886130348031, -STORE, 94886129770496, 94886130458623, -STORE, 94886129770496, 94886130606079, -SNULL, 94886130573311, 94886130606079, -STORE, 94886129770496, 94886130573311, -STORE, 94886130573312, 94886130606079, -ERASE, 94886130573312, 94886130606079, -STORE, 94886129770496, 94886130724863, -STORE, 94886129770496, 94886130876415, -STORE, 94886129770496, 94886131023871, -STORE, 94886129770496, 94886131175423, -STORE, 94886129770496, 94886131318783, -STORE, 94886129770496, 94886131453951, -SNULL, 94886131449855, 94886131453951, -STORE, 94886129770496, 94886131449855, -STORE, 94886131449856, 94886131453951, -ERASE, 94886131449856, 94886131453951, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735450779648, 140737488351231, -SNULL, 140735450787839, 140737488351231, -STORE, 140735450779648, 140735450787839, -STORE, 140735450648576, 140735450787839, -STORE, 93947794079744, 93947796414463, -SNULL, 93947794292735, 93947796414463, -STORE, 93947794079744, 93947794292735, -STORE, 93947794292736, 93947796414463, -ERASE, 93947794292736, 93947796414463, -STORE, 93947796389888, 93947796402175, -STORE, 93947796402176, 93947796414463, -STORE, 139841993433088, 139841995685887, -SNULL, 139841993576447, 139841995685887, -STORE, 139841993433088, 139841993576447, -STORE, 139841993576448, 139841995685887, -ERASE, 139841993576448, 139841995685887, -STORE, 139841995673600, 139841995681791, -STORE, 139841995681792, 139841995685887, -STORE, 140735451308032, 140735451312127, -STORE, 140735451295744, 140735451308031, -STORE, 139841995644928, 139841995673599, -STORE, 139841995636736, 139841995644927, -STORE, 139841991319552, 139841993433087, -SNULL, 139841991319552, 139841991331839, -STORE, 139841991331840, 139841993433087, -STORE, 139841991319552, 139841991331839, -SNULL, 139841993424895, 139841993433087, -STORE, 139841991331840, 139841993424895, -STORE, 139841993424896, 139841993433087, -ERASE, 139841993424896, 139841993433087, -STORE, 139841993424896, 139841993433087, -STORE, 139841987522560, 139841991319551, -SNULL, 139841987522560, 139841989181439, -STORE, 139841989181440, 139841991319551, -STORE, 139841987522560, 139841989181439, -SNULL, 139841991278591, 139841991319551, -STORE, 139841989181440, 139841991278591, -STORE, 139841991278592, 139841991319551, -SNULL, 139841991278592, 139841991303167, -STORE, 139841991303168, 139841991319551, -STORE, 139841991278592, 139841991303167, -ERASE, 139841991278592, 139841991303167, -STORE, 139841991278592, 139841991303167, -ERASE, 139841991303168, 139841991319551, -STORE, 139841991303168, 139841991319551, -STORE, 139841995628544, 139841995644927, -SNULL, 139841991294975, 139841991303167, -STORE, 139841991278592, 139841991294975, -STORE, 139841991294976, 139841991303167, -SNULL, 139841993428991, 139841993433087, -STORE, 139841993424896, 139841993428991, -STORE, 139841993428992, 139841993433087, -SNULL, 93947796393983, 93947796402175, -STORE, 93947796389888, 93947796393983, -STORE, 93947796393984, 93947796402175, -SNULL, 139841995677695, 139841995681791, -STORE, 139841995673600, 139841995677695, -STORE, 139841995677696, 139841995681791, -ERASE, 139841995644928, 139841995673599, -STORE, 93947829739520, 93947829874687, -STORE, 139841993945088, 139841995628543, -STORE, 93947829739520, 93947830009855, -STORE, 93947829739520, 93947830145023, -STORE, 94659351814144, 94659352027135, -STORE, 94659354124288, 94659354128383, -STORE, 94659354128384, 94659354136575, -STORE, 94659354136576, 94659354148863, -STORE, 94659383476224, 94659385057279, -STORE, 139959054557184, 139959056216063, -STORE, 139959056216064, 139959058313215, -STORE, 139959058313216, 139959058329599, -STORE, 139959058329600, 139959058337791, -STORE, 139959058337792, 139959058354175, -STORE, 139959058354176, 139959058366463, -STORE, 139959058366464, 139959060459519, -STORE, 139959060459520, 139959060463615, -STORE, 139959060463616, 139959060467711, -STORE, 139959060467712, 139959060611071, -STORE, 139959060979712, 139959062663167, -STORE, 139959062663168, 139959062679551, -STORE, 139959062708224, 139959062712319, -STORE, 139959062712320, 139959062716415, -STORE, 139959062716416, 139959062720511, -STORE, 140735532539904, 140735532679167, -STORE, 140735532830720, 140735532843007, -STORE, 140735532843008, 140735532847103, -STORE, 93894361829376, 93894362042367, -STORE, 93894364139520, 93894364143615, -STORE, 93894364143616, 93894364151807, -STORE, 93894364151808, 93894364164095, -STORE, 93894396944384, 93894397624319, -STORE, 140075612573696, 140075614232575, -STORE, 140075614232576, 140075616329727, -STORE, 140075616329728, 140075616346111, -STORE, 140075616346112, 140075616354303, -STORE, 140075616354304, 140075616370687, -STORE, 140075616370688, 140075616382975, -STORE, 140075616382976, 140075618476031, -STORE, 140075618476032, 140075618480127, -STORE, 140075618480128, 140075618484223, -STORE, 140075618484224, 140075618627583, -STORE, 140075618996224, 140075620679679, -STORE, 140075620679680, 140075620696063, -STORE, 140075620724736, 140075620728831, -STORE, 140075620728832, 140075620732927, -STORE, 140075620732928, 140075620737023, -STORE, 140720830312448, 140720830451711, -STORE, 140720830631936, 140720830644223, -STORE, 140720830644224, 140720830648319, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735116226560, 140737488351231, -SNULL, 140735116234751, 140737488351231, -STORE, 140735116226560, 140735116234751, -STORE, 140735116095488, 140735116234751, -STORE, 94873398054912, 94873400279039, -SNULL, 94873398165503, 94873400279039, -STORE, 94873398054912, 94873398165503, -STORE, 94873398165504, 94873400279039, -ERASE, 94873398165504, 94873400279039, -STORE, 94873400258560, 94873400270847, -STORE, 94873400270848, 94873400279039, -STORE, 140303828606976, 140303830859775, -SNULL, 140303828750335, 140303830859775, -STORE, 140303828606976, 140303828750335, -STORE, 140303828750336, 140303830859775, -ERASE, 140303828750336, 140303830859775, -STORE, 140303830847488, 140303830855679, -STORE, 140303830855680, 140303830859775, -STORE, 140735116251136, 140735116255231, -STORE, 140735116238848, 140735116251135, -STORE, 140303830818816, 140303830847487, -STORE, 140303830810624, 140303830818815, -STORE, 140303824809984, 140303828606975, -SNULL, 140303824809984, 140303826468863, -STORE, 140303826468864, 140303828606975, -STORE, 140303824809984, 140303826468863, -SNULL, 140303828566015, 140303828606975, -STORE, 140303826468864, 140303828566015, -STORE, 140303828566016, 140303828606975, -SNULL, 140303828566016, 140303828590591, -STORE, 140303828590592, 140303828606975, -STORE, 140303828566016, 140303828590591, -ERASE, 140303828566016, 140303828590591, -STORE, 140303828566016, 140303828590591, -ERASE, 140303828590592, 140303828606975, -STORE, 140303828590592, 140303828606975, -SNULL, 140303828582399, 140303828590591, -STORE, 140303828566016, 140303828582399, -STORE, 140303828582400, 140303828590591, -SNULL, 94873400266751, 94873400270847, -STORE, 94873400258560, 94873400266751, -STORE, 94873400266752, 94873400270847, -SNULL, 140303830851583, 140303830855679, -STORE, 140303830847488, 140303830851583, -STORE, 140303830851584, 140303830855679, -ERASE, 140303830818816, 140303830847487, -STORE, 94873413713920, 94873413849087, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140732349956096, 140737488351231, -SNULL, 140732349964287, 140737488351231, -STORE, 140732349956096, 140732349964287, -STORE, 140732349825024, 140732349964287, -STORE, 94009652736000, 94009655070719, -SNULL, 94009652948991, 94009655070719, -STORE, 94009652736000, 94009652948991, -STORE, 94009652948992, 94009655070719, -ERASE, 94009652948992, 94009655070719, -STORE, 94009655046144, 94009655058431, -STORE, 94009655058432, 94009655070719, -STORE, 140295688531968, 140295690784767, -SNULL, 140295688675327, 140295690784767, -STORE, 140295688531968, 140295688675327, -STORE, 140295688675328, 140295690784767, -ERASE, 140295688675328, 140295690784767, -STORE, 140295690772480, 140295690780671, -STORE, 140295690780672, 140295690784767, -STORE, 140732350005248, 140732350009343, -STORE, 140732349992960, 140732350005247, -STORE, 140295690743808, 140295690772479, -STORE, 140295690735616, 140295690743807, -STORE, 140295686418432, 140295688531967, -SNULL, 140295686418432, 140295686430719, -STORE, 140295686430720, 140295688531967, -STORE, 140295686418432, 140295686430719, -SNULL, 140295688523775, 140295688531967, -STORE, 140295686430720, 140295688523775, -STORE, 140295688523776, 140295688531967, -ERASE, 140295688523776, 140295688531967, -STORE, 140295688523776, 140295688531967, -STORE, 140295682621440, 140295686418431, -SNULL, 140295682621440, 140295684280319, -STORE, 140295684280320, 140295686418431, -STORE, 140295682621440, 140295684280319, -SNULL, 140295686377471, 140295686418431, -STORE, 140295684280320, 140295686377471, -STORE, 140295686377472, 140295686418431, -SNULL, 140295686377472, 140295686402047, -STORE, 140295686402048, 140295686418431, -STORE, 140295686377472, 140295686402047, -ERASE, 140295686377472, 140295686402047, -STORE, 140295686377472, 140295686402047, -ERASE, 140295686402048, 140295686418431, -STORE, 140295686402048, 140295686418431, -STORE, 140295690727424, 140295690743807, -SNULL, 140295686393855, 140295686402047, -STORE, 140295686377472, 140295686393855, -STORE, 140295686393856, 140295686402047, -SNULL, 140295688527871, 140295688531967, -STORE, 140295688523776, 140295688527871, -STORE, 140295688527872, 140295688531967, -SNULL, 94009655050239, 94009655058431, -STORE, 94009655046144, 94009655050239, -STORE, 94009655050240, 94009655058431, -SNULL, 140295690776575, 140295690780671, -STORE, 140295690772480, 140295690776575, -STORE, 140295690776576, 140295690780671, -ERASE, 140295690743808, 140295690772479, -STORE, 94009672114176, 94009672249343, -STORE, 140295689043968, 140295690727423, -STORE, 94009672114176, 94009672384511, -STORE, 94009672114176, 94009672519679, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722376515584, 140737488351231, -SNULL, 140722376523775, 140737488351231, -STORE, 140722376515584, 140722376523775, -STORE, 140722376384512, 140722376523775, -STORE, 94089815773184, 94089818107903, -SNULL, 94089815986175, 94089818107903, -STORE, 94089815773184, 94089815986175, -STORE, 94089815986176, 94089818107903, -ERASE, 94089815986176, 94089818107903, -STORE, 94089818083328, 94089818095615, -STORE, 94089818095616, 94089818107903, -STORE, 140265595711488, 140265597964287, -SNULL, 140265595854847, 140265597964287, -STORE, 140265595711488, 140265595854847, -STORE, 140265595854848, 140265597964287, -ERASE, 140265595854848, 140265597964287, -STORE, 140265597952000, 140265597960191, -STORE, 140265597960192, 140265597964287, -STORE, 140722378297344, 140722378301439, -STORE, 140722378285056, 140722378297343, -STORE, 140265597923328, 140265597951999, -STORE, 140265597915136, 140265597923327, -STORE, 140265593597952, 140265595711487, -SNULL, 140265593597952, 140265593610239, -STORE, 140265593610240, 140265595711487, -STORE, 140265593597952, 140265593610239, -SNULL, 140265595703295, 140265595711487, -STORE, 140265593610240, 140265595703295, -STORE, 140265595703296, 140265595711487, -ERASE, 140265595703296, 140265595711487, -STORE, 140265595703296, 140265595711487, -STORE, 140265589800960, 140265593597951, -SNULL, 140265589800960, 140265591459839, -STORE, 140265591459840, 140265593597951, -STORE, 140265589800960, 140265591459839, -SNULL, 140265593556991, 140265593597951, -STORE, 140265591459840, 140265593556991, -STORE, 140265593556992, 140265593597951, -SNULL, 140265593556992, 140265593581567, -STORE, 140265593581568, 140265593597951, -STORE, 140265593556992, 140265593581567, -ERASE, 140265593556992, 140265593581567, -STORE, 140265593556992, 140265593581567, -ERASE, 140265593581568, 140265593597951, -STORE, 140265593581568, 140265593597951, -STORE, 140265597906944, 140265597923327, -SNULL, 140265593573375, 140265593581567, -STORE, 140265593556992, 140265593573375, -STORE, 140265593573376, 140265593581567, -SNULL, 140265595707391, 140265595711487, -STORE, 140265595703296, 140265595707391, -STORE, 140265595707392, 140265595711487, -SNULL, 94089818087423, 94089818095615, -STORE, 94089818083328, 94089818087423, -STORE, 94089818087424, 94089818095615, -SNULL, 140265597956095, 140265597960191, -STORE, 140265597952000, 140265597956095, -STORE, 140265597956096, 140265597960191, -ERASE, 140265597923328, 140265597951999, -STORE, 94089837146112, 94089837281279, -STORE, 140265596223488, 140265597906943, -STORE, 94089837146112, 94089837416447, -STORE, 94089837146112, 94089837551615, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735265218560, 140737488351231, -SNULL, 140735265226751, 140737488351231, -STORE, 140735265218560, 140735265226751, -STORE, 140735265087488, 140735265226751, -STORE, 94250422370304, 94250424705023, -SNULL, 94250422583295, 94250424705023, -STORE, 94250422370304, 94250422583295, -STORE, 94250422583296, 94250424705023, -ERASE, 94250422583296, 94250424705023, -STORE, 94250424680448, 94250424692735, -STORE, 94250424692736, 94250424705023, -STORE, 140344442474496, 140344444727295, -SNULL, 140344442617855, 140344444727295, -STORE, 140344442474496, 140344442617855, -STORE, 140344442617856, 140344444727295, -ERASE, 140344442617856, 140344444727295, -STORE, 140344444715008, 140344444723199, -STORE, 140344444723200, 140344444727295, -STORE, 140735265341440, 140735265345535, -STORE, 140735265329152, 140735265341439, -STORE, 140344444686336, 140344444715007, -STORE, 140344444678144, 140344444686335, -STORE, 140344440360960, 140344442474495, -SNULL, 140344440360960, 140344440373247, -STORE, 140344440373248, 140344442474495, -STORE, 140344440360960, 140344440373247, -SNULL, 140344442466303, 140344442474495, -STORE, 140344440373248, 140344442466303, -STORE, 140344442466304, 140344442474495, -ERASE, 140344442466304, 140344442474495, -STORE, 140344442466304, 140344442474495, -STORE, 140344436563968, 140344440360959, -SNULL, 140344436563968, 140344438222847, -STORE, 140344438222848, 140344440360959, -STORE, 140344436563968, 140344438222847, -SNULL, 140344440319999, 140344440360959, -STORE, 140344438222848, 140344440319999, -STORE, 140344440320000, 140344440360959, -SNULL, 140344440320000, 140344440344575, -STORE, 140344440344576, 140344440360959, -STORE, 140344440320000, 140344440344575, -ERASE, 140344440320000, 140344440344575, -STORE, 140344440320000, 140344440344575, -ERASE, 140344440344576, 140344440360959, -STORE, 140344440344576, 140344440360959, -STORE, 140344444669952, 140344444686335, -SNULL, 140344440336383, 140344440344575, -STORE, 140344440320000, 140344440336383, -STORE, 140344440336384, 140344440344575, -SNULL, 140344442470399, 140344442474495, -STORE, 140344442466304, 140344442470399, -STORE, 140344442470400, 140344442474495, -SNULL, 94250424684543, 94250424692735, -STORE, 94250424680448, 94250424684543, -STORE, 94250424684544, 94250424692735, -SNULL, 140344444719103, 140344444723199, -STORE, 140344444715008, 140344444719103, -STORE, 140344444719104, 140344444723199, -ERASE, 140344444686336, 140344444715007, -STORE, 94250445512704, 94250445647871, -STORE, 140344442986496, 140344444669951, -STORE, 94250445512704, 94250445783039, -STORE, 94250445512704, 94250445918207, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140725762719744, 140737488351231, -SNULL, 140725762727935, 140737488351231, -STORE, 140725762719744, 140725762727935, -STORE, 140725762588672, 140725762727935, -STORE, 94819009097728, 94819011432447, -SNULL, 94819009310719, 94819011432447, -STORE, 94819009097728, 94819009310719, -STORE, 94819009310720, 94819011432447, -ERASE, 94819009310720, 94819011432447, -STORE, 94819011407872, 94819011420159, -STORE, 94819011420160, 94819011432447, -STORE, 139987985596416, 139987987849215, -SNULL, 139987985739775, 139987987849215, -STORE, 139987985596416, 139987985739775, -STORE, 139987985739776, 139987987849215, -ERASE, 139987985739776, 139987987849215, -STORE, 139987987836928, 139987987845119, -STORE, 139987987845120, 139987987849215, -STORE, 140725763072000, 140725763076095, -STORE, 140725763059712, 140725763071999, -STORE, 139987987808256, 139987987836927, -STORE, 139987987800064, 139987987808255, -STORE, 139987983482880, 139987985596415, -SNULL, 139987983482880, 139987983495167, -STORE, 139987983495168, 139987985596415, -STORE, 139987983482880, 139987983495167, -SNULL, 139987985588223, 139987985596415, -STORE, 139987983495168, 139987985588223, -STORE, 139987985588224, 139987985596415, -ERASE, 139987985588224, 139987985596415, -STORE, 139987985588224, 139987985596415, -STORE, 139987979685888, 139987983482879, -SNULL, 139987979685888, 139987981344767, -STORE, 139987981344768, 139987983482879, -STORE, 139987979685888, 139987981344767, -SNULL, 139987983441919, 139987983482879, -STORE, 139987981344768, 139987983441919, -STORE, 139987983441920, 139987983482879, -SNULL, 139987983441920, 139987983466495, -STORE, 139987983466496, 139987983482879, -STORE, 139987983441920, 139987983466495, -ERASE, 139987983441920, 139987983466495, -STORE, 139987983441920, 139987983466495, -ERASE, 139987983466496, 139987983482879, -STORE, 139987983466496, 139987983482879, -STORE, 139987987791872, 139987987808255, -SNULL, 139987983458303, 139987983466495, -STORE, 139987983441920, 139987983458303, -STORE, 139987983458304, 139987983466495, -SNULL, 139987985592319, 139987985596415, -STORE, 139987985588224, 139987985592319, -STORE, 139987985592320, 139987985596415, -SNULL, 94819011411967, 94819011420159, -STORE, 94819011407872, 94819011411967, -STORE, 94819011411968, 94819011420159, -SNULL, 139987987841023, 139987987845119, -STORE, 139987987836928, 139987987841023, -STORE, 139987987841024, 139987987845119, -ERASE, 139987987808256, 139987987836927, -STORE, 94819028176896, 94819028312063, -STORE, 139987986108416, 139987987791871, -STORE, 94819028176896, 94819028447231, -STORE, 94819028176896, 94819028582399, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722475413504, 140737488351231, -SNULL, 140722475421695, 140737488351231, -STORE, 140722475413504, 140722475421695, -STORE, 140722475282432, 140722475421695, -STORE, 94620599119872, 94620601343999, -SNULL, 94620599230463, 94620601343999, -STORE, 94620599119872, 94620599230463, -STORE, 94620599230464, 94620601343999, -ERASE, 94620599230464, 94620601343999, -STORE, 94620601323520, 94620601335807, -STORE, 94620601335808, 94620601343999, -STORE, 139891763060736, 139891765313535, -SNULL, 139891763204095, 139891765313535, -STORE, 139891763060736, 139891763204095, -STORE, 139891763204096, 139891765313535, -ERASE, 139891763204096, 139891765313535, -STORE, 139891765301248, 139891765309439, -STORE, 139891765309440, 139891765313535, -STORE, 140722475700224, 140722475704319, -STORE, 140722475687936, 140722475700223, -STORE, 139891765272576, 139891765301247, -STORE, 139891765264384, 139891765272575, -STORE, 139891759263744, 139891763060735, -SNULL, 139891759263744, 139891760922623, -STORE, 139891760922624, 139891763060735, -STORE, 139891759263744, 139891760922623, -SNULL, 139891763019775, 139891763060735, -STORE, 139891760922624, 139891763019775, -STORE, 139891763019776, 139891763060735, -SNULL, 139891763019776, 139891763044351, -STORE, 139891763044352, 139891763060735, -STORE, 139891763019776, 139891763044351, -ERASE, 139891763019776, 139891763044351, -STORE, 139891763019776, 139891763044351, -ERASE, 139891763044352, 139891763060735, -STORE, 139891763044352, 139891763060735, -SNULL, 139891763036159, 139891763044351, -STORE, 139891763019776, 139891763036159, -STORE, 139891763036160, 139891763044351, -SNULL, 94620601331711, 94620601335807, -STORE, 94620601323520, 94620601331711, -STORE, 94620601331712, 94620601335807, -SNULL, 139891765305343, 139891765309439, -STORE, 139891765301248, 139891765305343, -STORE, 139891765305344, 139891765309439, -ERASE, 139891765272576, 139891765301247, -STORE, 94620610027520, 94620610162687, -STORE, 94031976210432, 94031976423423, -STORE, 94031978520576, 94031978524671, -STORE, 94031978524672, 94031978532863, -STORE, 94031978532864, 94031978545151, -STORE, 94031990398976, 94031992565759, -STORE, 140336240640000, 140336242298879, -STORE, 140336242298880, 140336244396031, -STORE, 140336244396032, 140336244412415, -STORE, 140336244412416, 140336244420607, -STORE, 140336244420608, 140336244436991, -STORE, 140336244436992, 140336244449279, -STORE, 140336244449280, 140336246542335, -STORE, 140336246542336, 140336246546431, -STORE, 140336246546432, 140336246550527, -STORE, 140336246550528, 140336246693887, -STORE, 140336247062528, 140336248745983, -STORE, 140336248745984, 140336248762367, -STORE, 140336248791040, 140336248795135, -STORE, 140336248795136, 140336248799231, -STORE, 140336248799232, 140336248803327, -STORE, 140728500064256, 140728500203519, -STORE, 140728501501952, 140728501514239, -STORE, 140728501514240, 140728501518335, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140730503987200, 140737488351231, -SNULL, 140730503995391, 140737488351231, -STORE, 140730503987200, 140730503995391, -STORE, 140730503856128, 140730503995391, -STORE, 93866544205824, 93866546429951, -SNULL, 93866544316415, 93866546429951, -STORE, 93866544205824, 93866544316415, -STORE, 93866544316416, 93866546429951, -ERASE, 93866544316416, 93866546429951, -STORE, 93866546409472, 93866546421759, -STORE, 93866546421760, 93866546429951, -STORE, 140216311959552, 140216314212351, -SNULL, 140216312102911, 140216314212351, -STORE, 140216311959552, 140216312102911, -STORE, 140216312102912, 140216314212351, -ERASE, 140216312102912, 140216314212351, -STORE, 140216314200064, 140216314208255, -STORE, 140216314208256, 140216314212351, -STORE, 140730504626176, 140730504630271, -STORE, 140730504613888, 140730504626175, -STORE, 140216314171392, 140216314200063, -STORE, 140216314163200, 140216314171391, -STORE, 140216308162560, 140216311959551, -SNULL, 140216308162560, 140216309821439, -STORE, 140216309821440, 140216311959551, -STORE, 140216308162560, 140216309821439, -SNULL, 140216311918591, 140216311959551, -STORE, 140216309821440, 140216311918591, -STORE, 140216311918592, 140216311959551, -SNULL, 140216311918592, 140216311943167, -STORE, 140216311943168, 140216311959551, -STORE, 140216311918592, 140216311943167, -ERASE, 140216311918592, 140216311943167, -STORE, 140216311918592, 140216311943167, -ERASE, 140216311943168, 140216311959551, -STORE, 140216311943168, 140216311959551, -SNULL, 140216311934975, 140216311943167, -STORE, 140216311918592, 140216311934975, -STORE, 140216311934976, 140216311943167, -SNULL, 93866546417663, 93866546421759, -STORE, 93866546409472, 93866546417663, -STORE, 93866546417664, 93866546421759, -SNULL, 140216314204159, 140216314208255, -STORE, 140216314200064, 140216314204159, -STORE, 140216314204160, 140216314208255, -ERASE, 140216314171392, 140216314200063, -STORE, 93866550386688, 93866550521855, -STORE, 94074292674560, 94074292887551, -STORE, 94074294984704, 94074294988799, -STORE, 94074294988800, 94074294996991, -STORE, 94074294996992, 94074295009279, -STORE, 94074300219392, 94074301378559, -STORE, 139781563256832, 139781564915711, -STORE, 139781564915712, 139781567012863, -STORE, 139781567012864, 139781567029247, -STORE, 139781567029248, 139781567037439, -STORE, 139781567037440, 139781567053823, -STORE, 139781567053824, 139781567066111, -STORE, 139781567066112, 139781569159167, -STORE, 139781569159168, 139781569163263, -STORE, 139781569163264, 139781569167359, -STORE, 139781569167360, 139781569310719, -STORE, 139781569679360, 139781571362815, -STORE, 139781571362816, 139781571379199, -STORE, 139781571407872, 139781571411967, -STORE, 139781571411968, 139781571416063, -STORE, 139781571416064, 139781571420159, -STORE, 140723688488960, 140723688628223, -STORE, 140723689005056, 140723689017343, -STORE, 140723689017344, 140723689021439, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735189745664, 140737488351231, -SNULL, 140735189753855, 140737488351231, -STORE, 140735189745664, 140735189753855, -STORE, 140735189614592, 140735189753855, -STORE, 94172072177664, 94172074512383, -SNULL, 94172072390655, 94172074512383, -STORE, 94172072177664, 94172072390655, -STORE, 94172072390656, 94172074512383, -ERASE, 94172072390656, 94172074512383, -STORE, 94172074487808, 94172074500095, -STORE, 94172074500096, 94172074512383, -STORE, 140687827263488, 140687829516287, -SNULL, 140687827406847, 140687829516287, -STORE, 140687827263488, 140687827406847, -STORE, 140687827406848, 140687829516287, -ERASE, 140687827406848, 140687829516287, -STORE, 140687829504000, 140687829512191, -STORE, 140687829512192, 140687829516287, -STORE, 140735189766144, 140735189770239, -STORE, 140735189753856, 140735189766143, -STORE, 140687829475328, 140687829503999, -STORE, 140687829467136, 140687829475327, -STORE, 140687825149952, 140687827263487, -SNULL, 140687825149952, 140687825162239, -STORE, 140687825162240, 140687827263487, -STORE, 140687825149952, 140687825162239, -SNULL, 140687827255295, 140687827263487, -STORE, 140687825162240, 140687827255295, -STORE, 140687827255296, 140687827263487, -ERASE, 140687827255296, 140687827263487, -STORE, 140687827255296, 140687827263487, -STORE, 140687821352960, 140687825149951, -SNULL, 140687821352960, 140687823011839, -STORE, 140687823011840, 140687825149951, -STORE, 140687821352960, 140687823011839, -SNULL, 140687825108991, 140687825149951, -STORE, 140687823011840, 140687825108991, -STORE, 140687825108992, 140687825149951, -SNULL, 140687825108992, 140687825133567, -STORE, 140687825133568, 140687825149951, -STORE, 140687825108992, 140687825133567, -ERASE, 140687825108992, 140687825133567, -STORE, 140687825108992, 140687825133567, -ERASE, 140687825133568, 140687825149951, -STORE, 140687825133568, 140687825149951, -STORE, 140687829458944, 140687829475327, -SNULL, 140687825125375, 140687825133567, -STORE, 140687825108992, 140687825125375, -STORE, 140687825125376, 140687825133567, -SNULL, 140687827259391, 140687827263487, -STORE, 140687827255296, 140687827259391, -STORE, 140687827259392, 140687827263487, -SNULL, 94172074491903, 94172074500095, -STORE, 94172074487808, 94172074491903, -STORE, 94172074491904, 94172074500095, -SNULL, 140687829508095, 140687829512191, -STORE, 140687829504000, 140687829508095, -STORE, 140687829508096, 140687829512191, -ERASE, 140687829475328, 140687829503999, -STORE, 94172092432384, 94172092567551, -STORE, 140687827775488, 140687829458943, -STORE, 94172092432384, 94172092702719, -STORE, 94172092432384, 94172092837887, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140737229504512, 140737488351231, -SNULL, 140737229512703, 140737488351231, -STORE, 140737229504512, 140737229512703, -STORE, 140737229373440, 140737229512703, -STORE, 94155246866432, 94155249090559, -SNULL, 94155246977023, 94155249090559, -STORE, 94155246866432, 94155246977023, -STORE, 94155246977024, 94155249090559, -ERASE, 94155246977024, 94155249090559, -STORE, 94155249070080, 94155249082367, -STORE, 94155249082368, 94155249090559, -STORE, 140640993693696, 140640995946495, -SNULL, 140640993837055, 140640995946495, -STORE, 140640993693696, 140640993837055, -STORE, 140640993837056, 140640995946495, -ERASE, 140640993837056, 140640995946495, -STORE, 140640995934208, 140640995942399, -STORE, 140640995942400, 140640995946495, -STORE, 140737230004224, 140737230008319, -STORE, 140737229991936, 140737230004223, -STORE, 140640995905536, 140640995934207, -STORE, 140640995897344, 140640995905535, -STORE, 140640989896704, 140640993693695, -SNULL, 140640989896704, 140640991555583, -STORE, 140640991555584, 140640993693695, -STORE, 140640989896704, 140640991555583, -SNULL, 140640993652735, 140640993693695, -STORE, 140640991555584, 140640993652735, -STORE, 140640993652736, 140640993693695, -SNULL, 140640993652736, 140640993677311, -STORE, 140640993677312, 140640993693695, -STORE, 140640993652736, 140640993677311, -ERASE, 140640993652736, 140640993677311, -STORE, 140640993652736, 140640993677311, -ERASE, 140640993677312, 140640993693695, -STORE, 140640993677312, 140640993693695, -SNULL, 140640993669119, 140640993677311, -STORE, 140640993652736, 140640993669119, -STORE, 140640993669120, 140640993677311, -SNULL, 94155249078271, 94155249082367, -STORE, 94155249070080, 94155249078271, -STORE, 94155249078272, 94155249082367, -SNULL, 140640995938303, 140640995942399, -STORE, 140640995934208, 140640995938303, -STORE, 140640995938304, 140640995942399, -ERASE, 140640995905536, 140640995934207, -STORE, 94155281035264, 94155281170431, -STORE, 94088066453504, 94088066564095, -STORE, 94088068657152, 94088068665343, -STORE, 94088068665344, 94088068669439, -STORE, 94088068669440, 94088068677631, -STORE, 94088090214400, 94088090349567, -STORE, 140503024627712, 140503026286591, -STORE, 140503026286592, 140503028383743, -STORE, 140503028383744, 140503028400127, -STORE, 140503028400128, 140503028408319, -STORE, 140503028408320, 140503028424703, -STORE, 140503028424704, 140503028568063, -STORE, 140503030628352, 140503030636543, -STORE, 140503030665216, 140503030669311, -STORE, 140503030669312, 140503030673407, -STORE, 140503030673408, 140503030677503, -STORE, 140730894725120, 140730894864383, -STORE, 140730894880768, 140730894893055, -STORE, 140730894893056, 140730894897151, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140730434342912, 140737488351231, -SNULL, 140730434351103, 140737488351231, -STORE, 140730434342912, 140730434351103, -STORE, 140730434211840, 140730434351103, -STORE, 4194304, 5128191, -STORE, 7221248, 7241727, -STORE, 7241728, 7249919, -STORE, 140109041938432, 140109044191231, -SNULL, 140109042081791, 140109044191231, -STORE, 140109041938432, 140109042081791, -STORE, 140109042081792, 140109044191231, -ERASE, 140109042081792, 140109044191231, -STORE, 140109044178944, 140109044187135, -STORE, 140109044187136, 140109044191231, -STORE, 140730434850816, 140730434854911, -STORE, 140730434838528, 140730434850815, -STORE, 140109044150272, 140109044178943, -STORE, 140109044142080, 140109044150271, -STORE, 140109038776320, 140109041938431, -SNULL, 140109038776320, 140109039837183, -STORE, 140109039837184, 140109041938431, -STORE, 140109038776320, 140109039837183, -SNULL, 140109041930239, 140109041938431, -STORE, 140109039837184, 140109041930239, -STORE, 140109041930240, 140109041938431, -ERASE, 140109041930240, 140109041938431, -STORE, 140109041930240, 140109041938431, -STORE, 140109034979328, 140109038776319, -SNULL, 140109034979328, 140109036638207, -STORE, 140109036638208, 140109038776319, -STORE, 140109034979328, 140109036638207, -SNULL, 140109038735359, 140109038776319, -STORE, 140109036638208, 140109038735359, -STORE, 140109038735360, 140109038776319, -SNULL, 140109038735360, 140109038759935, -STORE, 140109038759936, 140109038776319, -STORE, 140109038735360, 140109038759935, -ERASE, 140109038735360, 140109038759935, -STORE, 140109038735360, 140109038759935, -ERASE, 140109038759936, 140109038776319, -STORE, 140109038759936, 140109038776319, -STORE, 140109044129792, 140109044150271, -SNULL, 140109038751743, 140109038759935, -STORE, 140109038735360, 140109038751743, -STORE, 140109038751744, 140109038759935, -SNULL, 140109041934335, 140109041938431, -STORE, 140109041930240, 140109041934335, -STORE, 140109041934336, 140109041938431, -SNULL, 7233535, 7241727, -STORE, 7221248, 7233535, -STORE, 7233536, 7241727, -SNULL, 140109044183039, 140109044187135, -STORE, 140109044178944, 140109044183039, -STORE, 140109044183040, 140109044187135, -ERASE, 140109044150272, 140109044178943, -STORE, 20000768, 20135935, -STORE, 20000768, 20283391, -STORE, 140109042446336, 140109044129791, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140730853408768, 140737488351231, -SNULL, 140730853416959, 140737488351231, -STORE, 140730853408768, 140730853416959, -STORE, 140730853277696, 140730853416959, -STORE, 94865902977024, 94865905311743, -SNULL, 94865903190015, 94865905311743, -STORE, 94865902977024, 94865903190015, -STORE, 94865903190016, 94865905311743, -ERASE, 94865903190016, 94865905311743, -STORE, 94865905287168, 94865905299455, -STORE, 94865905299456, 94865905311743, -STORE, 139768865738752, 139768867991551, -SNULL, 139768865882111, 139768867991551, -STORE, 139768865738752, 139768865882111, -STORE, 139768865882112, 139768867991551, -ERASE, 139768865882112, 139768867991551, -STORE, 139768867979264, 139768867987455, -STORE, 139768867987456, 139768867991551, -STORE, 140730853957632, 140730853961727, -STORE, 140730853945344, 140730853957631, -STORE, 139768867950592, 139768867979263, -STORE, 139768867942400, 139768867950591, -STORE, 139768863625216, 139768865738751, -SNULL, 139768863625216, 139768863637503, -STORE, 139768863637504, 139768865738751, -STORE, 139768863625216, 139768863637503, -SNULL, 139768865730559, 139768865738751, -STORE, 139768863637504, 139768865730559, -STORE, 139768865730560, 139768865738751, -ERASE, 139768865730560, 139768865738751, -STORE, 139768865730560, 139768865738751, -STORE, 139768859828224, 139768863625215, -SNULL, 139768859828224, 139768861487103, -STORE, 139768861487104, 139768863625215, -STORE, 139768859828224, 139768861487103, -SNULL, 139768863584255, 139768863625215, -STORE, 139768861487104, 139768863584255, -STORE, 139768863584256, 139768863625215, -SNULL, 139768863584256, 139768863608831, -STORE, 139768863608832, 139768863625215, -STORE, 139768863584256, 139768863608831, -ERASE, 139768863584256, 139768863608831, -STORE, 139768863584256, 139768863608831, -ERASE, 139768863608832, 139768863625215, -STORE, 139768863608832, 139768863625215, -STORE, 139768867934208, 139768867950591, -SNULL, 139768863600639, 139768863608831, -STORE, 139768863584256, 139768863600639, -STORE, 139768863600640, 139768863608831, -SNULL, 139768865734655, 139768865738751, -STORE, 139768865730560, 139768865734655, -STORE, 139768865734656, 139768865738751, -SNULL, 94865905291263, 94865905299455, -STORE, 94865905287168, 94865905291263, -STORE, 94865905291264, 94865905299455, -SNULL, 139768867983359, 139768867987455, -STORE, 139768867979264, 139768867983359, -STORE, 139768867983360, 139768867987455, -ERASE, 139768867950592, 139768867979263, -STORE, 94865923670016, 94865923805183, -STORE, 139768866250752, 139768867934207, -STORE, 94865923670016, 94865923940351, -STORE, 94865923670016, 94865924075519, -STORE, 94865923670016, 94865924222975, -SNULL, 94865924210687, 94865924222975, -STORE, 94865923670016, 94865924210687, -STORE, 94865924210688, 94865924222975, -ERASE, 94865924210688, 94865924222975, -STORE, 94865923670016, 94865924349951, -STORE, 94865923670016, 94865924493311, -STORE, 94865923670016, 94865924640767, -SNULL, 94865924603903, 94865924640767, -STORE, 94865923670016, 94865924603903, -STORE, 94865924603904, 94865924640767, -ERASE, 94865924603904, 94865924640767, -STORE, 94865923670016, 94865924747263, -STORE, 94865923670016, 94865924898815, -SNULL, 94865924874239, 94865924898815, -STORE, 94865923670016, 94865924874239, -STORE, 94865924874240, 94865924898815, -ERASE, 94865924874240, 94865924898815, -STORE, 94865923670016, 94865925025791, -SNULL, 94865925013503, 94865925025791, -STORE, 94865923670016, 94865925013503, -STORE, 94865925013504, 94865925025791, -ERASE, 94865925013504, 94865925025791, -SNULL, 94865924988927, 94865925013503, -STORE, 94865923670016, 94865924988927, -STORE, 94865924988928, 94865925013503, -ERASE, 94865924988928, 94865925013503, -STORE, 94865923670016, 94865925152767, -SNULL, 94865925136383, 94865925152767, -STORE, 94865923670016, 94865925136383, -STORE, 94865925136384, 94865925152767, -ERASE, 94865925136384, 94865925152767, -STORE, 94865923670016, 94865925292031, -SNULL, 94865925279743, 94865925292031, -STORE, 94865923670016, 94865925279743, -STORE, 94865925279744, 94865925292031, -ERASE, 94865925279744, 94865925292031, -SNULL, 94865925255167, 94865925279743, -STORE, 94865923670016, 94865925255167, -STORE, 94865925255168, 94865925279743, -ERASE, 94865925255168, 94865925279743, -STORE, 94865923670016, 94865925406719, -SNULL, 94865925394431, 94865925406719, -STORE, 94865923670016, 94865925394431, -STORE, 94865925394432, 94865925406719, -ERASE, 94865925394432, 94865925406719, -STORE, 94865923670016, 94865925545983, -SNULL, 94865925533695, 94865925545983, -STORE, 94865923670016, 94865925533695, -STORE, 94865925533696, 94865925545983, -ERASE, 94865925533696, 94865925545983, -SNULL, 94865925492735, 94865925533695, -STORE, 94865923670016, 94865925492735, -STORE, 94865925492736, 94865925533695, -ERASE, 94865925492736, 94865925533695, -STORE, 94865923670016, 94865925627903, -SNULL, 94865925599231, 94865925627903, -STORE, 94865923670016, 94865925599231, -STORE, 94865925599232, 94865925627903, -ERASE, 94865925599232, 94865925627903, -STORE, 94865923670016, 94865925738495, -SNULL, 94865925726207, 94865925738495, -STORE, 94865923670016, 94865925726207, -STORE, 94865925726208, 94865925738495, -ERASE, 94865925726208, 94865925738495, -STORE, 94865923670016, 94865925877759, -SNULL, 94865925865471, 94865925877759, -STORE, 94865923670016, 94865925865471, -STORE, 94865925865472, 94865925877759, -ERASE, 94865925865472, 94865925877759, -STORE, 94865923670016, 94865926021119, -SNULL, 94865926008831, 94865926021119, -STORE, 94865923670016, 94865926008831, -STORE, 94865926008832, 94865926021119, -ERASE, 94865926008832, 94865926021119, -SNULL, 94865925971967, 94865926008831, -STORE, 94865923670016, 94865925971967, -STORE, 94865925971968, 94865926008831, -ERASE, 94865925971968, 94865926008831, -STORE, 94865923670016, 94865926115327, -STORE, 94865923670016, 94865926254591, -SNULL, 94865926246399, 94865926254591, -STORE, 94865923670016, 94865926246399, -STORE, 94865926246400, 94865926254591, -ERASE, 94865926246400, 94865926254591, -STORE, 94865923670016, 94865926385663, -STORE, 94865923670016, 94865926537215, -STORE, 94865923670016, 94865926672383, -STORE, 94865923670016, 94865926815743, -STORE, 94865923670016, 94865926955007, -STORE, 94865923670016, 94865927094271, -STORE, 94865923670016, 94865927233535, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140731148435456, 140737488351231, -SNULL, 140731148443647, 140737488351231, -STORE, 140731148435456, 140731148443647, -STORE, 140731148304384, 140731148443647, -STORE, 94090775400448, 94090777735167, -SNULL, 94090775613439, 94090777735167, -STORE, 94090775400448, 94090775613439, -STORE, 94090775613440, 94090777735167, -ERASE, 94090775613440, 94090777735167, -STORE, 94090777710592, 94090777722879, -STORE, 94090777722880, 94090777735167, -STORE, 140301090283520, 140301092536319, -SNULL, 140301090426879, 140301092536319, -STORE, 140301090283520, 140301090426879, -STORE, 140301090426880, 140301092536319, -ERASE, 140301090426880, 140301092536319, -STORE, 140301092524032, 140301092532223, -STORE, 140301092532224, 140301092536319, -STORE, 140731148570624, 140731148574719, -STORE, 140731148558336, 140731148570623, -STORE, 140301092495360, 140301092524031, -STORE, 140301092487168, 140301092495359, -STORE, 140301088169984, 140301090283519, -SNULL, 140301088169984, 140301088182271, -STORE, 140301088182272, 140301090283519, -STORE, 140301088169984, 140301088182271, -SNULL, 140301090275327, 140301090283519, -STORE, 140301088182272, 140301090275327, -STORE, 140301090275328, 140301090283519, -ERASE, 140301090275328, 140301090283519, -STORE, 140301090275328, 140301090283519, -STORE, 140301084372992, 140301088169983, -SNULL, 140301084372992, 140301086031871, -STORE, 140301086031872, 140301088169983, -STORE, 140301084372992, 140301086031871, -SNULL, 140301088129023, 140301088169983, -STORE, 140301086031872, 140301088129023, -STORE, 140301088129024, 140301088169983, -SNULL, 140301088129024, 140301088153599, -STORE, 140301088153600, 140301088169983, -STORE, 140301088129024, 140301088153599, -ERASE, 140301088129024, 140301088153599, -STORE, 140301088129024, 140301088153599, -ERASE, 140301088153600, 140301088169983, -STORE, 140301088153600, 140301088169983, -STORE, 140301092478976, 140301092495359, -SNULL, 140301088145407, 140301088153599, -STORE, 140301088129024, 140301088145407, -STORE, 140301088145408, 140301088153599, -SNULL, 140301090279423, 140301090283519, -STORE, 140301090275328, 140301090279423, -STORE, 140301090279424, 140301090283519, -SNULL, 94090777714687, 94090777722879, -STORE, 94090777710592, 94090777714687, -STORE, 94090777714688, 94090777722879, -SNULL, 140301092528127, 140301092532223, -STORE, 140301092524032, 140301092528127, -STORE, 140301092528128, 140301092532223, -ERASE, 140301092495360, 140301092524031, -STORE, 94090794590208, 94090794725375, -STORE, 140301090795520, 140301092478975, -STORE, 94090794590208, 94090794860543, -STORE, 94090794590208, 94090794995711, -STORE, 94090794590208, 94090795163647, -SNULL, 94090795139071, 94090795163647, -STORE, 94090794590208, 94090795139071, -STORE, 94090795139072, 94090795163647, -ERASE, 94090795139072, 94090795163647, -STORE, 94090794590208, 94090795278335, -STORE, 94090794590208, 94090795425791, -SNULL, 94090795388927, 94090795425791, -STORE, 94090794590208, 94090795388927, -STORE, 94090795388928, 94090795425791, -ERASE, 94090795388928, 94090795425791, -STORE, 94090794590208, 94090795528191, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733084430336, 140737488351231, -SNULL, 140733084438527, 140737488351231, -STORE, 140733084430336, 140733084438527, -STORE, 140733084299264, 140733084438527, -STORE, 94116169183232, 94116171517951, -SNULL, 94116169396223, 94116171517951, -STORE, 94116169183232, 94116169396223, -STORE, 94116169396224, 94116171517951, -ERASE, 94116169396224, 94116171517951, -STORE, 94116171493376, 94116171505663, -STORE, 94116171505664, 94116171517951, -STORE, 139772214128640, 139772216381439, -SNULL, 139772214271999, 139772216381439, -STORE, 139772214128640, 139772214271999, -STORE, 139772214272000, 139772216381439, -ERASE, 139772214272000, 139772216381439, -STORE, 139772216369152, 139772216377343, -STORE, 139772216377344, 139772216381439, -STORE, 140733085270016, 140733085274111, -STORE, 140733085257728, 140733085270015, -STORE, 139772216340480, 139772216369151, -STORE, 139772216332288, 139772216340479, -STORE, 139772212015104, 139772214128639, -SNULL, 139772212015104, 139772212027391, -STORE, 139772212027392, 139772214128639, -STORE, 139772212015104, 139772212027391, -SNULL, 139772214120447, 139772214128639, -STORE, 139772212027392, 139772214120447, -STORE, 139772214120448, 139772214128639, -ERASE, 139772214120448, 139772214128639, -STORE, 139772214120448, 139772214128639, -STORE, 139772208218112, 139772212015103, -SNULL, 139772208218112, 139772209876991, -STORE, 139772209876992, 139772212015103, -STORE, 139772208218112, 139772209876991, -SNULL, 139772211974143, 139772212015103, -STORE, 139772209876992, 139772211974143, -STORE, 139772211974144, 139772212015103, -SNULL, 139772211974144, 139772211998719, -STORE, 139772211998720, 139772212015103, -STORE, 139772211974144, 139772211998719, -ERASE, 139772211974144, 139772211998719, -STORE, 139772211974144, 139772211998719, -ERASE, 139772211998720, 139772212015103, -STORE, 139772211998720, 139772212015103, -STORE, 139772216324096, 139772216340479, -SNULL, 139772211990527, 139772211998719, -STORE, 139772211974144, 139772211990527, -STORE, 139772211990528, 139772211998719, -SNULL, 139772214124543, 139772214128639, -STORE, 139772214120448, 139772214124543, -STORE, 139772214124544, 139772214128639, -SNULL, 94116171497471, 94116171505663, -STORE, 94116171493376, 94116171497471, -STORE, 94116171497472, 94116171505663, -SNULL, 139772216373247, 139772216377343, -STORE, 139772216369152, 139772216373247, -STORE, 139772216373248, 139772216377343, -ERASE, 139772216340480, 139772216369151, -STORE, 94116199383040, 94116199518207, -STORE, 139772214640640, 139772216324095, -STORE, 94116199383040, 94116199653375, -STORE, 94116199383040, 94116199788543, -STORE, 140737488347136, 140737488351231, -STORE, 140726067826688, 140737488351231, -SNULL, 140726067830783, 140737488351231, -STORE, 140726067826688, 140726067830783, -STORE, 140726067695616, 140726067830783, -STORE, 94535150673920, 94535152898047, -SNULL, 94535150784511, 94535152898047, -STORE, 94535150673920, 94535150784511, -STORE, 94535150784512, 94535152898047, -ERASE, 94535150784512, 94535152898047, -STORE, 94535152877568, 94535152889855, -STORE, 94535152889856, 94535152898047, -STORE, 140381257314304, 140381259567103, -SNULL, 140381257457663, 140381259567103, -STORE, 140381257314304, 140381257457663, -STORE, 140381257457664, 140381259567103, -ERASE, 140381257457664, 140381259567103, -STORE, 140381259554816, 140381259563007, -STORE, 140381259563008, 140381259567103, -STORE, 140726068060160, 140726068064255, -STORE, 140726068047872, 140726068060159, -STORE, 140381259526144, 140381259554815, -STORE, 140381259517952, 140381259526143, -STORE, 140381253517312, 140381257314303, -SNULL, 140381253517312, 140381255176191, -STORE, 140381255176192, 140381257314303, -STORE, 140381253517312, 140381255176191, -SNULL, 140381257273343, 140381257314303, -STORE, 140381255176192, 140381257273343, -STORE, 140381257273344, 140381257314303, -SNULL, 140381257273344, 140381257297919, -STORE, 140381257297920, 140381257314303, -STORE, 140381257273344, 140381257297919, -ERASE, 140381257273344, 140381257297919, -STORE, 140381257273344, 140381257297919, -ERASE, 140381257297920, 140381257314303, -STORE, 140381257297920, 140381257314303, -SNULL, 140381257289727, 140381257297919, -STORE, 140381257273344, 140381257289727, -STORE, 140381257289728, 140381257297919, -SNULL, 94535152885759, 94535152889855, -STORE, 94535152877568, 94535152885759, -STORE, 94535152885760, 94535152889855, -SNULL, 140381259558911, 140381259563007, -STORE, 140381259554816, 140381259558911, -STORE, 140381259558912, 140381259563007, -ERASE, 140381259526144, 140381259554815, -STORE, 94535186296832, 94535186431999, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140729189425152, 140737488351231, -SNULL, 140729189433343, 140737488351231, -STORE, 140729189425152, 140729189433343, -STORE, 140729189294080, 140729189433343, -STORE, 94428200128512, 94428202352639, -SNULL, 94428200239103, 94428202352639, -STORE, 94428200128512, 94428200239103, -STORE, 94428200239104, 94428202352639, -ERASE, 94428200239104, 94428202352639, -STORE, 94428202332160, 94428202344447, -STORE, 94428202344448, 94428202352639, -STORE, 139707216986112, 139707219238911, -SNULL, 139707217129471, 139707219238911, -STORE, 139707216986112, 139707217129471, -STORE, 139707217129472, 139707219238911, -ERASE, 139707217129472, 139707219238911, -STORE, 139707219226624, 139707219234815, -STORE, 139707219234816, 139707219238911, -STORE, 140729189785600, 140729189789695, -STORE, 140729189773312, 140729189785599, -STORE, 139707219197952, 139707219226623, -STORE, 139707219189760, 139707219197951, -STORE, 139707213189120, 139707216986111, -SNULL, 139707213189120, 139707214847999, -STORE, 139707214848000, 139707216986111, -STORE, 139707213189120, 139707214847999, -SNULL, 139707216945151, 139707216986111, -STORE, 139707214848000, 139707216945151, -STORE, 139707216945152, 139707216986111, -SNULL, 139707216945152, 139707216969727, -STORE, 139707216969728, 139707216986111, -STORE, 139707216945152, 139707216969727, -ERASE, 139707216945152, 139707216969727, -STORE, 139707216945152, 139707216969727, -ERASE, 139707216969728, 139707216986111, -STORE, 139707216969728, 139707216986111, -SNULL, 139707216961535, 139707216969727, -STORE, 139707216945152, 139707216961535, -STORE, 139707216961536, 139707216969727, -SNULL, 94428202340351, 94428202344447, -STORE, 94428202332160, 94428202340351, -STORE, 94428202340352, 94428202344447, -SNULL, 139707219230719, 139707219234815, -STORE, 139707219226624, 139707219230719, -STORE, 139707219230720, 139707219234815, -ERASE, 139707219197952, 139707219226623, -STORE, 94428208599040, 94428208734207, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722000953344, 140737488351231, -SNULL, 140722000961535, 140737488351231, -STORE, 140722000953344, 140722000961535, -STORE, 140722000822272, 140722000961535, -STORE, 94636494757888, 94636496982015, -SNULL, 94636494868479, 94636496982015, -STORE, 94636494757888, 94636494868479, -STORE, 94636494868480, 94636496982015, -ERASE, 94636494868480, 94636496982015, -STORE, 94636496961536, 94636496973823, -STORE, 94636496973824, 94636496982015, -STORE, 140142275100672, 140142277353471, -SNULL, 140142275244031, 140142277353471, -STORE, 140142275100672, 140142275244031, -STORE, 140142275244032, 140142277353471, -ERASE, 140142275244032, 140142277353471, -STORE, 140142277341184, 140142277349375, -STORE, 140142277349376, 140142277353471, -STORE, 140722002747392, 140722002751487, -STORE, 140722002735104, 140722002747391, -STORE, 140142277312512, 140142277341183, -STORE, 140142277304320, 140142277312511, -STORE, 140142271303680, 140142275100671, -SNULL, 140142271303680, 140142272962559, -STORE, 140142272962560, 140142275100671, -STORE, 140142271303680, 140142272962559, -SNULL, 140142275059711, 140142275100671, -STORE, 140142272962560, 140142275059711, -STORE, 140142275059712, 140142275100671, -SNULL, 140142275059712, 140142275084287, -STORE, 140142275084288, 140142275100671, -STORE, 140142275059712, 140142275084287, -ERASE, 140142275059712, 140142275084287, -STORE, 140142275059712, 140142275084287, -ERASE, 140142275084288, 140142275100671, -STORE, 140142275084288, 140142275100671, -SNULL, 140142275076095, 140142275084287, -STORE, 140142275059712, 140142275076095, -STORE, 140142275076096, 140142275084287, -SNULL, 94636496969727, 94636496973823, -STORE, 94636496961536, 94636496969727, -STORE, 94636496969728, 94636496973823, -SNULL, 140142277345279, 140142277349375, -STORE, 140142277341184, 140142277345279, -STORE, 140142277345280, 140142277349375, -ERASE, 140142277312512, 140142277341183, -STORE, 94636516286464, 94636516421631, -STORE, 94071103692800, 94071103905791, -STORE, 94071106002944, 94071106007039, -STORE, 94071106007040, 94071106015231, -STORE, 94071106015232, 94071106027519, -STORE, 94071138521088, 94071140368383, -STORE, 140145668190208, 140145669849087, -STORE, 140145669849088, 140145671946239, -STORE, 140145671946240, 140145671962623, -STORE, 140145671962624, 140145671970815, -STORE, 140145671970816, 140145671987199, -STORE, 140145671987200, 140145671999487, -STORE, 140145671999488, 140145674092543, -STORE, 140145674092544, 140145674096639, -STORE, 140145674096640, 140145674100735, -STORE, 140145674100736, 140145674244095, -STORE, 140145674612736, 140145676296191, -STORE, 140145676296192, 140145676312575, -STORE, 140145676341248, 140145676345343, -STORE, 140145676345344, 140145676349439, -STORE, 140145676349440, 140145676353535, -STORE, 140734927740928, 140734927880191, -STORE, 140734928842752, 140734928855039, -STORE, 140734928855040, 140734928859135, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722342535168, 140737488351231, -SNULL, 140722342543359, 140737488351231, -STORE, 140722342535168, 140722342543359, -STORE, 140722342404096, 140722342543359, -STORE, 94399699714048, 94399702048767, -SNULL, 94399699927039, 94399702048767, -STORE, 94399699714048, 94399699927039, -STORE, 94399699927040, 94399702048767, -ERASE, 94399699927040, 94399702048767, -STORE, 94399702024192, 94399702036479, -STORE, 94399702036480, 94399702048767, -STORE, 139811024748544, 139811027001343, -SNULL, 139811024891903, 139811027001343, -STORE, 139811024748544, 139811024891903, -STORE, 139811024891904, 139811027001343, -ERASE, 139811024891904, 139811027001343, -STORE, 139811026989056, 139811026997247, -STORE, 139811026997248, 139811027001343, -STORE, 140722342707200, 140722342711295, -STORE, 140722342694912, 140722342707199, -STORE, 139811026960384, 139811026989055, -STORE, 139811026952192, 139811026960383, -STORE, 139811022635008, 139811024748543, -SNULL, 139811022635008, 139811022647295, -STORE, 139811022647296, 139811024748543, -STORE, 139811022635008, 139811022647295, -SNULL, 139811024740351, 139811024748543, -STORE, 139811022647296, 139811024740351, -STORE, 139811024740352, 139811024748543, -ERASE, 139811024740352, 139811024748543, -STORE, 139811024740352, 139811024748543, -STORE, 139811018838016, 139811022635007, -SNULL, 139811018838016, 139811020496895, -STORE, 139811020496896, 139811022635007, -STORE, 139811018838016, 139811020496895, -SNULL, 139811022594047, 139811022635007, -STORE, 139811020496896, 139811022594047, -STORE, 139811022594048, 139811022635007, -SNULL, 139811022594048, 139811022618623, -STORE, 139811022618624, 139811022635007, -STORE, 139811022594048, 139811022618623, -ERASE, 139811022594048, 139811022618623, -STORE, 139811022594048, 139811022618623, -ERASE, 139811022618624, 139811022635007, -STORE, 139811022618624, 139811022635007, -STORE, 139811026944000, 139811026960383, -SNULL, 139811022610431, 139811022618623, -STORE, 139811022594048, 139811022610431, -STORE, 139811022610432, 139811022618623, -SNULL, 139811024744447, 139811024748543, -STORE, 139811024740352, 139811024744447, -STORE, 139811024744448, 139811024748543, -SNULL, 94399702028287, 94399702036479, -STORE, 94399702024192, 94399702028287, -STORE, 94399702028288, 94399702036479, -SNULL, 139811026993151, 139811026997247, -STORE, 139811026989056, 139811026993151, -STORE, 139811026993152, 139811026997247, -ERASE, 139811026960384, 139811026989055, -STORE, 94399723880448, 94399724015615, -STORE, 139811025260544, 139811026943999, -STORE, 94399723880448, 94399724150783, -STORE, 94399723880448, 94399724285951, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140735364939776, 140737488351231, -SNULL, 140735364947967, 140737488351231, -STORE, 140735364939776, 140735364947967, -STORE, 140735364808704, 140735364947967, -STORE, 94421528674304, 94421531009023, -SNULL, 94421528887295, 94421531009023, -STORE, 94421528674304, 94421528887295, -STORE, 94421528887296, 94421531009023, -ERASE, 94421528887296, 94421531009023, -STORE, 94421530984448, 94421530996735, -STORE, 94421530996736, 94421531009023, -STORE, 140162004742144, 140162006994943, -SNULL, 140162004885503, 140162006994943, -STORE, 140162004742144, 140162004885503, -STORE, 140162004885504, 140162006994943, -ERASE, 140162004885504, 140162006994943, -STORE, 140162006982656, 140162006990847, -STORE, 140162006990848, 140162006994943, -STORE, 140735365402624, 140735365406719, -STORE, 140735365390336, 140735365402623, -STORE, 140162006953984, 140162006982655, -STORE, 140162006945792, 140162006953983, -STORE, 140162002628608, 140162004742143, -SNULL, 140162002628608, 140162002640895, -STORE, 140162002640896, 140162004742143, -STORE, 140162002628608, 140162002640895, -SNULL, 140162004733951, 140162004742143, -STORE, 140162002640896, 140162004733951, -STORE, 140162004733952, 140162004742143, -ERASE, 140162004733952, 140162004742143, -STORE, 140162004733952, 140162004742143, -STORE, 140161998831616, 140162002628607, -SNULL, 140161998831616, 140162000490495, -STORE, 140162000490496, 140162002628607, -STORE, 140161998831616, 140162000490495, -SNULL, 140162002587647, 140162002628607, -STORE, 140162000490496, 140162002587647, -STORE, 140162002587648, 140162002628607, -SNULL, 140162002587648, 140162002612223, -STORE, 140162002612224, 140162002628607, -STORE, 140162002587648, 140162002612223, -ERASE, 140162002587648, 140162002612223, -STORE, 140162002587648, 140162002612223, -ERASE, 140162002612224, 140162002628607, -STORE, 140162002612224, 140162002628607, -STORE, 140162006937600, 140162006953983, -SNULL, 140162002604031, 140162002612223, -STORE, 140162002587648, 140162002604031, -STORE, 140162002604032, 140162002612223, -SNULL, 140162004738047, 140162004742143, -STORE, 140162004733952, 140162004738047, -STORE, 140162004738048, 140162004742143, -SNULL, 94421530988543, 94421530996735, -STORE, 94421530984448, 94421530988543, -STORE, 94421530988544, 94421530996735, -SNULL, 140162006986751, 140162006990847, -STORE, 140162006982656, 140162006986751, -STORE, 140162006986752, 140162006990847, -ERASE, 140162006953984, 140162006982655, -STORE, 94421551697920, 94421551833087, -STORE, 140162005254144, 140162006937599, -STORE, 94421551697920, 94421551968255, -STORE, 94421551697920, 94421552103423, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140733498486784, 140737488351231, -SNULL, 140733498494975, 140737488351231, -STORE, 140733498486784, 140733498494975, -STORE, 140733498355712, 140733498494975, -STORE, 94567985836032, 94567988170751, -SNULL, 94567986049023, 94567988170751, -STORE, 94567985836032, 94567986049023, -STORE, 94567986049024, 94567988170751, -ERASE, 94567986049024, 94567988170751, -STORE, 94567988146176, 94567988158463, -STORE, 94567988158464, 94567988170751, -STORE, 139634278572032, 139634280824831, -SNULL, 139634278715391, 139634280824831, -STORE, 139634278572032, 139634278715391, -STORE, 139634278715392, 139634280824831, -ERASE, 139634278715392, 139634280824831, -STORE, 139634280812544, 139634280820735, -STORE, 139634280820736, 139634280824831, -STORE, 140733498544128, 140733498548223, -STORE, 140733498531840, 140733498544127, -STORE, 139634280783872, 139634280812543, -STORE, 139634280775680, 139634280783871, -STORE, 139634276458496, 139634278572031, -SNULL, 139634276458496, 139634276470783, -STORE, 139634276470784, 139634278572031, -STORE, 139634276458496, 139634276470783, -SNULL, 139634278563839, 139634278572031, -STORE, 139634276470784, 139634278563839, -STORE, 139634278563840, 139634278572031, -ERASE, 139634278563840, 139634278572031, -STORE, 139634278563840, 139634278572031, -STORE, 139634272661504, 139634276458495, -SNULL, 139634272661504, 139634274320383, -STORE, 139634274320384, 139634276458495, -STORE, 139634272661504, 139634274320383, -SNULL, 139634276417535, 139634276458495, -STORE, 139634274320384, 139634276417535, -STORE, 139634276417536, 139634276458495, -SNULL, 139634276417536, 139634276442111, -STORE, 139634276442112, 139634276458495, -STORE, 139634276417536, 139634276442111, -ERASE, 139634276417536, 139634276442111, -STORE, 139634276417536, 139634276442111, -ERASE, 139634276442112, 139634276458495, -STORE, 139634276442112, 139634276458495, -STORE, 139634280767488, 139634280783871, -SNULL, 139634276433919, 139634276442111, -STORE, 139634276417536, 139634276433919, -STORE, 139634276433920, 139634276442111, -SNULL, 139634278567935, 139634278572031, -STORE, 139634278563840, 139634278567935, -STORE, 139634278567936, 139634278572031, -SNULL, 94567988150271, 94567988158463, -STORE, 94567988146176, 94567988150271, -STORE, 94567988150272, 94567988158463, -SNULL, 139634280816639, 139634280820735, -STORE, 139634280812544, 139634280816639, -STORE, 139634280816640, 139634280820735, -ERASE, 139634280783872, 139634280812543, -STORE, 94567996379136, 94567996514303, -STORE, 139634279084032, 139634280767487, -STORE, 94567996379136, 94567996649471, -STORE, 94567996379136, 94567996784639, -STORE, 94567996379136, 94567996960767, -SNULL, 94567996932095, 94567996960767, -STORE, 94567996379136, 94567996932095, -STORE, 94567996932096, 94567996960767, -ERASE, 94567996932096, 94567996960767, -STORE, 94567996379136, 94567997071359, -STORE, 94567996379136, 94567997206527, -SNULL, 94567997186047, 94567997206527, -STORE, 94567996379136, 94567997186047, -STORE, 94567997186048, 94567997206527, -ERASE, 94567997186048, 94567997206527, -STORE, 94567996379136, 94567997358079, -STORE, 94567996379136, 94567997493247, -SNULL, 94567997476863, 94567997493247, -STORE, 94567996379136, 94567997476863, -STORE, 94567997476864, 94567997493247, -ERASE, 94567997476864, 94567997493247, -STORE, 94567996379136, 94567997612031, -STORE, 94567996379136, 94567997767679, -SNULL, 94567997739007, 94567997767679, -STORE, 94567996379136, 94567997739007, -STORE, 94567997739008, 94567997767679, -ERASE, 94567997739008, 94567997767679, -SNULL, 94567997698047, 94567997739007, -STORE, 94567996379136, 94567997698047, -STORE, 94567997698048, 94567997739007, -ERASE, 94567997698048, 94567997739007, -STORE, 94567996379136, 94567997853695, -STORE, 94567996379136, 94567997988863, -STORE, 94567996379136, 94567998132223, -STORE, 94567996379136, 94567998275583, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140723667759104, 140737488351231, -SNULL, 140723667767295, 140737488351231, -STORE, 140723667759104, 140723667767295, -STORE, 140723667628032, 140723667767295, -STORE, 94231598800896, 94231601135615, -SNULL, 94231599013887, 94231601135615, -STORE, 94231598800896, 94231599013887, -STORE, 94231599013888, 94231601135615, -ERASE, 94231599013888, 94231601135615, -STORE, 94231601111040, 94231601123327, -STORE, 94231601123328, 94231601135615, -STORE, 140269472649216, 140269474902015, -SNULL, 140269472792575, 140269474902015, -STORE, 140269472649216, 140269472792575, -STORE, 140269472792576, 140269474902015, -ERASE, 140269472792576, 140269474902015, -STORE, 140269474889728, 140269474897919, -STORE, 140269474897920, 140269474902015, -STORE, 140723667836928, 140723667841023, -STORE, 140723667824640, 140723667836927, -STORE, 140269474861056, 140269474889727, -STORE, 140269474852864, 140269474861055, -STORE, 140269470535680, 140269472649215, -SNULL, 140269470535680, 140269470547967, -STORE, 140269470547968, 140269472649215, -STORE, 140269470535680, 140269470547967, -SNULL, 140269472641023, 140269472649215, -STORE, 140269470547968, 140269472641023, -STORE, 140269472641024, 140269472649215, -ERASE, 140269472641024, 140269472649215, -STORE, 140269472641024, 140269472649215, -STORE, 140269466738688, 140269470535679, -SNULL, 140269466738688, 140269468397567, -STORE, 140269468397568, 140269470535679, -STORE, 140269466738688, 140269468397567, -SNULL, 140269470494719, 140269470535679, -STORE, 140269468397568, 140269470494719, -STORE, 140269470494720, 140269470535679, -SNULL, 140269470494720, 140269470519295, -STORE, 140269470519296, 140269470535679, -STORE, 140269470494720, 140269470519295, -ERASE, 140269470494720, 140269470519295, -STORE, 140269470494720, 140269470519295, -ERASE, 140269470519296, 140269470535679, -STORE, 140269470519296, 140269470535679, -STORE, 140269474844672, 140269474861055, -SNULL, 140269470511103, 140269470519295, -STORE, 140269470494720, 140269470511103, -STORE, 140269470511104, 140269470519295, -SNULL, 140269472645119, 140269472649215, -STORE, 140269472641024, 140269472645119, -STORE, 140269472645120, 140269472649215, -SNULL, 94231601115135, 94231601123327, -STORE, 94231601111040, 94231601115135, -STORE, 94231601115136, 94231601123327, -SNULL, 140269474893823, 140269474897919, -STORE, 140269474889728, 140269474893823, -STORE, 140269474893824, 140269474897919, -ERASE, 140269474861056, 140269474889727, -STORE, 94231626592256, 94231626727423, -STORE, 140269473161216, 140269474844671, -STORE, 94231626592256, 94231626862591, -STORE, 94231626592256, 94231626997759, -STORE, 94327178862592, 94327179075583, -STORE, 94327181172736, 94327181176831, -STORE, 94327181176832, 94327181185023, -STORE, 94327181185024, 94327181197311, -STORE, 94327185715200, 94327186685951, -STORE, 140172071755776, 140172073414655, -STORE, 140172073414656, 140172075511807, -STORE, 140172075511808, 140172075528191, -STORE, 140172075528192, 140172075536383, -STORE, 140172075536384, 140172075552767, -STORE, 140172075552768, 140172075565055, -STORE, 140172075565056, 140172077658111, -STORE, 140172077658112, 140172077662207, -STORE, 140172077662208, 140172077666303, -STORE, 140172077666304, 140172077809663, -STORE, 140172078178304, 140172079861759, -STORE, 140172079861760, 140172079878143, -STORE, 140172079878144, 140172079906815, -STORE, 140172079906816, 140172079910911, -STORE, 140172079910912, 140172079915007, -STORE, 140172079915008, 140172079919103, -STORE, 140720358359040, 140720358494207, -STORE, 140720358498304, 140720358510591, -STORE, 140720358510592, 140720358514687, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140722548621312, 140737488351231, -SNULL, 140722548629503, 140737488351231, -STORE, 140722548621312, 140722548629503, -STORE, 140722548490240, 140722548629503, -STORE, 93949289504768, 93949291728895, -SNULL, 93949289615359, 93949291728895, -STORE, 93949289504768, 93949289615359, -STORE, 93949289615360, 93949291728895, -ERASE, 93949289615360, 93949291728895, -STORE, 93949291708416, 93949291720703, -STORE, 93949291720704, 93949291728895, -STORE, 140305861902336, 140305864155135, -SNULL, 140305862045695, 140305864155135, -STORE, 140305861902336, 140305862045695, -STORE, 140305862045696, 140305864155135, -ERASE, 140305862045696, 140305864155135, -STORE, 140305864142848, 140305864151039, -STORE, 140305864151040, 140305864155135, -STORE, 140722549821440, 140722549825535, -STORE, 140722549809152, 140722549821439, -STORE, 140305864114176, 140305864142847, -STORE, 140305864105984, 140305864114175, -STORE, 140305858105344, 140305861902335, -SNULL, 140305858105344, 140305859764223, -STORE, 140305859764224, 140305861902335, -STORE, 140305858105344, 140305859764223, -SNULL, 140305861861375, 140305861902335, -STORE, 140305859764224, 140305861861375, -STORE, 140305861861376, 140305861902335, -SNULL, 140305861861376, 140305861885951, -STORE, 140305861885952, 140305861902335, -STORE, 140305861861376, 140305861885951, -ERASE, 140305861861376, 140305861885951, -STORE, 140305861861376, 140305861885951, -ERASE, 140305861885952, 140305861902335, -STORE, 140305861885952, 140305861902335, -SNULL, 140305861877759, 140305861885951, -STORE, 140305861861376, 140305861877759, -STORE, 140305861877760, 140305861885951, -SNULL, 93949291716607, 93949291720703, -STORE, 93949291708416, 93949291716607, -STORE, 93949291716608, 93949291720703, -SNULL, 140305864146943, 140305864151039, -STORE, 140305864142848, 140305864146943, -STORE, 140305864146944, 140305864151039, -ERASE, 140305864114176, 140305864142847, -STORE, 93949324136448, 93949324271615, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140725754908672, 140737488351231, -SNULL, 140725754916863, 140737488351231, -STORE, 140725754908672, 140725754916863, -STORE, 140725754777600, 140725754916863, -STORE, 94831184375808, 94831186599935, -SNULL, 94831184486399, 94831186599935, -STORE, 94831184375808, 94831184486399, -STORE, 94831184486400, 94831186599935, -ERASE, 94831184486400, 94831186599935, -STORE, 94831186579456, 94831186591743, -STORE, 94831186591744, 94831186599935, -STORE, 140605482479616, 140605484732415, -SNULL, 140605482622975, 140605484732415, -STORE, 140605482479616, 140605482622975, -STORE, 140605482622976, 140605484732415, -ERASE, 140605482622976, 140605484732415, -STORE, 140605484720128, 140605484728319, -STORE, 140605484728320, 140605484732415, -STORE, 140725755670528, 140725755674623, -STORE, 140725755658240, 140725755670527, -STORE, 140605484691456, 140605484720127, -STORE, 140605484683264, 140605484691455, -STORE, 140605478682624, 140605482479615, -SNULL, 140605478682624, 140605480341503, -STORE, 140605480341504, 140605482479615, -STORE, 140605478682624, 140605480341503, -SNULL, 140605482438655, 140605482479615, -STORE, 140605480341504, 140605482438655, -STORE, 140605482438656, 140605482479615, -SNULL, 140605482438656, 140605482463231, -STORE, 140605482463232, 140605482479615, -STORE, 140605482438656, 140605482463231, -ERASE, 140605482438656, 140605482463231, -STORE, 140605482438656, 140605482463231, -ERASE, 140605482463232, 140605482479615, -STORE, 140605482463232, 140605482479615, -SNULL, 140605482455039, 140605482463231, -STORE, 140605482438656, 140605482455039, -STORE, 140605482455040, 140605482463231, -SNULL, 94831186587647, 94831186591743, -STORE, 94831186579456, 94831186587647, -STORE, 94831186587648, 94831186591743, -SNULL, 140605484724223, 140605484728319, -STORE, 140605484720128, 140605484724223, -STORE, 140605484724224, 140605484728319, -ERASE, 140605484691456, 140605484720127, -STORE, 94831217156096, 94831217291263, -STORE, 94327178862592, 94327179075583, -STORE, 94327181172736, 94327181176831, -STORE, 94327181176832, 94327181185023, -STORE, 94327181185024, 94327181197311, -STORE, 94327185715200, 94327186685951, -STORE, 140172071755776, 140172073414655, -STORE, 140172073414656, 140172075511807, -STORE, 140172075511808, 140172075528191, -STORE, 140172075528192, 140172075536383, -STORE, 140172075536384, 140172075552767, -STORE, 140172075552768, 140172075565055, -STORE, 140172075565056, 140172077658111, -STORE, 140172077658112, 140172077662207, -STORE, 140172077662208, 140172077666303, -STORE, 140172077666304, 140172077809663, -STORE, 140172078178304, 140172079861759, -STORE, 140172079861760, 140172079878143, -STORE, 140172079878144, 140172079906815, -STORE, 140172079906816, 140172079910911, -STORE, 140172079910912, 140172079915007, -STORE, 140172079915008, 140172079919103, -STORE, 140720358359040, 140720358494207, -STORE, 140720358498304, 140720358510591, -STORE, 140720358510592, 140720358514687, -STORE, 140737488347136, 140737488351231, -STORE, 140737488343040, 140737488351231, -STORE, 140737488338944, 140737488351231, -STORE, 140734529933312, 140737488351231, -SNULL, 140734529945599, 140737488351231, -STORE, 140734529933312, 140734529945599, -STORE, 140734529802240, 140734529945599, -STORE, 4194304, 26279935, -STORE, 28372992, 28454911, -STORE, 28454912, 29806591, -STORE, 140249744060416, 140249746313215, -SNULL, 140249744203775, 140249746313215, -STORE, 140249744060416, 140249744203775, -STORE, 140249744203776, 140249746313215, -ERASE, 140249744203776, 140249746313215, -STORE, 140249746300928, 140249746309119, -STORE, 140249746309120, 140249746313215, -STORE, 140734530174976, 140734530179071, -STORE, 140734530162688, 140734530174975, -STORE, 140249746272256, 140249746300927, -STORE, 140249746264064, 140249746272255, -STORE, 140249740226560, 140249744060415, -SNULL, 140249740226560, 140249741934591, -STORE, 140249741934592, 140249744060415, -STORE, 140249740226560, 140249741934591, -SNULL, 140249744027647, 140249744060415, -STORE, 140249741934592, 140249744027647, -STORE, 140249744027648, 140249744060415, -ERASE, 140249744027648, 140249744060415, -STORE, 140249744027648, 140249744060415, -STORE, 140249738031104, 140249740226559, -SNULL, 140249738031104, 140249738125311, -STORE, 140249738125312, 140249740226559, -STORE, 140249738031104, 140249738125311, -SNULL, 140249740218367, 140249740226559, -STORE, 140249738125312, 140249740218367, -STORE, 140249740218368, 140249740226559, -ERASE, 140249740218368, 140249740226559, -STORE, 140249740218368, 140249740226559, -STORE, 140249735512064, 140249738031103, -SNULL, 140249735512064, 140249735925759, -STORE, 140249735925760, 140249738031103, -STORE, 140249735512064, 140249735925759, -SNULL, 140249738018815, 140249738031103, -STORE, 140249735925760, 140249738018815, -STORE, 140249738018816, 140249738031103, -ERASE, 140249738018816, 140249738031103, -STORE, 140249738018816, 140249738031103, -STORE, 140249732878336, 140249735512063, -SNULL, 140249732878336, 140249733406719, -STORE, 140249733406720, 140249735512063, -STORE, 140249732878336, 140249733406719, -SNULL, 140249735503871, 140249735512063, -STORE, 140249733406720, 140249735503871, -STORE, 140249735503872, 140249735512063, -ERASE, 140249735503872, 140249735512063, -STORE, 140249735503872, 140249735512063, -STORE, 140249730764800, 140249732878335, -SNULL, 140249730764800, 140249730777087, -STORE, 140249730777088, 140249732878335, -STORE, 140249730764800, 140249730777087, -SNULL, 140249732870143, 140249732878335, -STORE, 140249730777088, 140249732870143, -STORE, 140249732870144, 140249732878335, -ERASE, 140249732870144, 140249732878335, -STORE, 140249732870144, 140249732878335, -STORE, 140249728561152, 140249730764799, -SNULL, 140249728561152, 140249728663551, -STORE, 140249728663552, 140249730764799, -STORE, 140249728561152, 140249728663551, -SNULL, 140249730756607, 140249730764799, -STORE, 140249728663552, 140249730756607, -STORE, 140249730756608, 140249730764799, -ERASE, 140249730756608, 140249730764799, -STORE, 140249730756608, 140249730764799, -STORE, 140249746255872, 140249746272255, -STORE, 140249725399040, 140249728561151, -SNULL, 140249725399040, 140249726459903, -STORE, 140249726459904, 140249728561151, -STORE, 140249725399040, 140249726459903, -SNULL, 140249728552959, 140249728561151, -STORE, 140249726459904, 140249728552959, -STORE, 140249728552960, 140249728561151, -ERASE, 140249728552960, 140249728561151, -STORE, 140249728552960, 140249728561151, -STORE, 140249721602048, 140249725399039, -SNULL, 140249721602048, 140249723260927, -STORE, 140249723260928, 140249725399039, -STORE, 140249721602048, 140249723260927, -SNULL, 140249725358079, 140249725399039, -STORE, 140249723260928, 140249725358079, -STORE, 140249725358080, 140249725399039, -SNULL, 140249725358080, 140249725382655, -STORE, 140249725382656, 140249725399039, -STORE, 140249725358080, 140249725382655, -ERASE, 140249725358080, 140249725382655, -STORE, 140249725358080, 140249725382655, -ERASE, 140249725382656, 140249725399039, -STORE, 140249725382656, 140249725399039, -STORE, 140249746243584, 140249746272255, -SNULL, 140249725374463, 140249725382655, -STORE, 140249725358080, 140249725374463, -STORE, 140249725374464, 140249725382655, -SNULL, 140249728557055, 140249728561151, -STORE, 140249728552960, 140249728557055, -STORE, 140249728557056, 140249728561151, -SNULL, 140249730760703, 140249730764799, -STORE, 140249730756608, 140249730760703, -STORE, 140249730760704, 140249730764799, -SNULL, 140249732874239, 140249732878335, -STORE, 140249732870144, 140249732874239, -STORE, 140249732874240, 140249732878335, -SNULL, 140249735507967, 140249735512063, -STORE, 140249735503872, 140249735507967, -STORE, 140249735507968, 140249735512063, -SNULL, 140249738027007, 140249738031103, -STORE, 140249738018816, 140249738027007, -STORE, 140249738027008, 140249738031103, -SNULL, 140249740222463, 140249740226559, -STORE, 140249740218368, 140249740222463, -STORE, 140249740222464, 140249740226559, -SNULL, 140249744031743, 140249744060415, -STORE, 140249744027648, 140249744031743, -STORE, 140249744031744, 140249744060415, -SNULL, 28405759, 28454911, -STORE, 28372992, 28405759, -STORE, 28405760, 28454911, -SNULL, 140249746305023, 140249746309119, -STORE, 140249746300928, 140249746305023, -STORE, 140249746305024, 140249746309119, -ERASE, 140249746272256, 140249746300927, -STORE, 33853440, 33988607, -STORE, 140249744560128, 140249746243583, -STORE, 140249746296832, 140249746300927, -STORE, 140249744424960, 140249744560127, -STORE, 33853440, 34131967, -STORE, 140249719504896, 140249721602047, -STORE, 140249746288640, 140249746300927, -STORE, 140249746280448, 140249746300927, -STORE, 140249746243584, 140249746280447, -STORE, 140249744408576, 140249744560127, -STORE, 33853440, 34267135, -STORE, 33853440, 34422783, -STORE, 140249744400384, 140249744560127, -STORE, 140249744392192, 140249744560127, -STORE, 33853440, 34557951, -STORE, 33853440, 34693119, -STORE, 140249744375808, 140249744560127, -STORE, 140249744367616, 140249744560127, -STORE, 33853440, 34832383, -STORE, 140249719230464, 140249721602047, -STORE, 140249744207872, 140249744560127, -STORE, 33853440, 34971647, -SNULL, 34963455, 34971647, -STORE, 33853440, 34963455, -STORE, 34963456, 34971647, -ERASE, 34963456, 34971647, -SNULL, 34955263, 34963455, -STORE, 33853440, 34955263, -STORE, 34955264, 34963455, -ERASE, 34955264, 34963455, -SNULL, 34947071, 34955263, -STORE, 33853440, 34947071, -STORE, 34947072, 34955263, -ERASE, 34947072, 34955263, -SNULL, 34938879, 34947071, -STORE, 33853440, 34938879, -STORE, 34938880, 34947071, -ERASE, 34938880, 34947071, -STORE, 140249719214080, 140249721602047, -STORE, 140249719148544, 140249721602047, -STORE, 140249719115776, 140249721602047, -STORE, 140249717018624, 140249721602047, -STORE, 140249716953088, 140249721602047, -STORE, 33853440, 35086335, -STORE, 140249716822016, 140249721602047, -STORE, 140249716559872, 140249721602047, -STORE, 140249716551680, 140249721602047, -STORE, 140249716535296, 140249721602047, -STORE, 140249716527104, 140249721602047, -STORE, 140249716518912, 140249721602047, -STORE, 33853440, 35221503, -SNULL, 35213311, 35221503, -STORE, 33853440, 35213311, -STORE, 35213312, 35221503, -ERASE, 35213312, 35221503, -SNULL, 35205119, 35213311, -STORE, 33853440, 35205119, -STORE, 35205120, 35213311, -ERASE, 35205120, 35213311, -SNULL, 35192831, 35205119, -STORE, 33853440, 35192831, -STORE, 35192832, 35205119, -ERASE, 35192832, 35205119, -SNULL, 35176447, 35192831, -STORE, 33853440, 35176447, -STORE, 35176448, 35192831, -ERASE, 35176448, 35192831, -STORE, 140249716502528, 140249721602047, -STORE, 33853440, 35311615, -SNULL, 35307519, 35311615, -STORE, 33853440, 35307519, -STORE, 35307520, 35311615, -ERASE, 35307520, 35311615, -SNULL, 35303423, 35307519, -STORE, 33853440, 35303423, -STORE, 35303424, 35307519, -ERASE, 35303424, 35307519, -SNULL, 35299327, 35303423, -STORE, 33853440, 35299327, -STORE, 35299328, 35303423, -ERASE, 35299328, 35303423, -SNULL, 35295231, 35299327, -STORE, 33853440, 35295231, -STORE, 35295232, 35299327, -ERASE, 35295232, 35299327, -SNULL, 35291135, 35295231, -STORE, 33853440, 35291135, -STORE, 35291136, 35295231, -ERASE, 35291136, 35295231, -SNULL, 35287039, 35291135, -STORE, 33853440, 35287039, -STORE, 35287040, 35291135, -ERASE, 35287040, 35291135, -SNULL, 35282943, 35287039, -STORE, 33853440, 35282943, -STORE, 35282944, 35287039, -ERASE, 35282944, 35287039, -STORE, 140249716486144, 140249721602047, -STORE, 140249716453376, 140249721602047, -STORE, 33853440, 35418111, -SNULL, 35401727, 35418111, -STORE, 33853440, 35401727, -STORE, 35401728, 35418111, -ERASE, 35401728, 35418111, -SNULL, 35389439, 35401727, -STORE, 33853440, 35389439, -STORE, 35389440, 35401727, -ERASE, 35389440, 35401727, -STORE, 140249714356224, 140249721602047, -STORE, 33853440, 35540991, -STORE, 140249714339840, 140249721602047, -STORE, 140249714077696, 140249721602047, -STORE, 140249714069504, 140249721602047, -STORE, 140249714061312, 140249721602047, -STORE, 33853440, 35680255, -SNULL, 35672063, 35680255, -STORE, 33853440, 35672063, -STORE, 35672064, 35680255, -ERASE, 35672064, 35680255, -SNULL, 35627007, 35672063, -STORE, 33853440, 35627007, -STORE, 35627008, 35672063, -ERASE, 35627008, 35672063, -STORE, 140249711964160, 140249721602047, -STORE, 33853440, 35762175, -SNULL, 35753983, 35762175, -STORE, 33853440, 35753983, -STORE, 35753984, 35762175, -ERASE, 35753984, 35762175, -SNULL, 35745791, 35753983, -STORE, 33853440, 35745791, -STORE, 35745792, 35753983, -ERASE, 35745792, 35753983, -STORE, 140249711955968, 140249721602047, -STORE, 140249711947776, 140249721602047, -STORE, 140249710899200, 140249721602047, -STORE, 140249710866432, 140249721602047, -STORE, 140249710600192, 140249721602047, -SNULL, 140249744424959, 140249744560127, -STORE, 140249744207872, 140249744424959, -STORE, 140249744424960, 140249744560127, -ERASE, 140249744424960, 140249744560127, -STORE, 140249708503040, 140249721602047, -STORE, 33853440, 35885055, -STORE, 140249707978752, 140249721602047, -STORE, 140249705881600, 140249721602047, -STORE, 33853440, 36036607, -STORE, 33853440, 36175871, -STORE, 140249744551936, 140249744560127, -STORE, 140249744543744, 140249744560127, -STORE, 140249744535552, 140249744560127, -STORE, 140249744527360, 140249744560127, -STORE, 140249744519168, 140249744560127, -STORE, 140249705619456, 140249721602047, -STORE, 140249744510976, 140249744560127, -STORE, 140249744502784, 140249744560127, -STORE, 140249744494592, 140249744560127, -STORE, 140249744486400, 140249744560127, -STORE, 140249744478208, 140249744560127, -STORE, 140249744470016, 140249744560127, -STORE, 140249744461824, 140249744560127, -STORE, 140249744453632, 140249744560127, -STORE, 140249744445440, 140249744560127, -STORE, 140249744437248, 140249744560127, -STORE, 140249744429056, 140249744560127, -STORE, 140249703522304, 140249721602047, -STORE, 33853440, 36311039, -STORE, 140249703489536, 140249721602047, -STORE, 33853440, 36474879, -STORE, 140249703456768, 140249721602047, -STORE, 33853440, 36622335, -STORE, 140249703424000, 140249721602047, -STORE, 140249703391232, 140249721602047, -STORE, 33853440, 36810751, -STORE, 140249703358464, 140249721602047, -STORE, 140249703325696, 140249721602047, -SNULL, 36655103, 36810751, -STORE, 33853440, 36655103, -STORE, 36655104, 36810751, -ERASE, 36655104, 36810751, -SNULL, 36438015, 36655103, -STORE, 33853440, 36438015, -STORE, 36438016, 36655103, -ERASE, 36438016, 36655103, -STORE, 140249703317504, 140249721602047, -STORE, 140249701220352, 140249721602047, -STORE, 33853440, 36585471, -STORE, 33853440, 36782079, -STORE, 140249701212160, 140249721602047, -STORE, 140249701203968, 140249721602047, -STORE, 140249701195776, 140249721602047, -STORE, 140249701187584, 140249721602047, -STORE, 140249701179392, 140249721602047, -STORE, 140249701171200, 140249721602047, -STORE, 140249701163008, 140249721602047, -STORE, 140249701154816, 140249721602047, -STORE, 140249701146624, 140249721602047, -STORE, 140249701138432, 140249721602047, -STORE, 140249701130240, 140249721602047, -STORE, 140249700081664, 140249721602047, -STORE, 140249700073472, 140249721602047, -STORE, 33853440, 36978687, -STORE, 140249697976320, 140249721602047, -STORE, 33853440, 37240831, -STORE, 140249695879168, 140249721602047, -STORE, 140249695870976, 140249721602047, -STORE, 140249695862784, 140249721602047, -STORE, 140249695854592, 140249721602047, -STORE, 140249695326208, 140249721602047, -SNULL, 140249710600191, 140249721602047, -STORE, 140249695326208, 140249710600191, -STORE, 140249710600192, 140249721602047, -SNULL, 140249710600192, 140249710866431, -STORE, 140249710866432, 140249721602047, -STORE, 140249710600192, 140249710866431, -ERASE, 140249710600192, 140249710866431, -STORE, 140249691131904, 140249710600191, -STORE, 33853440, 37474303, -STORE, 140249710858240, 140249721602047, -STORE, 140249710850048, 140249721602047, -STORE, 140249710841856, 140249721602047, -STORE, 140249710833664, 140249721602047, -STORE, 140249710825472, 140249721602047, -STORE, 140249710817280, 140249721602047, -STORE, 140249710809088, 140249721602047, -STORE, 140249710800896, 140249721602047, -STORE, 140249710792704, 140249721602047, -STORE, 140249710784512, 140249721602047, -STORE, 140249710776320, 140249721602047, -STORE, 140249710768128, 140249721602047, -STORE, 140249710759936, 140249721602047, -STORE, 140249710751744, 140249721602047, -STORE, 140249710743552, 140249721602047, -STORE, 140249710735360, 140249721602047, -STORE, 140249689034752, 140249710600191, -STORE, 140249710727168, 140249721602047, -STORE, 140249686937600, 140249710600191, -STORE, 33853440, 37867519, -STORE, 140249684840448, 140249710600191, -STORE, 140249710718976, 140249721602047, -STORE, 140249682743296, 140249710600191, -STORE, 140249710710784, 140249721602047, -STORE, 140249710702592, 140249721602047, -STORE, 140249710694400, 140249721602047, -STORE, 140249710686208, 140249721602047, -STORE, 140249710678016, 140249721602047, -STORE, 140249682612224, 140249710600191, -STORE, 140249682087936, 140249710600191, -SNULL, 140249705619455, 140249710600191, -STORE, 140249682087936, 140249705619455, -STORE, 140249705619456, 140249710600191, -SNULL, 140249705619456, 140249705881599, -STORE, 140249705881600, 140249710600191, -STORE, 140249705619456, 140249705881599, -ERASE, 140249705619456, 140249705881599, -STORE, 140249679990784, 140249705619455, -STORE, 140249710669824, 140249721602047, -STORE, 140249677893632, 140249705619455, -STORE, 140249710653440, 140249721602047, -STORE, 140249710645248, 140249721602047, -STORE, 140249710637056, 140249721602047, -STORE, 140249710628864, 140249721602047, -STORE, 140249710620672, 140249721602047, -STORE, 140249710612480, 140249721602047, -STORE, 140249710604288, 140249721602047, -STORE, 140249705873408, 140249710600191, -STORE, 140249705865216, 140249710600191, -STORE, 140249705857024, 140249710600191, -STORE, 140249705848832, 140249710600191, -STORE, 140249705840640, 140249710600191, -STORE, 140249705832448, 140249710600191, -STORE, 140249705824256, 140249710600191, -STORE, 140249705816064, 140249710600191, -STORE, 140249705807872, 140249710600191, -STORE, 140249705799680, 140249710600191, -STORE, 33853440, 38129663, -SNULL, 140249744207872, 140249744367615, -STORE, 140249744367616, 140249744424959, -STORE, 140249744207872, 140249744367615, -ERASE, 140249744207872, 140249744367615, -STORE, 140249677606912, 140249705619455, -STORE, 140249675509760, 140249705619455, -SNULL, 140249677606911, 140249705619455, -STORE, 140249675509760, 140249677606911, -STORE, 140249677606912, 140249705619455, -SNULL, 140249677606912, 140249677893631, -STORE, 140249677893632, 140249705619455, -STORE, 140249677606912, 140249677893631, -ERASE, 140249677606912, 140249677893631, -STORE, 140249744359424, 140249744424959, -STORE, 33853440, 38391807, -STORE, 140249674981376, 140249677606911, -STORE, 140249672884224, 140249677606911, -SNULL, 140249719230463, 140249721602047, -STORE, 140249710604288, 140249719230463, -STORE, 140249719230464, 140249721602047, -SNULL, 140249719230464, 140249719504895, -STORE, 140249719504896, 140249721602047, -STORE, 140249719230464, 140249719504895, -ERASE, 140249719230464, 140249719504895, -STORE, 140249744351232, 140249744424959, -STORE, 140249744343040, 140249744424959, -STORE, 140249744334848, 140249744424959, -STORE, 140249744326656, 140249744424959, -STORE, 140249744310272, 140249744424959, -STORE, 140249744302080, 140249744424959, -STORE, 140249744285696, 140249744424959, -STORE, 140249744277504, 140249744424959, -STORE, 140249744261120, 140249744424959, -STORE, 140249744252928, 140249744424959, -STORE, 140249744220160, 140249744424959, -STORE, 140249744211968, 140249744424959, -STORE, 140249719488512, 140249721602047, -STORE, 140249744203776, 140249744424959, -STORE, 140249719472128, 140249721602047, -STORE, 140249719463936, 140249721602047, -STORE, 140249719447552, 140249721602047, -STORE, 140249719439360, 140249721602047, -STORE, 140249719406592, 140249721602047, -STORE, 140249719398400, 140249721602047, -STORE, 140249719382016, 140249721602047, -STORE, 140249719373824, 140249721602047, -STORE, 140249719357440, 140249721602047, -STORE, 140249719349248, 140249721602047, -STORE, 140249719332864, 140249721602047, -STORE, 140249719324672, 140249721602047, -STORE, 140249719291904, 140249721602047, -STORE, 140249719283712, 140249721602047, -STORE, 140249719267328, 140249721602047, -STORE, 140249719259136, 140249721602047, -STORE, 140249719242752, 140249721602047, -STORE, 140249719234560, 140249721602047, -STORE, 140249705783296, 140249710600191, -STORE, 140249705775104, 140249710600191, -STORE, 140249705742336, 140249710600191, -STORE, 140249705734144, 140249710600191, -STORE, 140249705717760, 140249710600191, -STORE, 140249670787072, 140249677606911, -STORE, 140249705709568, 140249710600191, -STORE, 140249705693184, 140249710600191, -STORE, 140249705684992, 140249710600191, -STORE, 140249705668608, 140249710600191, -STORE, 140249705660416, 140249710600191, -STORE, 140249705627648, 140249710600191, -STORE, 140249677893632, 140249710600191, -STORE, 140249677877248, 140249710600191, -STORE, 140249677869056, 140249710600191, -STORE, 140249677852672, 140249710600191, -STORE, 140249677844480, 140249710600191, -STORE, 140249677828096, 140249710600191, -STORE, 140249668689920, 140249677606911, -STORE, 140249677819904, 140249710600191, -STORE, 140249677787136, 140249710600191, -STORE, 140249677778944, 140249710600191, -STORE, 140249677762560, 140249710600191, -STORE, 140249677754368, 140249710600191, -STORE, 140249677737984, 140249710600191, -STORE, 140249677729792, 140249710600191, -STORE, 140249677713408, 140249710600191, -STORE, 140249677705216, 140249710600191, -STORE, 140249677672448, 140249710600191, -STORE, 140249677664256, 140249710600191, -STORE, 140249677647872, 140249710600191, -STORE, 140249677639680, 140249710600191, -STORE, 140249677623296, 140249710600191, -STORE, 140249677615104, 140249710600191, -STORE, 140249668673536, 140249677606911, -STORE, 140249668673536, 140249710600191, -STORE, 140249668640768, 140249710600191, -STORE, 140249668632576, 140249710600191, -STORE, 140249668616192, 140249710600191, -STORE, 140249668608000, 140249710600191, -STORE, 140249668591616, 140249710600191, -STORE, 140249668583424, 140249710600191, -STORE, 140249668567040, 140249710600191, -STORE, 140249668558848, 140249710600191, -STORE, 140249668526080, 140249710600191, -STORE, 140249668517888, 140249710600191, -STORE, 140249668501504, 140249710600191, -STORE, 140249668493312, 140249710600191, -STORE, 140249668476928, 140249710600191, -STORE, 140249668468736, 140249710600191, -STORE, 140249668452352, 140249710600191, -STORE, 140249668444160, 140249710600191, -STORE, 140249668411392, 140249710600191, -STORE, 140249668403200, 140249710600191, -STORE, 140249668386816, 140249710600191, -STORE, 140249668378624, 140249710600191, -STORE, 140249668362240, 140249710600191, -STORE, 140249668354048, 140249710600191, -STORE, 140249668337664, 140249710600191, -STORE, 140249668329472, 140249710600191, -STORE, 140249668296704, 140249710600191, -STORE, 140249668288512, 140249710600191, -STORE, 140249668272128, 140249710600191, -STORE, 140249668263936, 140249710600191, -STORE, 140249668247552, 140249710600191, -STORE, 140249668239360, 140249710600191, -STORE, 140249668222976, 140249710600191, -STORE, 140249668214784, 140249710600191, -STORE, 140249668182016, 140249710600191, -STORE, 140249668173824, 140249710600191, -STORE, 140249668157440, 140249710600191, -STORE, 140249668149248, 140249710600191, -STORE, 140249668132864, 140249710600191, -STORE, 140249668124672, 140249710600191, -STORE, 140249668108288, 140249710600191, -STORE, 140249668100096, 140249710600191, -STORE, 140249668067328, 140249710600191, -STORE, 140249668059136, 140249710600191, -STORE, 140249668042752, 140249710600191, -STORE, 140249668034560, 140249710600191, -STORE, 140249668018176, 140249710600191, -STORE, 140249668009984, 140249710600191, -STORE, 140249667993600, 140249710600191, -STORE, 140249667985408, 140249710600191, -STORE, 140249667952640, 140249710600191, -STORE, 140249667944448, 140249710600191, -STORE, 140249667928064, 140249710600191, -STORE, 140249667919872, 140249710600191, -STORE, 140249667903488, 140249710600191, -STORE, 140249667895296, 140249710600191, -STORE, 140249667878912, 140249710600191, -STORE, 140249667870720, 140249710600191, -STORE, 140249667837952, 140249710600191, -STORE, 140249667829760, 140249710600191, -STORE, 140249667813376, 140249710600191, -STORE, 140249667805184, 140249710600191, -STORE, 140249667788800, 140249710600191, -STORE, 140249667780608, 140249710600191, -STORE, 140249667764224, 140249710600191, -STORE, 140249667756032, 140249710600191, -STORE, 140249667723264, 140249710600191, -STORE, 140249667715072, 140249710600191, -STORE, 140249667698688, 140249710600191, -STORE, 140249667690496, 140249710600191, -STORE, 140249667674112, 140249710600191, -STORE, 140249667665920, 140249710600191, -STORE, 140249667649536, 140249710600191, -STORE, 140249667641344, 140249710600191, -STORE, 140249667608576, 140249710600191, -STORE, 140249667600384, 140249710600191, -STORE, 140249667584000, 140249710600191, -STORE, 140249667575808, 140249710600191, -STORE, 140249667559424, 140249710600191, -STORE, 140249667551232, 140249710600191, -STORE, 140249667534848, 140249710600191, -STORE, 140249667526656, 140249710600191, -STORE, 140249667493888, 140249710600191, -STORE, 140249667485696, 140249710600191, -STORE, 140249667469312, 140249710600191, -STORE, 140249667461120, 140249710600191, -STORE, 140249667444736, 140249710600191, -STORE, 140249667436544, 140249710600191, -STORE, 140249667420160, 140249710600191, -STORE, 140249665323008, 140249710600191, -STORE, 140249665314816, 140249710600191, -STORE, 140249665282048, 140249710600191, -STORE, 140249665273856, 140249710600191, -STORE, 140249665257472, 140249710600191, -STORE, 140249665249280, 140249710600191, -STORE, 140249665232896, 140249710600191, -STORE, 140249665224704, 140249710600191, -STORE, 140249665208320, 140249710600191, -STORE, 140249665200128, 140249710600191, -STORE, 140249665167360, 140249710600191, -STORE, 140249665159168, 140249710600191, -STORE, 140249665142784, 140249710600191, -STORE, 140249665134592, 140249710600191, -STORE, 140249665118208, 140249710600191, -STORE, 140249665110016, 140249710600191, -STORE, 140249665093632, 140249710600191, -STORE, 140249665085440, 140249710600191, -STORE, 140249665052672, 140249710600191, -STORE, 140249665044480, 140249710600191, -STORE, 140249665028096, 140249710600191, -STORE, 140249665019904, 140249710600191, -STORE, 140249665003520, 140249710600191, -STORE, 140249664995328, 140249710600191, -STORE, 140249664978944, 140249710600191, -STORE, 140249664970752, 140249710600191, -STORE, 140249664937984, 140249710600191, -STORE, 140249664929792, 140249710600191, -STORE, 140249664913408, 140249710600191, -STORE, 140249664905216, 140249710600191, -STORE, 140249664888832, 140249710600191, -STORE, 140249664880640, 140249710600191, -STORE, 140249664864256, 140249710600191, -STORE, 140249664856064, 140249710600191, -STORE, 140249664823296, 140249710600191, -STORE, 140249664815104, 140249710600191, -STORE, 140249664798720, 140249710600191, -STORE, 140249664790528, 140249710600191, -STORE, 140249664774144, 140249710600191, -STORE, 140249664765952, 140249710600191, -STORE, 140249664749568, 140249710600191, -STORE, 140249664741376, 140249710600191, -STORE, 140249664708608, 140249710600191, -STORE, 140249664700416, 140249710600191, -STORE, 140249664684032, 140249710600191, -STORE, 140249664675840, 140249710600191, -STORE, 140249664659456, 140249710600191, -STORE, 140249664651264, 140249710600191, -STORE, 140249664634880, 140249710600191, -STORE, 140249664626688, 140249710600191, -STORE, 140249664593920, 140249710600191, -STORE, 140249664585728, 140249710600191, -STORE, 140249664569344, 140249710600191, -STORE, 140249664561152, 140249710600191, -STORE, 140249664544768, 140249710600191, -STORE, 140249664536576, 140249710600191, -STORE, 140249664520192, 140249710600191, -STORE, 140249664512000, 140249710600191, -STORE, 140249664479232, 140249710600191, -STORE, 140249664471040, 140249710600191, -STORE, 140249664454656, 140249710600191, -STORE, 140249664446464, 140249710600191, -STORE, 140249664430080, 140249710600191, -STORE, 140249664421888, 140249710600191, -STORE, 140249664405504, 140249710600191, -STORE, 140249664397312, 140249710600191, -STORE, 140249664364544, 140249710600191, -STORE, 140249664356352, 140249710600191, -STORE, 140249664339968, 140249710600191, -STORE, 140249664331776, 140249710600191, -STORE, 140249664315392, 140249710600191, -STORE, 140249664307200, 140249710600191, -STORE, 140249664290816, 140249710600191, -STORE, 140249664282624, 140249710600191, -STORE, 140249664249856, 140249710600191, -STORE, 140249664241664, 140249710600191, -STORE, 140249664225280, 140249710600191, -STORE, 140249664217088, 140249710600191, -STORE, 140249664200704, 140249710600191, -STORE, 140249664192512, 140249710600191, -STORE, 140249664176128, 140249710600191, -STORE, 140249664167936, 140249710600191, -STORE, 140249664135168, 140249710600191, -STORE, 140249664126976, 140249710600191, -STORE, 140249664110592, 140249710600191, -STORE, 140249664102400, 140249710600191, -STORE, 140249664086016, 140249710600191, -STORE, 140249664077824, 140249710600191, -STORE, 140249664061440, 140249710600191, -STORE, 140249664053248, 140249710600191, -STORE, 140249664020480, 140249710600191, -STORE, 140249664012288, 140249710600191, -STORE, 140249663995904, 140249710600191, -STORE, 140249663987712, 140249710600191, -STORE, 140249663971328, 140249710600191, -STORE, 140249663963136, 140249710600191, -STORE, 140249663946752, 140249710600191, -STORE, 140249663938560, 140249710600191, -STORE, 140249663905792, 140249710600191, -STORE, 140249663897600, 140249710600191, -STORE, 140249663881216, 140249710600191, -STORE, 140249663873024, 140249710600191, -STORE, 140249663856640, 140249710600191, -STORE, 140249663848448, 140249710600191, -STORE, 140249663832064, 140249710600191, -STORE, 140249663823872, 140249710600191, -STORE, 140249663791104, 140249710600191, -STORE, 140249663782912, 140249710600191, -STORE, 140249663766528, 140249710600191, -STORE, 140249663758336, 140249710600191, -STORE, 140249663741952, 140249710600191, -STORE, 140249663733760, 140249710600191, -STORE, 140249663717376, 140249710600191, -STORE, 140249663709184, 140249710600191, -STORE, 140249663676416, 140249710600191, -STORE, 140249663668224, 140249710600191, -STORE, 140249663651840, 140249710600191, -STORE, 140249663643648, 140249710600191, -STORE, 140249663627264, 140249710600191, -STORE, 33853440, 38526975, -STORE, 140249663619072, 140249710600191, -STORE, 140249663602688, 140249710600191, -STORE, 140249661505536, 140249710600191, -STORE, 140249661497344, 140249710600191, -STORE, 140249661464576, 140249710600191, -STORE, 140249661456384, 140249710600191, -STORE, 140249661440000, 140249710600191, -STORE, 140249661431808, 140249710600191, -STORE, 140249661415424, 140249710600191, -STORE, 140249661407232, 140249710600191, -STORE, 140249661390848, 140249710600191, -STORE, 140249661382656, 140249710600191, -STORE, 140249661349888, 140249710600191, -STORE, 140249661341696, 140249710600191, -STORE, 140249661325312, 140249710600191, -STORE, 140249661317120, 140249710600191, -STORE, 140249661300736, 140249710600191, -STORE, 140249661292544, 140249710600191, -STORE, 140249661276160, 140249710600191, -STORE, 140249661267968, 140249710600191, -STORE, 140249661235200, 140249710600191, -STORE, 140249661227008, 140249710600191, -STORE, 140249661210624, 140249710600191, -STORE, 140249661202432, 140249710600191, -STORE, 140249661186048, 140249710600191, -STORE, 140249661177856, 140249710600191, -STORE, 140249661161472, 140249710600191, -STORE, 140249661153280, 140249710600191, -STORE, 140249661120512, 140249710600191, -STORE, 140249661112320, 140249710600191, -STORE, 140249661095936, 140249710600191, -STORE, 140249661087744, 140249710600191, -STORE, 140249661071360, 140249710600191, -STORE, 140249661063168, 140249710600191, -STORE, 140249661046784, 140249710600191, -STORE, 140249661038592, 140249710600191, -STORE, 140249661005824, 140249710600191, -STORE, 140249660997632, 140249710600191, -STORE, 140249660981248, 140249710600191, -STORE, 140249660973056, 140249710600191, -STORE, 140249660956672, 140249710600191, -STORE, 140249660948480, 140249710600191, -STORE, 140249660932096, 140249710600191, -STORE, 140249660923904, 140249710600191, -STORE, 140249660891136, 140249710600191, -STORE, 140249660882944, 140249710600191, -STORE, 140249660866560, 140249710600191, -STORE, 140249660858368, 140249710600191, -STORE, 140249660841984, 140249710600191, -STORE, 140249660833792, 140249710600191, -STORE, 140249660817408, 140249710600191, -STORE, 140249660809216, 140249710600191, -STORE, 140249660776448, 140249710600191, -STORE, 140249660768256, 140249710600191, -STORE, 140249660751872, 140249710600191, -STORE, 140249660743680, 140249710600191, -STORE, 140249660727296, 140249710600191, -STORE, 140249660719104, 140249710600191, -STORE, 140249660702720, 140249710600191, -STORE, 140249660694528, 140249710600191, -STORE, 140249660661760, 140249710600191, -STORE, 140249660653568, 140249710600191, -STORE, 140249660637184, 140249710600191, -STORE, 140249660628992, 140249710600191, -STORE, 140249660612608, 140249710600191, -STORE, 140249660604416, 140249710600191, -STORE, 140249660588032, 140249710600191, -STORE, 140249660579840, 140249710600191, -STORE, 140249660547072, 140249710600191, -STORE, 140249660538880, 140249710600191, -STORE, 140249660522496, 140249710600191, -STORE, 140249660514304, 140249710600191, -STORE, 140249660497920, 140249710600191, -STORE, 140249660489728, 140249710600191, -STORE, 140249660473344, 140249710600191, -STORE, 140249660465152, 140249710600191, -STORE, 140249660432384, 140249710600191, -STORE, 140249660424192, 140249710600191, -STORE, 140249660407808, 140249710600191, -STORE, 140249660399616, 140249710600191, -STORE, 140249660383232, 140249710600191, -STORE, 140249660375040, 140249710600191, -STORE, 140249660358656, 140249710600191, -STORE, 140249660350464, 140249710600191, -STORE, 140249660317696, 140249710600191, -STORE, 140249660309504, 140249710600191, -STORE, 140249660293120, 140249710600191, -STORE, 140249660284928, 140249710600191, -STORE, 140249660268544, 140249710600191, -STORE, 140249660260352, 140249710600191, -STORE, 140249660243968, 140249710600191, -STORE, 140249660235776, 140249710600191, -STORE, 140249660203008, 140249710600191, -STORE, 140249660194816, 140249710600191, -STORE, 140249660178432, 140249710600191, -STORE, 140249660170240, 140249710600191, -STORE, 140249660153856, 140249710600191, -STORE, 140249660145664, 140249710600191, -STORE, 140249660129280, 140249710600191, -STORE, 140249660121088, 140249710600191, -STORE, 140249660088320, 140249710600191, -STORE, 140249660080128, 140249710600191, -STORE, 140249660063744, 140249710600191, -STORE, 140249660055552, 140249710600191, -STORE, 140249660039168, 140249710600191, -STORE, 140249660030976, 140249710600191, -STORE, 140249660014592, 140249710600191, -STORE, 140249660006400, 140249710600191, -STORE, 140249659973632, 140249710600191, -STORE, 140249659965440, 140249710600191, -STORE, 140249659949056, 140249710600191, -STORE, 140249659940864, 140249710600191, -STORE, 140249659924480, 140249710600191, -STORE, 140249659916288, 140249710600191, -STORE, 140249659899904, 140249710600191, -STORE, 140249659891712, 140249710600191, -STORE, 140249659858944, 140249710600191, -STORE, 140249659850752, 140249710600191, -STORE, 140249659834368, 140249710600191, -STORE, 140249659826176, 140249710600191, -STORE, 140249659809792, 140249710600191, -STORE, 140249659801600, 140249710600191, -STORE, 140249659785216, 140249710600191, -STORE, 140249657688064, 140249710600191, -STORE, 140249657679872, 140249710600191, -STORE, 140249657647104, 140249710600191, -STORE, 140249657638912, 140249710600191, -STORE, 140249657622528, 140249710600191, -STORE, 140249657614336, 140249710600191, -STORE, 140249657597952, 140249710600191, -STORE, 140249657589760, 140249710600191, -STORE, 140249657573376, 140249710600191, -STORE, 140249657565184, 140249710600191, -STORE, 140249657532416, 140249710600191, -STORE, 140249657524224, 140249710600191, -STORE, 140249657507840, 140249710600191, -STORE, 140249657499648, 140249710600191, -STORE, 140249657483264, 140249710600191, -STORE, 140249657475072, 140249710600191, -STORE, 140249657458688, 140249710600191, -STORE, 140249657450496, 140249710600191, -STORE, 140249657417728, 140249710600191, -STORE, 140249657409536, 140249710600191, -STORE, 140249657393152, 140249710600191, -STORE, 140249657384960, 140249710600191, -STORE, 140249657368576, 140249710600191, -STORE, 140249657360384, 140249710600191, -STORE, 140249657344000, 140249710600191, -STORE, 140249657335808, 140249710600191, -STORE, 140249657303040, 140249710600191, -STORE, 140249657294848, 140249710600191, -STORE, 140249657278464, 140249710600191, -STORE, 140249657270272, 140249710600191, -STORE, 140249657253888, 140249710600191, -STORE, 140249657245696, 140249710600191, -STORE, 140249657229312, 140249710600191, -STORE, 140249657221120, 140249710600191, -STORE, 140249657188352, 140249710600191, -STORE, 140249657180160, 140249710600191, -STORE, 140249657163776, 140249710600191, -STORE, 140249657155584, 140249710600191, -STORE, 140249657139200, 140249710600191, -STORE, 140249657131008, 140249710600191, -STORE, 140249657114624, 140249710600191, -STORE, 140249657106432, 140249710600191, -STORE, 140249657073664, 140249710600191, -STORE, 140249657065472, 140249710600191, -STORE, 140249657049088, 140249710600191, -STORE, 140249657040896, 140249710600191, -STORE, 140249657024512, 140249710600191, -STORE, 140249657016320, 140249710600191, -STORE, 140249656999936, 140249710600191, -STORE, 140249656991744, 140249710600191, -STORE, 140249656958976, 140249710600191, -STORE, 140249656950784, 140249710600191, -STORE, 140249656934400, 140249710600191, -STORE, 140249656926208, 140249710600191, -STORE, 140249656909824, 140249710600191, -STORE, 140249656901632, 140249710600191, -STORE, 140249656885248, 140249710600191, -STORE, 140249656877056, 140249710600191, -STORE, 140249656844288, 140249710600191, -STORE, 140249656836096, 140249710600191, -STORE, 140249656819712, 140249710600191, -STORE, 140249656811520, 140249710600191, -STORE, 140249656795136, 140249710600191, -STORE, 33853440, 38662143, -STORE, 140249656786944, 140249710600191, -STORE, 140249656770560, 140249710600191, -STORE, 140249656762368, 140249710600191, -STORE, 140249656729600, 140249710600191, -STORE, 140249656721408, 140249710600191, -STORE, 140249656705024, 140249710600191, -STORE, 140249656696832, 140249710600191, -STORE, 140249656680448, 140249710600191, -STORE, 140249656672256, 140249710600191, -STORE, 140249656655872, 140249710600191, -STORE, 140249656647680, 140249710600191, -STORE, 140249656614912, 140249710600191, -STORE, 140249656606720, 140249710600191, -STORE, 140249656590336, 140249710600191, -STORE, 140249656582144, 140249710600191, -STORE, 140249656565760, 140249710600191, -STORE, 140249656557568, 140249710600191, -STORE, 140249656541184, 140249710600191, -STORE, 140249656532992, 140249710600191, -STORE, 140249656500224, 140249710600191, -STORE, 140249656492032, 140249710600191, -STORE, 140249656475648, 140249710600191, -STORE, 140249656467456, 140249710600191, -STORE, 140249656451072, 140249710600191, -STORE, 140249656442880, 140249710600191, -STORE, 140249656426496, 140249710600191, -STORE, 140249656418304, 140249710600191, -STORE, 140249656385536, 140249710600191, -STORE, 140249656377344, 140249710600191, -STORE, 140249656360960, 140249710600191, -STORE, 140249656352768, 140249710600191, -STORE, 140249656336384, 140249710600191, -STORE, 140249656328192, 140249710600191, -STORE, 140249656311808, 140249710600191, -STORE, 140249656303616, 140249710600191, -STORE, 140249656270848, 140249710600191, -STORE, 140249656262656, 140249710600191, -STORE, 140249656246272, 140249710600191, -STORE, 140249656238080, 140249710600191, -STORE, 140249656221696, 140249710600191, -STORE, 140249656213504, 140249710600191, -STORE, 140249656197120, 140249710600191, -STORE, 140249656188928, 140249710600191, -STORE, 140249656156160, 140249710600191, -STORE, 140249656147968, 140249710600191, -STORE, 140249656131584, 140249710600191, -STORE, 140249656123392, 140249710600191, -STORE, 140249656107008, 140249710600191, -STORE, 140249656098816, 140249710600191, -STORE, 140249656082432, 140249710600191, -STORE, 140249656074240, 140249710600191, -STORE, 140249656041472, 140249710600191, -STORE, 140249656033280, 140249710600191, -STORE, 140249656016896, 140249710600191, -STORE, 140249656008704, 140249710600191, -STORE, 140249655992320, 140249710600191, -STORE, 140249655984128, 140249710600191, -STORE, 140249655967744, 140249710600191, -STORE, 140249653870592, 140249710600191, -STORE, 140249653862400, 140249710600191, -STORE, 140249653829632, 140249710600191, -STORE, 140249653821440, 140249710600191, -STORE, 140249653805056, 140249710600191, -STORE, 140249653796864, 140249710600191, -STORE, 140249653780480, 140249710600191, -STORE, 140249653772288, 140249710600191, -STORE, 140249653755904, 140249710600191, -STORE, 140249652703232, 140249710600191, -SNULL, 140249682087935, 140249710600191, -STORE, 140249652703232, 140249682087935, -STORE, 140249682087936, 140249710600191, - }; - - unsigned long set26[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140729464770560, 140737488351231, -SNULL, 140729464774655, 140737488351231, -STORE, 140729464770560, 140729464774655, -STORE, 140729464639488, 140729464774655, -STORE, 4194304, 5066751, -STORE, 7159808, 7172095, -STORE, 7172096, 7180287, -STORE, 140729465114624, 140729465118719, -STORE, 140729465102336, 140729465114623, -STORE, 30867456, 30875647, -STORE, 30867456, 31010815, -STORE, 140109040988160, 140109042671615, -STORE, 140109040959488, 140109040988159, -STORE, 140109040943104, 140109040959487, -ERASE, 140109040943104, 140109040959487, -STORE, 140109040840704, 140109040959487, -ERASE, 140109040840704, 140109040959487, -STORE, 140109040951296, 140109040959487, -ERASE, 140109040951296, 140109040959487, -STORE, 140109040955392, 140109040959487, -ERASE, 140109040955392, 140109040959487, - }; - unsigned long set27[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140726128070656, 140737488351231, -SNULL, 140726128074751, 140737488351231, -STORE, 140726128070656, 140726128074751, -STORE, 140726127939584, 140726128074751, -STORE, 94478497189888, 94478499303423, -SNULL, 94478497202175, 94478499303423, -STORE, 94478497189888, 94478497202175, -STORE, 94478497202176, 94478499303423, -ERASE, 94478497202176, 94478499303423, -STORE, 94478499295232, 94478499303423, -STORE, 140415605723136, 140415607975935, -SNULL, 140415605866495, 140415607975935, -STORE, 140415605723136, 140415605866495, -STORE, 140415605866496, 140415607975935, -ERASE, 140415605866496, 140415607975935, -STORE, 140415607963648, 140415607971839, -STORE, 140415607971840, 140415607975935, -STORE, 140726130024448, 140726130028543, -STORE, 140726130012160, 140726130024447, -STORE, 140415607934976, 140415607963647, -STORE, 140415607926784, 140415607934975, -STORE, 140415603245056, 140415605723135, -SNULL, 140415603245056, 140415603613695, -STORE, 140415603613696, 140415605723135, -STORE, 140415603245056, 140415603613695, -SNULL, 140415605710847, 140415605723135, -STORE, 140415603613696, 140415605710847, -STORE, 140415605710848, 140415605723135, -ERASE, 140415605710848, 140415605723135, -STORE, 140415605710848, 140415605723135, -STORE, 140415599370240, 140415603245055, -SNULL, 140415599370240, 140415601111039, -STORE, 140415601111040, 140415603245055, -STORE, 140415599370240, 140415601111039, -SNULL, 140415603208191, 140415603245055, -STORE, 140415601111040, 140415603208191, -STORE, 140415603208192, 140415603245055, -ERASE, 140415603208192, 140415603245055, -STORE, 140415603208192, 140415603245055, -STORE, 140415595692032, 140415599370239, -SNULL, 140415595692032, 140415597207551, -STORE, 140415597207552, 140415599370239, -STORE, 140415595692032, 140415597207551, -SNULL, 140415599304703, 140415599370239, -STORE, 140415597207552, 140415599304703, -STORE, 140415599304704, 140415599370239, -SNULL, 140415599304704, 140415599353855, -STORE, 140415599353856, 140415599370239, -STORE, 140415599304704, 140415599353855, -ERASE, 140415599304704, 140415599353855, -STORE, 140415599304704, 140415599353855, -ERASE, 140415599353856, 140415599370239, -STORE, 140415599353856, 140415599370239, -STORE, 140415593500672, 140415595692031, -SNULL, 140415593500672, 140415593590783, -STORE, 140415593590784, 140415595692031, -STORE, 140415593500672, 140415593590783, -SNULL, 140415595683839, 140415595692031, -STORE, 140415593590784, 140415595683839, -STORE, 140415595683840, 140415595692031, -ERASE, 140415595683840, 140415595692031, -STORE, 140415595683840, 140415595692031, -STORE, 140415589703680, 140415593500671, -SNULL, 140415589703680, 140415591362559, -STORE, 140415591362560, 140415593500671, -STORE, 140415589703680, 140415591362559, -SNULL, 140415593459711, 140415593500671, -STORE, 140415591362560, 140415593459711, -STORE, 140415593459712, 140415593500671, -SNULL, 140415593459712, 140415593484287, -STORE, 140415593484288, 140415593500671, -STORE, 140415593459712, 140415593484287, -ERASE, 140415593459712, 140415593484287, -STORE, 140415593459712, 140415593484287, -ERASE, 140415593484288, 140415593500671, -STORE, 140415593484288, 140415593500671, -STORE, 140415587590144, 140415589703679, -SNULL, 140415587590144, 140415587602431, -STORE, 140415587602432, 140415589703679, -STORE, 140415587590144, 140415587602431, -SNULL, 140415589695487, 140415589703679, -STORE, 140415587602432, 140415589695487, -STORE, 140415589695488, 140415589703679, -ERASE, 140415589695488, 140415589703679, -STORE, 140415589695488, 140415589703679, -STORE, 140415607918592, 140415607934975, -STORE, 140415585398784, 140415587590143, -SNULL, 140415585398784, 140415585480703, -STORE, 140415585480704, 140415587590143, -STORE, 140415585398784, 140415585480703, -SNULL, 140415587573759, 140415587590143, -STORE, 140415585480704, 140415587573759, -STORE, 140415587573760, 140415587590143, -SNULL, 140415587573760, 140415587581951, -STORE, 140415587581952, 140415587590143, -STORE, 140415587573760, 140415587581951, -ERASE, 140415587573760, 140415587581951, -STORE, 140415587573760, 140415587581951, -ERASE, 140415587581952, 140415587590143, -STORE, 140415587581952, 140415587590143, -STORE, 140415583182848, 140415585398783, -SNULL, 140415583182848, 140415583281151, -STORE, 140415583281152, 140415585398783, -STORE, 140415583182848, 140415583281151, -SNULL, 140415585374207, 140415585398783, -STORE, 140415583281152, 140415585374207, -STORE, 140415585374208, 140415585398783, -SNULL, 140415585374208, 140415585382399, -STORE, 140415585382400, 140415585398783, -STORE, 140415585374208, 140415585382399, -ERASE, 140415585374208, 140415585382399, -STORE, 140415585374208, 140415585382399, -ERASE, 140415585382400, 140415585398783, -STORE, 140415585382400, 140415585398783, -STORE, 140415580979200, 140415583182847, -SNULL, 140415580979200, 140415581081599, -STORE, 140415581081600, 140415583182847, -STORE, 140415580979200, 140415581081599, -SNULL, 140415583174655, 140415583182847, -STORE, 140415581081600, 140415583174655, -STORE, 140415583174656, 140415583182847, -ERASE, 140415583174656, 140415583182847, -STORE, 140415583174656, 140415583182847, -STORE, 140415578816512, 140415580979199, -SNULL, 140415578816512, 140415578877951, -STORE, 140415578877952, 140415580979199, -STORE, 140415578816512, 140415578877951, -SNULL, 140415580971007, 140415580979199, -STORE, 140415578877952, 140415580971007, -STORE, 140415580971008, 140415580979199, -ERASE, 140415580971008, 140415580979199, -STORE, 140415580971008, 140415580979199, -STORE, 140415576563712, 140415578816511, -SNULL, 140415576563712, 140415576715263, -STORE, 140415576715264, 140415578816511, -STORE, 140415576563712, 140415576715263, -SNULL, 140415578808319, 140415578816511, -STORE, 140415576715264, 140415578808319, -STORE, 140415578808320, 140415578816511, -ERASE, 140415578808320, 140415578816511, -STORE, 140415578808320, 140415578816511, -STORE, 140415574392832, 140415576563711, -SNULL, 140415574392832, 140415574462463, -STORE, 140415574462464, 140415576563711, -STORE, 140415574392832, 140415574462463, -SNULL, 140415576555519, 140415576563711, -STORE, 140415574462464, 140415576555519, -STORE, 140415576555520, 140415576563711, -ERASE, 140415576555520, 140415576563711, -STORE, 140415576555520, 140415576563711, -STORE, 140415607910400, 140415607934975, -STORE, 140415571230720, 140415574392831, -SNULL, 140415571230720, 140415572291583, -STORE, 140415572291584, 140415574392831, -STORE, 140415571230720, 140415572291583, -SNULL, 140415574384639, 140415574392831, -STORE, 140415572291584, 140415574384639, -STORE, 140415574384640, 140415574392831, -ERASE, 140415574384640, 140415574392831, -STORE, 140415574384640, 140415574392831, -STORE, 140415607902208, 140415607934975, -SNULL, 140415593476095, 140415593484287, -STORE, 140415593459712, 140415593476095, -STORE, 140415593476096, 140415593484287, -SNULL, 140415574388735, 140415574392831, -STORE, 140415574384640, 140415574388735, -STORE, 140415574388736, 140415574392831, -SNULL, 140415576559615, 140415576563711, -STORE, 140415576555520, 140415576559615, -STORE, 140415576559616, 140415576563711, -SNULL, 140415589699583, 140415589703679, -STORE, 140415589695488, 140415589699583, -STORE, 140415589699584, 140415589703679, -SNULL, 140415585378303, 140415585382399, -STORE, 140415585374208, 140415585378303, -STORE, 140415585378304, 140415585382399, -SNULL, 140415578812415, 140415578816511, -STORE, 140415578808320, 140415578812415, -STORE, 140415578812416, 140415578816511, -SNULL, 140415580975103, 140415580979199, -STORE, 140415580971008, 140415580975103, -STORE, 140415580975104, 140415580979199, -SNULL, 140415583178751, 140415583182847, -STORE, 140415583174656, 140415583178751, -STORE, 140415583178752, 140415583182847, -SNULL, 140415587577855, 140415587581951, -STORE, 140415587573760, 140415587577855, -STORE, 140415587577856, 140415587581951, -SNULL, 140415595687935, 140415595692031, -STORE, 140415595683840, 140415595687935, -STORE, 140415595687936, 140415595692031, -STORE, 140415607894016, 140415607934975, -SNULL, 140415599345663, 140415599353855, -STORE, 140415599304704, 140415599345663, -STORE, 140415599345664, 140415599353855, -SNULL, 140415603240959, 140415603245055, -STORE, 140415603208192, 140415603240959, -STORE, 140415603240960, 140415603245055, -SNULL, 140415605719039, 140415605723135, -STORE, 140415605710848, 140415605719039, -STORE, 140415605719040, 140415605723135, -SNULL, 94478499299327, 94478499303423, -STORE, 94478499295232, 94478499299327, -STORE, 94478499299328, 94478499303423, -SNULL, 140415607967743, 140415607971839, -STORE, 140415607963648, 140415607967743, -STORE, 140415607967744, 140415607971839, -ERASE, 140415607934976, 140415607963647, -STORE, 94478511173632, 94478511378431, -STORE, 140415606210560, 140415607894015, -STORE, 140415607934976, 140415607963647, -STORE, 94478511173632, 94478511513599, -STORE, 94478511173632, 94478511648767, -SNULL, 94478511615999, 94478511648767, -STORE, 94478511173632, 94478511615999, -STORE, 94478511616000, 94478511648767, -ERASE, 94478511616000, 94478511648767, -STORE, 94478511173632, 94478511751167, -SNULL, 94478511747071, 94478511751167, -STORE, 94478511173632, 94478511747071, -STORE, 94478511747072, 94478511751167, -ERASE, 94478511747072, 94478511751167, -STORE, 94478511173632, 94478511882239, -SNULL, 94478511878143, 94478511882239, -STORE, 94478511173632, 94478511878143, -STORE, 94478511878144, 94478511882239, -ERASE, 94478511878144, 94478511882239, -STORE, 94478511173632, 94478512013311, -SNULL, 94478512009215, 94478512013311, -STORE, 94478511173632, 94478512009215, -STORE, 94478512009216, 94478512013311, -ERASE, 94478512009216, 94478512013311, -STORE, 94478511173632, 94478512144383, -STORE, 94478511173632, 94478512279551, -STORE, 140415606181888, 140415606210559, -STORE, 140415569100800, 140415571230719, -SNULL, 140415569100800, 140415569129471, -STORE, 140415569129472, 140415571230719, -STORE, 140415569100800, 140415569129471, -SNULL, 140415571222527, 140415571230719, -STORE, 140415569129472, 140415571222527, -STORE, 140415571222528, 140415571230719, -ERASE, 140415571222528, 140415571230719, -STORE, 140415571222528, 140415571230719, -STORE, 140415566905344, 140415569100799, -SNULL, 140415566905344, 140415566987263, -STORE, 140415566987264, 140415569100799, -STORE, 140415566905344, 140415566987263, -SNULL, 140415569084415, 140415569100799, -STORE, 140415566987264, 140415569084415, -STORE, 140415569084416, 140415569100799, -SNULL, 140415569084416, 140415569092607, -STORE, 140415569092608, 140415569100799, -STORE, 140415569084416, 140415569092607, -ERASE, 140415569084416, 140415569092607, -STORE, 140415569084416, 140415569092607, -ERASE, 140415569092608, 140415569100799, -STORE, 140415569092608, 140415569100799, -SNULL, 140415569088511, 140415569092607, -STORE, 140415569084416, 140415569088511, -STORE, 140415569088512, 140415569092607, -SNULL, 140415571226623, 140415571230719, -STORE, 140415571222528, 140415571226623, -STORE, 140415571226624, 140415571230719, -ERASE, 140415606181888, 140415606210559, -STORE, 140415606181888, 140415606210559, -STORE, 140415564759040, 140415566905343, -SNULL, 140415564759040, 140415564804095, -STORE, 140415564804096, 140415566905343, -STORE, 140415564759040, 140415564804095, -SNULL, 140415566897151, 140415566905343, -STORE, 140415564804096, 140415566897151, -STORE, 140415566897152, 140415566905343, -ERASE, 140415566897152, 140415566905343, -STORE, 140415566897152, 140415566905343, -STORE, 140415562588160, 140415564759039, -SNULL, 140415562588160, 140415562629119, -STORE, 140415562629120, 140415564759039, -STORE, 140415562588160, 140415562629119, -SNULL, 140415564726271, 140415564759039, -STORE, 140415562629120, 140415564726271, -STORE, 140415564726272, 140415564759039, -SNULL, 140415564726272, 140415564734463, -STORE, 140415564734464, 140415564759039, -STORE, 140415564726272, 140415564734463, -ERASE, 140415564726272, 140415564734463, -STORE, 140415564726272, 140415564734463, -ERASE, 140415564734464, 140415564759039, -STORE, 140415564734464, 140415564759039, -SNULL, 140415564730367, 140415564734463, -STORE, 140415564726272, 140415564730367, -STORE, 140415564730368, 140415564734463, -SNULL, 140415566901247, 140415566905343, -STORE, 140415566897152, 140415566901247, -STORE, 140415566901248, 140415566905343, -ERASE, 140415606181888, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415605944320, 140415606210559, -ERASE, 140415605944320, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 94478511173632, 94478512414719, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 140415606206464, 140415606210559, -ERASE, 140415606206464, 140415606210559, -STORE, 94478511173632, 94478512652287, -STORE, 94478511173632, 94478512787455, -STORE, 94478511173632, 94478512922623, -STORE, 94478511173632, 94478513057791, -STORE, 140415537422336, 140415562588159, -STORE, 94478511173632, 94478513192959, -STORE, 94478511173632, 94478513356799, -STORE, 94478511173632, 94478513491967, -STORE, 94478511173632, 94478513627135, -STORE, 94478511173632, 94478513790975, -STORE, 94478511173632, 94478513926143, -STORE, 94478511173632, 94478514061311, -STORE, 94478511173632, 94478514196479, -STORE, 94478511173632, 94478514331647, -STORE, 94478511173632, 94478514606079, -STORE, 94478511173632, 94478514741247, -STORE, 94478511173632, 94478514876415, -STORE, 94478511173632, 94478515011583, -STORE, 94478511173632, 94478515146751, -STORE, 94478511173632, 94478515281919, -STORE, 94478511173632, 94478515474431, -STORE, 94478511173632, 94478515609599, -STORE, 94478511173632, 94478515744767, -STORE, 140415536922624, 140415562588159, -STORE, 94478511173632, 94478515879935, -STORE, 94478511173632, 94478516015103, -STORE, 94478511173632, 94478516150271, -STORE, 94478511173632, 94478516285439, -STORE, 94478511173632, 94478516420607, -STORE, 94478511173632, 94478516555775, -STORE, 94478511173632, 94478516690943, -STORE, 94478511173632, 94478516826111, -STORE, 94478511173632, 94478516961279, -STORE, 94478511173632, 94478517231615, -STORE, 94478511173632, 94478517366783, -STORE, 94478511173632, 94478517501951, -STORE, 94478511173632, 94478517637119, -STORE, 94478511173632, 94478517772287, -STORE, 94478511173632, 94478517907455, -STORE, 94478511173632, 94478518042623, -STORE, 94478511173632, 94478518177791, -STORE, 94478511173632, 94478518312959, -STORE, 94478511173632, 94478518448127, -STORE, 140415535910912, 140415562588159, -SNULL, 140415536922623, 140415562588159, -STORE, 140415535910912, 140415536922623, -STORE, 140415536922624, 140415562588159, -SNULL, 140415536922624, 140415537422335, -STORE, 140415537422336, 140415562588159, -STORE, 140415536922624, 140415537422335, -ERASE, 140415536922624, 140415537422335, -STORE, 94478511173632, 94478518583295, -STORE, 94478511173632, 94478518718463, -STORE, 94478511173632, 94478518853631, -STORE, 94478511173632, 94478518988799, -STORE, 94478511173632, 94478519123967, -STORE, 94478511173632, 94478519259135, -STORE, 140415509696512, 140415535910911, -ERASE, 140415537422336, 140415562588159, -STORE, 140415482433536, 140415509696511, - }; - unsigned long set28[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140722475622400, 140737488351231, -SNULL, 140722475626495, 140737488351231, -STORE, 140722475622400, 140722475626495, -STORE, 140722475491328, 140722475626495, -STORE, 93865834291200, 93865836548095, -SNULL, 93865834422271, 93865836548095, -STORE, 93865834291200, 93865834422271, -STORE, 93865834422272, 93865836548095, -ERASE, 93865834422272, 93865836548095, -STORE, 93865836519424, 93865836527615, -STORE, 93865836527616, 93865836548095, -STORE, 139918411104256, 139918413357055, -SNULL, 139918411247615, 139918413357055, -STORE, 139918411104256, 139918411247615, -STORE, 139918411247616, 139918413357055, -ERASE, 139918411247616, 139918413357055, -STORE, 139918413344768, 139918413352959, -STORE, 139918413352960, 139918413357055, -STORE, 140722476642304, 140722476646399, -STORE, 140722476630016, 140722476642303, -STORE, 139918413316096, 139918413344767, -STORE, 139918413307904, 139918413316095, -STORE, 139918408888320, 139918411104255, -SNULL, 139918408888320, 139918408986623, -STORE, 139918408986624, 139918411104255, -STORE, 139918408888320, 139918408986623, -SNULL, 139918411079679, 139918411104255, -STORE, 139918408986624, 139918411079679, -STORE, 139918411079680, 139918411104255, -SNULL, 139918411079680, 139918411087871, -STORE, 139918411087872, 139918411104255, -STORE, 139918411079680, 139918411087871, -ERASE, 139918411079680, 139918411087871, -STORE, 139918411079680, 139918411087871, -ERASE, 139918411087872, 139918411104255, -STORE, 139918411087872, 139918411104255, -STORE, 139918405091328, 139918408888319, -SNULL, 139918405091328, 139918406750207, -STORE, 139918406750208, 139918408888319, -STORE, 139918405091328, 139918406750207, -SNULL, 139918408847359, 139918408888319, -STORE, 139918406750208, 139918408847359, -STORE, 139918408847360, 139918408888319, -SNULL, 139918408847360, 139918408871935, -STORE, 139918408871936, 139918408888319, -STORE, 139918408847360, 139918408871935, -ERASE, 139918408847360, 139918408871935, -STORE, 139918408847360, 139918408871935, -ERASE, 139918408871936, 139918408888319, -STORE, 139918408871936, 139918408888319, -STORE, 139918413299712, 139918413316095, -SNULL, 139918408863743, 139918408871935, -STORE, 139918408847360, 139918408863743, -STORE, 139918408863744, 139918408871935, -SNULL, 139918411083775, 139918411087871, -STORE, 139918411079680, 139918411083775, -STORE, 139918411083776, 139918411087871, -SNULL, 93865836523519, 93865836527615, -STORE, 93865836519424, 93865836523519, -STORE, 93865836523520, 93865836527615, -SNULL, 139918413348863, 139918413352959, -STORE, 139918413344768, 139918413348863, -STORE, 139918413348864, 139918413352959, -ERASE, 139918413316096, 139918413344767, -STORE, 93865848528896, 93865848664063, - }; - unsigned long set29[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140734467944448, 140737488351231, -SNULL, 140734467948543, 140737488351231, -STORE, 140734467944448, 140734467948543, -STORE, 140734467813376, 140734467948543, -STORE, 94880407924736, 94880410177535, -SNULL, 94880408055807, 94880410177535, -STORE, 94880407924736, 94880408055807, -STORE, 94880408055808, 94880410177535, -ERASE, 94880408055808, 94880410177535, -STORE, 94880410148864, 94880410157055, -STORE, 94880410157056, 94880410177535, -STORE, 140143367815168, 140143370067967, -SNULL, 140143367958527, 140143370067967, -STORE, 140143367815168, 140143367958527, -STORE, 140143367958528, 140143370067967, -ERASE, 140143367958528, 140143370067967, -STORE, 140143370055680, 140143370063871, -STORE, 140143370063872, 140143370067967, -STORE, 140734468329472, 140734468333567, -STORE, 140734468317184, 140734468329471, -STORE, 140143370027008, 140143370055679, -STORE, 140143370018816, 140143370027007, -STORE, 140143365599232, 140143367815167, -SNULL, 140143365599232, 140143365697535, -STORE, 140143365697536, 140143367815167, -STORE, 140143365599232, 140143365697535, -SNULL, 140143367790591, 140143367815167, -STORE, 140143365697536, 140143367790591, -STORE, 140143367790592, 140143367815167, -SNULL, 140143367790592, 140143367798783, -STORE, 140143367798784, 140143367815167, -STORE, 140143367790592, 140143367798783, -ERASE, 140143367790592, 140143367798783, -STORE, 140143367790592, 140143367798783, -ERASE, 140143367798784, 140143367815167, -STORE, 140143367798784, 140143367815167, -STORE, 140143361802240, 140143365599231, -SNULL, 140143361802240, 140143363461119, -STORE, 140143363461120, 140143365599231, -STORE, 140143361802240, 140143363461119, -SNULL, 140143365558271, 140143365599231, -STORE, 140143363461120, 140143365558271, -STORE, 140143365558272, 140143365599231, -SNULL, 140143365558272, 140143365582847, -STORE, 140143365582848, 140143365599231, -STORE, 140143365558272, 140143365582847, -ERASE, 140143365558272, 140143365582847, -STORE, 140143365558272, 140143365582847, -ERASE, 140143365582848, 140143365599231, -STORE, 140143365582848, 140143365599231, -STORE, 140143370010624, 140143370027007, -SNULL, 140143365574655, 140143365582847, -STORE, 140143365558272, 140143365574655, -STORE, 140143365574656, 140143365582847, -SNULL, 140143367794687, 140143367798783, -STORE, 140143367790592, 140143367794687, -STORE, 140143367794688, 140143367798783, -SNULL, 94880410152959, 94880410157055, -STORE, 94880410148864, 94880410152959, -STORE, 94880410152960, 94880410157055, -SNULL, 140143370059775, 140143370063871, -STORE, 140143370055680, 140143370059775, -STORE, 140143370059776, 140143370063871, -ERASE, 140143370027008, 140143370055679, -STORE, 94880442400768, 94880442535935, -STORE, 140143353409536, 140143361802239, -SNULL, 140143353413631, 140143361802239, -STORE, 140143353409536, 140143353413631, -STORE, 140143353413632, 140143361802239, -STORE, 140143345016832, 140143353409535, -STORE, 140143210799104, 140143345016831, -SNULL, 140143210799104, 140143239364607, -STORE, 140143239364608, 140143345016831, -STORE, 140143210799104, 140143239364607, -ERASE, 140143210799104, 140143239364607, -SNULL, 140143306473471, 140143345016831, -STORE, 140143239364608, 140143306473471, -STORE, 140143306473472, 140143345016831, -ERASE, 140143306473472, 140143345016831, -SNULL, 140143239499775, 140143306473471, -STORE, 140143239364608, 140143239499775, -STORE, 140143239499776, 140143306473471, -SNULL, 140143345020927, 140143353409535, -STORE, 140143345016832, 140143345020927, -STORE, 140143345020928, 140143353409535, -STORE, 140143336624128, 140143345016831, -SNULL, 140143336628223, 140143345016831, -STORE, 140143336624128, 140143336628223, -STORE, 140143336628224, 140143345016831, -STORE, 140143328231424, 140143336624127, -SNULL, 140143328235519, 140143336624127, -STORE, 140143328231424, 140143328235519, -STORE, 140143328235520, 140143336624127, -STORE, 140143319838720, 140143328231423, -SNULL, 140143319842815, 140143328231423, -STORE, 140143319838720, 140143319842815, -STORE, 140143319842816, 140143328231423, -STORE, 140143311446016, 140143319838719, -STORE, 140143105146880, 140143239364607, -STORE, 140143096754176, 140143105146879, -STORE, 140143029645312, 140143096754175, -ERASE, 140143029645312, 140143096754175, -STORE, 140142962536448, 140143096754175, -SNULL, 140142962536448, 140142970929151, -STORE, 140142970929152, 140143096754175, -STORE, 140142962536448, 140142970929151, -ERASE, 140142962536448, 140142970929151, -STORE, 140142962536448, 140142970929151, -STORE, 140142828318720, 140142962536447, -STORE, 140142819926016, 140142828318719, -SNULL, 140142828318720, 140142836711423, -STORE, 140142836711424, 140142962536447, -STORE, 140142828318720, 140142836711423, -ERASE, 140142828318720, 140142836711423, -SNULL, 140143172255743, 140143239364607, -STORE, 140143105146880, 140143172255743, -STORE, 140143172255744, 140143239364607, -ERASE, 140143172255744, 140143239364607, -SNULL, 140143105282047, 140143172255743, -STORE, 140143105146880, 140143105282047, -STORE, 140143105282048, 140143172255743, -SNULL, 140143038038015, 140143096754175, -STORE, 140142970929152, 140143038038015, -STORE, 140143038038016, 140143096754175, -ERASE, 140143038038016, 140143096754175, -SNULL, 140142971064319, 140143038038015, -STORE, 140142970929152, 140142971064319, -STORE, 140142971064320, 140143038038015, -SNULL, 140142903820287, 140142962536447, -STORE, 140142836711424, 140142903820287, -STORE, 140142903820288, 140142962536447, -ERASE, 140142903820288, 140142962536447, -SNULL, 140142836846591, 140142903820287, -STORE, 140142836711424, 140142836846591, -STORE, 140142836846592, 140142903820287, -STORE, 140142685708288, 140142819926015, -SNULL, 140143311450111, 140143319838719, -STORE, 140143311446016, 140143311450111, -STORE, 140143311450112, 140143319838719, -SNULL, 140142962540543, 140142970929151, -STORE, 140142962536448, 140142962540543, -STORE, 140142962540544, 140142970929151, -SNULL, 140142685708288, 140142702493695, -STORE, 140142702493696, 140142819926015, -STORE, 140142685708288, 140142702493695, -ERASE, 140142685708288, 140142702493695, -SNULL, 140142769602559, 140142819926015, -STORE, 140142702493696, 140142769602559, -STORE, 140142769602560, 140142819926015, -ERASE, 140142769602560, 140142819926015, -SNULL, 140142702628863, 140142769602559, -STORE, 140142702493696, 140142702628863, -STORE, 140142702628864, 140142769602559, -STORE, 140143230971904, 140143239364607, -SNULL, 140143230975999, 140143239364607, -STORE, 140143230971904, 140143230975999, -STORE, 140143230976000, 140143239364607, -SNULL, 140143096758271, 140143105146879, -STORE, 140143096754176, 140143096758271, -STORE, 140143096758272, 140143105146879, -STORE, 140143222579200, 140143230971903, -SNULL, 140143222583295, 140143230971903, -STORE, 140143222579200, 140143222583295, -STORE, 140143222583296, 140143230971903, -STORE, 140143214186496, 140143222579199, -SNULL, 140142819930111, 140142828318719, -STORE, 140142819926016, 140142819930111, -STORE, 140142819930112, 140142828318719, -STORE, 140143205793792, 140143222579199, -SNULL, 140143205793792, 140143214186495, -STORE, 140143214186496, 140143222579199, -STORE, 140143205793792, 140143214186495, -SNULL, 140143214190591, 140143222579199, -STORE, 140143214186496, 140143214190591, -STORE, 140143214190592, 140143222579199, -SNULL, 140143205797887, 140143214186495, -STORE, 140143205793792, 140143205797887, -STORE, 140143205797888, 140143214186495, -STORE, 140143197401088, 140143205793791, -SNULL, 140143197405183, 140143205793791, -STORE, 140143197401088, 140143197405183, -STORE, 140143197405184, 140143205793791, -STORE, 140143189008384, 140143197401087, -STORE, 140143180615680, 140143197401087, -STORE, 140143088361472, 140143096754175, -SNULL, 140143180619775, 140143197401087, -STORE, 140143180615680, 140143180619775, -STORE, 140143180619776, 140143197401087, -SNULL, 140143180619776, 140143189008383, -STORE, 140143189008384, 140143197401087, -STORE, 140143180619776, 140143189008383, -SNULL, 140143189012479, 140143197401087, -STORE, 140143189008384, 140143189012479, -STORE, 140143189012480, 140143197401087, -SNULL, 140143088365567, 140143096754175, -STORE, 140143088361472, 140143088365567, -STORE, 140143088365568, 140143096754175, -STORE, 140143079968768, 140143088361471, -SNULL, 140143079972863, 140143088361471, -STORE, 140143079968768, 140143079972863, -STORE, 140143079972864, 140143088361471, -STORE, 140143071576064, 140143079968767, -SNULL, 140143071580159, 140143079968767, -STORE, 140143071576064, 140143071580159, -STORE, 140143071580160, 140143079968767, -STORE, 140143063183360, 140143071576063, -STORE, 140143054790656, 140143071576063, -SNULL, 140143054794751, 140143071576063, -STORE, 140143054790656, 140143054794751, -STORE, 140143054794752, 140143071576063, -SNULL, 140143054794752, 140143063183359, -STORE, 140143063183360, 140143071576063, -STORE, 140143054794752, 140143063183359, -SNULL, 140143063187455, 140143071576063, -STORE, 140143063183360, 140143063187455, -STORE, 140143063187456, 140143071576063, -STORE, 140143046397952, 140143054790655, -STORE, 140142954143744, 140142962536447, -STORE, 140142945751040, 140142962536447, -STORE, 140142937358336, 140142962536447, -STORE, 140142928965632, 140142962536447, -STORE, 140142568275968, 140142702493695, -SNULL, 140142635384831, 140142702493695, -STORE, 140142568275968, 140142635384831, -STORE, 140142635384832, 140142702493695, -ERASE, 140142635384832, 140142702493695, -STORE, 140142920572928, 140142962536447, -STORE, 140142912180224, 140142962536447, -STORE, 140142568275968, 140142702493695, -SNULL, 140142568275968, 140142635384831, -STORE, 140142635384832, 140142702493695, -STORE, 140142568275968, 140142635384831, -SNULL, 140142635519999, 140142702493695, -STORE, 140142635384832, 140142635519999, -STORE, 140142635520000, 140142702493695, -STORE, 140142819930112, 140142836711423, -STORE, 140142811533312, 140142819926015, -STORE, 140142434058240, 140142635384831, -SNULL, 140142501167103, 140142635384831, -STORE, 140142434058240, 140142501167103, -STORE, 140142501167104, 140142635384831, -SNULL, 140142501167104, 140142568275967, -STORE, 140142568275968, 140142635384831, -STORE, 140142501167104, 140142568275967, -ERASE, 140142501167104, 140142568275967, -STORE, 140142299840512, 140142501167103, -STORE, 140142803140608, 140142819926015, -SNULL, 140142366949375, 140142501167103, -STORE, 140142299840512, 140142366949375, -STORE, 140142366949376, 140142501167103, -SNULL, 140142366949376, 140142434058239, -STORE, 140142434058240, 140142501167103, -STORE, 140142366949376, 140142434058239, -ERASE, 140142366949376, 140142434058239, -STORE, 140142794747904, 140142819926015, -STORE, 140142786355200, 140142819926015, -STORE, 140142299840512, 140142501167103, -STORE, 140142777962496, 140142819926015, -STORE, 140142559883264, 140142568275967, -STORE, 140142232731648, 140142501167103, -STORE, 140142551490560, 140142568275967, -SNULL, 140142777962496, 140142803140607, -STORE, 140142803140608, 140142819926015, -STORE, 140142777962496, 140142803140607, -SNULL, 140142803144703, 140142819926015, -STORE, 140142803140608, 140142803144703, -STORE, 140142803144704, 140142819926015, -STORE, 140142543097856, 140142568275967, -STORE, 140142098513920, 140142501167103, -SNULL, 140142165622783, 140142501167103, -STORE, 140142098513920, 140142165622783, -STORE, 140142165622784, 140142501167103, -SNULL, 140142165622784, 140142232731647, -STORE, 140142232731648, 140142501167103, -STORE, 140142165622784, 140142232731647, -ERASE, 140142165622784, 140142232731647, -SNULL, 140142568411135, 140142635384831, -STORE, 140142568275968, 140142568411135, -STORE, 140142568411136, 140142635384831, -STORE, 140141964296192, 140142165622783, -SNULL, 140142912180224, 140142928965631, -STORE, 140142928965632, 140142962536447, -STORE, 140142912180224, 140142928965631, -SNULL, 140142928969727, 140142962536447, -STORE, 140142928965632, 140142928969727, -STORE, 140142928969728, 140142962536447, -STORE, 140141830078464, 140142165622783, -SNULL, 140142912184319, 140142928965631, -STORE, 140142912180224, 140142912184319, -STORE, 140142912184320, 140142928965631, -SNULL, 140142232731648, 140142434058239, -STORE, 140142434058240, 140142501167103, -STORE, 140142232731648, 140142434058239, -SNULL, 140142434193407, 140142501167103, -STORE, 140142434058240, 140142434193407, -STORE, 140142434193408, 140142501167103, -SNULL, 140142232731648, 140142299840511, -STORE, 140142299840512, 140142434058239, -STORE, 140142232731648, 140142299840511, -SNULL, 140142299975679, 140142434058239, -STORE, 140142299840512, 140142299975679, -STORE, 140142299975680, 140142434058239, -SNULL, 140142928969728, 140142954143743, -STORE, 140142954143744, 140142962536447, -STORE, 140142928969728, 140142954143743, -SNULL, 140142954147839, 140142962536447, -STORE, 140142954143744, 140142954147839, -STORE, 140142954147840, 140142962536447, -STORE, 140141830078464, 140142299840511, -SNULL, 140142543097856, 140142559883263, -STORE, 140142559883264, 140142568275967, -STORE, 140142543097856, 140142559883263, -SNULL, 140142559887359, 140142568275967, -STORE, 140142559883264, 140142559887359, -STORE, 140142559887360, 140142568275967, -STORE, 140142534705152, 140142559883263, -SNULL, 140142928969728, 140142945751039, -STORE, 140142945751040, 140142954143743, -STORE, 140142928969728, 140142945751039, -SNULL, 140142945755135, 140142954143743, -STORE, 140142945751040, 140142945755135, -STORE, 140142945755136, 140142954143743, -SNULL, 140142299975680, 140142366949375, -STORE, 140142366949376, 140142434058239, -STORE, 140142299975680, 140142366949375, -SNULL, 140142367084543, 140142434058239, -STORE, 140142366949376, 140142367084543, -STORE, 140142367084544, 140142434058239, -SNULL, 140142928969728, 140142937358335, -STORE, 140142937358336, 140142945751039, -STORE, 140142928969728, 140142937358335, -SNULL, 140142937362431, 140142945751039, -STORE, 140142937358336, 140142937362431, -STORE, 140142937362432, 140142945751039, -SNULL, 140141830078464, 140142232731647, -STORE, 140142232731648, 140142299840511, -STORE, 140141830078464, 140142232731647, -SNULL, 140142232866815, 140142299840511, -STORE, 140142232731648, 140142232866815, -STORE, 140142232866816, 140142299840511, -SNULL, 140142534705152, 140142543097855, -STORE, 140142543097856, 140142559883263, -STORE, 140142534705152, 140142543097855, -SNULL, 140142543101951, 140142559883263, -STORE, 140142543097856, 140142543101951, -STORE, 140142543101952, 140142559883263, -STORE, 140142526312448, 140142543097855, -STORE, 140142517919744, 140142543097855, -SNULL, 140141830078464, 140142098513919, -STORE, 140142098513920, 140142232731647, -STORE, 140141830078464, 140142098513919, -SNULL, 140142098649087, 140142232731647, -STORE, 140142098513920, 140142098649087, -STORE, 140142098649088, 140142232731647, -SNULL, 140142031405055, 140142098513919, -STORE, 140141830078464, 140142031405055, -STORE, 140142031405056, 140142098513919, -ERASE, 140142031405056, 140142098513919, -SNULL, 140141830078464, 140141964296191, -STORE, 140141964296192, 140142031405055, -STORE, 140141830078464, 140141964296191, -SNULL, 140141964431359, 140142031405055, -STORE, 140141964296192, 140141964431359, -STORE, 140141964431360, 140142031405055, -STORE, 140142509527040, 140142543097855, -SNULL, 140141897187327, 140141964296191, -STORE, 140141830078464, 140141897187327, -STORE, 140141897187328, 140141964296191, -ERASE, 140141897187328, 140141964296191, -SNULL, 140141830213631, 140141897187327, -STORE, 140141830078464, 140141830213631, -STORE, 140141830213632, 140141897187327, -SNULL, 140142803144704, 140142811533311, -STORE, 140142811533312, 140142819926015, -STORE, 140142803144704, 140142811533311, -SNULL, 140142811537407, 140142819926015, -STORE, 140142811533312, 140142811537407, -STORE, 140142811537408, 140142819926015, -SNULL, 140142098649088, 140142165622783, -STORE, 140142165622784, 140142232731647, -STORE, 140142098649088, 140142165622783, -SNULL, 140142165757951, 140142232731647, -STORE, 140142165622784, 140142165757951, -STORE, 140142165757952, 140142232731647, -STORE, 140142090121216, 140142098513919, -SNULL, 140142777962496, 140142786355199, -STORE, 140142786355200, 140142803140607, -STORE, 140142777962496, 140142786355199, -SNULL, 140142786359295, 140142803140607, -STORE, 140142786355200, 140142786359295, -STORE, 140142786359296, 140142803140607, -SNULL, 140142509527040, 140142534705151, -STORE, 140142534705152, 140142543097855, -STORE, 140142509527040, 140142534705151, -SNULL, 140142534709247, 140142543097855, -STORE, 140142534705152, 140142534709247, -STORE, 140142534709248, 140142543097855, -STORE, 140142081728512, 140142098513919, -SNULL, 140142786359296, 140142794747903, -STORE, 140142794747904, 140142803140607, -STORE, 140142786359296, 140142794747903, -SNULL, 140142794751999, 140142803140607, -STORE, 140142794747904, 140142794751999, -STORE, 140142794752000, 140142803140607, -STORE, 140142073335808, 140142098513919, -SNULL, 140142073339903, 140142098513919, -STORE, 140142073335808, 140142073339903, -STORE, 140142073339904, 140142098513919, -SNULL, 140142543101952, 140142551490559, -STORE, 140142551490560, 140142559883263, -STORE, 140142543101952, 140142551490559, -SNULL, 140142551494655, 140142559883263, -STORE, 140142551490560, 140142551494655, -STORE, 140142551494656, 140142559883263, -SNULL, 140142509527040, 140142517919743, -STORE, 140142517919744, 140142534705151, -STORE, 140142509527040, 140142517919743, -SNULL, 140142517923839, 140142534705151, -STORE, 140142517919744, 140142517923839, -STORE, 140142517923840, 140142534705151, -STORE, 140142064943104, 140142073335807, -SNULL, 140142073339904, 140142090121215, -STORE, 140142090121216, 140142098513919, -STORE, 140142073339904, 140142090121215, -SNULL, 140142090125311, 140142098513919, -STORE, 140142090121216, 140142090125311, -STORE, 140142090125312, 140142098513919, -STORE, 140142056550400, 140142073335807, -SNULL, 140142056554495, 140142073335807, -STORE, 140142056550400, 140142056554495, -STORE, 140142056554496, 140142073335807, -STORE, 140142048157696, 140142056550399, -SNULL, 140142509531135, 140142517919743, -STORE, 140142509527040, 140142509531135, -STORE, 140142509531136, 140142517919743, -SNULL, 140142777966591, 140142786355199, -STORE, 140142777962496, 140142777966591, -STORE, 140142777966592, 140142786355199, -SNULL, 140143046402047, 140143054790655, -STORE, 140143046397952, 140143046402047, -STORE, 140143046402048, 140143054790655, -SNULL, 140142912184320, 140142920572927, -STORE, 140142920572928, 140142928965631, -STORE, 140142912184320, 140142920572927, -SNULL, 140142920577023, 140142928965631, -STORE, 140142920572928, 140142920577023, -STORE, 140142920577024, 140142928965631, -STORE, 140142039764992, 140142056550399, -STORE, 140141955903488, 140141964296191, -SNULL, 140142819930112, 140142828318719, -STORE, 140142828318720, 140142836711423, -STORE, 140142819930112, 140142828318719, -SNULL, 140142828322815, 140142836711423, -STORE, 140142828318720, 140142828322815, -STORE, 140142828322816, 140142836711423, -SNULL, 140142517923840, 140142526312447, -STORE, 140142526312448, 140142534705151, -STORE, 140142517923840, 140142526312447, -SNULL, 140142526316543, 140142534705151, -STORE, 140142526312448, 140142526316543, -STORE, 140142526316544, 140142534705151, -STORE, 140141947510784, 140141964296191, -SNULL, 140142056554496, 140142064943103, -STORE, 140142064943104, 140142073335807, -STORE, 140142056554496, 140142064943103, -SNULL, 140142064947199, 140142073335807, -STORE, 140142064943104, 140142064947199, -STORE, 140142064947200, 140142073335807, -SNULL, 140142073339904, 140142081728511, -STORE, 140142081728512, 140142090121215, -STORE, 140142073339904, 140142081728511, -SNULL, 140142081732607, 140142090121215, -STORE, 140142081728512, 140142081732607, -STORE, 140142081732608, 140142090121215, -STORE, 140141939118080, 140141964296191, -STORE, 140141930725376, 140141964296191, -STORE, 140141922332672, 140141964296191, -STORE, 140141913939968, 140141964296191, -SNULL, 140141913939968, 140141922332671, -STORE, 140141922332672, 140141964296191, -STORE, 140141913939968, 140141922332671, -SNULL, 140141922336767, 140141964296191, -STORE, 140141922332672, 140141922336767, -STORE, 140141922336768, 140141964296191, -STORE, 140141905547264, 140141922332671, -SNULL, 140141905551359, 140141922332671, -STORE, 140141905547264, 140141905551359, -STORE, 140141905551360, 140141922332671, -STORE, 140141821685760, 140141830078463, -STORE, 140141813293056, 140141830078463, -STORE, 140141804900352, 140141830078463, -STORE, 140141796507648, 140141830078463, -SNULL, 140141796511743, 140141830078463, -STORE, 140141796507648, 140141796511743, -STORE, 140141796511744, 140141830078463, -SNULL, 140141922336768, 140141955903487, -STORE, 140141955903488, 140141964296191, -STORE, 140141922336768, 140141955903487, -SNULL, 140141955907583, 140141964296191, -STORE, 140141955903488, 140141955907583, -STORE, 140141955907584, 140141964296191, -STORE, 140141788114944, 140141796507647, -STORE, 140141779722240, 140141796507647, -SNULL, 140141779722240, 140141788114943, -STORE, 140141788114944, 140141796507647, -STORE, 140141779722240, 140141788114943, -SNULL, 140141788119039, 140141796507647, -STORE, 140141788114944, 140141788119039, -STORE, 140141788119040, 140141796507647, -SNULL, 140141922336768, 140141947510783, -STORE, 140141947510784, 140141955903487, -STORE, 140141922336768, 140141947510783, -SNULL, 140141947514879, 140141955903487, -STORE, 140141947510784, 140141947514879, -STORE, 140141947514880, 140141955903487, -SNULL, 140142039764992, 140142048157695, -STORE, 140142048157696, 140142056550399, -STORE, 140142039764992, 140142048157695, -SNULL, 140142048161791, 140142056550399, -STORE, 140142048157696, 140142048161791, -STORE, 140142048161792, 140142056550399, -SNULL, 140142039769087, 140142048157695, -STORE, 140142039764992, 140142039769087, -STORE, 140142039769088, 140142048157695, -SNULL, 140141796511744, 140141804900351, -STORE, 140141804900352, 140141830078463, -STORE, 140141796511744, 140141804900351, -SNULL, 140141804904447, 140141830078463, -STORE, 140141804900352, 140141804904447, -STORE, 140141804904448, 140141830078463, -STORE, 140141771329536, 140141788114943, -STORE, 140141762936832, 140141788114943, -STORE, 140141754544128, 140141788114943, -SNULL, 140141804904448, 140141821685759, -STORE, 140141821685760, 140141830078463, -STORE, 140141804904448, 140141821685759, -SNULL, 140141821689855, 140141830078463, -STORE, 140141821685760, 140141821689855, -STORE, 140141821689856, 140141830078463, -SNULL, 140141922336768, 140141939118079, -STORE, 140141939118080, 140141947510783, -STORE, 140141922336768, 140141939118079, -SNULL, 140141939122175, 140141947510783, -STORE, 140141939118080, 140141939122175, -STORE, 140141939122176, 140141947510783, -SNULL, 140141905551360, 140141913939967, -STORE, 140141913939968, 140141922332671, -STORE, 140141905551360, 140141913939967, -SNULL, 140141913944063, 140141922332671, -STORE, 140141913939968, 140141913944063, -STORE, 140141913944064, 140141922332671, -STORE, 140141746151424, 140141788114943, -STORE, 140141737758720, 140141788114943, -SNULL, 140141804904448, 140141813293055, -STORE, 140141813293056, 140141821685759, -STORE, 140141804904448, 140141813293055, -SNULL, 140141813297151, 140141821685759, -STORE, 140141813293056, 140141813297151, -STORE, 140141813297152, 140141821685759, -STORE, 140141729366016, 140141788114943, -STORE, 140141720973312, 140141788114943, -STORE, 140141712580608, 140141788114943, -SNULL, 140141712584703, 140141788114943, -STORE, 140141712580608, 140141712584703, -STORE, 140141712584704, 140141788114943, -SNULL, 140141922336768, 140141930725375, -STORE, 140141930725376, 140141939118079, -STORE, 140141922336768, 140141930725375, -SNULL, 140141930729471, 140141939118079, -STORE, 140141930725376, 140141930729471, -STORE, 140141930729472, 140141939118079, -STORE, 140141704187904, 140141712580607, -SNULL, 140141704191999, 140141712580607, -STORE, 140141704187904, 140141704191999, -STORE, 140141704192000, 140141712580607, -STORE, 140141695795200, 140141704187903, -STORE, 140141687402496, 140141704187903, -SNULL, 140141712584704, 140141771329535, -STORE, 140141771329536, 140141788114943, -STORE, 140141712584704, 140141771329535, -SNULL, 140141771333631, 140141788114943, -STORE, 140141771329536, 140141771333631, -STORE, 140141771333632, 140141788114943, -SNULL, 140141771333632, 140141779722239, -STORE, 140141779722240, 140141788114943, -STORE, 140141771333632, 140141779722239, -SNULL, 140141779726335, 140141788114943, -STORE, 140141779722240, 140141779726335, -STORE, 140141779726336, 140141788114943, -STORE, 140141679009792, 140141704187903, -SNULL, 140141679013887, 140141704187903, -STORE, 140141679009792, 140141679013887, -STORE, 140141679013888, 140141704187903, -STORE, 140141670617088, 140141679009791, -SNULL, 140141670621183, 140141679009791, -STORE, 140141670617088, 140141670621183, -STORE, 140141670621184, 140141679009791, -STORE, 140141662224384, 140141670617087, -SNULL, 140141712584704, 140141737758719, -STORE, 140141737758720, 140141771329535, -STORE, 140141712584704, 140141737758719, -SNULL, 140141737762815, 140141771329535, -STORE, 140141737758720, 140141737762815, -STORE, 140141737762816, 140141771329535, -SNULL, 140141712584704, 140141729366015, -STORE, 140141729366016, 140141737758719, -STORE, 140141712584704, 140141729366015, -SNULL, 140141729370111, 140141737758719, -STORE, 140141729366016, 140141729370111, -STORE, 140141729370112, 140141737758719, -SNULL, 140141737762816, 140141746151423, -STORE, 140141746151424, 140141771329535, -STORE, 140141737762816, 140141746151423, -SNULL, 140141746155519, 140141771329535, -STORE, 140141746151424, 140141746155519, -STORE, 140141746155520, 140141771329535, -STORE, 140141653831680, 140141670617087, -SNULL, 140141746155520, 140141762936831, -STORE, 140141762936832, 140141771329535, -STORE, 140141746155520, 140141762936831, -SNULL, 140141762940927, 140141771329535, -STORE, 140141762936832, 140141762940927, -STORE, 140141762940928, 140141771329535, -STORE, 140141645438976, 140141670617087, -SNULL, 140141645443071, 140141670617087, -STORE, 140141645438976, 140141645443071, -STORE, 140141645443072, 140141670617087, -SNULL, 140141712584704, 140141720973311, -STORE, 140141720973312, 140141729366015, -STORE, 140141712584704, 140141720973311, -SNULL, 140141720977407, 140141729366015, -STORE, 140141720973312, 140141720977407, -STORE, 140141720977408, 140141729366015, -STORE, 140141637046272, 140141645438975, -SNULL, 140141637050367, 140141645438975, -STORE, 140141637046272, 140141637050367, -STORE, 140141637050368, 140141645438975, -STORE, 140141628653568, 140141637046271, -SNULL, 140141628657663, 140141637046271, -STORE, 140141628653568, 140141628657663, -STORE, 140141628657664, 140141637046271, -STORE, 140141620260864, 140141628653567, -SNULL, 140141679013888, 140141687402495, -STORE, 140141687402496, 140141704187903, -STORE, 140141679013888, 140141687402495, -SNULL, 140141687406591, 140141704187903, -STORE, 140141687402496, 140141687406591, -STORE, 140141687406592, 140141704187903, -SNULL, 140141746155520, 140141754544127, -STORE, 140141754544128, 140141762936831, -STORE, 140141746155520, 140141754544127, -SNULL, 140141754548223, 140141762936831, -STORE, 140141754544128, 140141754548223, -STORE, 140141754548224, 140141762936831, -SNULL, 140141687406592, 140141695795199, -STORE, 140141695795200, 140141704187903, -STORE, 140141687406592, 140141695795199, -SNULL, 140141695799295, 140141704187903, -STORE, 140141695795200, 140141695799295, -STORE, 140141695799296, 140141704187903, -STORE, 140141611868160, 140141628653567, -SNULL, 140141611872255, 140141628653567, -STORE, 140141611868160, 140141611872255, -STORE, 140141611872256, 140141628653567, -SNULL, 140141645443072, 140141662224383, -STORE, 140141662224384, 140141670617087, -STORE, 140141645443072, 140141662224383, -SNULL, 140141662228479, 140141670617087, -STORE, 140141662224384, 140141662228479, -STORE, 140141662228480, 140141670617087, -STORE, 140141603475456, 140141611868159, -SNULL, 140141603479551, 140141611868159, -STORE, 140141603475456, 140141603479551, -STORE, 140141603479552, 140141611868159, -STORE, 140141595082752, 140141603475455, -SNULL, 140141645443072, 140141653831679, -STORE, 140141653831680, 140141662224383, -STORE, 140141645443072, 140141653831679, -SNULL, 140141653835775, 140141662224383, -STORE, 140141653831680, 140141653835775, -STORE, 140141653835776, 140141662224383, -STORE, 140141586690048, 140141603475455, -SNULL, 140141611872256, 140141620260863, -STORE, 140141620260864, 140141628653567, -STORE, 140141611872256, 140141620260863, -SNULL, 140141620264959, 140141628653567, -STORE, 140141620260864, 140141620264959, -STORE, 140141620264960, 140141628653567, -SNULL, 140141586690048, 140141595082751, -STORE, 140141595082752, 140141603475455, -STORE, 140141586690048, 140141595082751, -SNULL, 140141595086847, 140141603475455, -STORE, 140141595082752, 140141595086847, -STORE, 140141595086848, 140141603475455, -STORE, 140141578297344, 140141595082751, -SNULL, 140141578301439, 140141595082751, -STORE, 140141578297344, 140141578301439, -STORE, 140141578301440, 140141595082751, -SNULL, 140141578301440, 140141586690047, -STORE, 140141586690048, 140141595082751, -STORE, 140141578301440, 140141586690047, -SNULL, 140141586694143, 140141595082751, -STORE, 140141586690048, 140141586694143, -STORE, 140141586694144, 140141595082751, -STORE, 140143370027008, 140143370055679, -STORE, 140143309254656, 140143311446015, -SNULL, 140143309254656, 140143309344767, -STORE, 140143309344768, 140143311446015, -STORE, 140143309254656, 140143309344767, -SNULL, 140143311437823, 140143311446015, -STORE, 140143309344768, 140143311437823, -STORE, 140143311437824, 140143311446015, -ERASE, 140143311437824, 140143311446015, -STORE, 140143311437824, 140143311446015, -SNULL, 140143311441919, 140143311446015, -STORE, 140143311437824, 140143311441919, -STORE, 140143311441920, 140143311446015, -ERASE, 140143370027008, 140143370055679, -ERASE, 140142912180224, 140142912184319, -ERASE, 140142912184320, 140142920572927, -ERASE, 140142945751040, 140142945755135, -ERASE, 140142945755136, 140142954143743, -ERASE, 140142090121216, 140142090125311, -ERASE, 140142090125312, 140142098513919, -ERASE, 140142794747904, 140142794751999, -ERASE, 140142794752000, 140142803140607, -ERASE, 140141913939968, 140141913944063, -ERASE, 140141913944064, 140141922332671, -ERASE, 140141746151424, 140141746155519, -ERASE, 140141746155520, 140141754544127, -ERASE, 140142954143744, 140142954147839, -ERASE, 140142954147840, 140142962536447, -ERASE, 140142081728512, 140142081732607, -ERASE, 140142081732608, 140142090121215, -ERASE, 140141905547264, 140141905551359, -ERASE, 140141905551360, 140141913939967, -ERASE, 140141729366016, 140141729370111, -ERASE, 140141729370112, 140141737758719, -ERASE, 140142920572928, 140142920577023, -ERASE, 140142920577024, 140142928965631, -ERASE, 140142039764992, 140142039769087, -ERASE, 140142039769088, 140142048157695, -ERASE, 140141679009792, 140141679013887, -ERASE, 140141679013888, 140141687402495, -ERASE, 140142551490560, 140142551494655, -ERASE, 140142551494656, 140142559883263, -ERASE, 140141947510784, 140141947514879, -ERASE, 140141947514880, 140141955903487, -ERASE, 140141771329536, 140141771333631, -ERASE, 140141771333632, 140141779722239, -ERASE, 140142928965632, 140142928969727, -ERASE, 140142928969728, 140142937358335, -ERASE, 140142073335808, 140142073339903, -ERASE, 140142073339904, 140142081728511, -ERASE, 140142543097856, 140142543101951, -ERASE, 140142543101952, 140142551490559, -ERASE, 140141955903488, 140141955907583, -ERASE, 140141955907584, 140141964296191, -ERASE, 140141704187904, 140141704191999, -ERASE, 140141704192000, 140141712580607, -ERASE, 140142786355200, 140142786359295, -ERASE, 140142786359296, 140142794747903, -ERASE, 140142056550400, 140142056554495, -ERASE, 140142056554496, 140142064943103, -ERASE, 140142828318720, 140142828322815, -ERASE, 140142828322816, 140142836711423, -ERASE, 140141788114944, 140141788119039, -ERASE, 140141788119040, 140141796507647, -ERASE, 140141695795200, 140141695799295, -ERASE, 140141695799296, 140141704187903, -ERASE, 140141578297344, 140141578301439, -ERASE, 140141578301440, 140141586690047, -ERASE, 140141611868160, 140141611872255, -ERASE, 140141611872256, 140141620260863, -ERASE, 140142811533312, 140142811537407, -ERASE, 140142811537408, 140142819926015, -ERASE, 140142064943104, 140142064947199, -ERASE, 140142064947200, 140142073335807, -ERASE, 140141628653568, 140141628657663, -ERASE, 140141628657664, 140141637046271, -ERASE, 140143046397952, 140143046402047, -ERASE, 140143046402048, 140143054790655, -ERASE, 140141796507648, 140141796511743, -ERASE, 140141796511744, 140141804900351, -ERASE, 140142803140608, 140142803144703, -ERASE, 140142803144704, 140142811533311, -ERASE, 140142509527040, 140142509531135, -ERASE, 140142509531136, 140142517919743, -ERASE, 140141821685760, 140141821689855, -ERASE, 140141821689856, 140141830078463, -ERASE, 140142777962496, 140142777966591, -ERASE, 140142777966592, 140142786355199, -ERASE, 140141804900352, 140141804904447, -ERASE, 140141804904448, 140141813293055, -ERASE, 140141930725376, 140141930729471, -ERASE, 140141930729472, 140141939118079, -ERASE, 140142937358336, 140142937362431, -ERASE, 140142937362432, 140142945751039, -ERASE, 140142559883264, 140142559887359, -ERASE, 140142559887360, 140142568275967, -ERASE, 140142534705152, 140142534709247, -ERASE, 140142534709248, 140142543097855, -ERASE, 140142048157696, 140142048161791, -ERASE, 140142048161792, 140142056550399, -ERASE, 140141754544128, 140141754548223, -ERASE, 140141754548224, 140141762936831, -ERASE, 140141939118080, 140141939122175, -ERASE, 140141939122176, 140141947510783, -ERASE, 140141653831680, 140141653835775, -ERASE, 140141653835776, 140141662224383, -ERASE, 140141712580608, 140141712584703, -ERASE, 140141712584704, 140141720973311, -ERASE, 140141645438976, 140141645443071, -ERASE, 140141645443072, 140141653831679, -ERASE, 140141687402496, 140141687406591, -ERASE, 140141687406592, 140141695795199, -ERASE, 140141662224384, 140141662228479, -ERASE, 140141662228480, 140141670617087, -ERASE, 140141922332672, 140141922336767, -ERASE, 140141922336768, 140141930725375, -ERASE, 140141737758720, 140141737762815, -ERASE, 140141737762816, 140141746151423, -ERASE, 140141637046272, 140141637050367, -ERASE, 140141637050368, 140141645438975, -ERASE, 140142517919744, 140142517923839, -ERASE, 140142517923840, 140142526312447, -ERASE, 140143096754176, 140143096758271, -ERASE, 140143096758272, 140143105146879, -ERASE, 140141595082752, 140141595086847, -ERASE, 140141595086848, 140141603475455, -ERASE, 140141762936832, 140141762940927, -ERASE, 140141762940928, 140141771329535, -ERASE, 140143311446016, 140143311450111, -ERASE, 140143311450112, 140143319838719, -ERASE, 140142526312448, 140142526316543, -ERASE, 140142526316544, 140142534705151, -ERASE, 140142819926016, 140142819930111, -ERASE, 140142819930112, 140142828318719, -ERASE, 140143180615680, 140143180619775, -ERASE, 140143180619776, 140143189008383, -ERASE, 140142962536448, 140142962540543, -ERASE, 140142962540544, 140142970929151, -ERASE, 140143214186496, 140143214190591, -ERASE, 140143214190592, 140143222579199, -ERASE, 140143088361472, 140143088365567, -ERASE, 140143088365568, 140143096754175, -ERASE, 140141586690048, 140141586694143, -ERASE, 140141586694144, 140141595082751, -ERASE, 140143230971904, 140143230975999, -ERASE, 140143230976000, 140143239364607, -ERASE, 140141779722240, 140141779726335, -ERASE, 140141779726336, 140141788114943, -ERASE, 140141670617088, 140141670621183, -ERASE, 140141670621184, 140141679009791, -ERASE, 140141813293056, 140141813297151, -ERASE, 140141813297152, 140141821685759, -ERASE, 140143222579200, 140143222583295, -ERASE, 140143222583296, 140143230971903, -ERASE, 140143189008384, 140143189012479, -ERASE, 140143189012480, 140143197401087, -ERASE, 140143071576064, 140143071580159, -ERASE, 140143071580160, 140143079968767, -ERASE, 140141620260864, 140141620264959, -ERASE, 140141620264960, 140141628653567, -ERASE, 140141603475456, 140141603479551, -ERASE, 140141603479552, 140141611868159, -ERASE, 140141720973312, 140141720977407, -ERASE, 140141720977408, 140141729366015, -ERASE, 140143079968768, 140143079972863, -ERASE, 140143079972864, 140143088361471, -ERASE, 140143205793792, 140143205797887, -ERASE, 140143205797888, 140143214186495, - }; - unsigned long set30[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140733436743680, 140737488351231, -SNULL, 140733436747775, 140737488351231, -STORE, 140733436743680, 140733436747775, -STORE, 140733436612608, 140733436747775, -STORE, 94630728904704, 94630731157503, -SNULL, 94630729035775, 94630731157503, -STORE, 94630728904704, 94630729035775, -STORE, 94630729035776, 94630731157503, -ERASE, 94630729035776, 94630731157503, -STORE, 94630731128832, 94630731137023, -STORE, 94630731137024, 94630731157503, -STORE, 140165750841344, 140165753094143, -SNULL, 140165750984703, 140165753094143, -STORE, 140165750841344, 140165750984703, -STORE, 140165750984704, 140165753094143, -ERASE, 140165750984704, 140165753094143, -STORE, 140165753081856, 140165753090047, -STORE, 140165753090048, 140165753094143, -STORE, 140733436887040, 140733436891135, -STORE, 140733436874752, 140733436887039, -STORE, 140165753053184, 140165753081855, -STORE, 140165753044992, 140165753053183, -STORE, 140165748625408, 140165750841343, -SNULL, 140165748625408, 140165748723711, -STORE, 140165748723712, 140165750841343, -STORE, 140165748625408, 140165748723711, -SNULL, 140165750816767, 140165750841343, -STORE, 140165748723712, 140165750816767, -STORE, 140165750816768, 140165750841343, -SNULL, 140165750816768, 140165750824959, -STORE, 140165750824960, 140165750841343, -STORE, 140165750816768, 140165750824959, -ERASE, 140165750816768, 140165750824959, -STORE, 140165750816768, 140165750824959, -ERASE, 140165750824960, 140165750841343, -STORE, 140165750824960, 140165750841343, -STORE, 140165744828416, 140165748625407, -SNULL, 140165744828416, 140165746487295, -STORE, 140165746487296, 140165748625407, -STORE, 140165744828416, 140165746487295, -SNULL, 140165748584447, 140165748625407, -STORE, 140165746487296, 140165748584447, -STORE, 140165748584448, 140165748625407, -SNULL, 140165748584448, 140165748609023, -STORE, 140165748609024, 140165748625407, -STORE, 140165748584448, 140165748609023, -ERASE, 140165748584448, 140165748609023, -STORE, 140165748584448, 140165748609023, -ERASE, 140165748609024, 140165748625407, -STORE, 140165748609024, 140165748625407, -STORE, 140165753036800, 140165753053183, -SNULL, 140165748600831, 140165748609023, -STORE, 140165748584448, 140165748600831, -STORE, 140165748600832, 140165748609023, -SNULL, 140165750820863, 140165750824959, -STORE, 140165750816768, 140165750820863, -STORE, 140165750820864, 140165750824959, -SNULL, 94630731132927, 94630731137023, -STORE, 94630731128832, 94630731132927, -STORE, 94630731132928, 94630731137023, -SNULL, 140165753085951, 140165753090047, -STORE, 140165753081856, 140165753085951, -STORE, 140165753085952, 140165753090047, -ERASE, 140165753053184, 140165753081855, -STORE, 94630743547904, 94630743683071, -STORE, 140165736435712, 140165744828415, -SNULL, 140165736439807, 140165744828415, -STORE, 140165736435712, 140165736439807, -STORE, 140165736439808, 140165744828415, -STORE, 140165728043008, 140165736435711, -STORE, 140165593825280, 140165728043007, -SNULL, 140165593825280, 140165653725183, -STORE, 140165653725184, 140165728043007, -STORE, 140165593825280, 140165653725183, -ERASE, 140165593825280, 140165653725183, -SNULL, 140165720834047, 140165728043007, -STORE, 140165653725184, 140165720834047, -STORE, 140165720834048, 140165728043007, -ERASE, 140165720834048, 140165728043007, -SNULL, 140165653860351, 140165720834047, -STORE, 140165653725184, 140165653860351, -STORE, 140165653860352, 140165720834047, -SNULL, 140165728047103, 140165736435711, -STORE, 140165728043008, 140165728047103, -STORE, 140165728047104, 140165736435711, -STORE, 140165645332480, 140165653725183, -SNULL, 140165645336575, 140165653725183, -STORE, 140165645332480, 140165645336575, -STORE, 140165645336576, 140165653725183, -STORE, 140165636939776, 140165645332479, -SNULL, 140165636943871, 140165645332479, -STORE, 140165636939776, 140165636943871, -STORE, 140165636943872, 140165645332479, -STORE, 140165628547072, 140165636939775, -SNULL, 140165628551167, 140165636939775, -STORE, 140165628547072, 140165628551167, -STORE, 140165628551168, 140165636939775, -STORE, 140165620154368, 140165628547071, -STORE, 140165611761664, 140165628547071, -STORE, 140165603368960, 140165628547071, -STORE, 140165469151232, 140165603368959, -SNULL, 140165469151232, 140165519507455, -STORE, 140165519507456, 140165603368959, -STORE, 140165469151232, 140165519507455, -ERASE, 140165469151232, 140165519507455, -SNULL, 140165586616319, 140165603368959, -STORE, 140165519507456, 140165586616319, -STORE, 140165586616320, 140165603368959, -ERASE, 140165586616320, 140165603368959, -STORE, 140165594976256, 140165628547071, -STORE, 140165385289728, 140165586616319, -SNULL, 140165452398591, 140165586616319, -STORE, 140165385289728, 140165452398591, -STORE, 140165452398592, 140165586616319, -SNULL, 140165452398592, 140165519507455, -STORE, 140165519507456, 140165586616319, -STORE, 140165452398592, 140165519507455, -ERASE, 140165452398592, 140165519507455, -STORE, 140165251072000, 140165452398591, -SNULL, 140165318180863, 140165452398591, -STORE, 140165251072000, 140165318180863, -STORE, 140165318180864, 140165452398591, -SNULL, 140165318180864, 140165385289727, -STORE, 140165385289728, 140165452398591, -STORE, 140165318180864, 140165385289727, -ERASE, 140165318180864, 140165385289727, -SNULL, 140165519642623, 140165586616319, -STORE, 140165519507456, 140165519642623, -STORE, 140165519642624, 140165586616319, -SNULL, 140165594976256, 140165611761663, -STORE, 140165611761664, 140165628547071, -STORE, 140165594976256, 140165611761663, -SNULL, 140165611765759, 140165628547071, -STORE, 140165611761664, 140165611765759, -STORE, 140165611765760, 140165628547071, -STORE, 140165385289728, 140165519507455, -SNULL, 140165385424895, 140165519507455, -STORE, 140165385289728, 140165385424895, -STORE, 140165385424896, 140165519507455, -SNULL, 140165594976256, 140165603368959, -STORE, 140165603368960, 140165611761663, -STORE, 140165594976256, 140165603368959, -SNULL, 140165603373055, 140165611761663, -STORE, 140165603368960, 140165603373055, -STORE, 140165603373056, 140165611761663, -SNULL, 140165251207167, 140165318180863, -STORE, 140165251072000, 140165251207167, -STORE, 140165251207168, 140165318180863, -STORE, 140165376897024, 140165385289727, -SNULL, 140165376901119, 140165385289727, -STORE, 140165376897024, 140165376901119, -STORE, 140165376901120, 140165385289727, -SNULL, 140165385424896, 140165452398591, -STORE, 140165452398592, 140165519507455, -STORE, 140165385424896, 140165452398591, -SNULL, 140165452533759, 140165519507455, -STORE, 140165452398592, 140165452533759, -STORE, 140165452533760, 140165519507455, -STORE, 140165368504320, 140165376897023, -SNULL, 140165594980351, 140165603368959, -STORE, 140165594976256, 140165594980351, -STORE, 140165594980352, 140165603368959, -SNULL, 140165368508415, 140165376897023, -STORE, 140165368504320, 140165368508415, -STORE, 140165368508416, 140165376897023, -SNULL, 140165611765760, 140165620154367, -STORE, 140165620154368, 140165628547071, -STORE, 140165611765760, 140165620154367, -SNULL, 140165620158463, 140165628547071, -STORE, 140165620154368, 140165620158463, -STORE, 140165620158464, 140165628547071, -STORE, 140165360111616, 140165368504319, -STORE, 140165351718912, 140165368504319, -STORE, 140165343326208, 140165368504319, -SNULL, 140165343326208, 140165351718911, -STORE, 140165351718912, 140165368504319, -STORE, 140165343326208, 140165351718911, -SNULL, 140165351723007, 140165368504319, -STORE, 140165351718912, 140165351723007, -STORE, 140165351723008, 140165368504319, -SNULL, 140165343330303, 140165351718911, -STORE, 140165343326208, 140165343330303, -STORE, 140165343330304, 140165351718911, -SNULL, 140165351723008, 140165360111615, -STORE, 140165360111616, 140165368504319, -STORE, 140165351723008, 140165360111615, -SNULL, 140165360115711, 140165368504319, -STORE, 140165360111616, 140165360115711, -STORE, 140165360115712, 140165368504319, -STORE, 140165334933504, 140165343326207, -SNULL, 140165334937599, 140165343326207, -STORE, 140165334933504, 140165334937599, -STORE, 140165334937600, 140165343326207, -STORE, 140165326540800, 140165334933503, -STORE, 140165242679296, 140165251071999, -SNULL, 140165242683391, 140165251071999, -STORE, 140165242679296, 140165242683391, -STORE, 140165242683392, 140165251071999, -STORE, 140165234286592, 140165242679295, -STORE, 140165225893888, 140165242679295, -SNULL, 140165225897983, 140165242679295, -STORE, 140165225893888, 140165225897983, -STORE, 140165225897984, 140165242679295, -SNULL, 140165225897984, 140165234286591, -STORE, 140165234286592, 140165242679295, -STORE, 140165225897984, 140165234286591, -SNULL, 140165234290687, 140165242679295, -STORE, 140165234286592, 140165234290687, -STORE, 140165234290688, 140165242679295, -SNULL, 140165326544895, 140165334933503, -STORE, 140165326540800, 140165326544895, -STORE, 140165326544896, 140165334933503, -STORE, 140165217501184, 140165225893887, -STORE, 140165209108480, 140165225893887, -SNULL, 140165209108480, 140165217501183, -STORE, 140165217501184, 140165225893887, -STORE, 140165209108480, 140165217501183, -SNULL, 140165217505279, 140165225893887, -STORE, 140165217501184, 140165217505279, -STORE, 140165217505280, 140165225893887, -SNULL, 140165209112575, 140165217501183, -STORE, 140165209108480, 140165209112575, -STORE, 140165209112576, 140165217501183, -STORE, 140165200715776, 140165209108479, -STORE, 140165066498048, 140165200715775, -SNULL, 140165066498048, 140165116854271, -STORE, 140165116854272, 140165200715775, -STORE, 140165066498048, 140165116854271, -ERASE, 140165066498048, 140165116854271, -SNULL, 140165183963135, 140165200715775, -STORE, 140165116854272, 140165183963135, -STORE, 140165183963136, 140165200715775, -ERASE, 140165183963136, 140165200715775, -SNULL, 140165116989439, 140165183963135, -STORE, 140165116854272, 140165116989439, -STORE, 140165116989440, 140165183963135, -STORE, 140165192323072, 140165209108479, -STORE, 140165108461568, 140165116854271, -STORE, 140164974243840, 140165108461567, -STORE, 140164965851136, 140164974243839, -SNULL, 140164974243840, 140164982636543, -STORE, 140164982636544, 140165108461567, -STORE, 140164974243840, 140164982636543, -ERASE, 140164974243840, 140164982636543, -STORE, 140164965851136, 140164982636543, -STORE, 140164957458432, 140164982636543, -STORE, 140164949065728, 140164982636543, -STORE, 140164940673024, 140164982636543, -STORE, 140164806455296, 140164940673023, -STORE, 140164798062592, 140164806455295, -STORE, 140164789669888, 140164806455295, -STORE, 140164655452160, 140164789669887, -STORE, 140164647059456, 140164655452159, -STORE, 140164638666752, 140164655452159, -SNULL, 140164655452160, 140164714201087, -STORE, 140164714201088, 140164789669887, -STORE, 140164655452160, 140164714201087, -ERASE, 140164655452160, 140164714201087, -STORE, 140164705808384, 140164714201087, -STORE, 140164697415680, 140164714201087, -STORE, 140164504449024, 140164638666751, -SNULL, 140164504449024, 140164512874495, -STORE, 140164512874496, 140164638666751, -STORE, 140164504449024, 140164512874495, -ERASE, 140164504449024, 140164512874495, -STORE, 140164689022976, 140164714201087, -STORE, 140164680630272, 140164714201087, -SNULL, 140164680634367, 140164714201087, -STORE, 140164680630272, 140164680634367, -STORE, 140164680634368, 140164714201087, -STORE, 140164378656768, 140164638666751, -SNULL, 140165192323072, 140165200715775, -STORE, 140165200715776, 140165209108479, -STORE, 140165192323072, 140165200715775, -SNULL, 140165200719871, 140165209108479, -STORE, 140165200715776, 140165200719871, -STORE, 140165200719872, 140165209108479, -SNULL, 140165049745407, 140165108461567, -STORE, 140164982636544, 140165049745407, -STORE, 140165049745408, 140165108461567, -ERASE, 140165049745408, 140165108461567, -SNULL, 140164982771711, 140165049745407, -STORE, 140164982636544, 140164982771711, -STORE, 140164982771712, 140165049745407, -STORE, 140164244439040, 140164638666751, -SNULL, 140164311547903, 140164638666751, -STORE, 140164244439040, 140164311547903, -STORE, 140164311547904, 140164638666751, -SNULL, 140164311547904, 140164378656767, -STORE, 140164378656768, 140164638666751, -STORE, 140164311547904, 140164378656767, -ERASE, 140164311547904, 140164378656767, -SNULL, 140164806455296, 140164848418815, -STORE, 140164848418816, 140164940673023, -STORE, 140164806455296, 140164848418815, -ERASE, 140164806455296, 140164848418815, -SNULL, 140164915527679, 140164940673023, -STORE, 140164848418816, 140164915527679, -STORE, 140164915527680, 140164940673023, -ERASE, 140164915527680, 140164940673023, -STORE, 140164110221312, 140164311547903, -SNULL, 140164177330175, 140164311547903, -STORE, 140164110221312, 140164177330175, -STORE, 140164177330176, 140164311547903, -SNULL, 140164177330176, 140164244439039, -STORE, 140164244439040, 140164311547903, -STORE, 140164177330176, 140164244439039, -ERASE, 140164177330176, 140164244439039, -SNULL, 140164781309951, 140164789669887, -STORE, 140164714201088, 140164781309951, -STORE, 140164781309952, 140164789669887, -ERASE, 140164781309952, 140164789669887, -STORE, 140163976003584, 140164177330175, -SNULL, 140164043112447, 140164177330175, -STORE, 140163976003584, 140164043112447, -STORE, 140164043112448, 140164177330175, -SNULL, 140164043112448, 140164110221311, -STORE, 140164110221312, 140164177330175, -STORE, 140164043112448, 140164110221311, -ERASE, 140164043112448, 140164110221311, -SNULL, 140164579983359, 140164638666751, -STORE, 140164378656768, 140164579983359, -STORE, 140164579983360, 140164638666751, -ERASE, 140164579983360, 140164638666751, -STORE, 140163841785856, 140164043112447, -SNULL, 140163908894719, 140164043112447, -STORE, 140163841785856, 140163908894719, -STORE, 140163908894720, 140164043112447, -SNULL, 140163908894720, 140163976003583, -STORE, 140163976003584, 140164043112447, -STORE, 140163908894720, 140163976003583, -ERASE, 140163908894720, 140163976003583, -SNULL, 140164940673024, 140164965851135, -STORE, 140164965851136, 140164982636543, -STORE, 140164940673024, 140164965851135, -SNULL, 140164965855231, 140164982636543, -STORE, 140164965851136, 140164965855231, -STORE, 140164965855232, 140164982636543, -SNULL, 140164965855232, 140164974243839, -STORE, 140164974243840, 140164982636543, -STORE, 140164965855232, 140164974243839, -SNULL, 140164974247935, 140164982636543, -STORE, 140164974243840, 140164974247935, -STORE, 140164974247936, 140164982636543, -SNULL, 140164445765631, 140164579983359, -STORE, 140164378656768, 140164445765631, -STORE, 140164445765632, 140164579983359, -SNULL, 140164445765632, 140164512874495, -STORE, 140164512874496, 140164579983359, -STORE, 140164445765632, 140164512874495, -ERASE, 140164445765632, 140164512874495, -SNULL, 140164378791935, 140164445765631, -STORE, 140164378656768, 140164378791935, -STORE, 140164378791936, 140164445765631, -SNULL, 140164789673983, 140164806455295, -STORE, 140164789669888, 140164789673983, -STORE, 140164789673984, 140164806455295, -SNULL, 140164789673984, 140164798062591, -STORE, 140164798062592, 140164806455295, -STORE, 140164789673984, 140164798062591, -SNULL, 140164798066687, 140164806455295, -STORE, 140164798062592, 140164798066687, -STORE, 140164798066688, 140164806455295, -SNULL, 140164638670847, 140164655452159, -STORE, 140164638666752, 140164638670847, -STORE, 140164638670848, 140164655452159, -STORE, 140165100068864, 140165116854271, -STORE, 140165091676160, 140165116854271, -STORE, 140165083283456, 140165116854271, -SNULL, 140164244574207, 140164311547903, -STORE, 140164244439040, 140164244574207, -STORE, 140164244574208, 140164311547903, -SNULL, 140164848553983, 140164915527679, -STORE, 140164848418816, 140164848553983, -STORE, 140164848553984, 140164915527679, -SNULL, 140164110356479, 140164177330175, -STORE, 140164110221312, 140164110356479, -STORE, 140164110356480, 140164177330175, -SNULL, 140164714336255, 140164781309951, -STORE, 140164714201088, 140164714336255, -STORE, 140164714336256, 140164781309951, -SNULL, 140163976138751, 140164043112447, -STORE, 140163976003584, 140163976138751, -STORE, 140163976138752, 140164043112447, -SNULL, 140164513009663, 140164579983359, -STORE, 140164512874496, 140164513009663, -STORE, 140164513009664, 140164579983359, -SNULL, 140163841921023, 140163908894719, -STORE, 140163841785856, 140163841921023, -STORE, 140163841921024, 140163908894719, -SNULL, 140165083283456, 140165100068863, -STORE, 140165100068864, 140165116854271, -STORE, 140165083283456, 140165100068863, -SNULL, 140165100072959, 140165116854271, -STORE, 140165100068864, 140165100072959, -STORE, 140165100072960, 140165116854271, -SNULL, 140165100072960, 140165108461567, -STORE, 140165108461568, 140165116854271, -STORE, 140165100072960, 140165108461567, -SNULL, 140165108465663, 140165116854271, -STORE, 140165108461568, 140165108465663, -STORE, 140165108465664, 140165116854271, -STORE, 140165074890752, 140165100068863, -SNULL, 140165074894847, 140165100068863, -STORE, 140165074890752, 140165074894847, -STORE, 140165074894848, 140165100068863, -STORE, 140165066498048, 140165074890751, -STORE, 140165058105344, 140165074890751, -STORE, 140164932280320, 140164965851135, -SNULL, 140165192327167, 140165200715775, -STORE, 140165192323072, 140165192327167, -STORE, 140165192327168, 140165200715775, -STORE, 140164923887616, 140164965851135, -SNULL, 140164923891711, 140164965851135, -STORE, 140164923887616, 140164923891711, -STORE, 140164923891712, 140164965851135, -SNULL, 140164680634368, 140164705808383, -STORE, 140164705808384, 140164714201087, -STORE, 140164680634368, 140164705808383, -SNULL, 140164705812479, 140164714201087, -STORE, 140164705808384, 140164705812479, -STORE, 140164705812480, 140164714201087, -SNULL, 140164680634368, 140164697415679, -STORE, 140164697415680, 140164705808383, -STORE, 140164680634368, 140164697415679, -SNULL, 140164697419775, 140164705808383, -STORE, 140164697415680, 140164697419775, -STORE, 140164697419776, 140164705808383, -STORE, 140164840026112, 140164848418815, -STORE, 140164831633408, 140164848418815, -STORE, 140164823240704, 140164848418815, -SNULL, 140165074894848, 140165083283455, -STORE, 140165083283456, 140165100068863, -STORE, 140165074894848, 140165083283455, -SNULL, 140165083287551, 140165100068863, -STORE, 140165083283456, 140165083287551, -STORE, 140165083287552, 140165100068863, -SNULL, 140165083287552, 140165091676159, -STORE, 140165091676160, 140165100068863, -STORE, 140165083287552, 140165091676159, -SNULL, 140165091680255, 140165100068863, -STORE, 140165091676160, 140165091680255, -STORE, 140165091680256, 140165100068863, -SNULL, 140164638670848, 140164647059455, -STORE, 140164647059456, 140164655452159, -STORE, 140164638670848, 140164647059455, -SNULL, 140164647063551, 140164655452159, -STORE, 140164647059456, 140164647063551, -STORE, 140164647063552, 140164655452159, -SNULL, 140164923891712, 140164940673023, -STORE, 140164940673024, 140164965851135, -STORE, 140164923891712, 140164940673023, -SNULL, 140164940677119, 140164965851135, -STORE, 140164940673024, 140164940677119, -STORE, 140164940677120, 140164965851135, -SNULL, 140164940677120, 140164949065727, -STORE, 140164949065728, 140164965851135, -STORE, 140164940677120, 140164949065727, -SNULL, 140164949069823, 140164965851135, -STORE, 140164949065728, 140164949069823, -STORE, 140164949069824, 140164965851135, -SNULL, 140164949069824, 140164957458431, -STORE, 140164957458432, 140164965851135, -STORE, 140164949069824, 140164957458431, -SNULL, 140164957462527, 140164965851135, -STORE, 140164957458432, 140164957462527, -STORE, 140164957462528, 140164965851135, -SNULL, 140164680634368, 140164689022975, -STORE, 140164689022976, 140164697415679, -STORE, 140164680634368, 140164689022975, -SNULL, 140164689027071, 140164697415679, -STORE, 140164689022976, 140164689027071, -STORE, 140164689027072, 140164697415679, -STORE, 140164814848000, 140164848418815, -SNULL, 140165058105344, 140165066498047, -STORE, 140165066498048, 140165074890751, -STORE, 140165058105344, 140165066498047, -SNULL, 140165066502143, 140165074890751, -STORE, 140165066498048, 140165066502143, -STORE, 140165066502144, 140165074890751, -SNULL, 140165058109439, 140165066498047, -STORE, 140165058105344, 140165058109439, -STORE, 140165058109440, 140165066498047, -STORE, 140164798066688, 140164814847999, -SNULL, 140164798066688, 140164806455295, -STORE, 140164806455296, 140164814847999, -STORE, 140164798066688, 140164806455295, -SNULL, 140164806459391, 140164814847999, -STORE, 140164806455296, 140164806459391, -STORE, 140164806459392, 140164814847999, -SNULL, 140164923891712, 140164932280319, -STORE, 140164932280320, 140164940673023, -STORE, 140164923891712, 140164932280319, -SNULL, 140164932284415, 140164940673023, -STORE, 140164932280320, 140164932284415, -STORE, 140164932284416, 140164940673023, -STORE, 140164672237568, 140164680630271, -STORE, 140164663844864, 140164680630271, -STORE, 140164647063552, 140164680630271, -SNULL, 140164647063552, 140164655452159, -STORE, 140164655452160, 140164680630271, -STORE, 140164647063552, 140164655452159, -SNULL, 140164655456255, 140164680630271, -STORE, 140164655452160, 140164655456255, -STORE, 140164655456256, 140164680630271, -STORE, 140164630274048, 140164638666751, -SNULL, 140164814852095, 140164848418815, -STORE, 140164814848000, 140164814852095, -STORE, 140164814852096, 140164848418815, -SNULL, 140164814852096, 140164831633407, -STORE, 140164831633408, 140164848418815, -STORE, 140164814852096, 140164831633407, -SNULL, 140164831637503, 140164848418815, -STORE, 140164831633408, 140164831637503, -STORE, 140164831637504, 140164848418815, -STORE, 140164621881344, 140164638666751, -SNULL, 140164831637504, 140164840026111, -STORE, 140164840026112, 140164848418815, -STORE, 140164831637504, 140164840026111, -SNULL, 140164840030207, 140164848418815, -STORE, 140164840026112, 140164840030207, -STORE, 140164840030208, 140164848418815, -STORE, 140164613488640, 140164638666751, -SNULL, 140164613492735, 140164638666751, -STORE, 140164613488640, 140164613492735, -STORE, 140164613492736, 140164638666751, -STORE, 140164605095936, 140164613488639, -SNULL, 140164605100031, 140164613488639, -STORE, 140164605095936, 140164605100031, -STORE, 140164605100032, 140164613488639, -STORE, 140164596703232, 140164605095935, -STORE, 140164588310528, 140164605095935, -SNULL, 140164588314623, 140164605095935, -STORE, 140164588310528, 140164588314623, -STORE, 140164588314624, 140164605095935, -STORE, 140164504481792, 140164512874495, -STORE, 140164496089088, 140164512874495, -SNULL, 140164496089088, 140164504481791, -STORE, 140164504481792, 140164512874495, -STORE, 140164496089088, 140164504481791, -SNULL, 140164504485887, 140164512874495, -STORE, 140164504481792, 140164504485887, -STORE, 140164504485888, 140164512874495, -SNULL, 140164613492736, 140164630274047, -STORE, 140164630274048, 140164638666751, -STORE, 140164613492736, 140164630274047, -SNULL, 140164630278143, 140164638666751, -STORE, 140164630274048, 140164630278143, -STORE, 140164630278144, 140164638666751, -STORE, 140164487696384, 140164504481791, -STORE, 140164479303680, 140164504481791, -SNULL, 140164814852096, 140164823240703, -STORE, 140164823240704, 140164831633407, -STORE, 140164814852096, 140164823240703, -SNULL, 140164823244799, 140164831633407, -STORE, 140164823240704, 140164823244799, -STORE, 140164823244800, 140164831633407, -STORE, 140164470910976, 140164504481791, -SNULL, 140164470910976, 140164496089087, -STORE, 140164496089088, 140164504481791, -STORE, 140164470910976, 140164496089087, -SNULL, 140164496093183, 140164504481791, -STORE, 140164496089088, 140164496093183, -STORE, 140164496093184, 140164504481791, -SNULL, 140164655456256, 140164672237567, -STORE, 140164672237568, 140164680630271, -STORE, 140164655456256, 140164672237567, -SNULL, 140164672241663, 140164680630271, -STORE, 140164672237568, 140164672241663, -STORE, 140164672241664, 140164680630271, -STORE, 140164462518272, 140164496089087, -STORE, 140164454125568, 140164496089087, -SNULL, 140164655456256, 140164663844863, -STORE, 140164663844864, 140164672237567, -STORE, 140164655456256, 140164663844863, -SNULL, 140164663848959, 140164672237567, -STORE, 140164663844864, 140164663848959, -STORE, 140164663848960, 140164672237567, -STORE, 140164370264064, 140164378656767, -STORE, 140164361871360, 140164378656767, -STORE, 140164353478656, 140164378656767, -STORE, 140164345085952, 140164378656767, -SNULL, 140164345085952, 140164353478655, -STORE, 140164353478656, 140164378656767, -STORE, 140164345085952, 140164353478655, -SNULL, 140164353482751, 140164378656767, -STORE, 140164353478656, 140164353482751, -STORE, 140164353482752, 140164378656767, -SNULL, 140164454125568, 140164487696383, -STORE, 140164487696384, 140164496089087, -STORE, 140164454125568, 140164487696383, -SNULL, 140164487700479, 140164496089087, -STORE, 140164487696384, 140164487700479, -STORE, 140164487700480, 140164496089087, -STORE, 140164336693248, 140164353478655, -SNULL, 140164336697343, 140164353478655, -STORE, 140164336693248, 140164336697343, -STORE, 140164336697344, 140164353478655, -STORE, 140164328300544, 140164336693247, -SNULL, 140164454125568, 140164479303679, -STORE, 140164479303680, 140164487696383, -STORE, 140164454125568, 140164479303679, -SNULL, 140164479307775, 140164487696383, -STORE, 140164479303680, 140164479307775, -STORE, 140164479307776, 140164487696383, -STORE, 140164319907840, 140164336693247, -STORE, 140164236046336, 140164244439039, -SNULL, 140164588314624, 140164596703231, -STORE, 140164596703232, 140164605095935, -STORE, 140164588314624, 140164596703231, -SNULL, 140164596707327, 140164605095935, -STORE, 140164596703232, 140164596707327, -STORE, 140164596707328, 140164605095935, -SNULL, 140164454125568, 140164462518271, -STORE, 140164462518272, 140164479303679, -STORE, 140164454125568, 140164462518271, -SNULL, 140164462522367, 140164479303679, -STORE, 140164462518272, 140164462522367, -STORE, 140164462522368, 140164479303679, -STORE, 140164227653632, 140164244439039, -SNULL, 140164227657727, 140164244439039, -STORE, 140164227653632, 140164227657727, -STORE, 140164227657728, 140164244439039, -SNULL, 140164462522368, 140164470910975, -STORE, 140164470910976, 140164479303679, -STORE, 140164462522368, 140164470910975, -SNULL, 140164470915071, 140164479303679, -STORE, 140164470910976, 140164470915071, -STORE, 140164470915072, 140164479303679, -SNULL, 140164613492736, 140164621881343, -STORE, 140164621881344, 140164630274047, -STORE, 140164613492736, 140164621881343, -SNULL, 140164621885439, 140164630274047, -STORE, 140164621881344, 140164621885439, -STORE, 140164621885440, 140164630274047, -SNULL, 140164353482752, 140164370264063, -STORE, 140164370264064, 140164378656767, -STORE, 140164353482752, 140164370264063, -SNULL, 140164370268159, 140164378656767, -STORE, 140164370264064, 140164370268159, -STORE, 140164370268160, 140164378656767, -STORE, 140164219260928, 140164227653631, -SNULL, 140164319911935, 140164336693247, -STORE, 140164319907840, 140164319911935, -STORE, 140164319911936, 140164336693247, -SNULL, 140164336697344, 140164345085951, -STORE, 140164345085952, 140164353478655, -STORE, 140164336697344, 140164345085951, -SNULL, 140164345090047, 140164353478655, -STORE, 140164345085952, 140164345090047, -STORE, 140164345090048, 140164353478655, -SNULL, 140164319911936, 140164328300543, -STORE, 140164328300544, 140164336693247, -STORE, 140164319911936, 140164328300543, -SNULL, 140164328304639, 140164336693247, -STORE, 140164328300544, 140164328304639, -STORE, 140164328304640, 140164336693247, -SNULL, 140164454129663, 140164462518271, -STORE, 140164454125568, 140164454129663, -STORE, 140164454129664, 140164462518271, -STORE, 140164210868224, 140164227653631, -STORE, 140164202475520, 140164227653631, -STORE, 140164194082816, 140164227653631, -SNULL, 140164194086911, 140164227653631, -STORE, 140164194082816, 140164194086911, -STORE, 140164194086912, 140164227653631, -SNULL, 140164353482752, 140164361871359, -STORE, 140164361871360, 140164370264063, -STORE, 140164353482752, 140164361871359, -SNULL, 140164361875455, 140164370264063, -STORE, 140164361871360, 140164361875455, -STORE, 140164361875456, 140164370264063, -SNULL, 140164227657728, 140164236046335, -STORE, 140164236046336, 140164244439039, -STORE, 140164227657728, 140164236046335, -SNULL, 140164236050431, 140164244439039, -STORE, 140164236046336, 140164236050431, -STORE, 140164236050432, 140164244439039, -STORE, 140164185690112, 140164194082815, -SNULL, 140164194086912, 140164219260927, -STORE, 140164219260928, 140164227653631, -STORE, 140164194086912, 140164219260927, -SNULL, 140164219265023, 140164227653631, -STORE, 140164219260928, 140164219265023, -STORE, 140164219265024, 140164227653631, -STORE, 140164101828608, 140164110221311, -STORE, 140164093435904, 140164110221311, -STORE, 140164085043200, 140164110221311, -SNULL, 140164085047295, 140164110221311, -STORE, 140164085043200, 140164085047295, -STORE, 140164085047296, 140164110221311, -STORE, 140164076650496, 140164085043199, -SNULL, 140164185694207, 140164194082815, -STORE, 140164185690112, 140164185694207, -STORE, 140164185694208, 140164194082815, -SNULL, 140164085047296, 140164101828607, -STORE, 140164101828608, 140164110221311, -STORE, 140164085047296, 140164101828607, -SNULL, 140164101832703, 140164110221311, -STORE, 140164101828608, 140164101832703, -STORE, 140164101832704, 140164110221311, -SNULL, 140164085047296, 140164093435903, -STORE, 140164093435904, 140164101828607, -STORE, 140164085047296, 140164093435903, -SNULL, 140164093439999, 140164101828607, -STORE, 140164093435904, 140164093439999, -STORE, 140164093440000, 140164101828607, -SNULL, 140164194086912, 140164202475519, -STORE, 140164202475520, 140164219260927, -STORE, 140164194086912, 140164202475519, -SNULL, 140164202479615, 140164219260927, -STORE, 140164202475520, 140164202479615, -STORE, 140164202479616, 140164219260927, -SNULL, 140164202479616, 140164210868223, -STORE, 140164210868224, 140164219260927, -STORE, 140164202479616, 140164210868223, -SNULL, 140164210872319, 140164219260927, -STORE, 140164210868224, 140164210872319, -STORE, 140164210872320, 140164219260927, -SNULL, 140164076654591, 140164085043199, -STORE, 140164076650496, 140164076654591, -STORE, 140164076654592, 140164085043199, -STORE, 140164068257792, 140164076650495, -SNULL, 140164068261887, 140164076650495, -STORE, 140164068257792, 140164068261887, -STORE, 140164068261888, 140164076650495, -STORE, 140165753053184, 140165753081855, -STORE, 140165725851648, 140165728043007, -SNULL, 140165725851648, 140165725941759, -STORE, 140165725941760, 140165728043007, -STORE, 140165725851648, 140165725941759, -SNULL, 140165728034815, 140165728043007, -STORE, 140165725941760, 140165728034815, -STORE, 140165728034816, 140165728043007, -ERASE, 140165728034816, 140165728043007, -STORE, 140165728034816, 140165728043007, -SNULL, 140165728038911, 140165728043007, -STORE, 140165728034816, 140165728038911, -STORE, 140165728038912, 140165728043007, -ERASE, 140165753053184, 140165753081855, -ERASE, 140164638666752, 140164638670847, -ERASE, 140164638670848, 140164647059455, -ERASE, 140165091676160, 140165091680255, -ERASE, 140165091680256, 140165100068863, -ERASE, 140164613488640, 140164613492735, -ERASE, 140164613492736, 140164621881343, -ERASE, 140164319907840, 140164319911935, -ERASE, 140164319911936, 140164328300543, -ERASE, 140165620154368, 140165620158463, -ERASE, 140165620158464, 140165628547071, -ERASE, 140164798062592, 140164798066687, -ERASE, 140164798066688, 140164806455295, -ERASE, 140164789669888, 140164789673983, -ERASE, 140164789673984, 140164798062591, -ERASE, 140164965851136, 140164965855231, -ERASE, 140164965855232, 140164974243839, -ERASE, 140165074890752, 140165074894847, -ERASE, 140165074894848, 140165083283455, -ERASE, 140164672237568, 140164672241663, -ERASE, 140164672241664, 140164680630271, -ERASE, 140164454125568, 140164454129663, -ERASE, 140164454129664, 140164462518271, -ERASE, 140165200715776, 140165200719871, -ERASE, 140165200719872, 140165209108479, -ERASE, 140164932280320, 140164932284415, -ERASE, 140164932284416, 140164940673023, -ERASE, 140164663844864, 140164663848959, -ERASE, 140164663848960, 140164672237567, -ERASE, 140164697415680, 140164697419775, -ERASE, 140164697419776, 140164705808383, -ERASE, 140164831633408, 140164831637503, -ERASE, 140164831637504, 140164840026111, -ERASE, 140165192323072, 140165192327167, -ERASE, 140165192327168, 140165200715775, -ERASE, 140165108461568, 140165108465663, -ERASE, 140165108465664, 140165116854271, -ERASE, 140164840026112, 140164840030207, -ERASE, 140164840030208, 140164848418815, -ERASE, 140164647059456, 140164647063551, -ERASE, 140164647063552, 140164655452159, -ERASE, 140165083283456, 140165083287551, -ERASE, 140165083287552, 140165091676159, -ERASE, 140164923887616, 140164923891711, -ERASE, 140164923891712, 140164932280319, -ERASE, 140164823240704, 140164823244799, -ERASE, 140164823244800, 140164831633407, -ERASE, 140164227653632, 140164227657727, -ERASE, 140164227657728, 140164236046335, -ERASE, 140164957458432, 140164957462527, -ERASE, 140164957462528, 140164965851135, -ERASE, 140164680630272, 140164680634367, -ERASE, 140164680634368, 140164689022975, -ERASE, 140164974243840, 140164974247935, -ERASE, 140164974247936, 140164982636543, -ERASE, 140165066498048, 140165066502143, -ERASE, 140165066502144, 140165074890751, -ERASE, 140164621881344, 140164621885439, -ERASE, 140164621885440, 140164630274047, -ERASE, 140164949065728, 140164949069823, -ERASE, 140164949069824, 140164957458431, -ERASE, 140164588310528, 140164588314623, -ERASE, 140164588314624, 140164596703231, -ERASE, 140164806455296, 140164806459391, -ERASE, 140164806459392, 140164814847999, -ERASE, 140164940673024, 140164940677119, -ERASE, 140164940677120, 140164949065727, -ERASE, 140164596703232, 140164596707327, -ERASE, 140164596707328, 140164605095935, -ERASE, 140164605095936, 140164605100031, -ERASE, 140164605100032, 140164613488639, -ERASE, 140164655452160, 140164655456255, -ERASE, 140164655456256, 140164663844863, -ERASE, 140164705808384, 140164705812479, -ERASE, 140164705812480, 140164714201087, -ERASE, 140164689022976, 140164689027071, -ERASE, 140164689027072, 140164697415679, -ERASE, 140164630274048, 140164630278143, -ERASE, 140164630278144, 140164638666751, -ERASE, 140164479303680, 140164479307775, -ERASE, 140164479307776, 140164487696383, -ERASE, 140164236046336, 140164236050431, -ERASE, 140164236050432, 140164244439039, -ERASE, 140164085043200, 140164085047295, -ERASE, 140164085047296, 140164093435903, -ERASE, 140164345085952, 140164345090047, -ERASE, 140164345090048, 140164353478655, -ERASE, 140164101828608, 140164101832703, -ERASE, 140164101832704, 140164110221311, -ERASE, 140164370264064, 140164370268159, -ERASE, 140164370268160, 140164378656767, -ERASE, 140164336693248, 140164336697343, -ERASE, 140164336697344, 140164345085951, -ERASE, 140164194082816, 140164194086911, -ERASE, 140164194086912, 140164202475519, -ERASE, 140164353478656, 140164353482751, -ERASE, 140164353482752, 140164361871359, -ERASE, 140164210868224, 140164210872319, -ERASE, 140164210872320, 140164219260927, -ERASE, 140164814848000, 140164814852095, -ERASE, 140164814852096, 140164823240703, -ERASE, 140164504481792, 140164504485887, -ERASE, 140164504485888, 140164512874495, -ERASE, 140165100068864, 140165100072959, -ERASE, 140165100072960, 140165108461567, -ERASE, 140164361871360, 140164361875455, -ERASE, 140164361875456, 140164370264063, -ERASE, 140164470910976, 140164470915071, -ERASE, 140164470915072, 140164479303679, -ERASE, 140164076650496, 140164076654591, -ERASE, 140164076654592, 140164085043199, -ERASE, 140164202475520, 140164202479615, -ERASE, 140164202479616, 140164210868223, -ERASE, 140164462518272, 140164462522367, -ERASE, 140164462522368, 140164470910975, -ERASE, 140165351718912, 140165351723007, -ERASE, 140165351723008, 140165360111615, -ERASE, 140164328300544, 140164328304639, -ERASE, 140164328304640, 140164336693247, -ERASE, 140164093435904, 140164093439999, -ERASE, 140164093440000, 140164101828607, -ERASE, 140165603368960, 140165603373055, -ERASE, 140165603373056, 140165611761663, -ERASE, 140165368504320, 140165368508415, -ERASE, 140165368508416, 140165376897023, -ERASE, 140165334933504, 140165334937599, -ERASE, 140165334937600, 140165343326207, -ERASE, 140165594976256, 140165594980351, -ERASE, 140165594980352, 140165603368959, -ERASE, 140164487696384, 140164487700479, -ERASE, 140164487700480, 140164496089087, -ERASE, 140164219260928, 140164219265023, -ERASE, 140164219265024, 140164227653631, -ERASE, 140164185690112, 140164185694207, -ERASE, 140164185694208, 140164194082815, -ERASE, 140164068257792, 140164068261887, -ERASE, 140164068261888, 140164076650495, -ERASE, 140165225893888, 140165225897983, -ERASE, 140165225897984, 140165234286591, -ERASE, 140165058105344, 140165058109439, - }; - unsigned long set31[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140730890784768, 140737488351231, -SNULL, 140730890788863, 140737488351231, -STORE, 140730890784768, 140730890788863, -STORE, 140730890653696, 140730890788863, -STORE, 94577123659776, 94577125912575, -SNULL, 94577123790847, 94577125912575, -STORE, 94577123659776, 94577123790847, -STORE, 94577123790848, 94577125912575, -ERASE, 94577123790848, 94577125912575, -STORE, 94577125883904, 94577125892095, -STORE, 94577125892096, 94577125912575, -STORE, 140624060407808, 140624062660607, -SNULL, 140624060551167, 140624062660607, -STORE, 140624060407808, 140624060551167, -STORE, 140624060551168, 140624062660607, -ERASE, 140624060551168, 140624062660607, -STORE, 140624062648320, 140624062656511, -STORE, 140624062656512, 140624062660607, -STORE, 140730892140544, 140730892144639, -STORE, 140730892128256, 140730892140543, -STORE, 140624062619648, 140624062648319, -STORE, 140624062611456, 140624062619647, -STORE, 140624058191872, 140624060407807, -SNULL, 140624058191872, 140624058290175, -STORE, 140624058290176, 140624060407807, -STORE, 140624058191872, 140624058290175, -SNULL, 140624060383231, 140624060407807, -STORE, 140624058290176, 140624060383231, -STORE, 140624060383232, 140624060407807, -SNULL, 140624060383232, 140624060391423, -STORE, 140624060391424, 140624060407807, -STORE, 140624060383232, 140624060391423, -ERASE, 140624060383232, 140624060391423, -STORE, 140624060383232, 140624060391423, -ERASE, 140624060391424, 140624060407807, -STORE, 140624060391424, 140624060407807, -STORE, 140624054394880, 140624058191871, -SNULL, 140624054394880, 140624056053759, -STORE, 140624056053760, 140624058191871, -STORE, 140624054394880, 140624056053759, -SNULL, 140624058150911, 140624058191871, -STORE, 140624056053760, 140624058150911, -STORE, 140624058150912, 140624058191871, -SNULL, 140624058150912, 140624058175487, -STORE, 140624058175488, 140624058191871, -STORE, 140624058150912, 140624058175487, -ERASE, 140624058150912, 140624058175487, -STORE, 140624058150912, 140624058175487, -ERASE, 140624058175488, 140624058191871, -STORE, 140624058175488, 140624058191871, -STORE, 140624062603264, 140624062619647, -SNULL, 140624058167295, 140624058175487, -STORE, 140624058150912, 140624058167295, -STORE, 140624058167296, 140624058175487, -SNULL, 140624060387327, 140624060391423, -STORE, 140624060383232, 140624060387327, -STORE, 140624060387328, 140624060391423, -SNULL, 94577125887999, 94577125892095, -STORE, 94577125883904, 94577125887999, -STORE, 94577125888000, 94577125892095, -SNULL, 140624062652415, 140624062656511, -STORE, 140624062648320, 140624062652415, -STORE, 140624062652416, 140624062656511, -ERASE, 140624062619648, 140624062648319, -STORE, 94577157709824, 94577157844991, -STORE, 140624046002176, 140624054394879, -SNULL, 140624046006271, 140624054394879, -STORE, 140624046002176, 140624046006271, -STORE, 140624046006272, 140624054394879, -STORE, 140624037609472, 140624046002175, -STORE, 140623903391744, 140624037609471, -SNULL, 140623903391744, 140623940157439, -STORE, 140623940157440, 140624037609471, -STORE, 140623903391744, 140623940157439, -ERASE, 140623903391744, 140623940157439, -SNULL, 140624007266303, 140624037609471, -STORE, 140623940157440, 140624007266303, -STORE, 140624007266304, 140624037609471, -ERASE, 140624007266304, 140624037609471, -SNULL, 140623940292607, 140624007266303, -STORE, 140623940157440, 140623940292607, -STORE, 140623940292608, 140624007266303, -SNULL, 140624037613567, 140624046002175, -STORE, 140624037609472, 140624037613567, -STORE, 140624037613568, 140624046002175, -STORE, 140624029216768, 140624037609471, -SNULL, 140624029220863, 140624037609471, -STORE, 140624029216768, 140624029220863, -STORE, 140624029220864, 140624037609471, -STORE, 140624020824064, 140624029216767, -SNULL, 140624020828159, 140624029216767, -STORE, 140624020824064, 140624020828159, -STORE, 140624020828160, 140624029216767, -STORE, 140624012431360, 140624020824063, -SNULL, 140624012435455, 140624020824063, -STORE, 140624012431360, 140624012435455, -STORE, 140624012435456, 140624020824063, -STORE, 140623931764736, 140623940157439, -STORE, 140623797547008, 140623931764735, -SNULL, 140623797547008, 140623805939711, -STORE, 140623805939712, 140623931764735, -STORE, 140623797547008, 140623805939711, -ERASE, 140623797547008, 140623805939711, -SNULL, 140623873048575, 140623931764735, -STORE, 140623805939712, 140623873048575, -STORE, 140623873048576, 140623931764735, -ERASE, 140623873048576, 140623931764735, -STORE, 140623923372032, 140623940157439, -STORE, 140623914979328, 140623940157439, -STORE, 140623906586624, 140623940157439, -STORE, 140623671721984, 140623873048575, -SNULL, 140623738830847, 140623873048575, -STORE, 140623671721984, 140623738830847, -STORE, 140623738830848, 140623873048575, -SNULL, 140623738830848, 140623805939711, -STORE, 140623805939712, 140623873048575, -STORE, 140623738830848, 140623805939711, -ERASE, 140623738830848, 140623805939711, -SNULL, 140623806074879, 140623873048575, -STORE, 140623805939712, 140623806074879, -STORE, 140623806074880, 140623873048575, -SNULL, 140623906586624, 140623931764735, -STORE, 140623931764736, 140623940157439, -STORE, 140623906586624, 140623931764735, -SNULL, 140623931768831, 140623940157439, -STORE, 140623931764736, 140623931768831, -STORE, 140623931768832, 140623940157439, -STORE, 140623537504256, 140623738830847, -SNULL, 140623537504256, 140623671721983, -STORE, 140623671721984, 140623738830847, -STORE, 140623537504256, 140623671721983, -SNULL, 140623671857151, 140623738830847, -STORE, 140623671721984, 140623671857151, -STORE, 140623671857152, 140623738830847, -SNULL, 140623604613119, 140623671721983, -STORE, 140623537504256, 140623604613119, -STORE, 140623604613120, 140623671721983, -ERASE, 140623604613120, 140623671721983, -SNULL, 140623537639423, 140623604613119, -STORE, 140623537504256, 140623537639423, -STORE, 140623537639424, 140623604613119, -STORE, 140623537639424, 140623671721983, -SNULL, 140623537639424, 140623604613119, -STORE, 140623604613120, 140623671721983, -STORE, 140623537639424, 140623604613119, -SNULL, 140623604748287, 140623671721983, -STORE, 140623604613120, 140623604748287, -STORE, 140623604748288, 140623671721983, -STORE, 140623898193920, 140623931764735, -SNULL, 140623898193920, 140623923372031, -STORE, 140623923372032, 140623931764735, -STORE, 140623898193920, 140623923372031, -SNULL, 140623923376127, 140623931764735, -STORE, 140623923372032, 140623923376127, -STORE, 140623923376128, 140623931764735, -STORE, 140623889801216, 140623923372031, -SNULL, 140623889801216, 140623898193919, -STORE, 140623898193920, 140623923372031, -STORE, 140623889801216, 140623898193919, -SNULL, 140623898198015, 140623923372031, -STORE, 140623898193920, 140623898198015, -STORE, 140623898198016, 140623923372031, -SNULL, 140623889805311, 140623898193919, -STORE, 140623889801216, 140623889805311, -STORE, 140623889805312, 140623898193919, -SNULL, 140623898198016, 140623906586623, -STORE, 140623906586624, 140623923372031, -STORE, 140623898198016, 140623906586623, -SNULL, 140623906590719, 140623923372031, -STORE, 140623906586624, 140623906590719, -STORE, 140623906590720, 140623923372031, -STORE, 140623881408512, 140623889801215, -SNULL, 140623906590720, 140623914979327, -STORE, 140623914979328, 140623923372031, -STORE, 140623906590720, 140623914979327, -SNULL, 140623914983423, 140623923372031, -STORE, 140623914979328, 140623914983423, -STORE, 140623914983424, 140623923372031, -SNULL, 140623881412607, 140623889801215, -STORE, 140623881408512, 140623881412607, -STORE, 140623881412608, 140623889801215, -STORE, 140623797547008, 140623805939711, -STORE, 140623789154304, 140623805939711, -STORE, 140623780761600, 140623805939711, -SNULL, 140623780761600, 140623789154303, -STORE, 140623789154304, 140623805939711, -STORE, 140623780761600, 140623789154303, -SNULL, 140623789158399, 140623805939711, -STORE, 140623789154304, 140623789158399, -STORE, 140623789158400, 140623805939711, -STORE, 140623772368896, 140623789154303, -STORE, 140623763976192, 140623789154303, -SNULL, 140623763976192, 140623780761599, -STORE, 140623780761600, 140623789154303, -STORE, 140623763976192, 140623780761599, -SNULL, 140623780765695, 140623789154303, -STORE, 140623780761600, 140623780765695, -STORE, 140623780765696, 140623789154303, -SNULL, 140623789158400, 140623797547007, -STORE, 140623797547008, 140623805939711, -STORE, 140623789158400, 140623797547007, -SNULL, 140623797551103, 140623805939711, -STORE, 140623797547008, 140623797551103, -STORE, 140623797551104, 140623805939711, -SNULL, 140623763976192, 140623772368895, -STORE, 140623772368896, 140623780761599, -STORE, 140623763976192, 140623772368895, -SNULL, 140623772372991, 140623780761599, -STORE, 140623772368896, 140623772372991, -STORE, 140623772372992, 140623780761599, -SNULL, 140623763980287, 140623772368895, -STORE, 140623763976192, 140623763980287, -STORE, 140623763980288, 140623772368895, -STORE, 140623755583488, 140623763976191, -STORE, 140623747190784, 140623763976191, -SNULL, 140623747190784, 140623755583487, -STORE, 140623755583488, 140623763976191, -STORE, 140623747190784, 140623755583487, -SNULL, 140623755587583, 140623763976191, -STORE, 140623755583488, 140623755587583, -STORE, 140623755587584, 140623763976191, -STORE, 140623529111552, 140623537504255, -SNULL, 140623747194879, 140623755583487, -STORE, 140623747190784, 140623747194879, -STORE, 140623747194880, 140623755583487, -SNULL, 140623529115647, 140623537504255, -STORE, 140623529111552, 140623529115647, -STORE, 140623529115648, 140623537504255, -STORE, 140623520718848, 140623529111551, -SNULL, 140623520722943, 140623529111551, -STORE, 140623520718848, 140623520722943, -STORE, 140623520722944, 140623529111551, -STORE, 140623512326144, 140623520718847, -STORE, 140623503933440, 140623520718847, -STORE, 140623495540736, 140623520718847, -STORE, 140623361323008, 140623495540735, -STORE, 140623227105280, 140623495540735, -STORE, 140623218712576, 140623227105279, -STORE, 140623084494848, 140623218712575, -STORE, 140623076102144, 140623084494847, -STORE, 140622941884416, 140623076102143, -SNULL, 140622941884416, 140623000633343, -STORE, 140623000633344, 140623076102143, -STORE, 140622941884416, 140623000633343, -ERASE, 140622941884416, 140623000633343, -STORE, 140622992240640, 140623000633343, -STORE, 140622983847936, 140623000633343, -STORE, 140622849630208, 140622983847935, -STORE, 140622841237504, 140622849630207, -SNULL, 140622849630208, 140622866415615, -STORE, 140622866415616, 140622983847935, -STORE, 140622849630208, 140622866415615, -ERASE, 140622849630208, 140622866415615, -STORE, 140622858022912, 140622866415615, -SNULL, 140622933524479, 140622983847935, -STORE, 140622866415616, 140622933524479, -STORE, 140622933524480, 140622983847935, -ERASE, 140622933524480, 140622983847935, -STORE, 140622975455232, 140623000633343, -STORE, 140622707019776, 140622841237503, -STORE, 140622967062528, 140623000633343, -STORE, 140622572802048, 140622841237503, -STORE, 140622958669824, 140623000633343, -STORE, 140622438584320, 140622841237503, -STORE, 140622950277120, 140623000633343, -SNULL, 140622858027007, 140622866415615, -STORE, 140622858022912, 140622858027007, -STORE, 140622858027008, 140622866415615, -STORE, 140622941884416, 140623000633343, -STORE, 140622841237504, 140622858022911, -SNULL, 140622841237504, 140622849630207, -STORE, 140622849630208, 140622858022911, -STORE, 140622841237504, 140622849630207, -SNULL, 140622849634303, 140622858022911, -STORE, 140622849630208, 140622849634303, -STORE, 140622849634304, 140622858022911, -STORE, 140622430191616, 140622438584319, -SNULL, 140622430195711, 140622438584319, -STORE, 140622430191616, 140622430195711, -STORE, 140622430195712, 140622438584319, -SNULL, 140623361323007, 140623495540735, -STORE, 140623227105280, 140623361323007, -STORE, 140623361323008, 140623495540735, -SNULL, 140623361323008, 140623403286527, -STORE, 140623403286528, 140623495540735, -STORE, 140623361323008, 140623403286527, -ERASE, 140623361323008, 140623403286527, -SNULL, 140623470395391, 140623495540735, -STORE, 140623403286528, 140623470395391, -STORE, 140623470395392, 140623495540735, -ERASE, 140623470395392, 140623495540735, -SNULL, 140623227105280, 140623269068799, -STORE, 140623269068800, 140623361323007, -STORE, 140623227105280, 140623269068799, -ERASE, 140623227105280, 140623269068799, -SNULL, 140623084494848, 140623134851071, -STORE, 140623134851072, 140623218712575, -STORE, 140623084494848, 140623134851071, -ERASE, 140623084494848, 140623134851071, -SNULL, 140623201959935, 140623218712575, -STORE, 140623134851072, 140623201959935, -STORE, 140623201959936, 140623218712575, -ERASE, 140623201959936, 140623218712575, -SNULL, 140623067742207, 140623076102143, -STORE, 140623000633344, 140623067742207, -STORE, 140623067742208, 140623076102143, -ERASE, 140623067742208, 140623076102143, -STORE, 140622295973888, 140622430191615, -SNULL, 140622295973888, 140622329544703, -STORE, 140622329544704, 140622430191615, -STORE, 140622295973888, 140622329544703, -ERASE, 140622295973888, 140622329544703, -SNULL, 140622866550783, 140622933524479, -STORE, 140622866415616, 140622866550783, -STORE, 140622866550784, 140622933524479, -SNULL, 140622707019775, 140622841237503, -STORE, 140622438584320, 140622707019775, -STORE, 140622707019776, 140622841237503, -SNULL, 140622707019776, 140622732197887, -STORE, 140622732197888, 140622841237503, -STORE, 140622707019776, 140622732197887, -ERASE, 140622707019776, 140622732197887, -SNULL, 140622799306751, 140622841237503, -STORE, 140622732197888, 140622799306751, -STORE, 140622799306752, 140622841237503, -ERASE, 140622799306752, 140622841237503, -SNULL, 140622572802047, 140622707019775, -STORE, 140622438584320, 140622572802047, -STORE, 140622572802048, 140622707019775, -SNULL, 140622572802048, 140622597980159, -STORE, 140622597980160, 140622707019775, -STORE, 140622572802048, 140622597980159, -ERASE, 140622572802048, 140622597980159, -SNULL, 140622438584320, 140622463762431, -STORE, 140622463762432, 140622572802047, -STORE, 140622438584320, 140622463762431, -ERASE, 140622438584320, 140622463762431, -SNULL, 140622530871295, 140622572802047, -STORE, 140622463762432, 140622530871295, -STORE, 140622530871296, 140622572802047, -ERASE, 140622530871296, 140622572802047, -STORE, 140622195326976, 140622430191615, -SNULL, 140622262435839, 140622430191615, -STORE, 140622195326976, 140622262435839, -STORE, 140622262435840, 140622430191615, -SNULL, 140622262435840, 140622329544703, -STORE, 140622329544704, 140622430191615, -STORE, 140622262435840, 140622329544703, -ERASE, 140622262435840, 140622329544703, -SNULL, 140622841241599, 140622849630207, -STORE, 140622841237504, 140622841241599, -STORE, 140622841241600, 140622849630207, -STORE, 140623487148032, 140623520718847, -STORE, 140623478755328, 140623520718847, -SNULL, 140622941884416, 140622983847935, -STORE, 140622983847936, 140623000633343, -STORE, 140622941884416, 140622983847935, -SNULL, 140622983852031, 140623000633343, -STORE, 140622983847936, 140622983852031, -STORE, 140622983852032, 140623000633343, -STORE, 140623394893824, 140623403286527, -SNULL, 140623394897919, 140623403286527, -STORE, 140623394893824, 140623394897919, -STORE, 140623394897920, 140623403286527, -SNULL, 140623403421695, 140623470395391, -STORE, 140623403286528, 140623403421695, -STORE, 140623403421696, 140623470395391, -SNULL, 140623478755328, 140623503933439, -STORE, 140623503933440, 140623520718847, -STORE, 140623478755328, 140623503933439, -SNULL, 140623503937535, 140623520718847, -STORE, 140623503933440, 140623503937535, -STORE, 140623503937536, 140623520718847, -SNULL, 140623336177663, 140623361323007, -STORE, 140623269068800, 140623336177663, -STORE, 140623336177664, 140623361323007, -ERASE, 140623336177664, 140623361323007, -SNULL, 140623269203967, 140623336177663, -STORE, 140623269068800, 140623269203967, -STORE, 140623269203968, 140623336177663, -SNULL, 140623134986239, 140623201959935, -STORE, 140623134851072, 140623134986239, -STORE, 140623134986240, 140623201959935, -SNULL, 140623000768511, 140623067742207, -STORE, 140623000633344, 140623000768511, -STORE, 140623000768512, 140623067742207, -SNULL, 140622396653567, 140622430191615, -STORE, 140622329544704, 140622396653567, -STORE, 140622396653568, 140622430191615, -ERASE, 140622396653568, 140622430191615, -SNULL, 140622732333055, 140622799306751, -STORE, 140622732197888, 140622732333055, -STORE, 140622732333056, 140622799306751, -SNULL, 140622941884416, 140622975455231, -STORE, 140622975455232, 140622983847935, -STORE, 140622941884416, 140622975455231, -SNULL, 140622975459327, 140622983847935, -STORE, 140622975455232, 140622975459327, -STORE, 140622975459328, 140622983847935, -SNULL, 140622665089023, 140622707019775, -STORE, 140622597980160, 140622665089023, -STORE, 140622665089024, 140622707019775, -ERASE, 140622665089024, 140622707019775, -SNULL, 140622598115327, 140622665089023, -STORE, 140622597980160, 140622598115327, -STORE, 140622598115328, 140622665089023, -SNULL, 140622463897599, 140622530871295, -STORE, 140622463762432, 140622463897599, -STORE, 140622463897600, 140622530871295, -SNULL, 140622195462143, 140622262435839, -STORE, 140622195326976, 140622195462143, -STORE, 140622195462144, 140622262435839, -STORE, 140623386501120, 140623394893823, -SNULL, 140622941884416, 140622950277119, -STORE, 140622950277120, 140622975455231, -STORE, 140622941884416, 140622950277119, -SNULL, 140622950281215, 140622975455231, -STORE, 140622950277120, 140622950281215, -STORE, 140622950281216, 140622975455231, -SNULL, 140622941888511, 140622950277119, -STORE, 140622941884416, 140622941888511, -STORE, 140622941888512, 140622950277119, -STORE, 140623378108416, 140623394893823, -SNULL, 140623478755328, 140623495540735, -STORE, 140623495540736, 140623503933439, -STORE, 140623478755328, 140623495540735, -SNULL, 140623495544831, 140623503933439, -STORE, 140623495540736, 140623495544831, -STORE, 140623495544832, 140623503933439, -SNULL, 140623478755328, 140623487148031, -STORE, 140623487148032, 140623495540735, -STORE, 140623478755328, 140623487148031, -SNULL, 140623487152127, 140623495540735, -STORE, 140623487148032, 140623487152127, -STORE, 140623487152128, 140623495540735, -SNULL, 140623218716671, 140623227105279, -STORE, 140623218712576, 140623218716671, -STORE, 140623218716672, 140623227105279, -SNULL, 140623076106239, 140623084494847, -STORE, 140623076102144, 140623076106239, -STORE, 140623076106240, 140623084494847, -SNULL, 140622329679871, 140622396653567, -STORE, 140622329544704, 140622329679871, -STORE, 140622329679872, 140622396653567, -SNULL, 140622950281216, 140622958669823, -STORE, 140622958669824, 140622975455231, -STORE, 140622950281216, 140622958669823, -SNULL, 140622958673919, 140622975455231, -STORE, 140622958669824, 140622958673919, -STORE, 140622958673920, 140622975455231, -SNULL, 140623503937536, 140623512326143, -STORE, 140623512326144, 140623520718847, -STORE, 140623503937536, 140623512326143, -SNULL, 140623512330239, 140623520718847, -STORE, 140623512326144, 140623512330239, -STORE, 140623512330240, 140623520718847, -SNULL, 140623378108416, 140623386501119, -STORE, 140623386501120, 140623394893823, -STORE, 140623378108416, 140623386501119, -SNULL, 140623386505215, 140623394893823, -STORE, 140623386501120, 140623386505215, -STORE, 140623386505216, 140623394893823, -STORE, 140623369715712, 140623386501119, -STORE, 140623361323008, 140623386501119, -STORE, 140623352930304, 140623386501119, -SNULL, 140623352930304, 140623361323007, -STORE, 140623361323008, 140623386501119, -STORE, 140623352930304, 140623361323007, -SNULL, 140623361327103, 140623386501119, -STORE, 140623361323008, 140623361327103, -STORE, 140623361327104, 140623386501119, -SNULL, 140623478759423, 140623487148031, -STORE, 140623478755328, 140623478759423, -STORE, 140623478759424, 140623487148031, -STORE, 140623344537600, 140623361323007, -STORE, 140623260676096, 140623269068799, -SNULL, 140622958673920, 140622967062527, -STORE, 140622967062528, 140622975455231, -STORE, 140622958673920, 140622967062527, -SNULL, 140622967066623, 140622975455231, -STORE, 140622967062528, 140622967066623, -STORE, 140622967066624, 140622975455231, -STORE, 140623252283392, 140623269068799, -STORE, 140623243890688, 140623269068799, -SNULL, 140622983852032, 140622992240639, -STORE, 140622992240640, 140623000633343, -STORE, 140622983852032, 140622992240639, -SNULL, 140622992244735, 140623000633343, -STORE, 140622992240640, 140622992244735, -STORE, 140622992244736, 140623000633343, -STORE, 140623235497984, 140623269068799, -STORE, 140623218716672, 140623235497983, -STORE, 140623210319872, 140623218712575, -STORE, 140623126458368, 140623134851071, -SNULL, 140623210323967, 140623218712575, -STORE, 140623210319872, 140623210323967, -STORE, 140623210323968, 140623218712575, -SNULL, 140623218716672, 140623227105279, -STORE, 140623227105280, 140623235497983, -STORE, 140623218716672, 140623227105279, -SNULL, 140623227109375, 140623235497983, -STORE, 140623227105280, 140623227109375, -STORE, 140623227109376, 140623235497983, -STORE, 140623118065664, 140623134851071, -STORE, 140623109672960, 140623134851071, -SNULL, 140623109677055, 140623134851071, -STORE, 140623109672960, 140623109677055, -STORE, 140623109677056, 140623134851071, -STORE, 140623101280256, 140623109672959, -STORE, 140623092887552, 140623109672959, -SNULL, 140623092887552, 140623101280255, -STORE, 140623101280256, 140623109672959, -STORE, 140623092887552, 140623101280255, -SNULL, 140623101284351, 140623109672959, -STORE, 140623101280256, 140623101284351, -STORE, 140623101284352, 140623109672959, -SNULL, 140623361327104, 140623378108415, -STORE, 140623378108416, 140623386501119, -STORE, 140623361327104, 140623378108415, -SNULL, 140623378112511, 140623386501119, -STORE, 140623378108416, 140623378112511, -STORE, 140623378112512, 140623386501119, -SNULL, 140623235497984, 140623243890687, -STORE, 140623243890688, 140623269068799, -STORE, 140623235497984, 140623243890687, -SNULL, 140623243894783, 140623269068799, -STORE, 140623243890688, 140623243894783, -STORE, 140623243894784, 140623269068799, -SNULL, 140623361327104, 140623369715711, -STORE, 140623369715712, 140623378108415, -STORE, 140623361327104, 140623369715711, -SNULL, 140623369719807, 140623378108415, -STORE, 140623369715712, 140623369719807, -STORE, 140623369719808, 140623378108415, -SNULL, 140623243894784, 140623252283391, -STORE, 140623252283392, 140623269068799, -STORE, 140623243894784, 140623252283391, -SNULL, 140623252287487, 140623269068799, -STORE, 140623252283392, 140623252287487, -STORE, 140623252287488, 140623269068799, -SNULL, 140623235502079, 140623243890687, -STORE, 140623235497984, 140623235502079, -STORE, 140623235502080, 140623243890687, -SNULL, 140623344541695, 140623361323007, -STORE, 140623344537600, 140623344541695, -STORE, 140623344541696, 140623361323007, -STORE, 140623076106240, 140623092887551, -SNULL, 140623076106240, 140623084494847, -STORE, 140623084494848, 140623092887551, -STORE, 140623076106240, 140623084494847, -SNULL, 140623084498943, 140623092887551, -STORE, 140623084494848, 140623084498943, -STORE, 140623084498944, 140623092887551, -SNULL, 140623344541696, 140623352930303, -STORE, 140623352930304, 140623361323007, -STORE, 140623344541696, 140623352930303, -SNULL, 140623352934399, 140623361323007, -STORE, 140623352930304, 140623352934399, -STORE, 140623352934400, 140623361323007, -SNULL, 140623109677056, 140623118065663, -STORE, 140623118065664, 140623134851071, -STORE, 140623109677056, 140623118065663, -SNULL, 140623118069759, 140623134851071, -STORE, 140623118065664, 140623118069759, -STORE, 140623118069760, 140623134851071, -STORE, 140622832844800, 140622841237503, -STORE, 140622824452096, 140622841237503, -SNULL, 140622824452096, 140622832844799, -STORE, 140622832844800, 140622841237503, -STORE, 140622824452096, 140622832844799, -SNULL, 140622832848895, 140622841237503, -STORE, 140622832844800, 140622832848895, -STORE, 140622832848896, 140622841237503, -STORE, 140622816059392, 140622832844799, -SNULL, 140623092891647, 140623101280255, -STORE, 140623092887552, 140623092891647, -STORE, 140623092891648, 140623101280255, -SNULL, 140623118069760, 140623126458367, -STORE, 140623126458368, 140623134851071, -STORE, 140623118069760, 140623126458367, -SNULL, 140623126462463, 140623134851071, -STORE, 140623126458368, 140623126462463, -STORE, 140623126462464, 140623134851071, -SNULL, 140623252287488, 140623260676095, -STORE, 140623260676096, 140623269068799, -STORE, 140623252287488, 140623260676095, -SNULL, 140623260680191, 140623269068799, -STORE, 140623260676096, 140623260680191, -STORE, 140623260680192, 140623269068799, -STORE, 140622807666688, 140622832844799, -STORE, 140622723805184, 140622732197887, -STORE, 140622715412480, 140622732197887, -STORE, 140622707019776, 140622732197887, -SNULL, 140622707023871, 140622732197887, -STORE, 140622707019776, 140622707023871, -STORE, 140622707023872, 140622732197887, -STORE, 140622698627072, 140622707019775, -STORE, 140622690234368, 140622707019775, -SNULL, 140622690238463, 140622707019775, -STORE, 140622690234368, 140622690238463, -STORE, 140622690238464, 140622707019775, -SNULL, 140622807666688, 140622816059391, -STORE, 140622816059392, 140622832844799, -STORE, 140622807666688, 140622816059391, -SNULL, 140622816063487, 140622832844799, -STORE, 140622816059392, 140622816063487, -STORE, 140622816063488, 140622832844799, -STORE, 140622681841664, 140622690234367, -STORE, 140622673448960, 140622690234367, -SNULL, 140622673453055, 140622690234367, -STORE, 140622673448960, 140622673453055, -STORE, 140622673453056, 140622690234367, -STORE, 140622589587456, 140622597980159, -SNULL, 140622807670783, 140622816059391, -STORE, 140622807666688, 140622807670783, -STORE, 140622807670784, 140622816059391, -STORE, 140622581194752, 140622597980159, -SNULL, 140622581198847, 140622597980159, -STORE, 140622581194752, 140622581198847, -STORE, 140622581198848, 140622597980159, -SNULL, 140622816063488, 140622824452095, -STORE, 140622824452096, 140622832844799, -STORE, 140622816063488, 140622824452095, -SNULL, 140622824456191, 140622832844799, -STORE, 140622824452096, 140622824456191, -STORE, 140622824456192, 140622832844799, -STORE, 140622572802048, 140622581194751, -SNULL, 140622572806143, 140622581194751, -STORE, 140622572802048, 140622572806143, -STORE, 140622572806144, 140622581194751, -STORE, 140622564409344, 140622572802047, -STORE, 140622556016640, 140622572802047, -SNULL, 140622556016640, 140622564409343, -STORE, 140622564409344, 140622572802047, -STORE, 140622556016640, 140622564409343, -SNULL, 140622564413439, 140622572802047, -STORE, 140622564409344, 140622564413439, -STORE, 140622564413440, 140622572802047, -SNULL, 140622690238464, 140622698627071, -STORE, 140622698627072, 140622707019775, -STORE, 140622690238464, 140622698627071, -SNULL, 140622698631167, 140622707019775, -STORE, 140622698627072, 140622698631167, -STORE, 140622698631168, 140622707019775, -SNULL, 140622707023872, 140622723805183, -STORE, 140622723805184, 140622732197887, -STORE, 140622707023872, 140622723805183, -SNULL, 140622723809279, 140622732197887, -STORE, 140622723805184, 140622723809279, -STORE, 140622723809280, 140622732197887, -SNULL, 140622707023872, 140622715412479, -STORE, 140622715412480, 140622723805183, -STORE, 140622707023872, 140622715412479, -SNULL, 140622715416575, 140622723805183, -STORE, 140622715412480, 140622715416575, -STORE, 140622715416576, 140622723805183, -STORE, 140622547623936, 140622564409343, -SNULL, 140622547628031, 140622564409343, -STORE, 140622547623936, 140622547628031, -STORE, 140622547628032, 140622564409343, -STORE, 140622539231232, 140622547623935, -SNULL, 140622539235327, 140622547623935, -STORE, 140622539231232, 140622539235327, -STORE, 140622539235328, 140622547623935, -SNULL, 140622581198848, 140622589587455, -STORE, 140622589587456, 140622597980159, -STORE, 140622581198848, 140622589587455, -SNULL, 140622589591551, 140622597980159, -STORE, 140622589587456, 140622589591551, -STORE, 140622589591552, 140622597980159, -STORE, 140622455369728, 140622463762431, -SNULL, 140622455373823, 140622463762431, -STORE, 140622455369728, 140622455373823, -STORE, 140622455373824, 140622463762431, -STORE, 140622446977024, 140622455369727, -SNULL, 140622446981119, 140622455369727, -STORE, 140622446977024, 140622446981119, -STORE, 140622446981120, 140622455369727, -SNULL, 140622547628032, 140622556016639, -STORE, 140622556016640, 140622564409343, -STORE, 140622547628032, 140622556016639, -SNULL, 140622556020735, 140622564409343, -STORE, 140622556016640, 140622556020735, -STORE, 140622556020736, 140622564409343, -STORE, 140622430195712, 140622446977023, -STORE, 140622421798912, 140622430191615, -SNULL, 140622430195712, 140622438584319, -STORE, 140622438584320, 140622446977023, -STORE, 140622430195712, 140622438584319, -SNULL, 140622438588415, 140622446977023, -STORE, 140622438584320, 140622438588415, -STORE, 140622438588416, 140622446977023, -STORE, 140622413406208, 140622430191615, -STORE, 140622405013504, 140622430191615, -SNULL, 140622405013504, 140622413406207, -STORE, 140622413406208, 140622430191615, -STORE, 140622405013504, 140622413406207, -SNULL, 140622413410303, 140622430191615, -STORE, 140622413406208, 140622413410303, -STORE, 140622413410304, 140622430191615, -SNULL, 140622673453056, 140622681841663, -STORE, 140622681841664, 140622690234367, -STORE, 140622673453056, 140622681841663, -SNULL, 140622681845759, 140622690234367, -STORE, 140622681841664, 140622681845759, -STORE, 140622681845760, 140622690234367, -STORE, 140622321152000, 140622329544703, -SNULL, 140622413410304, 140622421798911, -STORE, 140622421798912, 140622430191615, -STORE, 140622413410304, 140622421798911, -SNULL, 140622421803007, 140622430191615, -STORE, 140622421798912, 140622421803007, -STORE, 140622421803008, 140622430191615, -STORE, 140622312759296, 140622329544703, -SNULL, 140622312763391, 140622329544703, -STORE, 140622312759296, 140622312763391, -STORE, 140622312763392, 140622329544703, -SNULL, 140622405017599, 140622413406207, -STORE, 140622405013504, 140622405017599, -STORE, 140622405017600, 140622413406207, -STORE, 140622304366592, 140622312759295, -SNULL, 140622304370687, 140622312759295, -STORE, 140622304366592, 140622304370687, -STORE, 140622304370688, 140622312759295, -SNULL, 140622312763392, 140622321151999, -STORE, 140622321152000, 140622329544703, -STORE, 140622312763392, 140622321151999, -SNULL, 140622321156095, 140622329544703, -STORE, 140622321152000, 140622321156095, -STORE, 140622321156096, 140622329544703, -STORE, 140624062619648, 140624062648319, -STORE, 140624010240000, 140624012431359, -SNULL, 140624010240000, 140624010330111, -STORE, 140624010330112, 140624012431359, -STORE, 140624010240000, 140624010330111, -SNULL, 140624012423167, 140624012431359, -STORE, 140624010330112, 140624012423167, -STORE, 140624012423168, 140624012431359, -ERASE, 140624012423168, 140624012431359, -STORE, 140624012423168, 140624012431359, -SNULL, 140624012427263, 140624012431359, -STORE, 140624012423168, 140624012427263, -STORE, 140624012427264, 140624012431359, -ERASE, 140624062619648, 140624062648319, -ERASE, 140622849630208, 140622849634303, -ERASE, 140622849634304, 140622858022911, -ERASE, 140623394893824, 140623394897919, -ERASE, 140623394897920, 140623403286527, -ERASE, 140623361323008, 140623361327103, -ERASE, 140623361327104, 140623369715711, -ERASE, 140623084494848, 140623084498943, -ERASE, 140623084498944, 140623092887551, -ERASE, 140623931764736, 140623931768831, -ERASE, 140623931768832, 140623940157439, -ERASE, 140622841237504, 140622841241599, -ERASE, 140622841241600, 140622849630207, -ERASE, 140623487148032, 140623487152127, -ERASE, 140623487152128, 140623495540735, -ERASE, 140623109672960, 140623109677055, -ERASE, 140623109677056, 140623118065663, -ERASE, 140622983847936, 140622983852031, -ERASE, 140622983852032, 140622992240639, -ERASE, 140623352930304, 140623352934399, -ERASE, 140623352934400, 140623361323007, -ERASE, 140622564409344, 140622564413439, -ERASE, 140622564413440, 140622572802047, -ERASE, 140622430191616, 140622430195711, -ERASE, 140622430195712, 140622438584319, -ERASE, 140622958669824, 140622958673919, -ERASE, 140622958673920, 140622967062527, -ERASE, 140622992240640, 140622992244735, -ERASE, 140622992244736, 140623000633343, -ERASE, 140623227105280, 140623227109375, -ERASE, 140623227109376, 140623235497983, -ERASE, 140622321152000, 140622321156095, -ERASE, 140622321156096, 140622329544703, -ERASE, 140622858022912, 140622858027007, -ERASE, 140622858027008, 140622866415615, -ERASE, 140622975455232, 140622975459327, -ERASE, 140622975459328, 140622983847935, -ERASE, 140623378108416, 140623378112511, -ERASE, 140623378112512, 140623386501119, -ERASE, 140623495540736, 140623495544831, -ERASE, 140623495544832, 140623503933439, -ERASE, 140623118065664, 140623118069759, -ERASE, 140623118069760, 140623126458367, -ERASE, 140622572802048, 140622572806143, -ERASE, 140622572806144, 140622581194751, -ERASE, 140622421798912, 140622421803007, -ERASE, 140622421803008, 140622430191615, -ERASE, 140622967062528, 140622967066623, -ERASE, 140622967066624, 140622975455231, -ERASE, 140623252283392, 140623252287487, -ERASE, 140623252287488, 140623260676095, -ERASE, 140622673448960, 140622673453055, -ERASE, 140622673453056, 140622681841663, -ERASE, 140623076102144, 140623076106239, -ERASE, 140623076106240, 140623084494847, -ERASE, 140623101280256, 140623101284351, -ERASE, 140623101284352, 140623109672959, -ERASE, 140622715412480, 140622715416575, -ERASE, 140622715416576, 140622723805183, -ERASE, 140622405013504, 140622405017599, -ERASE, 140622405017600, 140622413406207, -ERASE, 140623478755328, 140623478759423, -ERASE, 140623478759424, 140623487148031, -ERASE, 140623906586624, 140623906590719, -ERASE, 140623906590720, 140623914979327, -ERASE, 140622950277120, 140622950281215, -ERASE, 140622950281216, 140622958669823, - }; - unsigned long set32[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140731244212224, 140737488351231, -SNULL, 140731244216319, 140737488351231, -STORE, 140731244212224, 140731244216319, -STORE, 140731244081152, 140731244216319, -STORE, 94427773984768, 94427776237567, -SNULL, 94427774115839, 94427776237567, -STORE, 94427773984768, 94427774115839, -STORE, 94427774115840, 94427776237567, -ERASE, 94427774115840, 94427776237567, -STORE, 94427776208896, 94427776217087, -STORE, 94427776217088, 94427776237567, -STORE, 140401464893440, 140401467146239, -SNULL, 140401465036799, 140401467146239, -STORE, 140401464893440, 140401465036799, -STORE, 140401465036800, 140401467146239, -ERASE, 140401465036800, 140401467146239, -STORE, 140401467133952, 140401467142143, -STORE, 140401467142144, 140401467146239, -STORE, 140731244507136, 140731244511231, -STORE, 140731244494848, 140731244507135, -STORE, 140401467105280, 140401467133951, -STORE, 140401467097088, 140401467105279, -STORE, 140401462677504, 140401464893439, -SNULL, 140401462677504, 140401462775807, -STORE, 140401462775808, 140401464893439, -STORE, 140401462677504, 140401462775807, -SNULL, 140401464868863, 140401464893439, -STORE, 140401462775808, 140401464868863, -STORE, 140401464868864, 140401464893439, -SNULL, 140401464868864, 140401464877055, -STORE, 140401464877056, 140401464893439, -STORE, 140401464868864, 140401464877055, -ERASE, 140401464868864, 140401464877055, -STORE, 140401464868864, 140401464877055, -ERASE, 140401464877056, 140401464893439, -STORE, 140401464877056, 140401464893439, -STORE, 140401458880512, 140401462677503, -SNULL, 140401458880512, 140401460539391, -STORE, 140401460539392, 140401462677503, -STORE, 140401458880512, 140401460539391, -SNULL, 140401462636543, 140401462677503, -STORE, 140401460539392, 140401462636543, -STORE, 140401462636544, 140401462677503, -SNULL, 140401462636544, 140401462661119, -STORE, 140401462661120, 140401462677503, -STORE, 140401462636544, 140401462661119, -ERASE, 140401462636544, 140401462661119, -STORE, 140401462636544, 140401462661119, -ERASE, 140401462661120, 140401462677503, -STORE, 140401462661120, 140401462677503, -STORE, 140401467088896, 140401467105279, -SNULL, 140401462652927, 140401462661119, -STORE, 140401462636544, 140401462652927, -STORE, 140401462652928, 140401462661119, -SNULL, 140401464872959, 140401464877055, -STORE, 140401464868864, 140401464872959, -STORE, 140401464872960, 140401464877055, -SNULL, 94427776212991, 94427776217087, -STORE, 94427776208896, 94427776212991, -STORE, 94427776212992, 94427776217087, -SNULL, 140401467138047, 140401467142143, -STORE, 140401467133952, 140401467138047, -STORE, 140401467138048, 140401467142143, -ERASE, 140401467105280, 140401467133951, -STORE, 94427784683520, 94427784818687, -STORE, 140401450487808, 140401458880511, -SNULL, 140401450491903, 140401458880511, -STORE, 140401450487808, 140401450491903, -STORE, 140401450491904, 140401458880511, -STORE, 140401442095104, 140401450487807, -STORE, 140401307877376, 140401442095103, -SNULL, 140401307877376, 140401340055551, -STORE, 140401340055552, 140401442095103, -STORE, 140401307877376, 140401340055551, -ERASE, 140401307877376, 140401340055551, -SNULL, 140401407164415, 140401442095103, -STORE, 140401340055552, 140401407164415, -STORE, 140401407164416, 140401442095103, -ERASE, 140401407164416, 140401442095103, -SNULL, 140401340190719, 140401407164415, -STORE, 140401340055552, 140401340190719, -STORE, 140401340190720, 140401407164415, -SNULL, 140401442099199, 140401450487807, -STORE, 140401442095104, 140401442099199, -STORE, 140401442099200, 140401450487807, -STORE, 140401433702400, 140401442095103, -SNULL, 140401433706495, 140401442095103, -STORE, 140401433702400, 140401433706495, -STORE, 140401433706496, 140401442095103, -STORE, 140401425309696, 140401433702399, -SNULL, 140401425313791, 140401433702399, -STORE, 140401425309696, 140401425313791, -STORE, 140401425313792, 140401433702399, -STORE, 140401416916992, 140401425309695, -SNULL, 140401416921087, 140401425309695, -STORE, 140401416916992, 140401416921087, -STORE, 140401416921088, 140401425309695, -STORE, 140401408524288, 140401416916991, -STORE, 140401205837824, 140401340055551, -SNULL, 140401272946687, 140401340055551, -STORE, 140401205837824, 140401272946687, -STORE, 140401272946688, 140401340055551, -ERASE, 140401272946688, 140401340055551, -SNULL, 140401205972991, 140401272946687, -STORE, 140401205837824, 140401205972991, -STORE, 140401205972992, 140401272946687, -STORE, 140401331662848, 140401340055551, -STORE, 140401323270144, 140401340055551, -STORE, 140401138728960, 140401205837823, -STORE, 140401314877440, 140401340055551, -SNULL, 140401408528383, 140401416916991, -STORE, 140401408524288, 140401408528383, -STORE, 140401408528384, 140401416916991, -SNULL, 140401138864127, 140401205837823, -STORE, 140401138728960, 140401138864127, -STORE, 140401138864128, 140401205837823, -STORE, 140401004511232, 140401138728959, -SNULL, 140401071620095, 140401138728959, -STORE, 140401004511232, 140401071620095, -STORE, 140401071620096, 140401138728959, -ERASE, 140401071620096, 140401138728959, -STORE, 140400870293504, 140401071620095, -SNULL, 140400937402367, 140401071620095, -STORE, 140400870293504, 140400937402367, -STORE, 140400937402368, 140401071620095, -SNULL, 140400937402368, 140401004511231, -STORE, 140401004511232, 140401071620095, -STORE, 140400937402368, 140401004511231, -ERASE, 140400937402368, 140401004511231, -STORE, 140401306484736, 140401340055551, -SNULL, 140401306484736, 140401323270143, -STORE, 140401323270144, 140401340055551, -STORE, 140401306484736, 140401323270143, -SNULL, 140401323274239, 140401340055551, -STORE, 140401323270144, 140401323274239, -STORE, 140401323274240, 140401340055551, -SNULL, 140401004646399, 140401071620095, -STORE, 140401004511232, 140401004646399, -STORE, 140401004646400, 140401071620095, -SNULL, 140400870428671, 140400937402367, -STORE, 140400870293504, 140400870428671, -STORE, 140400870428672, 140400937402367, -SNULL, 140401306488831, 140401323270143, -STORE, 140401306484736, 140401306488831, -STORE, 140401306488832, 140401323270143, -STORE, 140401298092032, 140401306484735, -SNULL, 140401306488832, 140401314877439, -STORE, 140401314877440, 140401323270143, -STORE, 140401306488832, 140401314877439, -SNULL, 140401314881535, 140401323270143, -STORE, 140401314877440, 140401314881535, -STORE, 140401314881536, 140401323270143, -SNULL, 140401323274240, 140401331662847, -STORE, 140401331662848, 140401340055551, -STORE, 140401323274240, 140401331662847, -SNULL, 140401331666943, 140401340055551, -STORE, 140401331662848, 140401331666943, -STORE, 140401331666944, 140401340055551, -SNULL, 140401298096127, 140401306484735, -STORE, 140401298092032, 140401298096127, -STORE, 140401298096128, 140401306484735, -STORE, 140401289699328, 140401298092031, -STORE, 140401281306624, 140401298092031, -STORE, 140401130336256, 140401138728959, -SNULL, 140401281306624, 140401289699327, -STORE, 140401289699328, 140401298092031, -STORE, 140401281306624, 140401289699327, -SNULL, 140401289703423, 140401298092031, -STORE, 140401289699328, 140401289703423, -STORE, 140401289703424, 140401298092031, -STORE, 140401121943552, 140401138728959, -STORE, 140401113550848, 140401138728959, -SNULL, 140401281310719, 140401289699327, -STORE, 140401281306624, 140401281310719, -STORE, 140401281310720, 140401289699327, -SNULL, 140401113550848, 140401121943551, -STORE, 140401121943552, 140401138728959, -STORE, 140401113550848, 140401121943551, -SNULL, 140401121947647, 140401138728959, -STORE, 140401121943552, 140401121947647, -STORE, 140401121947648, 140401138728959, -STORE, 140401105158144, 140401121943551, -SNULL, 140401121947648, 140401130336255, -STORE, 140401130336256, 140401138728959, -STORE, 140401121947648, 140401130336255, -SNULL, 140401130340351, 140401138728959, -STORE, 140401130336256, 140401130340351, -STORE, 140401130340352, 140401138728959, -STORE, 140401096765440, 140401121943551, -SNULL, 140401096765440, 140401113550847, -STORE, 140401113550848, 140401121943551, -STORE, 140401096765440, 140401113550847, -SNULL, 140401113554943, 140401121943551, -STORE, 140401113550848, 140401113554943, -STORE, 140401113554944, 140401121943551, -STORE, 140401088372736, 140401113550847, -SNULL, 140401088372736, 140401096765439, -STORE, 140401096765440, 140401113550847, -STORE, 140401088372736, 140401096765439, -SNULL, 140401096769535, 140401113550847, -STORE, 140401096765440, 140401096769535, -STORE, 140401096769536, 140401113550847, -SNULL, 140401096769536, 140401105158143, -STORE, 140401105158144, 140401113550847, -STORE, 140401096769536, 140401105158143, -SNULL, 140401105162239, 140401113550847, -STORE, 140401105158144, 140401105162239, -STORE, 140401105162240, 140401113550847, -SNULL, 140401088376831, 140401096765439, -STORE, 140401088372736, 140401088376831, -STORE, 140401088376832, 140401096765439, -STORE, 140401079980032, 140401088372735, -STORE, 140400996118528, 140401004511231, -SNULL, 140401079984127, 140401088372735, -STORE, 140401079980032, 140401079984127, -STORE, 140401079984128, 140401088372735, -SNULL, 140400996122623, 140401004511231, -STORE, 140400996118528, 140400996122623, -STORE, 140400996122624, 140401004511231, -STORE, 140400987725824, 140400996118527, -STORE, 140400979333120, 140400996118527, -STORE, 140400803184640, 140400870293503, -SNULL, 140400803319807, 140400870293503, -STORE, 140400803184640, 140400803319807, -STORE, 140400803319808, 140400870293503, -SNULL, 140400979333120, 140400987725823, -STORE, 140400987725824, 140400996118527, -STORE, 140400979333120, 140400987725823, -SNULL, 140400987729919, 140400996118527, -STORE, 140400987725824, 140400987729919, -STORE, 140400987729920, 140400996118527, -STORE, 140400970940416, 140400987725823, -STORE, 140400962547712, 140400987725823, -STORE, 140400668966912, 140400803184639, -STORE, 140400954155008, 140400987725823, -STORE, 140400945762304, 140400987725823, -STORE, 140400660574208, 140400668966911, -STORE, 140400593465344, 140400660574207, -STORE, 140400585072640, 140400593465343, -STORE, 140400450854912, 140400585072639, -STORE, 140400442462208, 140400450854911, -STORE, 140400434069504, 140400450854911, -STORE, 140400299851776, 140400434069503, -STORE, 140400291459072, 140400299851775, -SNULL, 140400299851776, 140400333422591, -STORE, 140400333422592, 140400434069503, -STORE, 140400299851776, 140400333422591, -ERASE, 140400299851776, 140400333422591, -STORE, 140400325029888, 140400333422591, -STORE, 140400157241344, 140400291459071, -STORE, 140400316637184, 140400333422591, -STORE, 140400308244480, 140400333422591, -STORE, 140400023023616, 140400291459071, -STORE, 140400291459072, 140400333422591, -SNULL, 140400023023616, 140400064987135, -STORE, 140400064987136, 140400291459071, -STORE, 140400023023616, 140400064987135, -ERASE, 140400023023616, 140400064987135, -STORE, 140400056594432, 140400064987135, -SNULL, 140400056598527, 140400064987135, -STORE, 140400056594432, 140400056598527, -STORE, 140400056598528, 140400064987135, -STORE, 140399989485568, 140400056594431, -SNULL, 140400291459072, 140400316637183, -STORE, 140400316637184, 140400333422591, -STORE, 140400291459072, 140400316637183, -SNULL, 140400316641279, 140400333422591, -STORE, 140400316637184, 140400316641279, -STORE, 140400316641280, 140400333422591, -STORE, 140399855267840, 140400056594431, -SNULL, 140399855267840, 140399863660543, -STORE, 140399863660544, 140400056594431, -STORE, 140399855267840, 140399863660543, -ERASE, 140399855267840, 140399863660543, -SNULL, 140400736075775, 140400803184639, -STORE, 140400668966912, 140400736075775, -STORE, 140400736075776, 140400803184639, -ERASE, 140400736075776, 140400803184639, -SNULL, 140400669102079, 140400736075775, -STORE, 140400668966912, 140400669102079, -STORE, 140400669102080, 140400736075775, -STORE, 140400669102080, 140400803184639, -SNULL, 140400669102080, 140400736075775, -STORE, 140400736075776, 140400803184639, -STORE, 140400669102080, 140400736075775, -SNULL, 140400736210943, 140400803184639, -STORE, 140400736075776, 140400736210943, -STORE, 140400736210944, 140400803184639, -ERASE, 140400593465344, 140400660574207, -SNULL, 140400450854912, 140400467640319, -STORE, 140400467640320, 140400585072639, -STORE, 140400450854912, 140400467640319, -ERASE, 140400450854912, 140400467640319, -STORE, 140399729442816, 140400056594431, -SNULL, 140400400531455, 140400434069503, -STORE, 140400333422592, 140400400531455, -STORE, 140400400531456, 140400434069503, -ERASE, 140400400531456, 140400434069503, -SNULL, 140400333557759, 140400400531455, -STORE, 140400333422592, 140400333557759, -STORE, 140400333557760, 140400400531455, -SNULL, 140400157241343, 140400291459071, -STORE, 140400064987136, 140400157241343, -STORE, 140400157241344, 140400291459071, -SNULL, 140400157241344, 140400199204863, -STORE, 140400199204864, 140400291459071, -STORE, 140400157241344, 140400199204863, -ERASE, 140400157241344, 140400199204863, -SNULL, 140400266313727, 140400291459071, -STORE, 140400199204864, 140400266313727, -STORE, 140400266313728, 140400291459071, -ERASE, 140400266313728, 140400291459071, -SNULL, 140400132095999, 140400157241343, -STORE, 140400064987136, 140400132095999, -STORE, 140400132096000, 140400157241343, -ERASE, 140400132096000, 140400157241343, -SNULL, 140400065122303, 140400132095999, -STORE, 140400064987136, 140400065122303, -STORE, 140400065122304, 140400132095999, -SNULL, 140400945762304, 140400954155007, -STORE, 140400954155008, 140400987725823, -STORE, 140400945762304, 140400954155007, -SNULL, 140400954159103, 140400987725823, -STORE, 140400954155008, 140400954159103, -STORE, 140400954159104, 140400987725823, -SNULL, 140400434069504, 140400442462207, -STORE, 140400442462208, 140400450854911, -STORE, 140400434069504, 140400442462207, -SNULL, 140400442466303, 140400450854911, -STORE, 140400442462208, 140400442466303, -STORE, 140400442466304, 140400450854911, -SNULL, 140400291463167, 140400316637183, -STORE, 140400291459072, 140400291463167, -STORE, 140400291463168, 140400316637183, -STORE, 140400652181504, 140400668966911, -STORE, 140400643788800, 140400668966911, -SNULL, 140400291463168, 140400299851775, -STORE, 140400299851776, 140400316637183, -STORE, 140400291463168, 140400299851775, -SNULL, 140400299855871, 140400316637183, -STORE, 140400299851776, 140400299855871, -STORE, 140400299855872, 140400316637183, -STORE, 140400635396096, 140400668966911, -SNULL, 140400635396096, 140400643788799, -STORE, 140400643788800, 140400668966911, -STORE, 140400635396096, 140400643788799, -SNULL, 140400643792895, 140400668966911, -STORE, 140400643788800, 140400643792895, -STORE, 140400643792896, 140400668966911, -SNULL, 140399989485567, 140400056594431, -STORE, 140399729442816, 140399989485567, -STORE, 140399989485568, 140400056594431, -ERASE, 140399989485568, 140400056594431, -SNULL, 140399930769407, 140399989485567, -STORE, 140399729442816, 140399930769407, -STORE, 140399930769408, 140399989485567, -ERASE, 140399930769408, 140399989485567, -SNULL, 140400945766399, 140400954155007, -STORE, 140400945762304, 140400945766399, -STORE, 140400945766400, 140400954155007, -SNULL, 140400534749183, 140400585072639, -STORE, 140400467640320, 140400534749183, -STORE, 140400534749184, 140400585072639, -ERASE, 140400534749184, 140400585072639, -SNULL, 140399796551679, 140399930769407, -STORE, 140399729442816, 140399796551679, -STORE, 140399796551680, 140399930769407, -SNULL, 140399796551680, 140399863660543, -STORE, 140399863660544, 140399930769407, -STORE, 140399796551680, 140399863660543, -ERASE, 140399796551680, 140399863660543, -SNULL, 140400199340031, 140400266313727, -STORE, 140400199204864, 140400199340031, -STORE, 140400199340032, 140400266313727, -STORE, 140400627003392, 140400643788799, -SNULL, 140400316641280, 140400325029887, -STORE, 140400325029888, 140400333422591, -STORE, 140400316641280, 140400325029887, -SNULL, 140400325033983, 140400333422591, -STORE, 140400325029888, 140400325033983, -STORE, 140400325033984, 140400333422591, -SNULL, 140400627003392, 140400635396095, -STORE, 140400635396096, 140400643788799, -STORE, 140400627003392, 140400635396095, -SNULL, 140400635400191, 140400643788799, -STORE, 140400635396096, 140400635400191, -STORE, 140400635400192, 140400643788799, -SNULL, 140400434073599, 140400442462207, -STORE, 140400434069504, 140400434073599, -STORE, 140400434073600, 140400442462207, -STORE, 140400618610688, 140400635396095, -STORE, 140400610217984, 140400635396095, -SNULL, 140400954159104, 140400962547711, -STORE, 140400962547712, 140400987725823, -STORE, 140400954159104, 140400962547711, -SNULL, 140400962551807, 140400987725823, -STORE, 140400962547712, 140400962551807, -STORE, 140400962551808, 140400987725823, -SNULL, 140400299855872, 140400308244479, -STORE, 140400308244480, 140400316637183, -STORE, 140400299855872, 140400308244479, -SNULL, 140400308248575, 140400316637183, -STORE, 140400308244480, 140400308248575, -STORE, 140400308248576, 140400316637183, -STORE, 140400601825280, 140400635396095, -SNULL, 140400601829375, 140400635396095, -STORE, 140400601825280, 140400601829375, -STORE, 140400601829376, 140400635396095, -STORE, 140400576679936, 140400593465343, -SNULL, 140400576684031, 140400593465343, -STORE, 140400576679936, 140400576684031, -STORE, 140400576684032, 140400593465343, -SNULL, 140400643792896, 140400652181503, -STORE, 140400652181504, 140400668966911, -STORE, 140400643792896, 140400652181503, -SNULL, 140400652185599, 140400668966911, -STORE, 140400652181504, 140400652185599, -STORE, 140400652185600, 140400668966911, -STORE, 140399595225088, 140399796551679, -SNULL, 140399662333951, 140399796551679, -STORE, 140399595225088, 140399662333951, -STORE, 140399662333952, 140399796551679, -SNULL, 140399662333952, 140399729442815, -STORE, 140399729442816, 140399796551679, -STORE, 140399662333952, 140399729442815, -ERASE, 140399662333952, 140399729442815, -SNULL, 140399863795711, 140399930769407, -STORE, 140399863660544, 140399863795711, -STORE, 140399863795712, 140399930769407, -STORE, 140400568287232, 140400576679935, -SNULL, 140400568291327, 140400576679935, -STORE, 140400568287232, 140400568291327, -STORE, 140400568291328, 140400576679935, -SNULL, 140400467775487, 140400534749183, -STORE, 140400467640320, 140400467775487, -STORE, 140400467775488, 140400534749183, -SNULL, 140399729577983, 140399796551679, -STORE, 140399729442816, 140399729577983, -STORE, 140399729577984, 140399796551679, -SNULL, 140400601829376, 140400627003391, -STORE, 140400627003392, 140400635396095, -STORE, 140400601829376, 140400627003391, -SNULL, 140400627007487, 140400635396095, -STORE, 140400627003392, 140400627007487, -STORE, 140400627007488, 140400635396095, -STORE, 140400559894528, 140400568287231, -STORE, 140400551501824, 140400568287231, -STORE, 140400543109120, 140400568287231, -STORE, 140400459247616, 140400467640319, -STORE, 140400442466304, 140400467640319, -SNULL, 140399595360255, 140399662333951, -STORE, 140399595225088, 140399595360255, -STORE, 140399595360256, 140399662333951, -SNULL, 140400962551808, 140400970940415, -STORE, 140400970940416, 140400987725823, -STORE, 140400962551808, 140400970940415, -SNULL, 140400970944511, 140400987725823, -STORE, 140400970940416, 140400970944511, -STORE, 140400970944512, 140400987725823, -SNULL, 140400652185600, 140400660574207, -STORE, 140400660574208, 140400668966911, -STORE, 140400652185600, 140400660574207, -SNULL, 140400660578303, 140400668966911, -STORE, 140400660574208, 140400660578303, -STORE, 140400660578304, 140400668966911, -SNULL, 140400576684032, 140400585072639, -STORE, 140400585072640, 140400593465343, -STORE, 140400576684032, 140400585072639, -SNULL, 140400585076735, 140400593465343, -STORE, 140400585072640, 140400585076735, -STORE, 140400585076736, 140400593465343, -STORE, 140400425676800, 140400434069503, -STORE, 140400417284096, 140400434069503, -STORE, 140400408891392, 140400434069503, -SNULL, 140400408891392, 140400417284095, -STORE, 140400417284096, 140400434069503, -STORE, 140400408891392, 140400417284095, -SNULL, 140400417288191, 140400434069503, -STORE, 140400417284096, 140400417288191, -STORE, 140400417288192, 140400434069503, -STORE, 140400283066368, 140400291459071, -SNULL, 140400601829376, 140400618610687, -STORE, 140400618610688, 140400627003391, -STORE, 140400601829376, 140400618610687, -SNULL, 140400618614783, 140400627003391, -STORE, 140400618610688, 140400618614783, -STORE, 140400618614784, 140400627003391, -SNULL, 140400601829376, 140400610217983, -STORE, 140400610217984, 140400618610687, -STORE, 140400601829376, 140400610217983, -SNULL, 140400610222079, 140400618610687, -STORE, 140400610217984, 140400610222079, -STORE, 140400610222080, 140400618610687, -STORE, 140400274673664, 140400291459071, -STORE, 140400190812160, 140400199204863, -STORE, 140400182419456, 140400199204863, -SNULL, 140400442466304, 140400450854911, -STORE, 140400450854912, 140400467640319, -STORE, 140400442466304, 140400450854911, -SNULL, 140400450859007, 140400467640319, -STORE, 140400450854912, 140400450859007, -STORE, 140400450859008, 140400467640319, -SNULL, 140400543109120, 140400559894527, -STORE, 140400559894528, 140400568287231, -STORE, 140400543109120, 140400559894527, -SNULL, 140400559898623, 140400568287231, -STORE, 140400559894528, 140400559898623, -STORE, 140400559898624, 140400568287231, -SNULL, 140400450859008, 140400459247615, -STORE, 140400459247616, 140400467640319, -STORE, 140400450859008, 140400459247615, -SNULL, 140400459251711, 140400467640319, -STORE, 140400459247616, 140400459251711, -STORE, 140400459251712, 140400467640319, -SNULL, 140400543113215, 140400559894527, -STORE, 140400543109120, 140400543113215, -STORE, 140400543113216, 140400559894527, -SNULL, 140400970944512, 140400979333119, -STORE, 140400979333120, 140400987725823, -STORE, 140400970944512, 140400979333119, -SNULL, 140400979337215, 140400987725823, -STORE, 140400979333120, 140400979337215, -STORE, 140400979337216, 140400987725823, -STORE, 140400174026752, 140400199204863, -SNULL, 140400174030847, 140400199204863, -STORE, 140400174026752, 140400174030847, -STORE, 140400174030848, 140400199204863, -SNULL, 140400274673664, 140400283066367, -STORE, 140400283066368, 140400291459071, -STORE, 140400274673664, 140400283066367, -SNULL, 140400283070463, 140400291459071, -STORE, 140400283066368, 140400283070463, -STORE, 140400283070464, 140400291459071, -STORE, 140400165634048, 140400174026751, -SNULL, 140400165638143, 140400174026751, -STORE, 140400165634048, 140400165638143, -STORE, 140400165638144, 140400174026751, -SNULL, 140400174030848, 140400182419455, -STORE, 140400182419456, 140400199204863, -STORE, 140400174030848, 140400182419455, -SNULL, 140400182423551, 140400199204863, -STORE, 140400182419456, 140400182423551, -STORE, 140400182423552, 140400199204863, -SNULL, 140400182423552, 140400190812159, -STORE, 140400190812160, 140400199204863, -STORE, 140400182423552, 140400190812159, -SNULL, 140400190816255, 140400199204863, -STORE, 140400190812160, 140400190816255, -STORE, 140400190816256, 140400199204863, -STORE, 140400157241344, 140400165634047, -SNULL, 140400157245439, 140400165634047, -STORE, 140400157241344, 140400157245439, -STORE, 140400157245440, 140400165634047, -SNULL, 140400408895487, 140400417284095, -STORE, 140400408891392, 140400408895487, -STORE, 140400408895488, 140400417284095, -SNULL, 140400417288192, 140400425676799, -STORE, 140400425676800, 140400434069503, -STORE, 140400417288192, 140400425676799, -SNULL, 140400425680895, 140400434069503, -STORE, 140400425676800, 140400425680895, -STORE, 140400425680896, 140400434069503, -STORE, 140400148848640, 140400157241343, -SNULL, 140400148852735, 140400157241343, -STORE, 140400148848640, 140400148852735, -STORE, 140400148852736, 140400157241343, -SNULL, 140400543113216, 140400551501823, -STORE, 140400551501824, 140400559894527, -STORE, 140400543113216, 140400551501823, -SNULL, 140400551505919, 140400559894527, -STORE, 140400551501824, 140400551505919, -STORE, 140400551505920, 140400559894527, -STORE, 140400140455936, 140400148848639, -STORE, 140400048201728, 140400056594431, -SNULL, 140400140460031, 140400148848639, -STORE, 140400140455936, 140400140460031, -STORE, 140400140460032, 140400148848639, -STORE, 140400039809024, 140400056594431, -SNULL, 140400039813119, 140400056594431, -STORE, 140400039809024, 140400039813119, -STORE, 140400039813120, 140400056594431, -STORE, 140400031416320, 140400039809023, -STORE, 140400023023616, 140400039809023, -SNULL, 140400274677759, 140400283066367, -STORE, 140400274673664, 140400274677759, -STORE, 140400274677760, 140400283066367, -STORE, 140400014630912, 140400039809023, -STORE, 140400006238208, 140400039809023, -STORE, 140399997845504, 140400039809023, -SNULL, 140399997849599, 140400039809023, -STORE, 140399997845504, 140399997849599, -STORE, 140399997849600, 140400039809023, -STORE, 140399989452800, 140399997845503, -SNULL, 140399989456895, 140399997845503, -STORE, 140399989452800, 140399989456895, -STORE, 140399989456896, 140399997845503, -STORE, 140399981060096, 140399989452799, -SNULL, 140399981064191, 140399989452799, -STORE, 140399981060096, 140399981064191, -STORE, 140399981064192, 140399989452799, -STORE, 140399972667392, 140399981060095, -STORE, 140399964274688, 140399981060095, -SNULL, 140399964278783, 140399981060095, -STORE, 140399964274688, 140399964278783, -STORE, 140399964278784, 140399981060095, -SNULL, 140400039813120, 140400048201727, -STORE, 140400048201728, 140400056594431, -STORE, 140400039813120, 140400048201727, -SNULL, 140400048205823, 140400056594431, -STORE, 140400048201728, 140400048205823, -STORE, 140400048205824, 140400056594431, -SNULL, 140399997849600, 140400031416319, -STORE, 140400031416320, 140400039809023, -STORE, 140399997849600, 140400031416319, -SNULL, 140400031420415, 140400039809023, -STORE, 140400031416320, 140400031420415, -STORE, 140400031420416, 140400039809023, -STORE, 140399955881984, 140399964274687, -SNULL, 140399955886079, 140399964274687, -STORE, 140399955881984, 140399955886079, -STORE, 140399955886080, 140399964274687, -STORE, 140399947489280, 140399955881983, -STORE, 140399939096576, 140399955881983, -STORE, 140399855267840, 140399863660543, -SNULL, 140399939100671, 140399955881983, -STORE, 140399939096576, 140399939100671, -STORE, 140399939100672, 140399955881983, -SNULL, 140399997849600, 140400014630911, -STORE, 140400014630912, 140400031416319, -STORE, 140399997849600, 140400014630911, -SNULL, 140400014635007, 140400031416319, -STORE, 140400014630912, 140400014635007, -STORE, 140400014635008, 140400031416319, -SNULL, 140400014635008, 140400023023615, -STORE, 140400023023616, 140400031416319, -STORE, 140400014635008, 140400023023615, -SNULL, 140400023027711, 140400031416319, -STORE, 140400023023616, 140400023027711, -STORE, 140400023027712, 140400031416319, -SNULL, 140399997849600, 140400006238207, -STORE, 140400006238208, 140400014630911, -STORE, 140399997849600, 140400006238207, -SNULL, 140400006242303, 140400014630911, -STORE, 140400006238208, 140400006242303, -STORE, 140400006242304, 140400014630911, -STORE, 140399846875136, 140399863660543, -STORE, 140399838482432, 140399863660543, -SNULL, 140399838486527, 140399863660543, -STORE, 140399838482432, 140399838486527, -STORE, 140399838486528, 140399863660543, -SNULL, 140399939100672, 140399947489279, -STORE, 140399947489280, 140399955881983, -STORE, 140399939100672, 140399947489279, -SNULL, 140399947493375, 140399955881983, -STORE, 140399947489280, 140399947493375, -STORE, 140399947493376, 140399955881983, -SNULL, 140399964278784, 140399972667391, -STORE, 140399972667392, 140399981060095, -STORE, 140399964278784, 140399972667391, -SNULL, 140399972671487, 140399981060095, -STORE, 140399972667392, 140399972671487, -STORE, 140399972671488, 140399981060095, -SNULL, 140399838486528, 140399855267839, -STORE, 140399855267840, 140399863660543, -STORE, 140399838486528, 140399855267839, -SNULL, 140399855271935, 140399863660543, -STORE, 140399855267840, 140399855271935, -STORE, 140399855271936, 140399863660543, -STORE, 140399830089728, 140399838482431, -SNULL, 140399830093823, 140399838482431, -STORE, 140399830089728, 140399830093823, -STORE, 140399830093824, 140399838482431, -STORE, 140399821697024, 140399830089727, -SNULL, 140399821701119, 140399830089727, -STORE, 140399821697024, 140399821701119, -STORE, 140399821701120, 140399830089727, -SNULL, 140399838486528, 140399846875135, -STORE, 140399846875136, 140399855267839, -STORE, 140399838486528, 140399846875135, -SNULL, 140399846879231, 140399855267839, -STORE, 140399846875136, 140399846879231, -STORE, 140399846879232, 140399855267839, -STORE, 140399813304320, 140399821697023, -STORE, 140399804911616, 140399821697023, -SNULL, 140399804915711, 140399821697023, -STORE, 140399804911616, 140399804915711, -STORE, 140399804915712, 140399821697023, -STORE, 140399721050112, 140399729442815, -SNULL, 140399804915712, 140399813304319, -STORE, 140399813304320, 140399821697023, -STORE, 140399804915712, 140399813304319, -SNULL, 140399813308415, 140399821697023, -STORE, 140399813304320, 140399813308415, -STORE, 140399813308416, 140399821697023, -SNULL, 140399721054207, 140399729442815, -STORE, 140399721050112, 140399721054207, -STORE, 140399721054208, 140399729442815, -STORE, 140401467105280, 140401467133951, -STORE, 140401279115264, 140401281306623, -SNULL, 140401279115264, 140401279205375, -STORE, 140401279205376, 140401281306623, -STORE, 140401279115264, 140401279205375, -SNULL, 140401281298431, 140401281306623, -STORE, 140401279205376, 140401281298431, -STORE, 140401281298432, 140401281306623, -ERASE, 140401281298432, 140401281306623, -STORE, 140401281298432, 140401281306623, -SNULL, 140401281302527, 140401281306623, -STORE, 140401281298432, 140401281302527, -STORE, 140401281302528, 140401281306623, -ERASE, 140401467105280, 140401467133951, -ERASE, 140400056594432, 140400056598527, -ERASE, 140400056598528, 140400064987135, -ERASE, 140400635396096, 140400635400191, -ERASE, 140400635400192, 140400643788799, -ERASE, 140400408891392, 140400408895487, -ERASE, 140400408895488, 140400417284095, -ERASE, 140400299851776, 140400299855871, -ERASE, 140400299855872, 140400308244479, -ERASE, 140400627003392, 140400627007487, -ERASE, 140400627007488, 140400635396095, -ERASE, 140400954155008, 140400954159103, -ERASE, 140400954159104, 140400962547711, -ERASE, 140400291459072, 140400291463167, -ERASE, 140400291463168, 140400299851775, -ERASE, 140400643788800, 140400643792895, -ERASE, 140400643792896, 140400652181503, -ERASE, 140400325029888, 140400325033983, -ERASE, 140400325033984, 140400333422591, -ERASE, 140400610217984, 140400610222079, -ERASE, 140400610222080, 140400618610687, -ERASE, 140400190812160, 140400190816255, -ERASE, 140400190816256, 140400199204863, -ERASE, 140399964274688, 140399964278783, -ERASE, 140399964278784, 140399972667391, -ERASE, 140400945762304, 140400945766399, -ERASE, 140400945766400, 140400954155007, -ERASE, 140400568287232, 140400568291327, -ERASE, 140400568291328, 140400576679935, -ERASE, 140399972667392, 140399972671487, -ERASE, 140399972671488, 140399981060095, -ERASE, 140400962547712, 140400962551807, -ERASE, 140400962551808, 140400970940415, -ERASE, 140400987725824, 140400987729919, -ERASE, 140400987729920, 140400996118527, -ERASE, 140400652181504, 140400652185599, -ERASE, 140400652185600, 140400660574207, -ERASE, 140400450854912, 140400450859007, -ERASE, 140400450859008, 140400459247615, -ERASE, 140400031416320, 140400031420415, -ERASE, 140400031420416, 140400039809023, -ERASE, 140400308244480, 140400308248575, -ERASE, 140400308248576, 140400316637183, -ERASE, 140400434069504, 140400434073599, -ERASE, 140400434073600, 140400442462207, -ERASE, 140400543109120, 140400543113215, -ERASE, 140400543113216, 140400551501823, -ERASE, 140400023023616, 140400023027711, -ERASE, 140400023027712, 140400031416319, -ERASE, 140399813304320, 140399813308415, -ERASE, 140399813308416, 140399821697023, -ERASE, 140400316637184, 140400316641279, -ERASE, 140400316641280, 140400325029887, -ERASE, 140400585072640, 140400585076735, -ERASE, 140400585076736, 140400593465343, -ERASE, 140400148848640, 140400148852735, -ERASE, 140400148852736, 140400157241343, -ERASE, 140399955881984, 140399955886079, -ERASE, 140399955886080, 140399964274687, -ERASE, 140399821697024, 140399821701119, -ERASE, 140399821701120, 140399830089727, -ERASE, 140400601825280, 140400601829375, -ERASE, 140400601829376, 140400610217983, -ERASE, 140400979333120, 140400979337215, -ERASE, 140400979337216, 140400987725823, -ERASE, 140399997845504, 140399997849599, -ERASE, 140399997849600, 140400006238207, -ERASE, 140400459247616, 140400459251711, -ERASE, 140400459251712, 140400467640319, -ERASE, 140400551501824, 140400551505919, -ERASE, 140400551505920, 140400559894527, -ERASE, 140399939096576, 140399939100671, -ERASE, 140399939100672, 140399947489279, -ERASE, 140400442462208, 140400442466303, -ERASE, 140400442466304, 140400450854911, -ERASE, 140400576679936, 140400576684031, -ERASE, 140400576684032, 140400585072639, -ERASE, 140400559894528, 140400559898623, -ERASE, 140400559898624, 140400568287231, -ERASE, 140400417284096, 140400417288191, -ERASE, 140400417288192, 140400425676799, -ERASE, 140400283066368, 140400283070463, -ERASE, 140400283070464, 140400291459071, - }; - unsigned long set33[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140734562918400, 140737488351231, -SNULL, 140734562922495, 140737488351231, -STORE, 140734562918400, 140734562922495, -STORE, 140734562787328, 140734562922495, -STORE, 94133878984704, 94133881237503, -SNULL, 94133879115775, 94133881237503, -STORE, 94133878984704, 94133879115775, -STORE, 94133879115776, 94133881237503, -ERASE, 94133879115776, 94133881237503, -STORE, 94133881208832, 94133881217023, -STORE, 94133881217024, 94133881237503, -STORE, 140583654043648, 140583656296447, -SNULL, 140583654187007, 140583656296447, -STORE, 140583654043648, 140583654187007, -STORE, 140583654187008, 140583656296447, -ERASE, 140583654187008, 140583656296447, -STORE, 140583656284160, 140583656292351, -STORE, 140583656292352, 140583656296447, -STORE, 140734564319232, 140734564323327, -STORE, 140734564306944, 140734564319231, -STORE, 140583656255488, 140583656284159, -STORE, 140583656247296, 140583656255487, -STORE, 140583651827712, 140583654043647, -SNULL, 140583651827712, 140583651926015, -STORE, 140583651926016, 140583654043647, -STORE, 140583651827712, 140583651926015, -SNULL, 140583654019071, 140583654043647, -STORE, 140583651926016, 140583654019071, -STORE, 140583654019072, 140583654043647, -SNULL, 140583654019072, 140583654027263, -STORE, 140583654027264, 140583654043647, -STORE, 140583654019072, 140583654027263, -ERASE, 140583654019072, 140583654027263, -STORE, 140583654019072, 140583654027263, -ERASE, 140583654027264, 140583654043647, -STORE, 140583654027264, 140583654043647, -STORE, 140583648030720, 140583651827711, -SNULL, 140583648030720, 140583649689599, -STORE, 140583649689600, 140583651827711, -STORE, 140583648030720, 140583649689599, -SNULL, 140583651786751, 140583651827711, -STORE, 140583649689600, 140583651786751, -STORE, 140583651786752, 140583651827711, -SNULL, 140583651786752, 140583651811327, -STORE, 140583651811328, 140583651827711, -STORE, 140583651786752, 140583651811327, -ERASE, 140583651786752, 140583651811327, -STORE, 140583651786752, 140583651811327, -ERASE, 140583651811328, 140583651827711, -STORE, 140583651811328, 140583651827711, -STORE, 140583656239104, 140583656255487, -SNULL, 140583651803135, 140583651811327, -STORE, 140583651786752, 140583651803135, -STORE, 140583651803136, 140583651811327, -SNULL, 140583654023167, 140583654027263, -STORE, 140583654019072, 140583654023167, -STORE, 140583654023168, 140583654027263, -SNULL, 94133881212927, 94133881217023, -STORE, 94133881208832, 94133881212927, -STORE, 94133881212928, 94133881217023, -SNULL, 140583656288255, 140583656292351, -STORE, 140583656284160, 140583656288255, -STORE, 140583656288256, 140583656292351, -ERASE, 140583656255488, 140583656284159, -STORE, 94133881733120, 94133881868287, -STORE, 140583639638016, 140583648030719, -SNULL, 140583639642111, 140583648030719, -STORE, 140583639638016, 140583639642111, -STORE, 140583639642112, 140583648030719, -STORE, 140583631245312, 140583639638015, -STORE, 140583497027584, 140583631245311, -SNULL, 140583497027584, 140583540621311, -STORE, 140583540621312, 140583631245311, -STORE, 140583497027584, 140583540621311, -ERASE, 140583497027584, 140583540621311, -SNULL, 140583607730175, 140583631245311, -STORE, 140583540621312, 140583607730175, -STORE, 140583607730176, 140583631245311, -ERASE, 140583607730176, 140583631245311, -SNULL, 140583540756479, 140583607730175, -STORE, 140583540621312, 140583540756479, -STORE, 140583540756480, 140583607730175, -SNULL, 140583631249407, 140583639638015, -STORE, 140583631245312, 140583631249407, -STORE, 140583631249408, 140583639638015, -STORE, 140583622852608, 140583631245311, -SNULL, 140583622856703, 140583631245311, -STORE, 140583622852608, 140583622856703, -STORE, 140583622856704, 140583631245311, -STORE, 140583614459904, 140583622852607, -SNULL, 140583614463999, 140583622852607, -STORE, 140583614459904, 140583614463999, -STORE, 140583614464000, 140583622852607, -STORE, 140583532228608, 140583540621311, -SNULL, 140583532232703, 140583540621311, -STORE, 140583532228608, 140583532232703, -STORE, 140583532232704, 140583540621311, -STORE, 140583523835904, 140583532228607, -STORE, 140583515443200, 140583532228607, -STORE, 140583507050496, 140583532228607, -STORE, 140583372832768, 140583507050495, -STORE, 140583364440064, 140583372832767, -STORE, 140583230222336, 140583364440063, -STORE, 140583096004608, 140583364440063, -SNULL, 140583230222335, 140583364440063, -STORE, 140583096004608, 140583230222335, -STORE, 140583230222336, 140583364440063, -SNULL, 140583230222336, 140583272185855, -STORE, 140583272185856, 140583364440063, -STORE, 140583230222336, 140583272185855, -ERASE, 140583230222336, 140583272185855, -STORE, 140582961786880, 140583230222335, -SNULL, 140583372832768, 140583406403583, -STORE, 140583406403584, 140583507050495, -STORE, 140583372832768, 140583406403583, -ERASE, 140583372832768, 140583406403583, -SNULL, 140583473512447, 140583507050495, -STORE, 140583406403584, 140583473512447, -STORE, 140583473512448, 140583507050495, -ERASE, 140583473512448, 140583507050495, -SNULL, 140583096004607, 140583230222335, -STORE, 140582961786880, 140583096004607, -STORE, 140583096004608, 140583230222335, -SNULL, 140583096004608, 140583137968127, -STORE, 140583137968128, 140583230222335, -STORE, 140583096004608, 140583137968127, -ERASE, 140583096004608, 140583137968127, -SNULL, 140583339294719, 140583364440063, -STORE, 140583272185856, 140583339294719, -STORE, 140583339294720, 140583364440063, -ERASE, 140583339294720, 140583364440063, -SNULL, 140583272321023, 140583339294719, -STORE, 140583272185856, 140583272321023, -STORE, 140583272321024, 140583339294719, -SNULL, 140582961786880, 140583003750399, -STORE, 140583003750400, 140583096004607, -STORE, 140582961786880, 140583003750399, -ERASE, 140582961786880, 140583003750399, - }; - - unsigned long set34[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140731327180800, 140737488351231, -SNULL, 140731327184895, 140737488351231, -STORE, 140731327180800, 140731327184895, -STORE, 140731327049728, 140731327184895, -STORE, 94632924487680, 94632926740479, -SNULL, 94632924618751, 94632926740479, -STORE, 94632924487680, 94632924618751, -STORE, 94632924618752, 94632926740479, -ERASE, 94632924618752, 94632926740479, -STORE, 94632926711808, 94632926719999, -STORE, 94632926720000, 94632926740479, -STORE, 140012544888832, 140012547141631, -SNULL, 140012545032191, 140012547141631, -STORE, 140012544888832, 140012545032191, -STORE, 140012545032192, 140012547141631, -ERASE, 140012545032192, 140012547141631, -STORE, 140012547129344, 140012547137535, -STORE, 140012547137536, 140012547141631, -STORE, 140731327725568, 140731327729663, -STORE, 140731327713280, 140731327725567, -STORE, 140012547100672, 140012547129343, -STORE, 140012547092480, 140012547100671, -STORE, 140012542672896, 140012544888831, -SNULL, 140012542672896, 140012542771199, -STORE, 140012542771200, 140012544888831, -STORE, 140012542672896, 140012542771199, -SNULL, 140012544864255, 140012544888831, -STORE, 140012542771200, 140012544864255, -STORE, 140012544864256, 140012544888831, -SNULL, 140012544864256, 140012544872447, -STORE, 140012544872448, 140012544888831, -STORE, 140012544864256, 140012544872447, -ERASE, 140012544864256, 140012544872447, -STORE, 140012544864256, 140012544872447, -ERASE, 140012544872448, 140012544888831, -STORE, 140012544872448, 140012544888831, -STORE, 140012538875904, 140012542672895, -SNULL, 140012538875904, 140012540534783, -STORE, 140012540534784, 140012542672895, -STORE, 140012538875904, 140012540534783, -SNULL, 140012542631935, 140012542672895, -STORE, 140012540534784, 140012542631935, -STORE, 140012542631936, 140012542672895, -SNULL, 140012542631936, 140012542656511, -STORE, 140012542656512, 140012542672895, -STORE, 140012542631936, 140012542656511, -ERASE, 140012542631936, 140012542656511, -STORE, 140012542631936, 140012542656511, -ERASE, 140012542656512, 140012542672895, -STORE, 140012542656512, 140012542672895, -STORE, 140012547084288, 140012547100671, -SNULL, 140012542648319, 140012542656511, -STORE, 140012542631936, 140012542648319, -STORE, 140012542648320, 140012542656511, -SNULL, 140012544868351, 140012544872447, -STORE, 140012544864256, 140012544868351, -STORE, 140012544868352, 140012544872447, -SNULL, 94632926715903, 94632926719999, -STORE, 94632926711808, 94632926715903, -STORE, 94632926715904, 94632926719999, -SNULL, 140012547133439, 140012547137535, -STORE, 140012547129344, 140012547133439, -STORE, 140012547133440, 140012547137535, -ERASE, 140012547100672, 140012547129343, -STORE, 94632939606016, 94632939741183, -STORE, 140012530483200, 140012538875903, -SNULL, 140012530487295, 140012538875903, -STORE, 140012530483200, 140012530487295, -STORE, 140012530487296, 140012538875903, -STORE, 140012522090496, 140012530483199, -STORE, 140012387872768, 140012522090495, -SNULL, 140012387872768, 140012444188671, -STORE, 140012444188672, 140012522090495, -STORE, 140012387872768, 140012444188671, -ERASE, 140012387872768, 140012444188671, -SNULL, 140012511297535, 140012522090495, -STORE, 140012444188672, 140012511297535, -STORE, 140012511297536, 140012522090495, -ERASE, 140012511297536, 140012522090495, -SNULL, 140012444323839, 140012511297535, -STORE, 140012444188672, 140012444323839, -STORE, 140012444323840, 140012511297535, -SNULL, 140012522094591, 140012530483199, -STORE, 140012522090496, 140012522094591, -STORE, 140012522094592, 140012530483199, -STORE, 140012513697792, 140012522090495, -SNULL, 140012513701887, 140012522090495, -STORE, 140012513697792, 140012513701887, -STORE, 140012513701888, 140012522090495, -STORE, 140012435795968, 140012444188671, -SNULL, 140012435800063, 140012444188671, -STORE, 140012435795968, 140012435800063, -STORE, 140012435800064, 140012444188671, -STORE, 140012427403264, 140012435795967, -SNULL, 140012427407359, 140012435795967, -STORE, 140012427403264, 140012427407359, -STORE, 140012427407360, 140012435795967, -STORE, 140012419010560, 140012427403263, -STORE, 140012410617856, 140012427403263, -STORE, 140012276400128, 140012410617855, -STORE, 140012268007424, 140012276400127, -STORE, 140012133789696, 140012268007423, -SNULL, 140012133789696, 140012175753215, -STORE, 140012175753216, 140012268007423, -STORE, 140012133789696, 140012175753215, -ERASE, 140012133789696, 140012175753215, -STORE, 140012041535488, 140012268007423, -SNULL, 140012108644351, 140012268007423, -STORE, 140012041535488, 140012108644351, -STORE, 140012108644352, 140012268007423, -SNULL, 140012108644352, 140012175753215, -STORE, 140012175753216, 140012268007423, -STORE, 140012108644352, 140012175753215, -ERASE, 140012108644352, 140012175753215, -SNULL, 140012276400128, 140012309970943, -STORE, 140012309970944, 140012410617855, -STORE, 140012276400128, 140012309970943, -ERASE, 140012276400128, 140012309970943, -STORE, 140012301578240, 140012309970943, -STORE, 140012041535488, 140012268007423, -SNULL, 140012242862079, 140012268007423, -STORE, 140012041535488, 140012242862079, -STORE, 140012242862080, 140012268007423, -ERASE, 140012242862080, 140012268007423, -SNULL, 140012041670655, 140012242862079, -STORE, 140012041535488, 140012041670655, -STORE, 140012041670656, 140012242862079, -SNULL, 140012041670656, 140012108644351, -STORE, 140012108644352, 140012242862079, -STORE, 140012041670656, 140012108644351, -SNULL, 140012108779519, 140012242862079, -STORE, 140012108644352, 140012108779519, -STORE, 140012108779520, 140012242862079, -SNULL, 140012377079807, 140012410617855, -STORE, 140012309970944, 140012377079807, -STORE, 140012377079808, 140012410617855, -ERASE, 140012377079808, 140012410617855, -SNULL, 140012310106111, 140012377079807, -STORE, 140012309970944, 140012310106111, -STORE, 140012310106112, 140012377079807, -SNULL, 140012410621951, 140012427403263, -STORE, 140012410617856, 140012410621951, -STORE, 140012410621952, 140012427403263, -SNULL, 140012108779520, 140012175753215, -STORE, 140012175753216, 140012242862079, -STORE, 140012108779520, 140012175753215, -SNULL, 140012175888383, 140012242862079, -STORE, 140012175753216, 140012175888383, -STORE, 140012175888384, 140012242862079, -SNULL, 140012301582335, 140012309970943, -STORE, 140012301578240, 140012301582335, -STORE, 140012301582336, 140012309970943, -SNULL, 140012410621952, 140012419010559, -STORE, 140012419010560, 140012427403263, -STORE, 140012410621952, 140012419010559, -SNULL, 140012419014655, 140012427403263, -STORE, 140012419010560, 140012419014655, -STORE, 140012419014656, 140012427403263, -SNULL, 140012268011519, 140012276400127, -STORE, 140012268007424, 140012268011519, -STORE, 140012268011520, 140012276400127, -STORE, 140012402225152, 140012410617855, -STORE, 140012393832448, 140012410617855, -SNULL, 140012393832448, 140012402225151, -STORE, 140012402225152, 140012410617855, -STORE, 140012393832448, 140012402225151, -SNULL, 140012402229247, 140012410617855, -STORE, 140012402225152, 140012402229247, -STORE, 140012402229248, 140012410617855, -STORE, 140012385439744, 140012402225151, -SNULL, 140012385439744, 140012393832447, -STORE, 140012393832448, 140012402225151, -STORE, 140012385439744, 140012393832447, -SNULL, 140012393836543, 140012402225151, -STORE, 140012393832448, 140012393836543, -STORE, 140012393836544, 140012402225151, -STORE, 140012293185536, 140012301578239, -STORE, 140012284792832, 140012301578239, -SNULL, 140012284792832, 140012293185535, -STORE, 140012293185536, 140012301578239, -STORE, 140012284792832, 140012293185535, -SNULL, 140012293189631, 140012301578239, -STORE, 140012293185536, 140012293189631, -STORE, 140012293189632, 140012301578239, -STORE, 140012268011520, 140012284792831, -SNULL, 140012385443839, 140012393832447, -STORE, 140012385439744, 140012385443839, -STORE, 140012385443840, 140012393832447, -STORE, 140012259614720, 140012268007423, -SNULL, 140012259618815, 140012268007423, -STORE, 140012259614720, 140012259618815, -STORE, 140012259618816, 140012268007423, -STORE, 140012251222016, 140012259614719, -SNULL, 140012251226111, 140012259614719, -STORE, 140012251222016, 140012251226111, -STORE, 140012251226112, 140012259614719, -SNULL, 140012284796927, 140012293185535, -STORE, 140012284792832, 140012284796927, -STORE, 140012284796928, 140012293185535, -SNULL, 140012268011520, 140012276400127, -STORE, 140012276400128, 140012284792831, -STORE, 140012268011520, 140012276400127, -SNULL, 140012276404223, 140012284792831, -STORE, 140012276400128, 140012276404223, -STORE, 140012276404224, 140012284792831, -STORE, 140012033142784, 140012041535487, -SNULL, 140012033146879, 140012041535487, -STORE, 140012033142784, 140012033146879, -STORE, 140012033146880, 140012041535487, -STORE, 140012024750080, 140012033142783, -STORE, 140012016357376, 140012033142783, -SNULL, 140012016357376, 140012024750079, -STORE, 140012024750080, 140012033142783, -STORE, 140012016357376, 140012024750079, -SNULL, 140012024754175, 140012033142783, -STORE, 140012024750080, 140012024754175, -STORE, 140012024754176, 140012033142783, -SNULL, 140012016361471, 140012024750079, -STORE, 140012016357376, 140012016361471, -STORE, 140012016361472, 140012024750079, -STORE, 140012007964672, 140012016357375, -SNULL, 140012007968767, 140012016357375, -STORE, 140012007964672, 140012007968767, -STORE, 140012007968768, 140012016357375, -STORE, 140011999571968, 140012007964671, -STORE, 140011991179264, 140012007964671, -STORE, 140011856961536, 140011991179263, -STORE, 140011848568832, 140011856961535, -STORE, 140011714351104, 140011848568831, -SNULL, 140011714351104, 140011773100031, -STORE, 140011773100032, 140011848568831, -STORE, 140011714351104, 140011773100031, -ERASE, 140011714351104, 140011773100031, -STORE, 140011764707328, 140011773100031, -STORE, 140011756314624, 140011773100031, -STORE, 140011622096896, 140011756314623, -STORE, 140011613704192, 140011622096895, -STORE, 140011479486464, 140011613704191, -STORE, 140011471093760, 140011479486463, -SNULL, 140011479486464, 140011504664575, -STORE, 140011504664576, 140011613704191, -STORE, 140011479486464, 140011504664575, -ERASE, 140011479486464, 140011504664575, -STORE, 140011496271872, 140011504664575, -STORE, 140011487879168, 140011504664575, -STORE, 140011336876032, 140011471093759, -SNULL, 140011336876032, 140011370446847, -STORE, 140011370446848, 140011471093759, -STORE, 140011336876032, 140011370446847, -ERASE, 140011336876032, 140011370446847, -STORE, 140011471093760, 140011487879167, -STORE, 140011362054144, 140011370446847, -SNULL, 140011362058239, 140011370446847, -STORE, 140011362054144, 140011362058239, -STORE, 140011362058240, 140011370446847, -STORE, 140011353661440, 140011362054143, -STORE, 140011345268736, 140011362054143, -SNULL, 140011345272831, 140011362054143, -STORE, 140011345268736, 140011345272831, -STORE, 140011345272832, 140011362054143, -STORE, 140011336876032, 140011345268735, -STORE, 140011328483328, 140011345268735, -SNULL, 140011328487423, 140011345268735, -STORE, 140011328483328, 140011328487423, -STORE, 140011328487424, 140011345268735, -STORE, 140011320090624, 140011328483327, -STORE, 140011185872896, 140011320090623, -SNULL, 140011185872896, 140011236229119, -STORE, 140011236229120, 140011320090623, -STORE, 140011185872896, 140011236229119, -ERASE, 140011185872896, 140011236229119, -SNULL, 140011856961536, 140011907317759, -STORE, 140011907317760, 140011991179263, -STORE, 140011856961536, 140011907317759, -ERASE, 140011856961536, 140011907317759, -SNULL, 140011974426623, 140011991179263, -STORE, 140011907317760, 140011974426623, -STORE, 140011974426624, 140011991179263, -ERASE, 140011974426624, 140011991179263, -SNULL, 140011840208895, 140011848568831, -STORE, 140011773100032, 140011840208895, -STORE, 140011840208896, 140011848568831, -ERASE, 140011840208896, 140011848568831, -SNULL, 140011773235199, 140011840208895, -STORE, 140011773100032, 140011773235199, -STORE, 140011773235200, 140011840208895, -STORE, 140011102011392, 140011320090623, -SNULL, 140011169120255, 140011320090623, -STORE, 140011102011392, 140011169120255, -STORE, 140011169120256, 140011320090623, -SNULL, 140011169120256, 140011236229119, -STORE, 140011236229120, 140011320090623, -STORE, 140011169120256, 140011236229119, -ERASE, 140011169120256, 140011236229119, -SNULL, 140011622096896, 140011638882303, -STORE, 140011638882304, 140011756314623, -STORE, 140011622096896, 140011638882303, -ERASE, 140011622096896, 140011638882303, -SNULL, 140011705991167, 140011756314623, -STORE, 140011638882304, 140011705991167, -STORE, 140011705991168, 140011756314623, -ERASE, 140011705991168, 140011756314623, -SNULL, 140011571773439, 140011613704191, -STORE, 140011504664576, 140011571773439, -STORE, 140011571773440, 140011613704191, -ERASE, 140011571773440, 140011613704191, -STORE, 140010967793664, 140011169120255, -SNULL, 140011034902527, 140011169120255, -STORE, 140010967793664, 140011034902527, -STORE, 140011034902528, 140011169120255, -SNULL, 140011034902528, 140011102011391, -STORE, 140011102011392, 140011169120255, -STORE, 140011034902528, 140011102011391, -ERASE, 140011034902528, 140011102011391, -STORE, 140010833575936, 140011034902527, -SNULL, 140011437555711, 140011471093759, -STORE, 140011370446848, 140011437555711, -STORE, 140011437555712, 140011471093759, -ERASE, 140011437555712, 140011471093759, -SNULL, 140011370582015, 140011437555711, -STORE, 140011370446848, 140011370582015, -STORE, 140011370582016, 140011437555711, -STORE, 140010699358208, 140011034902527, -SNULL, 140011487883263, 140011504664575, -STORE, 140011487879168, 140011487883263, -STORE, 140011487883264, 140011504664575, -SNULL, 140011345272832, 140011353661439, -STORE, 140011353661440, 140011362054143, -STORE, 140011345272832, 140011353661439, -SNULL, 140011353665535, 140011362054143, -STORE, 140011353661440, 140011353665535, -STORE, 140011353665536, 140011362054143, -SNULL, 140011328487424, 140011336876031, -STORE, 140011336876032, 140011345268735, -STORE, 140011328487424, 140011336876031, -SNULL, 140011336880127, 140011345268735, -STORE, 140011336876032, 140011336880127, -STORE, 140011336880128, 140011345268735, -SNULL, 140011303337983, 140011320090623, -STORE, 140011236229120, 140011303337983, -STORE, 140011303337984, 140011320090623, -ERASE, 140011303337984, 140011320090623, -SNULL, 140011907452927, 140011974426623, -STORE, 140011907317760, 140011907452927, -STORE, 140011907452928, 140011974426623, -SNULL, 140011102146559, 140011169120255, -STORE, 140011102011392, 140011102146559, -STORE, 140011102146560, 140011169120255, -SNULL, 140011639017471, 140011705991167, -STORE, 140011638882304, 140011639017471, -STORE, 140011639017472, 140011705991167, -SNULL, 140011504799743, 140011571773439, -STORE, 140011504664576, 140011504799743, -STORE, 140011504799744, 140011571773439, -SNULL, 140011613708287, 140011622096895, -STORE, 140011613704192, 140011613708287, -STORE, 140011613708288, 140011622096895, -SNULL, 140010699358208, 140010967793663, -STORE, 140010967793664, 140011034902527, -STORE, 140010699358208, 140010967793663, -SNULL, 140010967928831, 140011034902527, -STORE, 140010967793664, 140010967928831, -STORE, 140010967928832, 140011034902527, -SNULL, 140010900684799, 140010967793663, -STORE, 140010699358208, 140010900684799, -STORE, 140010900684800, 140010967793663, -ERASE, 140010900684800, 140010967793663, -SNULL, 140010766467071, 140010900684799, -STORE, 140010699358208, 140010766467071, -STORE, 140010766467072, 140010900684799, -SNULL, 140010766467072, 140010833575935, -STORE, 140010833575936, 140010900684799, -STORE, 140010766467072, 140010833575935, -ERASE, 140010766467072, 140010833575935, -SNULL, 140010699493375, 140010766467071, -STORE, 140010699358208, 140010699493375, -STORE, 140010699493376, 140010766467071, -SNULL, 140011848572927, 140011856961535, -STORE, 140011848568832, 140011848572927, -STORE, 140011848572928, 140011856961535, -STORE, 140011982786560, 140012007964671, -STORE, 140011898925056, 140011907317759, -SNULL, 140011898929151, 140011907317759, -STORE, 140011898925056, 140011898929151, -STORE, 140011898929152, 140011907317759, -SNULL, 140011320094719, 140011328483327, -STORE, 140011320090624, 140011320094719, -STORE, 140011320094720, 140011328483327, -STORE, 140011890532352, 140011898925055, -STORE, 140011882139648, 140011898925055, -SNULL, 140011882143743, 140011898925055, -STORE, 140011882139648, 140011882143743, -STORE, 140011882143744, 140011898925055, -STORE, 140011873746944, 140011882139647, -SNULL, 140011873751039, 140011882139647, -STORE, 140011873746944, 140011873751039, -STORE, 140011873751040, 140011882139647, -SNULL, 140011236364287, 140011303337983, -STORE, 140011236229120, 140011236364287, -STORE, 140011236364288, 140011303337983, -SNULL, 140011756318719, 140011773100031, -STORE, 140011756314624, 140011756318719, -STORE, 140011756318720, 140011773100031, -SNULL, 140011756318720, 140011764707327, -STORE, 140011764707328, 140011773100031, -STORE, 140011756318720, 140011764707327, -SNULL, 140011764711423, 140011773100031, -STORE, 140011764707328, 140011764711423, -STORE, 140011764711424, 140011773100031, -SNULL, 140011471097855, 140011487879167, -STORE, 140011471093760, 140011471097855, -STORE, 140011471097856, 140011487879167, -SNULL, 140010833711103, 140010900684799, -STORE, 140010833575936, 140010833711103, -STORE, 140010833711104, 140010900684799, -SNULL, 140011982790655, 140012007964671, -STORE, 140011982786560, 140011982790655, -STORE, 140011982790656, 140012007964671, -STORE, 140011865354240, 140011873746943, -STORE, 140011848572928, 140011865354239, -SNULL, 140011848572928, 140011856961535, -STORE, 140011856961536, 140011865354239, -STORE, 140011848572928, 140011856961535, -SNULL, 140011856965631, 140011865354239, -STORE, 140011856961536, 140011856965631, -STORE, 140011856965632, 140011865354239, -STORE, 140011747921920, 140011756314623, -STORE, 140011739529216, 140011756314623, -SNULL, 140011471097856, 140011479486463, -STORE, 140011479486464, 140011487879167, -STORE, 140011471097856, 140011479486463, -SNULL, 140011479490559, 140011487879167, -STORE, 140011479486464, 140011479490559, -STORE, 140011479490560, 140011487879167, -STORE, 140011731136512, 140011756314623, -STORE, 140011722743808, 140011756314623, -SNULL, 140011982790656, 140011999571967, -STORE, 140011999571968, 140012007964671, -STORE, 140011982790656, 140011999571967, -SNULL, 140011999576063, 140012007964671, -STORE, 140011999571968, 140011999576063, -STORE, 140011999576064, 140012007964671, -STORE, 140011714351104, 140011756314623, -SNULL, 140011882143744, 140011890532351, -STORE, 140011890532352, 140011898925055, -STORE, 140011882143744, 140011890532351, -SNULL, 140011890536447, 140011898925055, -STORE, 140011890532352, 140011890536447, -STORE, 140011890536448, 140011898925055, -STORE, 140011630489600, 140011638882303, -STORE, 140011613708288, 140011638882303, -STORE, 140011605311488, 140011613704191, -STORE, 140011596918784, 140011613704191, -STORE, 140011588526080, 140011613704191, -SNULL, 140011487883264, 140011496271871, -STORE, 140011496271872, 140011504664575, -STORE, 140011487883264, 140011496271871, -SNULL, 140011496275967, 140011504664575, -STORE, 140011496271872, 140011496275967, -STORE, 140011496275968, 140011504664575, -STORE, 140011580133376, 140011613704191, -SNULL, 140011580137471, 140011613704191, -STORE, 140011580133376, 140011580137471, -STORE, 140011580137472, 140011613704191, -SNULL, 140011982790656, 140011991179263, -STORE, 140011991179264, 140011999571967, -STORE, 140011982790656, 140011991179263, -SNULL, 140011991183359, 140011999571967, -STORE, 140011991179264, 140011991183359, -STORE, 140011991183360, 140011999571967, -SNULL, 140011865358335, 140011873746943, -STORE, 140011865354240, 140011865358335, -STORE, 140011865358336, 140011873746943, -STORE, 140011462701056, 140011471093759, -SNULL, 140011714351104, 140011739529215, -STORE, 140011739529216, 140011756314623, -STORE, 140011714351104, 140011739529215, -SNULL, 140011739533311, 140011756314623, -STORE, 140011739529216, 140011739533311, -STORE, 140011739533312, 140011756314623, -SNULL, 140011739533312, 140011747921919, -STORE, 140011747921920, 140011756314623, -STORE, 140011739533312, 140011747921919, -SNULL, 140011747926015, 140011756314623, -STORE, 140011747921920, 140011747926015, -STORE, 140011747926016, 140011756314623, -SNULL, 140011613708288, 140011630489599, -STORE, 140011630489600, 140011638882303, -STORE, 140011613708288, 140011630489599, -SNULL, 140011630493695, 140011638882303, -STORE, 140011630489600, 140011630493695, -STORE, 140011630493696, 140011638882303, -SNULL, 140011714351104, 140011722743807, -STORE, 140011722743808, 140011739529215, -STORE, 140011714351104, 140011722743807, -SNULL, 140011722747903, 140011739529215, -STORE, 140011722743808, 140011722747903, -STORE, 140011722747904, 140011739529215, -SNULL, 140011714355199, 140011722743807, -STORE, 140011714351104, 140011714355199, -STORE, 140011714355200, 140011722743807, -SNULL, 140011722747904, 140011731136511, -STORE, 140011731136512, 140011739529215, -STORE, 140011722747904, 140011731136511, -SNULL, 140011731140607, 140011739529215, -STORE, 140011731136512, 140011731140607, -STORE, 140011731140608, 140011739529215, -STORE, 140011454308352, 140011471093759, -STORE, 140011445915648, 140011471093759, -SNULL, 140011580137472, 140011588526079, -STORE, 140011588526080, 140011613704191, -STORE, 140011580137472, 140011588526079, -SNULL, 140011588530175, 140011613704191, -STORE, 140011588526080, 140011588530175, -STORE, 140011588530176, 140011613704191, -SNULL, 140011445915648, 140011462701055, -STORE, 140011462701056, 140011471093759, -STORE, 140011445915648, 140011462701055, -SNULL, 140011462705151, 140011471093759, -STORE, 140011462701056, 140011462705151, -STORE, 140011462705152, 140011471093759, -SNULL, 140011588530176, 140011596918783, -STORE, 140011596918784, 140011613704191, -STORE, 140011588530176, 140011596918783, -SNULL, 140011596922879, 140011613704191, -STORE, 140011596918784, 140011596922879, -STORE, 140011596922880, 140011613704191, -SNULL, 140011596922880, 140011605311487, -STORE, 140011605311488, 140011613704191, -STORE, 140011596922880, 140011605311487, -SNULL, 140011605315583, 140011613704191, -STORE, 140011605311488, 140011605315583, -STORE, 140011605315584, 140011613704191, -SNULL, 140011613708288, 140011622096895, -STORE, 140011622096896, 140011630489599, -STORE, 140011613708288, 140011622096895, -SNULL, 140011622100991, 140011630489599, -STORE, 140011622096896, 140011622100991, -STORE, 140011622100992, 140011630489599, -STORE, 140011311697920, 140011320090623, -STORE, 140011227836416, 140011236229119, -STORE, 140011219443712, 140011236229119, -SNULL, 140011219447807, 140011236229119, -STORE, 140011219443712, 140011219447807, -STORE, 140011219447808, 140011236229119, -STORE, 140011211051008, 140011219443711, -STORE, 140011202658304, 140011219443711, -SNULL, 140011202662399, 140011219443711, -STORE, 140011202658304, 140011202662399, -STORE, 140011202662400, 140011219443711, -STORE, 140011194265600, 140011202658303, -STORE, 140011185872896, 140011202658303, -STORE, 140011177480192, 140011202658303, -STORE, 140011093618688, 140011102011391, -SNULL, 140011445915648, 140011454308351, -STORE, 140011454308352, 140011462701055, -STORE, 140011445915648, 140011454308351, -SNULL, 140011454312447, 140011462701055, -STORE, 140011454308352, 140011454312447, -STORE, 140011454312448, 140011462701055, -STORE, 140011085225984, 140011102011391, -SNULL, 140011085230079, 140011102011391, -STORE, 140011085225984, 140011085230079, -STORE, 140011085230080, 140011102011391, -SNULL, 140011177484287, 140011202658303, -STORE, 140011177480192, 140011177484287, -STORE, 140011177484288, 140011202658303, -SNULL, 140011445919743, 140011454308351, -STORE, 140011445915648, 140011445919743, -STORE, 140011445919744, 140011454308351, -SNULL, 140011177484288, 140011185872895, -STORE, 140011185872896, 140011202658303, -STORE, 140011177484288, 140011185872895, -SNULL, 140011185876991, 140011202658303, -STORE, 140011185872896, 140011185876991, -STORE, 140011185876992, 140011202658303, -STORE, 140011076833280, 140011085225983, -SNULL, 140011202662400, 140011211051007, -STORE, 140011211051008, 140011219443711, -STORE, 140011202662400, 140011211051007, -SNULL, 140011211055103, 140011219443711, -STORE, 140011211051008, 140011211055103, -STORE, 140011211055104, 140011219443711, -SNULL, 140011185876992, 140011194265599, -STORE, 140011194265600, 140011202658303, -STORE, 140011185876992, 140011194265599, -SNULL, 140011194269695, 140011202658303, -STORE, 140011194265600, 140011194269695, -STORE, 140011194269696, 140011202658303, -STORE, 140011068440576, 140011085225983, -SNULL, 140011311702015, 140011320090623, -STORE, 140011311697920, 140011311702015, -STORE, 140011311702016, 140011320090623, -STORE, 140011060047872, 140011085225983, -SNULL, 140011060051967, 140011085225983, -STORE, 140011060047872, 140011060051967, -STORE, 140011060051968, 140011085225983, -STORE, 140011051655168, 140011060047871, -STORE, 140011043262464, 140011060047871, -SNULL, 140011043266559, 140011060047871, -STORE, 140011043262464, 140011043266559, -STORE, 140011043266560, 140011060047871, -SNULL, 140011219447808, 140011227836415, -STORE, 140011227836416, 140011236229119, -STORE, 140011219447808, 140011227836415, -SNULL, 140011227840511, 140011236229119, -STORE, 140011227836416, 140011227840511, -STORE, 140011227840512, 140011236229119, -SNULL, 140011085230080, 140011093618687, -STORE, 140011093618688, 140011102011391, -STORE, 140011085230080, 140011093618687, -SNULL, 140011093622783, 140011102011391, -STORE, 140011093618688, 140011093622783, -STORE, 140011093622784, 140011102011391, -STORE, 140010959400960, 140010967793663, -STORE, 140010951008256, 140010967793663, -SNULL, 140010951008256, 140010959400959, -STORE, 140010959400960, 140010967793663, -STORE, 140010951008256, 140010959400959, -SNULL, 140010959405055, 140010967793663, -STORE, 140010959400960, 140010959405055, -STORE, 140010959405056, 140010967793663, -STORE, 140010942615552, 140010959400959, -STORE, 140010934222848, 140010959400959, -SNULL, 140011060051968, 140011076833279, -STORE, 140011076833280, 140011085225983, -STORE, 140011060051968, 140011076833279, -SNULL, 140011076837375, 140011085225983, -STORE, 140011076833280, 140011076837375, -STORE, 140011076837376, 140011085225983, -SNULL, 140011043266560, 140011051655167, -STORE, 140011051655168, 140011060047871, -STORE, 140011043266560, 140011051655167, -SNULL, 140011051659263, 140011060047871, -STORE, 140011051655168, 140011051659263, -STORE, 140011051659264, 140011060047871, -STORE, 140010925830144, 140010959400959, -SNULL, 140011060051968, 140011068440575, -STORE, 140011068440576, 140011076833279, -STORE, 140011060051968, 140011068440575, -SNULL, 140011068444671, 140011076833279, -STORE, 140011068440576, 140011068444671, -STORE, 140011068444672, 140011076833279, -STORE, 140010917437440, 140010959400959, -STORE, 140010909044736, 140010959400959, -STORE, 140010825183232, 140010833575935, -SNULL, 140010909044736, 140010942615551, -STORE, 140010942615552, 140010959400959, -STORE, 140010909044736, 140010942615551, -SNULL, 140010942619647, 140010959400959, -STORE, 140010942615552, 140010942619647, -STORE, 140010942619648, 140010959400959, -SNULL, 140010909044736, 140010934222847, -STORE, 140010934222848, 140010942615551, -STORE, 140010909044736, 140010934222847, -SNULL, 140010934226943, 140010942615551, -STORE, 140010934222848, 140010934226943, -STORE, 140010934226944, 140010942615551, -SNULL, 140010909048831, 140010934222847, -STORE, 140010909044736, 140010909048831, -STORE, 140010909048832, 140010934222847, -STORE, 140010816790528, 140010833575935, -SNULL, 140010816794623, 140010833575935, -STORE, 140010816790528, 140010816794623, -STORE, 140010816794624, 140010833575935, -STORE, 140010808397824, 140010816790527, -SNULL, 140010942619648, 140010951008255, -STORE, 140010951008256, 140010959400959, -STORE, 140010942619648, 140010951008255, -SNULL, 140010951012351, 140010959400959, -STORE, 140010951008256, 140010951012351, -STORE, 140010951012352, 140010959400959, -STORE, 140010800005120, 140010816790527, -SNULL, 140010800009215, 140010816790527, -STORE, 140010800005120, 140010800009215, -STORE, 140010800009216, 140010816790527, -SNULL, 140010909048832, 140010925830143, -STORE, 140010925830144, 140010934222847, -STORE, 140010909048832, 140010925830143, -SNULL, 140010925834239, 140010934222847, -STORE, 140010925830144, 140010925834239, -STORE, 140010925834240, 140010934222847, -SNULL, 140010816794624, 140010825183231, -STORE, 140010825183232, 140010833575935, -STORE, 140010816794624, 140010825183231, -SNULL, 140010825187327, 140010833575935, -STORE, 140010825183232, 140010825187327, -STORE, 140010825187328, 140010833575935, -SNULL, 140010909048832, 140010917437439, -STORE, 140010917437440, 140010925830143, -STORE, 140010909048832, 140010917437439, -SNULL, 140010917441535, 140010925830143, -STORE, 140010917437440, 140010917441535, -STORE, 140010917441536, 140010925830143, -SNULL, 140010800009216, 140010808397823, -STORE, 140010808397824, 140010816790527, -STORE, 140010800009216, 140010808397823, -SNULL, 140010808401919, 140010816790527, -STORE, 140010808397824, 140010808401919, -STORE, 140010808401920, 140010816790527, -STORE, 140010791612416, 140010800005119, -SNULL, 140010791616511, 140010800005119, -STORE, 140010791612416, 140010791616511, -STORE, 140010791616512, 140010800005119, -STORE, 140012547100672, 140012547129343, -STORE, 140012511506432, 140012513697791, -SNULL, 140012511506432, 140012511596543, -STORE, 140012511596544, 140012513697791, -STORE, 140012511506432, 140012511596543, -SNULL, 140012513689599, 140012513697791, -STORE, 140012511596544, 140012513689599, -STORE, 140012513689600, 140012513697791, -ERASE, 140012513689600, 140012513697791, -STORE, 140012513689600, 140012513697791, -SNULL, 140012513693695, 140012513697791, -STORE, 140012513689600, 140012513693695, -STORE, 140012513693696, 140012513697791, -ERASE, 140012547100672, 140012547129343, -ERASE, 140011362054144, 140011362058239, -ERASE, 140011362058240, 140011370446847, -ERASE, 140011882139648, 140011882143743, -ERASE, 140011882143744, 140011890532351, -ERASE, 140011873746944, 140011873751039, -ERASE, 140011873751040, 140011882139647, -ERASE, 140011588526080, 140011588530175, -ERASE, 140011588530176, 140011596918783, -ERASE, 140011328483328, 140011328487423, -ERASE, 140011328487424, 140011336876031, -ERASE, 140011898925056, 140011898929151, -ERASE, 140011898929152, 140011907317759, -ERASE, 140011353661440, 140011353665535, -ERASE, 140011353665536, 140011362054143, -ERASE, 140011336876032, 140011336880127, -ERASE, 140011336880128, 140011345268735, -ERASE, 140011731136512, 140011731140607, -ERASE, 140011731140608, 140011739529215, -ERASE, 140011479486464, 140011479490559, -ERASE, 140011479490560, 140011487879167, -ERASE, 140011756314624, 140011756318719, -ERASE, 140011756318720, 140011764707327, -ERASE, 140011580133376, 140011580137471, -ERASE, 140011580137472, 140011588526079, -ERASE, 140011219443712, 140011219447807, -ERASE, 140011219447808, 140011227836415, -ERASE, 140011051655168, 140011051659263, -ERASE, 140011051659264, 140011060047871, -ERASE, 140011999571968, 140011999576063, -ERASE, 140011999576064, 140012007964671, -ERASE, 140011714351104, 140011714355199, -ERASE, 140011714355200, 140011722743807, -ERASE, 140011739529216, 140011739533311, -ERASE, 140011739533312, 140011747921919, -ERASE, 140011320090624, 140011320094719, -ERASE, 140011320094720, 140011328483327, -ERASE, 140011630489600, 140011630493695, -ERASE, 140011630493696, 140011638882303, -ERASE, 140011345268736, 140011345272831, -ERASE, 140011345272832, 140011353661439, -ERASE, 140011496271872, 140011496275967, -ERASE, 140011496275968, 140011504664575, -ERASE, 140011194265600, 140011194269695, -ERASE, 140011194269696, 140011202658303, -ERASE, 140011068440576, 140011068444671, -ERASE, 140011068444672, 140011076833279, -ERASE, 140010909044736, 140010909048831, -ERASE, 140010909048832, 140010917437439, -ERASE, 140011764707328, 140011764711423, -ERASE, 140011764711424, 140011773100031, -ERASE, 140011462701056, 140011462705151, -ERASE, 140011462705152, 140011471093759, -ERASE, 140011076833280, 140011076837375, -ERASE, 140011076837376, 140011085225983, -ERASE, 140011991179264, 140011991183359, -ERASE, 140011991183360, 140011999571967, -ERASE, 140011211051008, 140011211055103, -ERASE, 140011211055104, 140011219443711, -ERASE, 140010917437440, 140010917441535, -ERASE, 140010917441536, 140010925830143, -ERASE, 140011085225984, 140011085230079, -ERASE, 140011085230080, 140011093618687, -ERASE, 140011487879168, 140011487883263, -ERASE, 140011487883264, 140011496271871, -ERASE, 140011856961536, 140011856965631, -ERASE, 140011856965632, 140011865354239, -ERASE, 140011982786560, 140011982790655, -ERASE, 140011982790656, 140011991179263, -ERASE, 140011722743808, 140011722747903, -ERASE, 140011722747904, 140011731136511, -ERASE, 140011177480192, 140011177484287, -ERASE, 140011177484288, 140011185872895, -ERASE, 140011848568832, 140011848572927, -ERASE, 140011848572928, 140011856961535, -ERASE, 140011890532352, 140011890536447, -ERASE, 140011890536448, 140011898925055, -ERASE, 140011622096896, 140011622100991, -ERASE, 140011622100992, 140011630489599, -ERASE, 140011311697920, 140011311702015, -ERASE, 140011311702016, 140011320090623, -ERASE, 140011471093760, 140011471097855, -ERASE, 140011471097856, 140011479486463, -ERASE, 140011605311488, 140011605315583, -ERASE, 140011605315584, 140011613704191, -ERASE, 140010791612416, 140010791616511, -ERASE, 140010791616512, 140010800005119, -ERASE, 140010959400960, 140010959405055, -ERASE, 140010959405056, 140010967793663, -ERASE, 140011185872896, 140011185876991, -ERASE, 140011185876992, 140011194265599, -ERASE, 140011454308352, 140011454312447, -ERASE, 140011454312448, 140011462701055, -ERASE, 140011596918784, 140011596922879, -ERASE, 140011596922880, 140011605311487, -ERASE, 140011060047872, 140011060051967, -ERASE, 140011060051968, 140011068440575, -ERASE, 140010925830144, 140010925834239, -ERASE, 140010925834240, 140010934222847, -ERASE, 140011747921920, 140011747926015, -ERASE, 140011747926016, 140011756314623, -ERASE, 140011202658304, 140011202662399, -ERASE, 140011202662400, 140011211051007, -ERASE, 140010800005120, 140010800009215, -ERASE, 140010800009216, 140010808397823, -ERASE, 140011093618688, 140011093622783, -ERASE, 140011093622784, 140011102011391, -ERASE, 140010808397824, 140010808401919, -ERASE, 140010808401920, 140010816790527, -ERASE, 140012419010560, 140012419014655, -ERASE, 140012419014656, 140012427403263, -ERASE, 140010934222848, 140010934226943, -ERASE, 140010934226944, 140010942615551, -ERASE, 140010942615552, 140010942619647, -ERASE, 140010942619648, 140010951008255, -ERASE, 140011613704192, 140011613708287, -ERASE, 140011613708288, 140011622096895, -ERASE, 140011865354240, 140011865358335, -ERASE, 140011865358336, 140011873746943, -ERASE, 140012301578240, 140012301582335, -ERASE, 140012301582336, 140012309970943, -ERASE, 140012393832448, 140012393836543, -ERASE, 140012393836544, 140012402225151, -ERASE, 140012410617856, 140012410621951, -ERASE, 140012410621952, 140012419010559, -ERASE, 140012402225152, 140012402229247, -ERASE, 140012402229248, 140012410617855, -ERASE, 140012259614720, 140012259618815, -ERASE, 140012259618816, 140012268007423, -ERASE, 140012251222016, 140012251226111, -ERASE, 140012251226112, 140012259614719, -ERASE, 140012284792832, 140012284796927, -ERASE, 140012284796928, 140012293185535, -ERASE, 140011445915648, 140011445919743, -ERASE, 140011445919744, 140011454308351, -ERASE, 140010951008256, 140010951012351, -ERASE, 140010951012352, 140010959400959, -ERASE, 140011043262464, 140011043266559, -ERASE, 140011043266560, 140011051655167, -ERASE, 140010825183232, 140010825187327, -ERASE, 140010825187328, 140010833575935, -ERASE, 140012293185536, 140012293189631, -ERASE, 140012293189632, 140012301578239, -ERASE, 140012276400128, 140012276404223, -ERASE, 140012276404224, 140012284792831, -ERASE, 140012016357376, 140012016361471, -ERASE, 140012016361472, 140012024750079, -ERASE, 140012024750080, 140012024754175, -ERASE, 140012024754176, 140012033142783, -ERASE, 140011227836416, 140011227840511, -ERASE, 140011227840512, 140011236229119, -ERASE, 140010816790528, 140010816794623, -ERASE, 140010816794624, 140010825183231, -ERASE, 140012268007424, 140012268011519, -ERASE, 140012268011520, 140012276400127, -ERASE, 140012385439744, 140012385443839, -ERASE, 140012385443840, 140012393832447, -ERASE, 140012522090496, 140012522094591, -ERASE, 140012522094592, 140012530483199, -ERASE, 140012033142784, 140012033146879, -ERASE, 140012033146880, 140012041535487, - }; - unsigned long set35[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140730536939520, 140737488351231, -SNULL, 140730536943615, 140737488351231, -STORE, 140730536939520, 140730536943615, -STORE, 140730536808448, 140730536943615, -STORE, 94245239877632, 94245242130431, -SNULL, 94245240008703, 94245242130431, -STORE, 94245239877632, 94245240008703, -STORE, 94245240008704, 94245242130431, -ERASE, 94245240008704, 94245242130431, -STORE, 94245242101760, 94245242109951, -STORE, 94245242109952, 94245242130431, -STORE, 140475575263232, 140475577516031, -SNULL, 140475575406591, 140475577516031, -STORE, 140475575263232, 140475575406591, -STORE, 140475575406592, 140475577516031, -ERASE, 140475575406592, 140475577516031, -STORE, 140475577503744, 140475577511935, -STORE, 140475577511936, 140475577516031, -STORE, 140730538164224, 140730538168319, -STORE, 140730538151936, 140730538164223, -STORE, 140475577475072, 140475577503743, -STORE, 140475577466880, 140475577475071, -STORE, 140475573047296, 140475575263231, -SNULL, 140475573047296, 140475573145599, -STORE, 140475573145600, 140475575263231, -STORE, 140475573047296, 140475573145599, -SNULL, 140475575238655, 140475575263231, -STORE, 140475573145600, 140475575238655, -STORE, 140475575238656, 140475575263231, -SNULL, 140475575238656, 140475575246847, -STORE, 140475575246848, 140475575263231, -STORE, 140475575238656, 140475575246847, -ERASE, 140475575238656, 140475575246847, -STORE, 140475575238656, 140475575246847, -ERASE, 140475575246848, 140475575263231, -STORE, 140475575246848, 140475575263231, -STORE, 140475569250304, 140475573047295, -SNULL, 140475569250304, 140475570909183, -STORE, 140475570909184, 140475573047295, -STORE, 140475569250304, 140475570909183, -SNULL, 140475573006335, 140475573047295, -STORE, 140475570909184, 140475573006335, -STORE, 140475573006336, 140475573047295, -SNULL, 140475573006336, 140475573030911, -STORE, 140475573030912, 140475573047295, -STORE, 140475573006336, 140475573030911, -ERASE, 140475573006336, 140475573030911, -STORE, 140475573006336, 140475573030911, -ERASE, 140475573030912, 140475573047295, -STORE, 140475573030912, 140475573047295, -STORE, 140475577458688, 140475577475071, -SNULL, 140475573022719, 140475573030911, -STORE, 140475573006336, 140475573022719, -STORE, 140475573022720, 140475573030911, -SNULL, 140475575242751, 140475575246847, -STORE, 140475575238656, 140475575242751, -STORE, 140475575242752, 140475575246847, -SNULL, 94245242105855, 94245242109951, -STORE, 94245242101760, 94245242105855, -STORE, 94245242105856, 94245242109951, -SNULL, 140475577507839, 140475577511935, -STORE, 140475577503744, 140475577507839, -STORE, 140475577507840, 140475577511935, -ERASE, 140475577475072, 140475577503743, -STORE, 94245271216128, 94245271351295, -STORE, 140475560857600, 140475569250303, -SNULL, 140475560861695, 140475569250303, -STORE, 140475560857600, 140475560861695, -STORE, 140475560861696, 140475569250303, -STORE, 140475552464896, 140475560857599, -STORE, 140475418247168, 140475552464895, -SNULL, 140475418247168, 140475428241407, -STORE, 140475428241408, 140475552464895, -STORE, 140475418247168, 140475428241407, -ERASE, 140475418247168, 140475428241407, -SNULL, 140475495350271, 140475552464895, -STORE, 140475428241408, 140475495350271, -STORE, 140475495350272, 140475552464895, -ERASE, 140475495350272, 140475552464895, -SNULL, 140475428376575, 140475495350271, -STORE, 140475428241408, 140475428376575, -STORE, 140475428376576, 140475495350271, -SNULL, 140475552468991, 140475560857599, -STORE, 140475552464896, 140475552468991, -STORE, 140475552468992, 140475560857599, -STORE, 140475544072192, 140475552464895, -SNULL, 140475544076287, 140475552464895, -STORE, 140475544072192, 140475544076287, -STORE, 140475544076288, 140475552464895, -STORE, 140475535679488, 140475544072191, -SNULL, 140475535683583, 140475544072191, -STORE, 140475535679488, 140475535683583, -STORE, 140475535683584, 140475544072191, -STORE, 140475527286784, 140475535679487, -SNULL, 140475527290879, 140475535679487, -STORE, 140475527286784, 140475527290879, -STORE, 140475527290880, 140475535679487, -STORE, 140475518894080, 140475527286783, -STORE, 140475510501376, 140475527286783, -STORE, 140475502108672, 140475527286783, -STORE, 140475419848704, 140475428241407, -STORE, 140475285630976, 140475419848703, -SNULL, 140475285630976, 140475294023679, -STORE, 140475294023680, 140475419848703, -STORE, 140475285630976, 140475294023679, -ERASE, 140475285630976, 140475294023679, -STORE, 140475159805952, 140475419848703, -STORE, 140475025588224, 140475419848703, -SNULL, 140475092697087, 140475419848703, -STORE, 140475025588224, 140475092697087, -STORE, 140475092697088, 140475419848703, -SNULL, 140475092697088, 140475159805951, -STORE, 140475159805952, 140475419848703, -STORE, 140475092697088, 140475159805951, -ERASE, 140475092697088, 140475159805951, -STORE, 140474891370496, 140475092697087, -SNULL, 140474958479359, 140475092697087, -STORE, 140474891370496, 140474958479359, -STORE, 140474958479360, 140475092697087, -SNULL, 140474958479360, 140475025588223, -STORE, 140475025588224, 140475092697087, -STORE, 140474958479360, 140475025588223, -ERASE, 140474958479360, 140475025588223, -SNULL, 140475361132543, 140475419848703, -STORE, 140475159805952, 140475361132543, -STORE, 140475361132544, 140475419848703, -ERASE, 140475361132544, 140475419848703, -SNULL, 140475159805952, 140475294023679, -STORE, 140475294023680, 140475361132543, -STORE, 140475159805952, 140475294023679, -SNULL, 140475294158847, 140475361132543, -STORE, 140475294023680, 140475294158847, -STORE, 140475294158848, 140475361132543, -SNULL, 140475226914815, 140475294023679, -STORE, 140475159805952, 140475226914815, -STORE, 140475226914816, 140475294023679, -ERASE, 140475226914816, 140475294023679, -SNULL, 140475025723391, 140475092697087, -STORE, 140475025588224, 140475025723391, -STORE, 140475025723392, 140475092697087, -SNULL, 140475159941119, 140475226914815, -STORE, 140475159805952, 140475159941119, -STORE, 140475159941120, 140475226914815, -SNULL, 140474891505663, 140474958479359, -STORE, 140474891370496, 140474891505663, -STORE, 140474891505664, 140474958479359, -SNULL, 140475502108672, 140475518894079, -STORE, 140475518894080, 140475527286783, -STORE, 140475502108672, 140475518894079, -SNULL, 140475518898175, 140475527286783, -STORE, 140475518894080, 140475518898175, -STORE, 140475518898176, 140475527286783, -STORE, 140475411456000, 140475428241407, -SNULL, 140475502112767, 140475518894079, -STORE, 140475502108672, 140475502112767, -STORE, 140475502112768, 140475518894079, -SNULL, 140475411460095, 140475428241407, -STORE, 140475411456000, 140475411460095, -STORE, 140475411460096, 140475428241407, -SNULL, 140475411460096, 140475419848703, -STORE, 140475419848704, 140475428241407, -STORE, 140475411460096, 140475419848703, -SNULL, 140475419852799, 140475428241407, -STORE, 140475419848704, 140475419852799, -STORE, 140475419852800, 140475428241407, -STORE, 140475403063296, 140475411455999, -SNULL, 140475502112768, 140475510501375, -STORE, 140475510501376, 140475518894079, -STORE, 140475502112768, 140475510501375, -SNULL, 140475510505471, 140475518894079, -STORE, 140475510501376, 140475510505471, -STORE, 140475510505472, 140475518894079, -SNULL, 140475403067391, 140475411455999, -STORE, 140475403063296, 140475403067391, -STORE, 140475403067392, 140475411455999, -STORE, 140475394670592, 140475403063295, -SNULL, 140475394674687, 140475403063295, -STORE, 140475394670592, 140475394674687, -STORE, 140475394674688, 140475403063295, -STORE, 140475386277888, 140475394670591, -STORE, 140475377885184, 140475394670591, -STORE, 140475369492480, 140475394670591, -SNULL, 140475369496575, 140475394670591, -STORE, 140475369492480, 140475369496575, -STORE, 140475369496576, 140475394670591, -SNULL, 140475369496576, 140475377885183, -STORE, 140475377885184, 140475394670591, -STORE, 140475369496576, 140475377885183, -SNULL, 140475377889279, 140475394670591, -STORE, 140475377885184, 140475377889279, -STORE, 140475377889280, 140475394670591, -STORE, 140475285630976, 140475294023679, -SNULL, 140475377889280, 140475386277887, -STORE, 140475386277888, 140475394670591, -STORE, 140475377889280, 140475386277887, -SNULL, 140475386281983, 140475394670591, -STORE, 140475386277888, 140475386281983, -STORE, 140475386281984, 140475394670591, -SNULL, 140475285635071, 140475294023679, -STORE, 140475285630976, 140475285635071, -STORE, 140475285635072, 140475294023679, -STORE, 140475277238272, 140475285630975, -STORE, 140475268845568, 140475285630975, -SNULL, 140475268845568, 140475277238271, -STORE, 140475277238272, 140475285630975, -STORE, 140475268845568, 140475277238271, -SNULL, 140475277242367, 140475285630975, -STORE, 140475277238272, 140475277242367, -STORE, 140475277242368, 140475285630975, -STORE, 140475260452864, 140475277238271, -SNULL, 140475260452864, 140475268845567, -STORE, 140475268845568, 140475277238271, -STORE, 140475260452864, 140475268845567, -SNULL, 140475268849663, 140475277238271, -STORE, 140475268845568, 140475268849663, -STORE, 140475268849664, 140475277238271, -SNULL, 140475260456959, 140475268845567, -STORE, 140475260452864, 140475260456959, -STORE, 140475260456960, 140475268845567, -STORE, 140475252060160, 140475260452863, -SNULL, 140475252064255, 140475260452863, -STORE, 140475252060160, 140475252064255, -STORE, 140475252064256, 140475260452863, -STORE, 140475243667456, 140475252060159, -SNULL, 140475243671551, 140475252060159, -STORE, 140475243667456, 140475243671551, -STORE, 140475243671552, 140475252060159, -STORE, 140475235274752, 140475243667455, -STORE, 140475151413248, 140475159805951, -STORE, 140474891505664, 140475025588223, -STORE, 140475143020544, 140475159805951, -SNULL, 140474891505664, 140474958479359, -STORE, 140474958479360, 140475025588223, -STORE, 140474891505664, 140474958479359, -SNULL, 140474958614527, 140475025588223, -STORE, 140474958479360, 140474958614527, -STORE, 140474958614528, 140475025588223, -STORE, 140474824261632, 140474891370495, -SNULL, 140474824396799, 140474891370495, -STORE, 140474824261632, 140474824396799, -STORE, 140474824396800, 140474891370495, -STORE, 140475134627840, 140475159805951, -STORE, 140474690043904, 140474824261631, -STORE, 140475126235136, 140475159805951, -STORE, 140475117842432, 140475159805951, -STORE, 140474622935040, 140474824261631, -STORE, 140475109449728, 140475159805951, -STORE, 140474488717312, 140474824261631, -STORE, 140475101057024, 140475159805951, -STORE, 140474480324608, 140474488717311, -STORE, 140474413215744, 140474480324607, -STORE, 140474404823040, 140474413215743, -ERASE, 140474413215744, 140474480324607, -STORE, 140474471931904, 140474488717311, -STORE, 140474270605312, 140474404823039, -SNULL, 140475101057024, 140475126235135, -STORE, 140475126235136, 140475159805951, -STORE, 140475101057024, 140475126235135, -SNULL, 140475126239231, 140475159805951, -STORE, 140475126235136, 140475126239231, -STORE, 140475126239232, 140475159805951, -STORE, 140474463539200, 140474488717311, -STORE, 140474455146496, 140474488717311, -SNULL, 140474455150591, 140474488717311, -STORE, 140474455146496, 140474455150591, -STORE, 140474455150592, 140474488717311, -STORE, 140474446753792, 140474455146495, -SNULL, 140474446757887, 140474455146495, -STORE, 140474446753792, 140474446757887, -STORE, 140474446757888, 140474455146495, -STORE, 140474438361088, 140474446753791, -STORE, 140474429968384, 140474446753791, -SNULL, 140474429972479, 140474446753791, -STORE, 140474429968384, 140474429972479, -STORE, 140474429972480, 140474446753791, -SNULL, 140475235278847, 140475243667455, -STORE, 140475235274752, 140475235278847, -STORE, 140475235278848, 140475243667455, -SNULL, 140474757152767, 140474824261631, -STORE, 140474488717312, 140474757152767, -STORE, 140474757152768, 140474824261631, -ERASE, 140474757152768, 140474824261631, -SNULL, 140474488717312, 140474690043903, -STORE, 140474690043904, 140474757152767, -STORE, 140474488717312, 140474690043903, -SNULL, 140474690179071, 140474757152767, -STORE, 140474690043904, 140474690179071, -STORE, 140474690179072, 140474757152767, -SNULL, 140474488717312, 140474622935039, -STORE, 140474622935040, 140474690043903, -STORE, 140474488717312, 140474622935039, -SNULL, 140474623070207, 140474690043903, -STORE, 140474622935040, 140474623070207, -STORE, 140474623070208, 140474690043903, -SNULL, 140475101057024, 140475117842431, -STORE, 140475117842432, 140475126235135, -STORE, 140475101057024, 140475117842431, -SNULL, 140475117846527, 140475126235135, -STORE, 140475117842432, 140475117846527, -STORE, 140475117846528, 140475126235135, -SNULL, 140474555826175, 140474622935039, -STORE, 140474488717312, 140474555826175, -STORE, 140474555826176, 140474622935039, -ERASE, 140474555826176, 140474622935039, -STORE, 140474136387584, 140474404823039, -SNULL, 140474136387584, 140474153172991, -STORE, 140474153172992, 140474404823039, -STORE, 140474136387584, 140474153172991, -ERASE, 140474136387584, 140474153172991, -STORE, 140474018955264, 140474404823039, -STORE, 140473884737536, 140474404823039, -SNULL, 140474086064127, 140474404823039, -STORE, 140473884737536, 140474086064127, -STORE, 140474086064128, 140474404823039, -SNULL, 140474086064128, 140474153172991, -STORE, 140474153172992, 140474404823039, -STORE, 140474086064128, 140474153172991, -ERASE, 140474086064128, 140474153172991, -STORE, 140473750519808, 140474086064127, -SNULL, 140473817628671, 140474086064127, -STORE, 140473750519808, 140473817628671, -STORE, 140473817628672, 140474086064127, -SNULL, 140473817628672, 140473884737535, -STORE, 140473884737536, 140474086064127, -STORE, 140473817628672, 140473884737535, -ERASE, 140473817628672, 140473884737535, -SNULL, 140475126239232, 140475151413247, -STORE, 140475151413248, 140475159805951, -STORE, 140475126239232, 140475151413247, -SNULL, 140475151417343, 140475159805951, -STORE, 140475151413248, 140475151417343, -STORE, 140475151417344, 140475159805951, -SNULL, 140474270605311, 140474404823039, -STORE, 140474153172992, 140474270605311, -STORE, 140474270605312, 140474404823039, -SNULL, 140474270605312, 140474287390719, -STORE, 140474287390720, 140474404823039, -STORE, 140474270605312, 140474287390719, -ERASE, 140474270605312, 140474287390719, -SNULL, 140474429972480, 140474438361087, -STORE, 140474438361088, 140474446753791, -STORE, 140474429972480, 140474438361087, -SNULL, 140474438365183, 140474446753791, -STORE, 140474438361088, 140474438365183, -STORE, 140474438365184, 140474446753791, -STORE, 140474815868928, 140474824261631, -SNULL, 140474815873023, 140474824261631, -STORE, 140474815868928, 140474815873023, -STORE, 140474815873024, 140474824261631, -SNULL, 140474220281855, 140474270605311, -STORE, 140474153172992, 140474220281855, -STORE, 140474220281856, 140474270605311, -ERASE, 140474220281856, 140474270605311, -SNULL, 140474488852479, 140474555826175, -STORE, 140474488717312, 140474488852479, -STORE, 140474488852480, 140474555826175, -SNULL, 140475101057024, 140475109449727, -STORE, 140475109449728, 140475117842431, -STORE, 140475101057024, 140475109449727, -SNULL, 140475109453823, 140475117842431, -STORE, 140475109449728, 140475109453823, -STORE, 140475109453824, 140475117842431, -SNULL, 140473951846399, 140474086064127, -STORE, 140473884737536, 140473951846399, -STORE, 140473951846400, 140474086064127, -SNULL, 140473951846400, 140474018955263, -STORE, 140474018955264, 140474086064127, -STORE, 140473951846400, 140474018955263, -ERASE, 140473951846400, 140474018955263, -SNULL, 140473884872703, 140473951846399, -STORE, 140473884737536, 140473884872703, -STORE, 140473884872704, 140473951846399, -SNULL, 140474019090431, 140474086064127, -STORE, 140474018955264, 140474019090431, -STORE, 140474019090432, 140474086064127, -SNULL, 140473750654975, 140473817628671, -STORE, 140473750519808, 140473750654975, -STORE, 140473750654976, 140473817628671, -SNULL, 140474455150592, 140474463539199, -STORE, 140474463539200, 140474488717311, -STORE, 140474455150592, 140474463539199, -SNULL, 140474463543295, 140474488717311, -STORE, 140474463539200, 140474463543295, -STORE, 140474463543296, 140474488717311, -STORE, 140474807476224, 140474815868927, -SNULL, 140474463543296, 140474471931903, -STORE, 140474471931904, 140474488717311, -STORE, 140474463543296, 140474471931903, -SNULL, 140474471935999, 140474488717311, -STORE, 140474471931904, 140474471935999, -STORE, 140474471936000, 140474488717311, -STORE, 140474799083520, 140474815868927, -STORE, 140474790690816, 140474815868927, -SNULL, 140474790690816, 140474799083519, -STORE, 140474799083520, 140474815868927, -STORE, 140474790690816, 140474799083519, -SNULL, 140474799087615, 140474815868927, -STORE, 140474799083520, 140474799087615, -STORE, 140474799087616, 140474815868927, -SNULL, 140474354499583, 140474404823039, -STORE, 140474287390720, 140474354499583, -STORE, 140474354499584, 140474404823039, -ERASE, 140474354499584, 140474404823039, -SNULL, 140474287525887, 140474354499583, -STORE, 140474287390720, 140474287525887, -STORE, 140474287525888, 140474354499583, -STORE, 140474782298112, 140474799083519, -STORE, 140474773905408, 140474799083519, -SNULL, 140474773909503, 140474799083519, -STORE, 140474773905408, 140474773909503, -STORE, 140474773909504, 140474799083519, -SNULL, 140475126239232, 140475134627839, -STORE, 140475134627840, 140475151413247, -STORE, 140475126239232, 140475134627839, -SNULL, 140475134631935, 140475151413247, -STORE, 140475134627840, 140475134631935, -STORE, 140475134631936, 140475151413247, -STORE, 140474765512704, 140474773905407, -STORE, 140474614542336, 140474622935039, -SNULL, 140474153308159, 140474220281855, -STORE, 140474153172992, 140474153308159, -STORE, 140474153308160, 140474220281855, -SNULL, 140474404827135, 140474413215743, -STORE, 140474404823040, 140474404827135, -STORE, 140474404827136, 140474413215743, -STORE, 140474606149632, 140474622935039, -SNULL, 140474606153727, 140474622935039, -STORE, 140474606149632, 140474606153727, -STORE, 140474606153728, 140474622935039, -STORE, 140474597756928, 140474606149631, -SNULL, 140474597761023, 140474606149631, -STORE, 140474597756928, 140474597761023, -STORE, 140474597761024, 140474606149631, -SNULL, 140475134631936, 140475143020543, -STORE, 140475143020544, 140475151413247, -STORE, 140475134631936, 140475143020543, -SNULL, 140475143024639, 140475151413247, -STORE, 140475143020544, 140475143024639, -STORE, 140475143024640, 140475151413247, -STORE, 140474589364224, 140474597756927, -SNULL, 140474606153728, 140474614542335, -STORE, 140474614542336, 140474622935039, -STORE, 140474606153728, 140474614542335, -SNULL, 140474614546431, 140474622935039, -STORE, 140474614542336, 140474614546431, -STORE, 140474614546432, 140474622935039, -SNULL, 140474765516799, 140474773905407, -STORE, 140474765512704, 140474765516799, -STORE, 140474765516800, 140474773905407, -STORE, 140474580971520, 140474597756927, -SNULL, 140474773909504, 140474782298111, -STORE, 140474782298112, 140474799083519, -STORE, 140474773909504, 140474782298111, -SNULL, 140474782302207, 140474799083519, -STORE, 140474782298112, 140474782302207, -STORE, 140474782302208, 140474799083519, -SNULL, 140474471936000, 140474480324607, -STORE, 140474480324608, 140474488717311, -STORE, 140474471936000, 140474480324607, -SNULL, 140474480328703, 140474488717311, -STORE, 140474480324608, 140474480328703, -STORE, 140474480328704, 140474488717311, -STORE, 140474572578816, 140474597756927, -SNULL, 140474572582911, 140474597756927, -STORE, 140474572578816, 140474572582911, -STORE, 140474572582912, 140474597756927, -SNULL, 140474782302208, 140474790690815, -STORE, 140474790690816, 140474799083519, -STORE, 140474782302208, 140474790690815, -SNULL, 140474790694911, 140474799083519, -STORE, 140474790690816, 140474790694911, -STORE, 140474790694912, 140474799083519, -STORE, 140474564186112, 140474572578815, -STORE, 140474421575680, 140474429968383, -STORE, 140474396430336, 140474404823039, -SNULL, 140474396434431, 140474404823039, -STORE, 140474396430336, 140474396434431, -STORE, 140474396434432, 140474404823039, -STORE, 140474388037632, 140474396430335, -SNULL, 140474799087616, 140474807476223, -STORE, 140474807476224, 140474815868927, -STORE, 140474799087616, 140474807476223, -SNULL, 140474807480319, 140474815868927, -STORE, 140474807476224, 140474807480319, -STORE, 140474807480320, 140474815868927, -SNULL, 140475101061119, 140475109449727, -STORE, 140475101057024, 140475101061119, -STORE, 140475101061120, 140475109449727, -STORE, 140474379644928, 140474396430335, -SNULL, 140474572582912, 140474589364223, -STORE, 140474589364224, 140474597756927, -STORE, 140474572582912, 140474589364223, -SNULL, 140474589368319, 140474597756927, -STORE, 140474589364224, 140474589368319, -STORE, 140474589368320, 140474597756927, -STORE, 140474371252224, 140474396430335, -STORE, 140474362859520, 140474396430335, -STORE, 140474278998016, 140474287390719, -STORE, 140474270605312, 140474287390719, -STORE, 140474262212608, 140474287390719, -SNULL, 140474262216703, 140474287390719, -STORE, 140474262212608, 140474262216703, -STORE, 140474262216704, 140474287390719, -STORE, 140474253819904, 140474262212607, -SNULL, 140474253823999, 140474262212607, -STORE, 140474253819904, 140474253823999, -STORE, 140474253824000, 140474262212607, -SNULL, 140474362859520, 140474388037631, -STORE, 140474388037632, 140474396430335, -STORE, 140474362859520, 140474388037631, -SNULL, 140474388041727, 140474396430335, -STORE, 140474388037632, 140474388041727, -STORE, 140474388041728, 140474396430335, -SNULL, 140474362859520, 140474379644927, -STORE, 140474379644928, 140474388037631, -STORE, 140474362859520, 140474379644927, -SNULL, 140474379649023, 140474388037631, -STORE, 140474379644928, 140474379649023, -STORE, 140474379649024, 140474388037631, -STORE, 140474245427200, 140474253819903, -STORE, 140474237034496, 140474253819903, -STORE, 140474228641792, 140474253819903, -STORE, 140474144780288, 140474153172991, -SNULL, 140474228645887, 140474253819903, -STORE, 140474228641792, 140474228645887, -STORE, 140474228645888, 140474253819903, -SNULL, 140474564190207, 140474572578815, -STORE, 140474564186112, 140474564190207, -STORE, 140474564190208, 140474572578815, -STORE, 140474136387584, 140474153172991, -SNULL, 140474362859520, 140474371252223, -STORE, 140474371252224, 140474379644927, -STORE, 140474362859520, 140474371252223, -SNULL, 140474371256319, 140474379644927, -STORE, 140474371252224, 140474371256319, -STORE, 140474371256320, 140474379644927, -STORE, 140474127994880, 140474153172991, -STORE, 140474119602176, 140474153172991, -SNULL, 140474421579775, 140474429968383, -STORE, 140474421575680, 140474421579775, -STORE, 140474421579776, 140474429968383, -STORE, 140474111209472, 140474153172991, -SNULL, 140474111213567, 140474153172991, -STORE, 140474111209472, 140474111213567, -STORE, 140474111213568, 140474153172991, -SNULL, 140474262216704, 140474270605311, -STORE, 140474270605312, 140474287390719, -STORE, 140474262216704, 140474270605311, -SNULL, 140474270609407, 140474287390719, -STORE, 140474270605312, 140474270609407, -STORE, 140474270609408, 140474287390719, -STORE, 140474102816768, 140474111209471, -SNULL, 140474102820863, 140474111209471, -STORE, 140474102816768, 140474102820863, -STORE, 140474102820864, 140474111209471, -SNULL, 140474270609408, 140474278998015, -STORE, 140474278998016, 140474287390719, -STORE, 140474270609408, 140474278998015, -SNULL, 140474279002111, 140474287390719, -STORE, 140474278998016, 140474279002111, -STORE, 140474279002112, 140474287390719, -STORE, 140474094424064, 140474102816767, -SNULL, 140474572582912, 140474580971519, -STORE, 140474580971520, 140474589364223, -STORE, 140474572582912, 140474580971519, -SNULL, 140474580975615, 140474589364223, -STORE, 140474580971520, 140474580975615, -STORE, 140474580975616, 140474589364223, -SNULL, 140474362863615, 140474371252223, -STORE, 140474362859520, 140474362863615, -STORE, 140474362863616, 140474371252223, -STORE, 140474010562560, 140474018955263, -SNULL, 140474228645888, 140474245427199, -STORE, 140474245427200, 140474253819903, -STORE, 140474228645888, 140474245427199, -SNULL, 140474245431295, 140474253819903, -STORE, 140474245427200, 140474245431295, -STORE, 140474245431296, 140474253819903, -SNULL, 140474111213568, 140474136387583, -STORE, 140474136387584, 140474153172991, -STORE, 140474111213568, 140474136387583, -SNULL, 140474136391679, 140474153172991, -STORE, 140474136387584, 140474136391679, -STORE, 140474136391680, 140474153172991, -STORE, 140474002169856, 140474018955263, -STORE, 140473993777152, 140474018955263, -SNULL, 140474111213568, 140474127994879, -STORE, 140474127994880, 140474136387583, -STORE, 140474111213568, 140474127994879, -SNULL, 140474127998975, 140474136387583, -STORE, 140474127994880, 140474127998975, -STORE, 140474127998976, 140474136387583, -SNULL, 140474228645888, 140474237034495, -STORE, 140474237034496, 140474245427199, -STORE, 140474228645888, 140474237034495, -SNULL, 140474237038591, 140474245427199, -STORE, 140474237034496, 140474237038591, -STORE, 140474237038592, 140474245427199, -SNULL, 140474136391680, 140474144780287, -STORE, 140474144780288, 140474153172991, -STORE, 140474136391680, 140474144780287, -SNULL, 140474144784383, 140474153172991, -STORE, 140474144780288, 140474144784383, -STORE, 140474144784384, 140474153172991, -STORE, 140473985384448, 140474018955263, -STORE, 140473976991744, 140474018955263, -STORE, 140473968599040, 140474018955263, -SNULL, 140473968603135, 140474018955263, -STORE, 140473968599040, 140473968603135, -STORE, 140473968603136, 140474018955263, -SNULL, 140474111213568, 140474119602175, -STORE, 140474119602176, 140474127994879, -STORE, 140474111213568, 140474119602175, -SNULL, 140474119606271, 140474127994879, -STORE, 140474119602176, 140474119606271, -STORE, 140474119606272, 140474127994879, -STORE, 140473960206336, 140473968599039, -SNULL, 140474094428159, 140474102816767, -STORE, 140474094424064, 140474094428159, -STORE, 140474094428160, 140474102816767, -STORE, 140473876344832, 140473884737535, -STORE, 140473867952128, 140473884737535, -STORE, 140473859559424, 140473884737535, -SNULL, 140473859563519, 140473884737535, -STORE, 140473859559424, 140473859563519, -STORE, 140473859563520, 140473884737535, -SNULL, 140473968603136, 140473993777151, -STORE, 140473993777152, 140474018955263, -STORE, 140473968603136, 140473993777151, -SNULL, 140473993781247, 140474018955263, -STORE, 140473993777152, 140473993781247, -STORE, 140473993781248, 140474018955263, -SNULL, 140473960210431, 140473968599039, -STORE, 140473960206336, 140473960210431, -STORE, 140473960210432, 140473968599039, -SNULL, 140473993781248, 140474010562559, -STORE, 140474010562560, 140474018955263, -STORE, 140473993781248, 140474010562559, -SNULL, 140474010566655, 140474018955263, -STORE, 140474010562560, 140474010566655, -STORE, 140474010566656, 140474018955263, -SNULL, 140473968603136, 140473985384447, -STORE, 140473985384448, 140473993777151, -STORE, 140473968603136, 140473985384447, -SNULL, 140473985388543, 140473993777151, -STORE, 140473985384448, 140473985388543, -STORE, 140473985388544, 140473993777151, -SNULL, 140473993781248, 140474002169855, -STORE, 140474002169856, 140474010562559, -STORE, 140473993781248, 140474002169855, -SNULL, 140474002173951, 140474010562559, -STORE, 140474002169856, 140474002173951, -STORE, 140474002173952, 140474010562559, -STORE, 140473851166720, 140473859559423, -SNULL, 140473851170815, 140473859559423, -STORE, 140473851166720, 140473851170815, -STORE, 140473851170816, 140473859559423, -SNULL, 140473968603136, 140473976991743, -STORE, 140473976991744, 140473985384447, -STORE, 140473968603136, 140473976991743, -SNULL, 140473976995839, 140473985384447, -STORE, 140473976991744, 140473976995839, -STORE, 140473976995840, 140473985384447, -STORE, 140473842774016, 140473851166719, -SNULL, 140473859563520, 140473867952127, -STORE, 140473867952128, 140473884737535, -STORE, 140473859563520, 140473867952127, -SNULL, 140473867956223, 140473884737535, -STORE, 140473867952128, 140473867956223, -STORE, 140473867956224, 140473884737535, -SNULL, 140473867956224, 140473876344831, -STORE, 140473876344832, 140473884737535, -STORE, 140473867956224, 140473876344831, -SNULL, 140473876348927, 140473884737535, -STORE, 140473876344832, 140473876348927, -STORE, 140473876348928, 140473884737535, -STORE, 140473834381312, 140473851166719, -SNULL, 140473834385407, 140473851166719, -STORE, 140473834381312, 140473834385407, -STORE, 140473834385408, 140473851166719, -SNULL, 140473834385408, 140473842774015, -STORE, 140473842774016, 140473851166719, -STORE, 140473834385408, 140473842774015, -SNULL, 140473842778111, 140473851166719, -STORE, 140473842774016, 140473842778111, -STORE, 140473842778112, 140473851166719, -STORE, 140473825988608, 140473834381311, -SNULL, 140473825992703, 140473834381311, -STORE, 140473825988608, 140473825992703, -STORE, 140473825992704, 140473834381311, -STORE, 140475577475072, 140475577503743, -STORE, 140475499917312, 140475502108671, -SNULL, 140475499917312, 140475500007423, -STORE, 140475500007424, 140475502108671, -STORE, 140475499917312, 140475500007423, -SNULL, 140475502100479, 140475502108671, -STORE, 140475500007424, 140475502100479, -STORE, 140475502100480, 140475502108671, -ERASE, 140475502100480, 140475502108671, -STORE, 140475502100480, 140475502108671, -SNULL, 140475502104575, 140475502108671, -STORE, 140475502100480, 140475502104575, -STORE, 140475502104576, 140475502108671, -ERASE, 140475577475072, 140475577503743, -ERASE, 140475235274752, 140475235278847, -ERASE, 140475235278848, 140475243667455, -ERASE, 140474815868928, 140474815873023, -ERASE, 140474815873024, 140474824261631, -ERASE, 140474606149632, 140474606153727, -ERASE, 140474606153728, 140474614542335, -ERASE, 140474270605312, 140474270609407, -ERASE, 140474270609408, 140474278998015, -ERASE, 140474438361088, 140474438365183, -ERASE, 140474438365184, 140474446753791, -ERASE, 140474597756928, 140474597761023, -ERASE, 140474597761024, 140474606149631, -ERASE, 140475126235136, 140475126239231, -ERASE, 140475126239232, 140475134627839, -ERASE, 140474463539200, 140474463543295, -ERASE, 140474463543296, 140474471931903, -ERASE, 140474388037632, 140474388041727, -ERASE, 140474388041728, 140474396430335, -ERASE, 140474404823040, 140474404827135, -ERASE, 140474404827136, 140474413215743, -ERASE, 140474278998016, 140474279002111, -ERASE, 140474279002112, 140474287390719, -ERASE, 140474094424064, 140474094428159, -ERASE, 140474094428160, 140474102816767, -ERASE, 140473867952128, 140473867956223, -ERASE, 140473867956224, 140473876344831, -ERASE, 140475151413248, 140475151417343, -ERASE, 140475151417344, 140475159805951, -ERASE, 140474455146496, 140474455150591, -ERASE, 140474455150592, 140474463539199, -ERASE, 140474807476224, 140474807480319, -ERASE, 140474807480320, 140474815868927, -ERASE, 140475117842432, 140475117846527, -ERASE, 140475117846528, 140475126235135, -ERASE, 140474446753792, 140474446757887, -ERASE, 140474446757888, 140474455146495, -ERASE, 140474429968384, 140474429972479, -ERASE, 140474429972480, 140474438361087, -ERASE, 140474782298112, 140474782302207, -ERASE, 140474782302208, 140474790690815, -ERASE, 140474136387584, 140474136391679, -ERASE, 140474136391680, 140474144780287, -ERASE, 140474002169856, 140474002173951, -ERASE, 140474002173952, 140474010562559, -ERASE, 140475134627840, 140475134631935, -ERASE, 140475134631936, 140475143020543, -ERASE, 140474471931904, 140474471935999, -ERASE, 140474471936000, 140474480324607, -ERASE, 140474396430336, 140474396434431, -ERASE, 140474396434432, 140474404823039, - }; - unsigned long set36[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140723893125120, 140737488351231, -SNULL, 140723893129215, 140737488351231, -STORE, 140723893125120, 140723893129215, -STORE, 140723892994048, 140723893129215, -STORE, 94076829786112, 94076832038911, -SNULL, 94076829917183, 94076832038911, -STORE, 94076829786112, 94076829917183, -STORE, 94076829917184, 94076832038911, -ERASE, 94076829917184, 94076832038911, -STORE, 94076832010240, 94076832018431, -STORE, 94076832018432, 94076832038911, -STORE, 140122444345344, 140122446598143, -SNULL, 140122444488703, 140122446598143, -STORE, 140122444345344, 140122444488703, -STORE, 140122444488704, 140122446598143, -ERASE, 140122444488704, 140122446598143, -STORE, 140122446585856, 140122446594047, -STORE, 140122446594048, 140122446598143, -STORE, 140723893538816, 140723893542911, -STORE, 140723893526528, 140723893538815, -STORE, 140122446557184, 140122446585855, -STORE, 140122446548992, 140122446557183, -STORE, 140122442129408, 140122444345343, -SNULL, 140122442129408, 140122442227711, -STORE, 140122442227712, 140122444345343, -STORE, 140122442129408, 140122442227711, -SNULL, 140122444320767, 140122444345343, -STORE, 140122442227712, 140122444320767, -STORE, 140122444320768, 140122444345343, -SNULL, 140122444320768, 140122444328959, -STORE, 140122444328960, 140122444345343, -STORE, 140122444320768, 140122444328959, -ERASE, 140122444320768, 140122444328959, -STORE, 140122444320768, 140122444328959, -ERASE, 140122444328960, 140122444345343, -STORE, 140122444328960, 140122444345343, -STORE, 140122438332416, 140122442129407, -SNULL, 140122438332416, 140122439991295, -STORE, 140122439991296, 140122442129407, -STORE, 140122438332416, 140122439991295, -SNULL, 140122442088447, 140122442129407, -STORE, 140122439991296, 140122442088447, -STORE, 140122442088448, 140122442129407, -SNULL, 140122442088448, 140122442113023, -STORE, 140122442113024, 140122442129407, -STORE, 140122442088448, 140122442113023, -ERASE, 140122442088448, 140122442113023, -STORE, 140122442088448, 140122442113023, -ERASE, 140122442113024, 140122442129407, -STORE, 140122442113024, 140122442129407, -STORE, 140122446540800, 140122446557183, -SNULL, 140122442104831, 140122442113023, -STORE, 140122442088448, 140122442104831, -STORE, 140122442104832, 140122442113023, -SNULL, 140122444324863, 140122444328959, -STORE, 140122444320768, 140122444324863, -STORE, 140122444324864, 140122444328959, -SNULL, 94076832014335, 94076832018431, -STORE, 94076832010240, 94076832014335, -STORE, 94076832014336, 94076832018431, -SNULL, 140122446589951, 140122446594047, -STORE, 140122446585856, 140122446589951, -STORE, 140122446589952, 140122446594047, -ERASE, 140122446557184, 140122446585855, -STORE, 94076845723648, 94076845858815, -STORE, 140122429939712, 140122438332415, -SNULL, 140122429943807, 140122438332415, -STORE, 140122429939712, 140122429943807, -STORE, 140122429943808, 140122438332415, -STORE, 140122421547008, 140122429939711, -STORE, 140122287329280, 140122421547007, -SNULL, 140122287329280, 140122301399039, -STORE, 140122301399040, 140122421547007, -STORE, 140122287329280, 140122301399039, -ERASE, 140122287329280, 140122301399039, -SNULL, 140122368507903, 140122421547007, -STORE, 140122301399040, 140122368507903, -STORE, 140122368507904, 140122421547007, -ERASE, 140122368507904, 140122421547007, -SNULL, 140122301534207, 140122368507903, -STORE, 140122301399040, 140122301534207, -STORE, 140122301534208, 140122368507903, -SNULL, 140122421551103, 140122429939711, -STORE, 140122421547008, 140122421551103, -STORE, 140122421551104, 140122429939711, -STORE, 140122413154304, 140122421547007, -SNULL, 140122413158399, 140122421547007, -STORE, 140122413154304, 140122413158399, -STORE, 140122413158400, 140122421547007, -STORE, 140122404761600, 140122413154303, -SNULL, 140122404765695, 140122413154303, -STORE, 140122404761600, 140122404765695, -STORE, 140122404765696, 140122413154303, -STORE, 140122396368896, 140122404761599, -SNULL, 140122396372991, 140122404761599, -STORE, 140122396368896, 140122396372991, -STORE, 140122396372992, 140122404761599, -STORE, 140122387976192, 140122396368895, -STORE, 140122167181312, 140122301399039, -SNULL, 140122234290175, 140122301399039, -STORE, 140122167181312, 140122234290175, -STORE, 140122234290176, 140122301399039, -ERASE, 140122234290176, 140122301399039, -SNULL, 140122167316479, 140122234290175, -STORE, 140122167181312, 140122167316479, -STORE, 140122167316480, 140122234290175, -STORE, 140122379583488, 140122396368895, -STORE, 140122371190784, 140122396368895, -STORE, 140122167316480, 140122301399039, -STORE, 140122158788608, 140122167181311, -SNULL, 140122371190784, 140122387976191, -STORE, 140122387976192, 140122396368895, -STORE, 140122371190784, 140122387976191, -SNULL, 140122387980287, 140122396368895, -STORE, 140122387976192, 140122387980287, -STORE, 140122387980288, 140122396368895, -SNULL, 140122167316480, 140122234290175, -STORE, 140122234290176, 140122301399039, -STORE, 140122167316480, 140122234290175, -SNULL, 140122234425343, 140122301399039, -STORE, 140122234290176, 140122234425343, -STORE, 140122234425344, 140122301399039, -STORE, 140122024570880, 140122158788607, -SNULL, 140122024570880, 140122032963583, -STORE, 140122032963584, 140122158788607, -STORE, 140122024570880, 140122032963583, -ERASE, 140122024570880, 140122032963583, -STORE, 140121898745856, 140122158788607, -STORE, 140121890353152, 140121898745855, -SNULL, 140122100072447, 140122158788607, -STORE, 140121898745856, 140122100072447, -STORE, 140122100072448, 140122158788607, -ERASE, 140122100072448, 140122158788607, -SNULL, 140121965854719, 140122100072447, -STORE, 140121898745856, 140121965854719, -STORE, 140121965854720, 140122100072447, -SNULL, 140121965854720, 140122032963583, -STORE, 140122032963584, 140122100072447, -STORE, 140121965854720, 140122032963583, -ERASE, 140121965854720, 140122032963583, -SNULL, 140121898881023, 140121965854719, -STORE, 140121898745856, 140121898881023, -STORE, 140121898881024, 140121965854719, -SNULL, 140121890357247, 140121898745855, -STORE, 140121890353152, 140121890357247, -STORE, 140121890357248, 140121898745855, -SNULL, 140122371190784, 140122379583487, -STORE, 140122379583488, 140122387976191, -STORE, 140122371190784, 140122379583487, -SNULL, 140122379587583, 140122387976191, -STORE, 140122379583488, 140122379587583, -STORE, 140122379587584, 140122387976191, -SNULL, 140122033098751, 140122100072447, -STORE, 140122032963584, 140122033098751, -STORE, 140122033098752, 140122100072447, -SNULL, 140122158792703, 140122167181311, -STORE, 140122158788608, 140122158792703, -STORE, 140122158792704, 140122167181311, -STORE, 140122150395904, 140122158788607, -STORE, 140122142003200, 140122158788607, -SNULL, 140122142007295, 140122158788607, -STORE, 140122142003200, 140122142007295, -STORE, 140122142007296, 140122158788607, -SNULL, 140122371194879, 140122379583487, -STORE, 140122371190784, 140122371194879, -STORE, 140122371194880, 140122379583487, -SNULL, 140122142007296, 140122150395903, -STORE, 140122150395904, 140122158788607, -STORE, 140122142007296, 140122150395903, -SNULL, 140122150399999, 140122158788607, -STORE, 140122150395904, 140122150399999, -STORE, 140122150400000, 140122158788607, -STORE, 140122133610496, 140122142003199, -STORE, 140122125217792, 140122142003199, -STORE, 140122116825088, 140122142003199, -SNULL, 140122116829183, 140122142003199, -STORE, 140122116825088, 140122116829183, -STORE, 140122116829184, 140122142003199, -SNULL, 140122116829184, 140122133610495, -STORE, 140122133610496, 140122142003199, -STORE, 140122116829184, 140122133610495, -SNULL, 140122133614591, 140122142003199, -STORE, 140122133610496, 140122133614591, -STORE, 140122133614592, 140122142003199, -SNULL, 140122116829184, 140122125217791, -STORE, 140122125217792, 140122133610495, -STORE, 140122116829184, 140122125217791, -SNULL, 140122125221887, 140122133610495, -STORE, 140122125217792, 140122125221887, -STORE, 140122125221888, 140122133610495, -STORE, 140122108432384, 140122116825087, -SNULL, 140122108436479, 140122116825087, -STORE, 140122108432384, 140122108436479, -STORE, 140122108436480, 140122116825087, -STORE, 140122024570880, 140122032963583, -STORE, 140122016178176, 140122032963583, -SNULL, 140122016182271, 140122032963583, -STORE, 140122016178176, 140122016182271, -STORE, 140122016182272, 140122032963583, -SNULL, 140122016182272, 140122024570879, -STORE, 140122024570880, 140122032963583, -STORE, 140122016182272, 140122024570879, -SNULL, 140122024574975, 140122032963583, -STORE, 140122024570880, 140122024574975, -STORE, 140122024574976, 140122032963583, -STORE, 140122007785472, 140122016178175, -SNULL, 140122007789567, 140122016178175, -STORE, 140122007785472, 140122007789567, -STORE, 140122007789568, 140122016178175, -STORE, 140121999392768, 140122007785471, -STORE, 140121991000064, 140122007785471, -SNULL, 140121991004159, 140122007785471, -STORE, 140121991000064, 140121991004159, -STORE, 140121991004160, 140122007785471, -SNULL, 140121991004160, 140121999392767, -STORE, 140121999392768, 140122007785471, -STORE, 140121991004160, 140121999392767, -SNULL, 140121999396863, 140122007785471, -STORE, 140121999392768, 140121999396863, -STORE, 140121999396864, 140122007785471, -STORE, 140121982607360, 140121991000063, -STORE, 140121823244288, 140121890353151, -ERASE, 140121823244288, 140121890353151, -STORE, 140121756135424, 140121890353151, -SNULL, 140121756135424, 140121764528127, -STORE, 140121764528128, 140121890353151, -STORE, 140121756135424, 140121764528127, -ERASE, 140121756135424, 140121764528127, -SNULL, 140121831636991, 140121890353151, -STORE, 140121764528128, 140121831636991, -STORE, 140121831636992, 140121890353151, -ERASE, 140121831636992, 140121890353151, -STORE, 140121974214656, 140121991000063, -STORE, 140121630310400, 140121831636991, -SNULL, 140121697419263, 140121831636991, -STORE, 140121630310400, 140121697419263, -STORE, 140121697419264, 140121831636991, -SNULL, 140121697419264, 140121764528127, -STORE, 140121764528128, 140121831636991, -STORE, 140121697419264, 140121764528127, -ERASE, 140121697419264, 140121764528127, -STORE, 140121881960448, 140121890353151, -STORE, 140121630310400, 140121831636991, -STORE, 140121873567744, 140121890353151, -SNULL, 140121630310400, 140121697419263, -STORE, 140121697419264, 140121831636991, -STORE, 140121630310400, 140121697419263, -SNULL, 140121697554431, 140121831636991, -STORE, 140121697419264, 140121697554431, -STORE, 140121697554432, 140121831636991, -STORE, 140121865175040, 140121890353151, -STORE, 140121856782336, 140121890353151, -STORE, 140121848389632, 140121890353151, -STORE, 140121839996928, 140121890353151, -STORE, 140121496092672, 140121697419263, -STORE, 140121487699968, 140121496092671, -STORE, 140121420591104, 140121487699967, -STORE, 140121412198400, 140121420591103, -ERASE, 140121420591104, 140121487699967, -STORE, 140121479307264, 140121496092671, -STORE, 140121277980672, 140121412198399, -SNULL, 140121277980672, 140121294766079, -STORE, 140121294766080, 140121412198399, -STORE, 140121277980672, 140121294766079, -ERASE, 140121277980672, 140121294766079, -STORE, 140121470914560, 140121496092671, -STORE, 140121462521856, 140121496092671, -STORE, 140121160548352, 140121412198399, -STORE, 140121454129152, 140121496092671, -SNULL, 140121227657215, 140121412198399, -STORE, 140121160548352, 140121227657215, -STORE, 140121227657216, 140121412198399, -SNULL, 140121227657216, 140121294766079, -STORE, 140121294766080, 140121412198399, -STORE, 140121227657216, 140121294766079, -ERASE, 140121227657216, 140121294766079, -STORE, 140121445736448, 140121496092671, -STORE, 140121437343744, 140121496092671, -SNULL, 140121437343744, 140121445736447, -STORE, 140121445736448, 140121496092671, -STORE, 140121437343744, 140121445736447, -SNULL, 140121445740543, 140121496092671, -STORE, 140121445736448, 140121445740543, -STORE, 140121445740544, 140121496092671, -SNULL, 140121697554432, 140121764528127, -STORE, 140121764528128, 140121831636991, -STORE, 140121697554432, 140121764528127, -SNULL, 140121764663295, 140121831636991, -STORE, 140121764528128, 140121764663295, -STORE, 140121764663296, 140121831636991, -SNULL, 140121496092672, 140121630310399, -STORE, 140121630310400, 140121697419263, -STORE, 140121496092672, 140121630310399, -SNULL, 140121630445567, 140121697419263, -STORE, 140121630310400, 140121630445567, -STORE, 140121630445568, 140121697419263, -SNULL, 140121445740544, 140121454129151, -STORE, 140121454129152, 140121496092671, -STORE, 140121445740544, 140121454129151, -SNULL, 140121454133247, 140121496092671, -STORE, 140121454129152, 140121454133247, -STORE, 140121454133248, 140121496092671, -STORE, 140121026330624, 140121227657215, -SNULL, 140121093439487, 140121227657215, -STORE, 140121026330624, 140121093439487, -STORE, 140121093439488, 140121227657215, -SNULL, 140121093439488, 140121160548351, -STORE, 140121160548352, 140121227657215, -STORE, 140121093439488, 140121160548351, -ERASE, 140121093439488, 140121160548351, -SNULL, 140121563201535, 140121630310399, -STORE, 140121496092672, 140121563201535, -STORE, 140121563201536, 140121630310399, -ERASE, 140121563201536, 140121630310399, -STORE, 140120892112896, 140121093439487, -SNULL, 140120959221759, 140121093439487, -STORE, 140120892112896, 140120959221759, -STORE, 140120959221760, 140121093439487, -SNULL, 140120959221760, 140121026330623, -STORE, 140121026330624, 140121093439487, -STORE, 140120959221760, 140121026330623, -ERASE, 140120959221760, 140121026330623, -STORE, 140120757895168, 140120959221759, -SNULL, 140121361874943, 140121412198399, -STORE, 140121294766080, 140121361874943, -STORE, 140121361874944, 140121412198399, -ERASE, 140121361874944, 140121412198399, -SNULL, 140121294901247, 140121361874943, -STORE, 140121294766080, 140121294901247, -STORE, 140121294901248, 140121361874943, -STORE, 140120623677440, 140120959221759, -SNULL, 140120690786303, 140120959221759, -STORE, 140120623677440, 140120690786303, -STORE, 140120690786304, 140120959221759, -SNULL, 140120690786304, 140120757895167, -STORE, 140120757895168, 140120959221759, -STORE, 140120690786304, 140120757895167, -ERASE, 140120690786304, 140120757895167, -SNULL, 140121160683519, 140121227657215, -STORE, 140121160548352, 140121160683519, -STORE, 140121160683520, 140121227657215, -SNULL, 140121974214656, 140121982607359, -STORE, 140121982607360, 140121991000063, -STORE, 140121974214656, 140121982607359, -SNULL, 140121982611455, 140121991000063, -STORE, 140121982607360, 140121982611455, -STORE, 140121982611456, 140121991000063, -SNULL, 140121839996928, 140121873567743, -STORE, 140121873567744, 140121890353151, -STORE, 140121839996928, 140121873567743, -SNULL, 140121873571839, 140121890353151, -STORE, 140121873567744, 140121873571839, -STORE, 140121873571840, 140121890353151, -SNULL, 140121873571840, 140121881960447, -STORE, 140121881960448, 140121890353151, -STORE, 140121873571840, 140121881960447, -SNULL, 140121881964543, 140121890353151, -STORE, 140121881960448, 140121881964543, -STORE, 140121881964544, 140121890353151, -SNULL, 140121840001023, 140121873567743, -STORE, 140121839996928, 140121840001023, -STORE, 140121840001024, 140121873567743, -SNULL, 140121840001024, 140121865175039, -STORE, 140121865175040, 140121873567743, -STORE, 140121840001024, 140121865175039, -SNULL, 140121865179135, 140121873567743, -STORE, 140121865175040, 140121865179135, -STORE, 140121865179136, 140121873567743, -SNULL, 140121437347839, 140121445736447, -STORE, 140121437343744, 140121437347839, -STORE, 140121437347840, 140121445736447, -STORE, 140121621917696, 140121630310399, -STORE, 140121613524992, 140121630310399, -SNULL, 140121026465791, 140121093439487, -STORE, 140121026330624, 140121026465791, -STORE, 140121026465792, 140121093439487, -SNULL, 140121496227839, 140121563201535, -STORE, 140121496092672, 140121496227839, -STORE, 140121496227840, 140121563201535, -SNULL, 140120757895168, 140120892112895, -STORE, 140120892112896, 140120959221759, -STORE, 140120757895168, 140120892112895, -SNULL, 140120892248063, 140120959221759, -STORE, 140120892112896, 140120892248063, -STORE, 140120892248064, 140120959221759, -SNULL, 140120825004031, 140120892112895, -STORE, 140120757895168, 140120825004031, -STORE, 140120825004032, 140120892112895, -ERASE, 140120825004032, 140120892112895, -SNULL, 140120623812607, 140120690786303, -STORE, 140120623677440, 140120623812607, -STORE, 140120623812608, 140120690786303, -SNULL, 140120758030335, 140120825004031, -STORE, 140120757895168, 140120758030335, -STORE, 140120758030336, 140120825004031, -SNULL, 140121454133248, 140121462521855, -STORE, 140121462521856, 140121496092671, -STORE, 140121454133248, 140121462521855, -SNULL, 140121462525951, 140121496092671, -STORE, 140121462521856, 140121462525951, -STORE, 140121462525952, 140121496092671, -STORE, 140121605132288, 140121630310399, -SNULL, 140121605136383, 140121630310399, -STORE, 140121605132288, 140121605136383, -STORE, 140121605136384, 140121630310399, -STORE, 140121596739584, 140121605132287, -SNULL, 140121605136384, 140121621917695, -STORE, 140121621917696, 140121630310399, -STORE, 140121605136384, 140121621917695, -SNULL, 140121621921791, 140121630310399, -STORE, 140121621917696, 140121621921791, -STORE, 140121621921792, 140121630310399, -STORE, 140121588346880, 140121605132287, -STORE, 140121579954176, 140121605132287, -SNULL, 140121412202495, 140121420591103, -STORE, 140121412198400, 140121412202495, -STORE, 140121412202496, 140121420591103, -SNULL, 140121974218751, 140121982607359, -STORE, 140121974214656, 140121974218751, -STORE, 140121974218752, 140121982607359, -SNULL, 140121462525952, 140121479307263, -STORE, 140121479307264, 140121496092671, -STORE, 140121462525952, 140121479307263, -SNULL, 140121479311359, 140121496092671, -STORE, 140121479307264, 140121479311359, -STORE, 140121479311360, 140121496092671, -STORE, 140121571561472, 140121605132287, -SNULL, 140121571565567, 140121605132287, -STORE, 140121571561472, 140121571565567, -STORE, 140121571565568, 140121605132287, -STORE, 140121428951040, 140121437343743, -SNULL, 140121428955135, 140121437343743, -STORE, 140121428951040, 140121428955135, -STORE, 140121428955136, 140121437343743, -SNULL, 140121840001024, 140121856782335, -STORE, 140121856782336, 140121865175039, -STORE, 140121840001024, 140121856782335, -SNULL, 140121856786431, 140121865175039, -STORE, 140121856782336, 140121856786431, -STORE, 140121856786432, 140121865175039, -STORE, 140121403805696, 140121412198399, -SNULL, 140121840001024, 140121848389631, -STORE, 140121848389632, 140121856782335, -STORE, 140121840001024, 140121848389631, -SNULL, 140121848393727, 140121856782335, -STORE, 140121848389632, 140121848393727, -STORE, 140121848393728, 140121856782335, -SNULL, 140121479311360, 140121487699967, -STORE, 140121487699968, 140121496092671, -STORE, 140121479311360, 140121487699967, -SNULL, 140121487704063, 140121496092671, -STORE, 140121487699968, 140121487704063, -STORE, 140121487704064, 140121496092671, -STORE, 140121395412992, 140121412198399, -STORE, 140121387020288, 140121412198399, -SNULL, 140121387024383, 140121412198399, -STORE, 140121387020288, 140121387024383, -STORE, 140121387024384, 140121412198399, -SNULL, 140121605136384, 140121613524991, -STORE, 140121613524992, 140121621917695, -STORE, 140121605136384, 140121613524991, -SNULL, 140121613529087, 140121621917695, -STORE, 140121613524992, 140121613529087, -STORE, 140121613529088, 140121621917695, -SNULL, 140121462525952, 140121470914559, -STORE, 140121470914560, 140121479307263, -STORE, 140121462525952, 140121470914559, -SNULL, 140121470918655, 140121479307263, -STORE, 140121470914560, 140121470918655, -STORE, 140121470918656, 140121479307263, -STORE, 140121378627584, 140121387020287, -SNULL, 140121378631679, 140121387020287, -STORE, 140121378627584, 140121378631679, -STORE, 140121378631680, 140121387020287, -SNULL, 140121571565568, 140121596739583, -STORE, 140121596739584, 140121605132287, -STORE, 140121571565568, 140121596739583, -SNULL, 140121596743679, 140121605132287, -STORE, 140121596739584, 140121596743679, -STORE, 140121596743680, 140121605132287, -SNULL, 140121387024384, 140121403805695, -STORE, 140121403805696, 140121412198399, -STORE, 140121387024384, 140121403805695, -SNULL, 140121403809791, 140121412198399, -STORE, 140121403805696, 140121403809791, -STORE, 140121403809792, 140121412198399, -STORE, 140121370234880, 140121378627583, -SNULL, 140121387024384, 140121395412991, -STORE, 140121395412992, 140121403805695, -STORE, 140121387024384, 140121395412991, -SNULL, 140121395417087, 140121403805695, -STORE, 140121395412992, 140121395417087, -STORE, 140121395417088, 140121403805695, -SNULL, 140121571565568, 140121588346879, -STORE, 140121588346880, 140121596739583, -STORE, 140121571565568, 140121588346879, -SNULL, 140121588350975, 140121596739583, -STORE, 140121588346880, 140121588350975, -STORE, 140121588350976, 140121596739583, -SNULL, 140121571565568, 140121579954175, -STORE, 140121579954176, 140121588346879, -STORE, 140121571565568, 140121579954175, -SNULL, 140121579958271, 140121588346879, -STORE, 140121579954176, 140121579958271, -STORE, 140121579958272, 140121588346879, -STORE, 140121286373376, 140121294766079, -STORE, 140121277980672, 140121294766079, -SNULL, 140121277980672, 140121286373375, -STORE, 140121286373376, 140121294766079, -STORE, 140121277980672, 140121286373375, -SNULL, 140121286377471, 140121294766079, -STORE, 140121286373376, 140121286377471, -STORE, 140121286377472, 140121294766079, -STORE, 140121269587968, 140121286373375, -STORE, 140121261195264, 140121286373375, -SNULL, 140121261195264, 140121269587967, -STORE, 140121269587968, 140121286373375, -STORE, 140121261195264, 140121269587967, -SNULL, 140121269592063, 140121286373375, -STORE, 140121269587968, 140121269592063, -STORE, 140121269592064, 140121286373375, -STORE, 140121252802560, 140121269587967, -SNULL, 140121252806655, 140121269587967, -STORE, 140121252802560, 140121252806655, -STORE, 140121252806656, 140121269587967, -STORE, 140121244409856, 140121252802559, -STORE, 140121236017152, 140121252802559, -SNULL, 140121236017152, 140121244409855, -STORE, 140121244409856, 140121252802559, -STORE, 140121236017152, 140121244409855, -SNULL, 140121244413951, 140121252802559, -STORE, 140121244409856, 140121244413951, -STORE, 140121244413952, 140121252802559, -SNULL, 140121370238975, 140121378627583, -STORE, 140121370234880, 140121370238975, -STORE, 140121370238976, 140121378627583, -STORE, 140121152155648, 140121160548351, -STORE, 140121143762944, 140121160548351, -STORE, 140121135370240, 140121160548351, -SNULL, 140121135374335, 140121160548351, -STORE, 140121135370240, 140121135374335, -STORE, 140121135374336, 140121160548351, -STORE, 140121126977536, 140121135370239, -STORE, 140121118584832, 140121135370239, -STORE, 140121110192128, 140121135370239, -SNULL, 140121110192128, 140121118584831, -STORE, 140121118584832, 140121135370239, -STORE, 140121110192128, 140121118584831, -SNULL, 140121118588927, 140121135370239, -STORE, 140121118584832, 140121118588927, -STORE, 140121118588928, 140121135370239, -STORE, 140121101799424, 140121118584831, -STORE, 140121017937920, 140121026330623, -STORE, 140121009545216, 140121026330623, -SNULL, 140121009545216, 140121017937919, -STORE, 140121017937920, 140121026330623, -STORE, 140121009545216, 140121017937919, -SNULL, 140121017942015, 140121026330623, -STORE, 140121017937920, 140121017942015, -STORE, 140121017942016, 140121026330623, -SNULL, 140121269592064, 140121277980671, -STORE, 140121277980672, 140121286373375, -STORE, 140121269592064, 140121277980671, -SNULL, 140121277984767, 140121286373375, -STORE, 140121277980672, 140121277984767, -STORE, 140121277984768, 140121286373375, -STORE, 140121001152512, 140121017937919, -SNULL, 140121252806656, 140121261195263, -STORE, 140121261195264, 140121269587967, -STORE, 140121252806656, 140121261195263, -SNULL, 140121261199359, 140121269587967, -STORE, 140121261195264, 140121261199359, -STORE, 140121261199360, 140121269587967, -SNULL, 140121135374336, 140121152155647, -STORE, 140121152155648, 140121160548351, -STORE, 140121135374336, 140121152155647, -SNULL, 140121152159743, 140121160548351, -STORE, 140121152155648, 140121152159743, -STORE, 140121152159744, 140121160548351, -STORE, 140120992759808, 140121017937919, -STORE, 140120984367104, 140121017937919, -STORE, 140120975974400, 140121017937919, -SNULL, 140121101799424, 140121110192127, -STORE, 140121110192128, 140121118584831, -STORE, 140121101799424, 140121110192127, -SNULL, 140121110196223, 140121118584831, -STORE, 140121110192128, 140121110196223, -STORE, 140121110196224, 140121118584831, -SNULL, 140121118588928, 140121126977535, -STORE, 140121126977536, 140121135370239, -STORE, 140121118588928, 140121126977535, -SNULL, 140121126981631, 140121135370239, -STORE, 140121126977536, 140121126981631, -STORE, 140121126981632, 140121135370239, -STORE, 140120967581696, 140121017937919, -STORE, 140120883720192, 140120892112895, -SNULL, 140120883724287, 140120892112895, -STORE, 140120883720192, 140120883724287, -STORE, 140120883724288, 140120892112895, -STORE, 140120875327488, 140120883720191, -SNULL, 140121101803519, 140121110192127, -STORE, 140121101799424, 140121101803519, -STORE, 140121101803520, 140121110192127, -SNULL, 140121135374336, 140121143762943, -STORE, 140121143762944, 140121152155647, -STORE, 140121135374336, 140121143762943, -SNULL, 140121143767039, 140121152155647, -STORE, 140121143762944, 140121143767039, -STORE, 140121143767040, 140121152155647, -STORE, 140120866934784, 140120883720191, -SNULL, 140120967581696, 140120984367103, -STORE, 140120984367104, 140121017937919, -STORE, 140120967581696, 140120984367103, -SNULL, 140120984371199, 140121017937919, -STORE, 140120984367104, 140120984371199, -STORE, 140120984371200, 140121017937919, -STORE, 140120858542080, 140120883720191, -SNULL, 140121236021247, 140121244409855, -STORE, 140121236017152, 140121236021247, -STORE, 140121236021248, 140121244409855, -SNULL, 140120984371200, 140121009545215, -STORE, 140121009545216, 140121017937919, -STORE, 140120984371200, 140121009545215, -SNULL, 140121009549311, 140121017937919, -STORE, 140121009545216, 140121009549311, -STORE, 140121009549312, 140121017937919, -SNULL, 140120984371200, 140120992759807, -STORE, 140120992759808, 140121009545215, -STORE, 140120984371200, 140120992759807, -SNULL, 140120992763903, 140121009545215, -STORE, 140120992759808, 140120992763903, -STORE, 140120992763904, 140121009545215, -SNULL, 140120992763904, 140121001152511, -STORE, 140121001152512, 140121009545215, -STORE, 140120992763904, 140121001152511, -SNULL, 140121001156607, 140121009545215, -STORE, 140121001152512, 140121001156607, -STORE, 140121001156608, 140121009545215, -STORE, 140120850149376, 140120883720191, -SNULL, 140120850153471, 140120883720191, -STORE, 140120850149376, 140120850153471, -STORE, 140120850153472, 140120883720191, -SNULL, 140120967585791, 140120984367103, -STORE, 140120967581696, 140120967585791, -STORE, 140120967585792, 140120984367103, -SNULL, 140120850153472, 140120866934783, -STORE, 140120866934784, 140120883720191, -STORE, 140120850153472, 140120866934783, -SNULL, 140120866938879, 140120883720191, -STORE, 140120866934784, 140120866938879, -STORE, 140120866938880, 140120883720191, -STORE, 140120841756672, 140120850149375, -SNULL, 140120967585792, 140120975974399, -STORE, 140120975974400, 140120984367103, -STORE, 140120967585792, 140120975974399, -SNULL, 140120975978495, 140120984367103, -STORE, 140120975974400, 140120975978495, -STORE, 140120975978496, 140120984367103, -SNULL, 140120866938880, 140120875327487, -STORE, 140120875327488, 140120883720191, -STORE, 140120866938880, 140120875327487, -SNULL, 140120875331583, 140120883720191, -STORE, 140120875327488, 140120875331583, -STORE, 140120875331584, 140120883720191, -STORE, 140120833363968, 140120850149375, -STORE, 140120749502464, 140120757895167, -STORE, 140120741109760, 140120757895167, -STORE, 140120732717056, 140120757895167, -STORE, 140120724324352, 140120757895167, -SNULL, 140120724324352, 140120732717055, -STORE, 140120732717056, 140120757895167, -STORE, 140120724324352, 140120732717055, -SNULL, 140120732721151, 140120757895167, -STORE, 140120732717056, 140120732721151, -STORE, 140120732721152, 140120757895167, -STORE, 140120715931648, 140120732717055, -SNULL, 140120715935743, 140120732717055, -STORE, 140120715931648, 140120715935743, -STORE, 140120715935744, 140120732717055, -SNULL, 140120850153472, 140120858542079, -STORE, 140120858542080, 140120866934783, -STORE, 140120850153472, 140120858542079, -SNULL, 140120858546175, 140120866934783, -STORE, 140120858542080, 140120858546175, -STORE, 140120858546176, 140120866934783, -STORE, 140120707538944, 140120715931647, -SNULL, 140120707543039, 140120715931647, -STORE, 140120707538944, 140120707543039, -STORE, 140120707543040, 140120715931647, -SNULL, 140120833368063, 140120850149375, -STORE, 140120833363968, 140120833368063, -STORE, 140120833368064, 140120850149375, -SNULL, 140120833368064, 140120841756671, -STORE, 140120841756672, 140120850149375, -STORE, 140120833368064, 140120841756671, -SNULL, 140120841760767, 140120850149375, -STORE, 140120841756672, 140120841760767, -STORE, 140120841760768, 140120850149375, -STORE, 140120699146240, 140120707538943, -SNULL, 140120715935744, 140120724324351, -STORE, 140120724324352, 140120732717055, -STORE, 140120715935744, 140120724324351, -SNULL, 140120724328447, 140120732717055, -STORE, 140120724324352, 140120724328447, -STORE, 140120724328448, 140120732717055, -SNULL, 140120732721152, 140120741109759, -STORE, 140120741109760, 140120757895167, -STORE, 140120732721152, 140120741109759, -SNULL, 140120741113855, 140120757895167, -STORE, 140120741109760, 140120741113855, -STORE, 140120741113856, 140120757895167, -SNULL, 140120741113856, 140120749502463, -STORE, 140120749502464, 140120757895167, -STORE, 140120741113856, 140120749502463, -SNULL, 140120749506559, 140120757895167, -STORE, 140120749502464, 140120749506559, -STORE, 140120749506560, 140120757895167, -SNULL, 140120699150335, 140120707538943, -STORE, 140120699146240, 140120699150335, -STORE, 140120699150336, 140120707538943, -STORE, 140122446557184, 140122446585855, -STORE, 140122368999424, 140122371190783, -SNULL, 140122368999424, 140122369089535, -STORE, 140122369089536, 140122371190783, -STORE, 140122368999424, 140122369089535, -SNULL, 140122371182591, 140122371190783, -STORE, 140122369089536, 140122371182591, -STORE, 140122371182592, 140122371190783, -ERASE, 140122371182592, 140122371190783, -STORE, 140122371182592, 140122371190783, -SNULL, 140122371186687, 140122371190783, -STORE, 140122371182592, 140122371186687, -STORE, 140122371186688, 140122371190783, -ERASE, 140122446557184, 140122446585855, -ERASE, 140121445736448, 140121445740543, -ERASE, 140121445740544, 140121454129151, -ERASE, 140121621917696, 140121621921791, -ERASE, 140121621921792, 140121630310399, -ERASE, 140121579954176, 140121579958271, -ERASE, 140121579958272, 140121588346879, -ERASE, 140121261195264, 140121261199359, -ERASE, 140121261199360, 140121269587967, -ERASE, 140121454129152, 140121454133247, -ERASE, 140121454133248, 140121462521855, -ERASE, 140121588346880, 140121588350975, -ERASE, 140121588350976, 140121596739583, -ERASE, 140121135370240, 140121135374335, -ERASE, 140121135374336, 140121143762943, -ERASE, 140121881960448, 140121881964543, -ERASE, 140121881964544, 140121890353151, -ERASE, 140121428951040, 140121428955135, -ERASE, 140121428955136, 140121437343743, -ERASE, 140121387020288, 140121387024383, -ERASE, 140121387024384, 140121395412991, -ERASE, 140121487699968, 140121487704063, -ERASE, 140121487704064, 140121496092671, -ERASE, 140121437343744, 140121437347839, -ERASE, 140121437347840, 140121445736447, -ERASE, 140121613524992, 140121613529087, -ERASE, 140121613529088, 140121621917695, -ERASE, 140121856782336, 140121856786431, -ERASE, 140121856786432, 140121865175039, -ERASE, 140121252802560, 140121252806655, -ERASE, 140121252806656, 140121261195263, -ERASE, 140121839996928, 140121840001023, -ERASE, 140121840001024, 140121848389631, -ERASE, 140121596739584, 140121596743679, -ERASE, 140121596743680, 140121605132287, -ERASE, 140121009545216, 140121009549311, -ERASE, 140121009549312, 140121017937919, -ERASE, 140120724324352, 140120724328447, -ERASE, 140120724328448, 140120732717055, -ERASE, 140120883720192, 140120883724287, -ERASE, 140120883724288, 140120892112895, -ERASE, 140121982607360, 140121982611455, -ERASE, 140121982611456, 140121991000063, -ERASE, 140121571561472, 140121571565567, -ERASE, 140121571565568, 140121579954175, -ERASE, 140121286373376, 140121286377471, -ERASE, 140121286377472, 140121294766079, -ERASE, 140120875327488, 140120875331583, -ERASE, 140120875331584, 140120883720191, -ERASE, 140121848389632, 140121848393727, -ERASE, 140121848393728, 140121856782335, -ERASE, 140121370234880, 140121370238975, -ERASE, 140121370238976, 140121378627583, -ERASE, 140121143762944, 140121143767039, -ERASE, 140121143767040, 140121152155647, -ERASE, 140121118584832, 140121118588927, -ERASE, 140121118588928, 140121126977535, -ERASE, 140120866934784, 140120866938879, -ERASE, 140120866938880, 140120875327487, -ERASE, 140120741109760, 140120741113855, -ERASE, 140120741113856, 140120749502463, -ERASE, 140121865175040, 140121865179135, -ERASE, 140121865179136, 140121873567743, -ERASE, 140121403805696, 140121403809791, -ERASE, 140121403809792, 140121412198399, -ERASE, 140121236017152, 140121236021247, -ERASE, 140121236021248, 140121244409855, -ERASE, 140120732717056, 140120732721151, -ERASE, 140120732721152, 140120741109759, -ERASE, 140121017937920, 140121017942015, -ERASE, 140121017942016, 140121026330623, -ERASE, 140121873567744, 140121873571839, -ERASE, 140121873571840, 140121881960447, -ERASE, 140121470914560, 140121470918655, -ERASE, 140121470918656, 140121479307263, -ERASE, 140121126977536, 140121126981631, -ERASE, 140121126981632, 140121135370239, -ERASE, 140120850149376, 140120850153471, -ERASE, 140120850153472, 140120858542079, -ERASE, 140120707538944, 140120707543039, -ERASE, 140120707543040, 140120715931647, -ERASE, 140121479307264, 140121479311359, -ERASE, 140121479311360, 140121487699967, -ERASE, 140120967581696, 140120967585791, -ERASE, 140120967585792, 140120975974399, -ERASE, 140120841756672, 140120841760767, -ERASE, 140120841760768, 140120850149375, -ERASE, 140121412198400, 140121412202495, -ERASE, 140121412202496, 140121420591103, -ERASE, 140122158788608, 140122158792703, -ERASE, 140122158792704, 140122167181311, -ERASE, 140122142003200, 140122142007295, -ERASE, 140122142007296, 140122150395903, -ERASE, 140121101799424, 140121101803519, -ERASE, 140121101803520, 140121110192127, -ERASE, 140120858542080, 140120858546175, -ERASE, 140120858546176, 140120866934783, -ERASE, 140120833363968, 140120833368063, -ERASE, 140120833368064, 140120841756671, -ERASE, 140121277980672, 140121277984767, -ERASE, 140121277984768, 140121286373375, -ERASE, 140121001152512, 140121001156607, -ERASE, 140121001156608, 140121009545215, -ERASE, 140120749502464, 140120749506559, -ERASE, 140120749506560, 140120757895167, -ERASE, 140121605132288, 140121605136383, -ERASE, 140121605136384, 140121613524991, -ERASE, 140121378627584, 140121378631679, -ERASE, 140121378631680, 140121387020287, -ERASE, 140121110192128, 140121110196223, -ERASE, 140121110196224, 140121118584831, -ERASE, 140121462521856, 140121462525951, -ERASE, 140121462525952, 140121470914559, -ERASE, 140121395412992, 140121395417087, -ERASE, 140121395417088, 140121403805695, -ERASE, 140121152155648, 140121152159743, -ERASE, 140121152159744, 140121160548351, -ERASE, 140120992759808, 140120992763903, -ERASE, 140120992763904, 140121001152511, -ERASE, 140122387976192, 140122387980287, -ERASE, 140122387980288, 140122396368895, -ERASE, 140121890353152, 140121890357247, -ERASE, 140121890357248, 140121898745855, -ERASE, 140121269587968, 140121269592063, -ERASE, 140121269592064, 140121277980671, - }; - unsigned long set37[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140722404016128, 140737488351231, -SNULL, 140722404020223, 140737488351231, -STORE, 140722404016128, 140722404020223, -STORE, 140722403885056, 140722404020223, -STORE, 94637010001920, 94637012254719, -SNULL, 94637010132991, 94637012254719, -STORE, 94637010001920, 94637010132991, -STORE, 94637010132992, 94637012254719, -ERASE, 94637010132992, 94637012254719, -STORE, 94637012226048, 94637012234239, -STORE, 94637012234240, 94637012254719, -STORE, 139760240594944, 139760242847743, -SNULL, 139760240738303, 139760242847743, -STORE, 139760240594944, 139760240738303, -STORE, 139760240738304, 139760242847743, -ERASE, 139760240738304, 139760242847743, -STORE, 139760242835456, 139760242843647, -STORE, 139760242843648, 139760242847743, -STORE, 140722405232640, 140722405236735, -STORE, 140722405220352, 140722405232639, -STORE, 139760242806784, 139760242835455, -STORE, 139760242798592, 139760242806783, -STORE, 139760238379008, 139760240594943, -SNULL, 139760238379008, 139760238477311, -STORE, 139760238477312, 139760240594943, -STORE, 139760238379008, 139760238477311, -SNULL, 139760240570367, 139760240594943, -STORE, 139760238477312, 139760240570367, -STORE, 139760240570368, 139760240594943, -SNULL, 139760240570368, 139760240578559, -STORE, 139760240578560, 139760240594943, -STORE, 139760240570368, 139760240578559, -ERASE, 139760240570368, 139760240578559, -STORE, 139760240570368, 139760240578559, -ERASE, 139760240578560, 139760240594943, -STORE, 139760240578560, 139760240594943, -STORE, 139760234582016, 139760238379007, -SNULL, 139760234582016, 139760236240895, -STORE, 139760236240896, 139760238379007, -STORE, 139760234582016, 139760236240895, -SNULL, 139760238338047, 139760238379007, -STORE, 139760236240896, 139760238338047, -STORE, 139760238338048, 139760238379007, -SNULL, 139760238338048, 139760238362623, -STORE, 139760238362624, 139760238379007, -STORE, 139760238338048, 139760238362623, -ERASE, 139760238338048, 139760238362623, -STORE, 139760238338048, 139760238362623, -ERASE, 139760238362624, 139760238379007, -STORE, 139760238362624, 139760238379007, -STORE, 139760242790400, 139760242806783, -SNULL, 139760238354431, 139760238362623, -STORE, 139760238338048, 139760238354431, -STORE, 139760238354432, 139760238362623, -SNULL, 139760240574463, 139760240578559, -STORE, 139760240570368, 139760240574463, -STORE, 139760240574464, 139760240578559, -SNULL, 94637012230143, 94637012234239, -STORE, 94637012226048, 94637012230143, -STORE, 94637012230144, 94637012234239, -SNULL, 139760242839551, 139760242843647, -STORE, 139760242835456, 139760242839551, -STORE, 139760242839552, 139760242843647, -ERASE, 139760242806784, 139760242835455, -STORE, 94637033324544, 94637033459711, -STORE, 139760226189312, 139760234582015, -SNULL, 139760226193407, 139760234582015, -STORE, 139760226189312, 139760226193407, -STORE, 139760226193408, 139760234582015, -STORE, 139760217796608, 139760226189311, -STORE, 139760083578880, 139760217796607, -SNULL, 139760083578880, 139760114860031, -STORE, 139760114860032, 139760217796607, -STORE, 139760083578880, 139760114860031, -ERASE, 139760083578880, 139760114860031, -SNULL, 139760181968895, 139760217796607, -STORE, 139760114860032, 139760181968895, -STORE, 139760181968896, 139760217796607, -ERASE, 139760181968896, 139760217796607, -SNULL, 139760114995199, 139760181968895, -STORE, 139760114860032, 139760114995199, -STORE, 139760114995200, 139760181968895, -SNULL, 139760217800703, 139760226189311, -STORE, 139760217796608, 139760217800703, -STORE, 139760217800704, 139760226189311, -STORE, 139760209403904, 139760217796607, -SNULL, 139760209407999, 139760217796607, -STORE, 139760209403904, 139760209407999, -STORE, 139760209408000, 139760217796607, -STORE, 139760201011200, 139760209403903, -SNULL, 139760201015295, 139760209403903, -STORE, 139760201011200, 139760201015295, -STORE, 139760201015296, 139760209403903, -STORE, 139760192618496, 139760201011199, -SNULL, 139760192622591, 139760201011199, -STORE, 139760192618496, 139760192622591, -STORE, 139760192622592, 139760201011199, -STORE, 139760184225792, 139760192618495, -STORE, 139759980642304, 139760114860031, -STORE, 139759972249600, 139759980642303, -STORE, 139759963856896, 139759980642303, -STORE, 139759955464192, 139759980642303, -STORE, 139759888355328, 139759955464191, -SNULL, 139760047751167, 139760114860031, -STORE, 139759980642304, 139760047751167, -STORE, 139760047751168, 139760114860031, -ERASE, 139760047751168, 139760114860031, -SNULL, 139759980777471, 139760047751167, -STORE, 139759980642304, 139759980777471, -STORE, 139759980777472, 139760047751167, -STORE, 139759980777472, 139760114860031, -SNULL, 139759980777472, 139760047751167, -STORE, 139760047751168, 139760114860031, -STORE, 139759980777472, 139760047751167, -SNULL, 139760047886335, 139760114860031, -STORE, 139760047751168, 139760047886335, -STORE, 139760047886336, 139760114860031, -STORE, 139759821246464, 139759955464191, -SNULL, 139759821246464, 139759888355327, -STORE, 139759888355328, 139759955464191, -STORE, 139759821246464, 139759888355327, -ERASE, 139759821246464, 139759888355327, -ERASE, 139759888355328, 139759955464191, - }; - unsigned long set38[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140730666221568, 140737488351231, -SNULL, 140730666225663, 140737488351231, -STORE, 140730666221568, 140730666225663, -STORE, 140730666090496, 140730666225663, -STORE, 94177584803840, 94177587056639, -SNULL, 94177584934911, 94177587056639, -STORE, 94177584803840, 94177584934911, -STORE, 94177584934912, 94177587056639, -ERASE, 94177584934912, 94177587056639, -STORE, 94177587027968, 94177587036159, -STORE, 94177587036160, 94177587056639, -STORE, 140614382714880, 140614384967679, -SNULL, 140614382858239, 140614384967679, -STORE, 140614382714880, 140614382858239, -STORE, 140614382858240, 140614384967679, -ERASE, 140614382858240, 140614384967679, -STORE, 140614384955392, 140614384963583, -STORE, 140614384963584, 140614384967679, -STORE, 140730666315776, 140730666319871, -STORE, 140730666303488, 140730666315775, -STORE, 140614384926720, 140614384955391, -STORE, 140614384918528, 140614384926719, -STORE, 140614380498944, 140614382714879, -SNULL, 140614380498944, 140614380597247, -STORE, 140614380597248, 140614382714879, -STORE, 140614380498944, 140614380597247, -SNULL, 140614382690303, 140614382714879, -STORE, 140614380597248, 140614382690303, -STORE, 140614382690304, 140614382714879, -SNULL, 140614382690304, 140614382698495, -STORE, 140614382698496, 140614382714879, -STORE, 140614382690304, 140614382698495, -ERASE, 140614382690304, 140614382698495, -STORE, 140614382690304, 140614382698495, -ERASE, 140614382698496, 140614382714879, -STORE, 140614382698496, 140614382714879, -STORE, 140614376701952, 140614380498943, -SNULL, 140614376701952, 140614378360831, -STORE, 140614378360832, 140614380498943, -STORE, 140614376701952, 140614378360831, -SNULL, 140614380457983, 140614380498943, -STORE, 140614378360832, 140614380457983, -STORE, 140614380457984, 140614380498943, -SNULL, 140614380457984, 140614380482559, -STORE, 140614380482560, 140614380498943, -STORE, 140614380457984, 140614380482559, -ERASE, 140614380457984, 140614380482559, -STORE, 140614380457984, 140614380482559, -ERASE, 140614380482560, 140614380498943, -STORE, 140614380482560, 140614380498943, -STORE, 140614384910336, 140614384926719, -SNULL, 140614380474367, 140614380482559, -STORE, 140614380457984, 140614380474367, -STORE, 140614380474368, 140614380482559, -SNULL, 140614382694399, 140614382698495, -STORE, 140614382690304, 140614382694399, -STORE, 140614382694400, 140614382698495, -SNULL, 94177587032063, 94177587036159, -STORE, 94177587027968, 94177587032063, -STORE, 94177587032064, 94177587036159, -SNULL, 140614384959487, 140614384963583, -STORE, 140614384955392, 140614384959487, -STORE, 140614384959488, 140614384963583, -ERASE, 140614384926720, 140614384955391, -STORE, 94177619791872, 94177619927039, -STORE, 140614368309248, 140614376701951, -SNULL, 140614368313343, 140614376701951, -STORE, 140614368309248, 140614368313343, -STORE, 140614368313344, 140614376701951, -STORE, 140614359916544, 140614368309247, -STORE, 140614225698816, 140614359916543, -SNULL, 140614225698816, 140614276481023, -STORE, 140614276481024, 140614359916543, -STORE, 140614225698816, 140614276481023, -ERASE, 140614225698816, 140614276481023, -SNULL, 140614343589887, 140614359916543, -STORE, 140614276481024, 140614343589887, -STORE, 140614343589888, 140614359916543, -ERASE, 140614343589888, 140614359916543, -SNULL, 140614276616191, 140614343589887, -STORE, 140614276481024, 140614276616191, -STORE, 140614276616192, 140614343589887, -SNULL, 140614359920639, 140614368309247, -STORE, 140614359916544, 140614359920639, -STORE, 140614359920640, 140614368309247, -STORE, 140614351523840, 140614359916543, -SNULL, 140614351527935, 140614359916543, -STORE, 140614351523840, 140614351527935, -STORE, 140614351527936, 140614359916543, -STORE, 140614268088320, 140614276481023, -SNULL, 140614268092415, 140614276481023, -STORE, 140614268088320, 140614268092415, -STORE, 140614268092416, 140614276481023, -STORE, 140614259695616, 140614268088319, -SNULL, 140614259699711, 140614268088319, -STORE, 140614259695616, 140614259699711, -STORE, 140614259699712, 140614268088319, -STORE, 140614251302912, 140614259695615, -STORE, 140614242910208, 140614259695615, -STORE, 140614108692480, 140614242910207, -SNULL, 140614108692480, 140614142263295, -STORE, 140614142263296, 140614242910207, -STORE, 140614108692480, 140614142263295, -ERASE, 140614108692480, 140614142263295, -STORE, 140614133870592, 140614142263295, -STORE, 140613999652864, 140614133870591, -SNULL, 140613999652864, 140614008045567, -STORE, 140614008045568, 140614133870591, -STORE, 140613999652864, 140614008045567, -ERASE, 140613999652864, 140614008045567, -STORE, 140613999652864, 140614008045567, -STORE, 140613865435136, 140613999652863, -SNULL, 140613865435136, 140613873827839, -STORE, 140613873827840, 140613999652863, -STORE, 140613865435136, 140613873827839, -ERASE, 140613865435136, 140613873827839, -SNULL, 140614209372159, 140614242910207, -STORE, 140614142263296, 140614209372159, -STORE, 140614209372160, 140614242910207, -ERASE, 140614209372160, 140614242910207, -SNULL, 140614142398463, 140614209372159, -STORE, 140614142263296, 140614142398463, -STORE, 140614142398464, 140614209372159, -SNULL, 140614075154431, 140614133870591, -STORE, 140614008045568, 140614075154431, -STORE, 140614075154432, 140614133870591, -ERASE, 140614075154432, 140614133870591, -SNULL, 140614008180735, 140614075154431, -STORE, 140614008045568, 140614008180735, -STORE, 140614008180736, 140614075154431, -SNULL, 140613940936703, 140613999652863, -STORE, 140613873827840, 140613940936703, -STORE, 140613940936704, 140613999652863, -ERASE, 140613940936704, 140613999652863, -SNULL, 140614242914303, 140614259695615, -STORE, 140614242910208, 140614242914303, -STORE, 140614242914304, 140614259695615, -STORE, 140613739610112, 140613940936703, -STORE, 140614234517504, 140614242910207, -SNULL, 140614242914304, 140614251302911, -STORE, 140614251302912, 140614259695615, -STORE, 140614242914304, 140614251302911, -SNULL, 140614251307007, 140614259695615, -STORE, 140614251302912, 140614251307007, -STORE, 140614251307008, 140614259695615, -SNULL, 140613739610112, 140613873827839, -STORE, 140613873827840, 140613940936703, -STORE, 140613739610112, 140613873827839, -SNULL, 140613873963007, 140613940936703, -STORE, 140613873827840, 140613873963007, -STORE, 140613873963008, 140613940936703, -SNULL, 140614133874687, 140614142263295, -STORE, 140614133870592, 140614133874687, -STORE, 140614133874688, 140614142263295, -SNULL, 140613806718975, 140613873827839, -STORE, 140613739610112, 140613806718975, -STORE, 140613806718976, 140613873827839, -ERASE, 140613806718976, 140613873827839, -STORE, 140614226124800, 140614242910207, -SNULL, 140613739745279, 140613806718975, -STORE, 140613739610112, 140613739745279, -STORE, 140613739745280, 140613806718975, -SNULL, 140613999656959, 140614008045567, -STORE, 140613999652864, 140613999656959, -STORE, 140613999656960, 140614008045567, -SNULL, 140614226124800, 140614234517503, -STORE, 140614234517504, 140614242910207, -STORE, 140614226124800, 140614234517503, -SNULL, 140614234521599, 140614242910207, -STORE, 140614234517504, 140614234521599, -STORE, 140614234521600, 140614242910207, -STORE, 140614217732096, 140614234517503, -STORE, 140614125477888, 140614133870591, -SNULL, 140614125481983, 140614133870591, -STORE, 140614125477888, 140614125481983, -STORE, 140614125481984, 140614133870591, -STORE, 140614117085184, 140614125477887, -SNULL, 140614217736191, 140614234517503, -STORE, 140614217732096, 140614217736191, -STORE, 140614217736192, 140614234517503, -SNULL, 140614117089279, 140614125477887, -STORE, 140614117085184, 140614117089279, -STORE, 140614117089280, 140614125477887, -SNULL, 140614217736192, 140614226124799, -STORE, 140614226124800, 140614234517503, -STORE, 140614217736192, 140614226124799, -SNULL, 140614226128895, 140614234517503, -STORE, 140614226124800, 140614226128895, -STORE, 140614226128896, 140614234517503, -STORE, 140614108692480, 140614117085183, -STORE, 140614100299776, 140614117085183, -STORE, 140614091907072, 140614117085183, -SNULL, 140614091907072, 140614108692479, -STORE, 140614108692480, 140614117085183, -STORE, 140614091907072, 140614108692479, -SNULL, 140614108696575, 140614117085183, -STORE, 140614108692480, 140614108696575, -STORE, 140614108696576, 140614117085183, -SNULL, 140614091907072, 140614100299775, -STORE, 140614100299776, 140614108692479, -STORE, 140614091907072, 140614100299775, -SNULL, 140614100303871, 140614108692479, -STORE, 140614100299776, 140614100303871, -STORE, 140614100303872, 140614108692479, -STORE, 140614083514368, 140614100299775, -SNULL, 140614083518463, 140614100299775, -STORE, 140614083514368, 140614083518463, -STORE, 140614083518464, 140614100299775, -STORE, 140613991260160, 140613999652863, -SNULL, 140614083518464, 140614091907071, -STORE, 140614091907072, 140614100299775, -STORE, 140614083518464, 140614091907071, -SNULL, 140614091911167, 140614100299775, -STORE, 140614091907072, 140614091911167, -STORE, 140614091911168, 140614100299775, -SNULL, 140613991264255, 140613999652863, -STORE, 140613991260160, 140613991264255, -STORE, 140613991264256, 140613999652863, -STORE, 140613982867456, 140613991260159, -SNULL, 140613982871551, 140613991260159, -STORE, 140613982867456, 140613982871551, -STORE, 140613982871552, 140613991260159, -STORE, 140613974474752, 140613982867455, -SNULL, 140613974478847, 140613982867455, -STORE, 140613974474752, 140613974478847, -STORE, 140613974478848, 140613982867455, -STORE, 140613966082048, 140613974474751, -STORE, 140613739745280, 140613873827839, -SNULL, 140613739745280, 140613806718975, -STORE, 140613806718976, 140613873827839, -STORE, 140613739745280, 140613806718975, -SNULL, 140613806854143, 140613873827839, -STORE, 140613806718976, 140613806854143, -STORE, 140613806854144, 140613873827839, -SNULL, 140613966086143, 140613974474751, -STORE, 140613966082048, 140613966086143, -STORE, 140613966086144, 140613974474751, -STORE, 140613957689344, 140613966082047, -STORE, 140613605392384, 140613739610111, -STORE, 140613949296640, 140613966082047, -STORE, 140613596999680, 140613605392383, -STORE, 140613529890816, 140613596999679, -STORE, 140613521498112, 140613529890815, -STORE, 140613513105408, 140613529890815, -STORE, 140613378887680, 140613513105407, -SNULL, 140613378887680, 140613404065791, -STORE, 140613404065792, 140613513105407, -STORE, 140613378887680, 140613404065791, -ERASE, 140613378887680, 140613404065791, -STORE, 140613395673088, 140613404065791, -STORE, 140613261455360, 140613395673087, -SNULL, 140613261455360, 140613269848063, -STORE, 140613269848064, 140613395673087, -STORE, 140613261455360, 140613269848063, -ERASE, 140613261455360, 140613269848063, -STORE, 140613261455360, 140613269848063, -STORE, 140613253062656, 140613269848063, -STORE, 140613118844928, 140613253062655, -STORE, 140613110452224, 140613118844927, -SNULL, 140613118844928, 140613135630335, -STORE, 140613135630336, 140613253062655, -STORE, 140613118844928, 140613135630335, -ERASE, 140613118844928, 140613135630335, -STORE, 140613127237632, 140613135630335, -STORE, 140613110452224, 140613135630335, -STORE, 140612976234496, 140613110452223, -STORE, 140612967841792, 140612976234495, -STORE, 140612833624064, 140612967841791, -STORE, 140612825231360, 140612833624063, -STORE, 140612816838656, 140612833624063, -STORE, 140612682620928, 140612816838655, -STORE, 140612674228224, 140612682620927, -SNULL, 140612682620928, 140612732977151, -STORE, 140612732977152, 140612816838655, -STORE, 140612682620928, 140612732977151, -ERASE, 140612682620928, 140612732977151, -SNULL, 140613672501247, 140613739610111, -STORE, 140613605392384, 140613672501247, -STORE, 140613672501248, 140613739610111, -ERASE, 140613672501248, 140613739610111, -SNULL, 140613605527551, 140613672501247, -STORE, 140613605392384, 140613605527551, -STORE, 140613605527552, 140613672501247, -ERASE, 140613529890816, 140613596999679, -STORE, 140612540010496, 140612674228223, -SNULL, 140612540010496, 140612598759423, -STORE, 140612598759424, 140612674228223, -STORE, 140612540010496, 140612598759423, -ERASE, 140612540010496, 140612598759423, -SNULL, 140613471174655, 140613513105407, -STORE, 140613404065792, 140613471174655, -STORE, 140613471174656, 140613513105407, -ERASE, 140613471174656, 140613513105407, -SNULL, 140613404200959, 140613471174655, -STORE, 140613404065792, 140613404200959, -STORE, 140613404200960, 140613471174655, -SNULL, 140613336956927, 140613395673087, -STORE, 140613269848064, 140613336956927, -STORE, 140613336956928, 140613395673087, -ERASE, 140613336956928, 140613395673087, -SNULL, 140612833624064, 140612867194879, -STORE, 140612867194880, 140612967841791, -STORE, 140612833624064, 140612867194879, -ERASE, 140612833624064, 140612867194879, -SNULL, 140612976234496, 140613001412607, -STORE, 140613001412608, 140613110452223, -STORE, 140612976234496, 140613001412607, -ERASE, 140612976234496, 140613001412607, -SNULL, 140613202739199, 140613253062655, -STORE, 140613135630336, 140613202739199, -STORE, 140613202739200, 140613253062655, -ERASE, 140613202739200, 140613253062655, -SNULL, 140613135765503, 140613202739199, -STORE, 140613135630336, 140613135765503, -STORE, 140613135765504, 140613202739199, -SNULL, 140612816842751, 140612833624063, -STORE, 140612816838656, 140612816842751, -STORE, 140612816842752, 140612833624063, -SNULL, 140613110456319, 140613135630335, -STORE, 140613110452224, 140613110456319, -STORE, 140613110456320, 140613135630335, -SNULL, 140613949300735, 140613966082047, -STORE, 140613949296640, 140613949300735, -STORE, 140613949300736, 140613966082047, -SNULL, 140613110456320, 140613118844927, -STORE, 140613118844928, 140613135630335, -STORE, 140613110456320, 140613118844927, -SNULL, 140613118849023, 140613135630335, -STORE, 140613118844928, 140613118849023, -STORE, 140613118849024, 140613135630335, -SNULL, 140612800086015, 140612816838655, -STORE, 140612732977152, 140612800086015, -STORE, 140612800086016, 140612816838655, -ERASE, 140612800086016, 140612816838655, -SNULL, 140613253062656, 140613261455359, -STORE, 140613261455360, 140613269848063, -STORE, 140613253062656, 140613261455359, -SNULL, 140613261459455, 140613269848063, -STORE, 140613261455360, 140613261459455, -STORE, 140613261459456, 140613269848063, -SNULL, 140612674232319, 140612682620927, -STORE, 140612674228224, 140612674232319, -STORE, 140612674232320, 140612682620927, -STORE, 140613731217408, 140613739610111, -STORE, 140613722824704, 140613739610111, -SNULL, 140613949300736, 140613957689343, -STORE, 140613957689344, 140613966082047, -STORE, 140613949300736, 140613957689343, -SNULL, 140613957693439, 140613966082047, -STORE, 140613957689344, 140613957693439, -STORE, 140613957693440, 140613966082047, -STORE, 140612464541696, 140612674228223, -SNULL, 140612531650559, 140612674228223, -STORE, 140612464541696, 140612531650559, -STORE, 140612531650560, 140612674228223, -SNULL, 140612531650560, 140612598759423, -STORE, 140612598759424, 140612674228223, -STORE, 140612531650560, 140612598759423, -ERASE, 140612531650560, 140612598759423, -SNULL, 140612665868287, 140612674228223, -STORE, 140612598759424, 140612665868287, -STORE, 140612665868288, 140612674228223, -ERASE, 140612665868288, 140612674228223, -SNULL, 140613269983231, 140613336956927, -STORE, 140613269848064, 140613269983231, -STORE, 140613269983232, 140613336956927, -SNULL, 140612934303743, 140612967841791, -STORE, 140612867194880, 140612934303743, -STORE, 140612934303744, 140612967841791, -ERASE, 140612934303744, 140612967841791, -SNULL, 140613068521471, 140613110452223, -STORE, 140613001412608, 140613068521471, -STORE, 140613068521472, 140613110452223, -ERASE, 140613068521472, 140613110452223, -STORE, 140613714432000, 140613739610111, -SNULL, 140613001547775, 140613068521471, -STORE, 140613001412608, 140613001547775, -STORE, 140613001547776, 140613068521471, -SNULL, 140612733112319, 140612800086015, -STORE, 140612732977152, 140612733112319, -STORE, 140612733112320, 140612800086015, -SNULL, 140613513109503, 140613529890815, -STORE, 140613513105408, 140613513109503, -STORE, 140613513109504, 140613529890815, -STORE, 140613706039296, 140613739610111, -STORE, 140613697646592, 140613739610111, -STORE, 140613689253888, 140613739610111, -SNULL, 140613689257983, 140613739610111, -STORE, 140613689253888, 140613689257983, -STORE, 140613689257984, 140613739610111, -SNULL, 140613253066751, 140613261455359, -STORE, 140613253062656, 140613253066751, -STORE, 140613253066752, 140613261455359, -STORE, 140613680861184, 140613689253887, -STORE, 140613588606976, 140613605392383, -SNULL, 140613689257984, 140613731217407, -STORE, 140613731217408, 140613739610111, -STORE, 140613689257984, 140613731217407, -SNULL, 140613731221503, 140613739610111, -STORE, 140613731217408, 140613731221503, -STORE, 140613731221504, 140613739610111, -STORE, 140613580214272, 140613605392383, -SNULL, 140612464676863, 140612531650559, -STORE, 140612464541696, 140612464676863, -STORE, 140612464676864, 140612531650559, -SNULL, 140612598894591, 140612665868287, -STORE, 140612598759424, 140612598894591, -STORE, 140612598894592, 140612665868287, -SNULL, 140612867330047, 140612934303743, -STORE, 140612867194880, 140612867330047, -STORE, 140612867330048, 140612934303743, -STORE, 140613571821568, 140613605392383, -SNULL, 140613571825663, 140613605392383, -STORE, 140613571821568, 140613571825663, -STORE, 140613571825664, 140613605392383, -SNULL, 140613689257984, 140613722824703, -STORE, 140613722824704, 140613731217407, -STORE, 140613689257984, 140613722824703, -SNULL, 140613722828799, 140613731217407, -STORE, 140613722824704, 140613722828799, -STORE, 140613722828800, 140613731217407, -SNULL, 140613689257984, 140613714431999, -STORE, 140613714432000, 140613722824703, -STORE, 140613689257984, 140613714431999, -SNULL, 140613714436095, 140613722824703, -STORE, 140613714432000, 140613714436095, -STORE, 140613714436096, 140613722824703, -SNULL, 140612816842752, 140612825231359, -STORE, 140612825231360, 140612833624063, -STORE, 140612816842752, 140612825231359, -SNULL, 140612825235455, 140612833624063, -STORE, 140612825231360, 140612825235455, -STORE, 140612825235456, 140612833624063, -SNULL, 140613395677183, 140613404065791, -STORE, 140613395673088, 140613395677183, -STORE, 140613395677184, 140613404065791, -SNULL, 140613689257984, 140613706039295, -STORE, 140613706039296, 140613714431999, -STORE, 140613689257984, 140613706039295, -SNULL, 140613706043391, 140613714431999, -STORE, 140613706039296, 140613706043391, -STORE, 140613706043392, 140613714431999, -SNULL, 140613118849024, 140613127237631, -STORE, 140613127237632, 140613135630335, -STORE, 140613118849024, 140613127237631, -SNULL, 140613127241727, 140613135630335, -STORE, 140613127237632, 140613127241727, -STORE, 140613127241728, 140613135630335, -SNULL, 140613571825664, 140613580214271, -STORE, 140613580214272, 140613605392383, -STORE, 140613571825664, 140613580214271, -SNULL, 140613580218367, 140613605392383, -STORE, 140613580214272, 140613580218367, -STORE, 140613580218368, 140613605392383, -SNULL, 140613689257984, 140613697646591, -STORE, 140613697646592, 140613706039295, -STORE, 140613689257984, 140613697646591, -SNULL, 140613697650687, 140613706039295, -STORE, 140613697646592, 140613697650687, -STORE, 140613697650688, 140613706039295, -SNULL, 140613680865279, 140613689253887, -STORE, 140613680861184, 140613680865279, -STORE, 140613680865280, 140613689253887, -STORE, 140613563428864, 140613571821567, -SNULL, 140613563432959, 140613571821567, -STORE, 140613563428864, 140613563432959, -STORE, 140613563432960, 140613571821567, -SNULL, 140613580218368, 140613588606975, -STORE, 140613588606976, 140613605392383, -STORE, 140613580218368, 140613588606975, -SNULL, 140613588611071, 140613605392383, -STORE, 140613588606976, 140613588611071, -STORE, 140613588611072, 140613605392383, -SNULL, 140613513109504, 140613521498111, -STORE, 140613521498112, 140613529890815, -STORE, 140613513109504, 140613521498111, -SNULL, 140613521502207, 140613529890815, -STORE, 140613521498112, 140613521502207, -STORE, 140613521502208, 140613529890815, -SNULL, 140613588611072, 140613596999679, -STORE, 140613596999680, 140613605392383, -STORE, 140613588611072, 140613596999679, -SNULL, 140613597003775, 140613605392383, -STORE, 140613596999680, 140613597003775, -STORE, 140613597003776, 140613605392383, -STORE, 140613555036160, 140613563428863, -SNULL, 140613555040255, 140613563428863, -STORE, 140613555036160, 140613555040255, -STORE, 140613555040256, 140613563428863, -STORE, 140613546643456, 140613555036159, -STORE, 140613538250752, 140613555036159, -SNULL, 140613538250752, 140613546643455, -STORE, 140613546643456, 140613555036159, -STORE, 140613538250752, 140613546643455, -SNULL, 140613546647551, 140613555036159, -STORE, 140613546643456, 140613546647551, -STORE, 140613546647552, 140613555036159, -STORE, 140613504712704, 140613513105407, -STORE, 140613496320000, 140613513105407, -SNULL, 140613496324095, 140613513105407, -STORE, 140613496320000, 140613496324095, -STORE, 140613496324096, 140613513105407, -STORE, 140613487927296, 140613496319999, -SNULL, 140613487931391, 140613496319999, -STORE, 140613487927296, 140613487931391, -STORE, 140613487931392, 140613496319999, -STORE, 140613479534592, 140613487927295, -SNULL, 140612967845887, 140612976234495, -STORE, 140612967841792, 140612967845887, -STORE, 140612967845888, 140612976234495, -STORE, 140613387280384, 140613395673087, -STORE, 140613378887680, 140613395673087, -SNULL, 140613378887680, 140613387280383, -STORE, 140613387280384, 140613395673087, -STORE, 140613378887680, 140613387280383, -SNULL, 140613387284479, 140613395673087, -STORE, 140613387280384, 140613387284479, -STORE, 140613387284480, 140613395673087, -STORE, 140613370494976, 140613387280383, -STORE, 140613362102272, 140613387280383, -SNULL, 140613479538687, 140613487927295, -STORE, 140613479534592, 140613479538687, -STORE, 140613479538688, 140613487927295, -STORE, 140613353709568, 140613387280383, -STORE, 140613345316864, 140613387280383, -STORE, 140613244669952, 140613253062655, -SNULL, 140613345320959, 140613387280383, -STORE, 140613345316864, 140613345320959, -STORE, 140613345320960, 140613387280383, -SNULL, 140613538254847, 140613546643455, -STORE, 140613538250752, 140613538254847, -STORE, 140613538254848, 140613546643455, -STORE, 140613236277248, 140613253062655, -STORE, 140613227884544, 140613253062655, -STORE, 140613219491840, 140613253062655, -STORE, 140613211099136, 140613253062655, -SNULL, 140613211103231, 140613253062655, -STORE, 140613211099136, 140613211103231, -STORE, 140613211103232, 140613253062655, -STORE, 140613102059520, 140613110452223, -STORE, 140613093666816, 140613110452223, -SNULL, 140613093670911, 140613110452223, -STORE, 140613093666816, 140613093670911, -STORE, 140613093670912, 140613110452223, -STORE, 140613085274112, 140613093666815, -SNULL, 140613496324096, 140613504712703, -STORE, 140613504712704, 140613513105407, -STORE, 140613496324096, 140613504712703, -SNULL, 140613504716799, 140613513105407, -STORE, 140613504712704, 140613504716799, -STORE, 140613504716800, 140613513105407, -SNULL, 140613345320960, 140613378887679, -STORE, 140613378887680, 140613387280383, -STORE, 140613345320960, 140613378887679, -SNULL, 140613378891775, 140613387280383, -STORE, 140613378887680, 140613378891775, -STORE, 140613378891776, 140613387280383, -SNULL, 140613345320960, 140613362102271, -STORE, 140613362102272, 140613378887679, -STORE, 140613345320960, 140613362102271, -SNULL, 140613362106367, 140613378887679, -STORE, 140613362102272, 140613362106367, -STORE, 140613362106368, 140613378887679, -SNULL, 140613362106368, 140613370494975, -STORE, 140613370494976, 140613378887679, -STORE, 140613362106368, 140613370494975, -SNULL, 140613370499071, 140613378887679, -STORE, 140613370494976, 140613370499071, -STORE, 140613370499072, 140613378887679, -STORE, 140613076881408, 140613093666815, -STORE, 140612993019904, 140613001412607, -SNULL, 140613076885503, 140613093666815, -STORE, 140613076881408, 140613076885503, -STORE, 140613076885504, 140613093666815, -SNULL, 140613093670912, 140613102059519, -STORE, 140613102059520, 140613110452223, -STORE, 140613093670912, 140613102059519, -SNULL, 140613102063615, 140613110452223, -STORE, 140613102059520, 140613102063615, -STORE, 140613102063616, 140613110452223, -SNULL, 140613076885504, 140613085274111, -STORE, 140613085274112, 140613093666815, -STORE, 140613076885504, 140613085274111, -SNULL, 140613085278207, 140613093666815, -STORE, 140613085274112, 140613085278207, -STORE, 140613085278208, 140613093666815, -STORE, 140612984627200, 140613001412607, -STORE, 140612967845888, 140612984627199, -SNULL, 140613211103232, 140613219491839, -STORE, 140613219491840, 140613253062655, -STORE, 140613211103232, 140613219491839, -SNULL, 140613219495935, 140613253062655, -STORE, 140613219491840, 140613219495935, -STORE, 140613219495936, 140613253062655, -STORE, 140612959449088, 140612967841791, -STORE, 140612951056384, 140612967841791, -SNULL, 140612951060479, 140612967841791, -STORE, 140612951056384, 140612951060479, -STORE, 140612951060480, 140612967841791, -SNULL, 140613345320960, 140613353709567, -STORE, 140613353709568, 140613362102271, -STORE, 140613345320960, 140613353709567, -SNULL, 140613353713663, 140613362102271, -STORE, 140613353709568, 140613353713663, -STORE, 140613353713664, 140613362102271, -SNULL, 140613219495936, 140613244669951, -STORE, 140613244669952, 140613253062655, -STORE, 140613219495936, 140613244669951, -SNULL, 140613244674047, 140613253062655, -STORE, 140613244669952, 140613244674047, -STORE, 140613244674048, 140613253062655, -STORE, 140612942663680, 140612951056383, -SNULL, 140613219495936, 140613236277247, -STORE, 140613236277248, 140613244669951, -STORE, 140613219495936, 140613236277247, -SNULL, 140613236281343, 140613244669951, -STORE, 140613236277248, 140613236281343, -STORE, 140613236281344, 140613244669951, -SNULL, 140613219495936, 140613227884543, -STORE, 140613227884544, 140613236277247, -STORE, 140613219495936, 140613227884543, -SNULL, 140613227888639, 140613236277247, -STORE, 140613227884544, 140613227888639, -STORE, 140613227888640, 140613236277247, -SNULL, 140612984627200, 140612993019903, -STORE, 140612993019904, 140613001412607, -STORE, 140612984627200, 140612993019903, -SNULL, 140612993023999, 140613001412607, -STORE, 140612993019904, 140612993023999, -STORE, 140612993024000, 140613001412607, -STORE, 140612858802176, 140612867194879, -STORE, 140612850409472, 140612867194879, -SNULL, 140612951060480, 140612959449087, -STORE, 140612959449088, 140612967841791, -STORE, 140612951060480, 140612959449087, -SNULL, 140612959453183, 140612967841791, -STORE, 140612959449088, 140612959453183, -STORE, 140612959453184, 140612967841791, -SNULL, 140612967845888, 140612976234495, -STORE, 140612976234496, 140612984627199, -STORE, 140612967845888, 140612976234495, -SNULL, 140612976238591, 140612984627199, -STORE, 140612976234496, 140612976238591, -STORE, 140612976238592, 140612984627199, -STORE, 140612842016768, 140612867194879, -SNULL, 140612842020863, 140612867194879, -STORE, 140612842016768, 140612842020863, -STORE, 140612842020864, 140612867194879, -SNULL, 140612984631295, 140612993019903, -STORE, 140612984627200, 140612984631295, -STORE, 140612984631296, 140612993019903, -STORE, 140612825235456, 140612842016767, -STORE, 140612808445952, 140612816838655, -SNULL, 140612942667775, 140612951056383, -STORE, 140612942663680, 140612942667775, -STORE, 140612942667776, 140612951056383, -STORE, 140612724584448, 140612732977151, -SNULL, 140612724588543, 140612732977151, -STORE, 140612724584448, 140612724588543, -STORE, 140612724588544, 140612732977151, -STORE, 140612716191744, 140612724584447, -SNULL, 140612842020864, 140612850409471, -STORE, 140612850409472, 140612867194879, -STORE, 140612842020864, 140612850409471, -SNULL, 140612850413567, 140612867194879, -STORE, 140612850409472, 140612850413567, -STORE, 140612850413568, 140612867194879, -SNULL, 140612850413568, 140612858802175, -STORE, 140612858802176, 140612867194879, -STORE, 140612850413568, 140612858802175, -SNULL, 140612858806271, 140612867194879, -STORE, 140612858802176, 140612858806271, -STORE, 140612858806272, 140612867194879, -STORE, 140612707799040, 140612724584447, -SNULL, 140612707803135, 140612724584447, -STORE, 140612707799040, 140612707803135, -STORE, 140612707803136, 140612724584447, -SNULL, 140612707803136, 140612716191743, -STORE, 140612716191744, 140612724584447, -STORE, 140612707803136, 140612716191743, -SNULL, 140612716195839, 140612724584447, -STORE, 140612716191744, 140612716195839, -STORE, 140612716195840, 140612724584447, -SNULL, 140612808450047, 140612816838655, -STORE, 140612808445952, 140612808450047, -STORE, 140612808450048, 140612816838655, -SNULL, 140612825235456, 140612833624063, -STORE, 140612833624064, 140612842016767, -STORE, 140612825235456, 140612833624063, -SNULL, 140612833628159, 140612842016767, -STORE, 140612833624064, 140612833628159, -STORE, 140612833628160, 140612842016767, -STORE, 140612699406336, 140612707799039, -SNULL, 140612699410431, 140612707799039, -STORE, 140612699406336, 140612699410431, -STORE, 140612699410432, 140612707799039, -STORE, 140614384926720, 140614384955391, -STORE, 140614349332480, 140614351523839, -SNULL, 140614349332480, 140614349422591, -STORE, 140614349422592, 140614351523839, -STORE, 140614349332480, 140614349422591, -SNULL, 140614351515647, 140614351523839, -STORE, 140614349422592, 140614351515647, -STORE, 140614351515648, 140614351523839, -ERASE, 140614351515648, 140614351523839, -STORE, 140614351515648, 140614351523839, -SNULL, 140614351519743, 140614351523839, -STORE, 140614351515648, 140614351519743, -STORE, 140614351519744, 140614351523839, -ERASE, 140614384926720, 140614384955391, -ERASE, 140613949296640, 140613949300735, -ERASE, 140613949300736, 140613957689343, -ERASE, 140613689253888, 140613689257983, -ERASE, 140613689257984, 140613697646591, -ERASE, 140613563428864, 140613563432959, -ERASE, 140613563432960, 140613571821567, -ERASE, 140613211099136, 140613211103231, -ERASE, 140613211103232, 140613219491839, -ERASE, 140614133870592, 140614133874687, -ERASE, 140614133874688, 140614142263295, -ERASE, 140612967841792, 140612967845887, -ERASE, 140612967845888, 140612976234495, -ERASE, 140613076881408, 140613076885503, -ERASE, 140613076885504, 140613085274111, -ERASE, 140612850409472, 140612850413567, -ERASE, 140612850413568, 140612858802175, -ERASE, 140613110452224, 140613110456319, -ERASE, 140613110456320, 140613118844927, -ERASE, 140613706039296, 140613706043391, -ERASE, 140613706043392, 140613714431999, -ERASE, 140613521498112, 140613521502207, -ERASE, 140613521502208, 140613529890815, -ERASE, 140613362102272, 140613362106367, -ERASE, 140613362106368, 140613370494975, -ERASE, 140613253062656, 140613253066751, -ERASE, 140613253066752, 140613261455359, -ERASE, 140612816838656, 140612816842751, -ERASE, 140612816842752, 140612825231359, -ERASE, 140613261455360, 140613261459455, -ERASE, 140613261459456, 140613269848063, -ERASE, 140613118844928, 140613118849023, -ERASE, 140613118849024, 140613127237631, -ERASE, 140613714432000, 140613714436095, -ERASE, 140613714436096, 140613722824703, -ERASE, 140613496320000, 140613496324095, -ERASE, 140613496324096, 140613504712703, -ERASE, 140613513105408, 140613513109503, -ERASE, 140613513109504, 140613521498111, -ERASE, 140613697646592, 140613697650687, -ERASE, 140613697650688, 140613706039295, -ERASE, 140613093666816, 140613093670911, -ERASE, 140613093670912, 140613102059519, -ERASE, 140612993019904, 140612993023999, -ERASE, 140612993024000, 140613001412607, -ERASE, 140613127237632, 140613127241727, -ERASE, 140613127241728, 140613135630335, -ERASE, 140613957689344, 140613957693439, -ERASE, 140613957693440, 140613966082047, -ERASE, 140613571821568, 140613571825663, -ERASE, 140613571825664, 140613580214271, -ERASE, 140613479534592, 140613479538687, -ERASE, 140613479538688, 140613487927295, -ERASE, 140612984627200, 140612984631295, -ERASE, 140612984631296, 140612993019903, -ERASE, 140613588606976, 140613588611071, -ERASE, 140613588611072, 140613596999679, -ERASE, 140613680861184, 140613680865279, -ERASE, 140613680865280, 140613689253887, -ERASE, 140613345316864, 140613345320959, -ERASE, 140613345320960, 140613353709567, -ERASE, 140613596999680, 140613597003775, -ERASE, 140613597003776, 140613605392383, -ERASE, 140613966082048, 140613966086143, -ERASE, 140613966086144, 140613974474751, -ERASE, 140613731217408, 140613731221503, -ERASE, 140613731221504, 140613739610111, -ERASE, 140613395673088, 140613395677183, -ERASE, 140613395677184, 140613404065791, -ERASE, 140612825231360, 140612825235455, -ERASE, 140612825235456, 140612833624063, -ERASE, 140612674228224, 140612674232319, -ERASE, 140612674232320, 140612682620927, -ERASE, 140613722824704, 140613722828799, -ERASE, 140613722828800, 140613731217407, -ERASE, 140613487927296, 140613487931391, -ERASE, 140613487931392, 140613496319999, -ERASE, 140613102059520, 140613102063615, -ERASE, 140613102063616, 140613110452223, -ERASE, 140614242910208, 140614242914303, -ERASE, 140614242914304, 140614251302911, -ERASE, 140612808445952, 140612808450047, -ERASE, 140612808450048, 140612816838655, -ERASE, 140613236277248, 140613236281343, -ERASE, 140613236281344, 140613244669951, -ERASE, 140613580214272, 140613580218367, -ERASE, 140613580218368, 140613588606975, -ERASE, 140613370494976, 140613370499071, -ERASE, 140613370499072, 140613378887679, -ERASE, 140613244669952, 140613244674047, -ERASE, 140613244674048, 140613253062655, -ERASE, 140612724584448, 140612724588543, -ERASE, 140612724588544, 140612732977151, -ERASE, 140612707799040, 140612707803135, -ERASE, 140612707803136, 140612716191743, -ERASE, 140613504712704, 140613504716799, -ERASE, 140613504716800, 140613513105407, - }; - - unsigned long set39[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140736271417344, 140737488351231, -SNULL, 140736271421439, 140737488351231, -STORE, 140736271417344, 140736271421439, -STORE, 140736271286272, 140736271421439, -STORE, 94412930822144, 94412933074943, -SNULL, 94412930953215, 94412933074943, -STORE, 94412930822144, 94412930953215, -STORE, 94412930953216, 94412933074943, -ERASE, 94412930953216, 94412933074943, -STORE, 94412933046272, 94412933054463, -STORE, 94412933054464, 94412933074943, -STORE, 140326136901632, 140326139154431, -SNULL, 140326137044991, 140326139154431, -STORE, 140326136901632, 140326137044991, -STORE, 140326137044992, 140326139154431, -ERASE, 140326137044992, 140326139154431, -STORE, 140326139142144, 140326139150335, -STORE, 140326139150336, 140326139154431, -STORE, 140736271585280, 140736271589375, -STORE, 140736271572992, 140736271585279, -STORE, 140326139113472, 140326139142143, -STORE, 140326139105280, 140326139113471, -STORE, 140326134685696, 140326136901631, -SNULL, 140326134685696, 140326134783999, -STORE, 140326134784000, 140326136901631, -STORE, 140326134685696, 140326134783999, -SNULL, 140326136877055, 140326136901631, -STORE, 140326134784000, 140326136877055, -STORE, 140326136877056, 140326136901631, -SNULL, 140326136877056, 140326136885247, -STORE, 140326136885248, 140326136901631, -STORE, 140326136877056, 140326136885247, -ERASE, 140326136877056, 140326136885247, -STORE, 140326136877056, 140326136885247, -ERASE, 140326136885248, 140326136901631, -STORE, 140326136885248, 140326136901631, -STORE, 140326130888704, 140326134685695, -SNULL, 140326130888704, 140326132547583, -STORE, 140326132547584, 140326134685695, -STORE, 140326130888704, 140326132547583, -SNULL, 140326134644735, 140326134685695, -STORE, 140326132547584, 140326134644735, -STORE, 140326134644736, 140326134685695, -SNULL, 140326134644736, 140326134669311, -STORE, 140326134669312, 140326134685695, -STORE, 140326134644736, 140326134669311, -ERASE, 140326134644736, 140326134669311, -STORE, 140326134644736, 140326134669311, -ERASE, 140326134669312, 140326134685695, -STORE, 140326134669312, 140326134685695, -STORE, 140326139097088, 140326139113471, -SNULL, 140326134661119, 140326134669311, -STORE, 140326134644736, 140326134661119, -STORE, 140326134661120, 140326134669311, -SNULL, 140326136881151, 140326136885247, -STORE, 140326136877056, 140326136881151, -STORE, 140326136881152, 140326136885247, -SNULL, 94412933050367, 94412933054463, -STORE, 94412933046272, 94412933050367, -STORE, 94412933050368, 94412933054463, -SNULL, 140326139146239, 140326139150335, -STORE, 140326139142144, 140326139146239, -STORE, 140326139146240, 140326139150335, -ERASE, 140326139113472, 140326139142143, -STORE, 94412939493376, 94412939628543, -STORE, 140326122496000, 140326130888703, -SNULL, 140326122500095, 140326130888703, -STORE, 140326122496000, 140326122500095, -STORE, 140326122500096, 140326130888703, -STORE, 140326114103296, 140326122495999, -STORE, 140325979885568, 140326114103295, -SNULL, 140325979885568, 140326043910143, -STORE, 140326043910144, 140326114103295, -STORE, 140325979885568, 140326043910143, -ERASE, 140325979885568, 140326043910143, -SNULL, 140326111019007, 140326114103295, -STORE, 140326043910144, 140326111019007, -STORE, 140326111019008, 140326114103295, -ERASE, 140326111019008, 140326114103295, -SNULL, 140326044045311, 140326111019007, -STORE, 140326043910144, 140326044045311, -STORE, 140326044045312, 140326111019007, -SNULL, 140326114107391, 140326122495999, -STORE, 140326114103296, 140326114107391, -STORE, 140326114107392, 140326122495999, -STORE, 140326035517440, 140326043910143, -SNULL, 140326035521535, 140326043910143, -STORE, 140326035517440, 140326035521535, -STORE, 140326035521536, 140326043910143, -STORE, 140326027124736, 140326035517439, -SNULL, 140326027128831, 140326035517439, -STORE, 140326027124736, 140326027128831, -STORE, 140326027128832, 140326035517439, -STORE, 140326018732032, 140326027124735, -SNULL, 140326018736127, 140326027124735, -STORE, 140326018732032, 140326018736127, -STORE, 140326018736128, 140326027124735, -STORE, 140326010339328, 140326018732031, -STORE, 140326001946624, 140326018732031, -STORE, 140325993553920, 140326018732031, -STORE, 140325859336192, 140325993553919, -SNULL, 140325859336192, 140325909692415, -STORE, 140325909692416, 140325993553919, -STORE, 140325859336192, 140325909692415, -ERASE, 140325859336192, 140325909692415, -SNULL, 140325976801279, 140325993553919, -STORE, 140325909692416, 140325976801279, -STORE, 140325976801280, 140325993553919, -ERASE, 140325976801280, 140325993553919, -STORE, 140325985161216, 140326018732031, -STORE, 140325775474688, 140325976801279, -STORE, 140325708365824, 140325976801279, -SNULL, 140325708500991, 140325976801279, -STORE, 140325708365824, 140325708500991, -STORE, 140325708500992, 140325976801279, -SNULL, 140325708500992, 140325909692415, -STORE, 140325909692416, 140325976801279, -STORE, 140325708500992, 140325909692415, -SNULL, 140325909827583, 140325976801279, -STORE, 140325909692416, 140325909827583, -STORE, 140325909827584, 140325976801279, -SNULL, 140325842583551, 140325909692415, -STORE, 140325708500992, 140325842583551, -STORE, 140325842583552, 140325909692415, -ERASE, 140325842583552, 140325909692415, -SNULL, 140325708500992, 140325775474687, -STORE, 140325775474688, 140325842583551, -STORE, 140325708500992, 140325775474687, -SNULL, 140325775609855, 140325842583551, -STORE, 140325775474688, 140325775609855, -STORE, 140325775609856, 140325842583551, -STORE, 140325775609856, 140325909692415, -SNULL, 140325775609856, 140325842583551, -STORE, 140325842583552, 140325909692415, -STORE, 140325775609856, 140325842583551, -SNULL, 140325842718719, 140325909692415, -STORE, 140325842583552, 140325842718719, -STORE, 140325842718720, 140325909692415, -SNULL, 140325985161216, 140325993553919, -STORE, 140325993553920, 140326018732031, -STORE, 140325985161216, 140325993553919, -SNULL, 140325993558015, 140326018732031, -STORE, 140325993553920, 140325993558015, -STORE, 140325993558016, 140326018732031, -SNULL, 140325985165311, 140325993553919, -STORE, 140325985161216, 140325985165311, -STORE, 140325985165312, 140325993553919, -SNULL, 140325993558016, 140326001946623, -STORE, 140326001946624, 140326018732031, -STORE, 140325993558016, 140326001946623, -SNULL, 140326001950719, 140326018732031, -STORE, 140326001946624, 140326001950719, -STORE, 140326001950720, 140326018732031, -SNULL, 140326001950720, 140326010339327, -STORE, 140326010339328, 140326018732031, -STORE, 140326001950720, 140326010339327, -SNULL, 140326010343423, 140326018732031, -STORE, 140326010339328, 140326010343423, -STORE, 140326010343424, 140326018732031, -STORE, 140325699973120, 140325708365823, -STORE, 140325691580416, 140325708365823, -STORE, 140325683187712, 140325708365823, -SNULL, 140325683191807, 140325708365823, -STORE, 140325683187712, 140325683191807, -STORE, 140325683191808, 140325708365823, -SNULL, 140325683191808, 140325699973119, -STORE, 140325699973120, 140325708365823, -STORE, 140325683191808, 140325699973119, -SNULL, 140325699977215, 140325708365823, -STORE, 140325699973120, 140325699977215, -STORE, 140325699977216, 140325708365823, -STORE, 140325674795008, 140325683187711, -STORE, 140325666402304, 140325683187711, -STORE, 140325658009600, 140325683187711, -SNULL, 140325658009600, 140325666402303, -STORE, 140325666402304, 140325683187711, -STORE, 140325658009600, 140325666402303, -SNULL, 140325666406399, 140325683187711, -STORE, 140325666402304, 140325666406399, -STORE, 140325666406400, 140325683187711, -SNULL, 140325683191808, 140325691580415, -STORE, 140325691580416, 140325699973119, -STORE, 140325683191808, 140325691580415, -SNULL, 140325691584511, 140325699973119, -STORE, 140325691580416, 140325691584511, -STORE, 140325691584512, 140325699973119, -SNULL, 140325666406400, 140325674795007, -STORE, 140325674795008, 140325683187711, -STORE, 140325666406400, 140325674795007, -SNULL, 140325674799103, 140325683187711, -STORE, 140325674795008, 140325674799103, -STORE, 140325674799104, 140325683187711, -STORE, 140325649616896, 140325666402303, -SNULL, 140325649616896, 140325658009599, -STORE, 140325658009600, 140325666402303, -STORE, 140325649616896, 140325658009599, -SNULL, 140325658013695, 140325666402303, -STORE, 140325658009600, 140325658013695, -STORE, 140325658013696, 140325666402303, -SNULL, 140325649620991, 140325658009599, -STORE, 140325649616896, 140325649620991, -STORE, 140325649620992, 140325658009599, -STORE, 140325641224192, 140325649616895, -STORE, 140325632831488, 140325649616895, -SNULL, 140325632835583, 140325649616895, -STORE, 140325632831488, 140325632835583, -STORE, 140325632835584, 140325649616895, -STORE, 140325624438784, 140325632831487, -SNULL, 140325624442879, 140325632831487, -STORE, 140325624438784, 140325624442879, -STORE, 140325624442880, 140325632831487, -SNULL, 140325632835584, 140325641224191, -STORE, 140325641224192, 140325649616895, -STORE, 140325632835584, 140325641224191, -SNULL, 140325641228287, 140325649616895, -STORE, 140325641224192, 140325641228287, -STORE, 140325641228288, 140325649616895, -STORE, 140325616046080, 140325624438783, -SNULL, 140325616050175, 140325624438783, -STORE, 140325616046080, 140325616050175, -STORE, 140325616050176, 140325624438783, -STORE, 140325607653376, 140325616046079, -SNULL, 140325607657471, 140325616046079, -STORE, 140325607653376, 140325607657471, -STORE, 140325607657472, 140325616046079, -STORE, 140325599260672, 140325607653375, -STORE, 140325590867968, 140325607653375, -STORE, 140325456650240, 140325590867967, -SNULL, 140325456650240, 140325507039231, -STORE, 140325507039232, 140325590867967, -STORE, 140325456650240, 140325507039231, -ERASE, 140325456650240, 140325507039231, -STORE, 140325498646528, 140325507039231, -STORE, 140325364428800, 140325498646527, -SNULL, 140325364428800, 140325372821503, -STORE, 140325372821504, 140325498646527, -STORE, 140325364428800, 140325372821503, -ERASE, 140325364428800, 140325372821503, -STORE, 140325364428800, 140325372821503, -STORE, 140325356036096, 140325372821503, -STORE, 140325221818368, 140325356036095, -SNULL, 140325221818368, 140325238603775, -STORE, 140325238603776, 140325356036095, -STORE, 140325221818368, 140325238603775, -ERASE, 140325221818368, 140325238603775, -STORE, 140325230211072, 140325238603775, -STORE, 140325221818368, 140325238603775, -STORE, 140325087600640, 140325221818367, -STORE, 140325079207936, 140325087600639, -SNULL, 140325087600640, 140325104386047, -STORE, 140325104386048, 140325221818367, -STORE, 140325087600640, 140325104386047, -ERASE, 140325087600640, 140325104386047, -STORE, 140325095993344, 140325104386047, -STORE, 140325079207936, 140325104386047, -STORE, 140324944990208, 140325079207935, -SNULL, 140324944990208, 140324970168319, -STORE, 140324970168320, 140325079207935, -STORE, 140324944990208, 140324970168319, -ERASE, 140324944990208, 140324970168319, -STORE, 140324961775616, 140324970168319, -STORE, 140324953382912, 140324970168319, -STORE, 140324819165184, 140324953382911, -STORE, 140324684947456, 140324953382911, -STORE, 140324676554752, 140324684947455, -STORE, 140324668162048, 140324684947455, -STORE, 140324533944320, 140324668162047, -STORE, 140324525551616, 140324533944319, -SNULL, 140324533944320, 140324567515135, -STORE, 140324567515136, 140324668162047, -STORE, 140324533944320, 140324567515135, -ERASE, 140324533944320, 140324567515135, -STORE, 140324559122432, 140324567515135, -STORE, 140324391333888, 140324525551615, -SNULL, 140325574148095, 140325590867967, -STORE, 140325507039232, 140325574148095, -STORE, 140325574148096, 140325590867967, -ERASE, 140325574148096, 140325590867967, -SNULL, 140325439930367, 140325498646527, -STORE, 140325372821504, 140325439930367, -STORE, 140325439930368, 140325498646527, -ERASE, 140325439930368, 140325498646527, -SNULL, 140325305712639, 140325356036095, -STORE, 140325238603776, 140325305712639, -STORE, 140325305712640, 140325356036095, -ERASE, 140325305712640, 140325356036095, -SNULL, 140325171494911, 140325221818367, -STORE, 140325104386048, 140325171494911, -STORE, 140325171494912, 140325221818367, -ERASE, 140325171494912, 140325221818367, -SNULL, 140325104521215, 140325171494911, -STORE, 140325104386048, 140325104521215, -STORE, 140325104521216, 140325171494911, -STORE, 140324257116160, 140324525551615, -SNULL, 140324257116160, 140324299079679, -STORE, 140324299079680, 140324525551615, -STORE, 140324257116160, 140324299079679, -ERASE, 140324257116160, 140324299079679, -SNULL, 140325037277183, 140325079207935, -STORE, 140324970168320, 140325037277183, -STORE, 140325037277184, 140325079207935, -ERASE, 140325037277184, 140325079207935, -SNULL, 140324819165183, 140324953382911, -STORE, 140324684947456, 140324819165183, -STORE, 140324819165184, 140324953382911, -SNULL, 140324819165184, 140324835950591, -STORE, 140324835950592, 140324953382911, -STORE, 140324819165184, 140324835950591, -ERASE, 140324819165184, 140324835950591, -SNULL, 140324903059455, 140324953382911, -STORE, 140324835950592, 140324903059455, -STORE, 140324903059456, 140324953382911, -ERASE, 140324903059456, 140324953382911, -SNULL, 140324684947456, 140324701732863, -STORE, 140324701732864, 140324819165183, -STORE, 140324684947456, 140324701732863, -ERASE, 140324684947456, 140324701732863, -SNULL, 140324768841727, 140324819165183, -STORE, 140324701732864, 140324768841727, -STORE, 140324768841728, 140324819165183, -ERASE, 140324768841728, 140324819165183, -SNULL, 140324634623999, 140324668162047, -STORE, 140324567515136, 140324634623999, -STORE, 140324634624000, 140324668162047, -ERASE, 140324634624000, 140324668162047, -SNULL, 140324391333887, 140324525551615, -STORE, 140324299079680, 140324391333887, -STORE, 140324391333888, 140324525551615, -SNULL, 140324391333888, 140324433297407, -STORE, 140324433297408, 140324525551615, -STORE, 140324391333888, 140324433297407, -ERASE, 140324391333888, 140324433297407, -SNULL, 140325507174399, 140325574148095, -STORE, 140325507039232, 140325507174399, -STORE, 140325507174400, 140325574148095, -SNULL, 140325590867968, 140325599260671, -STORE, 140325599260672, 140325607653375, -STORE, 140325590867968, 140325599260671, -SNULL, 140325599264767, 140325607653375, -STORE, 140325599260672, 140325599264767, -STORE, 140325599264768, 140325607653375, -SNULL, 140325372956671, 140325439930367, -STORE, 140325372821504, 140325372956671, -STORE, 140325372956672, 140325439930367, -SNULL, 140324668166143, 140324684947455, -STORE, 140324668162048, 140324668166143, -STORE, 140324668166144, 140324684947455, -SNULL, 140324525555711, 140324533944319, -STORE, 140324525551616, 140324525555711, -STORE, 140324525555712, 140324533944319, -SNULL, 140324953382912, 140324961775615, -STORE, 140324961775616, 140324970168319, -STORE, 140324953382912, 140324961775615, -SNULL, 140324961779711, 140324970168319, -STORE, 140324961775616, 140324961779711, -STORE, 140324961779712, 140324970168319, -SNULL, 140325079212031, 140325104386047, -STORE, 140325079207936, 140325079212031, -STORE, 140325079212032, 140325104386047, -SNULL, 140325221818368, 140325230211071, -STORE, 140325230211072, 140325238603775, -STORE, 140325221818368, 140325230211071, -SNULL, 140325230215167, 140325238603775, -STORE, 140325230211072, 140325230215167, -STORE, 140325230215168, 140325238603775, -SNULL, 140325356036096, 140325364428799, -STORE, 140325364428800, 140325372821503, -STORE, 140325356036096, 140325364428799, -SNULL, 140325364432895, 140325372821503, - }; - unsigned long set40[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140734309167104, 140737488351231, -SNULL, 140734309171199, 140737488351231, -STORE, 140734309167104, 140734309171199, -STORE, 140734309036032, 140734309171199, -STORE, 94270500081664, 94270502334463, -SNULL, 94270500212735, 94270502334463, -STORE, 94270500081664, 94270500212735, -STORE, 94270500212736, 94270502334463, -ERASE, 94270500212736, 94270502334463, -STORE, 94270502305792, 94270502313983, -STORE, 94270502313984, 94270502334463, -STORE, 140321935110144, 140321937362943, -SNULL, 140321935253503, 140321937362943, -STORE, 140321935110144, 140321935253503, -STORE, 140321935253504, 140321937362943, -ERASE, 140321935253504, 140321937362943, -STORE, 140321937350656, 140321937358847, -STORE, 140321937358848, 140321937362943, -STORE, 140734309625856, 140734309629951, -STORE, 140734309613568, 140734309625855, -STORE, 140321937321984, 140321937350655, -STORE, 140321937313792, 140321937321983, -STORE, 140321932894208, 140321935110143, -SNULL, 140321932894208, 140321932992511, -STORE, 140321932992512, 140321935110143, -STORE, 140321932894208, 140321932992511, -SNULL, 140321935085567, 140321935110143, -STORE, 140321932992512, 140321935085567, -STORE, 140321935085568, 140321935110143, -SNULL, 140321935085568, 140321935093759, -STORE, 140321935093760, 140321935110143, -STORE, 140321935085568, 140321935093759, -ERASE, 140321935085568, 140321935093759, -STORE, 140321935085568, 140321935093759, -ERASE, 140321935093760, 140321935110143, -STORE, 140321935093760, 140321935110143, -STORE, 140321929097216, 140321932894207, -SNULL, 140321929097216, 140321930756095, -STORE, 140321930756096, 140321932894207, -STORE, 140321929097216, 140321930756095, -SNULL, 140321932853247, 140321932894207, -STORE, 140321930756096, 140321932853247, -STORE, 140321932853248, 140321932894207, -SNULL, 140321932853248, 140321932877823, -STORE, 140321932877824, 140321932894207, -STORE, 140321932853248, 140321932877823, -ERASE, 140321932853248, 140321932877823, -STORE, 140321932853248, 140321932877823, -ERASE, 140321932877824, 140321932894207, -STORE, 140321932877824, 140321932894207, -STORE, 140321937305600, 140321937321983, -SNULL, 140321932869631, 140321932877823, -STORE, 140321932853248, 140321932869631, -STORE, 140321932869632, 140321932877823, -SNULL, 140321935089663, 140321935093759, -STORE, 140321935085568, 140321935089663, -STORE, 140321935089664, 140321935093759, -SNULL, 94270502309887, 94270502313983, -STORE, 94270502305792, 94270502309887, -STORE, 94270502309888, 94270502313983, -SNULL, 140321937354751, 140321937358847, -STORE, 140321937350656, 140321937354751, -STORE, 140321937354752, 140321937358847, -ERASE, 140321937321984, 140321937350655, -STORE, 94270507364352, 94270507499519, -STORE, 140321920704512, 140321929097215, -SNULL, 140321920708607, 140321929097215, -STORE, 140321920704512, 140321920708607, -STORE, 140321920708608, 140321929097215, -STORE, 140321912311808, 140321920704511, -STORE, 140321778094080, 140321912311807, -SNULL, 140321778094080, 140321816051711, -STORE, 140321816051712, 140321912311807, -STORE, 140321778094080, 140321816051711, -ERASE, 140321778094080, 140321816051711, -SNULL, 140321883160575, 140321912311807, -STORE, 140321816051712, 140321883160575, -STORE, 140321883160576, 140321912311807, -ERASE, 140321883160576, 140321912311807, -SNULL, 140321816186879, 140321883160575, -STORE, 140321816051712, 140321816186879, -STORE, 140321816186880, 140321883160575, -SNULL, 140321912315903, 140321920704511, -STORE, 140321912311808, 140321912315903, -STORE, 140321912315904, 140321920704511, -STORE, 140321903919104, 140321912311807, -SNULL, 140321903923199, 140321912311807, -STORE, 140321903919104, 140321903923199, -STORE, 140321903923200, 140321912311807, -STORE, 140321895526400, 140321903919103, -SNULL, 140321895530495, 140321903919103, -STORE, 140321895526400, 140321895530495, -STORE, 140321895530496, 140321903919103, -STORE, 140321887133696, 140321895526399, -SNULL, 140321887137791, 140321895526399, -STORE, 140321887133696, 140321887137791, -STORE, 140321887137792, 140321895526399, -STORE, 140321807659008, 140321816051711, -STORE, 140321673441280, 140321807659007, -SNULL, 140321673441280, 140321681833983, -STORE, 140321681833984, 140321807659007, -STORE, 140321673441280, 140321681833983, -ERASE, 140321673441280, 140321681833983, -SNULL, 140321748942847, 140321807659007, -STORE, 140321681833984, 140321748942847, -STORE, 140321748942848, 140321807659007, -ERASE, 140321748942848, 140321807659007, -STORE, 140321799266304, 140321816051711, -STORE, 140321790873600, 140321816051711, -STORE, 140321782480896, 140321816051711, -STORE, 140321547616256, 140321748942847, -SNULL, 140321614725119, 140321748942847, -STORE, 140321547616256, 140321614725119, -STORE, 140321614725120, 140321748942847, -SNULL, 140321614725120, 140321681833983, -STORE, 140321681833984, 140321748942847, -STORE, 140321614725120, 140321681833983, -ERASE, 140321614725120, 140321681833983, -SNULL, 140321681969151, 140321748942847, -STORE, 140321681833984, 140321681969151, -STORE, 140321681969152, 140321748942847, -STORE, 140321547616256, 140321681833983, -SNULL, 140321547616256, 140321614725119, -STORE, 140321614725120, 140321681833983, -STORE, 140321547616256, 140321614725119, -SNULL, 140321614860287, 140321681833983, -STORE, 140321614725120, 140321614860287, -STORE, 140321614860288, 140321681833983, -SNULL, 140321547751423, 140321614725119, -STORE, 140321547616256, 140321547751423, -STORE, 140321547751424, 140321614725119, -STORE, 140321480507392, 140321547616255, -SNULL, 140321782480896, 140321799266303, -STORE, 140321799266304, 140321816051711, -STORE, 140321782480896, 140321799266303, -SNULL, 140321799270399, 140321816051711, -STORE, 140321799266304, 140321799270399, -STORE, 140321799270400, 140321816051711, -STORE, 140321774088192, 140321799266303, -SNULL, 140321774088192, 140321790873599, -STORE, 140321790873600, 140321799266303, -STORE, 140321774088192, 140321790873599, -SNULL, 140321790877695, 140321799266303, -STORE, 140321790873600, 140321790877695, -STORE, 140321790877696, 140321799266303, -SNULL, 140321480642559, 140321547616255, -STORE, 140321480507392, 140321480642559, -STORE, 140321480642560, 140321547616255, -SNULL, 140321774088192, 140321782480895, -STORE, 140321782480896, 140321790873599, -STORE, 140321774088192, 140321782480895, -SNULL, 140321782484991, 140321790873599, -STORE, 140321782480896, 140321782484991, -STORE, 140321782484992, 140321790873599, -SNULL, 140321799270400, 140321807659007, -STORE, 140321807659008, 140321816051711, -STORE, 140321799270400, 140321807659007, -SNULL, 140321807663103, 140321816051711, -STORE, 140321807659008, 140321807663103, -STORE, 140321807663104, 140321816051711, -STORE, 140321765695488, 140321782480895, -STORE, 140321757302784, 140321782480895, -SNULL, 140321757306879, 140321782480895, -STORE, 140321757302784, 140321757306879, -STORE, 140321757306880, 140321782480895, -STORE, 140321472114688, 140321480507391, -STORE, 140321463721984, 140321480507391, -SNULL, 140321463726079, 140321480507391, -STORE, 140321463721984, 140321463726079, -STORE, 140321463726080, 140321480507391, -SNULL, 140321757306880, 140321774088191, -STORE, 140321774088192, 140321782480895, -STORE, 140321757306880, 140321774088191, -SNULL, 140321774092287, 140321782480895, -STORE, 140321774088192, 140321774092287, -STORE, 140321774092288, 140321782480895, -SNULL, 140321463726080, 140321472114687, -STORE, 140321472114688, 140321480507391, -STORE, 140321463726080, 140321472114687, -SNULL, 140321472118783, 140321480507391, -STORE, 140321472114688, 140321472118783, -STORE, 140321472118784, 140321480507391, -SNULL, 140321757306880, 140321765695487, -STORE, 140321765695488, 140321774088191, -STORE, 140321757306880, 140321765695487, -SNULL, 140321765699583, 140321774088191, -STORE, 140321765695488, 140321765699583, -STORE, 140321765699584, 140321774088191, -STORE, 140321455329280, 140321463721983, -SNULL, 140321455333375, 140321463721983, -STORE, 140321455329280, 140321455333375, -STORE, 140321455333376, 140321463721983, -STORE, 140321446936576, 140321455329279, -STORE, 140321438543872, 140321455329279, -STORE, 140321430151168, 140321455329279, -SNULL, 140321430155263, 140321455329279, -STORE, 140321430151168, 140321430155263, -STORE, 140321430155264, 140321455329279, -SNULL, 140321430155264, 140321446936575, -STORE, 140321446936576, 140321455329279, -STORE, 140321430155264, 140321446936575, -SNULL, 140321446940671, 140321455329279, -STORE, 140321446936576, 140321446940671, -STORE, 140321446940672, 140321455329279, -SNULL, 140321430155264, 140321438543871, -STORE, 140321438543872, 140321446936575, -STORE, 140321430155264, 140321438543871, -SNULL, 140321438547967, 140321446936575, -STORE, 140321438543872, 140321438547967, -STORE, 140321438547968, 140321446936575, -STORE, 140321421758464, 140321430151167, -SNULL, 140321421762559, 140321430151167, -STORE, 140321421758464, 140321421762559, -STORE, 140321421762560, 140321430151167, -STORE, 140321413365760, 140321421758463, -SNULL, 140321413369855, 140321421758463, -STORE, 140321413365760, 140321413369855, -STORE, 140321413369856, 140321421758463, -STORE, 140321404973056, 140321413365759, -SNULL, 140321404977151, 140321413365759, -STORE, 140321404973056, 140321404977151, -STORE, 140321404977152, 140321413365759, -STORE, 140321396580352, 140321404973055, -STORE, 140321388187648, 140321404973055, -STORE, 140321253969920, 140321388187647, -SNULL, 140321253969920, 140321279180799, -STORE, 140321279180800, 140321388187647, -STORE, 140321253969920, 140321279180799, -ERASE, 140321253969920, 140321279180799, -SNULL, 140321346289663, 140321388187647, -STORE, 140321279180800, 140321346289663, -STORE, 140321346289664, 140321388187647, -ERASE, 140321346289664, 140321388187647, -STORE, 140321144963072, 140321346289663, -STORE, 140321379794944, 140321404973055, -STORE, 140321371402240, 140321404973055, -STORE, 140321010745344, 140321346289663, -STORE, 140321363009536, 140321404973055, -SNULL, 140321077854207, 140321346289663, -STORE, 140321010745344, 140321077854207, -STORE, 140321077854208, 140321346289663, -SNULL, 140321077854208, 140321144963071, -STORE, 140321144963072, 140321346289663, -STORE, 140321077854208, 140321144963071, -ERASE, 140321077854208, 140321144963071, -STORE, 140321354616832, 140321404973055, -STORE, 140321136570368, 140321144963071, -STORE, 140320943636480, 140321077854207, -STORE, 140320876527616, 140321077854207, -STORE, 140321128177664, 140321144963071, -SNULL, 140320876662783, 140321077854207, -STORE, 140320876527616, 140320876662783, -STORE, 140320876662784, 140321077854207, -STORE, 140321119784960, 140321144963071, -STORE, 140321111392256, 140321144963071, -STORE, 140320742309888, 140320876527615, -STORE, 140321102999552, 140321144963071, -STORE, 140320608092160, 140320876527615, -SNULL, 140320675201023, 140320876527615, -STORE, 140320608092160, 140320675201023, -STORE, 140320675201024, 140320876527615, -SNULL, 140320675201024, 140320742309887, -STORE, 140320742309888, 140320876527615, -STORE, 140320675201024, 140320742309887, -ERASE, 140320675201024, 140320742309887, -STORE, 140321094606848, 140321144963071, -STORE, 140321086214144, 140321144963071, -STORE, 140320608092160, 140320876527615, -SNULL, 140320608092160, 140320675201023, -STORE, 140320675201024, 140320876527615, -STORE, 140320608092160, 140320675201023, -SNULL, 140320675336191, 140320876527615, -STORE, 140320675201024, 140320675336191, -STORE, 140320675336192, 140320876527615, -STORE, 140320599699456, 140320608092159, -STORE, 140320591306752, 140320608092159, -STORE, 140320457089024, 140320591306751, -STORE, 140320448696320, 140320457089023, -STORE, 140320314478592, 140320448696319, -SNULL, 140321144963072, 140321279180799, -STORE, 140321279180800, 140321346289663, -STORE, 140321144963072, 140321279180799, -SNULL, 140321279315967, 140321346289663, -STORE, 140321279180800, 140321279315967, -STORE, 140321279315968, 140321346289663, -SNULL, 140321086214144, 140321136570367, -STORE, 140321136570368, 140321144963071, -STORE, 140321086214144, 140321136570367, -SNULL, 140321136574463, 140321144963071, -STORE, 140321136570368, 140321136574463, -STORE, 140321136574464, 140321144963071, -SNULL, 140321212071935, 140321279180799, -STORE, 140321144963072, 140321212071935, -STORE, 140321212071936, 140321279180799, -ERASE, 140321212071936, 140321279180799, -SNULL, 140321145098239, 140321212071935, -STORE, 140321144963072, 140321145098239, -STORE, 140321145098240, 140321212071935, -SNULL, 140320876662784, 140321010745343, -STORE, 140321010745344, 140321077854207, -STORE, 140320876662784, 140321010745343, -SNULL, 140321010880511, 140321077854207, -STORE, 140321010745344, 140321010880511, -STORE, 140321010880512, 140321077854207, -SNULL, 140321354616832, 140321379794943, -STORE, 140321379794944, 140321404973055, -STORE, 140321354616832, 140321379794943, -SNULL, 140321379799039, 140321404973055, -STORE, 140321379794944, 140321379799039, -STORE, 140321379799040, 140321404973055, -SNULL, 140320876662784, 140320943636479, -STORE, 140320943636480, 140321010745343, -STORE, 140320876662784, 140320943636479, -SNULL, 140320943771647, 140321010745343, -STORE, 140320943636480, 140320943771647, -STORE, 140320943771648, 140321010745343, -SNULL, 140320809418751, 140320876527615, -STORE, 140320675336192, 140320809418751, -STORE, 140320809418752, 140320876527615, -ERASE, 140320809418752, 140320876527615, -SNULL, 140320675336192, 140320742309887, -STORE, 140320742309888, 140320809418751, -STORE, 140320675336192, 140320742309887, -SNULL, 140320742445055, 140320809418751, -STORE, 140320742309888, 140320742445055, -STORE, 140320742445056, 140320809418751, -SNULL, 140320608227327, 140320675201023, -STORE, 140320608092160, 140320608227327, -STORE, 140320608227328, 140320675201023, -SNULL, 140320457089024, 140320473874431, -STORE, 140320473874432, 140320591306751, -STORE, 140320457089024, 140320473874431, -ERASE, 140320457089024, 140320473874431, -SNULL, 140320540983295, 140320591306751, -STORE, 140320473874432, 140320540983295, -STORE, 140320540983296, 140320591306751, -ERASE, 140320540983296, 140320591306751, -SNULL, 140320314478592, 140320339656703, -STORE, 140320339656704, 140320448696319, -STORE, 140320314478592, 140320339656703, -ERASE, 140320314478592, 140320339656703, -SNULL, 140321086214144, 140321128177663, -STORE, 140321128177664, 140321136570367, -STORE, 140321086214144, 140321128177663, -SNULL, 140321128181759, 140321136570367, -STORE, 140321128177664, 140321128181759, -STORE, 140321128181760, 140321136570367, -SNULL, 140321354616832, 140321371402239, -STORE, 140321371402240, 140321379794943, -STORE, 140321354616832, 140321371402239, -SNULL, 140321371406335, 140321379794943, -STORE, 140321371402240, 140321371406335, -STORE, 140321371406336, 140321379794943, -SNULL, 140320591310847, 140320608092159, -STORE, 140320591306752, 140320591310847, -STORE, 140320591310848, 140320608092159, -SNULL, 140321354616832, 140321363009535, -STORE, 140321363009536, 140321371402239, -STORE, 140321354616832, 140321363009535, -SNULL, 140321363013631, 140321371402239, -STORE, 140321363009536, 140321363013631, -STORE, 140321363013632, 140321371402239, -SNULL, 140321086214144, 140321119784959, -STORE, 140321119784960, 140321128177663, -STORE, 140321086214144, 140321119784959, -SNULL, 140321119789055, 140321128177663, -STORE, 140321119784960, 140321119789055, -STORE, 140321119789056, 140321128177663, -SNULL, 140321086218239, 140321119784959, -STORE, 140321086214144, 140321086218239, -STORE, 140321086218240, 140321119784959, -SNULL, 140321086218240, 140321094606847, -STORE, 140321094606848, 140321119784959, -STORE, 140321086218240, 140321094606847, -SNULL, 140321094610943, 140321119784959, -STORE, 140321094606848, 140321094610943, -STORE, 140321094610944, 140321119784959, -SNULL, 140320474009599, 140320540983295, -STORE, 140320473874432, 140320474009599, -STORE, 140320474009600, 140320540983295, -SNULL, 140320406765567, 140320448696319, -STORE, 140320339656704, 140320406765567, -STORE, 140320406765568, 140320448696319, -ERASE, 140320406765568, 140320448696319, -SNULL, 140320339791871, 140320406765567, -STORE, 140320339656704, 140320339791871, -STORE, 140320339791872, 140320406765567, -STORE, 140321270788096, 140321279180799, -STORE, 140321262395392, 140321279180799, -STORE, 140321254002688, 140321279180799, -SNULL, 140321254002688, 140321262395391, -STORE, 140321262395392, 140321279180799, -STORE, 140321254002688, 140321262395391, -SNULL, 140321262399487, 140321279180799, -STORE, 140321262395392, 140321262399487, -STORE, 140321262399488, 140321279180799, -STORE, 140321245609984, 140321262395391, -STORE, 140321237217280, 140321262395391, -SNULL, 140321237217280, 140321245609983, -STORE, 140321245609984, 140321262395391, -STORE, 140321237217280, 140321245609983, -SNULL, 140321245614079, 140321262395391, -STORE, 140321245609984, 140321245614079, -STORE, 140321245614080, 140321262395391, -SNULL, 140321379799040, 140321388187647, -STORE, 140321388187648, 140321404973055, -STORE, 140321379799040, 140321388187647, -SNULL, 140321388191743, 140321404973055, -STORE, 140321388187648, 140321388191743, -STORE, 140321388191744, 140321404973055, -SNULL, 140321354620927, 140321363009535, -STORE, 140321354616832, 140321354620927, -STORE, 140321354620928, 140321363009535, -SNULL, 140321388191744, 140321396580351, -STORE, 140321396580352, 140321404973055, -STORE, 140321388191744, 140321396580351, -SNULL, 140321396584447, 140321404973055, -STORE, 140321396580352, 140321396584447, -STORE, 140321396584448, 140321404973055, -SNULL, 140321094610944, 140321111392255, -STORE, 140321111392256, 140321119784959, -STORE, 140321094610944, 140321111392255, -SNULL, 140321111396351, 140321119784959, -STORE, 140321111392256, 140321111396351, -STORE, 140321111396352, 140321119784959, -STORE, 140321228824576, 140321245609983, -SNULL, 140321094610944, 140321102999551, -STORE, 140321102999552, 140321111392255, -STORE, 140321094610944, 140321102999551, -SNULL, 140321103003647, 140321111392255, -STORE, 140321102999552, 140321103003647, -STORE, 140321103003648, 140321111392255, -STORE, 140321220431872, 140321245609983, -SNULL, 140321220435967, 140321245609983, -STORE, 140321220431872, 140321220435967, -STORE, 140321220435968, 140321245609983, -STORE, 140320868134912, 140320876527615, -SNULL, 140320868139007, 140320876527615, -STORE, 140320868134912, 140320868139007, -STORE, 140320868139008, 140320876527615, -SNULL, 140320591310848, 140320599699455, -STORE, 140320599699456, 140320608092159, -STORE, 140320591310848, 140320599699455, -SNULL, 140320599703551, 140320608092159, -STORE, 140320599699456, 140320599703551, -STORE, 140320599703552, 140320608092159, -STORE, 140320859742208, 140320868134911, -SNULL, 140321262399488, 140321270788095, -STORE, 140321270788096, 140321279180799, -STORE, 140321262399488, 140321270788095, -SNULL, 140321270792191, 140321279180799, -STORE, 140321270788096, 140321270792191, -STORE, 140321270792192, 140321279180799, -STORE, 140320851349504, 140320868134911, -STORE, 140320842956800, 140320868134911, -STORE, 140320834564096, 140320868134911, -STORE, 140320826171392, 140320868134911, -SNULL, 140320826171392, 140320834564095, -STORE, 140320834564096, 140320868134911, -STORE, 140320826171392, 140320834564095, -SNULL, 140320834568191, 140320868134911, -STORE, 140320834564096, 140320834568191, -STORE, 140320834568192, 140320868134911, -SNULL, 140321220435968, 140321228824575, -STORE, 140321228824576, 140321245609983, -STORE, 140321220435968, 140321228824575, -SNULL, 140321228828671, 140321245609983, -STORE, 140321228824576, 140321228828671, -STORE, 140321228828672, 140321245609983, -STORE, 140320817778688, 140320834564095, -SNULL, 140320817782783, 140320834564095, -STORE, 140320817778688, 140320817782783, -STORE, 140320817782784, 140320834564095, -STORE, 140320582914048, 140320591306751, -SNULL, 140321228828672, 140321237217279, -STORE, 140321237217280, 140321245609983, -STORE, 140321228828672, 140321237217279, -SNULL, 140321237221375, 140321245609983, -STORE, 140321237217280, 140321237221375, -STORE, 140321237221376, 140321245609983, -SNULL, 140320448700415, 140320457089023, -STORE, 140320448696320, 140320448700415, -STORE, 140320448700416, 140320457089023, -SNULL, 140321245614080, 140321254002687, -STORE, 140321254002688, 140321262395391, -STORE, 140321245614080, 140321254002687, -SNULL, 140321254006783, 140321262395391, -STORE, 140321254002688, 140321254006783, -STORE, 140321254006784, 140321262395391, -STORE, 140320574521344, 140320591306751, -SNULL, 140320574525439, 140320591306751, -STORE, 140320574521344, 140320574525439, -STORE, 140320574525440, 140320591306751, -STORE, 140320566128640, 140320574521343, -SNULL, 140320566132735, 140320574521343, -STORE, 140320566128640, 140320566132735, -STORE, 140320566132736, 140320574521343, -SNULL, 140320574525440, 140320582914047, -STORE, 140320582914048, 140320591306751, -STORE, 140320574525440, 140320582914047, -SNULL, 140320582918143, 140320591306751, -STORE, 140320582914048, 140320582918143, -STORE, 140320582918144, 140320591306751, -STORE, 140320557735936, 140320566128639, -SNULL, 140320557740031, 140320566128639, -STORE, 140320557735936, 140320557740031, -STORE, 140320557740032, 140320566128639, -STORE, 140320549343232, 140320557735935, -STORE, 140320465481728, 140320473874431, -STORE, 140320448700416, 140320473874431, -SNULL, 140320834568192, 140320859742207, -STORE, 140320859742208, 140320868134911, -STORE, 140320834568192, 140320859742207, -SNULL, 140320859746303, 140320868134911, -STORE, 140320859742208, 140320859746303, -STORE, 140320859746304, 140320868134911, -STORE, 140320440303616, 140320448696319, -STORE, 140320431910912, 140320448696319, -SNULL, 140320834568192, 140320851349503, -STORE, 140320851349504, 140320859742207, -STORE, 140320834568192, 140320851349503, -SNULL, 140320851353599, 140320859742207, -STORE, 140320851349504, 140320851353599, -STORE, 140320851353600, 140320859742207, -SNULL, 140320817782784, 140320826171391, -STORE, 140320826171392, 140320834564095, -STORE, 140320817782784, 140320826171391, -SNULL, 140320826175487, 140320834564095, -STORE, 140320826171392, 140320826175487, -STORE, 140320826175488, 140320834564095, -SNULL, 140320834568192, 140320842956799, -STORE, 140320842956800, 140320851349503, -STORE, 140320834568192, 140320842956799, -SNULL, 140320842960895, 140320851349503, -STORE, 140320842956800, 140320842960895, -STORE, 140320842960896, 140320851349503, -STORE, 140320423518208, 140320448696319, -SNULL, 140320423522303, 140320448696319, -STORE, 140320423518208, 140320423522303, -STORE, 140320423522304, 140320448696319, -STORE, 140320415125504, 140320423518207, -STORE, 140320331264000, 140320339656703, -STORE, 140320322871296, 140320339656703, -STORE, 140320314478592, 140320339656703, -SNULL, 140320314482687, 140320339656703, -STORE, 140320314478592, 140320314482687, -STORE, 140320314482688, 140320339656703, -STORE, 140320306085888, 140320314478591, -SNULL, 140320306089983, 140320314478591, -STORE, 140320306085888, 140320306089983, -STORE, 140320306089984, 140320314478591, -STORE, 140320297693184, 140320306085887, -SNULL, 140320297697279, 140320306085887, -STORE, 140320297693184, 140320297697279, -STORE, 140320297697280, 140320306085887, -STORE, 140320289300480, 140320297693183, -STORE, 140320280907776, 140320297693183, -SNULL, 140320280911871, 140320297693183, -STORE, 140320280907776, 140320280911871, -STORE, 140320280911872, 140320297693183, -SNULL, 140320423522304, 140320431910911, -STORE, 140320431910912, 140320448696319, -STORE, 140320423522304, 140320431910911, -SNULL, 140320431915007, 140320448696319, -STORE, 140320431910912, 140320431915007, -STORE, 140320431915008, 140320448696319, -SNULL, 140320549347327, 140320557735935, -STORE, 140320549343232, 140320549347327, -STORE, 140320549347328, 140320557735935, -STORE, 140320272515072, 140320280907775, -SNULL, 140320448700416, 140320457089023, -STORE, 140320457089024, 140320473874431, -STORE, 140320448700416, 140320457089023, -SNULL, 140320457093119, 140320473874431, -STORE, 140320457089024, 140320457093119, -STORE, 140320457093120, 140320473874431, -STORE, 140320264122368, 140320280907775, -SNULL, 140320457093120, 140320465481727, -STORE, 140320465481728, 140320473874431, -STORE, 140320457093120, 140320465481727, -SNULL, 140320465485823, 140320473874431, -STORE, 140320465481728, 140320465485823, -STORE, 140320465485824, 140320473874431, -SNULL, 140320431915008, 140320440303615, -STORE, 140320440303616, 140320448696319, -STORE, 140320431915008, 140320440303615, -SNULL, 140320440307711, 140320448696319, -STORE, 140320440303616, 140320440307711, -STORE, 140320440307712, 140320448696319, -STORE, 140320255729664, 140320280907775, -STORE, 140320247336960, 140320280907775, -SNULL, 140320247341055, 140320280907775, -STORE, 140320247336960, 140320247341055, -STORE, 140320247341056, 140320280907775, -STORE, 140320238944256, 140320247336959, -STORE, 140320230551552, 140320247336959, -SNULL, 140320230551552, 140320238944255, -STORE, 140320238944256, 140320247336959, -STORE, 140320230551552, 140320238944255, -SNULL, 140320238948351, 140320247336959, -STORE, 140320238944256, 140320238948351, -STORE, 140320238948352, 140320247336959, -SNULL, 140320314482688, 140320331263999, -STORE, 140320331264000, 140320339656703, -STORE, 140320314482688, 140320331263999, -SNULL, 140320331268095, 140320339656703, -STORE, 140320331264000, 140320331268095, -STORE, 140320331268096, 140320339656703, -SNULL, 140320280911872, 140320289300479, -STORE, 140320289300480, 140320297693183, -STORE, 140320280911872, 140320289300479, -SNULL, 140320289304575, 140320297693183, -STORE, 140320289300480, 140320289304575, -STORE, 140320289304576, 140320297693183, -SNULL, 140320415129599, 140320423518207, -STORE, 140320415125504, 140320415129599, -STORE, 140320415129600, 140320423518207, -STORE, 140320222158848, 140320238944255, -STORE, 140320213766144, 140320238944255, -STORE, 140320205373440, 140320238944255, -SNULL, 140320205377535, 140320238944255, -STORE, 140320205373440, 140320205377535, -STORE, 140320205377536, 140320238944255, -SNULL, 140320314482688, 140320322871295, -STORE, 140320322871296, 140320331263999, -STORE, 140320314482688, 140320322871295, -SNULL, 140320322875391, 140320331263999, -STORE, 140320322871296, 140320322875391, -STORE, 140320322875392, 140320331263999, -SNULL, 140320247341056, 140320272515071, -STORE, 140320272515072, 140320280907775, -STORE, 140320247341056, 140320272515071, -SNULL, 140320272519167, 140320280907775, -STORE, 140320272515072, 140320272519167, -STORE, 140320272519168, 140320280907775, -SNULL, 140320247341056, 140320264122367, -STORE, 140320264122368, 140320272515071, -STORE, 140320247341056, 140320264122367, -SNULL, 140320264126463, 140320272515071, -STORE, 140320264122368, 140320264126463, -STORE, 140320264126464, 140320272515071, -SNULL, 140320205377536, 140320230551551, -STORE, 140320230551552, 140320238944255, -STORE, 140320205377536, 140320230551551, -SNULL, 140320230555647, 140320238944255, -STORE, 140320230551552, 140320230555647, -STORE, 140320230555648, 140320238944255, -STORE, 140320196980736, 140320205373439, -SNULL, 140320196984831, 140320205373439, -STORE, 140320196980736, 140320196984831, -STORE, 140320196984832, 140320205373439, -STORE, 140320188588032, 140320196980735, -SNULL, 140320247341056, 140320255729663, -STORE, 140320255729664, 140320264122367, -STORE, 140320247341056, 140320255729663, -SNULL, 140320255733759, 140320264122367, -STORE, 140320255729664, 140320255733759, -STORE, 140320255733760, 140320264122367, -STORE, 140320180195328, 140320196980735, -SNULL, 140320180199423, 140320196980735, -STORE, 140320180195328, 140320180199423, -STORE, 140320180199424, 140320196980735, -STORE, 140320171802624, 140320180195327, -STORE, 140320163409920, 140320180195327, -SNULL, 140320163414015, 140320180195327, -STORE, 140320163409920, 140320163414015, -STORE, 140320163414016, 140320180195327, -SNULL, 140320205377536, 140320222158847, -STORE, 140320222158848, 140320230551551, -STORE, 140320205377536, 140320222158847, -SNULL, 140320222162943, 140320230551551, -STORE, 140320222158848, 140320222162943, -STORE, 140320222162944, 140320230551551, -SNULL, 140320205377536, 140320213766143, -STORE, 140320213766144, 140320222158847, -STORE, 140320205377536, 140320213766143, -SNULL, 140320213770239, 140320222158847, -STORE, 140320213766144, 140320213770239, -STORE, 140320213770240, 140320222158847, -STORE, 140320155017216, 140320163409919, -SNULL, 140320180199424, 140320188588031, -STORE, 140320188588032, 140320196980735, -STORE, 140320180199424, 140320188588031, -SNULL, 140320188592127, 140320196980735, -STORE, 140320188588032, 140320188592127, -STORE, 140320188592128, 140320196980735, -SNULL, 140320155021311, 140320163409919, -STORE, 140320155017216, 140320155021311, -STORE, 140320155021312, 140320163409919, -SNULL, 140320163414016, 140320171802623, -STORE, 140320171802624, 140320180195327, -STORE, 140320163414016, 140320171802623, -SNULL, 140320171806719, 140320180195327, -STORE, 140320171802624, 140320171806719, -STORE, 140320171806720, 140320180195327, -STORE, 140320146624512, 140320155017215, -SNULL, 140320146628607, 140320155017215, -STORE, 140320146624512, 140320146628607, -STORE, 140320146628608, 140320155017215, -STORE, 140321937321984, 140321937350655, -STORE, 140321884942336, 140321887133695, -SNULL, 140321884942336, 140321885032447, -STORE, 140321885032448, 140321887133695, -STORE, 140321884942336, 140321885032447, -SNULL, 140321887125503, 140321887133695, -STORE, 140321885032448, 140321887125503, -STORE, 140321887125504, 140321887133695, -ERASE, 140321887125504, 140321887133695, -STORE, 140321887125504, 140321887133695, -SNULL, 140321887129599, 140321887133695, -STORE, 140321887125504, 140321887129599, -STORE, 140321887129600, 140321887133695, -ERASE, 140321937321984, 140321937350655, -ERASE, 140321086214144, 140321086218239, -ERASE, 140321086218240, 140321094606847, -ERASE, 140321119784960, 140321119789055, -ERASE, 140321119789056, 140321128177663, -ERASE, 140321245609984, 140321245614079, -ERASE, 140321245614080, 140321254002687, -ERASE, 140320574521344, 140320574525439, -ERASE, 140320574525440, 140320582914047, -ERASE, 140320297693184, 140320297697279, -ERASE, 140320297697280, 140320306085887, -ERASE, 140321354616832, 140321354620927, -ERASE, 140321354620928, 140321363009535, -ERASE, 140320834564096, 140320834568191, -ERASE, 140320834568192, 140320842956799, -ERASE, 140320591306752, 140320591310847, -ERASE, 140320591310848, 140320599699455, -ERASE, 140321136570368, 140321136574463, -ERASE, 140321136574464, 140321144963071, -ERASE, 140321237217280, 140321237221375, -ERASE, 140321237221376, 140321245609983, -ERASE, 140321363009536, 140321363013631, -ERASE, 140321363013632, 140321371402239, -ERASE, 140320599699456, 140320599703551, -ERASE, 140320599703552, 140320608092159, -ERASE, 140321396580352, 140321396584447, -ERASE, 140321396584448, 140321404973055, -ERASE, 140320566128640, 140320566132735, -ERASE, 140320566132736, 140320574521343, -ERASE, 140321094606848, 140321094610943, -ERASE, 140321094610944, 140321102999551, -ERASE, 140320582914048, 140320582918143, -ERASE, 140320582918144, 140320591306751, -ERASE, 140320289300480, 140320289304575, -ERASE, 140320289304576, 140320297693183, -ERASE, 140320163409920, 140320163414015, - }; - unsigned long set41[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140728157171712, 140737488351231, -SNULL, 140728157175807, 140737488351231, -STORE, 140728157171712, 140728157175807, -STORE, 140728157040640, 140728157175807, -STORE, 94376106364928, 94376108613631, -SNULL, 94376106487807, 94376108613631, -STORE, 94376106364928, 94376106487807, -STORE, 94376106487808, 94376108613631, -SNULL, 94376106487808, 94376108613631, -STORE, 94376108584960, 94376108593151, -STORE, 94376108593152, 94376108613631, -STORE, 140113496432640, 140113498685439, -SNULL, 140113496575999, 140113498685439, -STORE, 140113496432640, 140113496575999, -STORE, 140113496576000, 140113498685439, -SNULL, 140113496576000, 140113498685439, -STORE, 140113498673152, 140113498681343, -STORE, 140113498681344, 140113498685439, -STORE, 140728157609984, 140728157618175, -STORE, 140728157593600, 140728157609983, -STORE, 140113498636288, 140113498673151, -STORE, 140113498628096, 140113498636287, -STORE, 140113492635648, 140113496432639, -SNULL, 140113492635648, 140113494294527, -STORE, 140113494294528, 140113496432639, -STORE, 140113492635648, 140113494294527, -SNULL, 140113496391679, 140113496432639, -STORE, 140113494294528, 140113496391679, -STORE, 140113496391680, 140113496432639, -SNULL, 140113496391680, 140113496416255, -STORE, 140113496416256, 140113496432639, -STORE, 140113496391680, 140113496416255, -SNULL, 140113496391680, 140113496416255, -STORE, 140113496391680, 140113496416255, -SNULL, 140113496416256, 140113496432639, -STORE, 140113496416256, 140113496432639, -SNULL, 140113496408063, 140113496416255, -STORE, 140113496391680, 140113496408063, -STORE, 140113496408064, 140113496416255, -SNULL, 94376108589055, 94376108593151, -STORE, 94376108584960, 94376108589055, -STORE, 94376108589056, 94376108593151, -SNULL, 140113498677247, 140113498681343, -STORE, 140113498673152, 140113498677247, -STORE, 140113498677248, 140113498681343, -SNULL, 140113498636288, 140113498673151, -STORE, 94376135090176, 94376135094271, -STORE, 94376135090176, 94376135098367, -STORE, 94376139288576, 94376139292671, -STORE, 94376143482880, 94376143486975, -STORE, 94376147677184, 94376147681279, -STORE, 94376151871488, 94376151875583, -STORE, 94376156065792, 94376156069887, -STORE, 94376160260096, 94376160264191, -STORE, 94376164454400, 94376164458495, -STORE, 94376168648704, 94376168652799, -STORE, 94376172843008, 94376172847103, -STORE, 94376177037312, 94376177041407, -STORE, 94376181231616, 94376181235711, -STORE, 94376185425920, 94376185430015, -STORE, 94376189620224, 94376189624319, -STORE, 94376193814528, 94376193818623, -STORE, 94376198008832, 94376198012927, -STORE, 94376202203136, 94376202207231, -STORE, 94376206397440, 94376206401535, -STORE, 94376210591744, 94376210595839, -STORE, 94376214786048, 94376214790143, -STORE, 94376218980352, 94376218984447, -STORE, 94376223174656, 94376223178751, -STORE, 94376227368960, 94376227373055, -STORE, 94376231563264, 94376231567359, -STORE, 94376235757568, 94376235761663, -STORE, 94376239951872, 94376239955967, -STORE, 94376244146176, 94376244150271, -STORE, 94376248340480, 94376248344575, -STORE, 94376252534784, 94376252538879, -STORE, 94376256729088, 94376256733183, -STORE, 94376260923392, 94376260927487, -STORE, 94376265117696, 94376265121791, -STORE, 94376269312000, 94376269316095, -STORE, 94376273506304, 94376273510399, -STORE, 94376277700608, 94376277704703, -STORE, 94376281894912, 94376281899007, -STORE, 94376286089216, 94376286093311, -STORE, 94376290283520, 94376290287615, -STORE, 94376294477824, 94376294481919, -STORE, 94376298672128, 94376298676223, -STORE, 94376302866432, 94376302870527, -STORE, 94376307060736, 94376307064831, -STORE, 94376311255040, 94376311259135, -STORE, 94376315449344, 94376315453439, -STORE, 94376319643648, 94376319647743, -STORE, 94376323837952, 94376323842047, -STORE, 94376328032256, 94376328036351, -STORE, 94376332226560, 94376332230655, -STORE, 94376336420864, 94376336424959, -STORE, 94376340615168, 94376340619263, -STORE, 94376344809472, 94376344813567, -STORE, 94376349003776, 94376349007871, -STORE, 94376353198080, 94376353202175, -STORE, 94376357392384, 94376357396479, -STORE, 94376361586688, 94376361590783, -STORE, 94376365780992, 94376365785087, -STORE, 94376369975296, 94376369979391, -STORE, 94376374169600, 94376374173695, -STORE, 94376378363904, 94376378367999, -STORE, 94376382558208, 94376382562303, -STORE, 94376386752512, 94376386756607, -STORE, 94376390946816, 94376390950911, -STORE, 94376395141120, 94376395145215, -STORE, 94376399335424, 94376399339519, -STORE, 94376403529728, 94376403533823, -STORE, 94376407724032, 94376407728127, -STORE, 94376411918336, 94376411922431, -STORE, 94376416112640, 94376416116735, -STORE, 94376420306944, 94376420311039, -STORE, 94376424501248, 94376424505343, -STORE, 94376428695552, 94376428699647, -STORE, 94376432889856, 94376432893951, -STORE, 94376437084160, 94376437088255, -STORE, 94376441278464, 94376441282559, -STORE, 94376445472768, 94376445476863, -STORE, 94376449667072, 94376449671167, -STORE, 94376453861376, 94376453865471, -STORE, 94376458055680, 94376458059775, -STORE, 94376462249984, 94376462254079, -STORE, 94376466444288, 94376466448383, -STORE, 94376470638592, 94376470642687, -STORE, 94376474832896, 94376474836991, -STORE, 94376479027200, 94376479031295, -STORE, 94376483221504, 94376483225599, -STORE, 94376487415808, 94376487419903, -STORE, 94376491610112, 94376491614207, -STORE, 94376495804416, 94376495808511, -STORE, 94376499998720, 94376500002815, -STORE, 94376504193024, 94376504197119, -STORE, 94376508387328, 94376508391423, -STORE, 94376512581632, 94376512585727, -STORE, 94376516775936, 94376516780031, -STORE, 94376520970240, 94376520974335, -STORE, 94376525164544, 94376525168639, -STORE, 94376529358848, 94376529362943, -STORE, 94376533553152, 94376533557247, -STORE, 94376537747456, 94376537751551, -STORE, 94376541941760, 94376541945855, -STORE, 94376546136064, 94376546140159, -STORE, 94376550330368, 94376550334463, -STORE, 94376554524672, 94376554528767, -STORE, 94376558718976, 94376558723071, -STORE, 94376562913280, 94376562917375, -STORE, 94376567107584, 94376567111679, -STORE, 94376571301888, 94376571305983, -STORE, 94376575496192, 94376575500287, -STORE, 94376579690496, 94376579694591, -STORE, 94376583884800, 94376583888895, -STORE, 94376588079104, 94376588083199, -STORE, 94376592273408, 94376592277503, -STORE, 94376596467712, 94376596471807, -STORE, 94376600662016, 94376600666111, -STORE, 94376604856320, 94376604860415, -STORE, 94376609050624, 94376609054719, -STORE, 94376613244928, 94376613249023, -STORE, 94376617439232, 94376617443327, -STORE, 94376621633536, 94376621637631, -STORE, 94376625827840, 94376625831935, -STORE, 94376630022144, 94376630026239, -STORE, 94376634216448, 94376634220543, -STORE, 94376638410752, 94376638414847, -STORE, 94376642605056, 94376642609151, -STORE, 94376646799360, 94376646803455, -STORE, 94376650993664, 94376650997759, -STORE, 94376655187968, 94376655192063, -STORE, 94376659382272, 94376659386367, -STORE, 94376663576576, 94376663580671, -STORE, 94376667770880, 94376667774975, -STORE, 94376671965184, 94376671969279, -STORE, 94376676159488, 94376676163583, -STORE, 94376680353792, 94376680357887, -STORE, 94376684548096, 94376684552191, -STORE, 94376688742400, 94376688746495, -STORE, 94376692936704, 94376692940799, -STORE, 94376697131008, 94376697135103, -STORE, 94376701325312, 94376701329407, -STORE, 94376705519616, 94376705523711, -STORE, 94376709713920, 94376709718015, -STORE, 94376713908224, 94376713912319, -STORE, 94376718102528, 94376718106623, -STORE, 94376722296832, 94376722300927, -STORE, 94376726491136, 94376726495231, -STORE, 94376730685440, 94376730689535, -STORE, 94376734879744, 94376734883839, -STORE, 94376739074048, 94376739078143, -STORE, 94376743268352, 94376743272447, -STORE, 94376747462656, 94376747466751, -STORE, 94376751656960, 94376751661055, -STORE, 94376755851264, 94376755855359, -STORE, 94376760045568, 94376760049663, -STORE, 94376764239872, 94376764243967, -STORE, 94376768434176, 94376768438271, -STORE, 94376772628480, 94376772632575, -STORE, 94376776822784, 94376776826879, -STORE, 94376781017088, 94376781021183, -STORE, 94376785211392, 94376785215487, -STORE, 94376789405696, 94376789409791, -STORE, 94376793600000, 94376793604095, -STORE, 94376797794304, 94376797798399, -STORE, 94376801988608, 94376801992703, -STORE, 94376806182912, 94376806187007, -STORE, 94376810377216, 94376810381311, -STORE, 94376814571520, 94376814575615, -STORE, 94376818765824, 94376818769919, -STORE, 94376822960128, 94376822964223, -STORE, 94376827154432, 94376827158527, -STORE, 94376831348736, 94376831352831, -STORE, 94376835543040, 94376835547135, -STORE, 94376839737344, 94376839741439, -STORE, 94376843931648, 94376843935743, -STORE, 94376848125952, 94376848130047, -STORE, 94376852320256, 94376852324351, -STORE, 94376856514560, 94376856518655, -STORE, 94376860708864, 94376860712959, -STORE, 94376864903168, 94376864907263, -STORE, 94376869097472, 94376869101567, -STORE, 94376873291776, 94376873295871, -STORE, 94376877486080, 94376877490175, -STORE, 94376881680384, 94376881684479, -STORE, 94376885874688, 94376885878783, -STORE, 94376890068992, 94376890073087, -STORE, 94376894263296, 94376894267391, -STORE, 94376898457600, 94376898461695, -STORE, 94376902651904, 94376902655999, -STORE, 94376906846208, 94376906850303, -STORE, 94376911040512, 94376911044607, -STORE, 94376915234816, 94376915238911, -STORE, 94376919429120, 94376919433215, -STORE, 94376923623424, 94376923627519, -STORE, 94376927817728, 94376927821823, -STORE, 94376932012032, 94376932016127, -STORE, 94376936206336, 94376936210431, -STORE, 94376940400640, 94376940404735, -STORE, 94376944594944, 94376944599039, -STORE, 94376948789248, 94376948793343, -STORE, 94376952983552, 94376952987647, -STORE, 94376957177856, 94376957181951, -STORE, 94376961372160, 94376961376255, -STORE, 94376965566464, 94376965570559, -STORE, 94376969760768, 94376969764863, -STORE, 94376973955072, 94376973959167, -STORE, 94376978149376, 94376978153471, -STORE, 94376982343680, 94376982347775, -STORE, 94376986537984, 94376986542079, -STORE, 94376990732288, 94376990736383, -STORE, 94376994926592, 94376994930687, -STORE, 94376999120896, 94376999124991, -STORE, 94377003315200, 94377003319295, -STORE, 94377007509504, 94377007513599, -STORE, 94377011703808, 94377011707903, -STORE, 94377015898112, 94377015902207, -STORE, 94377020092416, 94377020096511, -STORE, 94377024286720, 94377024290815, -STORE, 94377028481024, 94377028485119, -STORE, 94377032675328, 94377032679423, -STORE, 94377036869632, 94377036873727, -STORE, 94377041063936, 94377041068031, -STORE, 94377045258240, 94377045262335, -STORE, 94377049452544, 94377049456639, -STORE, 94377053646848, 94377053650943, -STORE, 94377057841152, 94377057845247, -STORE, 94377062035456, 94377062039551, -STORE, 94377066229760, 94377066233855, -STORE, 94377070424064, 94377070428159, -STORE, 94377074618368, 94377074622463, -STORE, 94377078812672, 94377078816767, -STORE, 94377083006976, 94377083011071, -STORE, 94377087201280, 94377087205375, -STORE, 94377091395584, 94377091399679, -STORE, 94377095589888, 94377095593983, -STORE, 94377099784192, 94377099788287, -STORE, 94377103978496, 94377103982591, -STORE, 94377108172800, 94377108176895, -STORE, 94377112367104, 94377112371199, -STORE, 94377116561408, 94377116565503, -STORE, 94377120755712, 94377120759807, -STORE, 94377124950016, 94377124954111, -STORE, 94377129144320, 94377129148415, -STORE, 94377133338624, 94377133342719, -STORE, 94377137532928, 94377137537023, -STORE, 94377141727232, 94377141731327, -STORE, 94377145921536, 94377145925631, -STORE, 94377150115840, 94377150119935, -STORE, 94377154310144, 94377154314239, -STORE, 94377158504448, 94377158508543, -STORE, 94377162698752, 94377162702847, -STORE, 94377166893056, 94377166897151, -STORE, 94377171087360, 94377171091455, -STORE, 94377175281664, 94377175285759, -STORE, 94377179475968, 94377179480063, -STORE, 94377183670272, 94377183674367, -STORE, 94377187864576, 94377187868671, -STORE, 94377192058880, 94377192062975, -STORE, 94377196253184, 94377196257279, -STORE, 94377200447488, 94377200451583, -STORE, 94377204641792, 94377204645887, -SNULL, 94376135094271, 94376135098367, -STORE, 94376135090176, 94376135094271, -STORE, 94376135094272, 94376135098367, -SNULL, 94376135094272, 94377208836095, - }; - unsigned long set42[] = { -STORE, 314572800, 1388314623, -STORE, 1462157312, 1462169599, -STORE, 1462169600, 1462185983, -STORE, 1462185984, 1462190079, -STORE, 1462190080, 1462194175, -STORE, 1462194176, 1462198271, -STORE, 1879986176, 1881800703, -STORE, 1881800704, 1882034175, -STORE, 1882034176, 1882193919, -STORE, 1882193920, 1882406911, -STORE, 1882406912, 1882451967, -STORE, 1882451968, 1882996735, -STORE, 1882996736, 1885892607, -STORE, 1885892608, 1885896703, -STORE, 1885896704, 1885904895, -STORE, 1885904896, 1885908991, -STORE, 1885908992, 1885913087, -STORE, 1885913088, 1885966335, -STORE, 1885966336, 1886232575, -STORE, 1886232576, 1886236671, -STORE, 1886236672, 1886240767, -STORE, 1886240768, 1886244863, -STORE, 1886244864, 1886248959, -STORE, 1886248960, 1886294015, -STORE, 1886294016, 1886494719, -STORE, 1886494720, 1886498815, -STORE, 1886498816, 1886502911, -STORE, 1886502912, 1886507007, -STORE, 1886507008, 1886511103, -STORE, 1886511104, 1886556159, -STORE, 1886556160, 1886629887, -STORE, 1886629888, 1886633983, -STORE, 1886633984, 1886638079, -STORE, 1886638080, 1886642175, -STORE, 1886642176, 1886646271, -STORE, 1886646272, 1886666751, -STORE, 1886666752, 1886670847, -STORE, 1886670848, 1886674943, -STORE, 1886674944, 1886679039, -STORE, 1886679040, 1895419903, -STORE, 1895419904, 1895550975, -STORE, 1895550976, 1896148991, -STORE, 1896148992, 1897189375, -STORE, 1897189376, 1897701375, -STORE, 1897701376, 1897803775, -STORE, 1897803776, 1897816063, -STORE, 1897816064, 1899913215, -STORE, 1899913216, 1909379071, -STORE, 1909379072, 1909387263, -STORE, 1909387264, 1909391359, -STORE, 1909391360, 1909432319, -STORE, 1909432320, 1909436415, -STORE, 1909436416, 1909440511, -STORE, 1909440512, 1909460991, -STORE, 1909460992, 1909547007, -STORE, 1909547008, 1909551103, -STORE, 1909551104, 1909555199, -STORE, 1909555200, 1909559295, -STORE, 1909559296, 1909563391, -STORE, 1909563392, 1909739519, -STORE, 1909739520, 1910566911, -STORE, 1910566912, 1910571007, -STORE, 1910571008, 1910575103, -STORE, 1910575104, 1910579199, -STORE, 1910579200, 1910583295, -STORE, 1910583296, 1910587391, -STORE, 1910587392, 1910620159, -STORE, 1910620160, 1910624255, -STORE, 1910624256, 1910628351, -STORE, 1910628352, 1910632447, -STORE, 1910632448, 1910652927, -STORE, 1910652928, 1910657023, -STORE, 1910657024, 1910661119, -STORE, 1910661120, 1910665215, -STORE, 1910665216, 1910669311, -STORE, 1910669312, 1910677503, -STORE, 1910677504, 1910681599, -STORE, 1910681600, 1910685695, -STORE, 1910685696, 1910689791, -STORE, 1910689792, 1910697983, -STORE, 1910697984, 1910702079, -STORE, 1910702080, 1910706175, -STORE, 1910706176, 1910710271, -STORE, 1910710272, 1914093567, -STORE, 1914093568, 1914097663, -STORE, 1914097664, 1969434623, -STORE, 1969434624, 1977819135, -STORE, 3290435584, 3426750463, -STORE, 3426750464, 3426754559, -STORE, 3426754560, 3426762751, -STORE, 3426762752, 3426766847, -STORE, 3426766848, 3426770943, -STORE, 3427037184, 3427061759, -STORE, 3427061760, 3427135487, -STORE, 3427135488, 3427143679, -STORE, 3427143680, 3427147775, -STORE, 3427147776, 3427209215, -STORE, 3427319808, 3432116223, -STORE, 3432116224, 3450130431, -STORE, 3450130432, 3451027455, -STORE, 3451027456, 3451031551, -STORE, 3451031552, 3451461631, -STORE, 3451736064, 3456688127, -STORE, 3456688128, 3475222527, -STORE, 3475222528, 3476119551, -STORE, 3476119552, 3476127743, -STORE, 3476127744, 3476553727, -STORE, 3476631552, 3477315583, -STORE, 3477315584, 3479949311, -STORE, 3479949312, 3480002559, -STORE, 3480002560, 3480006655, -STORE, 3480006656, 3480432639, -STORE, 3480539136, 3480543231, -STORE, 3480543232, 3480547327, -STORE, 3480547328, 3480555519, -STORE, 3480854528, 3480903679, -STORE, 3480903680, 3480969215, -STORE, 3480969216, 3480977407, -STORE, 3480977408, 3480981503, -STORE, 3481030656, 3481092095, -STORE, 3481092096, 3481235455, -STORE, 3481235456, 3481243647, -STORE, 3481243648, 3481247743, -STORE, 3481436160, 3481444351, -STORE, 3481444352, 3481456639, -STORE, 3481456640, 3481460735, -STORE, 3481460736, 3481464831, -STORE, 3481587712, 3481645055, -STORE, 3481645056, 3481772031, -STORE, 3481772032, 3481776127, -STORE, 3481776128, 3481780223, -STORE, 3481874432, 3481935871, -STORE, 3481935872, 3482030079, -STORE, 3482030080, 3482038271, -STORE, 3482038272, 3482042367, -STORE, 3482198016, 3482230783, -STORE, 3482230784, 3482271743, -STORE, 3482271744, 3482279935, -STORE, 3482279936, 3482284031, -STORE, 3482562560, 3482566655, -STORE, 3482566656, 3482570751, -STORE, 3482570752, 3482574847, -STORE, 3482636288, 3482689535, -STORE, 3482689536, 3482746879, -STORE, 3482746880, 3482755071, -STORE, 3482755072, 3482759167, -STORE, 3482972160, 3483062271, -STORE, 3483062272, 3483242495, -STORE, 3483242496, 3483246591, -STORE, 3483246592, 3483250687, -STORE, 3483398144, 3483688959, -STORE, 3483688960, 3484114943, -STORE, 3484114944, 3484131327, -STORE, 3484131328, 3484135423, -STORE, 3484135424, 3484143615, -STORE, 3484184576, 3484475391, -STORE, 3484475392, 3485028351, -STORE, 3485028352, 3485057023, -STORE, 3485057024, 3485061119, -STORE, 3485360128, 3485364223, -STORE, 3485364224, 3485368319, -STORE, 3485368320, 3485372415, -STORE, 3485589504, 3485593599, -STORE, 3485593600, 3485597695, -STORE, 3485597696, 3485601791, -STORE, 3485913088, 3485937663, -STORE, 3485937664, 3485974527, -STORE, 3485974528, 3485982719, -STORE, 3485982720, 3485986815, -STORE, 3486052352, 3486056447, -STORE, 3486056448, 3486064639, -STORE, 3486064640, 3486068735, -STORE, 3486068736, 3486072831, -STORE, 3486294016, 3486302207, -STORE, 3486302208, 3486306303, -STORE, 3486306304, 3486310399, -STORE, 3486310400, 3486314495, -STORE, 3486670848, 3486679039, -STORE, 3486679040, 3486683135, -STORE, 3486683136, 3486687231, -STORE, 3486687232, 3486691327, -STORE, 3486863360, 3486871551, -STORE, 3486871552, 3486875647, -STORE, 3486875648, 3486879743, -STORE, 3486879744, 3486883839, -STORE, 3487584256, 3522543615, -STORE, 3522543616, 3523321855, -STORE, 3523321856, 3523342335, -STORE, 3523342336, 3523387391, -STORE, 3523387392, 3523391487, -STORE, 3523391488, 3523395583, -STORE, 3523477504, 3523686399, -STORE, 3523686400, 3523981311, -STORE, 3523981312, 3523997695, -STORE, 3523997696, 3524001791, -STORE, 3524177920, 3525013503, -STORE, 3525013504, 3526582271, -STORE, 3526582272, 3526606847, -STORE, 3526606848, 3526610943, -STORE, 3526610944, 3526615039, -STORE, 3526672384, 3526746111, -STORE, 3526746112, 3526860799, -STORE, 3526860800, 3526868991, -STORE, 3526868992, 3526873087, -STORE, 3527000064, 3527475199, -STORE, 3527475200, 3527479295, -STORE, 3527479296, 3527573503, -STORE, 3527573504, 3527581695, -STORE, 3527581696, 3527585791, -STORE, 3527585792, 3527606271, -STORE, 3527909376, 3527913471, -STORE, 3527913472, 3527917567, -STORE, 3527917568, 3527921663, -STORE, 3527950336, 3528011775, -STORE, 3528011776, 3528093695, -STORE, 3528093696, 3528101887, -STORE, 3528101888, 3528105983, -STORE, 3528228864, 3528241151, -STORE, 3528241152, 3528261631, -STORE, 3528261632, 3528265727, -STORE, 3528273920, 3528593407, -STORE, 3528593408, 3528609791, -STORE, 3528609792, 3528638463, -STORE, 3528638464, 3528642559, -STORE, 3528642560, 3528646655, -STORE, 3528880128, 3528912895, -STORE, 3528912896, 3528962047, -STORE, 3528962048, 3528966143, -STORE, 3528966144, 3528970239, -STORE, 3528982528, 3530293247, -STORE, 3530366976, 3530825727, -STORE, 3530825728, 3531317247, -STORE, 3531317248, 3541041151, -STORE, 3541041152, 3541303295, -STORE, 3541430272, 3566206975, -STORE, 3566206976, 3566993407, -STORE, 3567239168, 3587571711, -STORE, 3587571712, 3588284415, -STORE, 3588284416, 3588661247, -STORE, 3588661248, 3589066751, -STORE, 3589066752, 3589574655, -STORE, 3589574656, 3590078463, -STORE, 3590078464, 3590373375, -STORE, 3590373376, 3590668287, -STORE, 3590668288, 3590963199, -STORE, 3590963200, 3591294975, -STORE, 3591294976, 3591602175, -STORE, 3591602176, 3591933951, -STORE, 3591933952, 3592241151, -STORE, 3592241152, 3592572927, -STORE, 3592572928, 3592876031, -STORE, 3592876032, 3593211903, -STORE, 3593211904, 3593547775, -STORE, 3593547776, 3593650175, -STORE, 3593650176, 3593928703, -STORE, 3593928704, 3593936895, -STORE, 3593936896, 3593940991, -STORE, 3594006528, 3594301439, -STORE, 3594301440, 3594739711, -STORE, 3594739712, 3594756095, -STORE, 3594756096, 3594760191, -STORE, 3594760192, 3594768383, -STORE, 3594952704, 3595051007, -STORE, 3595051008, 3595223039, -STORE, 3595223040, 3595227135, -STORE, 3595227136, 3595235327, -STORE, 3595431936, 3595775999, -STORE, 3595776000, 3596701695, -STORE, 3596701696, 3596742655, -STORE, 3596742656, 3596746751, -STORE, 3596746752, 3596750847, -STORE, 3596767232, 3597070335, -STORE, 3597070336, 3597402111, -STORE, 3597402112, 3598188543, -STORE, 3598262272, 3623428095, -STORE, 3623428096, 3623432191, -STORE, 3623432192, 3623436287, -STORE, 3623436288, 3623440383, -STORE, 3623616512, 3623878655, -STORE, 3624169472, 3624300543, -STORE, 3627524096, 3628523519, -STORE, 3628523520, 3629522943, -STORE, 3696631808, 3730186239, -STORE, 3730186240, 3763740671, -STORE, 3763740672, 3764027391, -STORE, 3764027392, 3765133311, -STORE, 3765133312, 3765145599, -STORE, 3765145600, 3765149695, -STORE, 3765178368, 3766022143, -STORE, 3766022144, 3768791039, -STORE, 3768791040, 3768840191, -STORE, 3768840192, 3768844287, -STORE, 3768897536, 3768913919, -STORE, 3768913920, 3768934399, -STORE, 3768934400, 3768938495, -STORE, 3769016320, 3769147391, -STORE, 3769147392, 3769233407, -STORE, 3769233408, 3769356287, -STORE, 3769356288, 3769360383, -STORE, 3769360384, 3769368575, -STORE, 3769376768, 3794542591, -STORE, 3794542592, 3794599935, -STORE, 3794599936, 3794731007, -STORE, 3794731008, 3794735103, -STORE, 3794735104, 3794743295, -STORE, 3794849792, 3794980863, -STORE, 3794980864, 3794984959, -STORE, 3794984960, 3794989055, -STORE, 3794989056, 3794993151, -STORE, 3794993152, 3794997247, -STORE, 3795103744, 3795128319, -STORE, 3795128320, 3795165183, -STORE, 3795165184, 3795169279, -STORE, 3795169280, 3795173375, -STORE, 3795210240, 3795357695, -STORE, 3795357696, 3795365887, -STORE, 3795365888, 3795374079, -STORE, 3795374080, 3795378175, -STORE, 3795378176, 3795382271, -STORE, 3795406848, 3795738623, -STORE, 3795738624, 3795742719, -STORE, 3795742720, 3795755007, -STORE, 3795755008, 3795759103, -STORE, 3795763200, 3795894271, -STORE, 3795894272, 3796041727, -STORE, 3796041728, 3796054015, -STORE, 3796054016, 3796066303, -STORE, 3796066304, 3796070399, -STORE, 3796176896, 3796205567, -STORE, 3796205568, 3796250623, -STORE, 3796250624, 3796254719, -STORE, 3796254720, 3796258815, -STORE, 3796262912, 3796393983, -STORE, 3796393984, 3796516863, -STORE, 3796516864, 3796873215, -STORE, 3796873216, 3796885503, -STORE, 3796885504, 3796889599, -STORE, 3796963328, 3796967423, -STORE, 3796967424, 3796975615, -STORE, 3796975616, 3796979711, -STORE, 3797000192, 3797307391, -STORE, 3797307392, 3797311487, -STORE, 3797311488, 3797315583, -STORE, 3797315584, 3797323775, -STORE, 3797327872, 3797450751, -STORE, 3797450752, 3797458943, -STORE, 3797458944, 3797471231, -STORE, 3797471232, 3797475327, -STORE, 3797577728, 3797700607, -STORE, 3797700608, 3797721087, -STORE, 3797721088, 3797733375, -STORE, 3797733376, 3797741567, -STORE, 3797741568, 3797864447, -STORE, 3797864448, 3797995519, -STORE, 3797995520, 3798048767, -STORE, 3798048768, 3798179839, -STORE, 3798179840, 3798188031, -STORE, 3798188032, 3798192127, -STORE, 3798290432, 3798302719, -STORE, 3798302720, 3798323199, -STORE, 3798323200, 3798327295, -STORE, 3798327296, 3798331391, -STORE, 3798429696, 3798433791, -STORE, 3798433792, 3798552575, -STORE, 3798552576, 3798556671, -STORE, 3798556672, 3798568959, -STORE, 3798568960, 3798573055, -STORE, 3798573056, 3798581247, -STORE, 3798618112, 3798749183, -STORE, 3798749184, 3798855679, -STORE, 3798855680, 3798966271, -STORE, 3798966272, 3798982655, -STORE, 3798982656, 3798986751, -STORE, 3799101440, 3799171071, -STORE, 3799171072, 3799240703, -STORE, 3799240704, 3799248895, -STORE, 3799248896, 3799252991, -STORE, 3799326720, 3799650303, -STORE, 3799650304, 3800629247, -STORE, 3800629248, 3800641535, -STORE, 3800641536, 3800645631, -STORE, 3800645632, 3800649727, -STORE, 3800649728, 3800903679, -STORE, 3800903680, 3800936447, -STORE, 3800936448, 3800969215, -STORE, 3800969216, 3800981503, -STORE, 3800981504, 3800985599, -STORE, 3801001984, 3801133055, -STORE, 3801133056, 3801202687, -STORE, 3801202688, 3801591807, -STORE, 3801591808, 3801599999, -STORE, 3801600000, 3801604095, -STORE, 3801604096, 3801608191, -STORE, 3801608192, 3801739263, -STORE, 3801739264, 3801755647, -STORE, 3801755648, 3801796607, -STORE, 3801796608, 3801804799, -STORE, 3801804800, 3801808895, -STORE, 3801878528, 3801944063, -STORE, 3801944064, 3802116095, -STORE, 3802116096, 3802124287, -STORE, 3802124288, 3802128383, -STORE, 3802136576, 3803447295, -STORE, 3803492352, 3803553791, -STORE, 3803553792, 3804233727, -STORE, 3804233728, 3806068735, -STORE, 3806121984, 3806253055, -STORE, 3806253056, 3806674943, -STORE, 3806674944, 3807117311, -STORE, 3807117312, 3807379455, -STORE, 3807379456, 3807432703, -STORE, 3807432704, 3807563775, -STORE, 3807563776, 3809202175, -STORE, 3809202176, 3810250751, -STORE, 3810250752, 3827027967, -STORE, 3827027968, 3829125119, -STORE, 3829125120, 3837513727, -STORE, 3837513728, 3839610879, -STORE, 3839610880, 3847999487, -STORE, 3847999488, 3856392191, -STORE, 3856392192, 3864784895, -STORE, 3864784896, 3868983295, -STORE, 3868983296, 3885760511, -STORE, 3885760512, 3886809087, -STORE, 3886809088, 3887857663, -STORE, 3887857664, 3888119807, -STORE, 3888144384, 3888148479, -STORE, 3888148480, 3888218111, -STORE, 3888218112, 3888222207, -STORE, 3888222208, 3888353279, -STORE, 3888353280, 3889172479, -STORE, 3889172480, 3892314111, -STORE, 3892314112, 3892576255, -STORE, 3892588544, 3892637695, -STORE, 3892637696, 3892686847, -STORE, 3892686848, 3892744191, -STORE, 3892748288, 3892785151, -STORE, 3892785152, 3895459839, -STORE, 3895459840, 3895721983, -STORE, 3895738368, 3895885823, -STORE, 3895885824, 3897081855, -STORE, 3897081856, 3906482175, -STORE, 3906482176, 3916144639, -STORE, 3916144640, 3925766143, -STORE, 3925766144, 3926974463, -STORE, 3926974464, 3928367103, -STORE, 3928367104, 3928911871, -STORE, 3928911872, 3933995007, -STORE, 3933995008, 3935830015, -STORE, 3935830016, 3935846399, -STORE, 3935879168, 3936010239, -STORE, 3936010240, 3936026623, -STORE, 3936026624, 3936034815, -STORE, 3936034816, 3936051199, -STORE, 3936051200, 3936055295, -STORE, 3936071680, 3936137215, -STORE, 3936137216, 3936202751, -STORE, 3936202752, 3936219135, -STORE, 3936235520, 3936251903, -STORE, 3936268288, 3936276479, -STORE, 3936276480, 3936284671, -STORE, 3936284672, 3936288767, -STORE, 3936288768, 3936292863, -STORE, 3936296960, 3936354303, -STORE, 3936354304, 3936616447, -STORE, 3936628736, 3936669695, -STORE, 3936669696, 3936747519, -STORE, 3936747520, 3936870399, -STORE, 3936870400, 3936874495, -STORE, 3936874496, 3936878591, -STORE, 3936882688, 3936903167, -STORE, 3936911360, 3936948223, -STORE, 3936948224, 3936964607, -STORE, 3936964608, 3937103871, -STORE, 3937103872, 3937107967, -STORE, 3937132544, 3937161215, -STORE, 3937189888, 3937255423, -STORE, 3937255424, 3938512895, -STORE, 3938512896, 3945435135, -STORE, 3945435136, 3945476095, -STORE, 3945476096, 3945484287, -STORE, 3945484288, 3945496575, -STORE, 3945500672, 3945541631, -STORE, 3945558016, 3945566207, -STORE, 3945566208, 3945594879, -STORE, 3945594880, 3945598975, -STORE, 3945598976, 3945603071, -STORE, 3945611264, 3945742335, -STORE, 3945742336, 3945844735, -STORE, 3945844736, 3945848831, -STORE, 3945848832, 3945861119, -STORE, 3945861120, 3945865215, -STORE, 3945869312, 3945897983, -STORE, 3945897984, 3946303487, -STORE, 3946303488, 3946397695, -STORE, 3946397696, 3946569727, -STORE, 3946569728, 3946573823, -STORE, 3946573824, 3946594303, -STORE, 3946594304, 3946663935, -STORE, 3946663936, 3946708991, -STORE, 3946708992, 3946823679, -STORE, 3946823680, 3946827775, -STORE, 3946827776, 3946831871, -STORE, 3946831872, 3946860543, -STORE, 3946893312, 3946897407, -STORE, 3946897408, 3946905599, -STORE, 3946905600, 3946909695, -STORE, 3946909696, 3946913791, -STORE, 3946913792, 3946930175, -STORE, 3946930176, 3946967039, -STORE, 3946967040, 3947102207, -STORE, 3947102208, 3948412927, -STORE, 3948441600, 3948556287, -STORE, 3948556288, 3948576767, -STORE, 3948576768, 3948597247, -STORE, 3948597248, 3948605439, -STORE, 3948605440, 3948609535, -STORE, 3948609536, 3948654591, -STORE, 3948654592, 3948781567, -STORE, 3948781568, 3948822527, -STORE, 3948822528, 3948904447, -STORE, 3948904448, 3948908543, -STORE, 3948908544, 3948912639, -STORE, 3948945408, 3949043711, -STORE, 3949043712, 3949174783, -STORE, 3949174784, 3949191167, -STORE, 3949191168, 3949195263, -STORE, 3949207552, 3949252607, -STORE, 3949252608, 3949256703, -STORE, 3949256704, 3949363199, -STORE, 3949363200, 3949367295, -STORE, 3949367296, 3949379583, -STORE, 3949379584, 3949383679, -STORE, 3949383680, 3949400063, -STORE, 3949400064, 3949404159, -STORE, 3949416448, 3949481983, -STORE, 3949481984, 3949486079, -STORE, 3949486080, 3949592575, -STORE, 3949592576, 3949596671, -STORE, 3949596672, 3949621247, -STORE, 3949621248, 3949662207, -STORE, 3949662208, 3949666303, -STORE, 3949694976, 3949727743, -STORE, 3949727744, 3949731839, -STORE, 3949731840, 3949838335, -STORE, 3949838336, 3949842431, -STORE, 3949842432, 3949846527, -STORE, 3949846528, 3949854719, -STORE, 3949854720, 3949858815, -STORE, 3949858816, 3949862911, -STORE, 3949867008, 3949891583, -STORE, 3949891584, 3949928447, -STORE, 3949928448, 3949993983, -STORE, 3949993984, 3950043135, -STORE, 3950043136, 3950059519, -STORE, 3950059520, 3950096383, -STORE, 3950096384, 3950100479, -STORE, 3950100480, 3950104575, -STORE, 3950104576, 3950157823, -STORE, 3950157824, 3950292991, -STORE, 3950292992, 3950346239, -STORE, 3950346240, 3950477311, -STORE, 3950477312, 3950485503, -STORE, 3950485504, 3950489599, -STORE, 3950493696, 3950510079, -STORE, 3950510080, 3950661631, -STORE, 3950661632, 3951005695, -STORE, 3951005696, 3951026175, -STORE, 3951026176, 3951030271, -STORE, 3951030272, 3951054847, -STORE, 3951054848, 3951116287, -STORE, 3951116288, 3951144959, -STORE, 3951144960, 3951149055, -STORE, 3951149056, 3951194111, -STORE, 3951194112, 3951202303, -STORE, 3951202304, 3951206399, -STORE, 3951210496, 3951226879, -STORE, 3951226880, 3951329279, -STORE, 3951329280, 3951366143, -STORE, 3951366144, 3951411199, -STORE, 3951411200, 3951415295, -STORE, 3951415296, 3951419391, -STORE, 3951419392, 3951452159, -STORE, 3951452160, 3951566847, -STORE, 3951566848, 3951812607, -STORE, 3951812608, 3952173055, -STORE, 3952173056, 3952214015, -STORE, 3952214016, 3952218111, -STORE, 3952222208, 3952250879, -STORE, 3952250880, 3952369663, -STORE, 3952369664, 3952488447, -STORE, 3952488448, 3952627711, -STORE, 3952627712, 3952635903, -STORE, 3952635904, 3952639999, -STORE, 3952652288, 3952668671, -STORE, 3952668672, 3953000447, -STORE, 3953000448, 3953004543, -STORE, 3953004544, 3953008639, -STORE, 3953008640, 3953012735, -STORE, 3953012736, 3953037311, -STORE, 3953037312, 3953151999, -STORE, 3953152000, 3953291263, -STORE, 3953291264, 3953324031, -STORE, 3953324032, 3953364991, -STORE, 3953364992, 3953373183, -STORE, 3953373184, 3953377279, -STORE, 3953381376, 3953410047, -STORE, 3953410048, 3953491967, -STORE, 3953491968, 3953643519, -STORE, 3953643520, 3953651711, -STORE, 3953651712, 3953655807, -STORE, 3953659904, 3953766399, -STORE, 3953766400, 3953774591, -STORE, 3953774592, 3953786879, -STORE, 3953786880, 3953790975, -STORE, 3953790976, 3953823743, -STORE, 3953823744, 3953963007, -STORE, 3953963008, 3954024447, -STORE, 3954024448, 3954118655, -STORE, 3954118656, 3954122751, -STORE, 3954122752, 3954126847, -STORE, 3954130944, 3954184191, -STORE, 3954184192, 3954294783, -STORE, 3954294784, 3954323455, -STORE, 3954323456, 3954393087, -STORE, 3954393088, 3954397183, -STORE, 3954397184, 3954401279, -STORE, 3954401280, 3954405375, -STORE, 3954409472, 3954528255, -STORE, 3954528256, 3954737151, -STORE, 3954737152, 3955052543, -STORE, 3955052544, 3955060735, -STORE, 3955060736, 3955064831, -STORE, 3955068928, 3955105791, -STORE, 3955105792, 3955167231, -STORE, 3955167232, 3955277823, -STORE, 3955277824, 3955310591, -STORE, 3955310592, 3955351551, -STORE, 3955351552, 3955359743, -STORE, 3955359744, 3955363839, -STORE, 3955363840, 3955392511, -STORE, 3955392512, 3955453951, -STORE, 3955453952, 3955601407, -STORE, 3955601408, 3955777535, -STORE, 3955777536, 3955982335, -STORE, 3955982336, 3956011007, -STORE, 3956011008, 3956015103, -STORE, 3956023296, 3956039679, -STORE, 3956039680, 3956125695, -STORE, 3956125696, 3956129791, -STORE, 3956129792, 3956133887, -STORE, 3956133888, 3956137983, -STORE, 3956142080, 3956449279, -STORE, 3956449280, 3956543487, -STORE, 3956543488, 3956719615, -STORE, 3956719616, 3956731903, -STORE, 3956731904, 3956735999, -STORE, 3956744192, 3956793343, -STORE, 3956793344, 3956887551, -STORE, 3956887552, 3956953087, -STORE, 3956953088, 3957035007, -STORE, 3957035008, 3957039103, -STORE, 3957039104, 3957047295, -STORE, 3957047296, 3957071871, -STORE, 3957071872, 3957231615, -STORE, 3957231616, 3957563391, -STORE, 3957563392, 3957579775, -STORE, 3957579776, 3957583871, -STORE, 3957592064, 3957608447, -STORE, 3957608448, 3957878783, -STORE, 3957878784, 3958591487, -STORE, 3958591488, 3958599679, -STORE, 3958599680, 3958607871, -STORE, 3958607872, 3958620159, -STORE, 3958620160, 3958624255, -STORE, 3958624256, 3963199487, -STORE, 3963199488, 3963285503, -STORE, 3963285504, 3963371519, -STORE, 3963371520, 3963428863, -STORE, 3963428864, 3963555839, -STORE, 3963555840, 3963559935, -STORE, 3963559936, 3963564031, -STORE, 3963568128, 3963596799, -STORE, 3963596800, 3963682815, -STORE, 3963682816, 3963695103, -STORE, 3963695104, 3963711487, -STORE, 3963711488, 3963715583, -STORE, 3963719680, 3963752447, -STORE, 3963752448, 3963846655, -STORE, 3963846656, 3963932671, -STORE, 3963932672, 3964444671, -STORE, 3964444672, 3964448767, -STORE, 3964448768, 3965808639, -STORE, 3965808640, 3965845503, -STORE, 3965845504, 3965849599, -STORE, 3965853696, 3965935615, -STORE, 3965935616, 3966017535, -STORE, 3966017536, 3966103551, -STORE, 3966103552, 3966685183, -STORE, 3966685184, 3967705087, -STORE, 3967705088, 3967758335, -STORE, 3967758336, 3967762431, -STORE, 3967762432, 3967770623, -STORE, 3967770624, 3967799295, -STORE, 3967799296, 3967848447, -STORE, 3967848448, 3967868927, -STORE, 3967868928, 3967901695, -STORE, 3967901696, 3967905791, -STORE, 3967905792, 3967909887, -STORE, 3967909888, 3967995903, -STORE, 3967995904, 3968077823, -STORE, 3968077824, 3968159743, -STORE, 3968159744, 3968167935, -STORE, 3968167936, 3968172031, -STORE, 3968172032, 3968192511, -STORE, 3968192512, 3968196607, -STORE, 3968196608, 3968200703, -STORE, 3968208896, 3968516095, -STORE, 3968516096, 3968528383, -STORE, 3968528384, 3968552959, -STORE, 3968552960, 3968557055, -STORE, 3968561152, 3968593919, -STORE, 3968593920, 3968626687, -STORE, 3968626688, 3971153919, -STORE, 3971153920, 3973754879, -STORE, 3973754880, 3973804031, -STORE, 3973804032, 3973820415, -STORE, 3973820416, 3973832703, -STORE, 3973840896, 3973873663, -STORE, 3973873664, 3973967871, -STORE, 3973967872, 3973976063, -STORE, 3973976064, 3973984255, -STORE, 3973984256, 3973988351, -STORE, 3973988352, 3973992447, -STORE, 3973996544, 3974008831, -STORE, 3974008832, 3974045695, -STORE, 3974045696, 3974139903, -STORE, 3974139904, 3974254591, -STORE, 3974254592, 3974275071, -STORE, 3974275072, 3974291455, -STORE, 3974291456, 3974295551, -STORE, 3974295552, 3974373375, -STORE, 3974373376, 3974524927, -STORE, 3974524928, 3974529023, -STORE, 3974529024, 3974537215, -STORE, 3974537216, 3974541311, -STORE, 3974541312, 3974545407, -STORE, 3974545408, 3974627327, -STORE, 3974627328, 3974680575, -STORE, 3974680576, 3974811647, -STORE, 3974811648, 3974819839, -STORE, 3974819840, 3974823935, -STORE, 3974832128, 3974918143, -STORE, 3974918144, 3974963199, -STORE, 3974963200, 3975077887, -STORE, 3975077888, 3975090175, -STORE, 3975090176, 3975094271, -STORE, 3975094272, 3975102463, -STORE, 3975102464, 3975114751, -STORE, 3975114752, 3975266303, -STORE, 3975266304, 3975274495, -STORE, 3975274496, 3975286783, -STORE, 3975286784, 3975290879, -STORE, 3975290880, 3975299071, -STORE, 3975299072, 3975315455, -STORE, 3975315456, 3975430143, -STORE, 3975430144, 3975536639, -STORE, 3975536640, 3975651327, -STORE, 3975651328, 3975655423, -STORE, 3975655424, 3975659519, -STORE, 3975659520, 3975770111, -STORE, 3975770112, 3975778303, -STORE, 3975778304, 3975790591, -STORE, 3975790592, 3975794687, -STORE, 3975794688, 3975798783, -STORE, 3975798784, 3975831551, -STORE, 3975831552, 3975872511, -STORE, 3975872512, 3975987199, -STORE, 3975987200, 3976134655, -STORE, 3976134656, 3977175039, -STORE, 3977175040, 3977183231, -STORE, 3977183232, 3977191423, -STORE, 3977191424, 3977195519, -STORE, 3977199616, 3977248767, -STORE, 3977248768, 3977539583, -STORE, 3977539584, 3977965567, -STORE, 3977965568, 3977981951, -STORE, 3977981952, 3977986047, -STORE, 3977986048, 3977994239, -STORE, 3977994240, 3978002431, -STORE, 3978002432, 3978084351, -STORE, 3978084352, 3978125311, -STORE, 3978125312, 3978174463, -STORE, 3978174464, 3978178559, -STORE, 3978178560, 3978182655, -STORE, 3978182656, 3978207231, -STORE, 3978207232, 3978297343, -STORE, 3978297344, 3978301439, -STORE, 3978301440, 3978305535, -STORE, 3978305536, 3978309631, -STORE, 3978309632, 3978317823, -STORE, 3978317824, 3978625023, -STORE, 3978625024, 3978657791, -STORE, 3978657792, 3978727423, -STORE, 3978727424, 3978735615, -STORE, 3978735616, 3978739711, -STORE, 3978739712, 3978760191, -STORE, 3978760192, 3978842111, -STORE, 3978842112, 3978850303, -STORE, 3978850304, 3978858495, -STORE, 3978858496, 3978862591, -STORE, 3978862592, 3978895359, -STORE, 3978895360, 3979014143, -STORE, 3979014144, 3979132927, -STORE, 3979132928, 3979288575, -STORE, 3979288576, 3979481087, -STORE, 3979481088, 3979489279, -STORE, 3979489280, 3979493375, -STORE, 3979497472, 3979583487, -STORE, 3979583488, 3979673599, -STORE, 3979673600, 3979718655, -STORE, 3979718656, 3979829247, -STORE, 3979829248, 3979841535, -STORE, 3979841536, 3979882495, -STORE, 3979882496, 3979964415, -STORE, 3979964416, 3980013567, -STORE, 3980013568, 3980148735, -STORE, 3980148736, 3980152831, -STORE, 3980152832, 3980320767, -STORE, 3980320768, 3980337151, -STORE, 3980337152, 3980341247, -STORE, 3980345344, 3980365823, -STORE, 3980365824, 3980423167, -STORE, 3980423168, 3980460031, -STORE, 3980460032, 3980500991, -STORE, 3980500992, 3980509183, -STORE, 3980509184, 3980513279, -STORE, 3980513280, 3980546047, -STORE, 3980546048, 3980660735, -STORE, 3980660736, 3980951551, -STORE, 3980951552, 3981500415, -STORE, 3981500416, 3981529087, -STORE, 3981529088, 3981533183, -STORE, 3981537280, 3981549567, -STORE, 3981549568, 3981598719, -STORE, 3981598720, 3981717503, -STORE, 3981717504, 3982127103, -STORE, 3982127104, 3982675967, -STORE, 3982675968, 3982733311, -STORE, 3982733312, 3982737407, -STORE, 3982741504, 3982860287, -STORE, 3982860288, 3982905343, -STORE, 3982905344, 3982966783, -STORE, 3982966784, 3982974975, -STORE, 3982974976, 3982979071, -STORE, 3982979072, 3983032319, -STORE, 3983032320, 3983085567, -STORE, 3983085568, 3983208447, -STORE, 3983208448, 3983212543, -STORE, 3983212544, 3983220735, -STORE, 3983220736, 3983224831, -STORE, 3983224832, 3983237119, -STORE, 3983237120, 3983351807, -STORE, 3983351808, 3983376383, -STORE, 3983376384, 3983392767, -STORE, 3983392768, 3983396863, -STORE, 3983396864, 3983400959, -STORE, 3983400960, 3983417343, -STORE, 3983417344, 3983753215, -STORE, 3983753216, 3983757311, -STORE, 3983757312, 3983761407, -STORE, 3983761408, 3983765503, -STORE, 3983765504, 3983769599, -STORE, 3983769600, 3983880191, -STORE, 3983880192, 3983892479, -STORE, 3983892480, 3983900671, -STORE, 3983900672, 3983904767, -STORE, 3983904768, 3983908863, -STORE, 3983908864, 3983941631, -STORE, 3983941632, 3983990783, -STORE, 3983990784, 3984097279, -STORE, 3984097280, 3984105471, -STORE, 3984105472, 3984117759, -STORE, 3984117760, 3984121855, -STORE, 3984121856, 3984125951, -STORE, 3984125952, 3984134143, -STORE, 3984134144, 3984150527, -STORE, 3984150528, 3984416767, -STORE, 3984416768, 3984470015, -STORE, 3984470016, 3984564223, -STORE, 3984564224, 3984568319, -STORE, 3984572416, 3984629759, -STORE, 3984629760, 3984805887, -STORE, 3984805888, 3985096703, -STORE, 3985096704, 3985104895, -STORE, 3985104896, 3985108991, -STORE, 3985113088, 3986862079, -STORE, 3986862080, 3993640959, -STORE, 3993640960, 3993739263, -STORE, 3993739264, 3993743359, -STORE, 3993743360, 3993759743, -STORE, 3993759744, 3993780223, -STORE, 3993780224, 3993784319, -STORE, 3993784320, 3993792511, -STORE, 3993792512, 3993796607, -STORE, 3993796608, 3993800703, -STORE, 3993804800, 3994214399, -STORE, 3994214400, 3994218495, -STORE, 3994218496, 3994222591, -STORE, 3994222592, 3994226687, -STORE, 3994230784, 3994243071, -STORE, 3994243072, 3994255359, -STORE, 3994255360, 3994304511, -STORE, 3994304512, 3994386431, -STORE, 3994386432, 3994509311, -STORE, 3994509312, 3994521599, -STORE, 3994521600, 3994525695, -STORE, 3994529792, 3994542079, -STORE, 3994542080, 3994660863, -STORE, 3994660864, 3994705919, -STORE, 3994705920, 3994796031, -STORE, 3994796032, 3994800127, -STORE, 3994800128, 3994804223, -STORE, 3994804224, 3994812415, -STORE, 3994812416, 3994845183, -STORE, 3994845184, 3994898431, -STORE, 3994898432, 3994902527, -STORE, 3994902528, 3994906623, -STORE, 3994910720, 3994931199, -STORE, 3994931200, 3995181055, -STORE, 3995181056, 3995222015, -STORE, 3995222016, 3995275263, -STORE, 3995275264, 3995279359, -STORE, 3995279360, 3995283455, -STORE, 3995283456, 3995291647, -STORE, 3995291648, 3995324415, -STORE, 3995324416, 3995451391, -STORE, 3995451392, 3995697151, -STORE, 3995697152, 3996078079, -STORE, 3996078080, 3996086271, -STORE, 3996086272, 3996090367, -STORE, 3996094464, 3996119039, -STORE, 3996119040, 3996200959, -STORE, 3996200960, 3996229631, -STORE, 3996229632, 3996233727, -STORE, 3996233728, 3996282879, -STORE, 3996282880, 3996291071, -STORE, 3996291072, 3996295167, -STORE, 3996299264, 3996311551, -STORE, 3996311552, 3996430335, -STORE, 3996430336, 3996467199, -STORE, 3996467200, 3996504063, -STORE, 3996504064, 3996512255, -STORE, 3996512256, 3996516351, -STORE, 3996516352, 3996540927, -STORE, 3996540928, 3996671999, -STORE, 3996672000, 3996676095, -STORE, 3996676096, 3996684287, -STORE, 3996684288, 3996688383, -STORE, 3996688384, 3996692479, -STORE, 3996692480, 3996717055, -STORE, 3996717056, 3997048831, -STORE, 3997048832, 3997057023, -STORE, 3997057024, 3997073407, -STORE, 3997073408, 3997077503, -STORE, 3997077504, 3997081599, -STORE, 3997081600, 3997097983, -STORE, 3997097984, 3997179903, -STORE, 3997179904, 3997356031, -STORE, 3997356032, 3997650943, -STORE, 3997650944, 3997675519, -STORE, 3997675520, 3997679615, -STORE, 3997683712, 3997700095, -STORE, 3997700096, 3997745151, -STORE, 3997745152, 3997802495, -STORE, 3997802496, 3997810687, -STORE, 3997810688, 3997814783, -STORE, 3997814784, 3998064639, -STORE, 3998064640, 3998081023, -STORE, 3998081024, 3998085119, -STORE, 3998085120, 3998130175, -STORE, 3998130176, 3998134271, -STORE, 3998134272, 3998142463, -STORE, 3998142464, 3998179327, -STORE, 3998179328, 3998212095, -STORE, 3998212096, 3998326783, -STORE, 3998326784, 3998351359, -STORE, 3998351360, 3998392319, -STORE, 3998392320, 3998396415, -STORE, 3998396416, 3998400511, -STORE, 3998400512, 3998433279, -STORE, 3998433280, 3998466047, -STORE, 3998466048, 3998613503, -STORE, 3998613504, 3998666751, -STORE, 3998666752, 3998724095, -STORE, 3998724096, 3998732287, -STORE, 3998732288, 3998736383, -STORE, 3998736384, 3998760959, -STORE, 3998760960, 3998777343, -STORE, 3998777344, 3998822399, -STORE, 3998822400, 3998826495, -STORE, 3998826496, 3998830591, -STORE, 3998830592, 3998863359, -STORE, 3998863360, 3998900223, -STORE, 3998900224, 3999043583, -STORE, 3999043584, 3999121407, -STORE, 3999121408, 3999215615, -STORE, 3999215616, 3999223807, -STORE, 3999223808, 3999227903, -STORE, 3999227904, 3999236095, -STORE, 3999236096, 3999268863, -STORE, 3999268864, 3999301631, -STORE, 3999301632, 3999354879, -STORE, 3999354880, 3999428607, -STORE, 3999428608, 3999436799, -STORE, 3999436800, 3999440895, -STORE, 3999444992, 3999461375, -STORE, 3999461376, 3999584255, -STORE, 3999584256, 3999760383, -STORE, 3999760384, 4000219135, -STORE, 4000219136, 4000235519, -STORE, 4000235520, 4000251903, -STORE, 4000251904, 4000501759, -STORE, 4000501760, 4000505855, -STORE, 4000505856, 4000509951, -STORE, 4000509952, 4000518143, -STORE, 4000518144, 4000522239, -STORE, 4000522240, 4000587775, -STORE, 4000587776, 4000645119, -STORE, 4000645120, 4000813055, -STORE, 4000813056, 4000817151, -STORE, 4000821248, 4000837631, -STORE, 4000837632, 4000870399, -STORE, 4000870400, 4000874495, -STORE, 4000874496, 4000878591, -STORE, 4000878592, 4000882687, -STORE, 4000882688, 4000886783, -STORE, 4000886784, 4000890879, -STORE, 4000890880, 4000907263, -STORE, 4000907264, 4001214463, -STORE, 4001214464, 4001558527, -STORE, 4001558528, 4002484223, -STORE, 4002484224, 4002525183, -STORE, 4002525184, 4002529279, -STORE, 4002529280, 4002533375, -STORE, 4002533376, 4002537471, -STORE, 4002537472, 4002660351, -STORE, 4002660352, 4002779135, -STORE, 4002779136, 4002791423, -STORE, 4002791424, 4002799615, -STORE, 4002799616, 4002807807, -STORE, 4002807808, 4002811903, -STORE, 4002811904, 4002828287, -STORE, 4002828288, 4002910207, -STORE, 4002910208, 4003028991, -STORE, 4003028992, 4003037183, -STORE, 4003037184, 4003045375, -STORE, 4003045376, 4003049471, -STORE, 4003049472, 4003053567, -STORE, 4003053568, 4003057663, -STORE, 4003057664, 4003065855, -STORE, 4003065856, 4003135487, -STORE, 4003135488, 4003446783, -STORE, 4003446784, 4003450879, -STORE, 4003450880, 4003454975, -STORE, 4003454976, 4003459071, -STORE, 4003459072, 4003463167, -STORE, 4003463168, 4003495935, -STORE, 4003495936, 4003569663, -STORE, 4003569664, 4003573759, -STORE, 4003573760, 4003704831, -STORE, 4003704832, 4003708927, -STORE, 4003708928, 4003713023, -STORE, 4003713024, 4003737599, -STORE, 4003737600, 4003770367, -STORE, 4003770368, 4003876863, -STORE, 4003876864, 4003880959, -STORE, 4003880960, 4003885055, -STORE, 4003885056, 4003889151, -STORE, 4003889152, 4003893247, -STORE, 4003893248, 4003897343, -STORE, 4003897344, 4003962879, -STORE, 4003962880, 4004069375, -STORE, 4004069376, 4004093951, -STORE, 4004093952, 4004118527, -STORE, 4004118528, 4004122623, -STORE, 4004122624, 4004126719, -STORE, 4004126720, 4004155391, -STORE, 4004155392, 4004286463, -STORE, 4004286464, 4004384767, -STORE, 4004384768, 4004388863, -STORE, 4004388864, 4004646911, -STORE, 4004646912, 4004655103, -STORE, 4004655104, 4004659199, -STORE, 4004659200, 4004667391, -STORE, 4004667392, 4004683775, -STORE, 4004683776, 4004814847, -STORE, 4004814848, 4004818943, -STORE, 4004818944, 4004823039, -STORE, 4004823040, 4004827135, -STORE, 4004827136, 4004835327, -STORE, 4004835328, 4004954111, -STORE, 4004954112, 4005085183, -STORE, 4005085184, 4005306367, -STORE, 4005306368, 4005765119, -STORE, 4005765120, 4005789695, -STORE, 4005789696, 4005793791, -STORE, 4005793792, 4005801983, -STORE, 4005801984, 4005920767, -STORE, 4005920768, 4005945343, -STORE, 4005945344, 4005949439, -STORE, 4005949440, 4005986303, -STORE, 4005986304, 4005990399, -STORE, 4005990400, 4005994495, -STORE, 4005994496, 4006002687, -STORE, 4006002688, 4006109183, -STORE, 4006109184, 4006117375, -STORE, 4006117376, 4006121471, -STORE, 4006121472, 4006133759, -STORE, 4006133760, 4006137855, -STORE, 4006137856, 4006141951, -STORE, 4006141952, 4006150143, -STORE, 4006150144, 4006391807, -STORE, 4006391808, 4006445055, -STORE, 4006445056, 4006563839, -STORE, 4006563840, 4006572031, -STORE, 4006572032, 4006576127, -STORE, 4006576128, 4006584319, -STORE, 4006584320, 4006694911, -STORE, 4006694912, 4006739967, -STORE, 4006739968, 4006776831, -STORE, 4006776832, 4006785023, -STORE, 4006785024, 4006789119, -STORE, 4006789120, 4006797311, -STORE, 4006797312, 4006813695, -STORE, 4006813696, 4006846463, -STORE, 4006846464, 4006977535, -STORE, 4006977536, 4007006207, -STORE, 4007006208, 4007010303, -STORE, 4007010304, 4007067647, -STORE, 4007067648, 4007075839, -STORE, 4007075840, 4007084031, -STORE, 4007084032, 4007100415, -STORE, 4007100416, 4007116799, -STORE, 4007116800, 4007133183, -STORE, 4007133184, 4007153663, -STORE, 4007153664, 4007178239, -STORE, 4007178240, 4007202815, -STORE, 4007202816, 4007206911, -STORE, 4007206912, 4007272447, -STORE, 4007272448, 4007276543, -STORE, 4007276544, 4007280639, -STORE, 4007280640, 4007284735, -STORE, 4007284736, 4007292927, -STORE, 4007292928, 4007423999, -STORE, 4007424000, 4007448575, -STORE, 4007448576, 4007452671, -STORE, 4007452672, 4007505919, -STORE, 4007505920, 4007510015, -STORE, 4007510016, 4007514111, -STORE, 4007514112, 4007645183, -STORE, 4007645184, 4007776255, -STORE, 4007776256, 4007780351, -STORE, 4007780352, 4007784447, -STORE, 4007784448, 4007788543, -STORE, 4007788544, 4007809023, -STORE, 4007809024, 4007829503, -STORE, 4007829504, 4007960575, -STORE, 4007960576, 4008091647, -STORE, 4008091648, 4008296447, -STORE, 4008296448, 4008890367, -STORE, 4008890368, 4008898559, -STORE, 4008898560, 4008902655, -STORE, 4008902656, 4008996863, -STORE, 4008996864, 4009041919, -STORE, 4009041920, 4009082879, -STORE, 4009082880, 4009091071, -STORE, 4009091072, 4009107455, -STORE, 4009107456, 4009349119, -STORE, 4009349120, 4009373695, -STORE, 4009373696, 4009414655, -STORE, 4009414656, 4009422847, -STORE, 4009422848, 4009426943, -STORE, 4009426944, 4009447423, -STORE, 4009447424, 4009471999, -STORE, 4009472000, 4009512959, -STORE, 4009512960, 4009594879, -STORE, 4009594880, 4009598975, -STORE, 4009598976, 4009697279, -STORE, 4009697280, 4009713663, -STORE, 4009713664, 4009717759, -STORE, 4009717760, 4009721855, -STORE, 4009721856, 4009730047, -STORE, 4009730048, 4009861119, -STORE, 4009861120, 4009951231, -STORE, 4009951232, 4010131455, -STORE, 4010131456, 4010135551, -STORE, 4010135552, 4010139647, -STORE, 4010139648, 4010143743, -STORE, 4010143744, 4010164223, -STORE, 4010164224, 4010295295, -STORE, 4010295296, 4010299391, -STORE, 4010299392, 4010491903, -STORE, 4010491904, 4010495999, -STORE, 4010496000, 4010668031, -STORE, 4010668032, 4011028479, -STORE, 4011028480, 4011053055, -STORE, 4011053056, 4011057151, -STORE, 4011057152, 4011118591, -STORE, 4011118592, 4011126783, -STORE, 4011126784, 4011130879, -STORE, 4011130880, 4011143167, -STORE, 4011143168, 4011147263, -STORE, 4011147264, 4011167743, -STORE, 4011167744, 4011171839, -STORE, 4011171840, 4011360255, -STORE, 4011360256, 4011364351, -STORE, 4011364352, 4011626495, -STORE, 4011626496, 4012216319, -STORE, 4012216320, 4012228607, -STORE, 4012228608, 4012232703, -STORE, 4012232704, 4012236799, -STORE, 4012236800, 4012240895, -STORE, 4012240896, 4012261375, -STORE, 4012261376, 4012392447, -STORE, 4012392448, 4012466175, -STORE, 4012466176, 4012597247, -STORE, 4012597248, 4012601343, -STORE, 4012601344, 4012605439, -STORE, 4012605440, 4012609535, -STORE, 4012609536, 4012679167, -STORE, 4012679168, 4013563903, -STORE, 4013563904, 4015366143, -STORE, 4015366144, 4015411199, -STORE, 4015411200, 4015415295, -STORE, 4015415296, 4015419391, -STORE, 4015419392, 4015542271, -STORE, 4015542272, 4015550463, -STORE, 4015550464, 4015558655, -STORE, 4015558656, 4015562751, -STORE, 4015562752, 4015583231, -STORE, 4015583232, 4015587327, -STORE, 4015587328, 4015603711, -STORE, 4015665152, 4015669247, -STORE, 4015669248, 4015812607, -STORE, 4015812608, 4015816703, -STORE, 4015816704, 4016111615, -STORE, 4016111616, 4016467967, -STORE, 4016467968, 4016508927, -STORE, 4016508928, 4016517119, -STORE, 4016517120, 4016525311, -STORE, 4016525312, 4016586751, -STORE, 4016586752, 4016664575, -STORE, 4016664576, 4016697343, -STORE, 4016697344, 4016742399, -STORE, 4016742400, 4016746495, -STORE, 4016746496, 4016750591, -STORE, 4016750592, 4016758783, -STORE, 4016799744, 4016844799, -STORE, 4016844800, 4016902143, -STORE, 4016902144, 4016992255, -STORE, 4016992256, 4017000447, -STORE, 4017000448, 4017004543, -STORE, 4017004544, 4017008639, -STORE, 4017008640, 4017016831, -STORE, 4017016832, 4017020927, -STORE, 4017020928, 4017127423, -STORE, 4017127424, 4017131519, -STORE, 4017131520, 4017229823, -STORE, 4017229824, 4017422335, -STORE, 4017422336, 4017438719, -STORE, 4017438720, 4017442815, -STORE, 4017442816, 4017446911, -STORE, 4017446912, 4017455103, -STORE, 4017455104, 4017766399, -STORE, 4017766400, 4017909759, -STORE, 4017909760, 4018081791, -STORE, 4018081792, 4018089983, -STORE, 4018089984, 4018094079, -STORE, 4018094080, 4018098175, -STORE, 4018098176, 4018327551, -STORE, 4018327552, 4018331647, -STORE, 4018331648, 4018339839, -STORE, 4018339840, 4018348031, -STORE, 4018348032, 4018610175, -STORE, 4018610176, 4018626559, -STORE, 4018626560, 4018647039, -STORE, 4018647040, 4018651135, -STORE, 4018651136, 4018749439, -STORE, 4018749440, 4018761727, -STORE, 4018761728, 4018802687, -STORE, 4018802688, 4018806783, -STORE, 4018806784, 4018810879, -STORE, 4018810880, 4018814975, -STORE, 4018814976, 4018823167, -STORE, 4018823168, 4018954239, -STORE, 4018954240, 4019007487, -STORE, 4019007488, 4019068927, -STORE, 4019068928, 4019077119, -STORE, 4019077120, 4019081215, -STORE, 4019081216, 4019093503, -STORE, 4019093504, 4019208191, -STORE, 4019208192, 4019232767, -STORE, 4019232768, 4019265535, -STORE, 4019265536, 4019269631, -STORE, 4019269632, 4019277823, -STORE, 4019277824, 4019458047, -STORE, 4019458048, 4019519487, -STORE, 4019519488, 4019613695, -STORE, 4019613696, 4019621887, -STORE, 4019621888, 4019625983, -STORE, 4019625984, 4019630079, -STORE, 4019630080, 4019744767, -STORE, 4019744768, 4019822591, -STORE, 4019822592, 4019929087, -STORE, 4019929088, 4019941375, -STORE, 4019941376, 4019945471, -STORE, 4019945472, 4019961855, -STORE, 4019961856, 4019994623, -STORE, 4019994624, 4019998719, -STORE, 4019998720, 4020002815, -STORE, 4020002816, 4020006911, -STORE, 4020006912, 4020011007, -STORE, 4020011008, 4020256767, -STORE, 4020256768, 4020326399, -STORE, 4020326400, 4020457471, -STORE, 4020457472, 4020469759, -STORE, 4020469760, 4020473855, -STORE, 4020473856, 4020482047, -STORE, 4020482048, 4020711423, -STORE, 4020711424, 4020715519, -STORE, 4020715520, 4020719615, -STORE, 4020719616, 4020723711, -STORE, 4020723712, 4020805631, -STORE, 4020805632, 4021051391, -STORE, 4021051392, 4021460991, -STORE, 4021460992, 4021469183, -STORE, 4021469184, 4021473279, -STORE, 4021473280, 4021571583, -STORE, 4021571584, 4021633023, -STORE, 4021633024, 4021727231, -STORE, 4021727232, 4021735423, -STORE, 4021735424, 4021739519, -STORE, 4021739520, 4021747711, -STORE, 4021747712, 4021829631, -STORE, 4021829632, 4021866495, -STORE, 4021866496, 4021919743, -STORE, 4021919744, 4021927935, -STORE, 4021927936, 4021932031, -STORE, 4021932032, 4021944319, -STORE, 4021944320, 4022157311, -STORE, 4022157312, 4022161407, -STORE, 4022161408, 4022173695, -STORE, 4022173696, 4022177791, -STORE, 4022177792, 4022472703, -STORE, 4022472704, 4022509567, -STORE, 4022509568, 4022583295, -STORE, 4022583296, 4022587391, -STORE, 4022587392, 4022591487, -STORE, 4022591488, 4022607871, -STORE, 4022607872, 4022657023, -STORE, 4022657024, 4022722559, -STORE, 4022722560, 4022730751, -STORE, 4022730752, 4022734847, -STORE, 4022734848, 4022865919, -STORE, 4022865920, 4022943743, -STORE, 4022943744, 4023062527, -STORE, 4023062528, 4023074815, -STORE, 4023074816, 4023078911, -STORE, 4023078912, 4023128063, -STORE, 4023128064, 4023218175, -STORE, 4023218176, 4023361535, -STORE, 4023361536, 4023373823, -STORE, 4023373824, 4023377919, -STORE, 4023377920, 4023558143, -STORE, 4023558144, 4023631871, -STORE, 4023631872, 4023816191, -STORE, 4023816192, 4023820287, -STORE, 4023820288, 4023824383, -STORE, 4023824384, 4023832575, -STORE, 4023832576, 4024078335, -STORE, 4024078336, 4024197119, -STORE, 4024197120, 4024389631, -STORE, 4024389632, 4024406015, -STORE, 4024406016, 4024410111, -STORE, 4024410112, 4024422399, -STORE, 4024422400, 4024619007, -STORE, 4024619008, 4024639487, -STORE, 4024639488, 4024655871, -STORE, 4024655872, 4024664063, -STORE, 4024664064, 4024668159, -STORE, 4024668160, 4024676351, -STORE, 4024676352, 4024905727, -STORE, 4024905728, 4024909823, -STORE, 4024909824, 4024918015, -STORE, 4024918016, 4024922111, -STORE, 4024922112, 4024930303, -STORE, 4024930304, 4025110527, -STORE, 4025110528, 4025176063, -STORE, 4025176064, 4025208831, -STORE, 4025208832, 4025212927, -STORE, 4025212928, 4025217023, -STORE, 4025217024, 4025348095, -STORE, 4025348096, 4025372671, -STORE, 4025372672, 4025458687, -STORE, 4025458688, 4025466879, -STORE, 4025466880, 4025565183, -STORE, 4025565184, 4025757695, -STORE, 4025757696, 4026249215, -STORE, 4026249216, 4026261503, -STORE, 4026261504, 4026265599, -STORE, 4026265600, 4026269695, -STORE, 4026269696, 4026302463, -STORE, 4026302464, 4026306559, -STORE, 4026306560, 4026314751, -STORE, 4026314752, 4026318847, -STORE, 4026318848, 4026322943, -STORE, 4026322944, 4026327039, -STORE, 4026327040, 4026654719, -STORE, 4026654720, 4026671103, -STORE, 4026671104, 4026720255, -STORE, 4026720256, 4026724351, -STORE, 4026724352, 4026728447, -STORE, 4026728448, 4026732543, -STORE, 4026732544, 4026863615, -STORE, 4026863616, 4027027455, -STORE, 4027027456, 4027031551, -STORE, 4027031552, 4027514879, -STORE, 4027514880, 4027531263, -STORE, 4027531264, 4027535359, -STORE, 4027535360, 4027539455, -STORE, 4027539456, 4027785215, -STORE, 4027785216, 4027789311, -STORE, 4027789312, 4027793407, -STORE, 4027793408, 4027797503, -STORE, 4027797504, 4027863039, -STORE, 4027863040, 4027899903, -STORE, 4027899904, 4027949055, -STORE, 4027949056, 4027957247, -STORE, 4027957248, 4027961343, -STORE, 4027961344, 4027965439, -STORE, 4027965440, 4028194815, -STORE, 4028194816, 4028252159, -STORE, 4028252160, 4028338175, -STORE, 4028338176, 4028350463, -STORE, 4028350464, 4028354559, -STORE, 4028354560, 4028452863, -STORE, 4028452864, 4028489727, -STORE, 4028489728, 4028530687, -STORE, 4028530688, 4028538879, -STORE, 4028538880, 4028542975, -STORE, 4028542976, 4028551167, -STORE, 4028551168, 4028665855, -STORE, 4028665856, 4029349887, -STORE, 4029349888, 4030468095, -STORE, 4030468096, 4030513151, -STORE, 4030513152, 4030517247, -STORE, 4030517248, 4030525439, -STORE, 4030525440, 4030529535, -STORE, 4030529536, 4030758911, -STORE, 4030758912, 4030828543, -STORE, 4030828544, 4030943231, -STORE, 4030943232, 4030951423, -STORE, 4030951424, 4030955519, -STORE, 4030955520, 4030967807, -STORE, 4030967808, 4031131647, -STORE, 4031131648, 4031135743, -STORE, 4031135744, 4031139839, -STORE, 4031139840, 4031148031, -STORE, 4031148032, 4031152127, -STORE, 4031152128, 4031160319, -STORE, 4031160320, 4031504383, -STORE, 4031504384, 4031598591, -STORE, 4031598592, 4031754239, -STORE, 4031754240, 4031766527, -STORE, 4031766528, 4031770623, -STORE, 4031770624, 4031774719, -STORE, 4031774720, 4031782911, -STORE, 4031782912, 4031799295, -STORE, 4031799296, 4031856639, -STORE, 4031856640, 4031983615, -STORE, 4031983616, 4031987711, -STORE, 4031987712, 4031991807, -STORE, 4031991808, 4032270335, -STORE, 4032270336, 4032274431, -STORE, 4032274432, 4032282623, -STORE, 4032282624, 4032286719, -STORE, 4032286720, 4032290815, -STORE, 4032290816, 4032389119, -STORE, 4032389120, 4032397311, -STORE, 4032397312, 4032405503, -STORE, 4032405504, 4032413695, -STORE, 4032413696, 4032417791, -STORE, 4032417792, 4032565247, -STORE, 4032565248, 4032593919, -STORE, 4032593920, 4032737279, -STORE, 4032737280, 4032741375, -STORE, 4032741376, 4032745471, -STORE, 4032745472, 4032770047, -STORE, 4032770048, 4032933887, -STORE, 4032933888, 4032999423, -STORE, 4032999424, 4033032191, -STORE, 4033032192, 4033036287, -STORE, 4033036288, 4033040383, -STORE, 4033040384, 4033105919, -STORE, 4033105920, 4033396735, -STORE, 4033396736, 4033822719, -STORE, 4033822720, 4033839103, -STORE, 4033839104, 4033843199, -STORE, 4033843200, 4033851391, -STORE, 4033851392, 4033863679, -STORE, 4033863680, 4033880063, -STORE, 4033880064, 4033933311, -STORE, 4033933312, 4034023423, -STORE, 4034023424, 4034031615, -STORE, 4034031616, 4034035711, -STORE, 4034035712, 4034043903, -STORE, 4034043904, 4034142207, -STORE, 4034142208, 4034191359, -STORE, 4034191360, 4034260991, -STORE, 4034260992, 4034269183, -STORE, 4034269184, 4034273279, -STORE, 4034273280, 4034281471, -STORE, 4034281472, 4034412543, -STORE, 4034412544, 4034445311, -STORE, 4034445312, 4034490367, -STORE, 4034490368, 4034494463, -STORE, 4034494464, 4034498559, -STORE, 4034498560, 4034662399, -STORE, 4034662400, 4034666495, -STORE, 4034666496, 4034670591, -STORE, 4034670592, 4034674687, -STORE, 4034674688, 4034678783, -STORE, 4034678784, 4034682879, -STORE, 4034682880, 4034781183, -STORE, 4034781184, 4035043327, -STORE, 4035043328, 4035047423, -STORE, 4035047424, 4035055615, -STORE, 4035055616, 4035059711, -STORE, 4035059712, 4035063807, -STORE, 4035063808, 4035067903, -STORE, 4035067904, 4035100671, -STORE, 4035100672, 4035375103, -STORE, 4035375104, 4035383295, -STORE, 4035383296, 4035395583, -STORE, 4035395584, 4035399679, -STORE, 4035399680, 4035403775, -STORE, 4035403776, 4035407871, -STORE, 4035407872, 4035411967, -STORE, 4035411968, 4035477503, -STORE, 4035477504, 4035608575, -STORE, 4035608576, 4035641343, -STORE, 4035641344, 4035682303, -STORE, 4035682304, 4035686399, -STORE, 4035686400, 4035690495, -STORE, 4035690496, 4035694591, -STORE, 4035694592, 4035743743, -STORE, 4035743744, 4035784703, -STORE, 4035784704, 4035829759, -STORE, 4035829760, 4035837951, -STORE, 4035837952, 4035842047, -STORE, 4035842048, 4035846143, -STORE, 4035846144, 4035850239, -STORE, 4035850240, 4036001791, -STORE, 4036001792, 4036005887, -STORE, 4036005888, 4036214783, -STORE, 4036214784, 4036218879, -STORE, 4036218880, 4036603903, -STORE, 4036603904, 4036648959, -STORE, 4036648960, 4036653055, -STORE, 4036653056, 4036657151, -STORE, 4036657152, 4036665343, -STORE, 4036665344, 4036780031, -STORE, 4036780032, 4036829183, -STORE, 4036829184, 4036984831, -STORE, 4036984832, 4036993023, -STORE, 4036993024, 4036997119, -STORE, 4036997120, 4037001215, -STORE, 4037001216, 4037009407, -STORE, 4037009408, 4037025791, -STORE, 4037025792, 4037095423, -STORE, 4037095424, 4037181439, -STORE, 4037181440, 4037193727, -STORE, 4037193728, 4037197823, -STORE, 4037197824, 4037206015, -STORE, 4037206016, 4037320703, -STORE, 4037320704, 4037337087, -STORE, 4037337088, 4037349375, -STORE, 4037349376, 4037357567, -STORE, 4037357568, 4037361663, -STORE, 4037369856, 4037386239, -STORE, 4037386240, 4037672959, -STORE, 4037672960, 4037689343, -STORE, 4037689344, 4037730303, -STORE, 4037730304, 4037734399, -STORE, 4037734400, 4037738495, -STORE, 4037738496, 4037742591, -STORE, 4037742592, 4037758975, -STORE, 4037758976, 4037890047, -STORE, 4037890048, 4037931007, -STORE, 4037931008, 4037976063, -STORE, 4037976064, 4037984255, -STORE, 4037984256, 4037988351, -STORE, 4037988352, 4038053887, -STORE, 4038053888, 4038184959, -STORE, 4038184960, 4038189055, -STORE, 4038189056, 4038197247, -STORE, 4038197248, 4038201343, -STORE, 4038201344, 4038205439, -STORE, 4038205440, 4038209535, -STORE, 4038217728, 4038250495, -STORE, 4038250496, 4038512639, -STORE, 4038512640, 4038516735, -STORE, 4038516736, 4038520831, -STORE, 4038520832, 4038524927, -STORE, 4038524928, 4038529023, -STORE, 4038529024, 4038533119, -STORE, 4038541312, 4038623231, -STORE, 4038623232, 4038754303, -STORE, 4038754304, 4038885375, -STORE, 4038885376, 4038889471, -STORE, 4038897664, 4038963199, -STORE, 4038963200, 4038967295, -STORE, 4038967296, 4038983679, -STORE, 4038983680, 4039114751, -STORE, 4039114752, 4039245823, -STORE, 4039245824, 4039376895, -STORE, 4039376896, 4040687615, -STORE, 4040687616, 4040691711, -STORE, 4040691712, 4040806399, -STORE, 4040806400, 4040937471, -STORE, 4040937472, 4040941567, -STORE, 4040945664, 4040949759, -STORE, 4040949760, 4041080831, -STORE, 4041080832, 4041211903, -STORE, 4041211904, 4043046911, -STORE, 4043046912, 4043051007, -STORE, 4043051008, 4043055103, -STORE, 4043055104, 4043137023, -STORE, 4043137024, 4043141119, -STORE, 4043141120, 4043145215, -STORE, 4043145216, 4043153407, -STORE, 4043153408, 4043186175, -STORE, 4043186176, 4043317247, -STORE, 4043317248, 4043448319, -STORE, 4043448320, 4043579391, -STORE, 4043579392, 4043583487, -STORE, 4043583488, 4043599871, -STORE, 4043599872, 4043661311, -STORE, 4043661312, 4043792383, -STORE, 4043792384, 4043796479, -STORE, 4043796480, 4043800575, -STORE, 4043800576, 4043816959, -STORE, 4043816960, 4043821055, -STORE, 4043821056, 4043825151, -STORE, 4043825152, 4043829247, -STORE, 4043829248, 4043833343, -STORE, 4043833344, 4047241215, -STORE, 4047241216, 4047249407, -STORE, 4047249408, 4047253503, -STORE, 4047253504, 4047323135, -STORE, 4047323136, 4047327231, -STORE, 4047327232, 4047458303, -STORE, 4047458304, 4047589375, -STORE, 4047589376, 4047720447, -STORE, 4047720448, 4047773695, -STORE, 4047773696, 4047790079, -STORE, 4047790080, 4047921151, -STORE, 4047921152, 4048052223, -STORE, 4048052224, 4048183295, -STORE, 4048183296, 4049002495, -STORE, 4049002496, 4049133567, -STORE, 4049133568, 4049154047, -STORE, 4049154048, 4049158143, -STORE, 4049158144, 4049162239, -STORE, 4049162240, 4049166335, -STORE, 4049166336, 4049174527, -STORE, 4049174528, 4049182719, -STORE, 4049182720, 4049186815, -STORE, 4049186816, 4049190911, -STORE, 4049190912, 4049195007, -STORE, 4049195008, 4049203199, -STORE, 4049203200, 4049207295, -STORE, 4049207296, 4049211391, -STORE, 4049211392, 4049215487, -STORE, 4049215488, 4049219583, -STORE, 4049219584, 4049227775, -STORE, 4049227776, 4049231871, -STORE, 4049231872, 4049235967, -STORE, 4049235968, 4049244159, -STORE, 4049244160, 4049248255, -STORE, 4049248256, 4049252351, -STORE, 4049252352, 4049256447, -STORE, 4049256448, 4049268735, -STORE, 4049268736, 4049272831, -STORE, 4049272832, 4049313791, -STORE, 4049313792, 4049723391, -STORE, 4049723392, 4049727487, -STORE, 4049727488, 4049858559, -STORE, 4049858560, 4049989631, -STORE, 4049989632, 4049993727, -STORE, 4049993728, 4050026495, -STORE, 4050026496, 4050030591, -STORE, 4050030592, 4050161663, -STORE, 4050161664, 4050169855, -STORE, 4050169856, 4050223103, -STORE, 4050223104, 4050632703, -STORE, 4050632704, 4050636799, -STORE, 4050636800, 4050640895, -STORE, 4050640896, 4050644991, -STORE, 4050644992, 4050661375, -STORE, 4050661376, 4050665471, -STORE, 4050665472, 4050673663, -STORE, 4050673664, 4050677759, -STORE, 4050677760, 4050694143, -STORE, 4050694144, 4050702335, -STORE, 4050702336, 4050956287, -STORE, 4050956288, 4051963903, -STORE, 4051963904, 4051980287, -STORE, 4051980288, 4051988479, -STORE, 4051988480, 4052000767, -STORE, 4052000768, 4052004863, -STORE, 4052004864, 4052029439, -STORE, 4284014592, 4284018687, -STORE, 4284018688, 4292403199, -SNULL, 4041080832, 4041211903, -SNULL, 3795763200, 3795894271, -STORE, 3629522944, 3696631807, -SNULL, 3663077375, 3696631807, -STORE, 3629522944, 3663077375, -STORE, 3663077376, 3696631807, -SNULL, 3663077376, 3696631807, -STORE, 3663077376, 3696631807, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3626471424, 3627524095, -SNULL, 3626471424, 3626475519, -STORE, 3626475520, 3627524095, -STORE, 3626471424, 3626475519, -SNULL, 3627519999, 3627524095, -STORE, 3626475520, 3627519999, -STORE, 3627520000, 3627524095, -STORE, 3625418752, 3626475519, -SNULL, 3625418752, 3625422847, -STORE, 3625422848, 3626475519, -STORE, 3625418752, 3625422847, -SNULL, 3626467327, 3626475519, -STORE, 3625422848, 3626467327, -STORE, 3626467328, 3626475519, -STORE, 3624366080, 3625422847, -SNULL, 3624366080, 3624370175, -STORE, 3624370176, 3625422847, -STORE, 3624366080, 3624370175, -SNULL, 3625414655, 3625422847, -STORE, 3624370176, 3625414655, -STORE, 3625414656, 3625422847, -STORE, 4041191424, 4041211903, -SNULL, 4041195519, 4041211903, -STORE, 4041191424, 4041195519, -STORE, 4041195520, 4041211903, -STORE, 4041170944, 4041191423, -SNULL, 4041175039, 4041191423, -STORE, 4041170944, 4041175039, -STORE, 4041175040, 4041191423, -SNULL, 3625426943, 3626467327, -STORE, 3625422848, 3625426943, -STORE, 3625426944, 3626467327, -STORE, 4041162752, 4041170943, -SNULL, 3626479615, 3627519999, -STORE, 3626475520, 3626479615, -STORE, 3626479616, 3627519999, -STORE, 4041154560, 4041162751, -STORE, 4041154560, 4041170943, -STORE, 4041134080, 4041154559, -SNULL, 4041138175, 4041154559, -STORE, 4041134080, 4041138175, -STORE, 4041138176, 4041154559, -SNULL, 3624374271, 3625414655, -STORE, 3624370176, 3624374271, -STORE, 3624374272, 3625414655, -STORE, 4041125888, 4041134079, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -STORE, 3487174656, 3487584255, -STORE, 4041121792, 4041125887, -SNULL, 4041121792, 4041125887, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 3487174656, 3487584255, -STORE, 3222274048, 3223326719, -SNULL, 3222274048, 3222278143, -STORE, 3222278144, 3223326719, -STORE, 3222274048, 3222278143, -SNULL, 3223322623, 3223326719, -STORE, 3222278144, 3223322623, -STORE, 3223322624, 3223326719, -STORE, 3221221376, 3222278143, -SNULL, 3221221376, 3221225471, -STORE, 3221225472, 3222278143, -STORE, 3221221376, 3221225471, -SNULL, 3222269951, 3222278143, -STORE, 3221225472, 3222269951, -STORE, 3222269952, 3222278143, -STORE, 3220168704, 3221225471, -SNULL, 3220168704, 3220172799, -STORE, 3220172800, 3221225471, -STORE, 3220168704, 3220172799, -SNULL, 3221217279, 3221225471, -STORE, 3220172800, 3221217279, -STORE, 3221217280, 3221225471, -STORE, 4041117696, 4041125887, -STORE, 4041117696, 4041134079, -STORE, 3219083264, 3220172799, -SNULL, 3219083264, 3219087359, -STORE, 3219087360, 3220172799, -STORE, 3219083264, 3219087359, -SNULL, 3220164607, 3220172799, -STORE, 3219087360, 3220164607, -STORE, 3220164608, 3220172799, -STORE, 4041109504, 4041117695, -STORE, 4041109504, 4041134079, -STORE, 3217997824, 3219087359, -SNULL, 3217997824, 3218001919, -STORE, 3218001920, 3219087359, -STORE, 3217997824, 3218001919, -SNULL, 3219079167, 3219087359, -STORE, 3218001920, 3219079167, -STORE, 3219079168, 3219087359, -STORE, 4041101312, 4041109503, -STORE, 4041101312, 4041134079, -STORE, 3216912384, 3218001919, -SNULL, 3216912384, 3216916479, -STORE, 3216916480, 3218001919, -STORE, 3216912384, 3216916479, -SNULL, 3217993727, 3218001919, -STORE, 3216916480, 3217993727, -STORE, 3217993728, 3218001919, -STORE, 4041093120, 4041101311, -STORE, 4041093120, 4041134079, -STORE, 3215826944, 3216916479, -SNULL, 3215826944, 3215831039, -STORE, 3215831040, 3216916479, -STORE, 3215826944, 3215831039, -SNULL, 3216908287, 3216916479, -STORE, 3215831040, 3216908287, -STORE, 3216908288, 3216916479, -STORE, 4016779264, 4016799743, -SNULL, 4016783359, 4016799743, -STORE, 4016779264, 4016783359, -STORE, 4016783360, 4016799743, -STORE, 4016758784, 4016779263, -SNULL, 4016762879, 4016779263, -STORE, 4016758784, 4016762879, -STORE, 4016762880, 4016779263, -SNULL, 3222282239, 3223322623, -STORE, 3222278144, 3222282239, -STORE, 3222282240, 3223322623, -STORE, 4041084928, 4041093119, -STORE, 4041084928, 4041134079, -SNULL, 3221229567, 3222269951, -STORE, 3221225472, 3221229567, -STORE, 3221229568, 3222269951, -STORE, 4015644672, 4015665151, -STORE, 4038889472, 4038897663, -SNULL, 4015648767, 4015665151, -STORE, 4015644672, 4015648767, -STORE, 4015648768, 4015665151, -STORE, 4015624192, 4015644671, -SNULL, 4015628287, 4015644671, -STORE, 4015624192, 4015628287, -STORE, 4015628288, 4015644671, -SNULL, 3219091455, 3220164607, -STORE, 3219087360, 3219091455, -STORE, 3219091456, 3220164607, -STORE, 4015603712, 4015624191, -SNULL, 4015607807, 4015624191, -STORE, 4015603712, 4015607807, -STORE, 4015607808, 4015624191, -SNULL, 3218006015, 3219079167, -STORE, 3218001920, 3218006015, -STORE, 3218006016, 3219079167, -STORE, 3949674496, 3949694975, -SNULL, 3949678591, 3949694975, -STORE, 3949674496, 3949678591, -STORE, 3949678592, 3949694975, -SNULL, 3216920575, 3217993727, -STORE, 3216916480, 3216920575, -STORE, 3216920576, 3217993727, -STORE, 3948924928, 3948945407, -SNULL, 3948929023, 3948945407, -STORE, 3948924928, 3948929023, -STORE, 3948929024, 3948945407, -SNULL, 3215835135, 3216908287, -STORE, 3215831040, 3215835135, -STORE, 3215835136, 3216908287, -SNULL, 3220176895, 3221217279, -STORE, 3220172800, 3220176895, -STORE, 3220176896, 3221217279, -STORE, 3214786560, 3215826943, -STORE, 3213733888, 3214786559, -SNULL, 3213733888, 3213737983, -STORE, 3213737984, 3214786559, -STORE, 3213733888, 3213737983, -SNULL, 3214782463, 3214786559, -STORE, 3213737984, 3214782463, -STORE, 3214782464, 3214786559, -STORE, 4038533120, 4038541311, -STORE, 3948421120, 3948441599, -SNULL, 3948425215, 3948441599, -STORE, 3948421120, 3948425215, -STORE, 3948425216, 3948441599, -SNULL, 3213742079, 3214782463, -STORE, 3213737984, 3213742079, -STORE, 3213742080, 3214782463, -STORE, 4038209536, 4038217727, -STORE, 3212681216, 3213737983, -SNULL, 3212681216, 3212685311, -STORE, 3212685312, 3213737983, -STORE, 3212681216, 3212685311, -SNULL, 3213729791, 3213737983, -STORE, 3212685312, 3213729791, -STORE, 3213729792, 3213737983, -STORE, 3795763200, 3795894271, -STORE, 3946872832, 3946893311, -SNULL, 3946876927, 3946893311, -STORE, 3946872832, 3946876927, -STORE, 3946876928, 3946893311, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -STORE, 3487174656, 3487584255, -SNULL, 3212689407, 3213729791, -STORE, 3212685312, 3212689407, -STORE, 3212689408, 3213729791, -STORE, 4041080832, 4041084927, -STORE, 4040941568, 4040945663, -STORE, 4037361664, 4037369855, -STORE, 4000817152, 4000821247, -STORE, 3999440896, 3999444991, -STORE, 3212161024, 3212681215, -SNULL, 3212161024, 3212439551, -STORE, 3212439552, 3212681215, -STORE, 3212161024, 3212439551, -SNULL, 3212161024, 3212439551, -SNULL, 3212464127, 3212681215, -STORE, 3212439552, 3212464127, -STORE, 3212464128, 3212681215, -SNULL, 3212464128, 3212681215, -SNULL, 3212439552, 3212451839, -STORE, 3212451840, 3212464127, -STORE, 3212439552, 3212451839, -SNULL, 3212439552, 3212451839, -STORE, 3212439552, 3212451839, -SNULL, 3212451840, 3212455935, -STORE, 3212455936, 3212464127, -STORE, 3212451840, 3212455935, -SNULL, 3212451840, 3212455935, -STORE, 3212451840, 3212455935, -SNULL, 3212455936, 3212460031, -STORE, 3212460032, 3212464127, -STORE, 3212455936, 3212460031, -SNULL, 3212455936, 3212460031, -STORE, 3212455936, 3212460031, -SNULL, 3212460032, 3212464127, -STORE, 3212460032, 3212464127, -STORE, 3997679616, 3997683711, -SNULL, 4049235968, 4049240063, -STORE, 4049240064, 4049244159, -STORE, 4049235968, 4049240063, -SNULL, 4049240064, 4049244159, -STORE, 4049240064, 4049244159, -SNULL, 3997679616, 3997683711, -SNULL, 3999440896, 3999444991, -SNULL, 4000817152, 4000821247, -SNULL, 4040941568, 4040945663, -SNULL, 4041080832, 4041084927, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 3487174656, 3487584255, -SNULL, 3212451840, 3212455935, -STORE, 3212451840, 3212455935, -STORE, 4041080832, 4041084927, -STORE, 3623890944, 3624169471, -SNULL, 4041080832, 4041084927, -STORE, 4041080832, 4041084927, -SNULL, 4041080832, 4041084927, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -STORE, 4041080832, 4041084927, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -STORE, 3211386880, 3212439551, -SNULL, 3211386880, 3211390975, -STORE, 3211390976, 3212439551, -STORE, 3211386880, 3211390975, -SNULL, 3212435455, 3212439551, -STORE, 3211390976, 3212435455, -STORE, 3212435456, 3212439551, -STORE, 4040941568, 4040945663, -STORE, 3937169408, 3937189887, -STORE, 3623485440, 3623616511, -SNULL, 717225983, 1388314623, -STORE, 314572800, 717225983, -STORE, 717225984, 1388314623, -SNULL, 717225984, 1388314623, -STORE, 3937112064, 3937132543, -SNULL, 3937116159, 3937132543, -STORE, 3937112064, 3937116159, -STORE, 3937116160, 3937132543, -SNULL, 3211395071, 3212435455, -STORE, 3211390976, 3211395071, -STORE, 3211395072, 3212435455, -STORE, 4000817152, 4000821247, -STORE, 3974823936, 3974832127, -STORE, 3595284480, 3595431935, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -STORE, 3487174656, 3487584255, -STORE, 3999440896, 3999444991, -STORE, 3997679616, 3997683711, -STORE, 3996295168, 3996299263, -STORE, 3996090368, 3996094463, -STORE, 3210866688, 3211386879, -SNULL, 3210866688, 3211001855, -STORE, 3211001856, 3211386879, -STORE, 3210866688, 3211001855, -SNULL, 3210866688, 3211001855, -SNULL, 3211038719, 3211386879, -STORE, 3211001856, 3211038719, -STORE, 3211038720, 3211386879, -SNULL, 3211038720, 3211386879, -SNULL, 3211001856, 3211022335, -STORE, 3211022336, 3211038719, -STORE, 3211001856, 3211022335, -SNULL, 3211001856, 3211022335, -STORE, 3211001856, 3211022335, -SNULL, 3211022336, 3211030527, -STORE, 3211030528, 3211038719, -STORE, 3211022336, 3211030527, -SNULL, 3211022336, 3211030527, -STORE, 3211022336, 3211030527, -SNULL, 3211030528, 3211034623, -STORE, 3211034624, 3211038719, -STORE, 3211030528, 3211034623, -SNULL, 3211030528, 3211034623, -STORE, 3211030528, 3211034623, -SNULL, 3211034624, 3211038719, -STORE, 3211034624, 3211038719, -STORE, 3994906624, 3994910719, -SNULL, 4049240064, 4049244159, -STORE, 4049240064, 4049244159, -SNULL, 3994906624, 3994910719, -SNULL, 3996090368, 3996094463, -SNULL, 3996295168, 3996299263, -SNULL, 3997679616, 3997683711, -SNULL, 3999440896, 3999444991, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 3487174656, 3487584255, -SNULL, 3211022336, 3211030527, -STORE, 3211022336, 3211030527, -STORE, 3999440896, 3999444991, -STORE, 3210199040, 3211001855, -SNULL, 3999440896, 3999444991, -STORE, 3999440896, 3999444991, -SNULL, 3999440896, 3999444991, -STORE, 3594821632, 3594952703, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 4048183296, 4048592895, -STORE, 4048592896, 4049002495, -STORE, 4048183296, 4048592895, -STORE, 4048183296, 4049002495, -SNULL, 1914101759, 1969434623, -STORE, 1914097664, 1914101759, -STORE, 1914101760, 1969434623, -STORE, 3567108096, 3567239167, -STORE, 3973832704, 3973840895, -STORE, 3209113600, 3210199039, -SNULL, 3209113600, 3209117695, -STORE, 3209117696, 3210199039, -STORE, 3209113600, 3209117695, -SNULL, 3210194943, 3210199039, -STORE, 3209117696, 3210194943, -STORE, 3210194944, 3210199039, -STORE, 3935858688, 3935879167, -SNULL, 3935862783, 3935879167, -STORE, 3935858688, 3935862783, -STORE, 3935862784, 3935879167, -SNULL, 3209121791, 3210194943, -STORE, 3209117696, 3209121791, -STORE, 3209121792, 3210194943, -STORE, 3528749056, 3528880127, -STORE, 3968200704, 3968208895, -STORE, 3208028160, 3209117695, -SNULL, 3208028160, 3208032255, -STORE, 3208032256, 3209117695, -STORE, 3208028160, 3208032255, -SNULL, 3209109503, 3209117695, -STORE, 3208032256, 3209109503, -STORE, 3209109504, 3209117695, -STORE, 3888123904, 3888144383, -SNULL, 3888127999, 3888144383, -STORE, 3888123904, 3888127999, -STORE, 3888128000, 3888144383, -SNULL, 3208036351, 3209109503, -STORE, 3208032256, 3208036351, -STORE, 3208036352, 3209109503, -SNULL, 3968200704, 3968208895, -SNULL, 3888123904, 3888144383, -SNULL, 3209109504, 3209113599, -STORE, 3209113600, 3209117695, -STORE, 3209109504, 3209113599, -SNULL, 3208028160, 3209113599, -STORE, 3208060928, 3209117695, -SNULL, 3208060928, 3208065023, -STORE, 3208065024, 3209117695, -STORE, 3208060928, 3208065023, -SNULL, 3209109503, 3209117695, -STORE, 3208065024, 3209109503, -STORE, 3209109504, 3209117695, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3888123904, 3888144383, -SNULL, 3888127999, 3888144383, -STORE, 3888123904, 3888127999, -STORE, 3888128000, 3888144383, -SNULL, 3208069119, 3209109503, -STORE, 3208065024, 3208069119, -STORE, 3208069120, 3209109503, -STORE, 3968200704, 3968208895, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3527778304, 3527909375, -STORE, 3999440896, 3999444991, -STORE, 3997679616, 3997683711, -STORE, 1914097664, 1914105855, -STORE, 1914105856, 1969434623, -STORE, 3957583872, 3957592063, -STORE, 3206975488, 3208065023, -SNULL, 3206975488, 3206979583, -STORE, 3206979584, 3208065023, -STORE, 3206975488, 3206979583, -SNULL, 3208056831, 3208065023, -STORE, 3206979584, 3208056831, -STORE, 3208056832, 3208065023, -STORE, 3956736000, 3956744191, -STORE, 3205890048, 3206979583, -SNULL, 3205890048, 3205894143, -STORE, 3205894144, 3206979583, -STORE, 3205890048, 3205894143, -SNULL, 3206971391, 3206979583, -STORE, 3205894144, 3206971391, -STORE, 3206971392, 3206979583, -STORE, 3806101504, 3806121983, -SNULL, 3806105599, 3806121983, -STORE, 3806101504, 3806105599, -STORE, 3806105600, 3806121983, -SNULL, 3206983679, 3208056831, -STORE, 3206979584, 3206983679, -STORE, 3206983680, 3208056831, -STORE, 3806081024, 3806101503, -SNULL, 3806085119, 3806101503, -STORE, 3806081024, 3806085119, -STORE, 3806085120, 3806101503, -SNULL, 3205898239, 3206971391, -STORE, 3205894144, 3205898239, -STORE, 3205898240, 3206971391, -STORE, 3956015104, 3956023295, -STORE, 3204804608, 3205894143, -SNULL, 3204804608, 3204808703, -STORE, 3204808704, 3205894143, -STORE, 3204804608, 3204808703, -SNULL, 3205885951, 3205894143, -STORE, 3204808704, 3205885951, -STORE, 3205885952, 3205894143, -STORE, 3803471872, 3803492351, -STORE, 3803451392, 3803471871, -STORE, 3803451392, 3803492351, -SNULL, 3957583872, 3957592063, -SNULL, 3806101504, 3806121983, -SNULL, 3206975487, 3206979583, -STORE, 3206971392, 3206975487, -STORE, 3206975488, 3206979583, -SNULL, 3208056832, 3208060927, -STORE, 3208060928, 3208065023, -STORE, 3208056832, 3208060927, -SNULL, 3206975488, 3208060927, -STORE, 3801845760, 3801878527, -STORE, 3806101504, 3806121983, -SNULL, 3806105599, 3806121983, -STORE, 3806101504, 3806105599, -STORE, 3806105600, 3806121983, -SNULL, 3204812799, 3205885951, -STORE, 3204808704, 3204812799, -STORE, 3204812800, 3205885951, -STORE, 1914097664, 1914109951, -STORE, 1914109952, 1969434623, -STORE, 3957583872, 3957592063, -STORE, 3206971392, 3208065023, -SNULL, 3206971392, 3206979583, -STORE, 3206979584, 3208065023, -STORE, 3206971392, 3206979583, -SNULL, 3208056831, 3208065023, -STORE, 3206979584, 3208056831, -STORE, 3208056832, 3208065023, -STORE, 3801825280, 3801845759, -SNULL, 3801829375, 3801845759, -STORE, 3801825280, 3801829375, -STORE, 3801829376, 3801845759, -SNULL, 3206983679, 3208056831, -STORE, 3206979584, 3206983679, -STORE, 3206983680, 3208056831, -STORE, 3202707456, 3204804607, -SNULL, 3202707456, 3204804607, -STORE, 3202707456, 3204804607, -STORE, 3200610304, 3202707455, -SNULL, 3202707456, 3204804607, -SNULL, 3200610304, 3202707455, -STORE, 3202707456, 3204804607, -SNULL, 3202707456, 3204804607, -STORE, 3202707456, 3204804607, -SNULL, 3202707456, 3204804607, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3527647232, 3527778303, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -STORE, 3487059968, 3487584255, -SNULL, 3487059968, 3487301631, -STORE, 3487301632, 3487584255, -STORE, 3487059968, 3487301631, -SNULL, 3487059968, 3487301631, -SNULL, 3487563775, 3487584255, -STORE, 3487301632, 3487563775, -STORE, 3487563776, 3487584255, -SNULL, 3487563776, 3487584255, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3524046848, 3524177919, -STORE, 3487170560, 3487301631, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3487039488, 3487170559, -STORE, 3487039488, 3487301631, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3204280320, 3204804607, -SNULL, 3204280320, 3204448255, -STORE, 3204448256, 3204804607, -STORE, 3204280320, 3204448255, -SNULL, 3204280320, 3204448255, -SNULL, 3204710399, 3204804607, -STORE, 3204448256, 3204710399, -STORE, 3204710400, 3204804607, -SNULL, 3204710400, 3204804607, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3996295168, 3996299263, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -SNULL, 3996295168, 3996299263, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3486908416, 3487039487, -STORE, 3486908416, 3487301631, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3223326720, 3290435583, -SNULL, 3223326720, 3256881151, -STORE, 3256881152, 3290435583, -STORE, 3223326720, 3256881151, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -STORE, 3201826816, 3202351103, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -STORE, 3202351104, 3204448255, -SNULL, 3202351104, 3204448255, -SNULL, 3803471871, 3803492351, -STORE, 3803451392, 3803471871, -STORE, 3803471872, 3803492351, -SNULL, 3803471872, 3803492351, -SNULL, 3803451392, 3803471871, -STORE, 3798999040, 3799101439, -SNULL, 3798999040, 3799101439, -STORE, 3952644096, 3952652287, -STORE, 3203362816, 3204448255, -SNULL, 3203362816, 3203366911, -STORE, 3203366912, 3204448255, -STORE, 3203362816, 3203366911, -SNULL, 3204444159, 3204448255, -STORE, 3203366912, 3204444159, -STORE, 3204444160, 3204448255, -STORE, 3803471872, 3803492351, -SNULL, 3803475967, 3803492351, -STORE, 3803471872, 3803475967, -STORE, 3803475968, 3803492351, -SNULL, 3203371007, 3204444159, -STORE, 3203366912, 3203371007, -STORE, 3203371008, 3204444159, -STORE, 3199729664, 3201826815, -SNULL, 3199729664, 3201826815, -STORE, 3199729664, 3201826815, -SNULL, 3199729664, 3201826815, -STORE, 3199729664, 3201826815, -SNULL, 3199729664, 3201826815, -STORE, 3199729664, 3201826815, -SNULL, 3199729664, 3201826815, -STORE, 3199729664, 3201826815, -SNULL, 3199729664, 3201826815, -STORE, 3200774144, 3201826815, -SNULL, 3200774144, 3200778239, -STORE, 3200778240, 3201826815, -STORE, 3200774144, 3200778239, -SNULL, 3201822719, 3201826815, -STORE, 3200778240, 3201822719, -STORE, 3201822720, 3201826815, -STORE, 3803451392, 3803471871, -SNULL, 3803455487, 3803471871, -STORE, 3803451392, 3803455487, -STORE, 3803455488, 3803471871, -SNULL, 3200782335, 3201822719, -STORE, 3200778240, 3200782335, -STORE, 3200782336, 3201822719, -STORE, 3949666304, 3949674495, -STORE, 3949408256, 3949416447, -STORE, 3199688704, 3200778239, -SNULL, 3199688704, 3199692799, -STORE, 3199692800, 3200778239, -STORE, 3199688704, 3199692799, -SNULL, 3200770047, 3200778239, -STORE, 3199692800, 3200770047, -STORE, 3200770048, 3200778239, -STORE, 3799306240, 3799326719, -SNULL, 3799310335, 3799326719, -STORE, 3799306240, 3799310335, -STORE, 3799310336, 3799326719, -SNULL, 3199696895, 3200770047, -STORE, 3199692800, 3199696895, -STORE, 3199696896, 3200770047, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -STORE, 3799277568, 3799306239, -SNULL, 3799277568, 3799306239, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -SNULL, 4041162751, 4041170943, -STORE, 4041154560, 4041162751, -STORE, 4041162752, 4041170943, -SNULL, 4041162752, 4041170943, -SNULL, 4041154560, 4041162751, -SNULL, 4041191424, 4041211903, -SNULL, 4041170944, 4041191423, -SNULL, 3626471423, 3626475519, -STORE, 3626467328, 3626471423, -STORE, 3626471424, 3626475519, -SNULL, 3626471424, 3627524095, -SNULL, 3625418751, 3625422847, -STORE, 3625414656, 3625418751, -STORE, 3625418752, 3625422847, -SNULL, 3625418752, 3626471423, -STORE, 3627393024, 3627524095, -STORE, 3627261952, 3627393023, -STORE, 3627261952, 3627524095, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -STORE, 3195494400, 3197591551, -SNULL, 3197591552, 3199688703, -SNULL, 3195494400, 3197591551, -STORE, 3197591552, 3199688703, -SNULL, 3197591552, 3199688703, -STORE, 3197591552, 3199688703, -STORE, 3195494400, 3197591551, -SNULL, 3197591552, 3199688703, -SNULL, 3195494400, 3197591551, -STORE, 3798999040, 3799101439, -SNULL, 3798999040, 3799101439, -/* - * mmap: unmapped_area_topdown: ffff9a9f14ddaa80 - * Gap was found: mt 4041162752 gap_end 4041183232 - * mmap: window was 4052029440 - 4096 size 28672 - * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423 - * mmap: mas.index 4041162752 align mask 0 offset 0 - * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678) - */ - }; - - unsigned long set43[] = { -STORE, 140737488347136, 140737488351231, -STORE, 140734187720704, 140737488351231, -SNULL, 140734187724800, 140737488351231, -STORE, 140734187589632, 140734187724799, -STORE, 4194304, 6443007, -STORE, 4337664, 6443007, -STORE, 4194304, 4337663, -SNULL, 4337664, 6443007, -STORE, 6430720, 6443007, -STORE, 206158430208, 206160674815, -STORE, 206158569472, 206160674815, -STORE, 206158430208, 206158569471, -SNULL, 206158569472, 206160674815, -STORE, 206160662528, 206160670719, -STORE, 206160670720, 206160674815, -STORE, 140734188756992, 140734188765183, -STORE, 140734188740608, 140734188756991, -STORE, 140501948112896, 140501948116991, - }; - - int count = 0; - void *ptr = NULL; - - MA_STATE(mas, mt, 0, 0); - - mt_set_non_kernel(3); - check_erase2_testset(mt, set, ARRAY_SIZE(set)); - mt_set_non_kernel(0); - mtree_destroy(mt); - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set2, ARRAY_SIZE(set2)); - start = 140735933894656; - MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL)); + /* Create tree of 1-100 */ + check_seq(mt, 100, false); + /* Store 45-168 */ + mt_set_non_kernel(10); + check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mt_set_non_kernel(2); - mt_init_flags(mt, 0); - check_erase2_testset(mt, set3, ARRAY_SIZE(set3)); - mt_set_non_kernel(0); + /* Create tree of 1-200 */ + check_seq(mt, 200, false); + /* Store 45-168 */ + check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mt_init_flags(mt, 0); - check_erase2_testset(mt, set4, ARRAY_SIZE(set4)); - rcu_read_lock(); - mas_for_each(&mas, entry, ULONG_MAX) { - if (xa_is_zero(entry)) - continue; - } - rcu_read_unlock(); - rcu_barrier(); + check_seq(mt, 30, false); + check_store_range(mt, 6, 18, xa_mk_value(6), 0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - mt_set_non_kernel(100); - check_erase2_testset(mt, set5, ARRAY_SIZE(set5)); - rcu_barrier(); + /* Overwrite across multiple levels. */ + /* Create tree of 1-400 */ + check_seq(mt, 400, false); + mt_set_non_kernel(50); + /* Store 118-128 */ + check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0); + mt_set_non_kernel(50); + mtree_test_erase(mt, 140); + mtree_test_erase(mt, 141); + mtree_test_erase(mt, 142); + mtree_test_erase(mt, 143); + mtree_test_erase(mt, 130); + mtree_test_erase(mt, 131); + mtree_test_erase(mt, 132); + mtree_test_erase(mt, 133); + mtree_test_erase(mt, 134); + mtree_test_erase(mt, 135); + check_load(mt, r[12], xa_mk_value(r[12])); + check_load(mt, r[13], xa_mk_value(r[12])); + check_load(mt, r[13] - 1, xa_mk_value(r[12])); + check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); + check_load(mt, 135, NULL); + check_load(mt, 140, NULL); mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set6, ARRAY_SIZE(set6)); - rcu_barrier(); - mtree_destroy(mt); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set7, ARRAY_SIZE(set7)); - rcu_barrier(); - mtree_destroy(mt); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set8, ARRAY_SIZE(set8)); - rcu_barrier(); - mtree_destroy(mt); + /* Overwrite multiple levels at the end of the tree (slot 7) */ + mt_set_non_kernel(50); + check_seq(mt, 400, false); + check_store_range(mt, 353, 361, xa_mk_value(353), 0); + check_store_range(mt, 347, 352, xa_mk_value(347), 0); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set9, ARRAY_SIZE(set9)); - rcu_barrier(); + check_load(mt, 346, xa_mk_value(346)); + for (i = 347; i <= 352; i++) + check_load(mt, i, xa_mk_value(347)); + for (i = 353; i <= 361; i++) + check_load(mt, i, xa_mk_value(353)); + check_load(mt, 362, xa_mk_value(362)); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set10, ARRAY_SIZE(set10)); - rcu_barrier(); + mt_set_non_kernel(50); + check_seq(mt, 400, false); + check_store_range(mt, 352, 364, NULL, 0); + check_store_range(mt, 351, 363, xa_mk_value(352), 0); + check_load(mt, 350, xa_mk_value(350)); + check_load(mt, 351, xa_mk_value(352)); + for (i = 352; i <= 363; i++) + check_load(mt, i, xa_mk_value(352)); + check_load(mt, 364, NULL); + check_load(mt, 365, xa_mk_value(365)); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set11, ARRAY_SIZE(set11)); - rcu_barrier(); - mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); - MT_BUG_ON(mt, mas.last != 140014592573439); + mt_set_non_kernel(5); + check_seq(mt, 400, false); + check_store_range(mt, 352, 364, NULL, 0); + check_store_range(mt, 351, 364, xa_mk_value(352), 0); + check_load(mt, 350, xa_mk_value(350)); + check_load(mt, 351, xa_mk_value(352)); + for (i = 352; i <= 364; i++) + check_load(mt, i, xa_mk_value(352)); + check_load(mt, 365, xa_mk_value(365)); + mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - mas_reset(&mas); - mas.tree = mt; - count = 0; - mas.index = 0; - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set12, ARRAY_SIZE(set12)); - rcu_barrier(); - mas_for_each(&mas, entry, ULONG_MAX) { - if (xa_is_zero(entry)) - continue; - BUG_ON(count > 12); - count++; - } - mtree_destroy(mt); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set13, ARRAY_SIZE(set13)); - mtree_erase(mt, 140373516443648); - rcu_read_lock(); - mas_empty_area_rev(&mas, 0, 140373518663680, 4096); - rcu_read_unlock(); - mtree_destroy(mt); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set14, ARRAY_SIZE(set14)); - rcu_barrier(); + mt_set_non_kernel(50); + check_seq(mt, 400, false); + check_store_range(mt, 362, 367, xa_mk_value(362), 0); + check_store_range(mt, 353, 361, xa_mk_value(353), 0); + mt_set_non_kernel(0); + mt_validate(mt); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); + /* + * Interesting cases: + * 1. Overwrite the end of a node and end in the first entry of the next + * node. + * 2. Split a single range + * 3. Overwrite the start of a range + * 4. Overwrite the end of a range + * 5. Overwrite the entire range + * 6. Overwrite a range that causes multiple parent nodes to be + * combined + * 7. Overwrite a range that causes multiple parent nodes and part of + * root to be combined + * 8. Overwrite the whole tree + * 9. Try to overwrite the zero entry of an alloc tree. + * 10. Write a range larger than a nodes current pivot + */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set15, ARRAY_SIZE(set15)); - rcu_barrier(); + mt_set_non_kernel(50); + for (i = 0; i <= 500; i++) { + val = i*5; + val2 = (i+1)*5; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0); + check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0); + check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0); + check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0); + check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0); mtree_destroy(mt); + mt_set_non_kernel(0); - /* set16 was to find a bug on limit updating at slot 0. */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set16, ARRAY_SIZE(set16)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); - MT_BUG_ON(mt, mas.last != 139921865547775); + mt_set_non_kernel(50); + for (i = 0; i <= 500; i++) { + val = i*5; + val2 = (i+1)*5; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0); + check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0); + check_store_range(mt, 2425, 2425, xa_mk_value(2), 0); + check_store_range(mt, 2460, 2470, NULL, 0); + check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0); + check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0); mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - /* - * set17 found a bug in walking backwards and not counting nulls at - * the end. This could cause a gap to be missed if the null had any - * size. - */ - mt_set_non_kernel(99); - mas_reset(&mas); + /* Test rebalance gaps */ mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set17, ARRAY_SIZE(set17)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); - MT_BUG_ON(mt, mas.last != 139953197322239); -/* MT_BUG_ON(mt, mas.index != 139953197318144); */ + mt_set_non_kernel(50); + for (i = 0; i <= 50; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 161, 161, xa_mk_value(161), 0); + check_store_range(mt, 162, 162, xa_mk_value(162), 0); + check_store_range(mt, 163, 163, xa_mk_value(163), 0); + check_store_range(mt, 240, 249, NULL, 0); + mtree_erase(mt, 200); + mtree_erase(mt, 210); + mtree_erase(mt, 220); + mtree_erase(mt, 230); mt_set_non_kernel(0); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); - /* - * set18 found a bug in walking backwards and not setting the max from - * the node, but using the parent node. This was only an issue if the - * next slot in the parent had what we needed. - */ - mt_set_non_kernel(99); - mas_reset(&mas); mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set18, ARRAY_SIZE(set18)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); - MT_BUG_ON(mt, mas.last != 140222968475647); - /*MT_BUG_ON(mt, mas.index != 140222966259712); */ - mt_set_non_kernel(0); - mtree_destroy(mt); - - /* - * set19 found 2 bugs in prev. - * 1. If we hit root without finding anything, then there was an - * infinite loop. - * 2. The first ascending wasn't using the correct slot which may have - * caused missed entries. - */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set19, ARRAY_SIZE(set19)); - rcu_barrier(); - mas.index = 140656779083776; - entry = mas_find(&mas, ULONG_MAX); - MT_BUG_ON(mt, entry != xa_mk_value(140656779083776)); - entry = mas_prev(&mas, 0); - MT_BUG_ON(mt, entry != xa_mk_value(140656766251008)); - mt_set_non_kernel(0); - mtree_destroy(mt); - - /* - * set20 found a bug in mas_may_move_gap due to the slot being - * overwritten during the __mas_add operation and setting it to zero. - */ - mt_set_non_kernel(99); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set20, ARRAY_SIZE(set20)); - rcu_barrier(); - check_load(mt, 94849009414144, NULL); - mt_set_non_kernel(0); - mtree_destroy(mt); - - mt_set_non_kernel(99); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set21, ARRAY_SIZE(set21)); - rcu_barrier(); - mt_validate(mt); - mt_set_non_kernel(0); - mtree_destroy(mt); - - mt_set_non_kernel(999); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set22, ARRAY_SIZE(set22)); - rcu_barrier(); - mt_validate(mt); - ptr = mtree_load(mt, 140551363362816); - MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159)); - mt_set_non_kernel(0); - mtree_destroy(mt); - - mt_set_non_kernel(99); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set23, ARRAY_SIZE(set23)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - - mt_set_non_kernel(99); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set24, ARRAY_SIZE(set24)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - mt_set_non_kernel(99); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set25, ARRAY_SIZE(set25)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* Split on NULL followed by delete - causes gap issues. */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set26, ARRAY_SIZE(set26)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); - MT_BUG_ON(mt, mas.last != 140109040959487); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* Split on NULL followed by delete - causes gap issues. */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set27, ARRAY_SIZE(set27)); - rcu_barrier(); - MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336)); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set28, ARRAY_SIZE(set28)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); - /* Search for the size of gap then align it (offset 0) */ - mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); - MT_BUG_ON(mt, mas.index != 139918401601536); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* This test found issues with retry moving rebalanced nodes so the - * incorrect parent pivot was updated. - */ - mt_set_non_kernel(999); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set29, ARRAY_SIZE(set29)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* This test found issues with deleting all entries in a node when - * surrounded by entries in the next nodes, then deleting the entries - * surrounding the node filled with deleted entries. - */ - mt_set_non_kernel(999); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set30, ARRAY_SIZE(set30)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* This test found an issue with deleting all entries in a node that was - * the end node and mas_gap incorrectly set next = curr, and curr = prev - * then moved next to the left, losing data. - */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set31, ARRAY_SIZE(set31)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set32, ARRAY_SIZE(set32)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - -/* - * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found: - * mt 140582827569152 gap_end 140582869532672 - * mmap: window was 140583656296448 - 4096 size 134217728 - * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879 - * mmap: mas.index 140582827569152 align mask 0 offset 0 - * mmap: rb_find_vma find on - * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00) - */ - - /* move gap failed due to an entirely empty node */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set33, ARRAY_SIZE(set33)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); - MT_BUG_ON(mt, mas.last != 140583003750399); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* - * Incorrect gap in tree caused by mas_prev not setting the limits - * correctly while walking down. - */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set34, ARRAY_SIZE(set34)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* Empty leaf at the end of a parent caused incorrect gap. */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set35, ARRAY_SIZE(set35)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - mt_set_non_kernel(99); - /* Empty leaf at the end of a parent caused incorrect gap. */ - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set36, ARRAY_SIZE(set36)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set37, ARRAY_SIZE(set37)); - rcu_barrier(); - MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); - mt_validate(mt); - mtree_destroy(mt); - - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set38, ARRAY_SIZE(set38)); - rcu_barrier(); - MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); - mt_validate(mt); - mtree_destroy(mt); - - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); - rcu_barrier(); - mt_validate(mt); - mtree_destroy(mt); - - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); - rcu_barrier(); - mt_validate(mt); - mtree_destroy(mt); - - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set41, ARRAY_SIZE(set41)); - rcu_barrier(); - mt_validate(mt); - mtree_destroy(mt); - - /* move gap failed due to an entirely empty node. */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set42, ARRAY_SIZE(set42)); - rcu_barrier(); - mas_empty_area_rev(&mas, 4096, 4052029440, 28672); - MT_BUG_ON(mt, mas.last != 4041211903); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); - - /* gap calc off by one */ - mt_set_non_kernel(99); - mas_reset(&mas); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_erase2_testset(mt, set43, ARRAY_SIZE(set43)); - rcu_barrier(); - mt_set_non_kernel(0); - mt_validate(mt); - mtree_destroy(mt); -} - -static noinline void check_alloc_rev_range(struct maple_tree *mt) -{ - /* - * Generated by: - * cat /proc/self/maps | awk '{print $1}'| - * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' - */ - - unsigned long range[] = { - /* Inclusive , Exclusive. */ - 0x565234af2000, 0x565234af4000, - 0x565234af4000, 0x565234af9000, - 0x565234af9000, 0x565234afb000, - 0x565234afc000, 0x565234afd000, - 0x565234afd000, 0x565234afe000, - 0x565235def000, 0x565235e10000, - 0x7f36d4bfd000, 0x7f36d4ee2000, - 0x7f36d4ee2000, 0x7f36d4f04000, - 0x7f36d4f04000, 0x7f36d504c000, - 0x7f36d504c000, 0x7f36d5098000, - 0x7f36d5098000, 0x7f36d5099000, - 0x7f36d5099000, 0x7f36d509d000, - 0x7f36d509d000, 0x7f36d509f000, - 0x7f36d509f000, 0x7f36d50a5000, - 0x7f36d50b9000, 0x7f36d50db000, - 0x7f36d50db000, 0x7f36d50dc000, - 0x7f36d50dc000, 0x7f36d50fa000, - 0x7f36d50fa000, 0x7f36d5102000, - 0x7f36d5102000, 0x7f36d5103000, - 0x7f36d5103000, 0x7f36d5104000, - 0x7f36d5104000, 0x7f36d5105000, - 0x7fff5876b000, 0x7fff5878d000, - 0x7fff5878e000, 0x7fff58791000, - 0x7fff58791000, 0x7fff58793000, - }; - - unsigned long holes[] = { - /* - * Note: start of hole is INCLUSIVE - * end of hole is EXCLUSIVE - * (opposite of the above table.) - * Start of hole, end of hole, size of hole (+1) - */ - 0x565234afb000, 0x565234afc000, 0x1000, - 0x565234afe000, 0x565235def000, 0x12F1000, - 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, - }; - - /* - * req_range consists of 4 values. - * 1. min index - * 2. max index - * 3. size - * 4. number that should be returned. - * 5. return value - */ - unsigned long req_range[] = { - 0x565234af9000, /* Min */ - 0x7fff58791000, /* Max */ - 0x1000, /* Size */ - 0x7fff5878d << 12, /* First rev hole of size 0x1000 */ - 0, /* Return value success. */ - - 0x0, /* Min */ - 0x565234AF1 << 12, /* Max */ - 0x3000, /* Size */ - 0x565234AEE << 12, /* max - 3. */ - 0, /* Return value success. */ - - 0x0, /* Min */ - -1, /* Max */ - 0x1000, /* Size */ - 562949953421311 << 12,/* First rev hole of size 0x1000 */ - 0, /* Return value success. */ - - 0x0, /* Min */ - 0x7F36D510A << 12, /* Max */ - 0x4000, /* Size */ - 0x7F36D5106 << 12, /* First rev hole of size 0x4000 */ - 0, /* Return value success. */ - - /* Ascend test. */ - 0x0, - 34148798629 << 12, - 19 << 12, - 34148797418 << 12, - 0x0, - - /* Too big test. */ - 0x0, - 18446744073709551615UL, - 562915594369134UL << 12, - 0x0, - -EBUSY, - - }; - - int i, range_count = ARRAY_SIZE(range); - int req_range_count = ARRAY_SIZE(req_range); - unsigned long min = 0; - - MA_STATE(mas, mt, 0, 0); - - mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, - GFP_KERNEL); -#define DEBUG_REV_RANGE 0 - for (i = 0; i < range_count; i += 2) { - /* Inclusive, Inclusive (with the -1) */ - -#if DEBUG_REV_RANGE - pr_debug("\t%s: Insert %lu-%lu\n", __func__, range[i] >> 12, - (range[i + 1] >> 12) - 1); -#endif - check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, - xa_mk_value(range[i] >> 12), 0); - mt_validate(mt); - } - - - for (i = 0; i < ARRAY_SIZE(holes); i += 3) { -#if DEBUG_REV_RANGE - pr_debug("Search from %lu-%lu for gap %lu should be at %lu\n", - min, holes[i+1]>>12, holes[i+2]>>12, - holes[i] >> 12); -#endif - MT_BUG_ON(mt, mas_empty_area_rev(&mas, min, - holes[i+1] >> 12, - holes[i+2] >> 12)); -#if DEBUG_REV_RANGE - pr_debug("Found %lu %lu\n", mas.index, mas.last); - pr_debug("gap %lu %lu\n", (holes[i] >> 12), - (holes[i+1] >> 12)); -#endif - MT_BUG_ON(mt, mas.last + 1 != (holes[i+1] >> 12)); - MT_BUG_ON(mt, mas.index != (holes[i+1] >> 12) - (holes[i+2] >> 12)); - min = holes[i+1] >> 12; - mas_reset(&mas); - } - - for (i = 0; i < req_range_count; i += 5) { -#if DEBUG_REV_RANGE - pr_debug("\tReverse request between %lu-%lu size %lu, should get %lu\n", - req_range[i] >> 12, - (req_range[i + 1] >> 12) - 1, - req_range[i+2] >> 12, - req_range[i+3] >> 12); -#endif - check_mtree_alloc_rrange(mt, - req_range[i] >> 12, /* start */ - req_range[i+1] >> 12, /* end */ - req_range[i+2] >> 12, /* size */ - req_range[i+3] >> 12, /* expected address */ - req_range[i+4], /* expected return */ - xa_mk_value(req_range[i] >> 12)); /* pointer */ - mt_validate(mt); - } - - mt_set_non_kernel(1); - mtree_erase(mt, 34148798727); /* create a deleted range. */ - check_mtree_alloc_rrange(mt, 0, 34359052173, 210253414, - 34148798725, 0, mt); - - mtree_destroy(mt); -} - -static noinline void check_alloc_range(struct maple_tree *mt) -{ - /* - * Generated by: - * cat /proc/self/maps|awk '{print $1}'| - * awk -F "-" '{printf "0x%s, 0x%s, ", $1, $2}' - */ - - unsigned long range[] = { - /* Inclusive , Exclusive. */ - 0x565234af2000, 0x565234af4000, - 0x565234af4000, 0x565234af9000, - 0x565234af9000, 0x565234afb000, - 0x565234afc000, 0x565234afd000, - 0x565234afd000, 0x565234afe000, - 0x565235def000, 0x565235e10000, - 0x7f36d4bfd000, 0x7f36d4ee2000, - 0x7f36d4ee2000, 0x7f36d4f04000, - 0x7f36d4f04000, 0x7f36d504c000, - 0x7f36d504c000, 0x7f36d5098000, - 0x7f36d5098000, 0x7f36d5099000, - 0x7f36d5099000, 0x7f36d509d000, - 0x7f36d509d000, 0x7f36d509f000, - 0x7f36d509f000, 0x7f36d50a5000, - 0x7f36d50b9000, 0x7f36d50db000, - 0x7f36d50db000, 0x7f36d50dc000, - 0x7f36d50dc000, 0x7f36d50fa000, - 0x7f36d50fa000, 0x7f36d5102000, - 0x7f36d5102000, 0x7f36d5103000, - 0x7f36d5103000, 0x7f36d5104000, - 0x7f36d5104000, 0x7f36d5105000, - 0x7fff5876b000, 0x7fff5878d000, - 0x7fff5878e000, 0x7fff58791000, - 0x7fff58791000, 0x7fff58793000, - }; - unsigned long holes[] = { - /* Start of hole, end of hole, size of hole (+1) */ - 0x565234afb000, 0x565234afc000, 0x1000, - 0x565234afe000, 0x565235def000, 0x12F1000, - 0x565235e10000, 0x7f36d4bfd000, 0x28E49EDED000, - }; - - /* - * req_range consists of 4 values. - * 1. min index - * 2. max index - * 3. size - * 4. number that should be returned. - * 5. return value - */ - unsigned long req_range[] = { - 0x565234af9000, /* Min */ - 0x7fff58791000, /* Max */ - 0x1000, /* Size */ - 0x565234afb000, /* First hole in our data of size 1000. */ - 0, /* Return value success. */ - - 0x0, /* Min */ - 0x7fff58791000, /* Max */ - 0x1F00, /* Size */ - 0x0, /* First hole in our data of size 2000. */ - 0, /* Return value success. */ - - /* Test ascend. */ - 34148797436 << 12, /* Min */ - 0x7fff587AF000, /* Max */ - 0x3000, /* Size */ - 34148798629 << 12, /* Expected location */ - 0, /* Return value success. */ - - /* Test failing. */ - 34148798623 << 12, /* Min */ - 34148798683 << 12, /* Max */ - 0x15000, /* Size */ - 0, /* Expected location */ - -EBUSY, /* Return value failed. */ - - /* Test filling entire gap. */ - 34148798623 << 12, /* Min */ - 0x7fff587AF000, /* Max */ - 0x10000, /* Size */ - 34148798632 << 12, /* Expected location */ - 0, /* Return value success. */ - - /* Test walking off the end of root. */ - 0, /* Min */ - -1, /* Max */ - -1, /* Size */ - 0, /* Expected location */ - -EBUSY, /* Return value failure. */ - - /* Test looking for too large a hole across entire range. */ - 0, /* Min */ - -1, /* Max */ - 4503599618982063UL << 12, /* Size */ - 34359052178 << 12, /* Expected location */ - -EBUSY, /* Return failure. */ - }; - int i, range_count = ARRAY_SIZE(range); - int req_range_count = ARRAY_SIZE(req_range); - unsigned long min = 0x565234af2000; - - mtree_store_range(mt, MTREE_ALLOC_MAX, ULONG_MAX, XA_ZERO_ENTRY, - GFP_KERNEL); - for (i = 0; i < range_count; i += 2) { -#define DEBUG_ALLOC_RANGE 0 -#if DEBUG_ALLOC_RANGE - pr_debug("\tInsert %lu-%lu\n", range[i] >> 12, - (range[i + 1] >> 12) - 1); - mt_dump(mt); -#endif - check_insert_range(mt, range[i] >> 12, (range[i + 1] >> 12) - 1, - xa_mk_value(range[i] >> 12), 0); - mt_validate(mt); - } - - - MA_STATE(mas, mt, 0, 0); - - for (i = 0; i < ARRAY_SIZE(holes); i += 3) { - -#if DEBUG_ALLOC_RANGE - pr_debug("\tGet empty %lu-%lu size %lu (%lx-%lx)\n", min >> 12, - holes[i+1] >> 12, holes[i+2] >> 12, - min, holes[i+1]); -#endif - MT_BUG_ON(mt, mas_empty_area(&mas, min >> 12, - holes[i+1] >> 12, - holes[i+2] >> 12)); - MT_BUG_ON(mt, mas.index != holes[i] >> 12); - min = holes[i+1]; - mas_reset(&mas); - } - for (i = 0; i < req_range_count; i += 5) { -#if DEBUG_ALLOC_RANGE - pr_debug("\tTest %d: %lu-%lu size %lu expected %lu (%lu-%lu)\n", - i/5, req_range[i] >> 12, req_range[i + 1] >> 12, - req_range[i + 2] >> 12, req_range[i + 3] >> 12, - req_range[i], req_range[i+1]); -#endif - check_mtree_alloc_range(mt, - req_range[i] >> 12, /* start */ - req_range[i+1] >> 12, /* end */ - req_range[i+2] >> 12, /* size */ - req_range[i+3] >> 12, /* expected address */ - req_range[i+4], /* expected return */ - xa_mk_value(req_range[i] >> 12)); /* pointer */ - mt_validate(mt); -#if DEBUG_ALLOC_RANGE - mt_dump(mt); -#endif - } - - mtree_destroy(mt); -} - -static noinline void check_ranges(struct maple_tree *mt) -{ - int i, val, val2; - unsigned long r[] = { - 10, 15, - 20, 25, - 17, 22, /* Overlaps previous range. */ - 9, 1000, /* Huge. */ - 100, 200, - 45, 168, - 118, 128, - }; - - MT_BUG_ON(mt, !mtree_empty(mt)); - check_insert_range(mt, r[0], r[1], xa_mk_value(r[0]), 0); - check_insert_range(mt, r[2], r[3], xa_mk_value(r[2]), 0); - check_insert_range(mt, r[4], r[5], xa_mk_value(r[4]), -EEXIST); - MT_BUG_ON(mt, !mt_height(mt)); - /* Store */ - check_store_range(mt, r[4], r[5], xa_mk_value(r[4]), 0); - check_store_range(mt, r[6], r[7], xa_mk_value(r[6]), 0); - check_store_range(mt, r[8], r[9], xa_mk_value(r[8]), 0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - MT_BUG_ON(mt, mt_height(mt)); - - check_seq(mt, 50, false); - mt_set_non_kernel(4); - check_store_range(mt, 5, 47, xa_mk_value(47), 0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - /* Create tree of 1-100 */ - check_seq(mt, 100, false); - /* Store 45-168 */ - mt_set_non_kernel(10); - check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - /* Create tree of 1-200 */ - check_seq(mt, 200, false); - /* Store 45-168 */ - check_store_range(mt, r[10], r[11], xa_mk_value(r[10]), 0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - check_seq(mt, 30, false); - check_store_range(mt, 6, 18, xa_mk_value(6), 0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - /* Overwrite across multiple levels. */ - /* Create tree of 1-400 */ - check_seq(mt, 400, false); - mt_set_non_kernel(50); - /* Store 118-128 */ - check_store_range(mt, r[12], r[13], xa_mk_value(r[12]), 0); - mt_set_non_kernel(50); - mtree_test_erase(mt, 140); - mtree_test_erase(mt, 141); - mtree_test_erase(mt, 142); - mtree_test_erase(mt, 143); - mtree_test_erase(mt, 130); - mtree_test_erase(mt, 131); - mtree_test_erase(mt, 132); - mtree_test_erase(mt, 133); - mtree_test_erase(mt, 134); - mtree_test_erase(mt, 135); - check_load(mt, r[12], xa_mk_value(r[12])); - check_load(mt, r[13], xa_mk_value(r[12])); - check_load(mt, r[13] - 1, xa_mk_value(r[12])); - check_load(mt, r[13] + 1, xa_mk_value(r[13] + 1)); - check_load(mt, 135, NULL); - check_load(mt, 140, NULL); - mt_set_non_kernel(0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - - - /* Overwrite multiple levels at the end of the tree (slot 7) */ - mt_set_non_kernel(50); - check_seq(mt, 400, false); - check_store_range(mt, 353, 361, xa_mk_value(353), 0); - check_store_range(mt, 347, 352, xa_mk_value(347), 0); - - check_load(mt, 346, xa_mk_value(346)); - for (i = 347; i <= 352; i++) - check_load(mt, i, xa_mk_value(347)); - for (i = 353; i <= 361; i++) - check_load(mt, i, xa_mk_value(353)); - check_load(mt, 362, xa_mk_value(362)); - mt_set_non_kernel(0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - mt_set_non_kernel(50); - check_seq(mt, 400, false); - check_store_range(mt, 352, 364, NULL, 0); - check_store_range(mt, 351, 363, xa_mk_value(352), 0); - check_load(mt, 350, xa_mk_value(350)); - check_load(mt, 351, xa_mk_value(352)); - for (i = 352; i <= 363; i++) - check_load(mt, i, xa_mk_value(352)); - check_load(mt, 364, NULL); - check_load(mt, 365, xa_mk_value(365)); - mt_set_non_kernel(0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - mt_set_non_kernel(5); - check_seq(mt, 400, false); - check_store_range(mt, 352, 364, NULL, 0); - check_store_range(mt, 351, 364, xa_mk_value(352), 0); - check_load(mt, 350, xa_mk_value(350)); - check_load(mt, 351, xa_mk_value(352)); - for (i = 352; i <= 364; i++) - check_load(mt, i, xa_mk_value(352)); - check_load(mt, 365, xa_mk_value(365)); - mt_set_non_kernel(0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - - mt_set_non_kernel(50); - check_seq(mt, 400, false); - check_store_range(mt, 362, 367, xa_mk_value(362), 0); - check_store_range(mt, 353, 361, xa_mk_value(353), 0); - mt_set_non_kernel(0); - mt_validate(mt); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - /* - * Interesting cases: - * 1. Overwrite the end of a node and end in the first entry of the next - * node. - * 2. Split a single range - * 3. Overwrite the start of a range - * 4. Overwrite the end of a range - * 5. Overwrite the entire range - * 6. Overwrite a range that causes multiple parent nodes to be - * combined - * 7. Overwrite a range that causes multiple parent nodes and part of - * root to be combined - * 8. Overwrite the whole tree - * 9. Try to overwrite the zero entry of an alloc tree. - * 10. Write a range larger than a nodes current pivot - */ - - mt_set_non_kernel(50); - for (i = 0; i <= 500; i++) { - val = i*5; - val2 = (i+1)*5; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - } - check_store_range(mt, 2400, 2400, xa_mk_value(2400), 0); - check_store_range(mt, 2411, 2411, xa_mk_value(2411), 0); - check_store_range(mt, 2412, 2412, xa_mk_value(2412), 0); - check_store_range(mt, 2396, 2400, xa_mk_value(4052020), 0); - check_store_range(mt, 2402, 2402, xa_mk_value(2402), 0); - mtree_destroy(mt); - mt_set_non_kernel(0); - - mt_set_non_kernel(50); - for (i = 0; i <= 500; i++) { - val = i*5; - val2 = (i+1)*5; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - } - check_store_range(mt, 2422, 2422, xa_mk_value(2422), 0); - check_store_range(mt, 2424, 2424, xa_mk_value(2424), 0); - check_store_range(mt, 2425, 2425, xa_mk_value(2), 0); - check_store_range(mt, 2460, 2470, NULL, 0); - check_store_range(mt, 2435, 2460, xa_mk_value(2435), 0); - check_store_range(mt, 2461, 2470, xa_mk_value(2461), 0); - mt_set_non_kernel(0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - /* Test rebalance gaps */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - mt_set_non_kernel(50); - for (i = 0; i <= 50; i++) { - val = i*10; - val2 = (i+1)*10; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - } - check_store_range(mt, 161, 161, xa_mk_value(161), 0); - check_store_range(mt, 162, 162, xa_mk_value(162), 0); - check_store_range(mt, 163, 163, xa_mk_value(163), 0); - check_store_range(mt, 240, 249, NULL, 0); - mtree_erase(mt, 200); - mtree_erase(mt, 210); - mtree_erase(mt, 220); - mtree_erase(mt, 230); - mt_set_non_kernel(0); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= 500; i++) { - val = i*10; - val2 = (i+1)*10; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - } - check_store_range(mt, 4600, 4959, xa_mk_value(1), 0); - mt_validate(mt); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= 500; i++) { - val = i*10; - val2 = (i+1)*10; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - } - check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0); - check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0); - check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0); - check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0); - check_store_range(mt, 4842, 4849, NULL, 0); - mt_validate(mt); - MT_BUG_ON(mt, !mt_height(mt)); - mtree_destroy(mt); - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= 1300; i++) { - val = i*10; - val2 = (i+1)*10; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - MT_BUG_ON(mt, mt_height(mt) >= 4); - } - /* Cause a 3 child split all the way up the tree. */ - for (i = 5; i < 215; i += 10) - check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); - for (i = 5; i < 65; i += 10) - check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); - - MT_BUG_ON(mt, mt_height(mt) >= 4); - for (i = 5; i < 45; i += 10) - check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); - MT_BUG_ON(mt, mt_height(mt) < 4); - mtree_destroy(mt); - - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= 1200; i++) { - val = i*10; - val2 = (i+1)*10; - check_store_range(mt, val, val2, xa_mk_value(val), 0); - MT_BUG_ON(mt, mt_height(mt) >= 4); - } - /* Fill parents and leaves before split. */ - for (i = 5; i < 455; i += 10) - check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); - - for (i = 1; i < 16; i++) - check_store_range(mt, 8185 + i, 8185 + i + 1, - xa_mk_value(8185+i), 0); - MT_BUG_ON(mt, mt_height(mt) >= 4); - /* triple split across multiple levels. */ - check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); - MT_BUG_ON(mt, mt_height(mt) != 4); -} - -static noinline void check_next_entry(struct maple_tree *mt) -{ - void *entry = NULL; - unsigned long limit = 30, i = 0; - - MT_BUG_ON(mt, !mtree_empty(mt)); - MA_STATE(mas, mt, i, i); - - check_seq(mt, limit, false); - rcu_read_lock(); - - /* Check the first one and get ma_state in the correct state. */ - MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++)); - for ( ; i <= limit + 1; i++) { - entry = mas_next(&mas, limit); - if (i > limit) - MT_BUG_ON(mt, entry != NULL); - else - MT_BUG_ON(mt, xa_mk_value(i) != entry); - } - rcu_read_unlock(); - mtree_destroy(mt); -} - -static noinline void check_prev_entry(struct maple_tree *mt) -{ - unsigned long index = 16; - void *value; - int i; - - MA_STATE(mas, mt, index, index); - - MT_BUG_ON(mt, !mtree_empty(mt)); - check_seq(mt, 30, false); - - rcu_read_lock(); - value = mas_find(&mas, ULONG_MAX); - MT_BUG_ON(mt, value != xa_mk_value(index)); - value = mas_prev(&mas, 0); - MT_BUG_ON(mt, value != xa_mk_value(index - 1)); - rcu_read_unlock(); - mtree_destroy(mt); - - /* Check limits on prev */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - mas_lock(&mas); - for (i = 0; i <= index; i++) { - mas_set_range(&mas, i*10, i*10+5); - mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL); - } - - mas_set(&mas, 20); - value = mas_walk(&mas); - MT_BUG_ON(mt, value != xa_mk_value(2)); - - value = mas_prev(&mas, 19); - MT_BUG_ON(mt, value != NULL); - - mas_set(&mas, 80); - value = mas_walk(&mas); - MT_BUG_ON(mt, value != xa_mk_value(8)); - - value = mas_prev(&mas, 76); - MT_BUG_ON(mt, value != NULL); - - mas_unlock(&mas); -} - -static noinline void check_root_expand(struct maple_tree *mt) -{ - MA_STATE(mas, mt, 0, 0); - void *ptr; - - - mas_lock(&mas); - mas_set(&mas, 3); - ptr = mas_walk(&mas); - MT_BUG_ON(mt, ptr != NULL); - MT_BUG_ON(mt, mas.index != 0); - MT_BUG_ON(mt, mas.last != ULONG_MAX); - - ptr = &check_prev_entry; - mas_set(&mas, 1); - mas_store_gfp(&mas, ptr, GFP_KERNEL); - - mas_set(&mas, 0); - ptr = mas_walk(&mas); - MT_BUG_ON(mt, ptr != NULL); - - mas_set(&mas, 1); - ptr = mas_walk(&mas); - MT_BUG_ON(mt, ptr != &check_prev_entry); - - mas_set(&mas, 2); - ptr = mas_walk(&mas); - MT_BUG_ON(mt, ptr != NULL); - mas_unlock(&mas); - mtree_destroy(mt); - - - mt_init_flags(mt, 0); - mas_lock(&mas); - - mas_set(&mas, 0); - ptr = &check_prev_entry; - mas_store_gfp(&mas, ptr, GFP_KERNEL); - - mas_set(&mas, 5); - ptr = mas_walk(&mas); - MT_BUG_ON(mt, ptr != NULL); - MT_BUG_ON(mt, mas.index != 1); - MT_BUG_ON(mt, mas.last != ULONG_MAX); - - mas_set_range(&mas, 0, 100); - ptr = mas_walk(&mas); - MT_BUG_ON(mt, ptr != &check_prev_entry); - MT_BUG_ON(mt, mas.last != 0); - mas_unlock(&mas); - mtree_destroy(mt); - - mt_init_flags(mt, 0); - mas_lock(&mas); - - mas_set(&mas, 0); - ptr = (void *)((unsigned long) check_prev_entry | 1UL); - mas_store_gfp(&mas, ptr, GFP_KERNEL); - ptr = mas_next(&mas, ULONG_MAX); - MT_BUG_ON(mt, ptr != NULL); - MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); - - mas_set(&mas, 1); - ptr = mas_prev(&mas, 0); - MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); - MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL)); - - mas_unlock(&mas); - - mtree_destroy(mt); - - mt_init_flags(mt, 0); - mas_lock(&mas); - mas_set(&mas, 0); - ptr = (void *)((unsigned long) check_prev_entry | 2UL); - mas_store_gfp(&mas, ptr, GFP_KERNEL); - ptr = mas_next(&mas, ULONG_MAX); - MT_BUG_ON(mt, ptr != NULL); - MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); - - mas_set(&mas, 1); - ptr = mas_prev(&mas, 0); - MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); - MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL)); - - - mas_unlock(&mas); -} - -static noinline void check_prealloc(struct maple_tree *mt) -{ - unsigned long i, max = 100; - unsigned long allocated; - unsigned char height; - struct maple_node *mn; - void *ptr = check_prealloc; - MA_STATE(mas, mt, 10, 20); - - mt_set_non_kernel(1000); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mas_destroy(&mas); - allocated = mas_allocated(&mas); - MT_BUG_ON(mt, allocated != 0); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - mas_destroy(&mas); - allocated = mas_allocated(&mas); - MT_BUG_ON(mt, allocated != 0); - - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); - ma_free_rcu(mn); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - mas_destroy(&mas); - allocated = mas_allocated(&mas); - MT_BUG_ON(mt, allocated != 0); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - mas_destroy(&mas); - allocated = mas_allocated(&mas); - MT_BUG_ON(mt, allocated != 0); - ma_free_rcu(mn); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mn = mas_pop_node(&mas); - MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); - mas_push_node(&mas, mn); - MT_BUG_ON(mt, mas_allocated(&mas) != allocated); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - mas_destroy(&mas); - allocated = mas_allocated(&mas); - MT_BUG_ON(mt, allocated != 0); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mas_store_prealloc(&mas, ptr); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mas_store_prealloc(&mas, ptr); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mas_store_prealloc(&mas, ptr); - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mas_store_prealloc(&mas, ptr); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - mt_set_non_kernel(1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated != 0); - mas_destroy(&mas); - - - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated == 0); - MT_BUG_ON(mt, allocated != 1 + height * 3); - mas_store_prealloc(&mas, ptr); - MT_BUG_ON(mt, mas_allocated(&mas) != 0); - mt_set_non_kernel(1); - MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); - allocated = mas_allocated(&mas); - height = mas_mt_height(&mas); - MT_BUG_ON(mt, allocated != 0); -} - -static noinline void check_spanning_write(struct maple_tree *mt) -{ - unsigned long i, max = 5000; - MA_STATE(mas, mt, 1200, 2380); - - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 1205); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - for (i = 1; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mtree_lock(mt); - mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 1205); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mt_validate(mt); - mtree_destroy(mt); - - /* Test spanning store that requires a right cousin rebalance */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 1205); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* Test non-alloc tree spanning store */ - mt_init_flags(mt, 0); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mas_set_range(&mas, 0, 300); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 15); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* Test spanning store that requires a right sibling rebalance */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mas_set_range(&mas, 0, 12865); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 15); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* Test spanning store that requires a left sibling rebalance */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mas_set_range(&mas, 90, 13665); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 95); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* Test spanning store that requires a left cousin rebalance */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mas_set_range(&mas, 46805, 49995); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 46815); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* - * Test spanning store that requires a left cousin rebalance all the way - * to root - */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mas_set_range(&mas, 32395, 49995); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 46815); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* - * Test spanning store that requires a right cousin rebalance all the - * way to root - */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - mas_set_range(&mas, 38875, 43190); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 38900); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* Test spanning store ending at full node (depth 2)*/ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - mtree_lock(mt); - mas_set(&mas, 47606); - mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); - mas_set(&mas, 47607); - mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); - mas_set(&mas, 47608); - mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); - mas_set(&mas, 47609); - mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); - /* Ensure the parent node is full */ - mas_ascend(&mas); - MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); - mas_set_range(&mas, 11516, 48940); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mtree_unlock(mt); - mtree_destroy(mt); - - /* Test spanning write with many levels of no siblings */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - mas_set_range(&mas, 43200, 49999); - mtree_lock(mt); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mas_set(&mas, 43200); - MT_BUG_ON(mt, mas_walk(&mas) != NULL); - mtree_unlock(mt); - mtree_destroy(mt); - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= 100; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - - mtree_lock(mt); - mas_set_range(&mas, 76, 875); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - mtree_unlock(mt); -} - -static noinline void check_null_expand(struct maple_tree *mt) -{ - unsigned long i, max = 100; - unsigned char data_end; - MA_STATE(mas, mt, 959, 959); - - for (i = 0; i <= max; i++) - mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); - /* Test expanding null at start. */ - mas_walk(&mas); - data_end = mas_data_end(&mas); - mas_set_range(&mas, 959, 963); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - MT_BUG_ON(mt, mtree_load(mt, 963) != NULL); - MT_BUG_ON(mt, data_end != mas_data_end(&mas)); - - /* Test expanding null at end. */ - mas_set(&mas, 880); - mas_walk(&mas); - data_end = mas_data_end(&mas); - mas_set_range(&mas, 884, 887); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - MT_BUG_ON(mt, mtree_load(mt, 884) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 889) != NULL); - MT_BUG_ON(mt, data_end != mas_data_end(&mas)); - - /* Test expanding null at start and end. */ - mas_set(&mas, 890); - mas_walk(&mas); - data_end = mas_data_end(&mas); - mas_set_range(&mas, 900, 905); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - MT_BUG_ON(mt, mtree_load(mt, 899) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 900) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 905) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 906) != NULL); - MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); - - /* Test expanding null across multiple slots. */ - mas_set(&mas, 800); - mas_walk(&mas); - data_end = mas_data_end(&mas); - mas_set_range(&mas, 810, 825); - mas_store_gfp(&mas, NULL, GFP_KERNEL); - MT_BUG_ON(mt, mtree_load(mt, 809) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 810) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 825) != NULL); - MT_BUG_ON(mt, mtree_load(mt, 826) != NULL); - MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); -} - -static noinline void check_gap_combining(struct maple_tree *mt) -{ - struct maple_enode *mn1, *mn2; - void *entry; - - unsigned long seq100[] = { - /* 0-5 */ - 74, 75, 76, - 50, 100, 2, - - /* 6-12 */ - 44, 45, 46, 43, - 20, 50, 3, - - /* 13-20*/ - 80, 81, 82, - 76, 2, 79, 85, 4, - }; - unsigned long seq2000[] = { - 1152, 1151, - 1100, 1200, 2, - }; - unsigned long seq400[] = { - 286, 318, - 256, 260, 266, 270, 275, 280, 290, 398, - 286, 310, - }; - - unsigned long index = seq100[0]; - - MA_STATE(mas, mt, index, index); - - MT_BUG_ON(mt, !mtree_empty(mt)); - check_seq(mt, 100, false); /* create 100 singletons. */ - - mt_set_non_kernel(1); - mtree_test_erase(mt, seq100[2]); - check_load(mt, seq100[2], NULL); - mtree_test_erase(mt, seq100[1]); - check_load(mt, seq100[1], NULL); - - rcu_read_lock(); - entry = mas_find(&mas, ULONG_MAX); - MT_BUG_ON(mt, entry != xa_mk_value(index)); - mn1 = mas.node; - mas_next(&mas, ULONG_MAX); - entry = mas_next(&mas, ULONG_MAX); - MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); - mn2 = mas.node; - MT_BUG_ON(mt, mn1 == mn2); /* test the test. */ - - /* - * At this point, there is a gap of 2 at index + 1 between seq100[3] and - * seq100[4]. Search for the gap. - */ - mt_set_non_kernel(1); - mas_reset(&mas); - MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4], - seq100[5])); - MT_BUG_ON(mt, mas.index != index + 1); - rcu_read_unlock(); - - mtree_test_erase(mt, seq100[6]); - check_load(mt, seq100[6], NULL); - mtree_test_erase(mt, seq100[7]); - check_load(mt, seq100[7], NULL); - mtree_test_erase(mt, seq100[8]); - index = seq100[9]; - - rcu_read_lock(); - mas.index = index; - mas.last = index; - mas_reset(&mas); - entry = mas_find(&mas, ULONG_MAX); - MT_BUG_ON(mt, entry != xa_mk_value(index)); - mn1 = mas.node; - entry = mas_next(&mas, ULONG_MAX); - MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); - mas_next(&mas, ULONG_MAX); /* go to the next entry. */ - mn2 = mas.node; - MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */ - - /* - * At this point, there is a gap of 3 at seq100[6]. Find it by - * searching 20 - 50 for size 3. - */ - mas_reset(&mas); - MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11], - seq100[12])); - MT_BUG_ON(mt, mas.index != seq100[6]); - rcu_read_unlock(); - - mt_set_non_kernel(1); - mtree_store(mt, seq100[13], NULL, GFP_KERNEL); - check_load(mt, seq100[13], NULL); - check_load(mt, seq100[14], xa_mk_value(seq100[14])); - mtree_store(mt, seq100[14], NULL, GFP_KERNEL); - check_load(mt, seq100[13], NULL); - check_load(mt, seq100[14], NULL); - - mas_reset(&mas); - rcu_read_lock(); - MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15], - seq100[17])); - MT_BUG_ON(mt, mas.index != seq100[13]); - mt_validate(mt); - rcu_read_unlock(); - - /* - * *DEPRECATED: no retries anymore* Test retry entry in the start of a - * gap. - */ - mt_set_non_kernel(2); - mtree_test_store_range(mt, seq100[18], seq100[14], NULL); - mtree_test_erase(mt, seq100[15]); - mas_reset(&mas); - rcu_read_lock(); - MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19], - seq100[20])); - rcu_read_unlock(); - MT_BUG_ON(mt, mas.index != seq100[18]); - mt_validate(mt); - mtree_destroy(mt); - - /* seq 2000 tests are for multi-level tree gaps */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_seq(mt, 2000, false); - mt_set_non_kernel(1); - mtree_test_erase(mt, seq2000[0]); - mtree_test_erase(mt, seq2000[1]); - - mt_set_non_kernel(2); - mas_reset(&mas); - rcu_read_lock(); - MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3], - seq2000[4])); - MT_BUG_ON(mt, mas.index != seq2000[1]); - rcu_read_unlock(); - mt_validate(mt); - mtree_destroy(mt); - - /* seq 400 tests rebalancing over two levels. */ - mt_set_non_kernel(99); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_seq(mt, 400, false); - mtree_test_store_range(mt, seq400[0], seq400[1], NULL); - mt_set_non_kernel(0); + for (i = 0; i <= 500; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + } + check_store_range(mt, 4600, 4959, xa_mk_value(1), 0); + mt_validate(mt); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - check_seq(mt, 400, false); - mt_set_non_kernel(50); - mtree_test_store_range(mt, seq400[2], seq400[9], - xa_mk_value(seq400[2])); - mtree_test_store_range(mt, seq400[3], seq400[9], - xa_mk_value(seq400[3])); - mtree_test_store_range(mt, seq400[4], seq400[9], - xa_mk_value(seq400[4])); - mtree_test_store_range(mt, seq400[5], seq400[9], - xa_mk_value(seq400[5])); - mtree_test_store_range(mt, seq400[0], seq400[9], - xa_mk_value(seq400[0])); - mtree_test_store_range(mt, seq400[6], seq400[9], - xa_mk_value(seq400[6])); - mtree_test_store_range(mt, seq400[7], seq400[9], - xa_mk_value(seq400[7])); - mtree_test_store_range(mt, seq400[8], seq400[9], - xa_mk_value(seq400[8])); - mtree_test_store_range(mt, seq400[10], seq400[11], - xa_mk_value(seq400[10])); - mt_validate(mt); - mt_set_non_kernel(0); - mtree_destroy(mt); -} -static noinline void check_node_overwrite(struct maple_tree *mt) -{ - int i, max = 4000; - - for (i = 0; i < max; i++) - mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100)); - - mtree_test_store_range(mt, 319951, 367950, NULL); - /*mt_dump(mt); */ - mt_validate(mt); -} - -static void mas_dfs_preorder(struct ma_state *mas) -{ - - struct maple_enode *prev; - unsigned char end, slot = 0; - - if (mas_is_start(mas)) { - mas_start(mas); - return; - } - - if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) - goto done; - -walk_up: - end = mas_data_end(mas); - if (mte_is_leaf(mas->node) || - (slot > end)) { - if (mte_is_root(mas->node)) - goto done; - - slot = mte_parent_slot(mas->node) + 1; - mas_ascend(mas); - goto walk_up; - } - - prev = mas->node; - mas->node = mas_get_slot(mas, slot); - if (!mas->node || slot > end) { - if (mte_is_root(prev)) - goto done; - - mas->node = prev; - slot = mte_parent_slot(mas->node) + 1; - mas_ascend(mas); - goto walk_up; + for (i = 0; i <= 500; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); } - - return; -done: - mas->node = MAS_NONE; -} - - -static void check_dfs_preorder(struct maple_tree *mt) -{ - unsigned long count = 0, max = 1000; - - MA_STATE(mas, mt, 0, 0); - - check_seq(mt, max, false); - do { - count++; - mas_dfs_preorder(&mas); - } while (!mas_is_none(&mas)); - MT_BUG_ON(mt, count != 74); + check_store_range(mt, 4811, 4811, xa_mk_value(4811), 0); + check_store_range(mt, 4812, 4812, xa_mk_value(4812), 0); + check_store_range(mt, 4861, 4861, xa_mk_value(4861), 0); + check_store_range(mt, 4862, 4862, xa_mk_value(4862), 0); + check_store_range(mt, 4842, 4849, NULL, 0); + mt_validate(mt); + MT_BUG_ON(mt, !mt_height(mt)); mtree_destroy(mt); mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - mas_reset(&mas); - count = 0; - check_seq(mt, max, false); - do { - count++; - mas_dfs_preorder(&mas); - } while (!mas_is_none(&mas)); - /*printk("count %lu\n", count); */ - MT_BUG_ON(mt, count != 77); - mtree_destroy(mt); + for (i = 0; i <= 1300; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + MT_BUG_ON(mt, mt_height(mt) >= 4); + } + /* Cause a 3 child split all the way up the tree. */ + for (i = 5; i < 215; i += 10) + check_store_range(mt, 11450 + i, 11450 + i + 1, NULL, 0); + for (i = 5; i < 65; i += 10) + check_store_range(mt, 11770 + i, 11770 + i + 1, NULL, 0); - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - mas_reset(&mas); - count = 0; - check_rev_seq(mt, max, false); - do { - count++; - mas_dfs_preorder(&mas); - } while (!mas_is_none(&mas)); - /*printk("count %lu\n", count); */ - MT_BUG_ON(mt, count != 77); + MT_BUG_ON(mt, mt_height(mt) >= 4); + for (i = 5; i < 45; i += 10) + check_store_range(mt, 11700 + i, 11700 + i + 1, NULL, 0); + if (!MAPLE_32BIT) + MT_BUG_ON(mt, mt_height(mt) < 4); mtree_destroy(mt); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - mas_reset(&mas); - mt_zero_nr_tallocated(); - mt_set_non_kernel(200); - mas_expected_entries(&mas, max); - for (count = 0; count <= max; count++) { - mas.index = mas.last = count; - mas_store(&mas, xa_mk_value(count)); - MT_BUG_ON(mt, mas_is_err(&mas)); + for (i = 0; i <= 1200; i++) { + val = i*10; + val2 = (i+1)*10; + check_store_range(mt, val, val2, xa_mk_value(val), 0); + MT_BUG_ON(mt, mt_height(mt) >= 4); } - mas_destroy(&mas); - rcu_barrier(); - /* - * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n", - * max, mt_get_alloc_size()/1024, mt_nr_allocated(), - * mt_nr_tallocated()); - */ + /* Fill parents and leaves before split. */ + for (i = 5; i < 455; i += 10) + check_store_range(mt, 7800 + i, 7800 + i + 1, NULL, 0); + for (i = 1; i < 16; i++) + check_store_range(mt, 8185 + i, 8185 + i + 1, + xa_mk_value(8185+i), 0); + MT_BUG_ON(mt, mt_height(mt) >= 4); + /* triple split across multiple levels. */ + check_store_range(mt, 8184, 8184, xa_mk_value(8184), 0); + if (!MAPLE_32BIT) + MT_BUG_ON(mt, mt_height(mt) != 4); } -#if defined(BENCH_SLOT_STORE) -static noinline void bench_slot_store(struct maple_tree *mt) +static noinline void check_next_entry(struct maple_tree *mt) { - int i, brk = 105, max = 1040, brk_start = 100, count = 20000000; + void *entry = NULL; + unsigned long limit = 30, i = 0; + MA_STATE(mas, mt, i, i); - for (i = 0; i < max; i += 10) - mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + MT_BUG_ON(mt, !mtree_empty(mt)); - for (i = 0; i < count; i++) { - mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL); - mtree_store_range(mt, brk_start, brk, xa_mk_value(brk), - GFP_KERNEL); + check_seq(mt, limit, false); + rcu_read_lock(); + + /* Check the first one and get ma_state in the correct state. */ + MT_BUG_ON(mt, mas_walk(&mas) != xa_mk_value(i++)); + for ( ; i <= limit + 1; i++) { + entry = mas_next(&mas, limit); + if (i > limit) + MT_BUG_ON(mt, entry != NULL); + else + MT_BUG_ON(mt, xa_mk_value(i) != entry); } + rcu_read_unlock(); + mtree_destroy(mt); } -#endif -#if defined(BENCH_NODE_STORE) -static noinline void bench_node_store(struct maple_tree *mt) +static noinline void check_prev_entry(struct maple_tree *mt) { - int i, overwrite = 76, max = 240, count = 20000000; - - for (i = 0; i < max; i += 10) - mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + unsigned long index = 16; + void *value; + int i; - for (i = 0; i < count; i++) { - mtree_store_range(mt, overwrite, overwrite + 15, - xa_mk_value(overwrite), GFP_KERNEL); + MA_STATE(mas, mt, index, index); - overwrite += 5; - if (overwrite >= 135) - overwrite = 76; - } -} -#endif + MT_BUG_ON(mt, !mtree_empty(mt)); + check_seq(mt, 30, false); -#if defined(BENCH_AWALK) -static noinline void bench_awalk(struct maple_tree *mt) -{ - int i, max = 2500, count = 50000000; - MA_STATE(mas, mt, 1470, 1470); + rcu_read_lock(); + value = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, value != xa_mk_value(index)); + value = mas_prev(&mas, 0); + MT_BUG_ON(mt, value != xa_mk_value(index - 1)); + rcu_read_unlock(); + mtree_destroy(mt); - for (i = 0; i < max; i += 10) - mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + /* Check limits on prev */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_lock(&mas); + for (i = 0; i <= index; i++) { + mas_set_range(&mas, i*10, i*10+5); + mas_store_gfp(&mas, xa_mk_value(i), GFP_KERNEL); + } - mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL); + mas_set(&mas, 20); + value = mas_walk(&mas); + MT_BUG_ON(mt, value != xa_mk_value(2)); - for (i = 0; i < count; i++) { - mas_empty_area_rev(&mas, 0, 2000, 10); - mas_reset(&mas); - } -} -#endif -#if defined(BENCH_WALK) -static noinline void bench_walk(struct maple_tree *mt) -{ - int i, max = 2500, count = 550000000; - MA_STATE(mas, mt, 1470, 1470); + value = mas_prev(&mas, 19); + MT_BUG_ON(mt, value != NULL); - for (i = 0; i < max; i += 10) - mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); + mas_set(&mas, 80); + value = mas_walk(&mas); + MT_BUG_ON(mt, value != xa_mk_value(8)); - for (i = 0; i < count; i++) { - mas_walk(&mas); - mas_reset(&mas); - } + value = mas_prev(&mas, 76); + MT_BUG_ON(mt, value != NULL); + mas_unlock(&mas); } -#endif -#if defined(BENCH_MT_FOR_EACH) -static noinline void bench_mt_for_each(struct maple_tree *mt) +static noinline void check_root_expand(struct maple_tree *mt) { - int i, count = 1000000; - unsigned long max = 2500, index = 0; - void *entry; + MA_STATE(mas, mt, 0, 0); + void *ptr; - for (i = 0; i < max; i += 5) - mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL); - for (i = 0; i < count; i++) { - unsigned long j = 0; + mas_lock(&mas); + mas_set(&mas, 3); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != ULONG_MAX); - mt_for_each(mt, entry, index, max) { - MT_BUG_ON(mt, entry != xa_mk_value(j)); - j += 5; - } + ptr = &check_prev_entry; + mas_set(&mas, 1); + mas_store_gfp(&mas, ptr, GFP_KERNEL); - index = 0; - } + mas_set(&mas, 0); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); -} -#endif + mas_set(&mas, 1); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != &check_prev_entry); -static noinline void check_forking(struct maple_tree *mt) -{ + mas_set(&mas, 2); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + mas_unlock(&mas); + mtree_destroy(mt); - struct maple_tree newmt; - int i, nr_entries = 134; - void *val; - MA_STATE(mas, mt, 0, 0); - MA_STATE(newmas, mt, 0, 0); - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); + mt_init_flags(mt, 0); + mas_lock(&mas); - mt_set_non_kernel(99999); - mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); - newmas.tree = &newmt; - mas_reset(&newmas); - mas_reset(&mas); - mas.index = 0; - mas.last = 0; - if (mas_expected_entries(&newmas, nr_entries)) { - pr_err("OOM!"); - BUG_ON(1); - } - mas_for_each(&mas, val, ULONG_MAX) { - newmas.index = mas.index; - newmas.last = mas.last; - mas_store(&newmas, val); - } - mas_destroy(&newmas); - mt_validate(&newmt); - mt_set_non_kernel(0); - mtree_destroy(&newmt); -} + mas_set(&mas, 0); + ptr = &check_prev_entry; + mas_store_gfp(&mas, ptr, GFP_KERNEL); -static noinline void check_mas_store_gfp(struct maple_tree *mt) -{ + mas_set(&mas, 5); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, mas.index != 1); + MT_BUG_ON(mt, mas.last != ULONG_MAX); - struct maple_tree newmt; - int i, nr_entries = 135; - void *val; - MA_STATE(mas, mt, 0, 0); - MA_STATE(newmas, mt, 0, 0); + mas_set_range(&mas, 0, 100); + ptr = mas_walk(&mas); + MT_BUG_ON(mt, ptr != &check_prev_entry); + MT_BUG_ON(mt, mas.last != 0); + mas_unlock(&mas); + mtree_destroy(mt); - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); + mt_init_flags(mt, 0); + mas_lock(&mas); + + mas_set(&mas, 0); + ptr = (void *)((unsigned long) check_prev_entry | 1UL); + mas_store_gfp(&mas, ptr, GFP_KERNEL); + ptr = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); + + mas_set(&mas, 1); + ptr = mas_prev(&mas, 0); + MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); + MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 1UL)); - mt_set_non_kernel(99999); - mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); - newmas.tree = &newmt; - mas_reset(&newmas); - mas_set(&mas, 0); - mas_for_each(&mas, val, ULONG_MAX) { - newmas.index = mas.index; - newmas.last = mas.last; - mas_store_gfp(&newmas, val, GFP_KERNEL); - } + mas_unlock(&mas); - mt_validate(&newmt); - mt_set_non_kernel(0); - mtree_destroy(&newmt); -} + mtree_destroy(mt); -#if defined(BENCH_FORK) -static noinline void bench_forking(struct maple_tree *mt) -{ + mt_init_flags(mt, 0); + mas_lock(&mas); + mas_set(&mas, 0); + ptr = (void *)((unsigned long) check_prev_entry | 2UL); + mas_store_gfp(&mas, ptr, GFP_KERNEL); + ptr = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, ptr != NULL); + MT_BUG_ON(mt, (mas.index != 1) && (mas.last != ULONG_MAX)); - struct maple_tree newmt; - int i, nr_entries = 134, nr_fork = 80000; - void *val; - MA_STATE(mas, mt, 0, 0); - MA_STATE(newmas, mt, 0, 0); + mas_set(&mas, 1); + ptr = mas_prev(&mas, 0); + MT_BUG_ON(mt, (mas.index != 0) && (mas.last != 0)); + MT_BUG_ON(mt, ptr != (void *)((unsigned long) check_prev_entry | 2UL)); - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); - for (i = 0; i < nr_fork; i++) { - mt_set_non_kernel(99999); - mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); - newmas.tree = &newmt; - mas_reset(&newmas); - mas_reset(&mas); - mas.index = 0; - mas.last = 0; - if (mas_expected_entries(&newmas, nr_entries)) { - printk("OOM!"); - BUG_ON(1); - } - mas_for_each(&mas, val, ULONG_MAX) { - newmas.index = mas.index; - newmas.last = mas.last; - mas_store(&newmas, val); - } - mas_destroy(&newmas); - mt_validate(&newmt); - mt_set_non_kernel(0); - mtree_destroy(&newmt); - } + mas_unlock(&mas); } -#endif -static noinline void next_prev_test(struct maple_tree *mt) +static noinline void check_gap_combining(struct maple_tree *mt) { - int i, nr_entries = 200; - void *val; - MA_STATE(mas, mt, 0, 0); - struct maple_enode *mn; + struct maple_enode *mn1, *mn2; + void *entry; + unsigned long singletons = 100; + unsigned long *seq100; + unsigned long seq100_64[] = { + /* 0-5 */ + 74, 75, 76, + 50, 100, 2, - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); + /* 6-12 */ + 44, 45, 46, 43, + 20, 50, 3, - for (i = 0; i <= nr_entries / 2; i++) { - mas_next(&mas, 1000); - if (mas_is_none(&mas)) - break; + /* 13-20*/ + 80, 81, 82, + 76, 2, 79, 85, 4, + }; - } - mas_reset(&mas); - mas_set(&mas, 0); - i = 0; - mas_for_each(&mas, val, 1000) { - i++; - } + unsigned long seq100_32[] = { + /* 0-5 */ + 61, 62, 63, + 50, 100, 2, - mas_reset(&mas); - mas_set(&mas, 0); - i = 0; - mas_for_each(&mas, val, 1000) { - mas_pause(&mas); - i++; - } + /* 6-12 */ + 31, 32, 33, 30, + 20, 50, 3, - /* - * 680 - 685 = 0x61a00001930c - * 686 - 689 = NULL; - * 690 - 695 = 0x61a00001930c - * Check simple next/prev - */ - mas_set(&mas, 686); - val = mas_walk(&mas); - MT_BUG_ON(mt, val != NULL); + /* 13-20*/ + 80, 81, 82, + 76, 2, 79, 85, 4, + }; - val = mas_next(&mas, 1000); - MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); - MT_BUG_ON(mt, mas.index != 690); - MT_BUG_ON(mt, mas.last != 695); + unsigned long seq2000[] = { + 1152, 1151, + 1100, 1200, 2, + }; + unsigned long seq400[] = { + 286, 318, + 256, 260, 266, 270, 275, 280, 290, 398, + 286, 310, + }; - val = mas_prev(&mas, 0); - MT_BUG_ON(mt, val != xa_mk_value(680 / 10)); - MT_BUG_ON(mt, mas.index != 680); - MT_BUG_ON(mt, mas.last != 685); + unsigned long index; - val = mas_next(&mas, 1000); - MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); - MT_BUG_ON(mt, mas.index != 690); - MT_BUG_ON(mt, mas.last != 695); + MA_STATE(mas, mt, 0, 0); - val = mas_next(&mas, 1000); - MT_BUG_ON(mt, val != xa_mk_value(700 / 10)); - MT_BUG_ON(mt, mas.index != 700); - MT_BUG_ON(mt, mas.last != 705); + if (MAPLE_32BIT) + seq100 = seq100_32; + else + seq100 = seq100_64; - /* Check across node boundaries of the tree */ - mas_set(&mas, 70); - val = mas_walk(&mas); - MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); - MT_BUG_ON(mt, mas.index != 70); - MT_BUG_ON(mt, mas.last != 75); + index = seq100[0]; + mas_set(&mas, index); + MT_BUG_ON(mt, !mtree_empty(mt)); + check_seq(mt, singletons, false); /* create 100 singletons. */ - val = mas_next(&mas, 1000); - MT_BUG_ON(mt, val != xa_mk_value(80 / 10)); - MT_BUG_ON(mt, mas.index != 80); - MT_BUG_ON(mt, mas.last != 85); + mt_set_non_kernel(1); + mtree_test_erase(mt, seq100[2]); + check_load(mt, seq100[2], NULL); + mtree_test_erase(mt, seq100[1]); + check_load(mt, seq100[1], NULL); - val = mas_prev(&mas, 70); - MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); - MT_BUG_ON(mt, mas.index != 70); - MT_BUG_ON(mt, mas.last != 75); + rcu_read_lock(); + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index)); + mn1 = mas.node; + mas_next(&mas, ULONG_MAX); + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); + mn2 = mas.node; + MT_BUG_ON(mt, mn1 == mn2); /* test the test. */ - /* Check across two levels of the tree */ + /* + * At this point, there is a gap of 2 at index + 1 between seq100[3] and + * seq100[4]. Search for the gap. + */ + mt_set_non_kernel(1); mas_reset(&mas); - mas_set(&mas, 707); - val = mas_walk(&mas); - MT_BUG_ON(mt, val != NULL); - val = mas_next(&mas, 1000); - MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); - MT_BUG_ON(mt, mas.index != 710); - MT_BUG_ON(mt, mas.last != 715); - mn = mas.node; - - val = mas_next(&mas, 1000); - MT_BUG_ON(mt, val != xa_mk_value(720 / 10)); - MT_BUG_ON(mt, mas.index != 720); - MT_BUG_ON(mt, mas.last != 725); - MT_BUG_ON(mt, mn == mas.node); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[3], seq100[4], + seq100[5])); + MT_BUG_ON(mt, mas.index != index + 1); + rcu_read_unlock(); - val = mas_prev(&mas, 0); - MT_BUG_ON(mt, val != xa_mk_value(710 / 10)); - MT_BUG_ON(mt, mas.index != 710); - MT_BUG_ON(mt, mas.last != 715); + mtree_test_erase(mt, seq100[6]); + check_load(mt, seq100[6], NULL); + mtree_test_erase(mt, seq100[7]); + check_load(mt, seq100[7], NULL); + mtree_test_erase(mt, seq100[8]); + index = seq100[9]; - /* Check running off the end and back on */ + rcu_read_lock(); + mas.index = index; + mas.last = index; mas_reset(&mas); - mas_set(&mas, 2000); - val = mas_walk(&mas); - MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); - MT_BUG_ON(mt, mas.index != 2000); - MT_BUG_ON(mt, mas.last != 2005); - - val = mas_next(&mas, ULONG_MAX); - MT_BUG_ON(mt, val != NULL); - MT_BUG_ON(mt, mas.index != ULONG_MAX); - MT_BUG_ON(mt, mas.last != ULONG_MAX); - - val = mas_prev(&mas, 0); - MT_BUG_ON(mt, val != xa_mk_value(2000 / 10)); - MT_BUG_ON(mt, mas.index != 2000); - MT_BUG_ON(mt, mas.last != 2005); + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index)); + mn1 = mas.node; + entry = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(index + 4)); + mas_next(&mas, ULONG_MAX); /* go to the next entry. */ + mn2 = mas.node; + MT_BUG_ON(mt, mn1 == mn2); /* test the next entry is in the next node. */ - /* Check running off the start and back on */ + /* + * At this point, there is a gap of 3 at seq100[6]. Find it by + * searching 20 - 50 for size 3. + */ mas_reset(&mas); - mas_set(&mas, 10); - val = mas_walk(&mas); - MT_BUG_ON(mt, val != xa_mk_value(1)); - MT_BUG_ON(mt, mas.index != 10); - MT_BUG_ON(mt, mas.last != 15); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[10], seq100[11], + seq100[12])); + MT_BUG_ON(mt, mas.index != seq100[6]); + rcu_read_unlock(); - val = mas_prev(&mas, 0); - MT_BUG_ON(mt, val != xa_mk_value(0)); - MT_BUG_ON(mt, mas.index != 0); - MT_BUG_ON(mt, mas.last != 5); + mt_set_non_kernel(1); + mtree_store(mt, seq100[13], NULL, GFP_KERNEL); + check_load(mt, seq100[13], NULL); + check_load(mt, seq100[14], xa_mk_value(seq100[14])); + mtree_store(mt, seq100[14], NULL, GFP_KERNEL); + check_load(mt, seq100[13], NULL); + check_load(mt, seq100[14], NULL); - val = mas_prev(&mas, 0); - MT_BUG_ON(mt, val != NULL); - MT_BUG_ON(mt, mas.index != 0); - MT_BUG_ON(mt, mas.last != 0); + mas_reset(&mas); + rcu_read_lock(); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[15], + seq100[17])); + MT_BUG_ON(mt, mas.index != seq100[13]); + mt_validate(mt); + rcu_read_unlock(); - mas.index = 0; - mas.last = 5; - mas_store(&mas, NULL); + /* + * *DEPRECATED: no retries anymore* Test retry entry in the start of a + * gap. + */ + mt_set_non_kernel(2); + mtree_test_store_range(mt, seq100[18], seq100[14], NULL); + mtree_test_erase(mt, seq100[15]); mas_reset(&mas); - mas_set(&mas, 10); - mas_walk(&mas); + rcu_read_lock(); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq100[16], seq100[19], + seq100[20])); + rcu_read_unlock(); + MT_BUG_ON(mt, mas.index != seq100[18]); + mt_validate(mt); + mtree_destroy(mt); - val = mas_prev(&mas, 0); - MT_BUG_ON(mt, val != NULL); - MT_BUG_ON(mt, mas.index != 0); - MT_BUG_ON(mt, mas.last != 0); + /* seq 2000 tests are for multi-level tree gaps */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_seq(mt, 2000, false); + mt_set_non_kernel(1); + mtree_test_erase(mt, seq2000[0]); + mtree_test_erase(mt, seq2000[1]); + mt_set_non_kernel(2); + mas_reset(&mas); + rcu_read_lock(); + MT_BUG_ON(mt, mas_empty_area_rev(&mas, seq2000[2], seq2000[3], + seq2000[4])); + MT_BUG_ON(mt, mas.index != seq2000[1]); + rcu_read_unlock(); + mt_validate(mt); mtree_destroy(mt); - mt_init(mt); - mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); - mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL); - mas_set(&mas, 5); - val = mas_prev(&mas, 4); - MT_BUG_ON(mt, val != NULL); -} + /* seq 400 tests rebalancing over two levels. */ + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_seq(mt, 400, false); + mtree_test_store_range(mt, seq400[0], seq400[1], NULL); + mt_set_non_kernel(0); + mtree_destroy(mt); -#define RCU_RANGE_COUNT 1000 -#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true;} MT_BUG_ON(test->mt, y);} -struct rcu_test_struct2 { - struct maple_tree *mt; - - bool start; - bool stop; - unsigned int thread_count; - - unsigned int seen_toggle; - unsigned int seen_added; - unsigned int seen_modified; - unsigned int seen_deleted; - int pause; - - unsigned long index[RCU_RANGE_COUNT]; - unsigned long last[RCU_RANGE_COUNT]; -}; - -struct rcu_reader_struct { - unsigned int id; - int mod; - int del; - int flip; - int add; - int next; - struct rcu_test_struct2 *test; -}; - -/* RCU reader helper function */ -static void rcu_reader_register(struct rcu_test_struct2 *test) + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_seq(mt, 400, false); + mt_set_non_kernel(50); + mtree_test_store_range(mt, seq400[2], seq400[9], + xa_mk_value(seq400[2])); + mtree_test_store_range(mt, seq400[3], seq400[9], + xa_mk_value(seq400[3])); + mtree_test_store_range(mt, seq400[4], seq400[9], + xa_mk_value(seq400[4])); + mtree_test_store_range(mt, seq400[5], seq400[9], + xa_mk_value(seq400[5])); + mtree_test_store_range(mt, seq400[0], seq400[9], + xa_mk_value(seq400[0])); + mtree_test_store_range(mt, seq400[6], seq400[9], + xa_mk_value(seq400[6])); + mtree_test_store_range(mt, seq400[7], seq400[9], + xa_mk_value(seq400[7])); + mtree_test_store_range(mt, seq400[8], seq400[9], + xa_mk_value(seq400[8])); + mtree_test_store_range(mt, seq400[10], seq400[11], + xa_mk_value(seq400[10])); + mt_validate(mt); + mt_set_non_kernel(0); + mtree_destroy(mt); +} +static noinline void check_node_overwrite(struct maple_tree *mt) { - rcu_register_thread(); - uatomic_inc(&test->thread_count); + int i, max = 4000; - while (!test->start) - usleep(test->pause * 100); -} + for (i = 0; i < max; i++) + mtree_test_store_range(mt, i*100, i*100 + 50, xa_mk_value(i*100)); -static void rcu_reader_setup(struct rcu_reader_struct *reader, - unsigned int id, struct rcu_test_struct2 *test) -{ - reader->id = id; - reader->test = test; - reader->mod = reader->id % 10; - reader->del = (reader->mod + 1) % 10; - reader->flip = (reader->mod + 2) % 10; - reader->add = (reader->mod + 3) % 10; - reader->next = (reader->mod + 4) % 10; + mtree_test_store_range(mt, 319951, 367950, NULL); + /*mt_dump(mt); */ + mt_validate(mt); } -/* RCU reader in increasing index */ -static void *rcu_reader_fwd(void *ptr) +#if defined(BENCH_SLOT_STORE) +static noinline void bench_slot_store(struct maple_tree *mt) { - struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; - struct rcu_test_struct2 *test = reader->test; - unsigned long index = reader->id; - bool toggled, modified, deleted, added; - int i; - void *entry, *prev = NULL; - MA_STATE(mas, test->mt, 0, 0); + int i, brk = 105, max = 1040, brk_start = 100, count = 20000000; - rcu_reader_register(test); - toggled = modified = deleted = added = false; + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); - while (!test->stop) { - i = 0; - /* mas_for_each ?*/ - rcu_read_lock(); - mas_set(&mas, test->index[index]); - mas_for_each(&mas, entry, test->last[index + 9]) { - unsigned long r_start, r_end, alt_start; - void *expected, *alt; - - r_start = test->index[index + i]; - r_end = test->last[index + i]; - expected = xa_mk_value(r_start); - - if (i == reader->del) { - if (!deleted) { - alt_start = test->index[index + reader->flip]; - /* delete occurred. */ - if (mas.index == alt_start) { - uatomic_inc(&test->seen_deleted); - deleted = true; - } - } - if (deleted) { - i = reader->flip; - r_start = test->index[index + i]; - r_end = test->last[index + i]; - expected = xa_mk_value(r_start); - } - } - - if (!added && (i == reader->add)) { - alt_start = test->index[index + reader->next]; - if (mas.index == r_start) { - uatomic_inc(&test->seen_added); - added = true; - } else if (mas.index == alt_start) { - i = reader->next; - r_start = test->index[index + i]; - r_end = test->last[index + i]; - expected = xa_mk_value(r_start); - } - } - - RCU_MT_BUG_ON(test, mas.index != r_start); - RCU_MT_BUG_ON(test, mas.last != r_end); - - if (i == reader->flip) { - alt = xa_mk_value(index + i + RCU_RANGE_COUNT); - if (prev) { - if (toggled && entry == expected) - uatomic_inc(&test->seen_toggle); - else if (!toggled && entry == alt) - uatomic_inc(&test->seen_toggle); - } - - if (entry == expected) - toggled = false; - else if (entry == alt) - toggled = true; - else { - printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); - RCU_MT_BUG_ON(test, 1); - } - - prev = entry; - } else if (i == reader->mod) { - alt = xa_mk_value(index + i * 2 + 1 + - RCU_RANGE_COUNT); - if (entry != expected) { - if (!modified) - uatomic_inc(&test->seen_modified); - modified = true; - } else { - if (modified) - uatomic_inc(&test->seen_modified); - modified = false; - } - - if (modified) - RCU_MT_BUG_ON(test, entry != alt); - - } else { - if (entry != expected) - printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); - RCU_MT_BUG_ON(test, entry != expected); - } - - i++; - } - rcu_read_unlock(); - usleep(test->pause); + for (i = 0; i < count; i++) { + mtree_store_range(mt, brk, brk, NULL, GFP_KERNEL); + mtree_store_range(mt, brk_start, brk, xa_mk_value(brk), + GFP_KERNEL); } - - rcu_unregister_thread(); - return NULL; } +#endif -/* RCU reader in decreasing index */ -static void *rcu_reader_rev(void *ptr) +#if defined(BENCH_NODE_STORE) +static noinline void bench_node_store(struct maple_tree *mt) { - struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; - struct rcu_test_struct2 *test = reader->test; - unsigned long index = reader->id; - bool toggled, modified, deleted, added; - int i; - void *prev = NULL; - MA_STATE(mas, test->mt, 0, 0); - - rcu_reader_register(test); - toggled = modified = deleted = added = false; - - - while (!test->stop) { - void *entry; - - i = 9; - mas_set(&mas, test->index[index + i]); - - rcu_read_lock(); - while (i--) { - unsigned long r_start, r_end, alt_start; - void *expected, *alt; - int line = __LINE__; - - entry = mas_prev(&mas, test->index[index]); - r_start = test->index[index + i]; - r_end = test->last[index + i]; - expected = xa_mk_value(r_start); - - if (i == reader->del) { - alt_start = test->index[index + reader->mod]; - if (mas.index == alt_start) { - line = __LINE__; - if (!deleted) - uatomic_inc(&test->seen_deleted); - deleted = true; - } - if (deleted) { - line = __LINE__; - i = reader->mod; - r_start = test->index[index + i]; - r_end = test->last[index + i]; - expected = xa_mk_value(r_start); - } - } - if (!added && (i == reader->add)) { - alt_start = test->index[index + reader->flip]; - if (mas.index == r_start) { - line = __LINE__; - uatomic_inc(&test->seen_added); - added = true; - } else if (mas.index == alt_start) { - line = __LINE__; - i = reader->flip; - r_start = test->index[index + i]; - r_end = test->last[index + i]; - expected = xa_mk_value(r_start); - } - } - - if (i == reader->mod) - line = __LINE__; - else if (i == reader->flip) - line = __LINE__; - - if (mas.index != r_start) { - alt = xa_mk_value(index + i * 2 + 1 + - RCU_RANGE_COUNT); - mt_dump(test->mt); - printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n", - mas.index, mas.last, entry, - r_start, r_end, expected, alt, - line, i); - } - RCU_MT_BUG_ON(test, mas.index != r_start); - RCU_MT_BUG_ON(test, mas.last != r_end); - - if (i == reader->mod) { - alt = xa_mk_value(index + i * 2 + 1 + - RCU_RANGE_COUNT); - - if (entry != expected) { - if (!modified) - uatomic_inc(&test->seen_modified); - modified = true; - } else { - if (modified) - uatomic_inc(&test->seen_modified); - modified = false; - } - if (modified) - RCU_MT_BUG_ON(test, entry != alt); - - - } else if (i == reader->flip) { - alt = xa_mk_value(index + i + - RCU_RANGE_COUNT); - if (prev) { - if (toggled && entry == expected) - uatomic_inc(&test->seen_toggle); - else if (!toggled && entry == alt) - uatomic_inc(&test->seen_toggle); - } - - if (entry == expected) - toggled = false; - else if (entry == alt) - toggled = true; - else { - printk("%lu-%lu %p != %p or %p\n", - mas.index, mas.last, entry, - expected, alt); - RCU_MT_BUG_ON(test, 1); - } - - prev = entry; - } else { - if (entry != expected) - printk("%lu-%lu %p != %p\n", mas.index, - mas.last, entry, expected); - RCU_MT_BUG_ON(test, entry != expected); - } - } - rcu_read_unlock(); - usleep(test->pause); - } + int i, overwrite = 76, max = 240, count = 20000000; - rcu_unregister_thread(); - return NULL; -} + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); -static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test, - int count, struct rcu_reader_struct *test_reader) -{ - int i, j = 10000; - bool toggle = true; - - test->start = true; /* Release the hounds! */ - usleep(5); - - while (j--) { - toggle = !toggle; - i = count; - while (i--) { - unsigned long start, end; - struct rcu_reader_struct *this = &test_reader[i]; - - /* Mod offset */ - if (j == 600) { - start = test->index[this->id + this->mod]; - end = test->last[this->id + this->mod]; - mtree_store_range(mt, start, end, - xa_mk_value(this->id + this->mod * 2 + - 1 + RCU_RANGE_COUNT), - GFP_KERNEL); - } - - /* Toggle */ - if (!(j % 5)) { - start = test->index[this->id + this->flip]; - end = test->last[this->id + this->flip]; - mtree_store_range(mt, start, end, - xa_mk_value((toggle ? start : - this->id + this->flip + - RCU_RANGE_COUNT)), - GFP_KERNEL); - } - - /* delete */ - if (j == 400) { - start = test->index[this->id + this->del]; - end = test->last[this->id + this->del]; - mtree_store_range(mt, start, end, NULL, GFP_KERNEL); - } - - /* add */ - if (j == 500) { - start = test->index[this->id + this->add]; - end = test->last[this->id + this->add]; - mtree_store_range(mt, start, end, - xa_mk_value(start), GFP_KERNEL); - } - } - usleep(test->pause); - /* If a test fails, don't flood the console */ - if (test->stop) - break; - } -} + for (i = 0; i < count; i++) { + mtree_store_range(mt, overwrite, overwrite + 15, + xa_mk_value(overwrite), GFP_KERNEL); -static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test, - int count, struct rcu_reader_struct *test_reader) -{ - int j, i; - bool toggle = true; - - test->start = true; /* Release the hounds! */ - usleep(5); - for (j = 0; j < 10000; j++) { - toggle = !toggle; - for (i = 0; i < count; i++) { - unsigned long start, end; - struct rcu_reader_struct *this = &test_reader[i]; - - /* Mod offset */ - if (j == 600) { - start = test->index[this->id + this->mod]; - end = test->last[this->id + this->mod]; - mtree_store_range(mt, start, end, - xa_mk_value(this->id + this->mod * 2 + - 1 + RCU_RANGE_COUNT), - GFP_KERNEL); - } - - /* Toggle */ - if (!(j % 5)) { - start = test->index[this->id + this->flip]; - end = test->last[this->id + this->flip]; - mtree_store_range(mt, start, end, - xa_mk_value((toggle ? start : - this->id + this->flip + - RCU_RANGE_COUNT)), - GFP_KERNEL); - } - - /* delete */ - if (j == 400) { - start = test->index[this->id + this->del]; - end = test->last[this->id + this->del]; - mtree_store_range(mt, start, end, NULL, GFP_KERNEL); - } - - /* add */ - if (j == 500) { - start = test->index[this->id + this->add]; - end = test->last[this->id + this->add]; - mtree_store_range(mt, start, end, - xa_mk_value(start), GFP_KERNEL); - } - } - usleep(test->pause); - /* If a test fails, don't flood the console */ - if (test->stop) - break; + overwrite += 5; + if (overwrite >= 135) + overwrite = 76; } } +#endif -/* - * This is to check: - * 1. Range that is not ever present - * 2. Range that is always present - * 3. Things being added but not removed. - * 4. Things being removed but not added. - * 5. Things are being added and removed, searches my succeed or fail - * - * This sets up two readers for every 10 entries; one forward and one reverse - * reading. - */ -static void rcu_stress(struct maple_tree *mt, bool forward) +#if defined(BENCH_AWALK) +static noinline void bench_awalk(struct maple_tree *mt) { - unsigned int count, i; - unsigned long r, seed; - pthread_t readers[RCU_RANGE_COUNT / 5]; - struct rcu_test_struct2 test; - struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5]; - void *(*function)(void *); - - /* Test setup */ - test.mt = mt; - test.pause = 5; - test.seen_toggle = 0; - test.seen_deleted = 0; - test.seen_added = 0; - test.seen_modified = 0; - test.thread_count = 0; - test.start = test.stop = false; - seed = time(NULL); - srand(seed); - for (i = 0; i < RCU_RANGE_COUNT; i++) { - r = seed + rand(); - mtree_store_range(mt, seed, r, - xa_mk_value(seed), GFP_KERNEL); - - /* Record start and end of entry */ - test.index[i] = seed; - test.last[i] = r; - seed = 1 + r + rand() % 10; - } + int i, max = 2500, count = 50000000; + MA_STATE(mas, mt, 1470, 1470); - i = count = ARRAY_SIZE(readers); - while (i--) { - unsigned long id; + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); - id = i / 2 * 10; - if (i % 2) - function = rcu_reader_fwd; - else - function = rcu_reader_rev; + mtree_store_range(mt, 1470, 1475, NULL, GFP_KERNEL); - rcu_reader_setup(&test_reader[i], id, &test); - if (pthread_create(&readers[i], NULL, *function, - &test_reader[i])) { - perror("creating reader thread"); - exit(1); - } + for (i = 0; i < count; i++) { + mas_empty_area_rev(&mas, 0, 2000, 10); + mas_reset(&mas); } +} +#endif +#if defined(BENCH_WALK) +static noinline void bench_walk(struct maple_tree *mt) +{ + int i, max = 2500, count = 550000000; + MA_STATE(mas, mt, 1470, 1470); - for (i = 0; i < ARRAY_SIZE(readers); i++) { - struct rcu_reader_struct *this = &test_reader[i]; - int add = this->id + this->add; + for (i = 0; i < max; i += 10) + mtree_store_range(mt, i, i + 5, xa_mk_value(i), GFP_KERNEL); - /* Remove add entries from the tree for later addition */ - mtree_store_range(mt, test.index[add], test.last[add], - NULL, GFP_KERNEL); + for (i = 0; i < count; i++) { + mas_walk(&mas); + mas_reset(&mas); } - mt_set_in_rcu(mt); - do { - usleep(5); - } while (test.thread_count > ARRAY_SIZE(readers)); - - if (forward) - rcu_stress_fwd(mt, &test, count, test_reader); - else - rcu_stress_rev(mt, &test, count, test_reader); - - test.stop = true; - while (count--) - pthread_join(readers[count], NULL); - - mt_validate(mt); } +#endif - -struct rcu_test_struct { - struct maple_tree *mt; /* the maple tree */ - int count; /* Number of times to check value(s) */ - unsigned long index; /* The first index to check */ - void *entry1; /* The first entry value */ - void *entry2; /* The second entry value */ - void *entry3; /* The third entry value */ - - bool update_2; - bool update_3; - unsigned long range_start; - unsigned long range_end; - unsigned int loop_sleep; - unsigned int val_sleep; - - unsigned int failed; /* failed detection for other threads */ - unsigned int seen_entry2; /* Number of threads that have seen the new value */ - unsigned int seen_entry3; /* Number of threads that have seen the new value */ - unsigned int seen_both; /* Number of threads that have seen both new values */ - unsigned int seen_toggle; - unsigned int seen_added; - unsigned int seen_removed; - unsigned long last; /* The end of the range to write. */ - - unsigned long removed; /* The index of the removed entry */ - unsigned long added; /* The index of the removed entry */ - unsigned long toggle; /* The index of the removed entry */ -}; - -static inline -int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2, - bool *update_3) +#if defined(BENCH_MT_FOR_EACH) +static noinline void bench_mt_for_each(struct maple_tree *mt) { - if (entry == test->entry1) - return 0; + int i, count = 1000000; + unsigned long max = 2500, index = 0; + void *entry; - if (entry == test->entry2) { - if (!(*update_2)) { - uatomic_inc(&test->seen_entry2); - *update_2 = true; - if (update_3) - uatomic_inc(&test->seen_both); - } - return 0; - } + for (i = 0; i < max; i += 5) + mtree_store_range(mt, i, i + 4, xa_mk_value(i), GFP_KERNEL); + + for (i = 0; i < count; i++) { + unsigned long j = 0; - if (entry == test->entry3) { - if (!(*update_3)) { - uatomic_inc(&test->seen_entry3); - *update_3 = true; - if (update_2) - uatomic_inc(&test->seen_both); + mt_for_each(mt, entry, index, max) { + MT_BUG_ON(mt, entry != xa_mk_value(j)); + j += 5; } - return 0; + + index = 0; } - return 1; } +#endif -/* - * rcu_val() - Read a given value in the tree test->count times using the - * regular API - * - * @ptr: The pointer to the rcu_test_struct - */ -static void *rcu_val(void *ptr) +/* check_forking - simulate the kernel forking sequence with the tree. */ +static noinline void check_forking(struct maple_tree *mt) { - struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; - unsigned long count = test->count; - bool update_2 = false; - bool update_3 = false; - void *entry; - rcu_register_thread(); - while (count--) { - usleep(test->val_sleep); - /* - * No locking required, regular API locking is handled in the - * maple tree code - */ - entry = mtree_load(test->mt, test->index); - MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2, - &update_3)); + struct maple_tree newmt; + int i, nr_entries = 134; + void *val; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, mt, 0, 0); + + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + mt_set_non_kernel(99999); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + newmas.tree = &newmt; + mas_reset(&newmas); + mas_reset(&mas); + mas_lock(&newmas); + mas.index = 0; + mas.last = 0; + if (mas_expected_entries(&newmas, nr_entries)) { + pr_err("OOM!"); + BUG_ON(1); + } + rcu_read_lock(); + mas_for_each(&mas, val, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store(&newmas, val); } - rcu_unregister_thread(); - return NULL; + rcu_read_unlock(); + mas_destroy(&newmas); + mas_unlock(&newmas); + mt_validate(&newmt); + mt_set_non_kernel(0); + mtree_destroy(&newmt); } -/* - * rcu_loop() - Loop over a section of the maple tree, checking for an expected - * value using the advanced API - * - * @ptr - The pointer to the rcu_test_struct - */ -static void *rcu_loop(void *ptr) +static noinline void check_mas_store_gfp(struct maple_tree *mt) { - struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; - unsigned long count = test->count; - void *entry, *expected; - bool update_2 = false; - bool update_3 = false; - MA_STATE(mas, test->mt, test->range_start, test->range_start); - rcu_register_thread(); - - /* - * Loop through the test->range_start - test->range_end test->count - * times - */ - while (count--) { - usleep(test->loop_sleep); - rcu_read_lock(); - mas_for_each(&mas, entry, test->range_end) { - /* The expected value is based on the start range. */ - expected = xa_mk_value(mas.index ? mas.index / 10 : 0); - - /* Out of the interesting range */ - if (mas.index < test->index || mas.index > test->last) { - if (entry != expected) { - printk("%lx - %lx = %p not %p\n", - mas.index, mas.last, entry, expected); - } - MT_BUG_ON(test->mt, entry != expected); - continue; - } + struct maple_tree newmt; + int i, nr_entries = 135; + void *val; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, mt, 0, 0); - if (entry == expected) - continue; /* Not seen. */ + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); - /* In the interesting range */ - MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, - &update_2, - &update_3)); - } - rcu_read_unlock(); - mas_set(&mas, test->range_start); + mt_set_non_kernel(99999); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + newmas.tree = &newmt; + rcu_read_lock(); + mas_lock(&newmas); + mas_reset(&newmas); + mas_set(&mas, 0); + mas_for_each(&mas, val, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store_gfp(&newmas, val, GFP_KERNEL); } - - rcu_unregister_thread(); - return NULL; + mas_unlock(&newmas); + rcu_read_unlock(); + mt_validate(&newmt); + mt_set_non_kernel(0); + mtree_destroy(&newmt); } -static noinline -void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals) +#if defined(BENCH_FORK) +static noinline void bench_forking(struct maple_tree *mt) { - int i; - void *(*function)(void *); - pthread_t readers[20]; - - mt_set_in_rcu(mt); - MT_BUG_ON(mt, !mt_in_rcu(mt)); + struct maple_tree newmt; + int i, nr_entries = 134, nr_fork = 80000; + void *val; + MA_STATE(mas, mt, 0, 0); + MA_STATE(newmas, mt, 0, 0); - for (i = 0; i < ARRAY_SIZE(readers); i++) { - if (i % 2) - function = rcu_loop; - else - function = rcu_val; + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); - if (pthread_create(&readers[i], NULL, *function, vals)) { - perror("creating reader thread"); - exit(1); + for (i = 0; i < nr_fork; i++) { + mt_set_non_kernel(99999); + mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); + newmas.tree = &newmt; + mas_reset(&newmas); + mas_reset(&mas); + mas.index = 0; + mas.last = 0; + rcu_read_lock(); + mas_lock(&newmas); + if (mas_expected_entries(&newmas, nr_entries)) { + printk("OOM!"); + BUG_ON(1); + } + mas_for_each(&mas, val, ULONG_MAX) { + newmas.index = mas.index; + newmas.last = mas.last; + mas_store(&newmas, val); } + mas_destroy(&newmas); + mas_unlock(&newmas); + rcu_read_unlock(); + mt_validate(&newmt); + mt_set_non_kernel(0); + mtree_destroy(&newmt); } - - usleep(5); /* small yield to ensure all threads are at least started. */ - mtree_store_range(mt, vals->index, vals->last, vals->entry2, - GFP_KERNEL); - while (i--) - pthread_join(readers[i], NULL); - - /* Make sure the test caught at least one update. */ - MT_BUG_ON(mt, !vals->seen_entry2); } +#endif -static noinline -void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals) +static noinline void next_prev_test(struct maple_tree *mt) { + int i, nr_entries; + void *val; + MA_STATE(mas, mt, 0, 0); + struct maple_enode *mn; + unsigned long *level2; + unsigned long level2_64[] = {707, 1000, 710, 715, 720, 725}; + unsigned long level2_32[] = {1747, 2000, 1750, 1755, 1760, 1765}; - int i; - void *(*function)(void *); - pthread_t readers[20]; - unsigned int index = vals->index; + if (MAPLE_32BIT) { + nr_entries = 500; + level2 = level2_32; + } else { + nr_entries = 200; + level2 = level2_64; + } - mt_set_in_rcu(mt); - MT_BUG_ON(mt, !mt_in_rcu(mt)); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); - for (i = 0; i < ARRAY_SIZE(readers); i++) { - if (i % 2) - function = rcu_loop; - else - function = rcu_val; + mas_lock(&mas); + for (i = 0; i <= nr_entries / 2; i++) { + mas_next(&mas, 1000); + if (mas_is_none(&mas)) + break; - if (pthread_create(&readers[i], NULL, *function, vals)) { - perror("creating reader thread"); - exit(1); - } } - - usleep(5); /* small yield to ensure all threads are at least started. */ - - while (index <= vals->last) { - mtree_store(mt, index, - (index % 2 ? vals->entry2 : vals->entry3), - GFP_KERNEL); - index++; - usleep(5); + mas_reset(&mas); + mas_set(&mas, 0); + i = 0; + mas_for_each(&mas, val, 1000) { + i++; } - while (i--) - pthread_join(readers[i], NULL); - - /* Make sure the test caught at least one update. */ - MT_BUG_ON(mt, !vals->seen_entry2); - MT_BUG_ON(mt, !vals->seen_entry3); - MT_BUG_ON(mt, !vals->seen_both); -} -static noinline void check_rcu_simulated(struct maple_tree *mt) -{ - unsigned long i, nr_entries = 1000; - unsigned long target = 4320; - unsigned long val = 0xDEAD; + mas_reset(&mas); + mas_set(&mas, 0); + i = 0; + mas_for_each(&mas, val, 1000) { + mas_pause(&mas); + i++; + } - MA_STATE(mas_writer, mt, 0, 0); - MA_STATE(mas_reader, mt, target, target); + /* + * 680 - 685 = 0x61a00001930c + * 686 - 689 = NULL; + * 690 - 695 = 0x61a00001930c + * Check simple next/prev + */ + mas_set(&mas, 686); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != NULL); - rcu_register_thread(); + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); + MT_BUG_ON(mt, mas.index != 690); + MT_BUG_ON(mt, mas.last != 695); - mt_set_in_rcu(mt); - mas_lock(&mas_writer); - for (i = 0; i <= nr_entries; i++) { - mas_writer.index = i * 10; - mas_writer.last = i * 10 + 5; - mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); - } - mas_unlock(&mas_writer); + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(680 / 10)); + MT_BUG_ON(mt, mas.index != 680); + MT_BUG_ON(mt, mas.last != 685); - /* Overwrite one entry with a new value. */ - mas_set_range(&mas_writer, target, target + 5); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); - rcu_read_unlock(); + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(690 / 10)); + MT_BUG_ON(mt, mas.index != 690); + MT_BUG_ON(mt, mas.last != 695); - /* Restore value. */ - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); - mas_unlock(&mas_writer); - mas_reset(&mas_reader); + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(700 / 10)); + MT_BUG_ON(mt, mas.index != 700); + MT_BUG_ON(mt, mas.last != 705); + /* Check across node boundaries of the tree */ + mas_set(&mas, 70); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); + MT_BUG_ON(mt, mas.index != 70); + MT_BUG_ON(mt, mas.last != 75); - /* Overwrite 1/2 the entry */ - mas_set_range(&mas_writer, target, target + 2); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); - rcu_read_unlock(); + val = mas_next(&mas, 1000); + MT_BUG_ON(mt, val != xa_mk_value(80 / 10)); + MT_BUG_ON(mt, mas.index != 80); + MT_BUG_ON(mt, mas.last != 85); + val = mas_prev(&mas, 70); + MT_BUG_ON(mt, val != xa_mk_value(70 / 10)); + MT_BUG_ON(mt, mas.index != 70); + MT_BUG_ON(mt, mas.last != 75); - /* Restore value. */ - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); - mas_unlock(&mas_writer); - mas_reset(&mas_reader); + /* Check across two levels of the tree */ + mas_reset(&mas); + mas_set(&mas, level2[0]); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != NULL); + val = mas_next(&mas, level2[1]); + MT_BUG_ON(mt, val != xa_mk_value(level2[2] / 10)); + MT_BUG_ON(mt, mas.index != level2[2]); + MT_BUG_ON(mt, mas.last != level2[3]); + mn = mas.node; - /* Overwrite last 1/2 the entry */ - mas_set_range(&mas_writer, target + 2, target + 5); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - rcu_read_unlock(); + val = mas_next(&mas, level2[1]); + MT_BUG_ON(mt, val != xa_mk_value(level2[4] / 10)); + MT_BUG_ON(mt, mas.index != level2[4]); + MT_BUG_ON(mt, mas.last != level2[5]); + MT_BUG_ON(mt, mn == mas.node); + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(level2[2] / 10)); + MT_BUG_ON(mt, mas.index != level2[2]); + MT_BUG_ON(mt, mas.last != level2[3]); - /* Restore value. */ - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); - mas_unlock(&mas_writer); - mas_reset(&mas_reader); + /* Check running off the end and back on */ + mas_set(&mas, nr_entries * 10); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != xa_mk_value(nr_entries)); + MT_BUG_ON(mt, mas.index != (nr_entries * 10)); + MT_BUG_ON(mt, mas.last != (nr_entries * 10 + 5)); - /* Overwrite more than the entry */ - mas_set_range(&mas_writer, target - 5, target + 15); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); - rcu_read_unlock(); + val = mas_next(&mas, ULONG_MAX); + MT_BUG_ON(mt, val != NULL); + MT_BUG_ON(mt, mas.index != ULONG_MAX); + MT_BUG_ON(mt, mas.last != ULONG_MAX); - /* Restore value. */ - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); - mas_unlock(&mas_writer); - mas_reset(&mas_reader); + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(nr_entries)); + MT_BUG_ON(mt, mas.index != (nr_entries * 10)); + MT_BUG_ON(mt, mas.last != (nr_entries * 10 + 5)); - /* Overwrite more than the node. */ - mas_set_range(&mas_writer, target - 400, target + 400); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); - rcu_read_unlock(); + /* Check running off the start and back on */ + mas_reset(&mas); + mas_set(&mas, 10); + val = mas_walk(&mas); + MT_BUG_ON(mt, val != xa_mk_value(1)); + MT_BUG_ON(mt, mas.index != 10); + MT_BUG_ON(mt, mas.last != 15); - /* Restore value. */ - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); - mas_unlock(&mas_writer); - mas_reset(&mas_reader); + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != xa_mk_value(0)); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 5); - /* Overwrite the tree */ - mas_set_range(&mas_writer, 0, ULONG_MAX); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); - rcu_read_unlock(); + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 0); - /* Clear out tree & recreate it */ - mas_lock(&mas_writer); - mas_set_range(&mas_writer, 0, ULONG_MAX); - mas_store_gfp(&mas_writer, NULL, GFP_KERNEL); - mas_set_range(&mas_writer, 0, 0); - for (i = 0; i <= nr_entries; i++) { - mas_writer.index = i * 10; - mas_writer.last = i * 10 + 5; - mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); - } - mas_unlock(&mas_writer); + mas.index = 0; + mas.last = 5; + mas_store(&mas, NULL); + mas_reset(&mas); + mas_set(&mas, 10); + mas_walk(&mas); - /* next check */ - /* Overwrite one entry with a new value. */ - mas_reset(&mas_reader); - mas_set_range(&mas_writer, target, target + 5); - mas_set_range(&mas_reader, target, target); - rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_prev(&mas_reader, 0); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val)); - rcu_read_unlock(); + val = mas_prev(&mas, 0); + MT_BUG_ON(mt, val != NULL); + MT_BUG_ON(mt, mas.index != 0); + MT_BUG_ON(mt, mas.last != 0); + mas_unlock(&mas); - /* Restore value. */ - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); - mas_unlock(&mas_writer); + mtree_destroy(mt); - /* prev check */ - /* Overwrite one entry with a new value. */ - mas_reset(&mas_reader); - mas_set_range(&mas_writer, target, target + 5); - mas_set_range(&mas_reader, target, target); + mt_init(mt); + mtree_store_range(mt, 0, 0, xa_mk_value(0), GFP_KERNEL); + mtree_store_range(mt, 5, 5, xa_mk_value(5), GFP_KERNEL); rcu_read_lock(); - MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); - mas_next(&mas_reader, ULONG_MAX); - mas_lock(&mas_writer); - mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); - mas_unlock(&mas_writer); - MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val)); + mas_set(&mas, 5); + val = mas_prev(&mas, 4); + MT_BUG_ON(mt, val != NULL); rcu_read_unlock(); - - rcu_unregister_thread(); } -static noinline void check_rcu_threaded(struct maple_tree *mt) -{ - unsigned long i, nr_entries = 1000; - struct rcu_test_struct vals; - - vals.val_sleep = 200; - vals.loop_sleep = 110; - - rcu_register_thread(); - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); - /* Store across several slots. */ - vals.count = 1000; - vals.mt = mt; - vals.index = 8650; - vals.last = 8666; - vals.entry1 = xa_mk_value(865); - vals.entry2 = xa_mk_value(8650); - vals.entry3 = xa_mk_value(8650); - vals.range_start = 0; - vals.range_end = ULONG_MAX; - vals.seen_entry2 = 0; - vals.seen_entry3 = 0; - - run_check_rcu(mt, &vals); - mtree_destroy(mt); - - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); - - /* 4390-4395: value 439 (0x1b7) [0x36f] */ - /* Store across several slots. */ - /* Spanning store. */ - vals.count = 10000; - vals.mt = mt; - vals.index = 4390; - vals.last = 4398; - vals.entry1 = xa_mk_value(4390); - vals.entry2 = xa_mk_value(439); - vals.entry3 = xa_mk_value(439); - vals.seen_entry2 = 0; - vals.range_start = 4316; - vals.range_end = 5035; - run_check_rcu(mt, &vals); - mtree_destroy(mt); - - - /* Forward writer for rcu stress */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - rcu_stress(mt, true); - mtree_destroy(mt); - - /* Reverse writer for rcu stress */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - rcu_stress(mt, false); - mtree_destroy(mt); - - /* Slow reader test with spanning store. */ - mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); - for (i = 0; i <= nr_entries; i++) - mtree_store_range(mt, i*10, i*10 + 5, - xa_mk_value(i), GFP_KERNEL); - - /* 4390-4395: value 439 (0x1b7) [0x36f] */ - /* Store across several slots. */ - /* Spanning store. */ - vals.count = 15000; - vals.mt = mt; - vals.index = 4390; - vals.last = 4398; - vals.entry1 = xa_mk_value(4390); - vals.entry2 = xa_mk_value(439); - vals.entry3 = xa_mk_value(4391); - vals.seen_toggle = 0; - vals.seen_added = 0; - vals.seen_removed = 0; - vals.range_start = 4316; - vals.range_end = 5035; - vals.removed = 4360; - vals.added = 4396; - vals.toggle = 4347; - vals.val_sleep = 400; - vals.loop_sleep = 200; - vals.seen_entry2 = 0; - vals.seen_entry3 = 0; - vals.seen_both = 0; - vals.entry3 = xa_mk_value(438); - - run_check_rcu_slowread(mt, &vals); - rcu_unregister_thread(); -} -extern void test_kmem_cache_bulk(void); /* Test spanning writes that require balancing right sibling or right cousin */ static noinline void check_spanning_relatives(struct maple_tree *mt) @@ -37556,7 +2021,7 @@ static noinline void check_fuzzer(struct maple_tree *mt) * Also discovered issue with metadata setting. */ mt_init_flags(mt, 0); - mtree_test_store_range(mt, 0, 18446744073709551615UL, (void *)0x1); + mtree_test_store_range(mt, 0, ULONG_MAX, (void *)0x1); mtree_test_store(mt, 4, (void *)0x9); mtree_test_erase(mt, 5); mtree_test_erase(mt, 0); @@ -37612,9 +2077,9 @@ static noinline void check_fuzzer(struct maple_tree *mt) mtree_test_insert(mt, 8, (void *)0x11); mtree_test_insert(mt, 4, (void *)0x9); mtree_test_insert(mt, 2480, (void *)0x1361); - mtree_test_insert(mt, 18446744073709551615UL, + mtree_test_insert(mt, ULONG_MAX, (void *)0xffffffffffffffff); - mtree_test_erase(mt, 18446744073709551615UL); + mtree_test_erase(mt, ULONG_MAX); mtree_destroy(mt); /* @@ -37883,8 +2348,8 @@ static noinline void check_fuzzer(struct maple_tree *mt) mtree_test_insert(mt, 8, (void *)0x11); mtree_test_insert(mt, 21, (void *)0x2b); mtree_test_insert(mt, 2, (void *)0x5); - mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); - mtree_test_erase(mt, 18446744073709551605UL); + mtree_test_insert(mt, ULONG_MAX - 10, (void *)0xffffffffffffffeb); + mtree_test_erase(mt, ULONG_MAX - 10); mtree_test_store_range(mt, 0, 281, (void *)0x1); mtree_test_erase(mt, 2); mtree_test_insert(mt, 1211, (void *)0x977); @@ -37900,9 +2365,11 @@ static noinline void check_fuzzer(struct maple_tree *mt) mtree_test_insert(mt, 8, (void *)0x11); mtree_test_insert(mt, 21, (void *)0x2b); mtree_test_insert(mt, 2, (void *)0x5); - mtree_test_insert(mt, 18446744073709551605UL, (void *)0xffffffffffffffeb); - mtree_test_erase(mt, 18446744073709551605UL); + mtree_test_insert(mt, ULONG_MAX - 10, (void *)0xffffffffffffffeb); + mtree_test_erase(mt, ULONG_MAX - 10); } + +/* duplicate the tree with a specific gap */ static noinline void check_dup_gaps(struct maple_tree *mt, unsigned long nr_entries, bool zero_start, unsigned long gap) @@ -37914,7 +2381,6 @@ static noinline void check_dup_gaps(struct maple_tree *mt, MA_STATE(mas, mt, 0, 0); MA_STATE(newmas, &newmt, 0, 0); - if (!zero_start) i = 1; @@ -37925,58 +2391,78 @@ static noinline void check_dup_gaps(struct maple_tree *mt, mt_init_flags(&newmt, MT_FLAGS_ALLOC_RANGE); mt_set_non_kernel(99999); + mas_lock(&newmas); ret = mas_expected_entries(&newmas, nr_entries); mt_set_non_kernel(0); MT_BUG_ON(mt, ret != 0); + rcu_read_lock(); mas_for_each(&mas, tmp, ULONG_MAX) { newmas.index = mas.index; newmas.last = mas.last; mas_store(&newmas, tmp); } - - mas_destroy(&mas); + rcu_read_unlock(); mas_destroy(&newmas); + mas_unlock(&newmas); + mtree_destroy(&newmt); } +/* Duplicate many sizes of trees. Mainly to test expected entry values */ static noinline void check_dup(struct maple_tree *mt) { int i; + int big_start = 100010; /* Check with a value at zero */ for (i = 10; i < 1000; i++) { mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); check_dup_gaps(mt, i, true, 5); mtree_destroy(mt); + rcu_barrier(); } + cond_resched(); + mt_cache_shrink(); /* Check with a value at zero, no gap */ for (i = 1000; i < 2000; i++) { mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); check_dup_gaps(mt, i, true, 0); mtree_destroy(mt); + rcu_barrier(); } + cond_resched(); + mt_cache_shrink(); /* Check with a value at zero and unreasonably large */ - for (i = 100010; i < 100020; i++) { + for (i = big_start; i < big_start + 10; i++) { mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); check_dup_gaps(mt, i, true, 5); mtree_destroy(mt); + rcu_barrier(); } + cond_resched(); + mt_cache_shrink(); /* Small to medium size not starting at zero*/ for (i = 200; i < 1000; i++) { mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); check_dup_gaps(mt, i, false, 5); mtree_destroy(mt); + rcu_barrier(); } + cond_resched(); + mt_cache_shrink(); /* Unreasonably large not starting at zero*/ - for (i = 100010; i < 100020; i++) { + for (i = big_start; i < big_start + 10; i++) { mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); check_dup_gaps(mt, i, false, 5); mtree_destroy(mt); + rcu_barrier(); + cond_resched(); + mt_cache_shrink(); } /* Check non-allocation tree not starting at zero */ @@ -37984,22 +2470,32 @@ static noinline void check_dup(struct maple_tree *mt) mt_init_flags(mt, 0); check_dup_gaps(mt, i, false, 5); mtree_destroy(mt); + rcu_barrier(); + cond_resched(); + if (i % 2 == 0) + mt_cache_shrink(); } + mt_cache_shrink(); /* Check non-allocation tree starting at zero */ for (i = 200; i < 1000; i++) { mt_init_flags(mt, 0); check_dup_gaps(mt, i, true, 5); mtree_destroy(mt); + rcu_barrier(); + cond_resched(); } + mt_cache_shrink(); /* Unreasonably large */ - for (i = 100015; i < 100020; i++) { + for (i = big_start + 5; i < big_start + 10; i++) { mt_init_flags(mt, 0); check_dup_gaps(mt, i, true, 5); mtree_destroy(mt); + rcu_barrier(); + mt_cache_shrink(); + cond_resched(); } - } static DEFINE_MTREE(tree); @@ -38059,28 +2555,6 @@ static int maple_tree_seed(void) goto skip; #endif - test_kmem_cache_bulk(); - - mt_init_flags(&tree, 0); - check_new_node(&tree); - mtree_destroy(&tree); - - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); - check_prealloc(&tree); - mtree_destroy(&tree); - - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); - check_spanning_write(&tree); - mtree_destroy(&tree); - - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); - check_null_expand(&tree); - mtree_destroy(&tree); - - mt_init_flags(&tree, 0); - check_dfs_preorder(&tree); - mtree_destroy(&tree); - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_forking(&tree); mtree_destroy(&tree); @@ -38094,6 +2568,8 @@ static int maple_tree_seed(void) check_ranges(&tree); mtree_destroy(&tree); +#if defined(CONFIG_64BIT) + /* These tests have ranges outside of 4GB */ mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_alloc_range(&tree); mtree_destroy(&tree); @@ -38101,6 +2577,7 @@ static int maple_tree_seed(void) mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_alloc_rev_range(&tree); mtree_destroy(&tree); +#endif mt_init_flags(&tree, 0); @@ -38159,10 +2636,6 @@ static int maple_tree_seed(void) /* Clear out the tree */ mtree_destroy(&tree); - mt_init_flags(&tree, 0); - check_erase_testset(&tree); - mtree_destroy(&tree); - mt_init_flags(&tree, 0); /* * set[] = {5015, 5014, 5017, 25, 1000, @@ -38215,7 +2688,6 @@ static int maple_tree_seed(void) check_load(&tree, set[9], &tree); mtree_destroy(&tree); - check_nomem(&tree); mt_init_flags(&tree, 0); check_seq(&tree, 16, false); mtree_destroy(&tree); @@ -38242,10 +2714,6 @@ static int maple_tree_seed(void) check_prev_entry(&tree); mtree_destroy(&tree); - mt_init_flags(&tree, 0); - check_erase2_sets(&tree); - mtree_destroy(&tree); - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_gap_combining(&tree); mtree_destroy(&tree); @@ -38258,14 +2726,6 @@ static int maple_tree_seed(void) next_prev_test(&tree); mtree_destroy(&tree); - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); - check_rcu_simulated(&tree); - mtree_destroy(&tree); - - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); - check_rcu_threaded(&tree); - mtree_destroy(&tree); - mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); check_spanning_relatives(&tree); mtree_destroy(&tree); diff --git a/tools/testing/radix-tree/.gitignore b/tools/testing/radix-tree/.gitignore index c901d96dd013e..49bccb90c35ba 100644 --- a/tools/testing/radix-tree/.gitignore +++ b/tools/testing/radix-tree/.gitignore @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only +generated/bit-length.h generated/map-shift.h idr.c idr-test diff --git a/tools/testing/radix-tree/Makefile b/tools/testing/radix-tree/Makefile index 89d613e0505b3..caf32a9b96089 100644 --- a/tools/testing/radix-tree/Makefile +++ b/tools/testing/radix-tree/Makefile @@ -18,9 +18,14 @@ endif ifeq ($(BUILD), 32) CFLAGS += -m32 LDFLAGS += -m32 +LONG_BIT := 32 endif -targets: generated/map-shift.h $(TARGETS) +ifndef LONG_BIT +LONG_BIT := $(shell getconf LONG_BIT) +endif + +targets: generated/map-shift.h generated/bit-length.h $(TARGETS) main: $(OFILES) @@ -34,11 +39,11 @@ maple: $(CORE_OFILES) multiorder: multiorder.o $(CORE_OFILES) clean: - $(RM) $(TARGETS) *.o radix-tree.c idr.c generated/map-shift.h + $(RM) $(TARGETS) *.o radix-tree.c idr.c generated/map-shift.h generated/bit-length.h vpath %.c ../../lib -$(OFILES): Makefile *.h */*.h generated/map-shift.h \ +$(OFILES): Makefile *.h */*.h generated/map-shift.h generated/bit-length.h \ ../../include/linux/*.h \ ../../include/asm/*.h \ ../../../include/linux/xarray.h \ @@ -61,3 +66,11 @@ generated/map-shift.h: echo "#define XA_CHUNK_SHIFT $(SHIFT)" > \ generated/map-shift.h; \ fi + +generated/bit-length.h: FORCE + @if ! grep -qws CONFIG_$(LONG_BIT)BIT generated/bit-length.h; then \ + echo "Generating $@"; \ + echo "#define CONFIG_$(LONG_BIT)BIT 1" > $@; \ + fi + +FORCE: ; diff --git a/tools/testing/radix-tree/generated/autoconf.h b/tools/testing/radix-tree/generated/autoconf.h index e7da803502362..92dc474c349b6 100644 --- a/tools/testing/radix-tree/generated/autoconf.h +++ b/tools/testing/radix-tree/generated/autoconf.h @@ -1,2 +1,2 @@ +#include "bit-length.h" #define CONFIG_XARRAY_MULTI 1 -#define CONFIG_64BIT 1 diff --git a/tools/testing/radix-tree/linux.c b/tools/testing/radix-tree/linux.c index 2048d12c31df3..d587a558997f8 100644 --- a/tools/testing/radix-tree/linux.c +++ b/tools/testing/radix-tree/linux.c @@ -129,6 +129,10 @@ void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list) pthread_mutex_unlock(&cachep->lock); } +void kmem_cache_shrink(struct kmem_cache *cachep) +{ +} + int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size, void **p) { diff --git a/tools/testing/radix-tree/maple.c b/tools/testing/radix-tree/maple.c index 35082671928ad..2e91973fbaa66 100644 --- a/tools/testing/radix-tree/maple.c +++ b/tools/testing/radix-tree/maple.c @@ -2,11 +2,17 @@ /* * maple_tree.c: Userspace shim for maple tree test-suite * Copyright (c) 2018 Liam R. Howlett + * + * Any tests that require internal knowledge of the tree or threads and other + * difficult to handle in kernel tests. */ #define CONFIG_DEBUG_MAPLE_TREE #define CONFIG_MAPLE_SEARCH +#define MAPLE_32BIT (MAPLE_NODE_SLOTS > 31) #include "test.h" +#include +#include #define module_init(x) #define module_exit(x) @@ -18,6 +24,35717 @@ #undef CONFIG_DEBUG_MAPLE_TREE #include "../../../lib/test_maple_tree.c" +#define RCU_RANGE_COUNT 1000 +#define RCU_MT_BUG_ON(test, y) {if (y) { test->stop = true; } MT_BUG_ON(test->mt, y); } + +struct rcu_test_struct2 { + struct maple_tree *mt; + + bool start; + bool stop; + unsigned int thread_count; + + unsigned int seen_toggle; + unsigned int seen_added; + unsigned int seen_modified; + unsigned int seen_deleted; + int pause; + + unsigned long index[RCU_RANGE_COUNT]; + unsigned long last[RCU_RANGE_COUNT]; +}; + +struct rcu_reader_struct { + unsigned int id; + int mod; + int del; + int flip; + int add; + int next; + struct rcu_test_struct2 *test; +}; + +/* + * check_new_node() - Check the creation of new nodes and error path + * verification. + */ +static noinline void check_new_node(struct maple_tree *mt) +{ + + struct maple_node *mn, *mn2, *mn3; + struct maple_alloc *smn; + struct maple_node *nodes[100]; + int i, j, total; + + MA_STATE(mas, mt, 0, 0); + + /* Try allocating 3 nodes */ + mtree_lock(mt); + mt_set_non_kernel(0); + /* request 3 nodes to be allocated. */ + mas_node_count(&mas, 3); + /* Allocation request of 3. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 3); + /* Allocate failed. */ + MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, mas.alloc == NULL); + MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); + mas_push_node(&mas, mn); + mas_nomem(&mas, GFP_KERNEL); /* free */ + mtree_unlock(mt); + + + /* Try allocating 1 node, then 2 more */ + mtree_lock(mt); + /* Set allocation request to 1. */ + mas_set_alloc_req(&mas, 1); + /* Check Allocation request of 1. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); + mas_set_err(&mas, -ENOMEM); + /* Validate allocation request. */ + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + /* Eat the requested node. */ + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, mn->slot[0] != NULL); + MT_BUG_ON(mt, mn->slot[1] != NULL); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + ma_free_rcu(mn); + mas.node = MAS_START; + mas_nomem(&mas, GFP_KERNEL); + /* Allocate 3 nodes, will fail. */ + mas_node_count(&mas, 3); + /* Drop the lock and allocate 3 nodes. */ + mas_nomem(&mas, GFP_KERNEL); + /* Ensure 3 are allocated. */ + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + /* Allocation request of 0. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 0); + + MT_BUG_ON(mt, mas.alloc == NULL); + MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); + MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); + /* Ensure we counted 3. */ + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + /* Free. */ + mas_nomem(&mas, GFP_KERNEL); + + /* Set allocation request to 1. */ + mas_set_alloc_req(&mas, 1); + MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); + mas_set_err(&mas, -ENOMEM); + /* Validate allocation request. */ + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_allocated(&mas) != 1); + /* Check the node is only one node. */ + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, mn->slot[0] != NULL); + MT_BUG_ON(mt, mn->slot[1] != NULL); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != 1); + MT_BUG_ON(mt, mas.alloc->node_count); + + mas_set_alloc_req(&mas, 2); /* request 2 more. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 2); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_allocated(&mas) != 3); + MT_BUG_ON(mt, mas.alloc == NULL); + MT_BUG_ON(mt, mas.alloc->slot[0] == NULL); + MT_BUG_ON(mt, mas.alloc->slot[1] == NULL); + for (i = 2; i >= 0; i--) { + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != i); + MT_BUG_ON(mt, !mn); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + + total = 64; + mas_set_alloc_req(&mas, total); /* request 2 more. */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != total); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + for (i = total; i > 0; i--) { + unsigned int e = 0; /* expected node_count */ + + if (!MAPLE_32BIT) { + if (i >= 35) + e = i - 35; + else if (i >= 5) + e = i - 5; + else if (i >= 2) + e = i - 2; + } else { + if (i >= 4) + e = i - 4; + else if (i == 3) + e = i - 2; + else + e = 0; + } + + MT_BUG_ON(mt, mas.alloc->node_count != e); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != i - 1); + MT_BUG_ON(mt, !mn); + ma_free_rcu(mn); + } + + total = 100; + for (i = 1; i < total; i++) { + mas_set_alloc_req(&mas, i); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + for (j = i; j > 0; j--) { + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); + MT_BUG_ON(mt, !mn); + MT_BUG_ON(mt, not_empty(mn)); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != j); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != j - 1); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + mas_set_alloc_req(&mas, i); + mas_set_err(&mas, -ENOMEM); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + for (j = 0; j <= i/2; j++) { + MT_BUG_ON(mt, mas_allocated(&mas) != i - j); + nodes[j] = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); + } + + while (j) { + j--; + mas_push_node(&mas, nodes[j]); + MT_BUG_ON(mt, mas_allocated(&mas) != i - j); + } + MT_BUG_ON(mt, mas_allocated(&mas) != i); + for (j = 0; j <= i/2; j++) { + MT_BUG_ON(mt, mas_allocated(&mas) != i - j); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + MT_BUG_ON(mt, mas_allocated(&mas) != i - j - 1); + } + MT_BUG_ON(mt, mas_nomem(&mas, GFP_KERNEL)); + + } + + /* Set allocation request. */ + total = 500; + mas_node_count(&mas, total); + /* Drop the lock and allocate the nodes. */ + mas_nomem(&mas, GFP_KERNEL); + MT_BUG_ON(mt, !mas.alloc); + i = 1; + smn = mas.alloc; + while (i < total) { + for (j = 0; j < MAPLE_ALLOC_SLOTS; j++) { + i++; + MT_BUG_ON(mt, !smn->slot[j]); + if (i == total) + break; + } + smn = smn->slot[0]; /* next. */ + } + MT_BUG_ON(mt, mas_allocated(&mas) != total); + mas_nomem(&mas, GFP_KERNEL); /* Free. */ + + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + for (i = 1; i < 128; i++) { + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ + for (j = i; j > 0; j--) { /*Free the requests */ + mn = mas_pop_node(&mas); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + } + + for (i = 1; i < MAPLE_NODE_MASK + 1; i++) { + MA_STATE(mas2, mt, 0, 0); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != i); /* check request filled */ + for (j = 1; j <= i; j++) { /* Move the allocations to mas2 */ + mn = mas_pop_node(&mas); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + mas_push_node(&mas2, mn); + MT_BUG_ON(mt, mas_allocated(&mas2) != j); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + MT_BUG_ON(mt, mas_allocated(&mas2) != i); + + for (j = i; j > 0; j--) { /*Free the requests */ + MT_BUG_ON(mt, mas_allocated(&mas2) != j); + mn = mas_pop_node(&mas2); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas2) != 0); + } + + + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 1); /* Request */ + MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); + + mn = mas_pop_node(&mas); /* get the next node. */ + MT_BUG_ON(mt, mn == NULL); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 2); + + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); + + /* Check the limit of pop/push/pop */ + mas_node_count(&mas, MAPLE_ALLOC_SLOTS + 2); /* Request */ + MT_BUG_ON(mt, mas_alloc_req(&mas) != 1); + MT_BUG_ON(mt, mas.node != MA_ERROR(-ENOMEM)); + MT_BUG_ON(mt, !mas_nomem(&mas, GFP_KERNEL)); + MT_BUG_ON(mt, mas_alloc_req(&mas)); + MT_BUG_ON(mt, mas.alloc->node_count); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 1); + MT_BUG_ON(mt, mas.alloc->node_count != MAPLE_ALLOC_SLOTS - 1); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas.alloc->node_count); + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS + 2); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + for (i = 1; i <= MAPLE_ALLOC_SLOTS + 1; i++) { + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, not_empty(mn)); + ma_free_rcu(mn); + } + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + + for (i = 3; i < MAPLE_NODE_MASK * 3; i++) { + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + mas_push_node(&mas, mn); /* put it back */ + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + mn2 = mas_pop_node(&mas); /* get the next node. */ + mas_push_node(&mas, mn); /* put them back */ + mas_push_node(&mas, mn2); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + mn2 = mas_pop_node(&mas); /* get the next node. */ + mn3 = mas_pop_node(&mas); /* get the next node. */ + mas_push_node(&mas, mn); /* put them back */ + mas_push_node(&mas, mn2); + mas_push_node(&mas, mn3); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, i); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mn = mas_pop_node(&mas); /* get the next node. */ + ma_free_rcu(mn); + mas_destroy(&mas); + } + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, 5); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != 5); + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, 10); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mas.node = MAS_START; + MT_BUG_ON(mt, mas_allocated(&mas) != 10); + mas_destroy(&mas); + + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, MAPLE_ALLOC_SLOTS - 1); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + MT_BUG_ON(mt, mas_allocated(&mas) != MAPLE_ALLOC_SLOTS - 1); + mas.node = MA_ERROR(-ENOMEM); + mas_node_count(&mas, 10 + MAPLE_ALLOC_SLOTS - 1); /* Request */ + mas_nomem(&mas, GFP_KERNEL); /* Fill request */ + mas.node = MAS_START; + MT_BUG_ON(mt, mas_allocated(&mas) != 10 + MAPLE_ALLOC_SLOTS - 1); + mas_destroy(&mas); + + mtree_unlock(mt); +} + +/* + * Check erasing including RCU. + */ +static noinline void check_erase(struct maple_tree *mt, unsigned long index, + void *ptr) +{ + MT_BUG_ON(mt, mtree_test_erase(mt, index) != ptr); +} + +#define erase_check_load(mt, i) check_load(mt, set[i], entry[i%2]) +#define erase_check_insert(mt, i) check_insert(mt, set[i], entry[i%2]) +#define erase_check_erase(mt, i) check_erase(mt, set[i], entry[i%2]) + +static noinline void check_erase_testset(struct maple_tree *mt) +{ + unsigned long set[] = { 5015, 5014, 5017, 25, 1000, + 1001, 1002, 1003, 1005, 0, + 6003, 6002, 6008, 6012, 6015, + 7003, 7002, 7008, 7012, 7015, + 8003, 8002, 8008, 8012, 8015, + 9003, 9002, 9008, 9012, 9015, + 10003, 10002, 10008, 10012, 10015, + 11003, 11002, 11008, 11012, 11015, + 12003, 12002, 12008, 12012, 12015, + 13003, 13002, 13008, 13012, 13015, + 14003, 14002, 14008, 14012, 14015, + 15003, 15002, 15008, 15012, 15015, + }; + + + void *ptr = &set; + void *entry[2] = { ptr, mt }; + void *root_node; + + + rcu_register_thread(); + mt_set_in_rcu(mt); + for (int i = 0; i < 4; i++) + erase_check_insert(mt, i); + for (int i = 0; i < 4; i++) + erase_check_load(mt, i); + + mt_set_non_kernel(2); + erase_check_erase(mt, 1); + erase_check_load(mt, 0); + check_load(mt, set[1], NULL); + for (int i = 2; i < 4; i++) + erase_check_load(mt, i); + + + erase_check_erase(mt, 2); + erase_check_load(mt, 0); + check_load(mt, set[1], NULL); + check_load(mt, set[2], NULL); + + erase_check_insert(mt, 1); + erase_check_insert(mt, 2); + + for (int i = 0; i < 4; i++) + erase_check_load(mt, i); + + /* Check erase and load without an allocation. */ + erase_check_load(mt, 3); + erase_check_erase(mt, 1); + erase_check_load(mt, 0); + check_load(mt, set[1], NULL); + for (int i = 2; i < 4; i++) + erase_check_load(mt, i); + + /* + * Set the newly erased node. This will produce a different allocated + * node to avoid busy slots. + */ + root_node = mt->ma_root; + erase_check_insert(mt, 1); + + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + erase_check_load(mt, 2); + check_load(mt, 5018, NULL); + erase_check_load(mt, 3); + + erase_check_erase(mt, 2); /* erase 5017 to check append */ + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + check_load(mt, set[2], NULL); + check_load(mt, 5018, NULL); + + erase_check_load(mt, 3); + + root_node = mt->ma_root; + erase_check_insert(mt, 2); + + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + erase_check_load(mt, 2); + check_load(mt, 5018, NULL); + erase_check_load(mt, 3); + + mt_set_non_kernel(1); + erase_check_erase(mt, 2); /* erase 5017 to check append */ + erase_check_load(mt, 0); + check_load(mt, 5016, NULL); + check_load(mt, set[2], NULL); + erase_check_erase(mt, 0); /* erase 5015 to check append */ + check_load(mt, set[0], NULL); + check_load(mt, 5016, NULL); + erase_check_insert(mt, 4); /* 1000 < Should not split. */ + check_load(mt, set[0], NULL); + check_load(mt, 5016, NULL); + erase_check_load(mt, 1); + check_load(mt, 5013, NULL); + check_load(mt, set[2], NULL); + check_load(mt, 5018, NULL); + erase_check_load(mt, 4); + check_load(mt, 999, NULL); + check_load(mt, 1001, NULL); + erase_check_load(mt, 4); + if (mt_in_rcu(mt)) + MT_BUG_ON(mt, root_node == mt->ma_root); + else + MT_BUG_ON(mt, root_node != mt->ma_root); + + /* Should not have split. */ + MT_BUG_ON(mt, !mte_is_leaf(mt->ma_root)); + + + /* Coalesce testing */ + erase_check_insert(mt, 0); + erase_check_insert(mt, 2); + + for (int i = 5; i < 25; i++) { + erase_check_insert(mt, i); + for (int j = i; j >= 0; j--) + erase_check_load(mt, j); + } + + erase_check_erase(mt, 14); /*6015 */ + for (int i = 0; i < 25; i++) { + if (i == 14) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + erase_check_erase(mt, 16); /*7002 */ + for (int i = 0; i < 25; i++) { + if (i == 16 || i == 14) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + + mt_set_non_kernel(1); + erase_check_erase(mt, 13); /*6012 */ + for (int i = 0; i < 25; i++) { + if (i == 16 || i == 14 || i == 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 15); /*7003 */ + for (int i = 0; i < 25; i++) { + if (i <= 16 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + mt_set_non_kernel(2); + erase_check_erase(mt, 17); /*7008 *should* cause coalesce. */ + for (int i = 0; i < 25; i++) { + if (i <= 17 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 18); /*7012 */ + for (int i = 0; i < 25; i++) { + if (i <= 18 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + mt_set_non_kernel(2); + erase_check_erase(mt, 19); /*7015 */ + for (int i = 0; i < 25; i++) { + if (i <= 19 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 20); /*8003 */ + for (int i = 0; i < 25; i++) { + if (i <= 20 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + erase_check_erase(mt, 21); /*8002 */ + for (int i = 0; i < 25; i++) { + if (i <= 21 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + mt_set_non_kernel(2); + erase_check_erase(mt, 22); /*8008 */ + for (int i = 0; i < 25; i++) { + if (i <= 22 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + for (int i = 23; i < 25; i++) + erase_check_erase(mt, i); + + for (int i = 0; i < 25; i++) { + if (i <= 25 && i >= 13) + check_load(mt, set[i], NULL); + else + erase_check_load(mt, i); + } + + /* Shrinking tree test. */ + + for (int i = 13; i < ARRAY_SIZE(set); i++) + erase_check_insert(mt, i); + + mt_set_non_kernel(99); + for (int i = 18; i < ARRAY_SIZE(set); i++) { + erase_check_erase(mt, i); + for (int j = 0; j < ARRAY_SIZE(set); j++) { + if (j < 18 || j > i) + erase_check_load(mt, j); + else + check_load(mt, set[j], NULL); + } + } + mt_set_non_kernel(35); + for (int i = 0; i < 18; i++) { + erase_check_erase(mt, i); + for (int j = 0; j < ARRAY_SIZE(set); j++) { + if (j < 18 && j > i) + erase_check_load(mt, j); + else + check_load(mt, set[j], NULL); + } + } + erase_check_insert(mt, 8); + erase_check_insert(mt, 9); + erase_check_erase(mt, 8); + rcu_unregister_thread(); +} + +/* End of erase testing */ + +/* VM Generated Crashes - uses its own tree walk for verification */ +#define erase_check_store_range(mt, a, i, ptr) mtree_test_store_range(mt, \ + a[(i)], a[(i + 1)], ptr) +#define STORE 1 +#define SNULL 2 +#define ERASE 3 +#define ec_type_str(x) \ + (((x) == STORE) ? \ + "STORE" : \ + (((x) == SNULL) ? \ + "SNULL" : "ERASE") \ + ) +#define check_erase2_debug 0 + +/* Calculate the overwritten entries. */ +int mas_ce2_over_count(struct ma_state *mas_start, struct ma_state *mas_end, + void *s_entry, unsigned long s_min, + void *e_entry, unsigned long e_max, + unsigned long *set, int i, bool null_entry) +{ + int count = 0, span = 0; + unsigned long retry = 0; + void *entry; + struct ma_state tmp; + + + /* count slots */ + memcpy(&tmp, mas_start, sizeof(tmp)); + entry = mas_next(&tmp, mas_end->last); + while (entry) { + BUG_ON(retry > 50); /* stop infinite retry on testing. */ + if (xa_is_zero(s_entry)) { + retry++; + continue; + } + count++; + span++; + entry = mas_next(&tmp, mas_end->last); + } + + if (null_entry) { + /* Check splitting end. */ + if (e_entry && (e_max > mas_end->last)) + count--; + + /* check overwrite of entire start */ + if (s_entry && (s_min == mas_start->index)) + count++; + } else { /* !null_entry (store) */ + bool esplit = e_max > mas_end->last; + bool ssplit = s_min != mas_start->index; + + if (s_entry && e_entry) { + if (esplit && ssplit) + count--; + else if (ssplit) + count--; + else if (esplit) { + if (span) + count--; + } + } else if (s_entry && !e_entry) { + if (ssplit) + count--; + } else if (!s_entry && e_entry) { + if (esplit) + count--; + count--; + } else { + count--; + } + } + return count; +} + +/* + * mas_node_walk() - Walk a maple node to offset of the index. + * @mas: The maple state + * @type: The maple node type + * @*range_min: Pointer to store the minimum range of the offset + * @*range_max: Pointer to store the maximum range of the offset + * + * The offset will be stored in the maple state. + * + */ +static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node, + enum maple_type type, unsigned long *range_min, + unsigned long *range_max) + +{ + unsigned long *pivots; + unsigned char count; + unsigned long prev, max; + unsigned char offset; + unsigned long index; + + if (unlikely(ma_is_dense(type))) { + (*range_max) = (*range_min) = mas->index; + if (unlikely(ma_dead_node(node))) + return; + + mas->offset = mas->index = mas->min; + return; + } + + pivots = ma_pivots(node, type); + max = pivots[0]; + if (unlikely(ma_dead_node(node))) + return; + + offset = 0; + prev = mas->min; + index = mas->index; + if (unlikely(index <= max)) + goto offset_zero; + + count = mt_pivots[type]; + while (++offset < count) { + prev = max; + max = pivots[offset]; + if (unlikely(ma_dead_node(node))) + return; + + if (index <= max) + goto offset_found; + else if (unlikely(!max)) + goto mas_max; + } + + prev = max; +mas_max: + max = mas->max; +offset_found: + prev++; +offset_zero: + mas->offset = offset; + if (ma_is_leaf(type)) { + *range_max = max; + *range_min = prev; + } else { + mas->max = max; + mas->min = prev; + } +} + +/* + * mas_descend_walk(): Locates a value and sets the mas->node and slot + * accordingly. range_min and range_max are set to the range which the entry is + * valid. + * @mas: The maple state + * @*range_min: A pointer to store the minimum of the range + * @*range_max: A pointer to store the maximum of the range + * + * Check mas->node is still valid on return of any value. + * + * Return: true if pointing to a valid node and offset. False otherwise. + */ +static inline bool mas_descend_walk(struct ma_state *mas, + unsigned long *range_min, unsigned long *range_max) +{ + struct maple_enode *next; + struct maple_node *node; + enum maple_type type; + + next = mas->node; + while (true) { + node = mte_to_node(next); + type = mte_node_type(next); + mas_node_walk(mas, node, type, range_min, range_max); + next = mas_slot(mas, ma_slots(node, type), mas->offset); + if (unlikely(ma_dead_node(node))) + return false; + + if (unlikely(ma_is_leaf(type))) + return true; + + /* Descend. */ + mas->node = next; + } + return false; +} + +/* + * mas_tree_walk() - Walk to @mas->index and set the range values. + * @mas: The maple state. + * @*range_min: The minimum range to be set. + * @*range_max: The maximum range to be set. + * + * Ranges are only valid if there is a valid entry at @mas->index. + * + * Return: True if a value exists, false otherwise. + */ +static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min, + unsigned long *range_max) +{ + bool ret; + +retry: + ret = false; + mas_start(mas); + if (mas_is_none(mas)) + goto not_found; + + if (mas_is_ptr(mas)) { + *range_min = *range_max = 0; + if (!mas->index) + return true; + + goto not_found; + } + + ret = mas_descend_walk(mas, range_min, range_max); + if (unlikely(mte_dead_node(mas->node))) { + mas->node = MAS_START; + goto retry; + } + + return ret; + +not_found: + mas->offset = MAPLE_NODE_SLOTS; + return false; +} + +static inline void *mas_range_load(struct ma_state *mas, + unsigned long *range_min, unsigned long *range_max) + +{ + void *entry = NULL; + unsigned long index = mas->index; + + if (mas_is_none(mas) || mas_is_paused(mas)) + mas->node = MAS_START; +retry: + if (mas_tree_walk(mas, range_min, range_max)) + if (unlikely(mas->node == MAS_ROOT)) + return mas_root(mas); + + if (likely(mas->offset != MAPLE_NODE_SLOTS)) + entry = mas_get_slot(mas, mas->offset); + + if (mas_dead_node(mas, index)) + goto retry; + + return entry; +} + +#if defined(CONFIG_64BIT) +static noinline void check_erase2_testset(struct maple_tree *mt, + unsigned long *set, unsigned long size) +{ + int entry_count = 0; + int check = 0; + void *foo; + unsigned long addr = 0; + void *s_entry = NULL, *e_entry = NULL; + + MA_STATE(mas, mt, 0, 0); + + for (int i = 0; i < size; i += 3) { + unsigned long s_min, s_max; + unsigned long e_min, e_max; + void *value = NULL; + + MA_STATE(mas_start, mt, set[i+1], set[i+1]); + MA_STATE(mas_end, mt, set[i+2], set[i+2]); + mt_set_non_kernel(127); +#if check_erase2_debug + pr_err("%s: %d %s %lu - %lu\n", __func__, i, + ec_type_str(set[i]), + set[i+1], set[i+2]); +#endif + s_entry = mas_range_load(&mas_start, &s_min, &s_max); + e_entry = mas_range_load(&mas_end, &e_min, &e_max); + + switch (set[i]) { + case SNULL: + if ((s_min == set[i+1]) && (s_max == set[i+2])) { + if (s_entry) + entry_count--; + } else if ((s_min != set[i+1]) && (s_max != set[i+2])) { + entry_count++; + } else if ((mas_start.node != mas_end.node) || + (mas_start.offset != mas_end.offset)) { + entry_count -= + mas_ce2_over_count(&mas_start, &mas_end, + s_entry, s_min, + e_entry, e_max, set, i, + true); + } + + + erase_check_store_range(mt, set, i + 1, value); + break; + case STORE: + value = xa_mk_value(set[i + 1]); + if (mas_start.offset > mt_slot_count(mas_start.node)) { + entry_count++; /* appending an entry. */ + } else if ((s_min == e_min) && (s_max == e_max)) { + if (!entry_count) + entry_count++; + + else if (s_entry) { + if (e_max > mas_end.last) + entry_count++; + + if (s_min < mas_start.index) + entry_count++; + + } else { + entry_count++; + } + } else { + entry_count -= + mas_ce2_over_count(&mas_start, &mas_end, + s_entry, s_min, + e_entry, e_max, set, i, + false); + } + + erase_check_store_range(mt, set, i + 1, value); + break; + case ERASE: + if (!s_entry) + break; + check_erase(mt, set[i+1], xa_mk_value(set[i+1])); + entry_count--; + break; + } + mt_validate(mt); + if (entry_count) + MT_BUG_ON(mt, !mt_height(mt)); +#if check_erase2_debug > 1 + mt_dump(mt); +#endif +#if check_erase2_debug + pr_err("Done\n"); +#endif + + check = 0; + addr = 0; + mt_for_each(mt, foo, addr, ULONG_MAX) { + check++; +#if check_erase2_debug > 2 + pr_err("mt: %lu -> %p (%d)\n", addr+1, foo, check); +#endif + if (check > entry_count) + break; + } + +#if check_erase2_debug > 2 + pr_err("mt_for_each %d and count %d\n", check, entry_count); +#endif + + MT_BUG_ON(mt, check != entry_count); + + check = 0; + addr = 0; + mas_reset(&mas); + mas.index = 0; + rcu_read_lock(); + mas_for_each(&mas, foo, ULONG_MAX) { + if (xa_is_zero(foo)) { + if (addr == mas.index) { + mt_dump(mas.tree); + pr_err("retry failed %lu - %lu\n", + mas.index, mas.last); + MT_BUG_ON(mt, 1); + } + addr = mas.index; + continue; + } +#if check_erase2_debug > 2 + pr_err("mas: %lu -> %p\n", mas.index, foo); +#endif + check++; + if (check > entry_count) + break; + } + rcu_read_unlock(); +#if check_erase2_debug > 2 + pr_err("mas_for_each %d and count %d\n", check, entry_count); + mt_validate(mt); +#endif + + MT_BUG_ON(mt, check != entry_count); + + MT_BUG_ON(mt, mtree_load(mas.tree, 0) != NULL); + } +} + + +/* These tests were pulled from KVM tree modifications which failed. */ +static noinline void check_erase2_sets(struct maple_tree *mt) +{ + void *entry; + unsigned long start = 0; + unsigned long set[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140721266458624, 140737488351231, +ERASE, 140721266458624, 140737488351231, +STORE, 140721266458624, 140721266462719, +STORE, 94735788949504, 94735789121535, +ERASE, 94735788949504, 94735789121535, +STORE, 94735788949504, 94735788965887, +STORE, 94735788965888, 94735789121535, +ERASE, 94735788965888, 94735789121535, +STORE, 94735788965888, 94735789068287, +STORE, 94735789068288, 94735789109247, +STORE, 94735789109248, 94735789121535, +STORE, 140253902692352, 140253902864383, +ERASE, 140253902692352, 140253902864383, +STORE, 140253902692352, 140253902696447, +STORE, 140253902696448, 140253902864383, + }; + unsigned long set2[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735933583360, 140737488351231, +ERASE, 140735933583360, 140737488351231, +STORE, 140735933583360, 140735933587455, +STORE, 94811003260928, 94811003432959, +ERASE, 94811003260928, 94811003432959, +STORE, 94811003260928, 94811003277311, +STORE, 94811003277312, 94811003432959, +ERASE, 94811003277312, 94811003432959, +STORE, 94811003277312, 94811003379711, +STORE, 94811003379712, 94811003420671, +STORE, 94811003420672, 94811003432959, +STORE, 140277094653952, 140277094825983, +ERASE, 140277094653952, 140277094825983, +STORE, 140277094653952, 140277094658047, +STORE, 140277094658048, 140277094825983, +ERASE, 140277094658048, 140277094825983, +STORE, 140277094658048, 140277094780927, +STORE, 140277094780928, 140277094813695, +STORE, 140277094813696, 140277094821887, +STORE, 140277094821888, 140277094825983, +STORE, 140735933906944, 140735933911039, + }; + unsigned long set3[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735790264320, 140737488351231, +ERASE, 140735790264320, 140737488351231, +STORE, 140735790264320, 140735790268415, +STORE, 94016597282816, 94016597454847, +ERASE, 94016597282816, 94016597454847, +STORE, 94016597282816, 94016597299199, +STORE, 94016597299200, 94016597454847, +ERASE, 94016597299200, 94016597454847, +STORE, 94016597299200, 94016597401599, +STORE, 94016597401600, 94016597442559, +STORE, 94016597442560, 94016597454847, +STORE, 140496959283200, 140496959455231, +ERASE, 140496959283200, 140496959455231, +STORE, 140496959283200, 140496959287295, +STORE, 140496959287296, 140496959455231, +ERASE, 140496959287296, 140496959455231, +STORE, 140496959287296, 140496959410175, +STORE, 140496959410176, 140496959442943, +STORE, 140496959442944, 140496959451135, +STORE, 140496959451136, 140496959455231, +STORE, 140735791718400, 140735791722495, +STORE, 140735791706112, 140735791718399, +STORE, 47135835713536, 47135835721727, +STORE, 47135835721728, 47135835729919, +STORE, 47135835729920, 47135835893759, +ERASE, 47135835729920, 47135835893759, +STORE, 47135835729920, 47135835742207, +STORE, 47135835742208, 47135835893759, +STORE, 47135835840512, 47135835893759, +STORE, 47135835742208, 47135835840511, +ERASE, 47135835742208, 47135835840511, +STORE, 47135835742208, 47135835840511, +STORE, 47135835885568, 47135835893759, +STORE, 47135835840512, 47135835885567, +ERASE, 47135835840512, 47135835885567, +STORE, 47135835840512, 47135835893759, +ERASE, 47135835840512, 47135835893759, +STORE, 47135835840512, 47135835885567, +STORE, 47135835885568, 47135835893759, + }; + + unsigned long set4[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140728251703296, 140737488351231, +ERASE, 140728251703296, 140737488351231, +STORE, 140728251703296, 140728251707391, +STORE, 94668429205504, 94668429377535, +ERASE, 94668429205504, 94668429377535, +STORE, 94668429205504, 94668429221887, +STORE, 94668429221888, 94668429377535, +ERASE, 94668429221888, 94668429377535, +STORE, 94668429221888, 94668429324287, +STORE, 94668429324288, 94668429365247, +STORE, 94668429365248, 94668429377535, +STORE, 47646523273216, 47646523445247, +ERASE, 47646523273216, 47646523445247, +STORE, 47646523273216, 47646523277311, +STORE, 47646523277312, 47646523445247, +ERASE, 47646523277312, 47646523445247, +STORE, 47646523277312, 47646523400191, + }; + + unsigned long set5[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140726874062848, 140737488351231, +ERASE, 140726874062848, 140737488351231, +STORE, 140726874062848, 140726874066943, +STORE, 94248892870656, 94248893042687, +ERASE, 94248892870656, 94248893042687, +STORE, 94248892870656, 94248892887039, +STORE, 94248892887040, 94248893042687, +ERASE, 94248892887040, 94248893042687, +STORE, 94248892887040, 94248892989439, +STORE, 94248892989440, 94248893030399, +STORE, 94248893030400, 94248893042687, +STORE, 47884786266112, 47884786438143, +ERASE, 47884786266112, 47884786438143, +STORE, 47884786266112, 47884786270207, +STORE, 47884786270208, 47884786438143, +ERASE, 47884786270208, 47884786438143, +STORE, 47884786270208, 47884786393087, +STORE, 47884786393088, 47884786425855, +STORE, 47884786425856, 47884786434047, +STORE, 47884786434048, 47884786438143, +STORE, 140726874513408, 140726874517503, +STORE, 140726874501120, 140726874513407, +STORE, 47884786438144, 47884786446335, +STORE, 47884786446336, 47884786454527, +STORE, 47884786454528, 47884786618367, +ERASE, 47884786454528, 47884786618367, +STORE, 47884786454528, 47884786466815, +STORE, 47884786466816, 47884786618367, +STORE, 47884786565120, 47884786618367, +STORE, 47884786466816, 47884786565119, +ERASE, 47884786466816, 47884786565119, +STORE, 47884786466816, 47884786565119, +STORE, 47884786610176, 47884786618367, +STORE, 47884786565120, 47884786610175, +ERASE, 47884786565120, 47884786610175, +STORE, 47884786565120, 47884786618367, +ERASE, 47884786565120, 47884786618367, +STORE, 47884786565120, 47884786610175, +STORE, 47884786610176, 47884786618367, +ERASE, 47884786610176, 47884786618367, +STORE, 47884786610176, 47884786618367, +STORE, 47884786618368, 47884789669887, +STORE, 47884787163136, 47884789669887, +STORE, 47884786618368, 47884787163135, +ERASE, 47884787163136, 47884789669887, +STORE, 47884787163136, 47884789448703, +STORE, 47884789448704, 47884789669887, +STORE, 47884788858880, 47884789448703, +STORE, 47884787163136, 47884788858879, +ERASE, 47884787163136, 47884788858879, +STORE, 47884787163136, 47884788858879, +STORE, 47884789444608, 47884789448703, +STORE, 47884788858880, 47884789444607, +ERASE, 47884788858880, 47884789444607, +STORE, 47884788858880, 47884789444607, +STORE, 47884789653504, 47884789669887, +STORE, 47884789448704, 47884789653503, +ERASE, 47884789448704, 47884789653503, +STORE, 47884789448704, 47884789653503, +ERASE, 47884789653504, 47884789669887, +STORE, 47884789653504, 47884789669887, +STORE, 47884789669888, 47884791508991, +STORE, 47884789809152, 47884791508991, +STORE, 47884789669888, 47884789809151, +ERASE, 47884789809152, 47884791508991, +STORE, 47884789809152, 47884791468031, +STORE, 47884791468032, 47884791508991, +STORE, 47884791152640, 47884791468031, +STORE, 47884789809152, 47884791152639, +ERASE, 47884789809152, 47884791152639, +STORE, 47884789809152, 47884791152639, +STORE, 47884791463936, 47884791468031, +STORE, 47884791152640, 47884791463935, +ERASE, 47884791152640, 47884791463935, +STORE, 47884791152640, 47884791463935, +STORE, 47884791492608, 47884791508991, +STORE, 47884791468032, 47884791492607, +ERASE, 47884791468032, 47884791492607, +STORE, 47884791468032, 47884791492607, +ERASE, 47884791492608, 47884791508991, +STORE, 47884791492608, 47884791508991, +STORE, 47884791508992, 47884791644159, +ERASE, 47884791508992, 47884791644159, +STORE, 47884791508992, 47884791533567, +STORE, 47884791533568, 47884791644159, +STORE, 47884791595008, 47884791644159, +STORE, 47884791533568, 47884791595007, +ERASE, 47884791533568, 47884791595007, +STORE, 47884791533568, 47884791595007, +STORE, 47884791619584, 47884791644159, +STORE, 47884791595008, 47884791619583, +ERASE, 47884791595008, 47884791619583, +STORE, 47884791595008, 47884791644159, +ERASE, 47884791595008, 47884791644159, +STORE, 47884791595008, 47884791619583, +STORE, 47884791619584, 47884791644159, +STORE, 47884791627776, 47884791644159, +STORE, 47884791619584, 47884791627775, +ERASE, 47884791619584, 47884791627775, +STORE, 47884791619584, 47884791627775, +ERASE, 47884791627776, 47884791644159, +STORE, 47884791627776, 47884791644159, +STORE, 47884791644160, 47884791664639, +ERASE, 47884791644160, 47884791664639, +STORE, 47884791644160, 47884791648255, +STORE, 47884791648256, 47884791664639, +STORE, 47884791652352, 47884791664639, +STORE, 47884791648256, 47884791652351, +ERASE, 47884791648256, 47884791652351, +STORE, 47884791648256, 47884791652351, +STORE, 47884791656448, 47884791664639, +STORE, 47884791652352, 47884791656447, +ERASE, 47884791652352, 47884791656447, +STORE, 47884791652352, 47884791664639, +ERASE, 47884791652352, 47884791664639, +STORE, 47884791652352, 47884791656447, +STORE, 47884791656448, 47884791664639, +ERASE, 47884791656448, 47884791664639, +STORE, 47884791656448, 47884791664639, +STORE, 47884791664640, 47884791672831, +ERASE, 47884791468032, 47884791492607, +STORE, 47884791468032, 47884791484415, +STORE, 47884791484416, 47884791492607, +ERASE, 47884791656448, 47884791664639, +STORE, 47884791656448, 47884791660543, +STORE, 47884791660544, 47884791664639, +ERASE, 47884791619584, 47884791627775, +STORE, 47884791619584, 47884791623679, +STORE, 47884791623680, 47884791627775, + }; + + unsigned long set6[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722999021568, 140737488351231, +ERASE, 140722999021568, 140737488351231, +STORE, 140722999021568, 140722999025663, +STORE, 94901500268544, 94901500440575, +ERASE, 94901500268544, 94901500440575, +STORE, 94901500268544, 94901500284927, +STORE, 94901500284928, 94901500440575, +ERASE, 94901500284928, 94901500440575, +STORE, 94901500284928, 94901500387327, +STORE, 94901500387328, 94901500428287, +STORE, 94901500428288, 94901500440575, +STORE, 47430426660864, 47430426832895, +ERASE, 47430426660864, 47430426832895, +STORE, 47430426660864, 47430426664959, +STORE, 47430426664960, 47430426832895, +ERASE, 47430426664960, 47430426832895, +STORE, 47430426664960, 47430426787839, +STORE, 47430426787840, 47430426820607, +STORE, 47430426820608, 47430426828799, +STORE, 47430426828800, 47430426832895, +STORE, 140722999115776, 140722999119871, +STORE, 140722999103488, 140722999115775, +STORE, 47430426832896, 47430426841087, +STORE, 47430426841088, 47430426849279, +STORE, 47430426849280, 47430427013119, +ERASE, 47430426849280, 47430427013119, +STORE, 47430426849280, 47430426861567, +STORE, 47430426861568, 47430427013119, +STORE, 47430426959872, 47430427013119, +STORE, 47430426861568, 47430426959871, +ERASE, 47430426861568, 47430426959871, +STORE, 47430426861568, 47430426959871, +STORE, 47430427004928, 47430427013119, +STORE, 47430426959872, 47430427004927, +ERASE, 47430426959872, 47430427004927, +STORE, 47430426959872, 47430427013119, +ERASE, 47430426959872, 47430427013119, +STORE, 47430426959872, 47430427004927, +STORE, 47430427004928, 47430427013119, +ERASE, 47430427004928, 47430427013119, +STORE, 47430427004928, 47430427013119, +STORE, 47430427013120, 47430430064639, +STORE, 47430427557888, 47430430064639, +STORE, 47430427013120, 47430427557887, +ERASE, 47430427557888, 47430430064639, +STORE, 47430427557888, 47430429843455, +STORE, 47430429843456, 47430430064639, +STORE, 47430429253632, 47430429843455, +STORE, 47430427557888, 47430429253631, +ERASE, 47430427557888, 47430429253631, +STORE, 47430427557888, 47430429253631, +STORE, 47430429839360, 47430429843455, +STORE, 47430429253632, 47430429839359, +ERASE, 47430429253632, 47430429839359, +STORE, 47430429253632, 47430429839359, +STORE, 47430430048256, 47430430064639, +STORE, 47430429843456, 47430430048255, +ERASE, 47430429843456, 47430430048255, +STORE, 47430429843456, 47430430048255, +ERASE, 47430430048256, 47430430064639, +STORE, 47430430048256, 47430430064639, +STORE, 47430430064640, 47430431903743, +STORE, 47430430203904, 47430431903743, +STORE, 47430430064640, 47430430203903, +ERASE, 47430430203904, 47430431903743, +STORE, 47430430203904, 47430431862783, +STORE, 47430431862784, 47430431903743, +STORE, 47430431547392, 47430431862783, +STORE, 47430430203904, 47430431547391, +ERASE, 47430430203904, 47430431547391, +STORE, 47430430203904, 47430431547391, +STORE, 47430431858688, 47430431862783, +STORE, 47430431547392, 47430431858687, +ERASE, 47430431547392, 47430431858687, +STORE, 47430431547392, 47430431858687, +STORE, 47430431887360, 47430431903743, +STORE, 47430431862784, 47430431887359, +ERASE, 47430431862784, 47430431887359, +STORE, 47430431862784, 47430431887359, +ERASE, 47430431887360, 47430431903743, +STORE, 47430431887360, 47430431903743, +STORE, 47430431903744, 47430432038911, +ERASE, 47430431903744, 47430432038911, +STORE, 47430431903744, 47430431928319, +STORE, 47430431928320, 47430432038911, +STORE, 47430431989760, 47430432038911, +STORE, 47430431928320, 47430431989759, +ERASE, 47430431928320, 47430431989759, +STORE, 47430431928320, 47430431989759, +STORE, 47430432014336, 47430432038911, +STORE, 47430431989760, 47430432014335, +ERASE, 47430431989760, 47430432014335, +STORE, 47430431989760, 47430432038911, +ERASE, 47430431989760, 47430432038911, +STORE, 47430431989760, 47430432014335, +STORE, 47430432014336, 47430432038911, +STORE, 47430432022528, 47430432038911, +STORE, 47430432014336, 47430432022527, +ERASE, 47430432014336, 47430432022527, +STORE, 47430432014336, 47430432022527, +ERASE, 47430432022528, 47430432038911, +STORE, 47430432022528, 47430432038911, +STORE, 47430432038912, 47430432059391, +ERASE, 47430432038912, 47430432059391, +STORE, 47430432038912, 47430432043007, +STORE, 47430432043008, 47430432059391, +STORE, 47430432047104, 47430432059391, +STORE, 47430432043008, 47430432047103, +ERASE, 47430432043008, 47430432047103, +STORE, 47430432043008, 47430432047103, +STORE, 47430432051200, 47430432059391, +STORE, 47430432047104, 47430432051199, +ERASE, 47430432047104, 47430432051199, +STORE, 47430432047104, 47430432059391, +ERASE, 47430432047104, 47430432059391, +STORE, 47430432047104, 47430432051199, +STORE, 47430432051200, 47430432059391, +ERASE, 47430432051200, 47430432059391, +STORE, 47430432051200, 47430432059391, +STORE, 47430432059392, 47430432067583, +ERASE, 47430431862784, 47430431887359, +STORE, 47430431862784, 47430431879167, +STORE, 47430431879168, 47430431887359, +ERASE, 47430432051200, 47430432059391, +STORE, 47430432051200, 47430432055295, +STORE, 47430432055296, 47430432059391, +ERASE, 47430432014336, 47430432022527, +STORE, 47430432014336, 47430432018431, +STORE, 47430432018432, 47430432022527, + }; + unsigned long set7[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140729808330752, 140737488351231, +ERASE, 140729808330752, 140737488351231, +STORE, 140729808330752, 140729808334847, +STORE, 94629632020480, 94629632192511, +ERASE, 94629632020480, 94629632192511, +STORE, 94629632020480, 94629632036863, +STORE, 94629632036864, 94629632192511, +ERASE, 94629632036864, 94629632192511, +STORE, 94629632036864, 94629632139263, +STORE, 94629632139264, 94629632180223, +STORE, 94629632180224, 94629632192511, +STORE, 47439981776896, 47439981948927, +ERASE, 47439981776896, 47439981948927, +STORE, 47439981776896, 47439981780991, +STORE, 47439981780992, 47439981948927, +ERASE, 47439981780992, 47439981948927, +STORE, 47439981780992, 47439981903871, +STORE, 47439981903872, 47439981936639, +STORE, 47439981936640, 47439981944831, +STORE, 47439981944832, 47439981948927, +STORE, 140729808474112, 140729808478207, +STORE, 140729808461824, 140729808474111, +STORE, 47439981948928, 47439981957119, +STORE, 47439981957120, 47439981965311, +STORE, 47439981965312, 47439982129151, +ERASE, 47439981965312, 47439982129151, +STORE, 47439981965312, 47439981977599, +STORE, 47439981977600, 47439982129151, +STORE, 47439982075904, 47439982129151, +STORE, 47439981977600, 47439982075903, +ERASE, 47439981977600, 47439982075903, +STORE, 47439981977600, 47439982075903, +STORE, 47439982120960, 47439982129151, +STORE, 47439982075904, 47439982120959, +ERASE, 47439982075904, 47439982120959, +STORE, 47439982075904, 47439982129151, +ERASE, 47439982075904, 47439982129151, +STORE, 47439982075904, 47439982120959, +STORE, 47439982120960, 47439982129151, +ERASE, 47439982120960, 47439982129151, +STORE, 47439982120960, 47439982129151, +STORE, 47439982129152, 47439985180671, +STORE, 47439982673920, 47439985180671, +STORE, 47439982129152, 47439982673919, +ERASE, 47439982673920, 47439985180671, +STORE, 47439982673920, 47439984959487, +STORE, 47439984959488, 47439985180671, +STORE, 47439984369664, 47439984959487, +STORE, 47439982673920, 47439984369663, +ERASE, 47439982673920, 47439984369663, +STORE, 47439982673920, 47439984369663, +STORE, 47439984955392, 47439984959487, +STORE, 47439984369664, 47439984955391, +ERASE, 47439984369664, 47439984955391, +STORE, 47439984369664, 47439984955391, +STORE, 47439985164288, 47439985180671, +STORE, 47439984959488, 47439985164287, +ERASE, 47439984959488, 47439985164287, +STORE, 47439984959488, 47439985164287, +ERASE, 47439985164288, 47439985180671, +STORE, 47439985164288, 47439985180671, +STORE, 47439985180672, 47439987019775, +STORE, 47439985319936, 47439987019775, +STORE, 47439985180672, 47439985319935, +ERASE, 47439985319936, 47439987019775, +STORE, 47439985319936, 47439986978815, +STORE, 47439986978816, 47439987019775, +STORE, 47439986663424, 47439986978815, +STORE, 47439985319936, 47439986663423, +ERASE, 47439985319936, 47439986663423, +STORE, 47439985319936, 47439986663423, +STORE, 47439986974720, 47439986978815, +STORE, 47439986663424, 47439986974719, +ERASE, 47439986663424, 47439986974719, +STORE, 47439986663424, 47439986974719, +STORE, 47439987003392, 47439987019775, +STORE, 47439986978816, 47439987003391, +ERASE, 47439986978816, 47439987003391, +STORE, 47439986978816, 47439987003391, +ERASE, 47439987003392, 47439987019775, +STORE, 47439987003392, 47439987019775, +STORE, 47439987019776, 47439987154943, +ERASE, 47439987019776, 47439987154943, +STORE, 47439987019776, 47439987044351, +STORE, 47439987044352, 47439987154943, +STORE, 47439987105792, 47439987154943, +STORE, 47439987044352, 47439987105791, +ERASE, 47439987044352, 47439987105791, +STORE, 47439987044352, 47439987105791, +STORE, 47439987130368, 47439987154943, +STORE, 47439987105792, 47439987130367, +ERASE, 47439987105792, 47439987130367, +STORE, 47439987105792, 47439987154943, +ERASE, 47439987105792, 47439987154943, +STORE, 47439987105792, 47439987130367, +STORE, 47439987130368, 47439987154943, +STORE, 47439987138560, 47439987154943, +STORE, 47439987130368, 47439987138559, +ERASE, 47439987130368, 47439987138559, +STORE, 47439987130368, 47439987138559, +ERASE, 47439987138560, 47439987154943, +STORE, 47439987138560, 47439987154943, +STORE, 47439987154944, 47439987175423, +ERASE, 47439987154944, 47439987175423, +STORE, 47439987154944, 47439987159039, +STORE, 47439987159040, 47439987175423, +STORE, 47439987163136, 47439987175423, +STORE, 47439987159040, 47439987163135, +ERASE, 47439987159040, 47439987163135, +STORE, 47439987159040, 47439987163135, +STORE, 47439987167232, 47439987175423, +STORE, 47439987163136, 47439987167231, +ERASE, 47439987163136, 47439987167231, +STORE, 47439987163136, 47439987175423, +ERASE, 47439987163136, 47439987175423, +STORE, 47439987163136, 47439987167231, +STORE, 47439987167232, 47439987175423, +ERASE, 47439987167232, 47439987175423, +STORE, 47439987167232, 47439987175423, +STORE, 47439987175424, 47439987183615, +ERASE, 47439986978816, 47439987003391, +STORE, 47439986978816, 47439986995199, +STORE, 47439986995200, 47439987003391, +ERASE, 47439987167232, 47439987175423, +STORE, 47439987167232, 47439987171327, +STORE, 47439987171328, 47439987175423, +ERASE, 47439987130368, 47439987138559, +STORE, 47439987130368, 47439987134463, +STORE, 47439987134464, 47439987138559, + }; + unsigned long set8[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722482974720, 140737488351231, +ERASE, 140722482974720, 140737488351231, +STORE, 140722482974720, 140722482978815, +STORE, 94121505034240, 94121505206271, +ERASE, 94121505034240, 94121505206271, +STORE, 94121505034240, 94121505050623, +STORE, 94121505050624, 94121505206271, +ERASE, 94121505050624, 94121505206271, +STORE, 94121505050624, 94121505153023, +STORE, 94121505153024, 94121505193983, +STORE, 94121505193984, 94121505206271, +STORE, 47708483284992, 47708483457023, +ERASE, 47708483284992, 47708483457023, +STORE, 47708483284992, 47708483289087, +STORE, 47708483289088, 47708483457023, +ERASE, 47708483289088, 47708483457023, +STORE, 47708483289088, 47708483411967, +STORE, 47708483411968, 47708483444735, +STORE, 47708483444736, 47708483452927, +STORE, 47708483452928, 47708483457023, +STORE, 140722483142656, 140722483146751, +STORE, 140722483130368, 140722483142655, +STORE, 47708483457024, 47708483465215, +STORE, 47708483465216, 47708483473407, +STORE, 47708483473408, 47708483637247, +ERASE, 47708483473408, 47708483637247, +STORE, 47708483473408, 47708483485695, +STORE, 47708483485696, 47708483637247, +STORE, 47708483584000, 47708483637247, +STORE, 47708483485696, 47708483583999, +ERASE, 47708483485696, 47708483583999, +STORE, 47708483485696, 47708483583999, +STORE, 47708483629056, 47708483637247, +STORE, 47708483584000, 47708483629055, +ERASE, 47708483584000, 47708483629055, +STORE, 47708483584000, 47708483637247, +ERASE, 47708483584000, 47708483637247, +STORE, 47708483584000, 47708483629055, +STORE, 47708483629056, 47708483637247, +ERASE, 47708483629056, 47708483637247, +STORE, 47708483629056, 47708483637247, +STORE, 47708483637248, 47708486688767, +STORE, 47708484182016, 47708486688767, +STORE, 47708483637248, 47708484182015, +ERASE, 47708484182016, 47708486688767, +STORE, 47708484182016, 47708486467583, +STORE, 47708486467584, 47708486688767, +STORE, 47708485877760, 47708486467583, +STORE, 47708484182016, 47708485877759, +ERASE, 47708484182016, 47708485877759, +STORE, 47708484182016, 47708485877759, +STORE, 47708486463488, 47708486467583, +STORE, 47708485877760, 47708486463487, +ERASE, 47708485877760, 47708486463487, +STORE, 47708485877760, 47708486463487, +STORE, 47708486672384, 47708486688767, +STORE, 47708486467584, 47708486672383, +ERASE, 47708486467584, 47708486672383, +STORE, 47708486467584, 47708486672383, +ERASE, 47708486672384, 47708486688767, +STORE, 47708486672384, 47708486688767, +STORE, 47708486688768, 47708488527871, +STORE, 47708486828032, 47708488527871, +STORE, 47708486688768, 47708486828031, +ERASE, 47708486828032, 47708488527871, +STORE, 47708486828032, 47708488486911, +STORE, 47708488486912, 47708488527871, +STORE, 47708488171520, 47708488486911, +STORE, 47708486828032, 47708488171519, +ERASE, 47708486828032, 47708488171519, +STORE, 47708486828032, 47708488171519, +STORE, 47708488482816, 47708488486911, +STORE, 47708488171520, 47708488482815, +ERASE, 47708488171520, 47708488482815, +STORE, 47708488171520, 47708488482815, +STORE, 47708488511488, 47708488527871, +STORE, 47708488486912, 47708488511487, +ERASE, 47708488486912, 47708488511487, +STORE, 47708488486912, 47708488511487, +ERASE, 47708488511488, 47708488527871, +STORE, 47708488511488, 47708488527871, +STORE, 47708488527872, 47708488663039, +ERASE, 47708488527872, 47708488663039, +STORE, 47708488527872, 47708488552447, +STORE, 47708488552448, 47708488663039, +STORE, 47708488613888, 47708488663039, +STORE, 47708488552448, 47708488613887, +ERASE, 47708488552448, 47708488613887, +STORE, 47708488552448, 47708488613887, +STORE, 47708488638464, 47708488663039, +STORE, 47708488613888, 47708488638463, +ERASE, 47708488613888, 47708488638463, +STORE, 47708488613888, 47708488663039, +ERASE, 47708488613888, 47708488663039, +STORE, 47708488613888, 47708488638463, +STORE, 47708488638464, 47708488663039, +STORE, 47708488646656, 47708488663039, +STORE, 47708488638464, 47708488646655, +ERASE, 47708488638464, 47708488646655, +STORE, 47708488638464, 47708488646655, +ERASE, 47708488646656, 47708488663039, +STORE, 47708488646656, 47708488663039, +STORE, 47708488663040, 47708488683519, +ERASE, 47708488663040, 47708488683519, +STORE, 47708488663040, 47708488667135, +STORE, 47708488667136, 47708488683519, +STORE, 47708488671232, 47708488683519, +STORE, 47708488667136, 47708488671231, +ERASE, 47708488667136, 47708488671231, +STORE, 47708488667136, 47708488671231, +STORE, 47708488675328, 47708488683519, +STORE, 47708488671232, 47708488675327, +ERASE, 47708488671232, 47708488675327, +STORE, 47708488671232, 47708488683519, +ERASE, 47708488671232, 47708488683519, +STORE, 47708488671232, 47708488675327, +STORE, 47708488675328, 47708488683519, +ERASE, 47708488675328, 47708488683519, +STORE, 47708488675328, 47708488683519, +STORE, 47708488683520, 47708488691711, +ERASE, 47708488486912, 47708488511487, +STORE, 47708488486912, 47708488503295, +STORE, 47708488503296, 47708488511487, +ERASE, 47708488675328, 47708488683519, +STORE, 47708488675328, 47708488679423, +STORE, 47708488679424, 47708488683519, +ERASE, 47708488638464, 47708488646655, +STORE, 47708488638464, 47708488642559, +STORE, 47708488642560, 47708488646655, + }; + + unsigned long set9[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736427839488, 140737488351231, +ERASE, 140736427839488, 140736427839488, +STORE, 140736427839488, 140736427843583, +STORE, 94071213395968, 94071213567999, +ERASE, 94071213395968, 94071213395968, +STORE, 94071213395968, 94071213412351, +STORE, 94071213412352, 94071213567999, +ERASE, 94071213412352, 94071213412352, +STORE, 94071213412352, 94071213514751, +STORE, 94071213514752, 94071213555711, +STORE, 94071213555712, 94071213567999, +STORE, 139968410644480, 139968410816511, +ERASE, 139968410644480, 139968410644480, +STORE, 139968410644480, 139968410648575, +STORE, 139968410648576, 139968410816511, +ERASE, 139968410648576, 139968410648576, +STORE, 139968410648576, 139968410771455, +STORE, 139968410771456, 139968410804223, +STORE, 139968410804224, 139968410812415, +STORE, 139968410812416, 139968410816511, +STORE, 140736429277184, 140736429281279, +STORE, 140736429264896, 140736429277183, +STORE, 47664384352256, 47664384360447, +STORE, 47664384360448, 47664384368639, +STORE, 47664384368640, 47664384532479, +ERASE, 47664384368640, 47664384368640, +STORE, 47664384368640, 47664384380927, +STORE, 47664384380928, 47664384532479, +STORE, 47664384479232, 47664384532479, +STORE, 47664384380928, 47664384479231, +ERASE, 47664384380928, 47664384380928, +STORE, 47664384380928, 47664384479231, +STORE, 47664384524288, 47664384532479, +STORE, 47664384479232, 47664384524287, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384532479, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384524287, +STORE, 47664384524288, 47664384532479, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384532479, +STORE, 47664384532480, 47664387583999, +STORE, 47664385077248, 47664387583999, +STORE, 47664384532480, 47664385077247, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664387362815, +STORE, 47664387362816, 47664387583999, +STORE, 47664386772992, 47664387362815, +STORE, 47664385077248, 47664386772991, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664386772991, +STORE, 47664387358720, 47664387362815, +STORE, 47664386772992, 47664387358719, +ERASE, 47664386772992, 47664386772992, +STORE, 47664386772992, 47664387358719, +STORE, 47664387567616, 47664387583999, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387567616, 47664387567616, +STORE, 47664387567616, 47664387583999, +STORE, 47664387584000, 47664389423103, +STORE, 47664387723264, 47664389423103, +STORE, 47664387584000, 47664387723263, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389382143, +STORE, 47664389382144, 47664389423103, +STORE, 47664389066752, 47664389382143, +STORE, 47664387723264, 47664389066751, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389066751, +STORE, 47664389378048, 47664389382143, +STORE, 47664389066752, 47664389378047, +ERASE, 47664389066752, 47664389066752, +STORE, 47664389066752, 47664389378047, +STORE, 47664389406720, 47664389423103, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389406720, 47664389406720, +STORE, 47664389406720, 47664389423103, +STORE, 47664389423104, 47664389558271, +ERASE, 47664389423104, 47664389423104, +STORE, 47664389423104, 47664389447679, +STORE, 47664389447680, 47664389558271, +STORE, 47664389509120, 47664389558271, +STORE, 47664389447680, 47664389509119, +ERASE, 47664389447680, 47664389447680, +STORE, 47664389447680, 47664389509119, +STORE, 47664389533696, 47664389558271, +STORE, 47664389509120, 47664389533695, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389558271, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389533695, +STORE, 47664389533696, 47664389558271, +STORE, 47664389541888, 47664389558271, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389541888, 47664389541888, +STORE, 47664389541888, 47664389558271, +STORE, 47664389558272, 47664389578751, +ERASE, 47664389558272, 47664389558272, +STORE, 47664389558272, 47664389562367, +STORE, 47664389562368, 47664389578751, +STORE, 47664389566464, 47664389578751, +STORE, 47664389562368, 47664389566463, +ERASE, 47664389562368, 47664389562368, +STORE, 47664389562368, 47664389566463, +STORE, 47664389570560, 47664389578751, +STORE, 47664389566464, 47664389570559, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389578751, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389570559, +STORE, 47664389570560, 47664389578751, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389578751, +STORE, 47664389578752, 47664389586943, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389398527, +STORE, 47664389398528, 47664389406719, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389574655, +STORE, 47664389574656, 47664389578751, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389537791, +STORE, 47664389537792, 47664389541887, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387559423, +STORE, 47664387559424, 47664387567615, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384528383, +STORE, 47664384528384, 47664384532479, +ERASE, 94071213555712, 94071213555712, +STORE, 94071213555712, 94071213563903, +STORE, 94071213563904, 94071213567999, +ERASE, 139968410804224, 139968410804224, +STORE, 139968410804224, 139968410808319, +STORE, 139968410808320, 139968410812415, +ERASE, 47664384352256, 47664384352256, +STORE, 94071244402688, 94071244537855, +STORE, 140737488347136, 140737488351231, +STORE, 140728271503360, 140737488351231, +ERASE, 140728271503360, 140728271503360, +STORE, 140728271503360, 140728271507455, +STORE, 94410361982976, 94410362155007, +ERASE, 94410361982976, 94410361982976, +STORE, 94410361982976, 94410361999359, +STORE, 94410361999360, 94410362155007, +ERASE, 94410361999360, 94410361999360, +STORE, 94410361999360, 94410362101759, +STORE, 94410362101760, 94410362142719, +STORE, 94410362142720, 94410362155007, +STORE, 140351953997824, 140351954169855, +ERASE, 140351953997824, 140351953997824, +STORE, 140351953997824, 140351954001919, +STORE, 140351954001920, 140351954169855, +ERASE, 140351954001920, 140351954001920, +STORE, 140351954001920, 140351954124799, +STORE, 140351954124800, 140351954157567, +STORE, 140351954157568, 140351954165759, +STORE, 140351954165760, 140351954169855, +STORE, 140728272429056, 140728272433151, +STORE, 140728272416768, 140728272429055, +STORE, 47280840998912, 47280841007103, +STORE, 47280841007104, 47280841015295, +STORE, 47280841015296, 47280841179135, +ERASE, 47280841015296, 47280841015296, +STORE, 47280841015296, 47280841027583, +STORE, 47280841027584, 47280841179135, +STORE, 47280841125888, 47280841179135, +STORE, 47280841027584, 47280841125887, +ERASE, 47280841027584, 47280841027584, +STORE, 47280841027584, 47280841125887, +STORE, 47280841170944, 47280841179135, +STORE, 47280841125888, 47280841170943, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841179135, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841170943, +STORE, 47280841170944, 47280841179135, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841179135, +STORE, 47280841179136, 47280844230655, +STORE, 47280841723904, 47280844230655, +STORE, 47280841179136, 47280841723903, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280844009471, +STORE, 47280844009472, 47280844230655, +STORE, 47280843419648, 47280844009471, +STORE, 47280841723904, 47280843419647, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280843419647, +STORE, 47280844005376, 47280844009471, +STORE, 47280843419648, 47280844005375, +ERASE, 47280843419648, 47280843419648, +STORE, 47280843419648, 47280844005375, +STORE, 47280844214272, 47280844230655, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844214272, 47280844214272, +STORE, 47280844214272, 47280844230655, +STORE, 47280844230656, 47280846069759, +STORE, 47280844369920, 47280846069759, +STORE, 47280844230656, 47280844369919, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280846028799, +STORE, 47280846028800, 47280846069759, +STORE, 47280845713408, 47280846028799, +STORE, 47280844369920, 47280845713407, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280845713407, +STORE, 47280846024704, 47280846028799, +STORE, 47280845713408, 47280846024703, +ERASE, 47280845713408, 47280845713408, +STORE, 47280845713408, 47280846024703, +STORE, 47280846053376, 47280846069759, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846053376, 47280846053376, +STORE, 47280846053376, 47280846069759, +STORE, 47280846069760, 47280846204927, +ERASE, 47280846069760, 47280846069760, +STORE, 47280846069760, 47280846094335, +STORE, 47280846094336, 47280846204927, +STORE, 47280846155776, 47280846204927, +STORE, 47280846094336, 47280846155775, +ERASE, 47280846094336, 47280846094336, +STORE, 47280846094336, 47280846155775, +STORE, 47280846180352, 47280846204927, +STORE, 47280846155776, 47280846180351, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846204927, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846180351, +STORE, 47280846180352, 47280846204927, +STORE, 47280846188544, 47280846204927, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846188544, 47280846188544, +STORE, 47280846188544, 47280846204927, +STORE, 47280846204928, 47280846225407, +ERASE, 47280846204928, 47280846204928, +STORE, 47280846204928, 47280846209023, +STORE, 47280846209024, 47280846225407, +STORE, 47280846213120, 47280846225407, +STORE, 47280846209024, 47280846213119, +ERASE, 47280846209024, 47280846209024, +STORE, 47280846209024, 47280846213119, +STORE, 47280846217216, 47280846225407, +STORE, 47280846213120, 47280846217215, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846225407, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846217215, +STORE, 47280846217216, 47280846225407, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846225407, +STORE, 47280846225408, 47280846233599, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846045183, +STORE, 47280846045184, 47280846053375, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846221311, +STORE, 47280846221312, 47280846225407, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846184447, +STORE, 47280846184448, 47280846188543, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844206079, +STORE, 47280844206080, 47280844214271, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841175039, +STORE, 47280841175040, 47280841179135, +ERASE, 94410362142720, 94410362142720, +STORE, 94410362142720, 94410362150911, +STORE, 94410362150912, 94410362155007, +ERASE, 140351954157568, 140351954157568, +STORE, 140351954157568, 140351954161663, +STORE, 140351954161664, 140351954165759, +ERASE, 47280840998912, 47280840998912, +STORE, 94410379456512, 94410379591679, +STORE, 140737488347136, 140737488351231, +STORE, 140732946362368, 140737488351231, +ERASE, 140732946362368, 140732946362368, +STORE, 140732946362368, 140732946366463, +STORE, 94352937934848, 94352938106879, +ERASE, 94352937934848, 94352937934848, +STORE, 94352937934848, 94352937951231, +STORE, 94352937951232, 94352938106879, +ERASE, 94352937951232, 94352937951232, +STORE, 94352937951232, 94352938053631, +STORE, 94352938053632, 94352938094591, +STORE, 94352938094592, 94352938106879, +STORE, 140595518742528, 140595518914559, +ERASE, 140595518742528, 140595518742528, +STORE, 140595518742528, 140595518746623, +STORE, 140595518746624, 140595518914559, +ERASE, 140595518746624, 140595518746624, +STORE, 140595518746624, 140595518869503, +STORE, 140595518869504, 140595518902271, +STORE, 140595518902272, 140595518910463, +STORE, 140595518910464, 140595518914559, +STORE, 140732947468288, 140732947472383, +STORE, 140732947456000, 140732947468287, +STORE, 47037276254208, 47037276262399, +STORE, 47037276262400, 47037276270591, +STORE, 47037276270592, 47037276434431, +ERASE, 47037276270592, 47037276270592, +STORE, 47037276270592, 47037276282879, +STORE, 47037276282880, 47037276434431, +STORE, 47037276381184, 47037276434431, +STORE, 47037276282880, 47037276381183, +ERASE, 47037276282880, 47037276282880, +STORE, 47037276282880, 47037276381183, +STORE, 47037276426240, 47037276434431, +STORE, 47037276381184, 47037276426239, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276434431, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276426239, +STORE, 47037276426240, 47037276434431, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276434431, +STORE, 47037276434432, 47037279485951, +STORE, 47037276979200, 47037279485951, +STORE, 47037276434432, 47037276979199, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037279264767, +STORE, 47037279264768, 47037279485951, +STORE, 47037278674944, 47037279264767, +STORE, 47037276979200, 47037278674943, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037278674943, +STORE, 47037279260672, 47037279264767, +STORE, 47037278674944, 47037279260671, +ERASE, 47037278674944, 47037278674944, +STORE, 47037278674944, 47037279260671, +STORE, 47037279469568, 47037279485951, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279469568, 47037279469568, +STORE, 47037279469568, 47037279485951, +STORE, 47037279485952, 47037281325055, +STORE, 47037279625216, 47037281325055, +STORE, 47037279485952, 47037279625215, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037281284095, +STORE, 47037281284096, 47037281325055, +STORE, 47037280968704, 47037281284095, +STORE, 47037279625216, 47037280968703, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037280968703, +STORE, 47037281280000, 47037281284095, +STORE, 47037280968704, 47037281279999, +ERASE, 47037280968704, 47037280968704, +STORE, 47037280968704, 47037281279999, +STORE, 47037281308672, 47037281325055, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281308672, 47037281308672, +STORE, 47037281308672, 47037281325055, +STORE, 47037281325056, 47037281460223, +ERASE, 47037281325056, 47037281325056, +STORE, 47037281325056, 47037281349631, +STORE, 47037281349632, 47037281460223, +STORE, 47037281411072, 47037281460223, +STORE, 47037281349632, 47037281411071, +ERASE, 47037281349632, 47037281349632, +STORE, 47037281349632, 47037281411071, +STORE, 47037281435648, 47037281460223, +STORE, 47037281411072, 47037281435647, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281460223, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281435647, +STORE, 47037281435648, 47037281460223, +STORE, 47037281443840, 47037281460223, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281443840, 47037281443840, +STORE, 47037281443840, 47037281460223, +STORE, 47037281460224, 47037281480703, +ERASE, 47037281460224, 47037281460224, +STORE, 47037281460224, 47037281464319, +STORE, 47037281464320, 47037281480703, +STORE, 47037281468416, 47037281480703, +STORE, 47037281464320, 47037281468415, +ERASE, 47037281464320, 47037281464320, +STORE, 47037281464320, 47037281468415, +STORE, 47037281472512, 47037281480703, +STORE, 47037281468416, 47037281472511, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281480703, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281472511, +STORE, 47037281472512, 47037281480703, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281480703, +STORE, 47037281480704, 47037281488895, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281300479, +STORE, 47037281300480, 47037281308671, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281476607, +STORE, 47037281476608, 47037281480703, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281439743, +STORE, 47037281439744, 47037281443839, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279461375, +STORE, 47037279461376, 47037279469567, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276430335, +STORE, 47037276430336, 47037276434431, +ERASE, 94352938094592, 94352938094592, +STORE, 94352938094592, 94352938102783, +STORE, 94352938102784, 94352938106879, +ERASE, 140595518902272, 140595518902272, +STORE, 140595518902272, 140595518906367, +STORE, 140595518906368, 140595518910463, +ERASE, 47037276254208, 47037276254208, +STORE, 94352938438656, 94352938573823, +STORE, 140737488347136, 140737488351231, +STORE, 140733506027520, 140737488351231, +ERASE, 140733506027520, 140733506027520, +STORE, 140733506027520, 140733506031615, +STORE, 94150123073536, 94150123245567, +ERASE, 94150123073536, 94150123073536, +STORE, 94150123073536, 94150123089919, +STORE, 94150123089920, 94150123245567, +ERASE, 94150123089920, 94150123089920, +STORE, 94150123089920, 94150123192319, +STORE, 94150123192320, 94150123233279, +STORE, 94150123233280, 94150123245567, +STORE, 140081290375168, 140081290547199, +ERASE, 140081290375168, 140081290375168, +STORE, 140081290375168, 140081290379263, +STORE, 140081290379264, 140081290547199, +ERASE, 140081290379264, 140081290379264, +STORE, 140081290379264, 140081290502143, +STORE, 140081290502144, 140081290534911, +STORE, 140081290534912, 140081290543103, +STORE, 140081290543104, 140081290547199, +STORE, 140733506707456, 140733506711551, +STORE, 140733506695168, 140733506707455, +STORE, 47551504621568, 47551504629759, +STORE, 47551504629760, 47551504637951, +STORE, 47551504637952, 47551504801791, +ERASE, 47551504637952, 47551504637952, +STORE, 47551504637952, 47551504650239, +STORE, 47551504650240, 47551504801791, +STORE, 47551504748544, 47551504801791, +STORE, 47551504650240, 47551504748543, +ERASE, 47551504650240, 47551504650240, +STORE, 47551504650240, 47551504748543, +STORE, 47551504793600, 47551504801791, +STORE, 47551504748544, 47551504793599, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504801791, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504793599, +STORE, 47551504793600, 47551504801791, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504801791, +STORE, 47551504801792, 47551507853311, +STORE, 47551505346560, 47551507853311, +STORE, 47551504801792, 47551505346559, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507632127, +STORE, 47551507632128, 47551507853311, +STORE, 47551507042304, 47551507632127, +STORE, 47551505346560, 47551507042303, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507042303, +STORE, 47551507628032, 47551507632127, +STORE, 47551507042304, 47551507628031, +ERASE, 47551507042304, 47551507042304, +STORE, 47551507042304, 47551507628031, +STORE, 47551507836928, 47551507853311, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507836928, 47551507836928, +STORE, 47551507836928, 47551507853311, +STORE, 47551507853312, 47551509692415, +STORE, 47551507992576, 47551509692415, +STORE, 47551507853312, 47551507992575, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509651455, +STORE, 47551509651456, 47551509692415, +STORE, 47551509336064, 47551509651455, +STORE, 47551507992576, 47551509336063, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509336063, +STORE, 47551509647360, 47551509651455, +STORE, 47551509336064, 47551509647359, +ERASE, 47551509336064, 47551509336064, +STORE, 47551509336064, 47551509647359, +STORE, 47551509676032, 47551509692415, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509676032, 47551509676032, +STORE, 47551509676032, 47551509692415, +STORE, 47551509692416, 47551509827583, +ERASE, 47551509692416, 47551509692416, +STORE, 47551509692416, 47551509716991, +STORE, 47551509716992, 47551509827583, +STORE, 47551509778432, 47551509827583, +STORE, 47551509716992, 47551509778431, +ERASE, 47551509716992, 47551509716992, +STORE, 47551509716992, 47551509778431, +STORE, 47551509803008, 47551509827583, +STORE, 47551509778432, 47551509803007, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509827583, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509803007, +STORE, 47551509803008, 47551509827583, +STORE, 47551509811200, 47551509827583, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509811200, 47551509811200, +STORE, 47551509811200, 47551509827583, +STORE, 47551509827584, 47551509848063, +ERASE, 47551509827584, 47551509827584, +STORE, 47551509827584, 47551509831679, +STORE, 47551509831680, 47551509848063, +STORE, 47551509835776, 47551509848063, +STORE, 47551509831680, 47551509835775, +ERASE, 47551509831680, 47551509831680, +STORE, 47551509831680, 47551509835775, +STORE, 47551509839872, 47551509848063, +STORE, 47551509835776, 47551509839871, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509848063, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509839871, +STORE, 47551509839872, 47551509848063, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509848063, +STORE, 47551509848064, 47551509856255, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509667839, +STORE, 47551509667840, 47551509676031, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509843967, +STORE, 47551509843968, 47551509848063, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509807103, +STORE, 47551509807104, 47551509811199, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507828735, +STORE, 47551507828736, 47551507836927, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504797695, +STORE, 47551504797696, 47551504801791, +ERASE, 94150123233280, 94150123233280, +STORE, 94150123233280, 94150123241471, +STORE, 94150123241472, 94150123245567, +ERASE, 140081290534912, 140081290534912, +STORE, 140081290534912, 140081290539007, +STORE, 140081290539008, 140081290543103, +ERASE, 47551504621568, 47551504621568, +STORE, 94150148112384, 94150148247551, +STORE, 140737488347136, 140737488351231, +STORE, 140734389334016, 140737488351231, +ERASE, 140734389334016, 140734389334016, +STORE, 140734389334016, 140734389338111, +STORE, 94844636606464, 94844636778495, +ERASE, 94844636606464, 94844636606464, +STORE, 94844636606464, 94844636622847, +STORE, 94844636622848, 94844636778495, +ERASE, 94844636622848, 94844636622848, +STORE, 94844636622848, 94844636725247, +STORE, 94844636725248, 94844636766207, +STORE, 94844636766208, 94844636778495, +STORE, 139922765217792, 139922765389823, +ERASE, 139922765217792, 139922765217792, +STORE, 139922765217792, 139922765221887, +STORE, 139922765221888, 139922765389823, +ERASE, 139922765221888, 139922765221888, +STORE, 139922765221888, 139922765344767, +STORE, 139922765344768, 139922765377535, +STORE, 139922765377536, 139922765385727, +STORE, 139922765385728, 139922765389823, +STORE, 140734389678080, 140734389682175, +STORE, 140734389665792, 140734389678079, +STORE, 47710029778944, 47710029787135, +STORE, 47710029787136, 47710029795327, +STORE, 47710029795328, 47710029959167, +ERASE, 47710029795328, 47710029795328, +STORE, 47710029795328, 47710029807615, +STORE, 47710029807616, 47710029959167, +STORE, 47710029905920, 47710029959167, +STORE, 47710029807616, 47710029905919, +ERASE, 47710029807616, 47710029807616, +STORE, 47710029807616, 47710029905919, +STORE, 47710029950976, 47710029959167, +STORE, 47710029905920, 47710029950975, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029959167, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029950975, +STORE, 47710029950976, 47710029959167, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029959167, +STORE, 47710029959168, 47710033010687, +STORE, 47710030503936, 47710033010687, +STORE, 47710029959168, 47710030503935, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032789503, +STORE, 47710032789504, 47710033010687, +STORE, 47710032199680, 47710032789503, +STORE, 47710030503936, 47710032199679, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032199679, +STORE, 47710032785408, 47710032789503, +STORE, 47710032199680, 47710032785407, +ERASE, 47710032199680, 47710032199680, +STORE, 47710032199680, 47710032785407, +STORE, 47710032994304, 47710033010687, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032994304, 47710032994304, +STORE, 47710032994304, 47710033010687, +STORE, 47710033010688, 47710034849791, +STORE, 47710033149952, 47710034849791, +STORE, 47710033010688, 47710033149951, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034808831, +STORE, 47710034808832, 47710034849791, +STORE, 47710034493440, 47710034808831, +STORE, 47710033149952, 47710034493439, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034493439, +STORE, 47710034804736, 47710034808831, +STORE, 47710034493440, 47710034804735, +ERASE, 47710034493440, 47710034493440, +STORE, 47710034493440, 47710034804735, +STORE, 47710034833408, 47710034849791, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034833408, 47710034833408, +STORE, 47710034833408, 47710034849791, +STORE, 47710034849792, 47710034984959, +ERASE, 47710034849792, 47710034849792, +STORE, 47710034849792, 47710034874367, +STORE, 47710034874368, 47710034984959, +STORE, 47710034935808, 47710034984959, +STORE, 47710034874368, 47710034935807, +ERASE, 47710034874368, 47710034874368, +STORE, 47710034874368, 47710034935807, +STORE, 47710034960384, 47710034984959, +STORE, 47710034935808, 47710034960383, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034984959, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034960383, +STORE, 47710034960384, 47710034984959, +STORE, 47710034968576, 47710034984959, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034968576, 47710034968576, +STORE, 47710034968576, 47710034984959, +STORE, 47710034984960, 47710035005439, +ERASE, 47710034984960, 47710034984960, +STORE, 47710034984960, 47710034989055, +STORE, 47710034989056, 47710035005439, +STORE, 47710034993152, 47710035005439, +STORE, 47710034989056, 47710034993151, +ERASE, 47710034989056, 47710034989056, +STORE, 47710034989056, 47710034993151, +STORE, 47710034997248, 47710035005439, +STORE, 47710034993152, 47710034997247, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710035005439, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710034997247, +STORE, 47710034997248, 47710035005439, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035005439, +STORE, 47710035005440, 47710035013631, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034825215, +STORE, 47710034825216, 47710034833407, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035001343, +STORE, 47710035001344, 47710035005439, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034964479, +STORE, 47710034964480, 47710034968575, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032986111, +STORE, 47710032986112, 47710032994303, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029955071, +STORE, 47710029955072, 47710029959167, +ERASE, 94844636766208, 94844636766208, +STORE, 94844636766208, 94844636774399, +STORE, 94844636774400, 94844636778495, +ERASE, 139922765377536, 139922765377536, +STORE, 139922765377536, 139922765381631, +STORE, 139922765381632, 139922765385727, +ERASE, 47710029778944, 47710029778944, +STORE, 94844641775616, 94844641910783, +STORE, 140737488347136, 140737488351231, +STORE, 140732213886976, 140737488351231, +ERASE, 140732213886976, 140732213886976, +STORE, 140732213886976, 140732213891071, +STORE, 94240508887040, 94240509059071, +ERASE, 94240508887040, 94240508887040, +STORE, 94240508887040, 94240508903423, +STORE, 94240508903424, 94240509059071, +ERASE, 94240508903424, 94240508903424, +STORE, 94240508903424, 94240509005823, +STORE, 94240509005824, 94240509046783, +STORE, 94240509046784, 94240509059071, +STORE, 140275106516992, 140275106689023, +ERASE, 140275106516992, 140275106516992, +STORE, 140275106516992, 140275106521087, +STORE, 140275106521088, 140275106689023, +ERASE, 140275106521088, 140275106521088, +STORE, 140275106521088, 140275106643967, +STORE, 140275106643968, 140275106676735, +STORE, 140275106676736, 140275106684927, +STORE, 140275106684928, 140275106689023, +STORE, 140732213977088, 140732213981183, +STORE, 140732213964800, 140732213977087, +STORE, 47357688479744, 47357688487935, +STORE, 47357688487936, 47357688496127, +STORE, 47357688496128, 47357688659967, +ERASE, 47357688496128, 47357688496128, +STORE, 47357688496128, 47357688508415, +STORE, 47357688508416, 47357688659967, +STORE, 47357688606720, 47357688659967, +STORE, 47357688508416, 47357688606719, +ERASE, 47357688508416, 47357688508416, +STORE, 47357688508416, 47357688606719, +STORE, 47357688651776, 47357688659967, +STORE, 47357688606720, 47357688651775, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688659967, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688651775, +STORE, 47357688651776, 47357688659967, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688659967, +STORE, 47357688659968, 47357691711487, +STORE, 47357689204736, 47357691711487, +STORE, 47357688659968, 47357689204735, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357691490303, +STORE, 47357691490304, 47357691711487, +STORE, 47357690900480, 47357691490303, +STORE, 47357689204736, 47357690900479, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357690900479, +STORE, 47357691486208, 47357691490303, +STORE, 47357690900480, 47357691486207, +ERASE, 47357690900480, 47357690900480, +STORE, 47357690900480, 47357691486207, +STORE, 47357691695104, 47357691711487, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691695104, 47357691695104, +STORE, 47357691695104, 47357691711487, +STORE, 47357691711488, 47357693550591, +STORE, 47357691850752, 47357693550591, +STORE, 47357691711488, 47357691850751, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693509631, +STORE, 47357693509632, 47357693550591, +STORE, 47357693194240, 47357693509631, +STORE, 47357691850752, 47357693194239, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693194239, +STORE, 47357693505536, 47357693509631, +STORE, 47357693194240, 47357693505535, +ERASE, 47357693194240, 47357693194240, +STORE, 47357693194240, 47357693505535, +STORE, 47357693534208, 47357693550591, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693534208, 47357693534208, +STORE, 47357693534208, 47357693550591, +STORE, 47357693550592, 47357693685759, +ERASE, 47357693550592, 47357693550592, +STORE, 47357693550592, 47357693575167, +STORE, 47357693575168, 47357693685759, +STORE, 47357693636608, 47357693685759, +STORE, 47357693575168, 47357693636607, +ERASE, 47357693575168, 47357693575168, +STORE, 47357693575168, 47357693636607, +STORE, 47357693661184, 47357693685759, +STORE, 47357693636608, 47357693661183, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693685759, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693661183, +STORE, 47357693661184, 47357693685759, +STORE, 47357693669376, 47357693685759, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693669376, 47357693669376, +STORE, 47357693669376, 47357693685759, +STORE, 47357693685760, 47357693706239, +ERASE, 47357693685760, 47357693685760, +STORE, 47357693685760, 47357693689855, +STORE, 47357693689856, 47357693706239, +STORE, 47357693693952, 47357693706239, +STORE, 47357693689856, 47357693693951, +ERASE, 47357693689856, 47357693689856, +STORE, 47357693689856, 47357693693951, +STORE, 47357693698048, 47357693706239, +STORE, 47357693693952, 47357693698047, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693706239, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693698047, +STORE, 47357693698048, 47357693706239, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693706239, +STORE, 47357693706240, 47357693714431, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693526015, +STORE, 47357693526016, 47357693534207, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693702143, +STORE, 47357693702144, 47357693706239, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693665279, +STORE, 47357693665280, 47357693669375, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691686911, +STORE, 47357691686912, 47357691695103, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688655871, +STORE, 47357688655872, 47357688659967, +ERASE, 94240509046784, 94240509046784, +STORE, 94240509046784, 94240509054975, +STORE, 94240509054976, 94240509059071, +ERASE, 140275106676736, 140275106676736, +STORE, 140275106676736, 140275106680831, +STORE, 140275106680832, 140275106684927, +ERASE, 47357688479744, 47357688479744, +STORE, 94240518361088, 94240518496255, +STORE, 140737488347136, 140737488351231, +STORE, 140732688277504, 140737488351231, +ERASE, 140732688277504, 140732688277504, +STORE, 140732688277504, 140732688281599, +STORE, 94629171351552, 94629172064255, +ERASE, 94629171351552, 94629171351552, +STORE, 94629171351552, 94629171400703, +STORE, 94629171400704, 94629172064255, +ERASE, 94629171400704, 94629171400704, +STORE, 94629171400704, 94629171945471, +STORE, 94629171945472, 94629172043775, +STORE, 94629172043776, 94629172064255, +STORE, 139770707644416, 139770707816447, +ERASE, 139770707644416, 139770707644416, +STORE, 139770707644416, 139770707648511, +STORE, 139770707648512, 139770707816447, +ERASE, 139770707648512, 139770707648512, +STORE, 139770707648512, 139770707771391, +STORE, 139770707771392, 139770707804159, +STORE, 139770707804160, 139770707812351, +STORE, 139770707812352, 139770707816447, +STORE, 140732689121280, 140732689125375, +STORE, 140732689108992, 140732689121279, +STORE, 47862087352320, 47862087360511, +STORE, 47862087360512, 47862087368703, +STORE, 47862087368704, 47862087475199, +STORE, 47862087385088, 47862087475199, +STORE, 47862087368704, 47862087385087, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087458815, +STORE, 47862087458816, 47862087475199, +STORE, 47862087438336, 47862087458815, +STORE, 47862087385088, 47862087438335, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087438335, +STORE, 47862087454720, 47862087458815, +STORE, 47862087438336, 47862087454719, +ERASE, 47862087438336, 47862087438336, +STORE, 47862087438336, 47862087454719, +STORE, 47862087467008, 47862087475199, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087467008, 47862087467008, +STORE, 47862087467008, 47862087475199, +STORE, 47862087475200, 47862089314303, +STORE, 47862087614464, 47862089314303, +STORE, 47862087475200, 47862087614463, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862089273343, +STORE, 47862089273344, 47862089314303, +STORE, 47862088957952, 47862089273343, +STORE, 47862087614464, 47862088957951, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862088957951, +STORE, 47862089269248, 47862089273343, +STORE, 47862088957952, 47862089269247, +ERASE, 47862088957952, 47862088957952, +STORE, 47862088957952, 47862089269247, +STORE, 47862089297920, 47862089314303, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089297920, 47862089297920, +STORE, 47862089297920, 47862089314303, +STORE, 47862089297920, 47862089326591, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089289727, +STORE, 47862089289728, 47862089297919, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087462911, +STORE, 47862087462912, 47862087467007, +ERASE, 94629172043776, 94629172043776, +STORE, 94629172043776, 94629172060159, +STORE, 94629172060160, 94629172064255, +ERASE, 139770707804160, 139770707804160, +STORE, 139770707804160, 139770707808255, +STORE, 139770707808256, 139770707812351, +ERASE, 47862087352320, 47862087352320, +STORE, 94629197533184, 94629197668351, +STORE, 140737488347136, 140737488351231, +STORE, 140727540711424, 140737488351231, +ERASE, 140727540711424, 140727540711424, +STORE, 140727540711424, 140727540715519, +STORE, 94299865313280, 94299866025983, +ERASE, 94299865313280, 94299865313280, +STORE, 94299865313280, 94299865362431, +STORE, 94299865362432, 94299866025983, +ERASE, 94299865362432, 94299865362432, +STORE, 94299865362432, 94299865907199, +STORE, 94299865907200, 94299866005503, +STORE, 94299866005504, 94299866025983, +STORE, 140680268763136, 140680268935167, +ERASE, 140680268763136, 140680268763136, +STORE, 140680268763136, 140680268767231, +STORE, 140680268767232, 140680268935167, +ERASE, 140680268767232, 140680268767232, +STORE, 140680268767232, 140680268890111, +STORE, 140680268890112, 140680268922879, +STORE, 140680268922880, 140680268931071, +STORE, 140680268931072, 140680268935167, +STORE, 140727541424128, 140727541428223, +STORE, 140727541411840, 140727541424127, +STORE, 46952526233600, 46952526241791, +STORE, 46952526241792, 46952526249983, +STORE, 46952526249984, 46952526356479, +STORE, 46952526266368, 46952526356479, +STORE, 46952526249984, 46952526266367, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526340095, +STORE, 46952526340096, 46952526356479, +STORE, 46952526319616, 46952526340095, +STORE, 46952526266368, 46952526319615, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526319615, +STORE, 46952526336000, 46952526340095, +STORE, 46952526319616, 46952526335999, +ERASE, 46952526319616, 46952526319616, +STORE, 46952526319616, 46952526335999, +STORE, 46952526348288, 46952526356479, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526348288, 46952526348288, +STORE, 46952526348288, 46952526356479, +STORE, 46952526356480, 46952528195583, +STORE, 46952526495744, 46952528195583, +STORE, 46952526356480, 46952526495743, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952528154623, +STORE, 46952528154624, 46952528195583, +STORE, 46952527839232, 46952528154623, +STORE, 46952526495744, 46952527839231, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952527839231, +STORE, 46952528150528, 46952528154623, +STORE, 46952527839232, 46952528150527, +ERASE, 46952527839232, 46952527839232, +STORE, 46952527839232, 46952528150527, +STORE, 46952528179200, 46952528195583, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528179200, 46952528179200, +STORE, 46952528179200, 46952528195583, +STORE, 46952528179200, 46952528207871, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528171007, +STORE, 46952528171008, 46952528179199, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526344191, +STORE, 46952526344192, 46952526348287, +ERASE, 94299866005504, 94299866005504, +STORE, 94299866005504, 94299866021887, +STORE, 94299866021888, 94299866025983, +ERASE, 140680268922880, 140680268922880, +STORE, 140680268922880, 140680268926975, +STORE, 140680268926976, 140680268931071, +ERASE, 46952526233600, 46952526233600, +STORE, 140737488347136, 140737488351231, +STORE, 140722874793984, 140737488351231, +ERASE, 140722874793984, 140722874793984, +STORE, 140722874793984, 140722874798079, +STORE, 94448916213760, 94448916926463, +ERASE, 94448916213760, 94448916213760, +STORE, 94448916213760, 94448916262911, +STORE, 94448916262912, 94448916926463, +ERASE, 94448916262912, 94448916262912, +STORE, 94448916262912, 94448916807679, +STORE, 94448916807680, 94448916905983, +STORE, 94448916905984, 94448916926463, +STORE, 140389117046784, 140389117218815, +ERASE, 140389117046784, 140389117046784, +STORE, 140389117046784, 140389117050879, +STORE, 140389117050880, 140389117218815, +ERASE, 140389117050880, 140389117050880, +STORE, 140389117050880, 140389117173759, +STORE, 140389117173760, 140389117206527, +STORE, 140389117206528, 140389117214719, +STORE, 140389117214720, 140389117218815, +STORE, 140722875297792, 140722875301887, +STORE, 140722875285504, 140722875297791, +STORE, 47243677949952, 47243677958143, +STORE, 47243677958144, 47243677966335, +STORE, 47243677966336, 47243678072831, +STORE, 47243677982720, 47243678072831, +STORE, 47243677966336, 47243677982719, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678056447, +STORE, 47243678056448, 47243678072831, +STORE, 47243678035968, 47243678056447, +STORE, 47243677982720, 47243678035967, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678035967, +STORE, 47243678052352, 47243678056447, +STORE, 47243678035968, 47243678052351, +ERASE, 47243678035968, 47243678035968, +STORE, 47243678035968, 47243678052351, +STORE, 47243678064640, 47243678072831, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678064640, 47243678064640, +STORE, 47243678064640, 47243678072831, +STORE, 47243678072832, 47243679911935, +STORE, 47243678212096, 47243679911935, +STORE, 47243678072832, 47243678212095, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679870975, +STORE, 47243679870976, 47243679911935, +STORE, 47243679555584, 47243679870975, +STORE, 47243678212096, 47243679555583, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679555583, +STORE, 47243679866880, 47243679870975, +STORE, 47243679555584, 47243679866879, +ERASE, 47243679555584, 47243679555584, +STORE, 47243679555584, 47243679866879, +STORE, 47243679895552, 47243679911935, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679895552, 47243679895552, +STORE, 47243679895552, 47243679911935, +STORE, 47243679895552, 47243679924223, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679887359, +STORE, 47243679887360, 47243679895551, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678060543, +STORE, 47243678060544, 47243678064639, +ERASE, 94448916905984, 94448916905984, +STORE, 94448916905984, 94448916922367, +STORE, 94448916922368, 94448916926463, +ERASE, 140389117206528, 140389117206528, +STORE, 140389117206528, 140389117210623, +STORE, 140389117210624, 140389117214719, +ERASE, 47243677949952, 47243677949952, +STORE, 140737488347136, 140737488351231, +STORE, 140733068505088, 140737488351231, +ERASE, 140733068505088, 140733068505088, +STORE, 140733068505088, 140733068509183, +STORE, 94207145750528, 94207146463231, +ERASE, 94207145750528, 94207145750528, +STORE, 94207145750528, 94207145799679, +STORE, 94207145799680, 94207146463231, +ERASE, 94207145799680, 94207145799680, +STORE, 94207145799680, 94207146344447, +STORE, 94207146344448, 94207146442751, +STORE, 94207146442752, 94207146463231, +STORE, 140684504911872, 140684505083903, +ERASE, 140684504911872, 140684504911872, +STORE, 140684504911872, 140684504915967, +STORE, 140684504915968, 140684505083903, +ERASE, 140684504915968, 140684504915968, +STORE, 140684504915968, 140684505038847, +STORE, 140684505038848, 140684505071615, +STORE, 140684505071616, 140684505079807, +STORE, 140684505079808, 140684505083903, +STORE, 140733068607488, 140733068611583, +STORE, 140733068595200, 140733068607487, +STORE, 46948290084864, 46948290093055, +STORE, 46948290093056, 46948290101247, +STORE, 46948290101248, 46948290207743, +STORE, 46948290117632, 46948290207743, +STORE, 46948290101248, 46948290117631, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290191359, +STORE, 46948290191360, 46948290207743, +STORE, 46948290170880, 46948290191359, +STORE, 46948290117632, 46948290170879, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290170879, +STORE, 46948290187264, 46948290191359, +STORE, 46948290170880, 46948290187263, +ERASE, 46948290170880, 46948290170880, +STORE, 46948290170880, 46948290187263, +STORE, 46948290199552, 46948290207743, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290199552, 46948290199552, +STORE, 46948290199552, 46948290207743, +STORE, 46948290207744, 46948292046847, +STORE, 46948290347008, 46948292046847, +STORE, 46948290207744, 46948290347007, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948292005887, +STORE, 46948292005888, 46948292046847, +STORE, 46948291690496, 46948292005887, +STORE, 46948290347008, 46948291690495, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948291690495, +STORE, 46948292001792, 46948292005887, +STORE, 46948291690496, 46948292001791, +ERASE, 46948291690496, 46948291690496, +STORE, 46948291690496, 46948292001791, +STORE, 46948292030464, 46948292046847, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292030464, 46948292030464, +STORE, 46948292030464, 46948292046847, +STORE, 46948292030464, 46948292059135, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292022271, +STORE, 46948292022272, 46948292030463, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290195455, +STORE, 46948290195456, 46948290199551, +ERASE, 94207146442752, 94207146442752, +STORE, 94207146442752, 94207146459135, +STORE, 94207146459136, 94207146463231, +ERASE, 140684505071616, 140684505071616, +STORE, 140684505071616, 140684505075711, +STORE, 140684505075712, 140684505079807, +ERASE, 46948290084864, 46948290084864, +STORE, 140737488347136, 140737488351231, +STORE, 140726367158272, 140737488351231, +ERASE, 140726367158272, 140726367158272, +STORE, 140726367158272, 140726367162367, +STORE, 94436124106752, 94436124819455, +ERASE, 94436124106752, 94436124106752, +STORE, 94436124106752, 94436124155903, +STORE, 94436124155904, 94436124819455, +ERASE, 94436124155904, 94436124155904, +STORE, 94436124155904, 94436124700671, +STORE, 94436124700672, 94436124798975, +STORE, 94436124798976, 94436124819455, +STORE, 140049025044480, 140049025216511, +ERASE, 140049025044480, 140049025044480, +STORE, 140049025044480, 140049025048575, +STORE, 140049025048576, 140049025216511, +ERASE, 140049025048576, 140049025048576, +STORE, 140049025048576, 140049025171455, +STORE, 140049025171456, 140049025204223, +STORE, 140049025204224, 140049025212415, +STORE, 140049025212416, 140049025216511, +STORE, 140726367256576, 140726367260671, +STORE, 140726367244288, 140726367256575, +STORE, 47583769952256, 47583769960447, +STORE, 47583769960448, 47583769968639, +STORE, 47583769968640, 47583770075135, +STORE, 47583769985024, 47583770075135, +STORE, 47583769968640, 47583769985023, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770058751, +STORE, 47583770058752, 47583770075135, +STORE, 47583770038272, 47583770058751, +STORE, 47583769985024, 47583770038271, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770038271, +STORE, 47583770054656, 47583770058751, +STORE, 47583770038272, 47583770054655, +ERASE, 47583770038272, 47583770038272, +STORE, 47583770038272, 47583770054655, +STORE, 47583770066944, 47583770075135, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770066944, 47583770066944, +STORE, 47583770066944, 47583770075135, +STORE, 47583770075136, 47583771914239, +STORE, 47583770214400, 47583771914239, +STORE, 47583770075136, 47583770214399, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771873279, +STORE, 47583771873280, 47583771914239, +STORE, 47583771557888, 47583771873279, +STORE, 47583770214400, 47583771557887, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771557887, +STORE, 47583771869184, 47583771873279, +STORE, 47583771557888, 47583771869183, +ERASE, 47583771557888, 47583771557888, +STORE, 47583771557888, 47583771869183, +STORE, 47583771897856, 47583771914239, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771897856, 47583771897856, +STORE, 47583771897856, 47583771914239, +STORE, 47583771897856, 47583771926527, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771889663, +STORE, 47583771889664, 47583771897855, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770062847, +STORE, 47583770062848, 47583770066943, +ERASE, 94436124798976, 94436124798976, +STORE, 94436124798976, 94436124815359, +STORE, 94436124815360, 94436124819455, +ERASE, 140049025204224, 140049025204224, +STORE, 140049025204224, 140049025208319, +STORE, 140049025208320, 140049025212415, +ERASE, 47583769952256, 47583769952256, +STORE, 140737488347136, 140737488351231, +STORE, 140727116099584, 140737488351231, +ERASE, 140727116099584, 140727116099584, +STORE, 140727116099584, 140727116103679, +STORE, 94166319734784, 94166320447487, +ERASE, 94166319734784, 94166319734784, +STORE, 94166319734784, 94166319783935, +STORE, 94166319783936, 94166320447487, +ERASE, 94166319783936, 94166319783936, +STORE, 94166319783936, 94166320328703, +STORE, 94166320328704, 94166320427007, +STORE, 94166320427008, 94166320447487, +STORE, 139976559542272, 139976559714303, +ERASE, 139976559542272, 139976559542272, +STORE, 139976559542272, 139976559546367, +STORE, 139976559546368, 139976559714303, +ERASE, 139976559546368, 139976559546368, +STORE, 139976559546368, 139976559669247, +STORE, 139976559669248, 139976559702015, +STORE, 139976559702016, 139976559710207, +STORE, 139976559710208, 139976559714303, +STORE, 140727116222464, 140727116226559, +STORE, 140727116210176, 140727116222463, +STORE, 47656235454464, 47656235462655, +STORE, 47656235462656, 47656235470847, +STORE, 47656235470848, 47656235577343, +STORE, 47656235487232, 47656235577343, +STORE, 47656235470848, 47656235487231, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235560959, +STORE, 47656235560960, 47656235577343, +STORE, 47656235540480, 47656235560959, +STORE, 47656235487232, 47656235540479, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235540479, +STORE, 47656235556864, 47656235560959, +STORE, 47656235540480, 47656235556863, +ERASE, 47656235540480, 47656235540480, +STORE, 47656235540480, 47656235556863, +STORE, 47656235569152, 47656235577343, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235569152, 47656235569152, +STORE, 47656235569152, 47656235577343, +STORE, 47656235577344, 47656237416447, +STORE, 47656235716608, 47656237416447, +STORE, 47656235577344, 47656235716607, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237375487, +STORE, 47656237375488, 47656237416447, +STORE, 47656237060096, 47656237375487, +STORE, 47656235716608, 47656237060095, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237060095, +STORE, 47656237371392, 47656237375487, +STORE, 47656237060096, 47656237371391, +ERASE, 47656237060096, 47656237060096, +STORE, 47656237060096, 47656237371391, +STORE, 47656237400064, 47656237416447, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237400064, 47656237400064, +STORE, 47656237400064, 47656237416447, +STORE, 47656237400064, 47656237428735, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237391871, +STORE, 47656237391872, 47656237400063, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235565055, +STORE, 47656235565056, 47656235569151, +ERASE, 94166320427008, 94166320427008, +STORE, 94166320427008, 94166320443391, +STORE, 94166320443392, 94166320447487, +ERASE, 139976559702016, 139976559702016, +STORE, 139976559702016, 139976559706111, +STORE, 139976559706112, 139976559710207, +ERASE, 47656235454464, 47656235454464, +STORE, 94166332153856, 94166332289023, +STORE, 140737488347136, 140737488351231, +STORE, 140726412816384, 140737488351231, +ERASE, 140726412816384, 140726412816384, +STORE, 140726412816384, 140726412820479, +STORE, 94094884507648, 94094885220351, +ERASE, 94094884507648, 94094884507648, +STORE, 94094884507648, 94094884556799, +STORE, 94094884556800, 94094885220351, +ERASE, 94094884556800, 94094884556800, +STORE, 94094884556800, 94094885101567, +STORE, 94094885101568, 94094885199871, +STORE, 94094885199872, 94094885220351, +STORE, 139773773938688, 139773774110719, +ERASE, 139773773938688, 139773773938688, +STORE, 139773773938688, 139773773942783, +STORE, 139773773942784, 139773774110719, +ERASE, 139773773942784, 139773773942784, +STORE, 139773773942784, 139773774065663, +STORE, 139773774065664, 139773774098431, +STORE, 139773774098432, 139773774106623, +STORE, 139773774106624, 139773774110719, +STORE, 140726412963840, 140726412967935, +STORE, 140726412951552, 140726412963839, +STORE, 47859021058048, 47859021066239, +STORE, 47859021066240, 47859021074431, +STORE, 47859021074432, 47859021180927, +STORE, 47859021090816, 47859021180927, +STORE, 47859021074432, 47859021090815, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021164543, +STORE, 47859021164544, 47859021180927, +STORE, 47859021144064, 47859021164543, +STORE, 47859021090816, 47859021144063, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021144063, +STORE, 47859021160448, 47859021164543, +STORE, 47859021144064, 47859021160447, +ERASE, 47859021144064, 47859021144064, +STORE, 47859021144064, 47859021160447, +STORE, 47859021172736, 47859021180927, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021172736, 47859021172736, +STORE, 47859021172736, 47859021180927, +STORE, 47859021180928, 47859023020031, +STORE, 47859021320192, 47859023020031, +STORE, 47859021180928, 47859021320191, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022979071, +STORE, 47859022979072, 47859023020031, +STORE, 47859022663680, 47859022979071, +STORE, 47859021320192, 47859022663679, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022663679, +STORE, 47859022974976, 47859022979071, +STORE, 47859022663680, 47859022974975, +ERASE, 47859022663680, 47859022663680, +STORE, 47859022663680, 47859022974975, +STORE, 47859023003648, 47859023020031, +STORE, 47859022979072, 47859023003647, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859023003647, +ERASE, 47859023003648, 47859023003648, +STORE, 47859023003648, 47859023020031, +STORE, 47859023003648, 47859023032319, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859022995455, +STORE, 47859022995456, 47859023003647, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021168639, +STORE, 47859021168640, 47859021172735, +ERASE, 94094885199872, 94094885199872, +STORE, 94094885199872, 94094885216255, +STORE, 94094885216256, 94094885220351, +ERASE, 139773774098432, 139773774098432, +STORE, 139773774098432, 139773774102527, +STORE, 139773774102528, 139773774106623, +ERASE, 47859021058048, 47859021058048, +STORE, 94094901108736, 94094901243903, +STORE, 140737488347136, 140737488351231, +STORE, 140736567963648, 140737488351231, +ERASE, 140736567963648, 140736567963648, +STORE, 140736567963648, 140736567967743, +STORE, 94924425748480, 94924426461183, +ERASE, 94924425748480, 94924425748480, +STORE, 94924425748480, 94924425797631, +STORE, 94924425797632, 94924426461183, +ERASE, 94924425797632, 94924425797632, +STORE, 94924425797632, 94924426342399, +STORE, 94924426342400, 94924426440703, +STORE, 94924426440704, 94924426461183, +STORE, 140042126319616, 140042126491647, +ERASE, 140042126319616, 140042126319616, +STORE, 140042126319616, 140042126323711, +STORE, 140042126323712, 140042126491647, +ERASE, 140042126323712, 140042126323712, +STORE, 140042126323712, 140042126446591, +STORE, 140042126446592, 140042126479359, +STORE, 140042126479360, 140042126487551, +STORE, 140042126487552, 140042126491647, +STORE, 140736568672256, 140736568676351, +STORE, 140736568659968, 140736568672255, +STORE, 47590668677120, 47590668685311, +STORE, 47590668685312, 47590668693503, +STORE, 47590668693504, 47590668799999, +STORE, 47590668709888, 47590668799999, +STORE, 47590668693504, 47590668709887, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668783615, +STORE, 47590668783616, 47590668799999, +STORE, 47590668763136, 47590668783615, +STORE, 47590668709888, 47590668763135, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668763135, +STORE, 47590668779520, 47590668783615, +STORE, 47590668763136, 47590668779519, +ERASE, 47590668763136, 47590668763136, +STORE, 47590668763136, 47590668779519, +STORE, 47590668791808, 47590668799999, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668791808, 47590668791808, +STORE, 47590668791808, 47590668799999, +STORE, 47590668800000, 47590670639103, +STORE, 47590668939264, 47590670639103, +STORE, 47590668800000, 47590668939263, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670598143, +STORE, 47590670598144, 47590670639103, +STORE, 47590670282752, 47590670598143, +STORE, 47590668939264, 47590670282751, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670282751, +STORE, 47590670594048, 47590670598143, +STORE, 47590670282752, 47590670594047, +ERASE, 47590670282752, 47590670282752, +STORE, 47590670282752, 47590670594047, +STORE, 47590670622720, 47590670639103, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670622720, 47590670622720, +STORE, 47590670622720, 47590670639103, +STORE, 47590670622720, 47590670651391, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670614527, +STORE, 47590670614528, 47590670622719, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668787711, +STORE, 47590668787712, 47590668791807, +ERASE, 94924426440704, 94924426440704, +STORE, 94924426440704, 94924426457087, +STORE, 94924426457088, 94924426461183, +ERASE, 140042126479360, 140042126479360, +STORE, 140042126479360, 140042126483455, +STORE, 140042126483456, 140042126487551, +ERASE, 47590668677120, 47590668677120, +STORE, 140737488347136, 140737488351231, +STORE, 140733281439744, 140737488351231, +ERASE, 140733281439744, 140733281439744, +STORE, 140733281439744, 140733281443839, +STORE, 94490667069440, 94490667782143, +ERASE, 94490667069440, 94490667069440, +STORE, 94490667069440, 94490667118591, +STORE, 94490667118592, 94490667782143, +ERASE, 94490667118592, 94490667118592, +STORE, 94490667118592, 94490667663359, +STORE, 94490667663360, 94490667761663, +STORE, 94490667761664, 94490667782143, +STORE, 139878215118848, 139878215290879, +ERASE, 139878215118848, 139878215118848, +STORE, 139878215118848, 139878215122943, +STORE, 139878215122944, 139878215290879, +ERASE, 139878215122944, 139878215122944, +STORE, 139878215122944, 139878215245823, +STORE, 139878215245824, 139878215278591, +STORE, 139878215278592, 139878215286783, +STORE, 139878215286784, 139878215290879, +STORE, 140733281464320, 140733281468415, +STORE, 140733281452032, 140733281464319, +STORE, 47754579877888, 47754579886079, +STORE, 47754579886080, 47754579894271, +STORE, 47754579894272, 47754580000767, +STORE, 47754579910656, 47754580000767, +STORE, 47754579894272, 47754579910655, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579984383, +STORE, 47754579984384, 47754580000767, +STORE, 47754579963904, 47754579984383, +STORE, 47754579910656, 47754579963903, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579963903, +STORE, 47754579980288, 47754579984383, +STORE, 47754579963904, 47754579980287, +ERASE, 47754579963904, 47754579963904, +STORE, 47754579963904, 47754579980287, +STORE, 47754579992576, 47754580000767, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579992576, 47754579992576, +STORE, 47754579992576, 47754580000767, +STORE, 47754580000768, 47754581839871, +STORE, 47754580140032, 47754581839871, +STORE, 47754580000768, 47754580140031, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581798911, +STORE, 47754581798912, 47754581839871, +STORE, 47754581483520, 47754581798911, +STORE, 47754580140032, 47754581483519, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581483519, +STORE, 47754581794816, 47754581798911, +STORE, 47754581483520, 47754581794815, +ERASE, 47754581483520, 47754581483520, +STORE, 47754581483520, 47754581794815, +STORE, 47754581823488, 47754581839871, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581823488, 47754581823488, +STORE, 47754581823488, 47754581839871, +STORE, 47754581823488, 47754581852159, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581815295, +STORE, 47754581815296, 47754581823487, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579988479, +STORE, 47754579988480, 47754579992575, +ERASE, 94490667761664, 94490667761664, +STORE, 94490667761664, 94490667778047, +STORE, 94490667778048, 94490667782143, +ERASE, 139878215278592, 139878215278592, +STORE, 139878215278592, 139878215282687, +STORE, 139878215282688, 139878215286783, +ERASE, 47754579877888, 47754579877888, +STORE, 94490669649920, 94490669785087, +STORE, 140737488347136, 140737488351231, +STORE, 140735382188032, 140737488351231, +ERASE, 140735382188032, 140735382188032, +STORE, 140735382188032, 140735382192127, +STORE, 94150181302272, 94150182014975, +ERASE, 94150181302272, 94150181302272, +STORE, 94150181302272, 94150181351423, +STORE, 94150181351424, 94150182014975, +ERASE, 94150181351424, 94150181351424, +STORE, 94150181351424, 94150181896191, +STORE, 94150181896192, 94150181994495, +STORE, 94150181994496, 94150182014975, +STORE, 139679752458240, 139679752630271, +ERASE, 139679752458240, 139679752458240, +STORE, 139679752458240, 139679752462335, +STORE, 139679752462336, 139679752630271, +ERASE, 139679752462336, 139679752462336, +STORE, 139679752462336, 139679752585215, +STORE, 139679752585216, 139679752617983, +STORE, 139679752617984, 139679752626175, +STORE, 139679752626176, 139679752630271, +STORE, 140735382536192, 140735382540287, +STORE, 140735382523904, 140735382536191, +STORE, 47953042538496, 47953042546687, +STORE, 47953042546688, 47953042554879, +STORE, 47953042554880, 47953042661375, +STORE, 47953042571264, 47953042661375, +STORE, 47953042554880, 47953042571263, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042644991, +STORE, 47953042644992, 47953042661375, +STORE, 47953042624512, 47953042644991, +STORE, 47953042571264, 47953042624511, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042624511, +STORE, 47953042640896, 47953042644991, +STORE, 47953042624512, 47953042640895, +ERASE, 47953042624512, 47953042624512, +STORE, 47953042624512, 47953042640895, +STORE, 47953042653184, 47953042661375, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042653184, 47953042653184, +STORE, 47953042653184, 47953042661375, +STORE, 47953042661376, 47953044500479, +STORE, 47953042800640, 47953044500479, +STORE, 47953042661376, 47953042800639, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044459519, +STORE, 47953044459520, 47953044500479, +STORE, 47953044144128, 47953044459519, +STORE, 47953042800640, 47953044144127, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044144127, +STORE, 47953044455424, 47953044459519, +STORE, 47953044144128, 47953044455423, +ERASE, 47953044144128, 47953044144128, +STORE, 47953044144128, 47953044455423, +STORE, 47953044484096, 47953044500479, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044484096, 47953044484096, +STORE, 47953044484096, 47953044500479, +STORE, 47953044484096, 47953044512767, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044475903, +STORE, 47953044475904, 47953044484095, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042649087, +STORE, 47953042649088, 47953042653183, +ERASE, 94150181994496, 94150181994496, +STORE, 94150181994496, 94150182010879, +STORE, 94150182010880, 94150182014975, +ERASE, 139679752617984, 139679752617984, +STORE, 139679752617984, 139679752622079, +STORE, 139679752622080, 139679752626175, +ERASE, 47953042538496, 47953042538496, +STORE, 140737488347136, 140737488351231, +STORE, 140737044123648, 140737488351231, +ERASE, 140737044123648, 140737044123648, +STORE, 140737044123648, 140737044127743, +STORE, 94425324294144, 94425325006847, +ERASE, 94425324294144, 94425324294144, +STORE, 94425324294144, 94425324343295, +STORE, 94425324343296, 94425325006847, +ERASE, 94425324343296, 94425324343296, +STORE, 94425324343296, 94425324888063, +STORE, 94425324888064, 94425324986367, +STORE, 94425324986368, 94425325006847, +STORE, 140382015016960, 140382015188991, +ERASE, 140382015016960, 140382015016960, +STORE, 140382015016960, 140382015021055, +STORE, 140382015021056, 140382015188991, +ERASE, 140382015021056, 140382015021056, +STORE, 140382015021056, 140382015143935, +STORE, 140382015143936, 140382015176703, +STORE, 140382015176704, 140382015184895, +STORE, 140382015184896, 140382015188991, +STORE, 140737045585920, 140737045590015, +STORE, 140737045573632, 140737045585919, +STORE, 47250779979776, 47250779987967, +STORE, 47250779987968, 47250779996159, +STORE, 47250779996160, 47250780102655, +STORE, 47250780012544, 47250780102655, +STORE, 47250779996160, 47250780012543, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780086271, +STORE, 47250780086272, 47250780102655, +STORE, 47250780065792, 47250780086271, +STORE, 47250780012544, 47250780065791, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780065791, +STORE, 47250780082176, 47250780086271, +STORE, 47250780065792, 47250780082175, +ERASE, 47250780065792, 47250780065792, +STORE, 47250780065792, 47250780082175, +STORE, 47250780094464, 47250780102655, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780094464, 47250780094464, +STORE, 47250780094464, 47250780102655, +STORE, 47250780102656, 47250781941759, +STORE, 47250780241920, 47250781941759, +STORE, 47250780102656, 47250780241919, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781900799, +STORE, 47250781900800, 47250781941759, +STORE, 47250781585408, 47250781900799, +STORE, 47250780241920, 47250781585407, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781585407, +STORE, 47250781896704, 47250781900799, +STORE, 47250781585408, 47250781896703, +ERASE, 47250781585408, 47250781585408, +STORE, 47250781585408, 47250781896703, +STORE, 47250781925376, 47250781941759, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781925376, 47250781925376, +STORE, 47250781925376, 47250781941759, +STORE, 47250781925376, 47250781954047, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781917183, +STORE, 47250781917184, 47250781925375, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780090367, +STORE, 47250780090368, 47250780094463, +ERASE, 94425324986368, 94425324986368, +STORE, 94425324986368, 94425325002751, +STORE, 94425325002752, 94425325006847, +ERASE, 140382015176704, 140382015176704, +STORE, 140382015176704, 140382015180799, +STORE, 140382015180800, 140382015184895, +ERASE, 47250779979776, 47250779979776, +STORE, 94425351438336, 94425351573503, +STORE, 140737488347136, 140737488351231, +STORE, 140736801144832, 140737488351231, +ERASE, 140736801144832, 140736801144832, +STORE, 140736801144832, 140736801148927, +STORE, 94629429358592, 94629430071295, +ERASE, 94629429358592, 94629429358592, +STORE, 94629429358592, 94629429407743, +STORE, 94629429407744, 94629430071295, +ERASE, 94629429407744, 94629429407744, +STORE, 94629429407744, 94629429952511, +STORE, 94629429952512, 94629430050815, +STORE, 94629430050816, 94629430071295, +STORE, 139801685483520, 139801685655551, +ERASE, 139801685483520, 139801685483520, +STORE, 139801685483520, 139801685487615, +STORE, 139801685487616, 139801685655551, +ERASE, 139801685487616, 139801685487616, +STORE, 139801685487616, 139801685610495, +STORE, 139801685610496, 139801685643263, +STORE, 139801685643264, 139801685651455, +STORE, 139801685651456, 139801685655551, +STORE, 140736801198080, 140736801202175, +STORE, 140736801185792, 140736801198079, +STORE, 47831109513216, 47831109521407, +STORE, 47831109521408, 47831109529599, +STORE, 47831109529600, 47831109636095, +STORE, 47831109545984, 47831109636095, +STORE, 47831109529600, 47831109545983, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109619711, +STORE, 47831109619712, 47831109636095, +STORE, 47831109599232, 47831109619711, +STORE, 47831109545984, 47831109599231, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109599231, +STORE, 47831109615616, 47831109619711, +STORE, 47831109599232, 47831109615615, +ERASE, 47831109599232, 47831109599232, +STORE, 47831109599232, 47831109615615, +STORE, 47831109627904, 47831109636095, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109627904, 47831109627904, +STORE, 47831109627904, 47831109636095, +STORE, 47831109636096, 47831111475199, +STORE, 47831109775360, 47831111475199, +STORE, 47831109636096, 47831109775359, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111434239, +STORE, 47831111434240, 47831111475199, +STORE, 47831111118848, 47831111434239, +STORE, 47831109775360, 47831111118847, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111118847, +STORE, 47831111430144, 47831111434239, +STORE, 47831111118848, 47831111430143, +ERASE, 47831111118848, 47831111118848, +STORE, 47831111118848, 47831111430143, +STORE, 47831111458816, 47831111475199, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111458816, 47831111458816, +STORE, 47831111458816, 47831111475199, +STORE, 47831111458816, 47831111487487, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111450623, +STORE, 47831111450624, 47831111458815, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109623807, +STORE, 47831109623808, 47831109627903, +ERASE, 94629430050816, 94629430050816, +STORE, 94629430050816, 94629430067199, +STORE, 94629430067200, 94629430071295, +ERASE, 139801685643264, 139801685643264, +STORE, 139801685643264, 139801685647359, +STORE, 139801685647360, 139801685651455, +ERASE, 47831109513216, 47831109513216, +STORE, 140737488347136, 140737488351231, +STORE, 140729419612160, 140737488351231, +ERASE, 140729419612160, 140729419612160, +STORE, 140729419612160, 140729419616255, +STORE, 94443354148864, 94443354861567, +ERASE, 94443354148864, 94443354148864, +STORE, 94443354148864, 94443354198015, +STORE, 94443354198016, 94443354861567, +ERASE, 94443354198016, 94443354198016, +STORE, 94443354198016, 94443354742783, +STORE, 94443354742784, 94443354841087, +STORE, 94443354841088, 94443354861567, +STORE, 139741700038656, 139741700210687, +ERASE, 139741700038656, 139741700038656, +STORE, 139741700038656, 139741700042751, +STORE, 139741700042752, 139741700210687, +ERASE, 139741700042752, 139741700042752, +STORE, 139741700042752, 139741700165631, +STORE, 139741700165632, 139741700198399, +STORE, 139741700198400, 139741700206591, +STORE, 139741700206592, 139741700210687, +STORE, 140729420574720, 140729420578815, +STORE, 140729420562432, 140729420574719, +STORE, 47891094958080, 47891094966271, +STORE, 47891094966272, 47891094974463, +STORE, 47891094974464, 47891095080959, +STORE, 47891094990848, 47891095080959, +STORE, 47891094974464, 47891094990847, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095064575, +STORE, 47891095064576, 47891095080959, +STORE, 47891095044096, 47891095064575, +STORE, 47891094990848, 47891095044095, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095044095, +STORE, 47891095060480, 47891095064575, +STORE, 47891095044096, 47891095060479, +ERASE, 47891095044096, 47891095044096, +STORE, 47891095044096, 47891095060479, +STORE, 47891095072768, 47891095080959, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095072768, 47891095072768, +STORE, 47891095072768, 47891095080959, +STORE, 47891095080960, 47891096920063, +STORE, 47891095220224, 47891096920063, +STORE, 47891095080960, 47891095220223, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096879103, +STORE, 47891096879104, 47891096920063, +STORE, 47891096563712, 47891096879103, +STORE, 47891095220224, 47891096563711, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096563711, +STORE, 47891096875008, 47891096879103, +STORE, 47891096563712, 47891096875007, +ERASE, 47891096563712, 47891096563712, +STORE, 47891096563712, 47891096875007, +STORE, 47891096903680, 47891096920063, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096903680, 47891096903680, +STORE, 47891096903680, 47891096920063, +STORE, 47891096903680, 47891096932351, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096895487, +STORE, 47891096895488, 47891096903679, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095068671, +STORE, 47891095068672, 47891095072767, +ERASE, 94443354841088, 94443354841088, +STORE, 94443354841088, 94443354857471, +STORE, 94443354857472, 94443354861567, +ERASE, 139741700198400, 139741700198400, +STORE, 139741700198400, 139741700202495, +STORE, 139741700202496, 139741700206591, +ERASE, 47891094958080, 47891094958080, +STORE, 94443360825344, 94443360960511, +STORE, 140737488347136, 140737488351231, +STORE, 140722961661952, 140737488351231, +ERASE, 140722961661952, 140722961661952, +STORE, 140722961661952, 140722961666047, +STORE, 94878388944896, 94878389657599, +ERASE, 94878388944896, 94878388944896, +STORE, 94878388944896, 94878388994047, +STORE, 94878388994048, 94878389657599, +ERASE, 94878388994048, 94878388994048, +STORE, 94878388994048, 94878389538815, +STORE, 94878389538816, 94878389637119, +STORE, 94878389637120, 94878389657599, +STORE, 140210690056192, 140210690228223, +ERASE, 140210690056192, 140210690056192, +STORE, 140210690056192, 140210690060287, +STORE, 140210690060288, 140210690228223, +ERASE, 140210690060288, 140210690060288, +STORE, 140210690060288, 140210690183167, +STORE, 140210690183168, 140210690215935, +STORE, 140210690215936, 140210690224127, +STORE, 140210690224128, 140210690228223, +STORE, 140722963148800, 140722963152895, +STORE, 140722963136512, 140722963148799, +STORE, 47422104940544, 47422104948735, +STORE, 47422104948736, 47422104956927, +STORE, 47422104956928, 47422105063423, +STORE, 47422104973312, 47422105063423, +STORE, 47422104956928, 47422104973311, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105047039, +STORE, 47422105047040, 47422105063423, +STORE, 47422105026560, 47422105047039, +STORE, 47422104973312, 47422105026559, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105026559, +STORE, 47422105042944, 47422105047039, +STORE, 47422105026560, 47422105042943, +ERASE, 47422105026560, 47422105026560, +STORE, 47422105026560, 47422105042943, +STORE, 47422105055232, 47422105063423, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105055232, 47422105055232, +STORE, 47422105055232, 47422105063423, +STORE, 47422105063424, 47422106902527, +STORE, 47422105202688, 47422106902527, +STORE, 47422105063424, 47422105202687, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106861567, +STORE, 47422106861568, 47422106902527, +STORE, 47422106546176, 47422106861567, +STORE, 47422105202688, 47422106546175, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106546175, +STORE, 47422106857472, 47422106861567, +STORE, 47422106546176, 47422106857471, +ERASE, 47422106546176, 47422106546176, +STORE, 47422106546176, 47422106857471, +STORE, 47422106886144, 47422106902527, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106886144, 47422106886144, +STORE, 47422106886144, 47422106902527, +STORE, 47422106886144, 47422106914815, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106877951, +STORE, 47422106877952, 47422106886143, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105051135, +STORE, 47422105051136, 47422105055231, +ERASE, 94878389637120, 94878389637120, +STORE, 94878389637120, 94878389653503, +STORE, 94878389653504, 94878389657599, +ERASE, 140210690215936, 140210690215936, +STORE, 140210690215936, 140210690220031, +STORE, 140210690220032, 140210690224127, +ERASE, 47422104940544, 47422104940544, +STORE, 140737488347136, 140737488351231, +STORE, 140727690309632, 140737488351231, +ERASE, 140727690309632, 140727690309632, +STORE, 140727690309632, 140727690313727, +STORE, 94121892208640, 94121892921343, +ERASE, 94121892208640, 94121892208640, +STORE, 94121892208640, 94121892257791, +STORE, 94121892257792, 94121892921343, +ERASE, 94121892257792, 94121892257792, +STORE, 94121892257792, 94121892802559, +STORE, 94121892802560, 94121892900863, +STORE, 94121892900864, 94121892921343, +STORE, 140662438326272, 140662438498303, +ERASE, 140662438326272, 140662438326272, +STORE, 140662438326272, 140662438330367, +STORE, 140662438330368, 140662438498303, +ERASE, 140662438330368, 140662438330368, +STORE, 140662438330368, 140662438453247, +STORE, 140662438453248, 140662438486015, +STORE, 140662438486016, 140662438494207, +STORE, 140662438494208, 140662438498303, +STORE, 140727690379264, 140727690383359, +STORE, 140727690366976, 140727690379263, +STORE, 46970356670464, 46970356678655, +STORE, 46970356678656, 46970356686847, +STORE, 46970356686848, 46970356793343, +STORE, 46970356703232, 46970356793343, +STORE, 46970356686848, 46970356703231, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356776959, +STORE, 46970356776960, 46970356793343, +STORE, 46970356756480, 46970356776959, +STORE, 46970356703232, 46970356756479, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356756479, +STORE, 46970356772864, 46970356776959, +STORE, 46970356756480, 46970356772863, +ERASE, 46970356756480, 46970356756480, +STORE, 46970356756480, 46970356772863, +STORE, 46970356785152, 46970356793343, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356785152, 46970356785152, +STORE, 46970356785152, 46970356793343, +STORE, 46970356793344, 46970358632447, +STORE, 46970356932608, 46970358632447, +STORE, 46970356793344, 46970356932607, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358591487, +STORE, 46970358591488, 46970358632447, +STORE, 46970358276096, 46970358591487, +STORE, 46970356932608, 46970358276095, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358276095, +STORE, 46970358587392, 46970358591487, +STORE, 46970358276096, 46970358587391, +ERASE, 46970358276096, 46970358276096, +STORE, 46970358276096, 46970358587391, +STORE, 46970358616064, 46970358632447, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358616064, 46970358616064, +STORE, 46970358616064, 46970358632447, +STORE, 46970358616064, 46970358644735, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358607871, +STORE, 46970358607872, 46970358616063, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356781055, +STORE, 46970356781056, 46970356785151, +ERASE, 94121892900864, 94121892900864, +STORE, 94121892900864, 94121892917247, +STORE, 94121892917248, 94121892921343, +ERASE, 140662438486016, 140662438486016, +STORE, 140662438486016, 140662438490111, +STORE, 140662438490112, 140662438494207, +ERASE, 46970356670464, 46970356670464, +STORE, 94121898610688, 94121898745855, +STORE, 140737488347136, 140737488351231, +STORE, 140737189351424, 140737488351231, +ERASE, 140737189351424, 140737189351424, +STORE, 140737189351424, 140737189355519, +STORE, 93847948832768, 93847949545471, +ERASE, 93847948832768, 93847948832768, +STORE, 93847948832768, 93847948881919, +STORE, 93847948881920, 93847949545471, +ERASE, 93847948881920, 93847948881920, +STORE, 93847948881920, 93847949426687, +STORE, 93847949426688, 93847949524991, +STORE, 93847949524992, 93847949545471, +STORE, 139698989985792, 139698990157823, +ERASE, 139698989985792, 139698989985792, +STORE, 139698989985792, 139698989989887, +STORE, 139698989989888, 139698990157823, +ERASE, 139698989989888, 139698989989888, +STORE, 139698989989888, 139698990112767, +STORE, 139698990112768, 139698990145535, +STORE, 139698990145536, 139698990153727, +STORE, 139698990153728, 139698990157823, +STORE, 140737189744640, 140737189748735, +STORE, 140737189732352, 140737189744639, +STORE, 47933805010944, 47933805019135, +STORE, 47933805019136, 47933805027327, +STORE, 47933805027328, 47933805133823, +STORE, 47933805043712, 47933805133823, +STORE, 47933805027328, 47933805043711, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805117439, +STORE, 47933805117440, 47933805133823, +STORE, 47933805096960, 47933805117439, +STORE, 47933805043712, 47933805096959, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805096959, +STORE, 47933805113344, 47933805117439, +STORE, 47933805096960, 47933805113343, +ERASE, 47933805096960, 47933805096960, +STORE, 47933805096960, 47933805113343, +STORE, 47933805125632, 47933805133823, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805125632, 47933805125632, +STORE, 47933805125632, 47933805133823, +STORE, 47933805133824, 47933806972927, +STORE, 47933805273088, 47933806972927, +STORE, 47933805133824, 47933805273087, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806931967, +STORE, 47933806931968, 47933806972927, +STORE, 47933806616576, 47933806931967, +STORE, 47933805273088, 47933806616575, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806616575, +STORE, 47933806927872, 47933806931967, +STORE, 47933806616576, 47933806927871, +ERASE, 47933806616576, 47933806616576, +STORE, 47933806616576, 47933806927871, +STORE, 47933806956544, 47933806972927, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806956544, 47933806956544, +STORE, 47933806956544, 47933806972927, +STORE, 47933806956544, 47933806985215, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806948351, +STORE, 47933806948352, 47933806956543, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805121535, +STORE, 47933805121536, 47933805125631, +ERASE, 93847949524992, 93847949524992, +STORE, 93847949524992, 93847949541375, +STORE, 93847949541376, 93847949545471, +ERASE, 139698990145536, 139698990145536, +STORE, 139698990145536, 139698990149631, +STORE, 139698990149632, 139698990153727, +ERASE, 47933805010944, 47933805010944, +STORE, 140737488347136, 140737488351231, +STORE, 140725553991680, 140737488351231, +ERASE, 140725553991680, 140725553991680, +STORE, 140725553991680, 140725553995775, +STORE, 93980056248320, 93980056961023, +ERASE, 93980056248320, 93980056248320, +STORE, 93980056248320, 93980056297471, +STORE, 93980056297472, 93980056961023, +ERASE, 93980056297472, 93980056297472, +STORE, 93980056297472, 93980056842239, +STORE, 93980056842240, 93980056940543, +STORE, 93980056940544, 93980056961023, +STORE, 140146588971008, 140146589143039, +ERASE, 140146588971008, 140146588971008, +STORE, 140146588971008, 140146588975103, +STORE, 140146588975104, 140146589143039, +ERASE, 140146588975104, 140146588975104, +STORE, 140146588975104, 140146589097983, +STORE, 140146589097984, 140146589130751, +STORE, 140146589130752, 140146589138943, +STORE, 140146589138944, 140146589143039, +STORE, 140725554860032, 140725554864127, +STORE, 140725554847744, 140725554860031, +STORE, 47486206025728, 47486206033919, +STORE, 47486206033920, 47486206042111, +STORE, 47486206042112, 47486206148607, +STORE, 47486206058496, 47486206148607, +STORE, 47486206042112, 47486206058495, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206132223, +STORE, 47486206132224, 47486206148607, +STORE, 47486206111744, 47486206132223, +STORE, 47486206058496, 47486206111743, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206111743, +STORE, 47486206128128, 47486206132223, +STORE, 47486206111744, 47486206128127, +ERASE, 47486206111744, 47486206111744, +STORE, 47486206111744, 47486206128127, +STORE, 47486206140416, 47486206148607, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206140416, 47486206140416, +STORE, 47486206140416, 47486206148607, +STORE, 47486206148608, 47486207987711, +STORE, 47486206287872, 47486207987711, +STORE, 47486206148608, 47486206287871, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207946751, +STORE, 47486207946752, 47486207987711, +STORE, 47486207631360, 47486207946751, +STORE, 47486206287872, 47486207631359, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207631359, +STORE, 47486207942656, 47486207946751, +STORE, 47486207631360, 47486207942655, +ERASE, 47486207631360, 47486207631360, +STORE, 47486207631360, 47486207942655, +STORE, 47486207971328, 47486207987711, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207971328, 47486207971328, +STORE, 47486207971328, 47486207987711, +STORE, 47486207971328, 47486207999999, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207963135, +STORE, 47486207963136, 47486207971327, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206136319, +STORE, 47486206136320, 47486206140415, +ERASE, 93980056940544, 93980056940544, +STORE, 93980056940544, 93980056956927, +STORE, 93980056956928, 93980056961023, +ERASE, 140146589130752, 140146589130752, +STORE, 140146589130752, 140146589134847, +STORE, 140146589134848, 140146589138943, +ERASE, 47486206025728, 47486206025728, +STORE, 93980070006784, 93980070141951, +STORE, 140737488347136, 140737488351231, +STORE, 140727334776832, 140737488351231, +ERASE, 140727334776832, 140727334776832, +STORE, 140727334776832, 140727334780927, +STORE, 94049747247104, 94049747959807, +ERASE, 94049747247104, 94049747247104, +STORE, 94049747247104, 94049747296255, +STORE, 94049747296256, 94049747959807, +ERASE, 94049747296256, 94049747296256, +STORE, 94049747296256, 94049747841023, +STORE, 94049747841024, 94049747939327, +STORE, 94049747939328, 94049747959807, +STORE, 140227307216896, 140227307388927, +ERASE, 140227307216896, 140227307216896, +STORE, 140227307216896, 140227307220991, +STORE, 140227307220992, 140227307388927, +ERASE, 140227307220992, 140227307220992, +STORE, 140227307220992, 140227307343871, +STORE, 140227307343872, 140227307376639, +STORE, 140227307376640, 140227307384831, +STORE, 140227307384832, 140227307388927, +STORE, 140727335337984, 140727335342079, +STORE, 140727335325696, 140727335337983, +STORE, 47405487779840, 47405487788031, +STORE, 47405487788032, 47405487796223, +STORE, 47405487796224, 47405487902719, +STORE, 47405487812608, 47405487902719, +STORE, 47405487796224, 47405487812607, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487886335, +STORE, 47405487886336, 47405487902719, +STORE, 47405487865856, 47405487886335, +STORE, 47405487812608, 47405487865855, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487865855, +STORE, 47405487882240, 47405487886335, +STORE, 47405487865856, 47405487882239, +ERASE, 47405487865856, 47405487865856, +STORE, 47405487865856, 47405487882239, +STORE, 47405487894528, 47405487902719, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487894528, 47405487894528, +STORE, 47405487894528, 47405487902719, +STORE, 47405487902720, 47405489741823, +STORE, 47405488041984, 47405489741823, +STORE, 47405487902720, 47405488041983, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489700863, +STORE, 47405489700864, 47405489741823, +STORE, 47405489385472, 47405489700863, +STORE, 47405488041984, 47405489385471, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489385471, +STORE, 47405489696768, 47405489700863, +STORE, 47405489385472, 47405489696767, +ERASE, 47405489385472, 47405489385472, +STORE, 47405489385472, 47405489696767, +STORE, 47405489725440, 47405489741823, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489725440, 47405489725440, +STORE, 47405489725440, 47405489741823, +STORE, 47405489725440, 47405489754111, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489717247, +STORE, 47405489717248, 47405489725439, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487890431, +STORE, 47405487890432, 47405487894527, +ERASE, 94049747939328, 94049747939328, +STORE, 94049747939328, 94049747955711, +STORE, 94049747955712, 94049747959807, +ERASE, 140227307376640, 140227307376640, +STORE, 140227307376640, 140227307380735, +STORE, 140227307380736, 140227307384831, +ERASE, 47405487779840, 47405487779840, +STORE, 94049758810112, 94049758945279, +STORE, 140737488347136, 140737488351231, +STORE, 140727079718912, 140737488351231, +ERASE, 140727079718912, 140727079718912, +STORE, 140727079718912, 140727079723007, +STORE, 94250996527104, 94250997239807, +ERASE, 94250996527104, 94250996527104, +STORE, 94250996527104, 94250996576255, +STORE, 94250996576256, 94250997239807, +ERASE, 94250996576256, 94250996576256, +STORE, 94250996576256, 94250997121023, +STORE, 94250997121024, 94250997219327, +STORE, 94250997219328, 94250997239807, +STORE, 140060022587392, 140060022759423, +ERASE, 140060022587392, 140060022587392, +STORE, 140060022587392, 140060022591487, +STORE, 140060022591488, 140060022759423, +ERASE, 140060022591488, 140060022591488, +STORE, 140060022591488, 140060022714367, +STORE, 140060022714368, 140060022747135, +STORE, 140060022747136, 140060022755327, +STORE, 140060022755328, 140060022759423, +STORE, 140727079788544, 140727079792639, +STORE, 140727079776256, 140727079788543, +/* this next one caused issues when lowering the efficiency */ +STORE, 47572772409344, 47572772417535, +STORE, 47572772417536, 47572772425727, +STORE, 47572772425728, 47572772532223, +STORE, 47572772442112, 47572772532223, +STORE, 47572772425728, 47572772442111, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772515839, +STORE, 47572772515840, 47572772532223, +STORE, 47572772495360, 47572772515839, +STORE, 47572772442112, 47572772495359, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772495359, +STORE, 47572772511744, 47572772515839, +STORE, 47572772495360, 47572772511743, +ERASE, 47572772495360, 47572772495360, +STORE, 47572772495360, 47572772511743, +STORE, 47572772524032, 47572772532223, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772524032, 47572772524032, +STORE, 47572772524032, 47572772532223, +STORE, 47572772532224, 47572774371327, +STORE, 47572772671488, 47572774371327, +STORE, 47572772532224, 47572772671487, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774330367, +STORE, 47572774330368, 47572774371327, +STORE, 47572774014976, 47572774330367, +STORE, 47572772671488, 47572774014975, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774014975, +STORE, 47572774326272, 47572774330367, +STORE, 47572774014976, 47572774326271, +ERASE, 47572774014976, 47572774014976, +STORE, 47572774014976, 47572774326271, +STORE, 47572774354944, 47572774371327, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774354944, 47572774354944, +STORE, 47572774354944, 47572774371327, +STORE, 47572774354944, 47572774383615, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774346751, +STORE, 47572774346752, 47572774354943, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772519935, +STORE, 47572772519936, 47572772524031, +ERASE, 94250997219328, 94250997219328, +STORE, 94250997219328, 94250997235711, +STORE, 94250997235712, 94250997239807, +ERASE, 140060022747136, 140060022747136, +STORE, 140060022747136, 140060022751231, +STORE, 140060022751232, 140060022755327, +ERASE, 47572772409344, 47572772409344, +STORE, 94251018305536, 94251018440703, +STORE, 140737488347136, 140737488351231, +STORE, 140730012389376, 140737488351231, +ERASE, 140730012389376, 140730012389376, +STORE, 140730012389376, 140730012393471, +STORE, 94382607675392, 94382607695871, +ERASE, 94382607675392, 94382607675392, +STORE, 94382607675392, 94382607679487, +STORE, 94382607679488, 94382607695871, +ERASE, 94382607679488, 94382607679488, +STORE, 94382607679488, 94382607683583, +STORE, 94382607683584, 94382607687679, +STORE, 94382607687680, 94382607695871, +STORE, 140252451454976, 140252451627007, +ERASE, 140252451454976, 140252451454976, +STORE, 140252451454976, 140252451459071, +STORE, 140252451459072, 140252451627007, +ERASE, 140252451459072, 140252451459072, +STORE, 140252451459072, 140252451581951, +STORE, 140252451581952, 140252451614719, +STORE, 140252451614720, 140252451622911, +STORE, 140252451622912, 140252451627007, +STORE, 140730013548544, 140730013552639, +STORE, 140730013536256, 140730013548543, +STORE, 47380343541760, 47380343549951, +STORE, 47380343549952, 47380343558143, +STORE, 47380343558144, 47380345397247, +STORE, 47380343697408, 47380345397247, +STORE, 47380343558144, 47380343697407, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345356287, +STORE, 47380345356288, 47380345397247, +STORE, 47380345040896, 47380345356287, +STORE, 47380343697408, 47380345040895, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345040895, +STORE, 47380345352192, 47380345356287, +STORE, 47380345040896, 47380345352191, +ERASE, 47380345040896, 47380345040896, +STORE, 47380345040896, 47380345352191, +STORE, 47380345380864, 47380345397247, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345380864, 47380345380864, +STORE, 47380345380864, 47380345397247, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345372671, +STORE, 47380345372672, 47380345380863, +ERASE, 94382607687680, 94382607687680, +STORE, 94382607687680, 94382607691775, +STORE, 94382607691776, 94382607695871, +ERASE, 140252451614720, 140252451614720, +STORE, 140252451614720, 140252451618815, +STORE, 140252451618816, 140252451622911, +ERASE, 47380343541760, 47380343541760, +STORE, 94382626803712, 94382626938879, +STORE, 140737488347136, 140737488351231, +STORE, 140730900271104, 140737488351231, +ERASE, 140730900271104, 140730900271104, +STORE, 140730900271104, 140730900275199, +STORE, 93855478120448, 93855478337535, +ERASE, 93855478120448, 93855478120448, +STORE, 93855478120448, 93855478198271, +STORE, 93855478198272, 93855478337535, +ERASE, 93855478198272, 93855478198272, +STORE, 93855478198272, 93855478243327, +STORE, 93855478243328, 93855478288383, +STORE, 93855478288384, 93855478337535, +STORE, 140092686573568, 140092686745599, +ERASE, 140092686573568, 140092686573568, +STORE, 140092686573568, 140092686577663, +STORE, 140092686577664, 140092686745599, +ERASE, 140092686577664, 140092686577664, +STORE, 140092686577664, 140092686700543, +STORE, 140092686700544, 140092686733311, +STORE, 140092686733312, 140092686741503, +STORE, 140092686741504, 140092686745599, +STORE, 140730900537344, 140730900541439, +STORE, 140730900525056, 140730900537343, +STORE, 47540108423168, 47540108431359, +STORE, 47540108431360, 47540108439551, +STORE, 47540108439552, 47540110278655, +STORE, 47540108578816, 47540110278655, +STORE, 47540108439552, 47540108578815, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540110237695, +STORE, 47540110237696, 47540110278655, +STORE, 47540109922304, 47540110237695, +STORE, 47540108578816, 47540109922303, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540109922303, +STORE, 47540110233600, 47540110237695, +STORE, 47540109922304, 47540110233599, +ERASE, 47540109922304, 47540109922304, +STORE, 47540109922304, 47540110233599, +STORE, 47540110262272, 47540110278655, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110262272, 47540110262272, +STORE, 47540110262272, 47540110278655, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110254079, +STORE, 47540110254080, 47540110262271, +ERASE, 93855478288384, 93855478288384, +STORE, 93855478288384, 93855478333439, +STORE, 93855478333440, 93855478337535, +ERASE, 140092686733312, 140092686733312, +STORE, 140092686733312, 140092686737407, +STORE, 140092686737408, 140092686741503, +ERASE, 47540108423168, 47540108423168, +STORE, 93855492222976, 93855492358143, +STORE, 93855492222976, 93855492493311, +STORE, 140737488347136, 140737488351231, +STORE, 140733498146816, 140737488351231, +ERASE, 140733498146816, 140733498146816, +STORE, 140733498146816, 140733498150911, +STORE, 94170739654656, 94170740367359, +ERASE, 94170739654656, 94170739654656, +STORE, 94170739654656, 94170739703807, +STORE, 94170739703808, 94170740367359, +ERASE, 94170739703808, 94170739703808, +STORE, 94170739703808, 94170740248575, +STORE, 94170740248576, 94170740346879, +STORE, 94170740346880, 94170740367359, +STORE, 140024788877312, 140024789049343, +ERASE, 140024788877312, 140024788877312, +STORE, 140024788877312, 140024788881407, +STORE, 140024788881408, 140024789049343, +ERASE, 140024788881408, 140024788881408, +STORE, 140024788881408, 140024789004287, +STORE, 140024789004288, 140024789037055, +STORE, 140024789037056, 140024789045247, +STORE, 140024789045248, 140024789049343, +STORE, 140733499023360, 140733499027455, +STORE, 140733499011072, 140733499023359, +STORE, 47608006119424, 47608006127615, +STORE, 47608006127616, 47608006135807, +STORE, 47608006135808, 47608006242303, +STORE, 47608006152192, 47608006242303, +STORE, 47608006135808, 47608006152191, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006225919, +STORE, 47608006225920, 47608006242303, +STORE, 47608006205440, 47608006225919, +STORE, 47608006152192, 47608006205439, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006205439, +STORE, 47608006221824, 47608006225919, +STORE, 47608006205440, 47608006221823, +ERASE, 47608006205440, 47608006205440, +STORE, 47608006205440, 47608006221823, +STORE, 47608006234112, 47608006242303, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006234112, 47608006234112, +STORE, 47608006234112, 47608006242303, +STORE, 47608006242304, 47608008081407, +STORE, 47608006381568, 47608008081407, +STORE, 47608006242304, 47608006381567, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608008040447, +STORE, 47608008040448, 47608008081407, +STORE, 47608007725056, 47608008040447, +STORE, 47608006381568, 47608007725055, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608007725055, +STORE, 47608008036352, 47608008040447, +STORE, 47608007725056, 47608008036351, +ERASE, 47608007725056, 47608007725056, +STORE, 47608007725056, 47608008036351, +STORE, 47608008065024, 47608008081407, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008065024, 47608008065024, +STORE, 47608008065024, 47608008081407, +STORE, 47608008065024, 47608008093695, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008056831, +STORE, 47608008056832, 47608008065023, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006230015, +STORE, 47608006230016, 47608006234111, +ERASE, 94170740346880, 94170740346880, +STORE, 94170740346880, 94170740363263, +STORE, 94170740363264, 94170740367359, +ERASE, 140024789037056, 140024789037056, +STORE, 140024789037056, 140024789041151, +STORE, 140024789041152, 140024789045247, +ERASE, 47608006119424, 47608006119424, +STORE, 140737488347136, 140737488351231, +STORE, 140730264326144, 140737488351231, +ERASE, 140730264326144, 140730264326144, +STORE, 140730264326144, 140730264330239, +STORE, 94653216407552, 94653217120255, +ERASE, 94653216407552, 94653216407552, +STORE, 94653216407552, 94653216456703, +STORE, 94653216456704, 94653217120255, +ERASE, 94653216456704, 94653216456704, +STORE, 94653216456704, 94653217001471, +STORE, 94653217001472, 94653217099775, +STORE, 94653217099776, 94653217120255, +STORE, 140103617011712, 140103617183743, +ERASE, 140103617011712, 140103617011712, +STORE, 140103617011712, 140103617015807, +STORE, 140103617015808, 140103617183743, +ERASE, 140103617015808, 140103617015808, +STORE, 140103617015808, 140103617138687, +STORE, 140103617138688, 140103617171455, +STORE, 140103617171456, 140103617179647, +STORE, 140103617179648, 140103617183743, +STORE, 140730265427968, 140730265432063, +STORE, 140730265415680, 140730265427967, +STORE, 47529177985024, 47529177993215, +STORE, 47529177993216, 47529178001407, +STORE, 47529178001408, 47529178107903, +STORE, 47529178017792, 47529178107903, +STORE, 47529178001408, 47529178017791, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178091519, +STORE, 47529178091520, 47529178107903, +STORE, 47529178071040, 47529178091519, +STORE, 47529178017792, 47529178071039, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178071039, +STORE, 47529178087424, 47529178091519, +STORE, 47529178071040, 47529178087423, +ERASE, 47529178071040, 47529178071040, +STORE, 47529178071040, 47529178087423, +STORE, 47529178099712, 47529178107903, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178099712, 47529178099712, +STORE, 47529178099712, 47529178107903, +STORE, 47529178107904, 47529179947007, +STORE, 47529178247168, 47529179947007, +STORE, 47529178107904, 47529178247167, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179906047, +STORE, 47529179906048, 47529179947007, +STORE, 47529179590656, 47529179906047, +STORE, 47529178247168, 47529179590655, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179590655, +STORE, 47529179901952, 47529179906047, +STORE, 47529179590656, 47529179901951, +ERASE, 47529179590656, 47529179590656, +STORE, 47529179590656, 47529179901951, +STORE, 47529179930624, 47529179947007, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179930624, 47529179930624, +STORE, 47529179930624, 47529179947007, +STORE, 47529179930624, 47529179959295, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179922431, +STORE, 47529179922432, 47529179930623, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178095615, +STORE, 47529178095616, 47529178099711, +ERASE, 94653217099776, 94653217099776, +STORE, 94653217099776, 94653217116159, +STORE, 94653217116160, 94653217120255, +ERASE, 140103617171456, 140103617171456, +STORE, 140103617171456, 140103617175551, +STORE, 140103617175552, 140103617179647, +ERASE, 47529177985024, 47529177985024, +STORE, 94653241135104, 94653241270271, +STORE, 140737488347136, 140737488351231, +STORE, 140736284549120, 140737488351231, +ERASE, 140736284549120, 140736284549120, +STORE, 140736284549120, 140736284553215, +STORE, 93963663822848, 93963664506879, +ERASE, 93963663822848, 93963663822848, +STORE, 93963663822848, 93963663884287, +STORE, 93963663884288, 93963664506879, +ERASE, 93963663884288, 93963663884288, +STORE, 93963663884288, 93963664240639, +STORE, 93963664240640, 93963664379903, +STORE, 93963664379904, 93963664506879, +STORE, 140450188439552, 140450188611583, +ERASE, 140450188439552, 140450188439552, +STORE, 140450188439552, 140450188443647, +STORE, 140450188443648, 140450188611583, +ERASE, 140450188443648, 140450188443648, +STORE, 140450188443648, 140450188566527, +STORE, 140450188566528, 140450188599295, +STORE, 140450188599296, 140450188607487, +STORE, 140450188607488, 140450188611583, +STORE, 140736284577792, 140736284581887, +STORE, 140736284565504, 140736284577791, +STORE, 47182606557184, 47182606565375, +STORE, 47182606565376, 47182606573567, +STORE, 47182606573568, 47182608412671, +STORE, 47182606712832, 47182608412671, +STORE, 47182606573568, 47182606712831, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608371711, +STORE, 47182608371712, 47182608412671, +STORE, 47182608056320, 47182608371711, +STORE, 47182606712832, 47182608056319, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608056319, +STORE, 47182608367616, 47182608371711, +STORE, 47182608056320, 47182608367615, +ERASE, 47182608056320, 47182608056320, +STORE, 47182608056320, 47182608367615, +STORE, 47182608396288, 47182608412671, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608396288, 47182608396288, +STORE, 47182608396288, 47182608412671, +STORE, 47182608412672, 47182608523263, +STORE, 47182608429056, 47182608523263, +STORE, 47182608412672, 47182608429055, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608515071, +STORE, 47182608515072, 47182608523263, +STORE, 47182608490496, 47182608515071, +STORE, 47182608429056, 47182608490495, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608490495, +STORE, 47182608510976, 47182608515071, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608490496, 47182608490496, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608523263, +STORE, 47182608523264, 47182608568319, +ERASE, 47182608523264, 47182608523264, +STORE, 47182608523264, 47182608531455, +STORE, 47182608531456, 47182608568319, +STORE, 47182608551936, 47182608568319, +STORE, 47182608531456, 47182608551935, +ERASE, 47182608531456, 47182608531456, +STORE, 47182608531456, 47182608551935, +STORE, 47182608560128, 47182608568319, +STORE, 47182608551936, 47182608560127, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608568319, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608560127, +STORE, 47182608560128, 47182608568319, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608568319, +STORE, 47182608568320, 47182608916479, +STORE, 47182608609280, 47182608916479, +STORE, 47182608568320, 47182608609279, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608891903, +STORE, 47182608891904, 47182608916479, +STORE, 47182608822272, 47182608891903, +STORE, 47182608609280, 47182608822271, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608822271, +STORE, 47182608887808, 47182608891903, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608822272, 47182608822272, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608916479, +STORE, 47182608916480, 47182611177471, +STORE, 47182609068032, 47182611177471, +STORE, 47182608916480, 47182609068031, +ERASE, 47182609068032, 47182609068032, +STORE, 47182609068032, 47182611161087, +STORE, 47182611161088, 47182611177471, +STORE, 47182611169280, 47182611177471, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611169280, 47182611169280, +STORE, 47182611169280, 47182611177471, +STORE, 47182611177472, 47182611312639, +ERASE, 47182611177472, 47182611177472, +STORE, 47182611177472, 47182611202047, +STORE, 47182611202048, 47182611312639, +STORE, 47182611263488, 47182611312639, +STORE, 47182611202048, 47182611263487, +ERASE, 47182611202048, 47182611202048, +STORE, 47182611202048, 47182611263487, +STORE, 47182611288064, 47182611312639, +STORE, 47182611263488, 47182611288063, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611312639, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611288063, +STORE, 47182611288064, 47182611312639, +STORE, 47182611296256, 47182611312639, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611296256, 47182611296256, +STORE, 47182611296256, 47182611312639, +STORE, 47182611296256, 47182611320831, +STORE, 47182611320832, 47182611484671, +ERASE, 47182611320832, 47182611320832, +STORE, 47182611320832, 47182611333119, +STORE, 47182611333120, 47182611484671, +STORE, 47182611431424, 47182611484671, +STORE, 47182611333120, 47182611431423, +ERASE, 47182611333120, 47182611333120, +STORE, 47182611333120, 47182611431423, +STORE, 47182611476480, 47182611484671, +STORE, 47182611431424, 47182611476479, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611484671, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611476479, +STORE, 47182611476480, 47182611484671, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611484671, +STORE, 47182611484672, 47182612082687, +STORE, 47182611603456, 47182612082687, +STORE, 47182611484672, 47182611603455, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182612029439, +STORE, 47182612029440, 47182612082687, +STORE, 47182611918848, 47182612029439, +STORE, 47182611603456, 47182611918847, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182611918847, +STORE, 47182612025344, 47182612029439, +STORE, 47182611918848, 47182612025343, +ERASE, 47182611918848, 47182611918848, +STORE, 47182611918848, 47182612025343, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612082687, +STORE, 47182612082688, 47182615134207, +STORE, 47182612627456, 47182615134207, +STORE, 47182612082688, 47182612627455, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614913023, +STORE, 47182614913024, 47182615134207, +STORE, 47182614323200, 47182614913023, +STORE, 47182612627456, 47182614323199, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614323199, +STORE, 47182614908928, 47182614913023, +STORE, 47182614323200, 47182614908927, +ERASE, 47182614323200, 47182614323200, +STORE, 47182614323200, 47182614908927, +STORE, 47182615117824, 47182615134207, +STORE, 47182614913024, 47182615117823, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615117823, +ERASE, 47182615117824, 47182615117824, +STORE, 47182615117824, 47182615134207, +STORE, 47182615134208, 47182615166975, +ERASE, 47182615134208, 47182615134208, +STORE, 47182615134208, 47182615142399, +STORE, 47182615142400, 47182615166975, +STORE, 47182615154688, 47182615166975, +STORE, 47182615142400, 47182615154687, +ERASE, 47182615142400, 47182615142400, +STORE, 47182615142400, 47182615154687, +STORE, 47182615158784, 47182615166975, +STORE, 47182615154688, 47182615158783, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615166975, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615158783, +STORE, 47182615158784, 47182615166975, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615166975, +STORE, 47182615166976, 47182615203839, +ERASE, 47182615166976, 47182615166976, +STORE, 47182615166976, 47182615175167, +STORE, 47182615175168, 47182615203839, +STORE, 47182615191552, 47182615203839, +STORE, 47182615175168, 47182615191551, +ERASE, 47182615175168, 47182615175168, +STORE, 47182615175168, 47182615191551, +STORE, 47182615195648, 47182615203839, +STORE, 47182615191552, 47182615195647, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615203839, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615195647, +STORE, 47182615195648, 47182615203839, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615203839, +STORE, 47182615203840, 47182615678975, +ERASE, 47182615203840, 47182615203840, +STORE, 47182615203840, 47182615212031, +STORE, 47182615212032, 47182615678975, +STORE, 47182615547904, 47182615678975, +STORE, 47182615212032, 47182615547903, +ERASE, 47182615212032, 47182615212032, +STORE, 47182615212032, 47182615547903, +STORE, 47182615670784, 47182615678975, +STORE, 47182615547904, 47182615670783, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615678975, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615670783, +STORE, 47182615670784, 47182615678975, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615678975, +STORE, 47182615678976, 47182615687167, +STORE, 47182615687168, 47182615707647, +ERASE, 47182615687168, 47182615687168, +STORE, 47182615687168, 47182615691263, +STORE, 47182615691264, 47182615707647, +STORE, 47182615695360, 47182615707647, +STORE, 47182615691264, 47182615695359, +ERASE, 47182615691264, 47182615691264, +STORE, 47182615691264, 47182615695359, +STORE, 47182615699456, 47182615707647, +STORE, 47182615695360, 47182615699455, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615707647, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615699455, +STORE, 47182615699456, 47182615707647, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615707647, +STORE, 47182615707648, 47182615715839, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608388095, +STORE, 47182608388096, 47182608396287, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615703551, +STORE, 47182615703552, 47182615707647, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611292159, +STORE, 47182611292160, 47182611296255, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615674879, +STORE, 47182615674880, 47182615678975, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615199743, +STORE, 47182615199744, 47182615203839, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615162879, +STORE, 47182615162880, 47182615166975, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615109631, +STORE, 47182615109632, 47182615117823, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612066303, +STORE, 47182612066304, 47182612082687, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611480575, +STORE, 47182611480576, 47182611484671, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611165183, +STORE, 47182611165184, 47182611169279, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608912383, +STORE, 47182608912384, 47182608916479, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608564223, +STORE, 47182608564224, 47182608568319, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608519167, +STORE, 47182608519168, 47182608523263, +ERASE, 93963664379904, 93963664379904, +STORE, 93963664379904, 93963664502783, +STORE, 93963664502784, 93963664506879, +ERASE, 140450188599296, 140450188599296, +STORE, 140450188599296, 140450188603391, +STORE, 140450188603392, 140450188607487, +ERASE, 47182606557184, 47182606557184, +STORE, 93963694723072, 93963694858239, +STORE, 140737488347136, 140737488351231, +STORE, 140730313261056, 140737488351231, +ERASE, 140730313261056, 140730313261056, +STORE, 140730313261056, 140730313265151, +STORE, 94386579017728, 94386579697663, +ERASE, 94386579017728, 94386579017728, +STORE, 94386579017728, 94386579083263, +STORE, 94386579083264, 94386579697663, +ERASE, 94386579083264, 94386579083264, +STORE, 94386579083264, 94386579431423, +STORE, 94386579431424, 94386579570687, +STORE, 94386579570688, 94386579697663, +STORE, 140124810838016, 140124811010047, +ERASE, 140124810838016, 140124810838016, +STORE, 140124810838016, 140124810842111, +STORE, 140124810842112, 140124811010047, +ERASE, 140124810842112, 140124810842112, +STORE, 140124810842112, 140124810964991, +STORE, 140124810964992, 140124810997759, +STORE, 140124810997760, 140124811005951, +STORE, 140124811005952, 140124811010047, +STORE, 140730313601024, 140730313605119, +STORE, 140730313588736, 140730313601023, +STORE, 47507984158720, 47507984166911, +STORE, 47507984166912, 47507984175103, +STORE, 47507984175104, 47507986014207, +STORE, 47507984314368, 47507986014207, +STORE, 47507984175104, 47507984314367, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985973247, +STORE, 47507985973248, 47507986014207, +STORE, 47507985657856, 47507985973247, +STORE, 47507984314368, 47507985657855, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985657855, +STORE, 47507985969152, 47507985973247, +STORE, 47507985657856, 47507985969151, +ERASE, 47507985657856, 47507985657856, +STORE, 47507985657856, 47507985969151, +STORE, 47507985997824, 47507986014207, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985997824, 47507985997824, +STORE, 47507985997824, 47507986014207, +STORE, 47507986014208, 47507986124799, +STORE, 47507986030592, 47507986124799, +STORE, 47507986014208, 47507986030591, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986116607, +STORE, 47507986116608, 47507986124799, +STORE, 47507986092032, 47507986116607, +STORE, 47507986030592, 47507986092031, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986092031, +STORE, 47507986112512, 47507986116607, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986092032, 47507986092032, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986124799, +STORE, 47507986124800, 47507986169855, +ERASE, 47507986124800, 47507986124800, +STORE, 47507986124800, 47507986132991, +STORE, 47507986132992, 47507986169855, +STORE, 47507986153472, 47507986169855, +STORE, 47507986132992, 47507986153471, +ERASE, 47507986132992, 47507986132992, +STORE, 47507986132992, 47507986153471, +STORE, 47507986161664, 47507986169855, +STORE, 47507986153472, 47507986161663, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986169855, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986161663, +STORE, 47507986161664, 47507986169855, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986169855, +STORE, 47507986169856, 47507986518015, +STORE, 47507986210816, 47507986518015, +STORE, 47507986169856, 47507986210815, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986493439, +STORE, 47507986493440, 47507986518015, +STORE, 47507986423808, 47507986493439, +STORE, 47507986210816, 47507986423807, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986423807, +STORE, 47507986489344, 47507986493439, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986423808, 47507986423808, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986518015, +STORE, 47507986518016, 47507988779007, +STORE, 47507986669568, 47507988779007, +STORE, 47507986518016, 47507986669567, +ERASE, 47507986669568, 47507986669568, +STORE, 47507986669568, 47507988762623, +STORE, 47507988762624, 47507988779007, +STORE, 47507988770816, 47507988779007, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988770816, 47507988770816, +STORE, 47507988770816, 47507988779007, +STORE, 47507988779008, 47507988914175, +ERASE, 47507988779008, 47507988779008, +STORE, 47507988779008, 47507988803583, +STORE, 47507988803584, 47507988914175, +STORE, 47507988865024, 47507988914175, +STORE, 47507988803584, 47507988865023, +ERASE, 47507988803584, 47507988803584, +STORE, 47507988803584, 47507988865023, +STORE, 47507988889600, 47507988914175, +STORE, 47507988865024, 47507988889599, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988914175, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988889599, +STORE, 47507988889600, 47507988914175, +STORE, 47507988897792, 47507988914175, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988897792, 47507988897792, +STORE, 47507988897792, 47507988914175, +STORE, 47507988897792, 47507988922367, +STORE, 47507988922368, 47507989086207, +ERASE, 47507988922368, 47507988922368, +STORE, 47507988922368, 47507988934655, +STORE, 47507988934656, 47507989086207, +STORE, 47507989032960, 47507989086207, +STORE, 47507988934656, 47507989032959, +ERASE, 47507988934656, 47507988934656, +STORE, 47507988934656, 47507989032959, +STORE, 47507989078016, 47507989086207, +STORE, 47507989032960, 47507989078015, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989086207, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989078015, +STORE, 47507989078016, 47507989086207, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989086207, +STORE, 47507989086208, 47507989684223, +STORE, 47507989204992, 47507989684223, +STORE, 47507989086208, 47507989204991, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989630975, +STORE, 47507989630976, 47507989684223, +STORE, 47507989520384, 47507989630975, +STORE, 47507989204992, 47507989520383, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989520383, +STORE, 47507989626880, 47507989630975, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989520384, 47507989520384, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989684223, +STORE, 47507989684224, 47507992735743, +STORE, 47507990228992, 47507992735743, +STORE, 47507989684224, 47507990228991, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507992514559, +STORE, 47507992514560, 47507992735743, +STORE, 47507991924736, 47507992514559, +STORE, 47507990228992, 47507991924735, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507991924735, +STORE, 47507992510464, 47507992514559, +STORE, 47507991924736, 47507992510463, +ERASE, 47507991924736, 47507991924736, +STORE, 47507991924736, 47507992510463, +STORE, 47507992719360, 47507992735743, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992719360, 47507992719360, +STORE, 47507992719360, 47507992735743, +STORE, 47507992735744, 47507992768511, +ERASE, 47507992735744, 47507992735744, +STORE, 47507992735744, 47507992743935, +STORE, 47507992743936, 47507992768511, +STORE, 47507992756224, 47507992768511, +STORE, 47507992743936, 47507992756223, +ERASE, 47507992743936, 47507992743936, +STORE, 47507992743936, 47507992756223, +STORE, 47507992760320, 47507992768511, +STORE, 47507992756224, 47507992760319, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992768511, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992760319, +STORE, 47507992760320, 47507992768511, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992768511, +STORE, 47507992768512, 47507992805375, +ERASE, 47507992768512, 47507992768512, +STORE, 47507992768512, 47507992776703, +STORE, 47507992776704, 47507992805375, +STORE, 47507992793088, 47507992805375, +STORE, 47507992776704, 47507992793087, +ERASE, 47507992776704, 47507992776704, +STORE, 47507992776704, 47507992793087, +STORE, 47507992797184, 47507992805375, +STORE, 47507992793088, 47507992797183, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992805375, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992797183, +STORE, 47507992797184, 47507992805375, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992805375, +STORE, 47507992805376, 47507993280511, +ERASE, 47507992805376, 47507992805376, +STORE, 47507992805376, 47507992813567, +STORE, 47507992813568, 47507993280511, +STORE, 47507993149440, 47507993280511, +STORE, 47507992813568, 47507993149439, +ERASE, 47507992813568, 47507992813568, +STORE, 47507992813568, 47507993149439, +STORE, 47507993272320, 47507993280511, +STORE, 47507993149440, 47507993272319, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993280511, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993272319, +STORE, 47507993272320, 47507993280511, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993280511, +STORE, 47507993280512, 47507993288703, +STORE, 47507993288704, 47507993309183, +ERASE, 47507993288704, 47507993288704, +STORE, 47507993288704, 47507993292799, +STORE, 47507993292800, 47507993309183, +STORE, 47507993296896, 47507993309183, +STORE, 47507993292800, 47507993296895, +ERASE, 47507993292800, 47507993292800, +STORE, 47507993292800, 47507993296895, +STORE, 47507993300992, 47507993309183, +STORE, 47507993296896, 47507993300991, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993309183, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993300991, +STORE, 47507993300992, 47507993309183, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993309183, +STORE, 47507993309184, 47507993317375, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985989631, +STORE, 47507985989632, 47507985997823, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993305087, +STORE, 47507993305088, 47507993309183, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988893695, +STORE, 47507988893696, 47507988897791, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993276415, +STORE, 47507993276416, 47507993280511, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992801279, +STORE, 47507992801280, 47507992805375, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992764415, +STORE, 47507992764416, 47507992768511, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992711167, +STORE, 47507992711168, 47507992719359, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989667839, +STORE, 47507989667840, 47507989684223, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989082111, +STORE, 47507989082112, 47507989086207, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988766719, +STORE, 47507988766720, 47507988770815, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986513919, +STORE, 47507986513920, 47507986518015, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986165759, +STORE, 47507986165760, 47507986169855, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986120703, +STORE, 47507986120704, 47507986124799, +ERASE, 94386579570688, 94386579570688, +STORE, 94386579570688, 94386579693567, +STORE, 94386579693568, 94386579697663, +ERASE, 140124810997760, 140124810997760, +STORE, 140124810997760, 140124811001855, +STORE, 140124811001856, 140124811005951, +ERASE, 47507984158720, 47507984158720, +STORE, 94386583982080, 94386584117247, +STORE, 94386583982080, 94386584256511, +ERASE, 94386583982080, 94386583982080, +STORE, 94386583982080, 94386584223743, +STORE, 94386584223744, 94386584256511, +ERASE, 94386584223744, 94386584223744, +STORE, 140737488347136, 140737488351231, +STORE, 140733763395584, 140737488351231, +ERASE, 140733763395584, 140733763395584, +STORE, 140733763395584, 140733763399679, +STORE, 94011546472448, 94011547152383, +ERASE, 94011546472448, 94011546472448, +STORE, 94011546472448, 94011546537983, +STORE, 94011546537984, 94011547152383, +ERASE, 94011546537984, 94011546537984, +STORE, 94011546537984, 94011546886143, +STORE, 94011546886144, 94011547025407, +STORE, 94011547025408, 94011547152383, +STORE, 139757597949952, 139757598121983, +ERASE, 139757597949952, 139757597949952, +STORE, 139757597949952, 139757597954047, +STORE, 139757597954048, 139757598121983, +ERASE, 139757597954048, 139757597954048, +STORE, 139757597954048, 139757598076927, +STORE, 139757598076928, 139757598109695, +STORE, 139757598109696, 139757598117887, +STORE, 139757598117888, 139757598121983, +STORE, 140733763596288, 140733763600383, +STORE, 140733763584000, 140733763596287, +STORE, 47875197046784, 47875197054975, +STORE, 47875197054976, 47875197063167, +STORE, 47875197063168, 47875198902271, +STORE, 47875197202432, 47875198902271, +STORE, 47875197063168, 47875197202431, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198861311, +STORE, 47875198861312, 47875198902271, +STORE, 47875198545920, 47875198861311, +STORE, 47875197202432, 47875198545919, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198545919, +STORE, 47875198857216, 47875198861311, +STORE, 47875198545920, 47875198857215, +ERASE, 47875198545920, 47875198545920, +STORE, 47875198545920, 47875198857215, +STORE, 47875198885888, 47875198902271, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198885888, 47875198885888, +STORE, 47875198885888, 47875198902271, +STORE, 47875198902272, 47875199012863, +STORE, 47875198918656, 47875199012863, +STORE, 47875198902272, 47875198918655, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875199004671, +STORE, 47875199004672, 47875199012863, +STORE, 47875198980096, 47875199004671, +STORE, 47875198918656, 47875198980095, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875198980095, +STORE, 47875199000576, 47875199004671, +STORE, 47875198980096, 47875199000575, +ERASE, 47875198980096, 47875198980096, +STORE, 47875198980096, 47875199000575, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199012863, +STORE, 47875199012864, 47875199057919, +ERASE, 47875199012864, 47875199012864, +STORE, 47875199012864, 47875199021055, +STORE, 47875199021056, 47875199057919, +STORE, 47875199041536, 47875199057919, +STORE, 47875199021056, 47875199041535, +ERASE, 47875199021056, 47875199021056, +STORE, 47875199021056, 47875199041535, +STORE, 47875199049728, 47875199057919, +STORE, 47875199041536, 47875199049727, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199057919, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199049727, +STORE, 47875199049728, 47875199057919, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199057919, +STORE, 47875199057920, 47875199406079, +STORE, 47875199098880, 47875199406079, +STORE, 47875199057920, 47875199098879, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199381503, +STORE, 47875199381504, 47875199406079, +STORE, 47875199311872, 47875199381503, +STORE, 47875199098880, 47875199311871, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199311871, +STORE, 47875199377408, 47875199381503, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199311872, 47875199311872, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199406079, +STORE, 47875199406080, 47875201667071, +STORE, 47875199557632, 47875201667071, +STORE, 47875199406080, 47875199557631, +ERASE, 47875199557632, 47875199557632, +STORE, 47875199557632, 47875201650687, +STORE, 47875201650688, 47875201667071, +STORE, 47875201658880, 47875201667071, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201658880, 47875201658880, +STORE, 47875201658880, 47875201667071, +STORE, 47875201667072, 47875201802239, +ERASE, 47875201667072, 47875201667072, +STORE, 47875201667072, 47875201691647, +STORE, 47875201691648, 47875201802239, +STORE, 47875201753088, 47875201802239, +STORE, 47875201691648, 47875201753087, +ERASE, 47875201691648, 47875201691648, +STORE, 47875201691648, 47875201753087, +STORE, 47875201777664, 47875201802239, +STORE, 47875201753088, 47875201777663, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201802239, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201777663, +STORE, 47875201777664, 47875201802239, +STORE, 47875201785856, 47875201802239, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201785856, 47875201785856, +STORE, 47875201785856, 47875201802239, +STORE, 47875201785856, 47875201810431, +STORE, 47875201810432, 47875201974271, +ERASE, 47875201810432, 47875201810432, +STORE, 47875201810432, 47875201822719, +STORE, 47875201822720, 47875201974271, +STORE, 47875201921024, 47875201974271, +STORE, 47875201822720, 47875201921023, +ERASE, 47875201822720, 47875201822720, +STORE, 47875201822720, 47875201921023, +STORE, 47875201966080, 47875201974271, +STORE, 47875201921024, 47875201966079, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201974271, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201966079, +STORE, 47875201966080, 47875201974271, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201974271, +STORE, 47875201974272, 47875202572287, +STORE, 47875202093056, 47875202572287, +STORE, 47875201974272, 47875202093055, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202519039, +STORE, 47875202519040, 47875202572287, +STORE, 47875202408448, 47875202519039, +STORE, 47875202093056, 47875202408447, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202408447, +STORE, 47875202514944, 47875202519039, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202408448, 47875202408448, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202572287, +STORE, 47875202572288, 47875205623807, +STORE, 47875203117056, 47875205623807, +STORE, 47875202572288, 47875203117055, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875205402623, +STORE, 47875205402624, 47875205623807, +STORE, 47875204812800, 47875205402623, +STORE, 47875203117056, 47875204812799, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875204812799, +STORE, 47875205398528, 47875205402623, +STORE, 47875204812800, 47875205398527, +ERASE, 47875204812800, 47875204812800, +STORE, 47875204812800, 47875205398527, +STORE, 47875205607424, 47875205623807, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205607424, 47875205607424, +STORE, 47875205607424, 47875205623807, +STORE, 47875205623808, 47875205656575, +ERASE, 47875205623808, 47875205623808, +STORE, 47875205623808, 47875205631999, +STORE, 47875205632000, 47875205656575, +STORE, 47875205644288, 47875205656575, +STORE, 47875205632000, 47875205644287, +ERASE, 47875205632000, 47875205632000, +STORE, 47875205632000, 47875205644287, +STORE, 47875205648384, 47875205656575, +STORE, 47875205644288, 47875205648383, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205656575, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205648383, +STORE, 47875205648384, 47875205656575, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205656575, +STORE, 47875205656576, 47875205693439, +ERASE, 47875205656576, 47875205656576, +STORE, 47875205656576, 47875205664767, +STORE, 47875205664768, 47875205693439, +STORE, 47875205681152, 47875205693439, +STORE, 47875205664768, 47875205681151, +ERASE, 47875205664768, 47875205664768, +STORE, 47875205664768, 47875205681151, +STORE, 47875205685248, 47875205693439, +STORE, 47875205681152, 47875205685247, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205693439, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205685247, +STORE, 47875205685248, 47875205693439, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205693439, +STORE, 47875205693440, 47875206168575, +ERASE, 47875205693440, 47875205693440, +STORE, 47875205693440, 47875205701631, +STORE, 47875205701632, 47875206168575, +STORE, 47875206037504, 47875206168575, +STORE, 47875205701632, 47875206037503, +ERASE, 47875205701632, 47875205701632, +STORE, 47875205701632, 47875206037503, +STORE, 47875206160384, 47875206168575, +STORE, 47875206037504, 47875206160383, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206168575, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206160383, +STORE, 47875206160384, 47875206168575, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206168575, +STORE, 47875206168576, 47875206176767, +STORE, 47875206176768, 47875206197247, +ERASE, 47875206176768, 47875206176768, +STORE, 47875206176768, 47875206180863, +STORE, 47875206180864, 47875206197247, +STORE, 47875206184960, 47875206197247, +STORE, 47875206180864, 47875206184959, +ERASE, 47875206180864, 47875206180864, +STORE, 47875206180864, 47875206184959, +STORE, 47875206189056, 47875206197247, +STORE, 47875206184960, 47875206189055, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206197247, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206189055, +STORE, 47875206189056, 47875206197247, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206197247, +STORE, 47875206197248, 47875206205439, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198877695, +STORE, 47875198877696, 47875198885887, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206193151, +STORE, 47875206193152, 47875206197247, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201781759, +STORE, 47875201781760, 47875201785855, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206164479, +STORE, 47875206164480, 47875206168575, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205689343, +STORE, 47875205689344, 47875205693439, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205652479, +STORE, 47875205652480, 47875205656575, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205599231, +STORE, 47875205599232, 47875205607423, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202555903, +STORE, 47875202555904, 47875202572287, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201970175, +STORE, 47875201970176, 47875201974271, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201654783, +STORE, 47875201654784, 47875201658879, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199401983, +STORE, 47875199401984, 47875199406079, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199053823, +STORE, 47875199053824, 47875199057919, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199008767, +STORE, 47875199008768, 47875199012863, +ERASE, 94011547025408, 94011547025408, +STORE, 94011547025408, 94011547148287, +STORE, 94011547148288, 94011547152383, +ERASE, 139757598109696, 139757598109696, +STORE, 139757598109696, 139757598113791, +STORE, 139757598113792, 139757598117887, +ERASE, 47875197046784, 47875197046784, +STORE, 94011557584896, 94011557720063, +STORE, 94011557584896, 94011557855231, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557851135, +STORE, 94011557851136, 94011557855231, +ERASE, 94011557851136, 94011557851136, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557847039, +STORE, 94011557847040, 94011557851135, +ERASE, 94011557847040, 94011557847040, +STORE, 94011557584896, 94011557982207, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557978111, +STORE, 94011557978112, 94011557982207, +ERASE, 94011557978112, 94011557978112, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557974015, +STORE, 94011557974016, 94011557978111, +ERASE, 94011557974016, 94011557974016, +STORE, 140737488347136, 140737488351231, +STORE, 140734130360320, 140737488351231, +ERASE, 140734130360320, 140734130360320, +STORE, 140734130360320, 140734130364415, +STORE, 94641232105472, 94641232785407, +ERASE, 94641232105472, 94641232105472, +STORE, 94641232105472, 94641232171007, +STORE, 94641232171008, 94641232785407, +ERASE, 94641232171008, 94641232171008, +STORE, 94641232171008, 94641232519167, +STORE, 94641232519168, 94641232658431, +STORE, 94641232658432, 94641232785407, +STORE, 139726599516160, 139726599688191, +ERASE, 139726599516160, 139726599516160, +STORE, 139726599516160, 139726599520255, +STORE, 139726599520256, 139726599688191, +ERASE, 139726599520256, 139726599520256, +STORE, 139726599520256, 139726599643135, +STORE, 139726599643136, 139726599675903, +STORE, 139726599675904, 139726599684095, +STORE, 139726599684096, 139726599688191, +STORE, 140734130446336, 140734130450431, +STORE, 140734130434048, 140734130446335, +STORE, 47906195480576, 47906195488767, +STORE, 47906195488768, 47906195496959, +STORE, 47906195496960, 47906197336063, +STORE, 47906195636224, 47906197336063, +STORE, 47906195496960, 47906195636223, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906197295103, +STORE, 47906197295104, 47906197336063, +STORE, 47906196979712, 47906197295103, +STORE, 47906195636224, 47906196979711, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906196979711, +STORE, 47906197291008, 47906197295103, +STORE, 47906196979712, 47906197291007, +ERASE, 47906196979712, 47906196979712, +STORE, 47906196979712, 47906197291007, +STORE, 47906197319680, 47906197336063, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197319680, 47906197319680, +STORE, 47906197319680, 47906197336063, +STORE, 47906197336064, 47906197446655, +STORE, 47906197352448, 47906197446655, +STORE, 47906197336064, 47906197352447, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197438463, +STORE, 47906197438464, 47906197446655, +STORE, 47906197413888, 47906197438463, +STORE, 47906197352448, 47906197413887, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197413887, +STORE, 47906197434368, 47906197438463, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197413888, 47906197413888, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197446655, +STORE, 47906197446656, 47906197491711, +ERASE, 47906197446656, 47906197446656, +STORE, 47906197446656, 47906197454847, +STORE, 47906197454848, 47906197491711, +STORE, 47906197475328, 47906197491711, +STORE, 47906197454848, 47906197475327, +ERASE, 47906197454848, 47906197454848, +STORE, 47906197454848, 47906197475327, +STORE, 47906197483520, 47906197491711, +STORE, 47906197475328, 47906197483519, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197491711, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197483519, +STORE, 47906197483520, 47906197491711, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197491711, +STORE, 47906197491712, 47906197839871, +STORE, 47906197532672, 47906197839871, +STORE, 47906197491712, 47906197532671, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197815295, +STORE, 47906197815296, 47906197839871, +STORE, 47906197745664, 47906197815295, +STORE, 47906197532672, 47906197745663, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197745663, +STORE, 47906197811200, 47906197815295, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197745664, 47906197745664, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197839871, +STORE, 47906197839872, 47906200100863, +STORE, 47906197991424, 47906200100863, +STORE, 47906197839872, 47906197991423, +ERASE, 47906197991424, 47906197991424, +STORE, 47906197991424, 47906200084479, +STORE, 47906200084480, 47906200100863, +STORE, 47906200092672, 47906200100863, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200092672, 47906200092672, +STORE, 47906200092672, 47906200100863, +STORE, 47906200100864, 47906200236031, +ERASE, 47906200100864, 47906200100864, +STORE, 47906200100864, 47906200125439, +STORE, 47906200125440, 47906200236031, +STORE, 47906200186880, 47906200236031, +STORE, 47906200125440, 47906200186879, +ERASE, 47906200125440, 47906200125440, +STORE, 47906200125440, 47906200186879, +STORE, 47906200211456, 47906200236031, +STORE, 47906200186880, 47906200211455, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200236031, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200211455, +STORE, 47906200211456, 47906200236031, +STORE, 47906200219648, 47906200236031, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200219648, 47906200219648, +STORE, 47906200219648, 47906200236031, +STORE, 47906200219648, 47906200244223, +STORE, 47906200244224, 47906200408063, +ERASE, 47906200244224, 47906200244224, +STORE, 47906200244224, 47906200256511, +STORE, 47906200256512, 47906200408063, +STORE, 47906200354816, 47906200408063, +STORE, 47906200256512, 47906200354815, +ERASE, 47906200256512, 47906200256512, +STORE, 47906200256512, 47906200354815, +STORE, 47906200399872, 47906200408063, +STORE, 47906200354816, 47906200399871, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200408063, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200399871, +STORE, 47906200399872, 47906200408063, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200408063, +STORE, 47906200408064, 47906201006079, +STORE, 47906200526848, 47906201006079, +STORE, 47906200408064, 47906200526847, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200952831, +STORE, 47906200952832, 47906201006079, +STORE, 47906200842240, 47906200952831, +STORE, 47906200526848, 47906200842239, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200842239, +STORE, 47906200948736, 47906200952831, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200842240, 47906200842240, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906201006079, +STORE, 47906201006080, 47906204057599, +STORE, 47906201550848, 47906204057599, +STORE, 47906201006080, 47906201550847, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203836415, +STORE, 47906203836416, 47906204057599, +STORE, 47906203246592, 47906203836415, +STORE, 47906201550848, 47906203246591, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203246591, +STORE, 47906203832320, 47906203836415, +STORE, 47906203246592, 47906203832319, +ERASE, 47906203246592, 47906203246592, +STORE, 47906203246592, 47906203832319, +STORE, 47906204041216, 47906204057599, +STORE, 47906203836416, 47906204041215, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204041215, +ERASE, 47906204041216, 47906204041216, +STORE, 47906204041216, 47906204057599, +STORE, 47906204057600, 47906204090367, +ERASE, 47906204057600, 47906204057600, +STORE, 47906204057600, 47906204065791, +STORE, 47906204065792, 47906204090367, +STORE, 47906204078080, 47906204090367, +STORE, 47906204065792, 47906204078079, +ERASE, 47906204065792, 47906204065792, +STORE, 47906204065792, 47906204078079, +STORE, 47906204082176, 47906204090367, +STORE, 47906204078080, 47906204082175, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204090367, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204082175, +STORE, 47906204082176, 47906204090367, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204090367, +STORE, 47906204090368, 47906204127231, +ERASE, 47906204090368, 47906204090368, +STORE, 47906204090368, 47906204098559, +STORE, 47906204098560, 47906204127231, +STORE, 47906204114944, 47906204127231, +STORE, 47906204098560, 47906204114943, +ERASE, 47906204098560, 47906204098560, +STORE, 47906204098560, 47906204114943, +STORE, 47906204119040, 47906204127231, +STORE, 47906204114944, 47906204119039, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204127231, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204119039, +STORE, 47906204119040, 47906204127231, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204127231, +STORE, 47906204127232, 47906204602367, +ERASE, 47906204127232, 47906204127232, +STORE, 47906204127232, 47906204135423, +STORE, 47906204135424, 47906204602367, +STORE, 47906204471296, 47906204602367, +STORE, 47906204135424, 47906204471295, +ERASE, 47906204135424, 47906204135424, +STORE, 47906204135424, 47906204471295, +STORE, 47906204594176, 47906204602367, +STORE, 47906204471296, 47906204594175, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204602367, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204594175, +STORE, 47906204594176, 47906204602367, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204602367, +STORE, 47906204602368, 47906204610559, +STORE, 47906204610560, 47906204631039, +ERASE, 47906204610560, 47906204610560, +STORE, 47906204610560, 47906204614655, +STORE, 47906204614656, 47906204631039, +STORE, 47906204618752, 47906204631039, +STORE, 47906204614656, 47906204618751, +ERASE, 47906204614656, 47906204614656, +STORE, 47906204614656, 47906204618751, +STORE, 47906204622848, 47906204631039, +STORE, 47906204618752, 47906204622847, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204631039, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204622847, +STORE, 47906204622848, 47906204631039, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204631039, +STORE, 47906204631040, 47906204639231, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197311487, +STORE, 47906197311488, 47906197319679, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204626943, +STORE, 47906204626944, 47906204631039, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200215551, +STORE, 47906200215552, 47906200219647, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204598271, +STORE, 47906204598272, 47906204602367, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204123135, +STORE, 47906204123136, 47906204127231, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204086271, +STORE, 47906204086272, 47906204090367, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204033023, +STORE, 47906204033024, 47906204041215, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906200989695, +STORE, 47906200989696, 47906201006079, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200403967, +STORE, 47906200403968, 47906200408063, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200088575, +STORE, 47906200088576, 47906200092671, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197835775, +STORE, 47906197835776, 47906197839871, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197487615, +STORE, 47906197487616, 47906197491711, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197442559, +STORE, 47906197442560, 47906197446655, +ERASE, 94641232658432, 94641232658432, +STORE, 94641232658432, 94641232781311, +STORE, 94641232781312, 94641232785407, +ERASE, 139726599675904, 139726599675904, +STORE, 139726599675904, 139726599679999, +STORE, 139726599680000, 139726599684095, +ERASE, 47906195480576, 47906195480576, +STORE, 94641242615808, 94641242750975, + }; + + unsigned long set10[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736427839488, 140737488351231, +ERASE, 140736427839488, 140736427839488, +STORE, 140736427839488, 140736427843583, +STORE, 94071213395968, 94071213567999, +ERASE, 94071213395968, 94071213395968, +STORE, 94071213395968, 94071213412351, +STORE, 94071213412352, 94071213567999, +ERASE, 94071213412352, 94071213412352, +STORE, 94071213412352, 94071213514751, +STORE, 94071213514752, 94071213555711, +STORE, 94071213555712, 94071213567999, +STORE, 139968410644480, 139968410816511, +ERASE, 139968410644480, 139968410644480, +STORE, 139968410644480, 139968410648575, +STORE, 139968410648576, 139968410816511, +ERASE, 139968410648576, 139968410648576, +STORE, 139968410648576, 139968410771455, +STORE, 139968410771456, 139968410804223, +STORE, 139968410804224, 139968410812415, +STORE, 139968410812416, 139968410816511, +STORE, 140736429277184, 140736429281279, +STORE, 140736429264896, 140736429277183, +STORE, 47664384352256, 47664384360447, +STORE, 47664384360448, 47664384368639, +STORE, 47664384368640, 47664384532479, +ERASE, 47664384368640, 47664384368640, +STORE, 47664384368640, 47664384380927, +STORE, 47664384380928, 47664384532479, +STORE, 47664384479232, 47664384532479, +STORE, 47664384380928, 47664384479231, +ERASE, 47664384380928, 47664384380928, +STORE, 47664384380928, 47664384479231, +STORE, 47664384524288, 47664384532479, +STORE, 47664384479232, 47664384524287, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384532479, +ERASE, 47664384479232, 47664384479232, +STORE, 47664384479232, 47664384524287, +STORE, 47664384524288, 47664384532479, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384532479, +STORE, 47664384532480, 47664387583999, +STORE, 47664385077248, 47664387583999, +STORE, 47664384532480, 47664385077247, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664387362815, +STORE, 47664387362816, 47664387583999, +STORE, 47664386772992, 47664387362815, +STORE, 47664385077248, 47664386772991, +ERASE, 47664385077248, 47664385077248, +STORE, 47664385077248, 47664386772991, +STORE, 47664387358720, 47664387362815, +STORE, 47664386772992, 47664387358719, +ERASE, 47664386772992, 47664386772992, +STORE, 47664386772992, 47664387358719, +STORE, 47664387567616, 47664387583999, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387567615, +ERASE, 47664387567616, 47664387567616, +STORE, 47664387567616, 47664387583999, +STORE, 47664387584000, 47664389423103, +STORE, 47664387723264, 47664389423103, +STORE, 47664387584000, 47664387723263, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389382143, +STORE, 47664389382144, 47664389423103, +STORE, 47664389066752, 47664389382143, +STORE, 47664387723264, 47664389066751, +ERASE, 47664387723264, 47664387723264, +STORE, 47664387723264, 47664389066751, +STORE, 47664389378048, 47664389382143, +STORE, 47664389066752, 47664389378047, +ERASE, 47664389066752, 47664389066752, +STORE, 47664389066752, 47664389378047, +STORE, 47664389406720, 47664389423103, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389406719, +ERASE, 47664389406720, 47664389406720, +STORE, 47664389406720, 47664389423103, +STORE, 47664389423104, 47664389558271, +ERASE, 47664389423104, 47664389423104, +STORE, 47664389423104, 47664389447679, +STORE, 47664389447680, 47664389558271, +STORE, 47664389509120, 47664389558271, +STORE, 47664389447680, 47664389509119, +ERASE, 47664389447680, 47664389447680, +STORE, 47664389447680, 47664389509119, +STORE, 47664389533696, 47664389558271, +STORE, 47664389509120, 47664389533695, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389558271, +ERASE, 47664389509120, 47664389509120, +STORE, 47664389509120, 47664389533695, +STORE, 47664389533696, 47664389558271, +STORE, 47664389541888, 47664389558271, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389541887, +ERASE, 47664389541888, 47664389541888, +STORE, 47664389541888, 47664389558271, +STORE, 47664389558272, 47664389578751, +ERASE, 47664389558272, 47664389558272, +STORE, 47664389558272, 47664389562367, +STORE, 47664389562368, 47664389578751, +STORE, 47664389566464, 47664389578751, +STORE, 47664389562368, 47664389566463, +ERASE, 47664389562368, 47664389562368, +STORE, 47664389562368, 47664389566463, +STORE, 47664389570560, 47664389578751, +STORE, 47664389566464, 47664389570559, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389578751, +ERASE, 47664389566464, 47664389566464, +STORE, 47664389566464, 47664389570559, +STORE, 47664389570560, 47664389578751, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389578751, +STORE, 47664389578752, 47664389586943, +ERASE, 47664389382144, 47664389382144, +STORE, 47664389382144, 47664389398527, +STORE, 47664389398528, 47664389406719, +ERASE, 47664389570560, 47664389570560, +STORE, 47664389570560, 47664389574655, +STORE, 47664389574656, 47664389578751, +ERASE, 47664389533696, 47664389533696, +STORE, 47664389533696, 47664389537791, +STORE, 47664389537792, 47664389541887, +ERASE, 47664387362816, 47664387362816, +STORE, 47664387362816, 47664387559423, +STORE, 47664387559424, 47664387567615, +ERASE, 47664384524288, 47664384524288, +STORE, 47664384524288, 47664384528383, +STORE, 47664384528384, 47664384532479, +ERASE, 94071213555712, 94071213555712, +STORE, 94071213555712, 94071213563903, +STORE, 94071213563904, 94071213567999, +ERASE, 139968410804224, 139968410804224, +STORE, 139968410804224, 139968410808319, +STORE, 139968410808320, 139968410812415, +ERASE, 47664384352256, 47664384352256, +STORE, 94071244402688, 94071244537855, +STORE, 140737488347136, 140737488351231, +STORE, 140728271503360, 140737488351231, +ERASE, 140728271503360, 140728271503360, +STORE, 140728271503360, 140728271507455, +STORE, 94410361982976, 94410362155007, +ERASE, 94410361982976, 94410361982976, +STORE, 94410361982976, 94410361999359, +STORE, 94410361999360, 94410362155007, +ERASE, 94410361999360, 94410361999360, +STORE, 94410361999360, 94410362101759, +STORE, 94410362101760, 94410362142719, +STORE, 94410362142720, 94410362155007, +STORE, 140351953997824, 140351954169855, +ERASE, 140351953997824, 140351953997824, +STORE, 140351953997824, 140351954001919, +STORE, 140351954001920, 140351954169855, +ERASE, 140351954001920, 140351954001920, +STORE, 140351954001920, 140351954124799, +STORE, 140351954124800, 140351954157567, +STORE, 140351954157568, 140351954165759, +STORE, 140351954165760, 140351954169855, +STORE, 140728272429056, 140728272433151, +STORE, 140728272416768, 140728272429055, +STORE, 47280840998912, 47280841007103, +STORE, 47280841007104, 47280841015295, +STORE, 47280841015296, 47280841179135, +ERASE, 47280841015296, 47280841015296, +STORE, 47280841015296, 47280841027583, +STORE, 47280841027584, 47280841179135, +STORE, 47280841125888, 47280841179135, +STORE, 47280841027584, 47280841125887, +ERASE, 47280841027584, 47280841027584, +STORE, 47280841027584, 47280841125887, +STORE, 47280841170944, 47280841179135, +STORE, 47280841125888, 47280841170943, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841179135, +ERASE, 47280841125888, 47280841125888, +STORE, 47280841125888, 47280841170943, +STORE, 47280841170944, 47280841179135, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841179135, +STORE, 47280841179136, 47280844230655, +STORE, 47280841723904, 47280844230655, +STORE, 47280841179136, 47280841723903, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280844009471, +STORE, 47280844009472, 47280844230655, +STORE, 47280843419648, 47280844009471, +STORE, 47280841723904, 47280843419647, +ERASE, 47280841723904, 47280841723904, +STORE, 47280841723904, 47280843419647, +STORE, 47280844005376, 47280844009471, +STORE, 47280843419648, 47280844005375, +ERASE, 47280843419648, 47280843419648, +STORE, 47280843419648, 47280844005375, +STORE, 47280844214272, 47280844230655, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844214271, +ERASE, 47280844214272, 47280844214272, +STORE, 47280844214272, 47280844230655, +STORE, 47280844230656, 47280846069759, +STORE, 47280844369920, 47280846069759, +STORE, 47280844230656, 47280844369919, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280846028799, +STORE, 47280846028800, 47280846069759, +STORE, 47280845713408, 47280846028799, +STORE, 47280844369920, 47280845713407, +ERASE, 47280844369920, 47280844369920, +STORE, 47280844369920, 47280845713407, +STORE, 47280846024704, 47280846028799, +STORE, 47280845713408, 47280846024703, +ERASE, 47280845713408, 47280845713408, +STORE, 47280845713408, 47280846024703, +STORE, 47280846053376, 47280846069759, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846053375, +ERASE, 47280846053376, 47280846053376, +STORE, 47280846053376, 47280846069759, +STORE, 47280846069760, 47280846204927, +ERASE, 47280846069760, 47280846069760, +STORE, 47280846069760, 47280846094335, +STORE, 47280846094336, 47280846204927, +STORE, 47280846155776, 47280846204927, +STORE, 47280846094336, 47280846155775, +ERASE, 47280846094336, 47280846094336, +STORE, 47280846094336, 47280846155775, +STORE, 47280846180352, 47280846204927, +STORE, 47280846155776, 47280846180351, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846204927, +ERASE, 47280846155776, 47280846155776, +STORE, 47280846155776, 47280846180351, +STORE, 47280846180352, 47280846204927, +STORE, 47280846188544, 47280846204927, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846188543, +ERASE, 47280846188544, 47280846188544, +STORE, 47280846188544, 47280846204927, +STORE, 47280846204928, 47280846225407, +ERASE, 47280846204928, 47280846204928, +STORE, 47280846204928, 47280846209023, +STORE, 47280846209024, 47280846225407, +STORE, 47280846213120, 47280846225407, +STORE, 47280846209024, 47280846213119, +ERASE, 47280846209024, 47280846209024, +STORE, 47280846209024, 47280846213119, +STORE, 47280846217216, 47280846225407, +STORE, 47280846213120, 47280846217215, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846225407, +ERASE, 47280846213120, 47280846213120, +STORE, 47280846213120, 47280846217215, +STORE, 47280846217216, 47280846225407, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846225407, +STORE, 47280846225408, 47280846233599, +ERASE, 47280846028800, 47280846028800, +STORE, 47280846028800, 47280846045183, +STORE, 47280846045184, 47280846053375, +ERASE, 47280846217216, 47280846217216, +STORE, 47280846217216, 47280846221311, +STORE, 47280846221312, 47280846225407, +ERASE, 47280846180352, 47280846180352, +STORE, 47280846180352, 47280846184447, +STORE, 47280846184448, 47280846188543, +ERASE, 47280844009472, 47280844009472, +STORE, 47280844009472, 47280844206079, +STORE, 47280844206080, 47280844214271, +ERASE, 47280841170944, 47280841170944, +STORE, 47280841170944, 47280841175039, +STORE, 47280841175040, 47280841179135, +ERASE, 94410362142720, 94410362142720, +STORE, 94410362142720, 94410362150911, +STORE, 94410362150912, 94410362155007, +ERASE, 140351954157568, 140351954157568, +STORE, 140351954157568, 140351954161663, +STORE, 140351954161664, 140351954165759, +ERASE, 47280840998912, 47280840998912, +STORE, 94410379456512, 94410379591679, +STORE, 140737488347136, 140737488351231, +STORE, 140732946362368, 140737488351231, +ERASE, 140732946362368, 140732946362368, +STORE, 140732946362368, 140732946366463, +STORE, 94352937934848, 94352938106879, +ERASE, 94352937934848, 94352937934848, +STORE, 94352937934848, 94352937951231, +STORE, 94352937951232, 94352938106879, +ERASE, 94352937951232, 94352937951232, +STORE, 94352937951232, 94352938053631, +STORE, 94352938053632, 94352938094591, +STORE, 94352938094592, 94352938106879, +STORE, 140595518742528, 140595518914559, +ERASE, 140595518742528, 140595518742528, +STORE, 140595518742528, 140595518746623, +STORE, 140595518746624, 140595518914559, +ERASE, 140595518746624, 140595518746624, +STORE, 140595518746624, 140595518869503, +STORE, 140595518869504, 140595518902271, +STORE, 140595518902272, 140595518910463, +STORE, 140595518910464, 140595518914559, +STORE, 140732947468288, 140732947472383, +STORE, 140732947456000, 140732947468287, +STORE, 47037276254208, 47037276262399, +STORE, 47037276262400, 47037276270591, +STORE, 47037276270592, 47037276434431, +ERASE, 47037276270592, 47037276270592, +STORE, 47037276270592, 47037276282879, +STORE, 47037276282880, 47037276434431, +STORE, 47037276381184, 47037276434431, +STORE, 47037276282880, 47037276381183, +ERASE, 47037276282880, 47037276282880, +STORE, 47037276282880, 47037276381183, +STORE, 47037276426240, 47037276434431, +STORE, 47037276381184, 47037276426239, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276434431, +ERASE, 47037276381184, 47037276381184, +STORE, 47037276381184, 47037276426239, +STORE, 47037276426240, 47037276434431, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276434431, +STORE, 47037276434432, 47037279485951, +STORE, 47037276979200, 47037279485951, +STORE, 47037276434432, 47037276979199, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037279264767, +STORE, 47037279264768, 47037279485951, +STORE, 47037278674944, 47037279264767, +STORE, 47037276979200, 47037278674943, +ERASE, 47037276979200, 47037276979200, +STORE, 47037276979200, 47037278674943, +STORE, 47037279260672, 47037279264767, +STORE, 47037278674944, 47037279260671, +ERASE, 47037278674944, 47037278674944, +STORE, 47037278674944, 47037279260671, +STORE, 47037279469568, 47037279485951, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279469567, +ERASE, 47037279469568, 47037279469568, +STORE, 47037279469568, 47037279485951, +STORE, 47037279485952, 47037281325055, +STORE, 47037279625216, 47037281325055, +STORE, 47037279485952, 47037279625215, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037281284095, +STORE, 47037281284096, 47037281325055, +STORE, 47037280968704, 47037281284095, +STORE, 47037279625216, 47037280968703, +ERASE, 47037279625216, 47037279625216, +STORE, 47037279625216, 47037280968703, +STORE, 47037281280000, 47037281284095, +STORE, 47037280968704, 47037281279999, +ERASE, 47037280968704, 47037280968704, +STORE, 47037280968704, 47037281279999, +STORE, 47037281308672, 47037281325055, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281308671, +ERASE, 47037281308672, 47037281308672, +STORE, 47037281308672, 47037281325055, +STORE, 47037281325056, 47037281460223, +ERASE, 47037281325056, 47037281325056, +STORE, 47037281325056, 47037281349631, +STORE, 47037281349632, 47037281460223, +STORE, 47037281411072, 47037281460223, +STORE, 47037281349632, 47037281411071, +ERASE, 47037281349632, 47037281349632, +STORE, 47037281349632, 47037281411071, +STORE, 47037281435648, 47037281460223, +STORE, 47037281411072, 47037281435647, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281460223, +ERASE, 47037281411072, 47037281411072, +STORE, 47037281411072, 47037281435647, +STORE, 47037281435648, 47037281460223, +STORE, 47037281443840, 47037281460223, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281443839, +ERASE, 47037281443840, 47037281443840, +STORE, 47037281443840, 47037281460223, +STORE, 47037281460224, 47037281480703, +ERASE, 47037281460224, 47037281460224, +STORE, 47037281460224, 47037281464319, +STORE, 47037281464320, 47037281480703, +STORE, 47037281468416, 47037281480703, +STORE, 47037281464320, 47037281468415, +ERASE, 47037281464320, 47037281464320, +STORE, 47037281464320, 47037281468415, +STORE, 47037281472512, 47037281480703, +STORE, 47037281468416, 47037281472511, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281480703, +ERASE, 47037281468416, 47037281468416, +STORE, 47037281468416, 47037281472511, +STORE, 47037281472512, 47037281480703, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281480703, +STORE, 47037281480704, 47037281488895, +ERASE, 47037281284096, 47037281284096, +STORE, 47037281284096, 47037281300479, +STORE, 47037281300480, 47037281308671, +ERASE, 47037281472512, 47037281472512, +STORE, 47037281472512, 47037281476607, +STORE, 47037281476608, 47037281480703, +ERASE, 47037281435648, 47037281435648, +STORE, 47037281435648, 47037281439743, +STORE, 47037281439744, 47037281443839, +ERASE, 47037279264768, 47037279264768, +STORE, 47037279264768, 47037279461375, +STORE, 47037279461376, 47037279469567, +ERASE, 47037276426240, 47037276426240, +STORE, 47037276426240, 47037276430335, +STORE, 47037276430336, 47037276434431, +ERASE, 94352938094592, 94352938094592, +STORE, 94352938094592, 94352938102783, +STORE, 94352938102784, 94352938106879, +ERASE, 140595518902272, 140595518902272, +STORE, 140595518902272, 140595518906367, +STORE, 140595518906368, 140595518910463, +ERASE, 47037276254208, 47037276254208, +STORE, 94352938438656, 94352938573823, +STORE, 140737488347136, 140737488351231, +STORE, 140733506027520, 140737488351231, +ERASE, 140733506027520, 140733506027520, +STORE, 140733506027520, 140733506031615, +STORE, 94150123073536, 94150123245567, +ERASE, 94150123073536, 94150123073536, +STORE, 94150123073536, 94150123089919, +STORE, 94150123089920, 94150123245567, +ERASE, 94150123089920, 94150123089920, +STORE, 94150123089920, 94150123192319, +STORE, 94150123192320, 94150123233279, +STORE, 94150123233280, 94150123245567, +STORE, 140081290375168, 140081290547199, +ERASE, 140081290375168, 140081290375168, +STORE, 140081290375168, 140081290379263, +STORE, 140081290379264, 140081290547199, +ERASE, 140081290379264, 140081290379264, +STORE, 140081290379264, 140081290502143, +STORE, 140081290502144, 140081290534911, +STORE, 140081290534912, 140081290543103, +STORE, 140081290543104, 140081290547199, +STORE, 140733506707456, 140733506711551, +STORE, 140733506695168, 140733506707455, +STORE, 47551504621568, 47551504629759, +STORE, 47551504629760, 47551504637951, +STORE, 47551504637952, 47551504801791, +ERASE, 47551504637952, 47551504637952, +STORE, 47551504637952, 47551504650239, +STORE, 47551504650240, 47551504801791, +STORE, 47551504748544, 47551504801791, +STORE, 47551504650240, 47551504748543, +ERASE, 47551504650240, 47551504650240, +STORE, 47551504650240, 47551504748543, +STORE, 47551504793600, 47551504801791, +STORE, 47551504748544, 47551504793599, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504801791, +ERASE, 47551504748544, 47551504748544, +STORE, 47551504748544, 47551504793599, +STORE, 47551504793600, 47551504801791, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504801791, +STORE, 47551504801792, 47551507853311, +STORE, 47551505346560, 47551507853311, +STORE, 47551504801792, 47551505346559, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507632127, +STORE, 47551507632128, 47551507853311, +STORE, 47551507042304, 47551507632127, +STORE, 47551505346560, 47551507042303, +ERASE, 47551505346560, 47551505346560, +STORE, 47551505346560, 47551507042303, +STORE, 47551507628032, 47551507632127, +STORE, 47551507042304, 47551507628031, +ERASE, 47551507042304, 47551507042304, +STORE, 47551507042304, 47551507628031, +STORE, 47551507836928, 47551507853311, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507836927, +ERASE, 47551507836928, 47551507836928, +STORE, 47551507836928, 47551507853311, +STORE, 47551507853312, 47551509692415, +STORE, 47551507992576, 47551509692415, +STORE, 47551507853312, 47551507992575, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509651455, +STORE, 47551509651456, 47551509692415, +STORE, 47551509336064, 47551509651455, +STORE, 47551507992576, 47551509336063, +ERASE, 47551507992576, 47551507992576, +STORE, 47551507992576, 47551509336063, +STORE, 47551509647360, 47551509651455, +STORE, 47551509336064, 47551509647359, +ERASE, 47551509336064, 47551509336064, +STORE, 47551509336064, 47551509647359, +STORE, 47551509676032, 47551509692415, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509676031, +ERASE, 47551509676032, 47551509676032, +STORE, 47551509676032, 47551509692415, +STORE, 47551509692416, 47551509827583, +ERASE, 47551509692416, 47551509692416, +STORE, 47551509692416, 47551509716991, +STORE, 47551509716992, 47551509827583, +STORE, 47551509778432, 47551509827583, +STORE, 47551509716992, 47551509778431, +ERASE, 47551509716992, 47551509716992, +STORE, 47551509716992, 47551509778431, +STORE, 47551509803008, 47551509827583, +STORE, 47551509778432, 47551509803007, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509827583, +ERASE, 47551509778432, 47551509778432, +STORE, 47551509778432, 47551509803007, +STORE, 47551509803008, 47551509827583, +STORE, 47551509811200, 47551509827583, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509811199, +ERASE, 47551509811200, 47551509811200, +STORE, 47551509811200, 47551509827583, +STORE, 47551509827584, 47551509848063, +ERASE, 47551509827584, 47551509827584, +STORE, 47551509827584, 47551509831679, +STORE, 47551509831680, 47551509848063, +STORE, 47551509835776, 47551509848063, +STORE, 47551509831680, 47551509835775, +ERASE, 47551509831680, 47551509831680, +STORE, 47551509831680, 47551509835775, +STORE, 47551509839872, 47551509848063, +STORE, 47551509835776, 47551509839871, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509848063, +ERASE, 47551509835776, 47551509835776, +STORE, 47551509835776, 47551509839871, +STORE, 47551509839872, 47551509848063, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509848063, +STORE, 47551509848064, 47551509856255, +ERASE, 47551509651456, 47551509651456, +STORE, 47551509651456, 47551509667839, +STORE, 47551509667840, 47551509676031, +ERASE, 47551509839872, 47551509839872, +STORE, 47551509839872, 47551509843967, +STORE, 47551509843968, 47551509848063, +ERASE, 47551509803008, 47551509803008, +STORE, 47551509803008, 47551509807103, +STORE, 47551509807104, 47551509811199, +ERASE, 47551507632128, 47551507632128, +STORE, 47551507632128, 47551507828735, +STORE, 47551507828736, 47551507836927, +ERASE, 47551504793600, 47551504793600, +STORE, 47551504793600, 47551504797695, +STORE, 47551504797696, 47551504801791, +ERASE, 94150123233280, 94150123233280, +STORE, 94150123233280, 94150123241471, +STORE, 94150123241472, 94150123245567, +ERASE, 140081290534912, 140081290534912, +STORE, 140081290534912, 140081290539007, +STORE, 140081290539008, 140081290543103, +ERASE, 47551504621568, 47551504621568, +STORE, 94150148112384, 94150148247551, +STORE, 140737488347136, 140737488351231, +STORE, 140734389334016, 140737488351231, +ERASE, 140734389334016, 140734389334016, +STORE, 140734389334016, 140734389338111, +STORE, 94844636606464, 94844636778495, +ERASE, 94844636606464, 94844636606464, +STORE, 94844636606464, 94844636622847, +STORE, 94844636622848, 94844636778495, +ERASE, 94844636622848, 94844636622848, +STORE, 94844636622848, 94844636725247, +STORE, 94844636725248, 94844636766207, +STORE, 94844636766208, 94844636778495, +STORE, 139922765217792, 139922765389823, +ERASE, 139922765217792, 139922765217792, +STORE, 139922765217792, 139922765221887, +STORE, 139922765221888, 139922765389823, +ERASE, 139922765221888, 139922765221888, +STORE, 139922765221888, 139922765344767, +STORE, 139922765344768, 139922765377535, +STORE, 139922765377536, 139922765385727, +STORE, 139922765385728, 139922765389823, +STORE, 140734389678080, 140734389682175, +STORE, 140734389665792, 140734389678079, +STORE, 47710029778944, 47710029787135, +STORE, 47710029787136, 47710029795327, +STORE, 47710029795328, 47710029959167, +ERASE, 47710029795328, 47710029795328, +STORE, 47710029795328, 47710029807615, +STORE, 47710029807616, 47710029959167, +STORE, 47710029905920, 47710029959167, +STORE, 47710029807616, 47710029905919, +ERASE, 47710029807616, 47710029807616, +STORE, 47710029807616, 47710029905919, +STORE, 47710029950976, 47710029959167, +STORE, 47710029905920, 47710029950975, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029959167, +ERASE, 47710029905920, 47710029905920, +STORE, 47710029905920, 47710029950975, +STORE, 47710029950976, 47710029959167, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029959167, +STORE, 47710029959168, 47710033010687, +STORE, 47710030503936, 47710033010687, +STORE, 47710029959168, 47710030503935, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032789503, +STORE, 47710032789504, 47710033010687, +STORE, 47710032199680, 47710032789503, +STORE, 47710030503936, 47710032199679, +ERASE, 47710030503936, 47710030503936, +STORE, 47710030503936, 47710032199679, +STORE, 47710032785408, 47710032789503, +STORE, 47710032199680, 47710032785407, +ERASE, 47710032199680, 47710032199680, +STORE, 47710032199680, 47710032785407, +STORE, 47710032994304, 47710033010687, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032994303, +ERASE, 47710032994304, 47710032994304, +STORE, 47710032994304, 47710033010687, +STORE, 47710033010688, 47710034849791, +STORE, 47710033149952, 47710034849791, +STORE, 47710033010688, 47710033149951, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034808831, +STORE, 47710034808832, 47710034849791, +STORE, 47710034493440, 47710034808831, +STORE, 47710033149952, 47710034493439, +ERASE, 47710033149952, 47710033149952, +STORE, 47710033149952, 47710034493439, +STORE, 47710034804736, 47710034808831, +STORE, 47710034493440, 47710034804735, +ERASE, 47710034493440, 47710034493440, +STORE, 47710034493440, 47710034804735, +STORE, 47710034833408, 47710034849791, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034833407, +ERASE, 47710034833408, 47710034833408, +STORE, 47710034833408, 47710034849791, +STORE, 47710034849792, 47710034984959, +ERASE, 47710034849792, 47710034849792, +STORE, 47710034849792, 47710034874367, +STORE, 47710034874368, 47710034984959, +STORE, 47710034935808, 47710034984959, +STORE, 47710034874368, 47710034935807, +ERASE, 47710034874368, 47710034874368, +STORE, 47710034874368, 47710034935807, +STORE, 47710034960384, 47710034984959, +STORE, 47710034935808, 47710034960383, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034984959, +ERASE, 47710034935808, 47710034935808, +STORE, 47710034935808, 47710034960383, +STORE, 47710034960384, 47710034984959, +STORE, 47710034968576, 47710034984959, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034968575, +ERASE, 47710034968576, 47710034968576, +STORE, 47710034968576, 47710034984959, +STORE, 47710034984960, 47710035005439, +ERASE, 47710034984960, 47710034984960, +STORE, 47710034984960, 47710034989055, +STORE, 47710034989056, 47710035005439, +STORE, 47710034993152, 47710035005439, +STORE, 47710034989056, 47710034993151, +ERASE, 47710034989056, 47710034989056, +STORE, 47710034989056, 47710034993151, +STORE, 47710034997248, 47710035005439, +STORE, 47710034993152, 47710034997247, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710035005439, +ERASE, 47710034993152, 47710034993152, +STORE, 47710034993152, 47710034997247, +STORE, 47710034997248, 47710035005439, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035005439, +STORE, 47710035005440, 47710035013631, +ERASE, 47710034808832, 47710034808832, +STORE, 47710034808832, 47710034825215, +STORE, 47710034825216, 47710034833407, +ERASE, 47710034997248, 47710034997248, +STORE, 47710034997248, 47710035001343, +STORE, 47710035001344, 47710035005439, +ERASE, 47710034960384, 47710034960384, +STORE, 47710034960384, 47710034964479, +STORE, 47710034964480, 47710034968575, +ERASE, 47710032789504, 47710032789504, +STORE, 47710032789504, 47710032986111, +STORE, 47710032986112, 47710032994303, +ERASE, 47710029950976, 47710029950976, +STORE, 47710029950976, 47710029955071, +STORE, 47710029955072, 47710029959167, +ERASE, 94844636766208, 94844636766208, +STORE, 94844636766208, 94844636774399, +STORE, 94844636774400, 94844636778495, +ERASE, 139922765377536, 139922765377536, +STORE, 139922765377536, 139922765381631, +STORE, 139922765381632, 139922765385727, +ERASE, 47710029778944, 47710029778944, +STORE, 94844641775616, 94844641910783, +STORE, 140737488347136, 140737488351231, +STORE, 140732213886976, 140737488351231, +ERASE, 140732213886976, 140732213886976, +STORE, 140732213886976, 140732213891071, +STORE, 94240508887040, 94240509059071, +ERASE, 94240508887040, 94240508887040, +STORE, 94240508887040, 94240508903423, +STORE, 94240508903424, 94240509059071, +ERASE, 94240508903424, 94240508903424, +STORE, 94240508903424, 94240509005823, +STORE, 94240509005824, 94240509046783, +STORE, 94240509046784, 94240509059071, +STORE, 140275106516992, 140275106689023, +ERASE, 140275106516992, 140275106516992, +STORE, 140275106516992, 140275106521087, +STORE, 140275106521088, 140275106689023, +ERASE, 140275106521088, 140275106521088, +STORE, 140275106521088, 140275106643967, +STORE, 140275106643968, 140275106676735, +STORE, 140275106676736, 140275106684927, +STORE, 140275106684928, 140275106689023, +STORE, 140732213977088, 140732213981183, +STORE, 140732213964800, 140732213977087, +STORE, 47357688479744, 47357688487935, +STORE, 47357688487936, 47357688496127, +STORE, 47357688496128, 47357688659967, +ERASE, 47357688496128, 47357688496128, +STORE, 47357688496128, 47357688508415, +STORE, 47357688508416, 47357688659967, +STORE, 47357688606720, 47357688659967, +STORE, 47357688508416, 47357688606719, +ERASE, 47357688508416, 47357688508416, +STORE, 47357688508416, 47357688606719, +STORE, 47357688651776, 47357688659967, +STORE, 47357688606720, 47357688651775, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688659967, +ERASE, 47357688606720, 47357688606720, +STORE, 47357688606720, 47357688651775, +STORE, 47357688651776, 47357688659967, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688659967, +STORE, 47357688659968, 47357691711487, +STORE, 47357689204736, 47357691711487, +STORE, 47357688659968, 47357689204735, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357691490303, +STORE, 47357691490304, 47357691711487, +STORE, 47357690900480, 47357691490303, +STORE, 47357689204736, 47357690900479, +ERASE, 47357689204736, 47357689204736, +STORE, 47357689204736, 47357690900479, +STORE, 47357691486208, 47357691490303, +STORE, 47357690900480, 47357691486207, +ERASE, 47357690900480, 47357690900480, +STORE, 47357690900480, 47357691486207, +STORE, 47357691695104, 47357691711487, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691695103, +ERASE, 47357691695104, 47357691695104, +STORE, 47357691695104, 47357691711487, +STORE, 47357691711488, 47357693550591, +STORE, 47357691850752, 47357693550591, +STORE, 47357691711488, 47357691850751, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693509631, +STORE, 47357693509632, 47357693550591, +STORE, 47357693194240, 47357693509631, +STORE, 47357691850752, 47357693194239, +ERASE, 47357691850752, 47357691850752, +STORE, 47357691850752, 47357693194239, +STORE, 47357693505536, 47357693509631, +STORE, 47357693194240, 47357693505535, +ERASE, 47357693194240, 47357693194240, +STORE, 47357693194240, 47357693505535, +STORE, 47357693534208, 47357693550591, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693534207, +ERASE, 47357693534208, 47357693534208, +STORE, 47357693534208, 47357693550591, +STORE, 47357693550592, 47357693685759, +ERASE, 47357693550592, 47357693550592, +STORE, 47357693550592, 47357693575167, +STORE, 47357693575168, 47357693685759, +STORE, 47357693636608, 47357693685759, +STORE, 47357693575168, 47357693636607, +ERASE, 47357693575168, 47357693575168, +STORE, 47357693575168, 47357693636607, +STORE, 47357693661184, 47357693685759, +STORE, 47357693636608, 47357693661183, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693685759, +ERASE, 47357693636608, 47357693636608, +STORE, 47357693636608, 47357693661183, +STORE, 47357693661184, 47357693685759, +STORE, 47357693669376, 47357693685759, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693669375, +ERASE, 47357693669376, 47357693669376, +STORE, 47357693669376, 47357693685759, +STORE, 47357693685760, 47357693706239, +ERASE, 47357693685760, 47357693685760, +STORE, 47357693685760, 47357693689855, +STORE, 47357693689856, 47357693706239, +STORE, 47357693693952, 47357693706239, +STORE, 47357693689856, 47357693693951, +ERASE, 47357693689856, 47357693689856, +STORE, 47357693689856, 47357693693951, +STORE, 47357693698048, 47357693706239, +STORE, 47357693693952, 47357693698047, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693706239, +ERASE, 47357693693952, 47357693693952, +STORE, 47357693693952, 47357693698047, +STORE, 47357693698048, 47357693706239, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693706239, +STORE, 47357693706240, 47357693714431, +ERASE, 47357693509632, 47357693509632, +STORE, 47357693509632, 47357693526015, +STORE, 47357693526016, 47357693534207, +ERASE, 47357693698048, 47357693698048, +STORE, 47357693698048, 47357693702143, +STORE, 47357693702144, 47357693706239, +ERASE, 47357693661184, 47357693661184, +STORE, 47357693661184, 47357693665279, +STORE, 47357693665280, 47357693669375, +ERASE, 47357691490304, 47357691490304, +STORE, 47357691490304, 47357691686911, +STORE, 47357691686912, 47357691695103, +ERASE, 47357688651776, 47357688651776, +STORE, 47357688651776, 47357688655871, +STORE, 47357688655872, 47357688659967, +ERASE, 94240509046784, 94240509046784, +STORE, 94240509046784, 94240509054975, +STORE, 94240509054976, 94240509059071, +ERASE, 140275106676736, 140275106676736, +STORE, 140275106676736, 140275106680831, +STORE, 140275106680832, 140275106684927, +ERASE, 47357688479744, 47357688479744, +STORE, 94240518361088, 94240518496255, +STORE, 140737488347136, 140737488351231, +STORE, 140732688277504, 140737488351231, +ERASE, 140732688277504, 140732688277504, +STORE, 140732688277504, 140732688281599, +STORE, 94629171351552, 94629172064255, +ERASE, 94629171351552, 94629171351552, +STORE, 94629171351552, 94629171400703, +STORE, 94629171400704, 94629172064255, +ERASE, 94629171400704, 94629171400704, +STORE, 94629171400704, 94629171945471, +STORE, 94629171945472, 94629172043775, +STORE, 94629172043776, 94629172064255, +STORE, 139770707644416, 139770707816447, +ERASE, 139770707644416, 139770707644416, +STORE, 139770707644416, 139770707648511, +STORE, 139770707648512, 139770707816447, +ERASE, 139770707648512, 139770707648512, +STORE, 139770707648512, 139770707771391, +STORE, 139770707771392, 139770707804159, +STORE, 139770707804160, 139770707812351, +STORE, 139770707812352, 139770707816447, +STORE, 140732689121280, 140732689125375, +STORE, 140732689108992, 140732689121279, +STORE, 47862087352320, 47862087360511, +STORE, 47862087360512, 47862087368703, +STORE, 47862087368704, 47862087475199, +STORE, 47862087385088, 47862087475199, +STORE, 47862087368704, 47862087385087, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087458815, +STORE, 47862087458816, 47862087475199, +STORE, 47862087438336, 47862087458815, +STORE, 47862087385088, 47862087438335, +ERASE, 47862087385088, 47862087385088, +STORE, 47862087385088, 47862087438335, +STORE, 47862087454720, 47862087458815, +STORE, 47862087438336, 47862087454719, +ERASE, 47862087438336, 47862087438336, +STORE, 47862087438336, 47862087454719, +STORE, 47862087467008, 47862087475199, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087467007, +ERASE, 47862087467008, 47862087467008, +STORE, 47862087467008, 47862087475199, +STORE, 47862087475200, 47862089314303, +STORE, 47862087614464, 47862089314303, +STORE, 47862087475200, 47862087614463, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862089273343, +STORE, 47862089273344, 47862089314303, +STORE, 47862088957952, 47862089273343, +STORE, 47862087614464, 47862088957951, +ERASE, 47862087614464, 47862087614464, +STORE, 47862087614464, 47862088957951, +STORE, 47862089269248, 47862089273343, +STORE, 47862088957952, 47862089269247, +ERASE, 47862088957952, 47862088957952, +STORE, 47862088957952, 47862089269247, +STORE, 47862089297920, 47862089314303, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089297919, +ERASE, 47862089297920, 47862089297920, +STORE, 47862089297920, 47862089314303, +STORE, 47862089297920, 47862089326591, +ERASE, 47862089273344, 47862089273344, +STORE, 47862089273344, 47862089289727, +STORE, 47862089289728, 47862089297919, +ERASE, 47862087458816, 47862087458816, +STORE, 47862087458816, 47862087462911, +STORE, 47862087462912, 47862087467007, +ERASE, 94629172043776, 94629172043776, +STORE, 94629172043776, 94629172060159, +STORE, 94629172060160, 94629172064255, +ERASE, 139770707804160, 139770707804160, +STORE, 139770707804160, 139770707808255, +STORE, 139770707808256, 139770707812351, +ERASE, 47862087352320, 47862087352320, +STORE, 94629197533184, 94629197668351, +STORE, 140737488347136, 140737488351231, +STORE, 140727540711424, 140737488351231, +ERASE, 140727540711424, 140727540711424, +STORE, 140727540711424, 140727540715519, +STORE, 94299865313280, 94299866025983, +ERASE, 94299865313280, 94299865313280, +STORE, 94299865313280, 94299865362431, +STORE, 94299865362432, 94299866025983, +ERASE, 94299865362432, 94299865362432, +STORE, 94299865362432, 94299865907199, +STORE, 94299865907200, 94299866005503, +STORE, 94299866005504, 94299866025983, +STORE, 140680268763136, 140680268935167, +ERASE, 140680268763136, 140680268763136, +STORE, 140680268763136, 140680268767231, +STORE, 140680268767232, 140680268935167, +ERASE, 140680268767232, 140680268767232, +STORE, 140680268767232, 140680268890111, +STORE, 140680268890112, 140680268922879, +STORE, 140680268922880, 140680268931071, +STORE, 140680268931072, 140680268935167, +STORE, 140727541424128, 140727541428223, +STORE, 140727541411840, 140727541424127, +STORE, 46952526233600, 46952526241791, +STORE, 46952526241792, 46952526249983, +STORE, 46952526249984, 46952526356479, +STORE, 46952526266368, 46952526356479, +STORE, 46952526249984, 46952526266367, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526340095, +STORE, 46952526340096, 46952526356479, +STORE, 46952526319616, 46952526340095, +STORE, 46952526266368, 46952526319615, +ERASE, 46952526266368, 46952526266368, +STORE, 46952526266368, 46952526319615, +STORE, 46952526336000, 46952526340095, +STORE, 46952526319616, 46952526335999, +ERASE, 46952526319616, 46952526319616, +STORE, 46952526319616, 46952526335999, +STORE, 46952526348288, 46952526356479, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526348287, +ERASE, 46952526348288, 46952526348288, +STORE, 46952526348288, 46952526356479, +STORE, 46952526356480, 46952528195583, +STORE, 46952526495744, 46952528195583, +STORE, 46952526356480, 46952526495743, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952528154623, +STORE, 46952528154624, 46952528195583, +STORE, 46952527839232, 46952528154623, +STORE, 46952526495744, 46952527839231, +ERASE, 46952526495744, 46952526495744, +STORE, 46952526495744, 46952527839231, +STORE, 46952528150528, 46952528154623, +STORE, 46952527839232, 46952528150527, +ERASE, 46952527839232, 46952527839232, +STORE, 46952527839232, 46952528150527, +STORE, 46952528179200, 46952528195583, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528179199, +ERASE, 46952528179200, 46952528179200, +STORE, 46952528179200, 46952528195583, +STORE, 46952528179200, 46952528207871, +ERASE, 46952528154624, 46952528154624, +STORE, 46952528154624, 46952528171007, +STORE, 46952528171008, 46952528179199, +ERASE, 46952526340096, 46952526340096, +STORE, 46952526340096, 46952526344191, +STORE, 46952526344192, 46952526348287, +ERASE, 94299866005504, 94299866005504, +STORE, 94299866005504, 94299866021887, +STORE, 94299866021888, 94299866025983, +ERASE, 140680268922880, 140680268922880, +STORE, 140680268922880, 140680268926975, +STORE, 140680268926976, 140680268931071, +ERASE, 46952526233600, 46952526233600, +STORE, 140737488347136, 140737488351231, +STORE, 140722874793984, 140737488351231, +ERASE, 140722874793984, 140722874793984, +STORE, 140722874793984, 140722874798079, +STORE, 94448916213760, 94448916926463, +ERASE, 94448916213760, 94448916213760, +STORE, 94448916213760, 94448916262911, +STORE, 94448916262912, 94448916926463, +ERASE, 94448916262912, 94448916262912, +STORE, 94448916262912, 94448916807679, +STORE, 94448916807680, 94448916905983, +STORE, 94448916905984, 94448916926463, +STORE, 140389117046784, 140389117218815, +ERASE, 140389117046784, 140389117046784, +STORE, 140389117046784, 140389117050879, +STORE, 140389117050880, 140389117218815, +ERASE, 140389117050880, 140389117050880, +STORE, 140389117050880, 140389117173759, +STORE, 140389117173760, 140389117206527, +STORE, 140389117206528, 140389117214719, +STORE, 140389117214720, 140389117218815, +STORE, 140722875297792, 140722875301887, +STORE, 140722875285504, 140722875297791, +STORE, 47243677949952, 47243677958143, +STORE, 47243677958144, 47243677966335, +STORE, 47243677966336, 47243678072831, +STORE, 47243677982720, 47243678072831, +STORE, 47243677966336, 47243677982719, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678056447, +STORE, 47243678056448, 47243678072831, +STORE, 47243678035968, 47243678056447, +STORE, 47243677982720, 47243678035967, +ERASE, 47243677982720, 47243677982720, +STORE, 47243677982720, 47243678035967, +STORE, 47243678052352, 47243678056447, +STORE, 47243678035968, 47243678052351, +ERASE, 47243678035968, 47243678035968, +STORE, 47243678035968, 47243678052351, +STORE, 47243678064640, 47243678072831, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678064639, +ERASE, 47243678064640, 47243678064640, +STORE, 47243678064640, 47243678072831, +STORE, 47243678072832, 47243679911935, +STORE, 47243678212096, 47243679911935, +STORE, 47243678072832, 47243678212095, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679870975, +STORE, 47243679870976, 47243679911935, +STORE, 47243679555584, 47243679870975, +STORE, 47243678212096, 47243679555583, +ERASE, 47243678212096, 47243678212096, +STORE, 47243678212096, 47243679555583, +STORE, 47243679866880, 47243679870975, +STORE, 47243679555584, 47243679866879, +ERASE, 47243679555584, 47243679555584, +STORE, 47243679555584, 47243679866879, +STORE, 47243679895552, 47243679911935, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679895551, +ERASE, 47243679895552, 47243679895552, +STORE, 47243679895552, 47243679911935, +STORE, 47243679895552, 47243679924223, +ERASE, 47243679870976, 47243679870976, +STORE, 47243679870976, 47243679887359, +STORE, 47243679887360, 47243679895551, +ERASE, 47243678056448, 47243678056448, +STORE, 47243678056448, 47243678060543, +STORE, 47243678060544, 47243678064639, +ERASE, 94448916905984, 94448916905984, +STORE, 94448916905984, 94448916922367, +STORE, 94448916922368, 94448916926463, +ERASE, 140389117206528, 140389117206528, +STORE, 140389117206528, 140389117210623, +STORE, 140389117210624, 140389117214719, +ERASE, 47243677949952, 47243677949952, +STORE, 140737488347136, 140737488351231, +STORE, 140733068505088, 140737488351231, +ERASE, 140733068505088, 140733068505088, +STORE, 140733068505088, 140733068509183, +STORE, 94207145750528, 94207146463231, +ERASE, 94207145750528, 94207145750528, +STORE, 94207145750528, 94207145799679, +STORE, 94207145799680, 94207146463231, +ERASE, 94207145799680, 94207145799680, +STORE, 94207145799680, 94207146344447, +STORE, 94207146344448, 94207146442751, +STORE, 94207146442752, 94207146463231, +STORE, 140684504911872, 140684505083903, +ERASE, 140684504911872, 140684504911872, +STORE, 140684504911872, 140684504915967, +STORE, 140684504915968, 140684505083903, +ERASE, 140684504915968, 140684504915968, +STORE, 140684504915968, 140684505038847, +STORE, 140684505038848, 140684505071615, +STORE, 140684505071616, 140684505079807, +STORE, 140684505079808, 140684505083903, +STORE, 140733068607488, 140733068611583, +STORE, 140733068595200, 140733068607487, +STORE, 46948290084864, 46948290093055, +STORE, 46948290093056, 46948290101247, +STORE, 46948290101248, 46948290207743, +STORE, 46948290117632, 46948290207743, +STORE, 46948290101248, 46948290117631, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290191359, +STORE, 46948290191360, 46948290207743, +STORE, 46948290170880, 46948290191359, +STORE, 46948290117632, 46948290170879, +ERASE, 46948290117632, 46948290117632, +STORE, 46948290117632, 46948290170879, +STORE, 46948290187264, 46948290191359, +STORE, 46948290170880, 46948290187263, +ERASE, 46948290170880, 46948290170880, +STORE, 46948290170880, 46948290187263, +STORE, 46948290199552, 46948290207743, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290199551, +ERASE, 46948290199552, 46948290199552, +STORE, 46948290199552, 46948290207743, +STORE, 46948290207744, 46948292046847, +STORE, 46948290347008, 46948292046847, +STORE, 46948290207744, 46948290347007, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948292005887, +STORE, 46948292005888, 46948292046847, +STORE, 46948291690496, 46948292005887, +STORE, 46948290347008, 46948291690495, +ERASE, 46948290347008, 46948290347008, +STORE, 46948290347008, 46948291690495, +STORE, 46948292001792, 46948292005887, +STORE, 46948291690496, 46948292001791, +ERASE, 46948291690496, 46948291690496, +STORE, 46948291690496, 46948292001791, +STORE, 46948292030464, 46948292046847, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292030463, +ERASE, 46948292030464, 46948292030464, +STORE, 46948292030464, 46948292046847, +STORE, 46948292030464, 46948292059135, +ERASE, 46948292005888, 46948292005888, +STORE, 46948292005888, 46948292022271, +STORE, 46948292022272, 46948292030463, +ERASE, 46948290191360, 46948290191360, +STORE, 46948290191360, 46948290195455, +STORE, 46948290195456, 46948290199551, +ERASE, 94207146442752, 94207146442752, +STORE, 94207146442752, 94207146459135, +STORE, 94207146459136, 94207146463231, +ERASE, 140684505071616, 140684505071616, +STORE, 140684505071616, 140684505075711, +STORE, 140684505075712, 140684505079807, +ERASE, 46948290084864, 46948290084864, +STORE, 140737488347136, 140737488351231, +STORE, 140726367158272, 140737488351231, +ERASE, 140726367158272, 140726367158272, +STORE, 140726367158272, 140726367162367, +STORE, 94436124106752, 94436124819455, +ERASE, 94436124106752, 94436124106752, +STORE, 94436124106752, 94436124155903, +STORE, 94436124155904, 94436124819455, +ERASE, 94436124155904, 94436124155904, +STORE, 94436124155904, 94436124700671, +STORE, 94436124700672, 94436124798975, +STORE, 94436124798976, 94436124819455, +STORE, 140049025044480, 140049025216511, +ERASE, 140049025044480, 140049025044480, +STORE, 140049025044480, 140049025048575, +STORE, 140049025048576, 140049025216511, +ERASE, 140049025048576, 140049025048576, +STORE, 140049025048576, 140049025171455, +STORE, 140049025171456, 140049025204223, +STORE, 140049025204224, 140049025212415, +STORE, 140049025212416, 140049025216511, +STORE, 140726367256576, 140726367260671, +STORE, 140726367244288, 140726367256575, +STORE, 47583769952256, 47583769960447, +STORE, 47583769960448, 47583769968639, +STORE, 47583769968640, 47583770075135, +STORE, 47583769985024, 47583770075135, +STORE, 47583769968640, 47583769985023, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770058751, +STORE, 47583770058752, 47583770075135, +STORE, 47583770038272, 47583770058751, +STORE, 47583769985024, 47583770038271, +ERASE, 47583769985024, 47583769985024, +STORE, 47583769985024, 47583770038271, +STORE, 47583770054656, 47583770058751, +STORE, 47583770038272, 47583770054655, +ERASE, 47583770038272, 47583770038272, +STORE, 47583770038272, 47583770054655, +STORE, 47583770066944, 47583770075135, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770066943, +ERASE, 47583770066944, 47583770066944, +STORE, 47583770066944, 47583770075135, +STORE, 47583770075136, 47583771914239, +STORE, 47583770214400, 47583771914239, +STORE, 47583770075136, 47583770214399, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771873279, +STORE, 47583771873280, 47583771914239, +STORE, 47583771557888, 47583771873279, +STORE, 47583770214400, 47583771557887, +ERASE, 47583770214400, 47583770214400, +STORE, 47583770214400, 47583771557887, +STORE, 47583771869184, 47583771873279, +STORE, 47583771557888, 47583771869183, +ERASE, 47583771557888, 47583771557888, +STORE, 47583771557888, 47583771869183, +STORE, 47583771897856, 47583771914239, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771897855, +ERASE, 47583771897856, 47583771897856, +STORE, 47583771897856, 47583771914239, +STORE, 47583771897856, 47583771926527, +ERASE, 47583771873280, 47583771873280, +STORE, 47583771873280, 47583771889663, +STORE, 47583771889664, 47583771897855, +ERASE, 47583770058752, 47583770058752, +STORE, 47583770058752, 47583770062847, +STORE, 47583770062848, 47583770066943, +ERASE, 94436124798976, 94436124798976, +STORE, 94436124798976, 94436124815359, +STORE, 94436124815360, 94436124819455, +ERASE, 140049025204224, 140049025204224, +STORE, 140049025204224, 140049025208319, +STORE, 140049025208320, 140049025212415, +ERASE, 47583769952256, 47583769952256, +STORE, 140737488347136, 140737488351231, +STORE, 140727116099584, 140737488351231, +ERASE, 140727116099584, 140727116099584, +STORE, 140727116099584, 140727116103679, +STORE, 94166319734784, 94166320447487, +ERASE, 94166319734784, 94166319734784, +STORE, 94166319734784, 94166319783935, +STORE, 94166319783936, 94166320447487, +ERASE, 94166319783936, 94166319783936, +STORE, 94166319783936, 94166320328703, +STORE, 94166320328704, 94166320427007, +STORE, 94166320427008, 94166320447487, +STORE, 139976559542272, 139976559714303, +ERASE, 139976559542272, 139976559542272, +STORE, 139976559542272, 139976559546367, +STORE, 139976559546368, 139976559714303, +ERASE, 139976559546368, 139976559546368, +STORE, 139976559546368, 139976559669247, +STORE, 139976559669248, 139976559702015, +STORE, 139976559702016, 139976559710207, +STORE, 139976559710208, 139976559714303, +STORE, 140727116222464, 140727116226559, +STORE, 140727116210176, 140727116222463, +STORE, 47656235454464, 47656235462655, +STORE, 47656235462656, 47656235470847, +STORE, 47656235470848, 47656235577343, +STORE, 47656235487232, 47656235577343, +STORE, 47656235470848, 47656235487231, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235560959, +STORE, 47656235560960, 47656235577343, +STORE, 47656235540480, 47656235560959, +STORE, 47656235487232, 47656235540479, +ERASE, 47656235487232, 47656235487232, +STORE, 47656235487232, 47656235540479, +STORE, 47656235556864, 47656235560959, +STORE, 47656235540480, 47656235556863, +ERASE, 47656235540480, 47656235540480, +STORE, 47656235540480, 47656235556863, +STORE, 47656235569152, 47656235577343, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235569151, +ERASE, 47656235569152, 47656235569152, +STORE, 47656235569152, 47656235577343, +STORE, 47656235577344, 47656237416447, +STORE, 47656235716608, 47656237416447, +STORE, 47656235577344, 47656235716607, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237375487, +STORE, 47656237375488, 47656237416447, +STORE, 47656237060096, 47656237375487, +STORE, 47656235716608, 47656237060095, +ERASE, 47656235716608, 47656235716608, +STORE, 47656235716608, 47656237060095, +STORE, 47656237371392, 47656237375487, +STORE, 47656237060096, 47656237371391, +ERASE, 47656237060096, 47656237060096, +STORE, 47656237060096, 47656237371391, +STORE, 47656237400064, 47656237416447, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237400063, +ERASE, 47656237400064, 47656237400064, +STORE, 47656237400064, 47656237416447, +STORE, 47656237400064, 47656237428735, +ERASE, 47656237375488, 47656237375488, +STORE, 47656237375488, 47656237391871, +STORE, 47656237391872, 47656237400063, +ERASE, 47656235560960, 47656235560960, +STORE, 47656235560960, 47656235565055, +STORE, 47656235565056, 47656235569151, +ERASE, 94166320427008, 94166320427008, +STORE, 94166320427008, 94166320443391, +STORE, 94166320443392, 94166320447487, +ERASE, 139976559702016, 139976559702016, +STORE, 139976559702016, 139976559706111, +STORE, 139976559706112, 139976559710207, +ERASE, 47656235454464, 47656235454464, +STORE, 94166332153856, 94166332289023, +STORE, 140737488347136, 140737488351231, +STORE, 140726412816384, 140737488351231, +ERASE, 140726412816384, 140726412816384, +STORE, 140726412816384, 140726412820479, +STORE, 94094884507648, 94094885220351, +ERASE, 94094884507648, 94094884507648, +STORE, 94094884507648, 94094884556799, +STORE, 94094884556800, 94094885220351, +ERASE, 94094884556800, 94094884556800, +STORE, 94094884556800, 94094885101567, +STORE, 94094885101568, 94094885199871, +STORE, 94094885199872, 94094885220351, +STORE, 139773773938688, 139773774110719, +ERASE, 139773773938688, 139773773938688, +STORE, 139773773938688, 139773773942783, +STORE, 139773773942784, 139773774110719, +ERASE, 139773773942784, 139773773942784, +STORE, 139773773942784, 139773774065663, +STORE, 139773774065664, 139773774098431, +STORE, 139773774098432, 139773774106623, +STORE, 139773774106624, 139773774110719, +STORE, 140726412963840, 140726412967935, +STORE, 140726412951552, 140726412963839, +STORE, 47859021058048, 47859021066239, +STORE, 47859021066240, 47859021074431, +STORE, 47859021074432, 47859021180927, +STORE, 47859021090816, 47859021180927, +STORE, 47859021074432, 47859021090815, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021164543, +STORE, 47859021164544, 47859021180927, +STORE, 47859021144064, 47859021164543, +STORE, 47859021090816, 47859021144063, +ERASE, 47859021090816, 47859021090816, +STORE, 47859021090816, 47859021144063, +STORE, 47859021160448, 47859021164543, +STORE, 47859021144064, 47859021160447, +ERASE, 47859021144064, 47859021144064, +STORE, 47859021144064, 47859021160447, +STORE, 47859021172736, 47859021180927, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021172735, +ERASE, 47859021172736, 47859021172736, +STORE, 47859021172736, 47859021180927, +STORE, 47859021180928, 47859023020031, +STORE, 47859021320192, 47859023020031, +STORE, 47859021180928, 47859021320191, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022979071, +STORE, 47859022979072, 47859023020031, +STORE, 47859022663680, 47859022979071, +STORE, 47859021320192, 47859022663679, +ERASE, 47859021320192, 47859021320192, +STORE, 47859021320192, 47859022663679, +STORE, 47859022974976, 47859022979071, +STORE, 47859022663680, 47859022974975, +ERASE, 47859022663680, 47859022663680, +STORE, 47859022663680, 47859022974975, +STORE, 47859023003648, 47859023020031, +STORE, 47859022979072, 47859023003647, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859023003647, +ERASE, 47859023003648, 47859023003648, +STORE, 47859023003648, 47859023020031, +STORE, 47859023003648, 47859023032319, +ERASE, 47859022979072, 47859022979072, +STORE, 47859022979072, 47859022995455, +STORE, 47859022995456, 47859023003647, +ERASE, 47859021164544, 47859021164544, +STORE, 47859021164544, 47859021168639, +STORE, 47859021168640, 47859021172735, +ERASE, 94094885199872, 94094885199872, +STORE, 94094885199872, 94094885216255, +STORE, 94094885216256, 94094885220351, +ERASE, 139773774098432, 139773774098432, +STORE, 139773774098432, 139773774102527, +STORE, 139773774102528, 139773774106623, +ERASE, 47859021058048, 47859021058048, +STORE, 94094901108736, 94094901243903, +STORE, 140737488347136, 140737488351231, +STORE, 140736567963648, 140737488351231, +ERASE, 140736567963648, 140736567963648, +STORE, 140736567963648, 140736567967743, +STORE, 94924425748480, 94924426461183, +ERASE, 94924425748480, 94924425748480, +STORE, 94924425748480, 94924425797631, +STORE, 94924425797632, 94924426461183, +ERASE, 94924425797632, 94924425797632, +STORE, 94924425797632, 94924426342399, +STORE, 94924426342400, 94924426440703, +STORE, 94924426440704, 94924426461183, +STORE, 140042126319616, 140042126491647, +ERASE, 140042126319616, 140042126319616, +STORE, 140042126319616, 140042126323711, +STORE, 140042126323712, 140042126491647, +ERASE, 140042126323712, 140042126323712, +STORE, 140042126323712, 140042126446591, +STORE, 140042126446592, 140042126479359, +STORE, 140042126479360, 140042126487551, +STORE, 140042126487552, 140042126491647, +STORE, 140736568672256, 140736568676351, +STORE, 140736568659968, 140736568672255, +STORE, 47590668677120, 47590668685311, +STORE, 47590668685312, 47590668693503, +STORE, 47590668693504, 47590668799999, +STORE, 47590668709888, 47590668799999, +STORE, 47590668693504, 47590668709887, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668783615, +STORE, 47590668783616, 47590668799999, +STORE, 47590668763136, 47590668783615, +STORE, 47590668709888, 47590668763135, +ERASE, 47590668709888, 47590668709888, +STORE, 47590668709888, 47590668763135, +STORE, 47590668779520, 47590668783615, +STORE, 47590668763136, 47590668779519, +ERASE, 47590668763136, 47590668763136, +STORE, 47590668763136, 47590668779519, +STORE, 47590668791808, 47590668799999, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668791807, +ERASE, 47590668791808, 47590668791808, +STORE, 47590668791808, 47590668799999, +STORE, 47590668800000, 47590670639103, +STORE, 47590668939264, 47590670639103, +STORE, 47590668800000, 47590668939263, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670598143, +STORE, 47590670598144, 47590670639103, +STORE, 47590670282752, 47590670598143, +STORE, 47590668939264, 47590670282751, +ERASE, 47590668939264, 47590668939264, +STORE, 47590668939264, 47590670282751, +STORE, 47590670594048, 47590670598143, +STORE, 47590670282752, 47590670594047, +ERASE, 47590670282752, 47590670282752, +STORE, 47590670282752, 47590670594047, +STORE, 47590670622720, 47590670639103, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670622719, +ERASE, 47590670622720, 47590670622720, +STORE, 47590670622720, 47590670639103, +STORE, 47590670622720, 47590670651391, +ERASE, 47590670598144, 47590670598144, +STORE, 47590670598144, 47590670614527, +STORE, 47590670614528, 47590670622719, +ERASE, 47590668783616, 47590668783616, +STORE, 47590668783616, 47590668787711, +STORE, 47590668787712, 47590668791807, +ERASE, 94924426440704, 94924426440704, +STORE, 94924426440704, 94924426457087, +STORE, 94924426457088, 94924426461183, +ERASE, 140042126479360, 140042126479360, +STORE, 140042126479360, 140042126483455, +STORE, 140042126483456, 140042126487551, +ERASE, 47590668677120, 47590668677120, +STORE, 140737488347136, 140737488351231, +STORE, 140733281439744, 140737488351231, +ERASE, 140733281439744, 140733281439744, +STORE, 140733281439744, 140733281443839, +STORE, 94490667069440, 94490667782143, +ERASE, 94490667069440, 94490667069440, +STORE, 94490667069440, 94490667118591, +STORE, 94490667118592, 94490667782143, +ERASE, 94490667118592, 94490667118592, +STORE, 94490667118592, 94490667663359, +STORE, 94490667663360, 94490667761663, +STORE, 94490667761664, 94490667782143, +STORE, 139878215118848, 139878215290879, +ERASE, 139878215118848, 139878215118848, +STORE, 139878215118848, 139878215122943, +STORE, 139878215122944, 139878215290879, +ERASE, 139878215122944, 139878215122944, +STORE, 139878215122944, 139878215245823, +STORE, 139878215245824, 139878215278591, +STORE, 139878215278592, 139878215286783, +STORE, 139878215286784, 139878215290879, +STORE, 140733281464320, 140733281468415, +STORE, 140733281452032, 140733281464319, +STORE, 47754579877888, 47754579886079, +STORE, 47754579886080, 47754579894271, +STORE, 47754579894272, 47754580000767, +STORE, 47754579910656, 47754580000767, +STORE, 47754579894272, 47754579910655, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579984383, +STORE, 47754579984384, 47754580000767, +STORE, 47754579963904, 47754579984383, +STORE, 47754579910656, 47754579963903, +ERASE, 47754579910656, 47754579910656, +STORE, 47754579910656, 47754579963903, +STORE, 47754579980288, 47754579984383, +STORE, 47754579963904, 47754579980287, +ERASE, 47754579963904, 47754579963904, +STORE, 47754579963904, 47754579980287, +STORE, 47754579992576, 47754580000767, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579992575, +ERASE, 47754579992576, 47754579992576, +STORE, 47754579992576, 47754580000767, +STORE, 47754580000768, 47754581839871, +STORE, 47754580140032, 47754581839871, +STORE, 47754580000768, 47754580140031, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581798911, +STORE, 47754581798912, 47754581839871, +STORE, 47754581483520, 47754581798911, +STORE, 47754580140032, 47754581483519, +ERASE, 47754580140032, 47754580140032, +STORE, 47754580140032, 47754581483519, +STORE, 47754581794816, 47754581798911, +STORE, 47754581483520, 47754581794815, +ERASE, 47754581483520, 47754581483520, +STORE, 47754581483520, 47754581794815, +STORE, 47754581823488, 47754581839871, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581823487, +ERASE, 47754581823488, 47754581823488, +STORE, 47754581823488, 47754581839871, +STORE, 47754581823488, 47754581852159, +ERASE, 47754581798912, 47754581798912, +STORE, 47754581798912, 47754581815295, +STORE, 47754581815296, 47754581823487, +ERASE, 47754579984384, 47754579984384, +STORE, 47754579984384, 47754579988479, +STORE, 47754579988480, 47754579992575, +ERASE, 94490667761664, 94490667761664, +STORE, 94490667761664, 94490667778047, +STORE, 94490667778048, 94490667782143, +ERASE, 139878215278592, 139878215278592, +STORE, 139878215278592, 139878215282687, +STORE, 139878215282688, 139878215286783, +ERASE, 47754579877888, 47754579877888, +STORE, 94490669649920, 94490669785087, +STORE, 140737488347136, 140737488351231, +STORE, 140735382188032, 140737488351231, +ERASE, 140735382188032, 140735382188032, +STORE, 140735382188032, 140735382192127, +STORE, 94150181302272, 94150182014975, +ERASE, 94150181302272, 94150181302272, +STORE, 94150181302272, 94150181351423, +STORE, 94150181351424, 94150182014975, +ERASE, 94150181351424, 94150181351424, +STORE, 94150181351424, 94150181896191, +STORE, 94150181896192, 94150181994495, +STORE, 94150181994496, 94150182014975, +STORE, 139679752458240, 139679752630271, +ERASE, 139679752458240, 139679752458240, +STORE, 139679752458240, 139679752462335, +STORE, 139679752462336, 139679752630271, +ERASE, 139679752462336, 139679752462336, +STORE, 139679752462336, 139679752585215, +STORE, 139679752585216, 139679752617983, +STORE, 139679752617984, 139679752626175, +STORE, 139679752626176, 139679752630271, +STORE, 140735382536192, 140735382540287, +STORE, 140735382523904, 140735382536191, +STORE, 47953042538496, 47953042546687, +STORE, 47953042546688, 47953042554879, +STORE, 47953042554880, 47953042661375, +STORE, 47953042571264, 47953042661375, +STORE, 47953042554880, 47953042571263, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042644991, +STORE, 47953042644992, 47953042661375, +STORE, 47953042624512, 47953042644991, +STORE, 47953042571264, 47953042624511, +ERASE, 47953042571264, 47953042571264, +STORE, 47953042571264, 47953042624511, +STORE, 47953042640896, 47953042644991, +STORE, 47953042624512, 47953042640895, +ERASE, 47953042624512, 47953042624512, +STORE, 47953042624512, 47953042640895, +STORE, 47953042653184, 47953042661375, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042653183, +ERASE, 47953042653184, 47953042653184, +STORE, 47953042653184, 47953042661375, +STORE, 47953042661376, 47953044500479, +STORE, 47953042800640, 47953044500479, +STORE, 47953042661376, 47953042800639, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044459519, +STORE, 47953044459520, 47953044500479, +STORE, 47953044144128, 47953044459519, +STORE, 47953042800640, 47953044144127, +ERASE, 47953042800640, 47953042800640, +STORE, 47953042800640, 47953044144127, +STORE, 47953044455424, 47953044459519, +STORE, 47953044144128, 47953044455423, +ERASE, 47953044144128, 47953044144128, +STORE, 47953044144128, 47953044455423, +STORE, 47953044484096, 47953044500479, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044484095, +ERASE, 47953044484096, 47953044484096, +STORE, 47953044484096, 47953044500479, +STORE, 47953044484096, 47953044512767, +ERASE, 47953044459520, 47953044459520, +STORE, 47953044459520, 47953044475903, +STORE, 47953044475904, 47953044484095, +ERASE, 47953042644992, 47953042644992, +STORE, 47953042644992, 47953042649087, +STORE, 47953042649088, 47953042653183, +ERASE, 94150181994496, 94150181994496, +STORE, 94150181994496, 94150182010879, +STORE, 94150182010880, 94150182014975, +ERASE, 139679752617984, 139679752617984, +STORE, 139679752617984, 139679752622079, +STORE, 139679752622080, 139679752626175, +ERASE, 47953042538496, 47953042538496, +STORE, 140737488347136, 140737488351231, +STORE, 140737044123648, 140737488351231, +ERASE, 140737044123648, 140737044123648, +STORE, 140737044123648, 140737044127743, +STORE, 94425324294144, 94425325006847, +ERASE, 94425324294144, 94425324294144, +STORE, 94425324294144, 94425324343295, +STORE, 94425324343296, 94425325006847, +ERASE, 94425324343296, 94425324343296, +STORE, 94425324343296, 94425324888063, +STORE, 94425324888064, 94425324986367, +STORE, 94425324986368, 94425325006847, +STORE, 140382015016960, 140382015188991, +ERASE, 140382015016960, 140382015016960, +STORE, 140382015016960, 140382015021055, +STORE, 140382015021056, 140382015188991, +ERASE, 140382015021056, 140382015021056, +STORE, 140382015021056, 140382015143935, +STORE, 140382015143936, 140382015176703, +STORE, 140382015176704, 140382015184895, +STORE, 140382015184896, 140382015188991, +STORE, 140737045585920, 140737045590015, +STORE, 140737045573632, 140737045585919, +STORE, 47250779979776, 47250779987967, +STORE, 47250779987968, 47250779996159, +STORE, 47250779996160, 47250780102655, +STORE, 47250780012544, 47250780102655, +STORE, 47250779996160, 47250780012543, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780086271, +STORE, 47250780086272, 47250780102655, +STORE, 47250780065792, 47250780086271, +STORE, 47250780012544, 47250780065791, +ERASE, 47250780012544, 47250780012544, +STORE, 47250780012544, 47250780065791, +STORE, 47250780082176, 47250780086271, +STORE, 47250780065792, 47250780082175, +ERASE, 47250780065792, 47250780065792, +STORE, 47250780065792, 47250780082175, +STORE, 47250780094464, 47250780102655, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780094463, +ERASE, 47250780094464, 47250780094464, +STORE, 47250780094464, 47250780102655, +STORE, 47250780102656, 47250781941759, +STORE, 47250780241920, 47250781941759, +STORE, 47250780102656, 47250780241919, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781900799, +STORE, 47250781900800, 47250781941759, +STORE, 47250781585408, 47250781900799, +STORE, 47250780241920, 47250781585407, +ERASE, 47250780241920, 47250780241920, +STORE, 47250780241920, 47250781585407, +STORE, 47250781896704, 47250781900799, +STORE, 47250781585408, 47250781896703, +ERASE, 47250781585408, 47250781585408, +STORE, 47250781585408, 47250781896703, +STORE, 47250781925376, 47250781941759, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781925375, +ERASE, 47250781925376, 47250781925376, +STORE, 47250781925376, 47250781941759, +STORE, 47250781925376, 47250781954047, +ERASE, 47250781900800, 47250781900800, +STORE, 47250781900800, 47250781917183, +STORE, 47250781917184, 47250781925375, +ERASE, 47250780086272, 47250780086272, +STORE, 47250780086272, 47250780090367, +STORE, 47250780090368, 47250780094463, +ERASE, 94425324986368, 94425324986368, +STORE, 94425324986368, 94425325002751, +STORE, 94425325002752, 94425325006847, +ERASE, 140382015176704, 140382015176704, +STORE, 140382015176704, 140382015180799, +STORE, 140382015180800, 140382015184895, +ERASE, 47250779979776, 47250779979776, +STORE, 94425351438336, 94425351573503, +STORE, 140737488347136, 140737488351231, +STORE, 140736801144832, 140737488351231, +ERASE, 140736801144832, 140736801144832, +STORE, 140736801144832, 140736801148927, +STORE, 94629429358592, 94629430071295, +ERASE, 94629429358592, 94629429358592, +STORE, 94629429358592, 94629429407743, +STORE, 94629429407744, 94629430071295, +ERASE, 94629429407744, 94629429407744, +STORE, 94629429407744, 94629429952511, +STORE, 94629429952512, 94629430050815, +STORE, 94629430050816, 94629430071295, +STORE, 139801685483520, 139801685655551, +ERASE, 139801685483520, 139801685483520, +STORE, 139801685483520, 139801685487615, +STORE, 139801685487616, 139801685655551, +ERASE, 139801685487616, 139801685487616, +STORE, 139801685487616, 139801685610495, +STORE, 139801685610496, 139801685643263, +STORE, 139801685643264, 139801685651455, +STORE, 139801685651456, 139801685655551, +STORE, 140736801198080, 140736801202175, +STORE, 140736801185792, 140736801198079, +STORE, 47831109513216, 47831109521407, +STORE, 47831109521408, 47831109529599, +STORE, 47831109529600, 47831109636095, +STORE, 47831109545984, 47831109636095, +STORE, 47831109529600, 47831109545983, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109619711, +STORE, 47831109619712, 47831109636095, +STORE, 47831109599232, 47831109619711, +STORE, 47831109545984, 47831109599231, +ERASE, 47831109545984, 47831109545984, +STORE, 47831109545984, 47831109599231, +STORE, 47831109615616, 47831109619711, +STORE, 47831109599232, 47831109615615, +ERASE, 47831109599232, 47831109599232, +STORE, 47831109599232, 47831109615615, +STORE, 47831109627904, 47831109636095, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109627903, +ERASE, 47831109627904, 47831109627904, +STORE, 47831109627904, 47831109636095, +STORE, 47831109636096, 47831111475199, +STORE, 47831109775360, 47831111475199, +STORE, 47831109636096, 47831109775359, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111434239, +STORE, 47831111434240, 47831111475199, +STORE, 47831111118848, 47831111434239, +STORE, 47831109775360, 47831111118847, +ERASE, 47831109775360, 47831109775360, +STORE, 47831109775360, 47831111118847, +STORE, 47831111430144, 47831111434239, +STORE, 47831111118848, 47831111430143, +ERASE, 47831111118848, 47831111118848, +STORE, 47831111118848, 47831111430143, +STORE, 47831111458816, 47831111475199, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111458815, +ERASE, 47831111458816, 47831111458816, +STORE, 47831111458816, 47831111475199, +STORE, 47831111458816, 47831111487487, +ERASE, 47831111434240, 47831111434240, +STORE, 47831111434240, 47831111450623, +STORE, 47831111450624, 47831111458815, +ERASE, 47831109619712, 47831109619712, +STORE, 47831109619712, 47831109623807, +STORE, 47831109623808, 47831109627903, +ERASE, 94629430050816, 94629430050816, +STORE, 94629430050816, 94629430067199, +STORE, 94629430067200, 94629430071295, +ERASE, 139801685643264, 139801685643264, +STORE, 139801685643264, 139801685647359, +STORE, 139801685647360, 139801685651455, +ERASE, 47831109513216, 47831109513216, +STORE, 140737488347136, 140737488351231, +STORE, 140729419612160, 140737488351231, +ERASE, 140729419612160, 140729419612160, +STORE, 140729419612160, 140729419616255, +STORE, 94443354148864, 94443354861567, +ERASE, 94443354148864, 94443354148864, +STORE, 94443354148864, 94443354198015, +STORE, 94443354198016, 94443354861567, +ERASE, 94443354198016, 94443354198016, +STORE, 94443354198016, 94443354742783, +STORE, 94443354742784, 94443354841087, +STORE, 94443354841088, 94443354861567, +STORE, 139741700038656, 139741700210687, +ERASE, 139741700038656, 139741700038656, +STORE, 139741700038656, 139741700042751, +STORE, 139741700042752, 139741700210687, +ERASE, 139741700042752, 139741700042752, +STORE, 139741700042752, 139741700165631, +STORE, 139741700165632, 139741700198399, +STORE, 139741700198400, 139741700206591, +STORE, 139741700206592, 139741700210687, +STORE, 140729420574720, 140729420578815, +STORE, 140729420562432, 140729420574719, +STORE, 47891094958080, 47891094966271, +STORE, 47891094966272, 47891094974463, +STORE, 47891094974464, 47891095080959, +STORE, 47891094990848, 47891095080959, +STORE, 47891094974464, 47891094990847, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095064575, +STORE, 47891095064576, 47891095080959, +STORE, 47891095044096, 47891095064575, +STORE, 47891094990848, 47891095044095, +ERASE, 47891094990848, 47891094990848, +STORE, 47891094990848, 47891095044095, +STORE, 47891095060480, 47891095064575, +STORE, 47891095044096, 47891095060479, +ERASE, 47891095044096, 47891095044096, +STORE, 47891095044096, 47891095060479, +STORE, 47891095072768, 47891095080959, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095072767, +ERASE, 47891095072768, 47891095072768, +STORE, 47891095072768, 47891095080959, +STORE, 47891095080960, 47891096920063, +STORE, 47891095220224, 47891096920063, +STORE, 47891095080960, 47891095220223, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096879103, +STORE, 47891096879104, 47891096920063, +STORE, 47891096563712, 47891096879103, +STORE, 47891095220224, 47891096563711, +ERASE, 47891095220224, 47891095220224, +STORE, 47891095220224, 47891096563711, +STORE, 47891096875008, 47891096879103, +STORE, 47891096563712, 47891096875007, +ERASE, 47891096563712, 47891096563712, +STORE, 47891096563712, 47891096875007, +STORE, 47891096903680, 47891096920063, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096903679, +ERASE, 47891096903680, 47891096903680, +STORE, 47891096903680, 47891096920063, +STORE, 47891096903680, 47891096932351, +ERASE, 47891096879104, 47891096879104, +STORE, 47891096879104, 47891096895487, +STORE, 47891096895488, 47891096903679, +ERASE, 47891095064576, 47891095064576, +STORE, 47891095064576, 47891095068671, +STORE, 47891095068672, 47891095072767, +ERASE, 94443354841088, 94443354841088, +STORE, 94443354841088, 94443354857471, +STORE, 94443354857472, 94443354861567, +ERASE, 139741700198400, 139741700198400, +STORE, 139741700198400, 139741700202495, +STORE, 139741700202496, 139741700206591, +ERASE, 47891094958080, 47891094958080, +STORE, 94443360825344, 94443360960511, +STORE, 140737488347136, 140737488351231, +STORE, 140722961661952, 140737488351231, +ERASE, 140722961661952, 140722961661952, +STORE, 140722961661952, 140722961666047, +STORE, 94878388944896, 94878389657599, +ERASE, 94878388944896, 94878388944896, +STORE, 94878388944896, 94878388994047, +STORE, 94878388994048, 94878389657599, +ERASE, 94878388994048, 94878388994048, +STORE, 94878388994048, 94878389538815, +STORE, 94878389538816, 94878389637119, +STORE, 94878389637120, 94878389657599, +STORE, 140210690056192, 140210690228223, +ERASE, 140210690056192, 140210690056192, +STORE, 140210690056192, 140210690060287, +STORE, 140210690060288, 140210690228223, +ERASE, 140210690060288, 140210690060288, +STORE, 140210690060288, 140210690183167, +STORE, 140210690183168, 140210690215935, +STORE, 140210690215936, 140210690224127, +STORE, 140210690224128, 140210690228223, +STORE, 140722963148800, 140722963152895, +STORE, 140722963136512, 140722963148799, +STORE, 47422104940544, 47422104948735, +STORE, 47422104948736, 47422104956927, +STORE, 47422104956928, 47422105063423, +STORE, 47422104973312, 47422105063423, +STORE, 47422104956928, 47422104973311, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105047039, +STORE, 47422105047040, 47422105063423, +STORE, 47422105026560, 47422105047039, +STORE, 47422104973312, 47422105026559, +ERASE, 47422104973312, 47422104973312, +STORE, 47422104973312, 47422105026559, +STORE, 47422105042944, 47422105047039, +STORE, 47422105026560, 47422105042943, +ERASE, 47422105026560, 47422105026560, +STORE, 47422105026560, 47422105042943, +STORE, 47422105055232, 47422105063423, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105055231, +ERASE, 47422105055232, 47422105055232, +STORE, 47422105055232, 47422105063423, +STORE, 47422105063424, 47422106902527, +STORE, 47422105202688, 47422106902527, +STORE, 47422105063424, 47422105202687, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106861567, +STORE, 47422106861568, 47422106902527, +STORE, 47422106546176, 47422106861567, +STORE, 47422105202688, 47422106546175, +ERASE, 47422105202688, 47422105202688, +STORE, 47422105202688, 47422106546175, +STORE, 47422106857472, 47422106861567, +STORE, 47422106546176, 47422106857471, +ERASE, 47422106546176, 47422106546176, +STORE, 47422106546176, 47422106857471, +STORE, 47422106886144, 47422106902527, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106886143, +ERASE, 47422106886144, 47422106886144, +STORE, 47422106886144, 47422106902527, +STORE, 47422106886144, 47422106914815, +ERASE, 47422106861568, 47422106861568, +STORE, 47422106861568, 47422106877951, +STORE, 47422106877952, 47422106886143, +ERASE, 47422105047040, 47422105047040, +STORE, 47422105047040, 47422105051135, +STORE, 47422105051136, 47422105055231, +ERASE, 94878389637120, 94878389637120, +STORE, 94878389637120, 94878389653503, +STORE, 94878389653504, 94878389657599, +ERASE, 140210690215936, 140210690215936, +STORE, 140210690215936, 140210690220031, +STORE, 140210690220032, 140210690224127, +ERASE, 47422104940544, 47422104940544, +STORE, 140737488347136, 140737488351231, +STORE, 140727690309632, 140737488351231, +ERASE, 140727690309632, 140727690309632, +STORE, 140727690309632, 140727690313727, +STORE, 94121892208640, 94121892921343, +ERASE, 94121892208640, 94121892208640, +STORE, 94121892208640, 94121892257791, +STORE, 94121892257792, 94121892921343, +ERASE, 94121892257792, 94121892257792, +STORE, 94121892257792, 94121892802559, +STORE, 94121892802560, 94121892900863, +STORE, 94121892900864, 94121892921343, +STORE, 140662438326272, 140662438498303, +ERASE, 140662438326272, 140662438326272, +STORE, 140662438326272, 140662438330367, +STORE, 140662438330368, 140662438498303, +ERASE, 140662438330368, 140662438330368, +STORE, 140662438330368, 140662438453247, +STORE, 140662438453248, 140662438486015, +STORE, 140662438486016, 140662438494207, +STORE, 140662438494208, 140662438498303, +STORE, 140727690379264, 140727690383359, +STORE, 140727690366976, 140727690379263, +STORE, 46970356670464, 46970356678655, +STORE, 46970356678656, 46970356686847, +STORE, 46970356686848, 46970356793343, +STORE, 46970356703232, 46970356793343, +STORE, 46970356686848, 46970356703231, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356776959, +STORE, 46970356776960, 46970356793343, +STORE, 46970356756480, 46970356776959, +STORE, 46970356703232, 46970356756479, +ERASE, 46970356703232, 46970356703232, +STORE, 46970356703232, 46970356756479, +STORE, 46970356772864, 46970356776959, +STORE, 46970356756480, 46970356772863, +ERASE, 46970356756480, 46970356756480, +STORE, 46970356756480, 46970356772863, +STORE, 46970356785152, 46970356793343, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356785151, +ERASE, 46970356785152, 46970356785152, +STORE, 46970356785152, 46970356793343, +STORE, 46970356793344, 46970358632447, +STORE, 46970356932608, 46970358632447, +STORE, 46970356793344, 46970356932607, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358591487, +STORE, 46970358591488, 46970358632447, +STORE, 46970358276096, 46970358591487, +STORE, 46970356932608, 46970358276095, +ERASE, 46970356932608, 46970356932608, +STORE, 46970356932608, 46970358276095, +STORE, 46970358587392, 46970358591487, +STORE, 46970358276096, 46970358587391, +ERASE, 46970358276096, 46970358276096, +STORE, 46970358276096, 46970358587391, +STORE, 46970358616064, 46970358632447, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358616063, +ERASE, 46970358616064, 46970358616064, +STORE, 46970358616064, 46970358632447, +STORE, 46970358616064, 46970358644735, +ERASE, 46970358591488, 46970358591488, +STORE, 46970358591488, 46970358607871, +STORE, 46970358607872, 46970358616063, +ERASE, 46970356776960, 46970356776960, +STORE, 46970356776960, 46970356781055, +STORE, 46970356781056, 46970356785151, +ERASE, 94121892900864, 94121892900864, +STORE, 94121892900864, 94121892917247, +STORE, 94121892917248, 94121892921343, +ERASE, 140662438486016, 140662438486016, +STORE, 140662438486016, 140662438490111, +STORE, 140662438490112, 140662438494207, +ERASE, 46970356670464, 46970356670464, +STORE, 94121898610688, 94121898745855, +STORE, 140737488347136, 140737488351231, +STORE, 140737189351424, 140737488351231, +ERASE, 140737189351424, 140737189351424, +STORE, 140737189351424, 140737189355519, +STORE, 93847948832768, 93847949545471, +ERASE, 93847948832768, 93847948832768, +STORE, 93847948832768, 93847948881919, +STORE, 93847948881920, 93847949545471, +ERASE, 93847948881920, 93847948881920, +STORE, 93847948881920, 93847949426687, +STORE, 93847949426688, 93847949524991, +STORE, 93847949524992, 93847949545471, +STORE, 139698989985792, 139698990157823, +ERASE, 139698989985792, 139698989985792, +STORE, 139698989985792, 139698989989887, +STORE, 139698989989888, 139698990157823, +ERASE, 139698989989888, 139698989989888, +STORE, 139698989989888, 139698990112767, +STORE, 139698990112768, 139698990145535, +STORE, 139698990145536, 139698990153727, +STORE, 139698990153728, 139698990157823, +STORE, 140737189744640, 140737189748735, +STORE, 140737189732352, 140737189744639, +STORE, 47933805010944, 47933805019135, +STORE, 47933805019136, 47933805027327, +STORE, 47933805027328, 47933805133823, +STORE, 47933805043712, 47933805133823, +STORE, 47933805027328, 47933805043711, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805117439, +STORE, 47933805117440, 47933805133823, +STORE, 47933805096960, 47933805117439, +STORE, 47933805043712, 47933805096959, +ERASE, 47933805043712, 47933805043712, +STORE, 47933805043712, 47933805096959, +STORE, 47933805113344, 47933805117439, +STORE, 47933805096960, 47933805113343, +ERASE, 47933805096960, 47933805096960, +STORE, 47933805096960, 47933805113343, +STORE, 47933805125632, 47933805133823, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805125631, +ERASE, 47933805125632, 47933805125632, +STORE, 47933805125632, 47933805133823, +STORE, 47933805133824, 47933806972927, +STORE, 47933805273088, 47933806972927, +STORE, 47933805133824, 47933805273087, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806931967, +STORE, 47933806931968, 47933806972927, +STORE, 47933806616576, 47933806931967, +STORE, 47933805273088, 47933806616575, +ERASE, 47933805273088, 47933805273088, +STORE, 47933805273088, 47933806616575, +STORE, 47933806927872, 47933806931967, +STORE, 47933806616576, 47933806927871, +ERASE, 47933806616576, 47933806616576, +STORE, 47933806616576, 47933806927871, +STORE, 47933806956544, 47933806972927, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806956543, +ERASE, 47933806956544, 47933806956544, +STORE, 47933806956544, 47933806972927, +STORE, 47933806956544, 47933806985215, +ERASE, 47933806931968, 47933806931968, +STORE, 47933806931968, 47933806948351, +STORE, 47933806948352, 47933806956543, +ERASE, 47933805117440, 47933805117440, +STORE, 47933805117440, 47933805121535, +STORE, 47933805121536, 47933805125631, +ERASE, 93847949524992, 93847949524992, +STORE, 93847949524992, 93847949541375, +STORE, 93847949541376, 93847949545471, +ERASE, 139698990145536, 139698990145536, +STORE, 139698990145536, 139698990149631, +STORE, 139698990149632, 139698990153727, +ERASE, 47933805010944, 47933805010944, +STORE, 140737488347136, 140737488351231, +STORE, 140725553991680, 140737488351231, +ERASE, 140725553991680, 140725553991680, +STORE, 140725553991680, 140725553995775, +STORE, 93980056248320, 93980056961023, +ERASE, 93980056248320, 93980056248320, +STORE, 93980056248320, 93980056297471, +STORE, 93980056297472, 93980056961023, +ERASE, 93980056297472, 93980056297472, +STORE, 93980056297472, 93980056842239, +STORE, 93980056842240, 93980056940543, +STORE, 93980056940544, 93980056961023, +STORE, 140146588971008, 140146589143039, +ERASE, 140146588971008, 140146588971008, +STORE, 140146588971008, 140146588975103, +STORE, 140146588975104, 140146589143039, +ERASE, 140146588975104, 140146588975104, +STORE, 140146588975104, 140146589097983, +STORE, 140146589097984, 140146589130751, +STORE, 140146589130752, 140146589138943, +STORE, 140146589138944, 140146589143039, +STORE, 140725554860032, 140725554864127, +STORE, 140725554847744, 140725554860031, +STORE, 47486206025728, 47486206033919, +STORE, 47486206033920, 47486206042111, +STORE, 47486206042112, 47486206148607, +STORE, 47486206058496, 47486206148607, +STORE, 47486206042112, 47486206058495, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206132223, +STORE, 47486206132224, 47486206148607, +STORE, 47486206111744, 47486206132223, +STORE, 47486206058496, 47486206111743, +ERASE, 47486206058496, 47486206058496, +STORE, 47486206058496, 47486206111743, +STORE, 47486206128128, 47486206132223, +STORE, 47486206111744, 47486206128127, +ERASE, 47486206111744, 47486206111744, +STORE, 47486206111744, 47486206128127, +STORE, 47486206140416, 47486206148607, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206140415, +ERASE, 47486206140416, 47486206140416, +STORE, 47486206140416, 47486206148607, +STORE, 47486206148608, 47486207987711, +STORE, 47486206287872, 47486207987711, +STORE, 47486206148608, 47486206287871, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207946751, +STORE, 47486207946752, 47486207987711, +STORE, 47486207631360, 47486207946751, +STORE, 47486206287872, 47486207631359, +ERASE, 47486206287872, 47486206287872, +STORE, 47486206287872, 47486207631359, +STORE, 47486207942656, 47486207946751, +STORE, 47486207631360, 47486207942655, +ERASE, 47486207631360, 47486207631360, +STORE, 47486207631360, 47486207942655, +STORE, 47486207971328, 47486207987711, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207971327, +ERASE, 47486207971328, 47486207971328, +STORE, 47486207971328, 47486207987711, +STORE, 47486207971328, 47486207999999, +ERASE, 47486207946752, 47486207946752, +STORE, 47486207946752, 47486207963135, +STORE, 47486207963136, 47486207971327, +ERASE, 47486206132224, 47486206132224, +STORE, 47486206132224, 47486206136319, +STORE, 47486206136320, 47486206140415, +ERASE, 93980056940544, 93980056940544, +STORE, 93980056940544, 93980056956927, +STORE, 93980056956928, 93980056961023, +ERASE, 140146589130752, 140146589130752, +STORE, 140146589130752, 140146589134847, +STORE, 140146589134848, 140146589138943, +ERASE, 47486206025728, 47486206025728, +STORE, 93980070006784, 93980070141951, +STORE, 140737488347136, 140737488351231, +STORE, 140727334776832, 140737488351231, +ERASE, 140727334776832, 140727334776832, +STORE, 140727334776832, 140727334780927, +STORE, 94049747247104, 94049747959807, +ERASE, 94049747247104, 94049747247104, +STORE, 94049747247104, 94049747296255, +STORE, 94049747296256, 94049747959807, +ERASE, 94049747296256, 94049747296256, +STORE, 94049747296256, 94049747841023, +STORE, 94049747841024, 94049747939327, +STORE, 94049747939328, 94049747959807, +STORE, 140227307216896, 140227307388927, +ERASE, 140227307216896, 140227307216896, +STORE, 140227307216896, 140227307220991, +STORE, 140227307220992, 140227307388927, +ERASE, 140227307220992, 140227307220992, +STORE, 140227307220992, 140227307343871, +STORE, 140227307343872, 140227307376639, +STORE, 140227307376640, 140227307384831, +STORE, 140227307384832, 140227307388927, +STORE, 140727335337984, 140727335342079, +STORE, 140727335325696, 140727335337983, +STORE, 47405487779840, 47405487788031, +STORE, 47405487788032, 47405487796223, +STORE, 47405487796224, 47405487902719, +STORE, 47405487812608, 47405487902719, +STORE, 47405487796224, 47405487812607, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487886335, +STORE, 47405487886336, 47405487902719, +STORE, 47405487865856, 47405487886335, +STORE, 47405487812608, 47405487865855, +ERASE, 47405487812608, 47405487812608, +STORE, 47405487812608, 47405487865855, +STORE, 47405487882240, 47405487886335, +STORE, 47405487865856, 47405487882239, +ERASE, 47405487865856, 47405487865856, +STORE, 47405487865856, 47405487882239, +STORE, 47405487894528, 47405487902719, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487894527, +ERASE, 47405487894528, 47405487894528, +STORE, 47405487894528, 47405487902719, +STORE, 47405487902720, 47405489741823, +STORE, 47405488041984, 47405489741823, +STORE, 47405487902720, 47405488041983, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489700863, +STORE, 47405489700864, 47405489741823, +STORE, 47405489385472, 47405489700863, +STORE, 47405488041984, 47405489385471, +ERASE, 47405488041984, 47405488041984, +STORE, 47405488041984, 47405489385471, +STORE, 47405489696768, 47405489700863, +STORE, 47405489385472, 47405489696767, +ERASE, 47405489385472, 47405489385472, +STORE, 47405489385472, 47405489696767, +STORE, 47405489725440, 47405489741823, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489725439, +ERASE, 47405489725440, 47405489725440, +STORE, 47405489725440, 47405489741823, +STORE, 47405489725440, 47405489754111, +ERASE, 47405489700864, 47405489700864, +STORE, 47405489700864, 47405489717247, +STORE, 47405489717248, 47405489725439, +ERASE, 47405487886336, 47405487886336, +STORE, 47405487886336, 47405487890431, +STORE, 47405487890432, 47405487894527, +ERASE, 94049747939328, 94049747939328, +STORE, 94049747939328, 94049747955711, +STORE, 94049747955712, 94049747959807, +ERASE, 140227307376640, 140227307376640, +STORE, 140227307376640, 140227307380735, +STORE, 140227307380736, 140227307384831, +ERASE, 47405487779840, 47405487779840, +STORE, 94049758810112, 94049758945279, +STORE, 140737488347136, 140737488351231, +STORE, 140727079718912, 140737488351231, +ERASE, 140727079718912, 140727079718912, +STORE, 140727079718912, 140727079723007, +STORE, 94250996527104, 94250997239807, +ERASE, 94250996527104, 94250996527104, +STORE, 94250996527104, 94250996576255, +STORE, 94250996576256, 94250997239807, +ERASE, 94250996576256, 94250996576256, +STORE, 94250996576256, 94250997121023, +STORE, 94250997121024, 94250997219327, +STORE, 94250997219328, 94250997239807, +STORE, 140060022587392, 140060022759423, +ERASE, 140060022587392, 140060022587392, +STORE, 140060022587392, 140060022591487, +STORE, 140060022591488, 140060022759423, +ERASE, 140060022591488, 140060022591488, +STORE, 140060022591488, 140060022714367, +STORE, 140060022714368, 140060022747135, +STORE, 140060022747136, 140060022755327, +STORE, 140060022755328, 140060022759423, +STORE, 140727079788544, 140727079792639, +STORE, 140727079776256, 140727079788543, +STORE, 47572772409344, 47572772417535, +STORE, 47572772417536, 47572772425727, +STORE, 47572772425728, 47572772532223, +STORE, 47572772442112, 47572772532223, +STORE, 47572772425728, 47572772442111, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772515839, +STORE, 47572772515840, 47572772532223, +STORE, 47572772495360, 47572772515839, +STORE, 47572772442112, 47572772495359, +ERASE, 47572772442112, 47572772442112, +STORE, 47572772442112, 47572772495359, +STORE, 47572772511744, 47572772515839, +STORE, 47572772495360, 47572772511743, +ERASE, 47572772495360, 47572772495360, +STORE, 47572772495360, 47572772511743, +STORE, 47572772524032, 47572772532223, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772524031, +ERASE, 47572772524032, 47572772524032, +STORE, 47572772524032, 47572772532223, +STORE, 47572772532224, 47572774371327, +STORE, 47572772671488, 47572774371327, +STORE, 47572772532224, 47572772671487, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774330367, +STORE, 47572774330368, 47572774371327, +STORE, 47572774014976, 47572774330367, +STORE, 47572772671488, 47572774014975, +ERASE, 47572772671488, 47572772671488, +STORE, 47572772671488, 47572774014975, +STORE, 47572774326272, 47572774330367, +STORE, 47572774014976, 47572774326271, +ERASE, 47572774014976, 47572774014976, +STORE, 47572774014976, 47572774326271, +STORE, 47572774354944, 47572774371327, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774354943, +ERASE, 47572774354944, 47572774354944, +STORE, 47572774354944, 47572774371327, +STORE, 47572774354944, 47572774383615, +ERASE, 47572774330368, 47572774330368, +STORE, 47572774330368, 47572774346751, +STORE, 47572774346752, 47572774354943, +ERASE, 47572772515840, 47572772515840, +STORE, 47572772515840, 47572772519935, +STORE, 47572772519936, 47572772524031, +ERASE, 94250997219328, 94250997219328, +STORE, 94250997219328, 94250997235711, +STORE, 94250997235712, 94250997239807, +ERASE, 140060022747136, 140060022747136, +STORE, 140060022747136, 140060022751231, +STORE, 140060022751232, 140060022755327, +ERASE, 47572772409344, 47572772409344, +STORE, 94251018305536, 94251018440703, +STORE, 140737488347136, 140737488351231, +STORE, 140730012389376, 140737488351231, +ERASE, 140730012389376, 140730012389376, +STORE, 140730012389376, 140730012393471, +STORE, 94382607675392, 94382607695871, +ERASE, 94382607675392, 94382607675392, +STORE, 94382607675392, 94382607679487, +STORE, 94382607679488, 94382607695871, +ERASE, 94382607679488, 94382607679488, +STORE, 94382607679488, 94382607683583, +STORE, 94382607683584, 94382607687679, +STORE, 94382607687680, 94382607695871, +STORE, 140252451454976, 140252451627007, +ERASE, 140252451454976, 140252451454976, +STORE, 140252451454976, 140252451459071, +STORE, 140252451459072, 140252451627007, +ERASE, 140252451459072, 140252451459072, +STORE, 140252451459072, 140252451581951, +STORE, 140252451581952, 140252451614719, +STORE, 140252451614720, 140252451622911, +STORE, 140252451622912, 140252451627007, +STORE, 140730013548544, 140730013552639, +STORE, 140730013536256, 140730013548543, +STORE, 47380343541760, 47380343549951, +STORE, 47380343549952, 47380343558143, +STORE, 47380343558144, 47380345397247, +STORE, 47380343697408, 47380345397247, +STORE, 47380343558144, 47380343697407, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345356287, +STORE, 47380345356288, 47380345397247, +STORE, 47380345040896, 47380345356287, +STORE, 47380343697408, 47380345040895, +ERASE, 47380343697408, 47380343697408, +STORE, 47380343697408, 47380345040895, +STORE, 47380345352192, 47380345356287, +STORE, 47380345040896, 47380345352191, +ERASE, 47380345040896, 47380345040896, +STORE, 47380345040896, 47380345352191, +STORE, 47380345380864, 47380345397247, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345380863, +ERASE, 47380345380864, 47380345380864, +STORE, 47380345380864, 47380345397247, +ERASE, 47380345356288, 47380345356288, +STORE, 47380345356288, 47380345372671, +STORE, 47380345372672, 47380345380863, +ERASE, 94382607687680, 94382607687680, +STORE, 94382607687680, 94382607691775, +STORE, 94382607691776, 94382607695871, +ERASE, 140252451614720, 140252451614720, +STORE, 140252451614720, 140252451618815, +STORE, 140252451618816, 140252451622911, +ERASE, 47380343541760, 47380343541760, +STORE, 94382626803712, 94382626938879, +STORE, 140737488347136, 140737488351231, +STORE, 140730900271104, 140737488351231, +ERASE, 140730900271104, 140730900271104, +STORE, 140730900271104, 140730900275199, +STORE, 93855478120448, 93855478337535, +ERASE, 93855478120448, 93855478120448, +STORE, 93855478120448, 93855478198271, +STORE, 93855478198272, 93855478337535, +ERASE, 93855478198272, 93855478198272, +STORE, 93855478198272, 93855478243327, +STORE, 93855478243328, 93855478288383, +STORE, 93855478288384, 93855478337535, +STORE, 140092686573568, 140092686745599, +ERASE, 140092686573568, 140092686573568, +STORE, 140092686573568, 140092686577663, +STORE, 140092686577664, 140092686745599, +ERASE, 140092686577664, 140092686577664, +STORE, 140092686577664, 140092686700543, +STORE, 140092686700544, 140092686733311, +STORE, 140092686733312, 140092686741503, +STORE, 140092686741504, 140092686745599, +STORE, 140730900537344, 140730900541439, +STORE, 140730900525056, 140730900537343, +STORE, 47540108423168, 47540108431359, +STORE, 47540108431360, 47540108439551, +STORE, 47540108439552, 47540110278655, +STORE, 47540108578816, 47540110278655, +STORE, 47540108439552, 47540108578815, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540110237695, +STORE, 47540110237696, 47540110278655, +STORE, 47540109922304, 47540110237695, +STORE, 47540108578816, 47540109922303, +ERASE, 47540108578816, 47540108578816, +STORE, 47540108578816, 47540109922303, +STORE, 47540110233600, 47540110237695, +STORE, 47540109922304, 47540110233599, +ERASE, 47540109922304, 47540109922304, +STORE, 47540109922304, 47540110233599, +STORE, 47540110262272, 47540110278655, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110262271, +ERASE, 47540110262272, 47540110262272, +STORE, 47540110262272, 47540110278655, +ERASE, 47540110237696, 47540110237696, +STORE, 47540110237696, 47540110254079, +STORE, 47540110254080, 47540110262271, +ERASE, 93855478288384, 93855478288384, +STORE, 93855478288384, 93855478333439, +STORE, 93855478333440, 93855478337535, +ERASE, 140092686733312, 140092686733312, +STORE, 140092686733312, 140092686737407, +STORE, 140092686737408, 140092686741503, +ERASE, 47540108423168, 47540108423168, +STORE, 93855492222976, 93855492358143, +STORE, 93855492222976, 93855492493311, +STORE, 140737488347136, 140737488351231, +STORE, 140733498146816, 140737488351231, +ERASE, 140733498146816, 140733498146816, +STORE, 140733498146816, 140733498150911, +STORE, 94170739654656, 94170740367359, +ERASE, 94170739654656, 94170739654656, +STORE, 94170739654656, 94170739703807, +STORE, 94170739703808, 94170740367359, +ERASE, 94170739703808, 94170739703808, +STORE, 94170739703808, 94170740248575, +STORE, 94170740248576, 94170740346879, +STORE, 94170740346880, 94170740367359, +STORE, 140024788877312, 140024789049343, +ERASE, 140024788877312, 140024788877312, +STORE, 140024788877312, 140024788881407, +STORE, 140024788881408, 140024789049343, +ERASE, 140024788881408, 140024788881408, +STORE, 140024788881408, 140024789004287, +STORE, 140024789004288, 140024789037055, +STORE, 140024789037056, 140024789045247, +STORE, 140024789045248, 140024789049343, +STORE, 140733499023360, 140733499027455, +STORE, 140733499011072, 140733499023359, +STORE, 47608006119424, 47608006127615, +STORE, 47608006127616, 47608006135807, +STORE, 47608006135808, 47608006242303, +STORE, 47608006152192, 47608006242303, +STORE, 47608006135808, 47608006152191, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006225919, +STORE, 47608006225920, 47608006242303, +STORE, 47608006205440, 47608006225919, +STORE, 47608006152192, 47608006205439, +ERASE, 47608006152192, 47608006152192, +STORE, 47608006152192, 47608006205439, +STORE, 47608006221824, 47608006225919, +STORE, 47608006205440, 47608006221823, +ERASE, 47608006205440, 47608006205440, +STORE, 47608006205440, 47608006221823, +STORE, 47608006234112, 47608006242303, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006234111, +ERASE, 47608006234112, 47608006234112, +STORE, 47608006234112, 47608006242303, +STORE, 47608006242304, 47608008081407, +STORE, 47608006381568, 47608008081407, +STORE, 47608006242304, 47608006381567, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608008040447, +STORE, 47608008040448, 47608008081407, +STORE, 47608007725056, 47608008040447, +STORE, 47608006381568, 47608007725055, +ERASE, 47608006381568, 47608006381568, +STORE, 47608006381568, 47608007725055, +STORE, 47608008036352, 47608008040447, +STORE, 47608007725056, 47608008036351, +ERASE, 47608007725056, 47608007725056, +STORE, 47608007725056, 47608008036351, +STORE, 47608008065024, 47608008081407, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008065023, +ERASE, 47608008065024, 47608008065024, +STORE, 47608008065024, 47608008081407, +STORE, 47608008065024, 47608008093695, +ERASE, 47608008040448, 47608008040448, +STORE, 47608008040448, 47608008056831, +STORE, 47608008056832, 47608008065023, +ERASE, 47608006225920, 47608006225920, +STORE, 47608006225920, 47608006230015, +STORE, 47608006230016, 47608006234111, +ERASE, 94170740346880, 94170740346880, +STORE, 94170740346880, 94170740363263, +STORE, 94170740363264, 94170740367359, +ERASE, 140024789037056, 140024789037056, +STORE, 140024789037056, 140024789041151, +STORE, 140024789041152, 140024789045247, +ERASE, 47608006119424, 47608006119424, +STORE, 140737488347136, 140737488351231, +STORE, 140730264326144, 140737488351231, +ERASE, 140730264326144, 140730264326144, +STORE, 140730264326144, 140730264330239, +STORE, 94653216407552, 94653217120255, +ERASE, 94653216407552, 94653216407552, +STORE, 94653216407552, 94653216456703, +STORE, 94653216456704, 94653217120255, +ERASE, 94653216456704, 94653216456704, +STORE, 94653216456704, 94653217001471, +STORE, 94653217001472, 94653217099775, +STORE, 94653217099776, 94653217120255, +STORE, 140103617011712, 140103617183743, +ERASE, 140103617011712, 140103617011712, +STORE, 140103617011712, 140103617015807, +STORE, 140103617015808, 140103617183743, +ERASE, 140103617015808, 140103617015808, +STORE, 140103617015808, 140103617138687, +STORE, 140103617138688, 140103617171455, +STORE, 140103617171456, 140103617179647, +STORE, 140103617179648, 140103617183743, +STORE, 140730265427968, 140730265432063, +STORE, 140730265415680, 140730265427967, +STORE, 47529177985024, 47529177993215, +STORE, 47529177993216, 47529178001407, +STORE, 47529178001408, 47529178107903, +STORE, 47529178017792, 47529178107903, +STORE, 47529178001408, 47529178017791, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178091519, +STORE, 47529178091520, 47529178107903, +STORE, 47529178071040, 47529178091519, +STORE, 47529178017792, 47529178071039, +ERASE, 47529178017792, 47529178017792, +STORE, 47529178017792, 47529178071039, +STORE, 47529178087424, 47529178091519, +STORE, 47529178071040, 47529178087423, +ERASE, 47529178071040, 47529178071040, +STORE, 47529178071040, 47529178087423, +STORE, 47529178099712, 47529178107903, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178099711, +ERASE, 47529178099712, 47529178099712, +STORE, 47529178099712, 47529178107903, +STORE, 47529178107904, 47529179947007, +STORE, 47529178247168, 47529179947007, +STORE, 47529178107904, 47529178247167, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179906047, +STORE, 47529179906048, 47529179947007, +STORE, 47529179590656, 47529179906047, +STORE, 47529178247168, 47529179590655, +ERASE, 47529178247168, 47529178247168, +STORE, 47529178247168, 47529179590655, +STORE, 47529179901952, 47529179906047, +STORE, 47529179590656, 47529179901951, +ERASE, 47529179590656, 47529179590656, +STORE, 47529179590656, 47529179901951, +STORE, 47529179930624, 47529179947007, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179930623, +ERASE, 47529179930624, 47529179930624, +STORE, 47529179930624, 47529179947007, +STORE, 47529179930624, 47529179959295, +ERASE, 47529179906048, 47529179906048, +STORE, 47529179906048, 47529179922431, +STORE, 47529179922432, 47529179930623, +ERASE, 47529178091520, 47529178091520, +STORE, 47529178091520, 47529178095615, +STORE, 47529178095616, 47529178099711, +ERASE, 94653217099776, 94653217099776, +STORE, 94653217099776, 94653217116159, +STORE, 94653217116160, 94653217120255, +ERASE, 140103617171456, 140103617171456, +STORE, 140103617171456, 140103617175551, +STORE, 140103617175552, 140103617179647, +ERASE, 47529177985024, 47529177985024, +STORE, 94653241135104, 94653241270271, +STORE, 140737488347136, 140737488351231, +STORE, 140736284549120, 140737488351231, +ERASE, 140736284549120, 140736284549120, +STORE, 140736284549120, 140736284553215, +STORE, 93963663822848, 93963664506879, +ERASE, 93963663822848, 93963663822848, +STORE, 93963663822848, 93963663884287, +STORE, 93963663884288, 93963664506879, +ERASE, 93963663884288, 93963663884288, +STORE, 93963663884288, 93963664240639, +STORE, 93963664240640, 93963664379903, +STORE, 93963664379904, 93963664506879, +STORE, 140450188439552, 140450188611583, +ERASE, 140450188439552, 140450188439552, +STORE, 140450188439552, 140450188443647, +STORE, 140450188443648, 140450188611583, +ERASE, 140450188443648, 140450188443648, +STORE, 140450188443648, 140450188566527, +STORE, 140450188566528, 140450188599295, +STORE, 140450188599296, 140450188607487, +STORE, 140450188607488, 140450188611583, +STORE, 140736284577792, 140736284581887, +STORE, 140736284565504, 140736284577791, +STORE, 47182606557184, 47182606565375, +STORE, 47182606565376, 47182606573567, +STORE, 47182606573568, 47182608412671, +STORE, 47182606712832, 47182608412671, +STORE, 47182606573568, 47182606712831, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608371711, +STORE, 47182608371712, 47182608412671, +STORE, 47182608056320, 47182608371711, +STORE, 47182606712832, 47182608056319, +ERASE, 47182606712832, 47182606712832, +STORE, 47182606712832, 47182608056319, +STORE, 47182608367616, 47182608371711, +STORE, 47182608056320, 47182608367615, +ERASE, 47182608056320, 47182608056320, +STORE, 47182608056320, 47182608367615, +STORE, 47182608396288, 47182608412671, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608396287, +ERASE, 47182608396288, 47182608396288, +STORE, 47182608396288, 47182608412671, +STORE, 47182608412672, 47182608523263, +STORE, 47182608429056, 47182608523263, +STORE, 47182608412672, 47182608429055, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608515071, +STORE, 47182608515072, 47182608523263, +STORE, 47182608490496, 47182608515071, +STORE, 47182608429056, 47182608490495, +ERASE, 47182608429056, 47182608429056, +STORE, 47182608429056, 47182608490495, +STORE, 47182608510976, 47182608515071, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608490496, 47182608490496, +STORE, 47182608490496, 47182608510975, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608523263, +STORE, 47182608523264, 47182608568319, +ERASE, 47182608523264, 47182608523264, +STORE, 47182608523264, 47182608531455, +STORE, 47182608531456, 47182608568319, +STORE, 47182608551936, 47182608568319, +STORE, 47182608531456, 47182608551935, +ERASE, 47182608531456, 47182608531456, +STORE, 47182608531456, 47182608551935, +STORE, 47182608560128, 47182608568319, +STORE, 47182608551936, 47182608560127, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608568319, +ERASE, 47182608551936, 47182608551936, +STORE, 47182608551936, 47182608560127, +STORE, 47182608560128, 47182608568319, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608568319, +STORE, 47182608568320, 47182608916479, +STORE, 47182608609280, 47182608916479, +STORE, 47182608568320, 47182608609279, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608891903, +STORE, 47182608891904, 47182608916479, +STORE, 47182608822272, 47182608891903, +STORE, 47182608609280, 47182608822271, +ERASE, 47182608609280, 47182608609280, +STORE, 47182608609280, 47182608822271, +STORE, 47182608887808, 47182608891903, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608822272, 47182608822272, +STORE, 47182608822272, 47182608887807, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608916479, +STORE, 47182608916480, 47182611177471, +STORE, 47182609068032, 47182611177471, +STORE, 47182608916480, 47182609068031, +ERASE, 47182609068032, 47182609068032, +STORE, 47182609068032, 47182611161087, +STORE, 47182611161088, 47182611177471, +STORE, 47182611169280, 47182611177471, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611169279, +ERASE, 47182611169280, 47182611169280, +STORE, 47182611169280, 47182611177471, +STORE, 47182611177472, 47182611312639, +ERASE, 47182611177472, 47182611177472, +STORE, 47182611177472, 47182611202047, +STORE, 47182611202048, 47182611312639, +STORE, 47182611263488, 47182611312639, +STORE, 47182611202048, 47182611263487, +ERASE, 47182611202048, 47182611202048, +STORE, 47182611202048, 47182611263487, +STORE, 47182611288064, 47182611312639, +STORE, 47182611263488, 47182611288063, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611312639, +ERASE, 47182611263488, 47182611263488, +STORE, 47182611263488, 47182611288063, +STORE, 47182611288064, 47182611312639, +STORE, 47182611296256, 47182611312639, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611296255, +ERASE, 47182611296256, 47182611296256, +STORE, 47182611296256, 47182611312639, +STORE, 47182611296256, 47182611320831, +STORE, 47182611320832, 47182611484671, +ERASE, 47182611320832, 47182611320832, +STORE, 47182611320832, 47182611333119, +STORE, 47182611333120, 47182611484671, +STORE, 47182611431424, 47182611484671, +STORE, 47182611333120, 47182611431423, +ERASE, 47182611333120, 47182611333120, +STORE, 47182611333120, 47182611431423, +STORE, 47182611476480, 47182611484671, +STORE, 47182611431424, 47182611476479, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611484671, +ERASE, 47182611431424, 47182611431424, +STORE, 47182611431424, 47182611476479, +STORE, 47182611476480, 47182611484671, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611484671, +STORE, 47182611484672, 47182612082687, +STORE, 47182611603456, 47182612082687, +STORE, 47182611484672, 47182611603455, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182612029439, +STORE, 47182612029440, 47182612082687, +STORE, 47182611918848, 47182612029439, +STORE, 47182611603456, 47182611918847, +ERASE, 47182611603456, 47182611603456, +STORE, 47182611603456, 47182611918847, +STORE, 47182612025344, 47182612029439, +STORE, 47182611918848, 47182612025343, +ERASE, 47182611918848, 47182611918848, +STORE, 47182611918848, 47182612025343, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612082687, +STORE, 47182612082688, 47182615134207, +STORE, 47182612627456, 47182615134207, +STORE, 47182612082688, 47182612627455, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614913023, +STORE, 47182614913024, 47182615134207, +STORE, 47182614323200, 47182614913023, +STORE, 47182612627456, 47182614323199, +ERASE, 47182612627456, 47182612627456, +STORE, 47182612627456, 47182614323199, +STORE, 47182614908928, 47182614913023, +STORE, 47182614323200, 47182614908927, +ERASE, 47182614323200, 47182614323200, +STORE, 47182614323200, 47182614908927, +STORE, 47182615117824, 47182615134207, +STORE, 47182614913024, 47182615117823, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615117823, +ERASE, 47182615117824, 47182615117824, +STORE, 47182615117824, 47182615134207, +STORE, 47182615134208, 47182615166975, +ERASE, 47182615134208, 47182615134208, +STORE, 47182615134208, 47182615142399, +STORE, 47182615142400, 47182615166975, +STORE, 47182615154688, 47182615166975, +STORE, 47182615142400, 47182615154687, +ERASE, 47182615142400, 47182615142400, +STORE, 47182615142400, 47182615154687, +STORE, 47182615158784, 47182615166975, +STORE, 47182615154688, 47182615158783, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615166975, +ERASE, 47182615154688, 47182615154688, +STORE, 47182615154688, 47182615158783, +STORE, 47182615158784, 47182615166975, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615166975, +STORE, 47182615166976, 47182615203839, +ERASE, 47182615166976, 47182615166976, +STORE, 47182615166976, 47182615175167, +STORE, 47182615175168, 47182615203839, +STORE, 47182615191552, 47182615203839, +STORE, 47182615175168, 47182615191551, +ERASE, 47182615175168, 47182615175168, +STORE, 47182615175168, 47182615191551, +STORE, 47182615195648, 47182615203839, +STORE, 47182615191552, 47182615195647, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615203839, +ERASE, 47182615191552, 47182615191552, +STORE, 47182615191552, 47182615195647, +STORE, 47182615195648, 47182615203839, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615203839, +STORE, 47182615203840, 47182615678975, +ERASE, 47182615203840, 47182615203840, +STORE, 47182615203840, 47182615212031, +STORE, 47182615212032, 47182615678975, +STORE, 47182615547904, 47182615678975, +STORE, 47182615212032, 47182615547903, +ERASE, 47182615212032, 47182615212032, +STORE, 47182615212032, 47182615547903, +STORE, 47182615670784, 47182615678975, +STORE, 47182615547904, 47182615670783, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615678975, +ERASE, 47182615547904, 47182615547904, +STORE, 47182615547904, 47182615670783, +STORE, 47182615670784, 47182615678975, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615678975, +STORE, 47182615678976, 47182615687167, +STORE, 47182615687168, 47182615707647, +ERASE, 47182615687168, 47182615687168, +STORE, 47182615687168, 47182615691263, +STORE, 47182615691264, 47182615707647, +STORE, 47182615695360, 47182615707647, +STORE, 47182615691264, 47182615695359, +ERASE, 47182615691264, 47182615691264, +STORE, 47182615691264, 47182615695359, +STORE, 47182615699456, 47182615707647, +STORE, 47182615695360, 47182615699455, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615707647, +ERASE, 47182615695360, 47182615695360, +STORE, 47182615695360, 47182615699455, +STORE, 47182615699456, 47182615707647, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615707647, +STORE, 47182615707648, 47182615715839, +ERASE, 47182608371712, 47182608371712, +STORE, 47182608371712, 47182608388095, +STORE, 47182608388096, 47182608396287, +ERASE, 47182615699456, 47182615699456, +STORE, 47182615699456, 47182615703551, +STORE, 47182615703552, 47182615707647, +ERASE, 47182611288064, 47182611288064, +STORE, 47182611288064, 47182611292159, +STORE, 47182611292160, 47182611296255, +ERASE, 47182615670784, 47182615670784, +STORE, 47182615670784, 47182615674879, +STORE, 47182615674880, 47182615678975, +ERASE, 47182615195648, 47182615195648, +STORE, 47182615195648, 47182615199743, +STORE, 47182615199744, 47182615203839, +ERASE, 47182615158784, 47182615158784, +STORE, 47182615158784, 47182615162879, +STORE, 47182615162880, 47182615166975, +ERASE, 47182614913024, 47182614913024, +STORE, 47182614913024, 47182615109631, +STORE, 47182615109632, 47182615117823, +ERASE, 47182612029440, 47182612029440, +STORE, 47182612029440, 47182612066303, +STORE, 47182612066304, 47182612082687, +ERASE, 47182611476480, 47182611476480, +STORE, 47182611476480, 47182611480575, +STORE, 47182611480576, 47182611484671, +ERASE, 47182611161088, 47182611161088, +STORE, 47182611161088, 47182611165183, +STORE, 47182611165184, 47182611169279, +ERASE, 47182608891904, 47182608891904, +STORE, 47182608891904, 47182608912383, +STORE, 47182608912384, 47182608916479, +ERASE, 47182608560128, 47182608560128, +STORE, 47182608560128, 47182608564223, +STORE, 47182608564224, 47182608568319, +ERASE, 47182608515072, 47182608515072, +STORE, 47182608515072, 47182608519167, +STORE, 47182608519168, 47182608523263, +ERASE, 93963664379904, 93963664379904, +STORE, 93963664379904, 93963664502783, +STORE, 93963664502784, 93963664506879, +ERASE, 140450188599296, 140450188599296, +STORE, 140450188599296, 140450188603391, +STORE, 140450188603392, 140450188607487, +ERASE, 47182606557184, 47182606557184, +STORE, 93963694723072, 93963694858239, +STORE, 140737488347136, 140737488351231, +STORE, 140730313261056, 140737488351231, +ERASE, 140730313261056, 140730313261056, +STORE, 140730313261056, 140730313265151, +STORE, 94386579017728, 94386579697663, +ERASE, 94386579017728, 94386579017728, +STORE, 94386579017728, 94386579083263, +STORE, 94386579083264, 94386579697663, +ERASE, 94386579083264, 94386579083264, +STORE, 94386579083264, 94386579431423, +STORE, 94386579431424, 94386579570687, +STORE, 94386579570688, 94386579697663, +STORE, 140124810838016, 140124811010047, +ERASE, 140124810838016, 140124810838016, +STORE, 140124810838016, 140124810842111, +STORE, 140124810842112, 140124811010047, +ERASE, 140124810842112, 140124810842112, +STORE, 140124810842112, 140124810964991, +STORE, 140124810964992, 140124810997759, +STORE, 140124810997760, 140124811005951, +STORE, 140124811005952, 140124811010047, +STORE, 140730313601024, 140730313605119, +STORE, 140730313588736, 140730313601023, +STORE, 47507984158720, 47507984166911, +STORE, 47507984166912, 47507984175103, +STORE, 47507984175104, 47507986014207, +STORE, 47507984314368, 47507986014207, +STORE, 47507984175104, 47507984314367, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985973247, +STORE, 47507985973248, 47507986014207, +STORE, 47507985657856, 47507985973247, +STORE, 47507984314368, 47507985657855, +ERASE, 47507984314368, 47507984314368, +STORE, 47507984314368, 47507985657855, +STORE, 47507985969152, 47507985973247, +STORE, 47507985657856, 47507985969151, +ERASE, 47507985657856, 47507985657856, +STORE, 47507985657856, 47507985969151, +STORE, 47507985997824, 47507986014207, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985997823, +ERASE, 47507985997824, 47507985997824, +STORE, 47507985997824, 47507986014207, +STORE, 47507986014208, 47507986124799, +STORE, 47507986030592, 47507986124799, +STORE, 47507986014208, 47507986030591, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986116607, +STORE, 47507986116608, 47507986124799, +STORE, 47507986092032, 47507986116607, +STORE, 47507986030592, 47507986092031, +ERASE, 47507986030592, 47507986030592, +STORE, 47507986030592, 47507986092031, +STORE, 47507986112512, 47507986116607, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986092032, 47507986092032, +STORE, 47507986092032, 47507986112511, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986124799, +STORE, 47507986124800, 47507986169855, +ERASE, 47507986124800, 47507986124800, +STORE, 47507986124800, 47507986132991, +STORE, 47507986132992, 47507986169855, +STORE, 47507986153472, 47507986169855, +STORE, 47507986132992, 47507986153471, +ERASE, 47507986132992, 47507986132992, +STORE, 47507986132992, 47507986153471, +STORE, 47507986161664, 47507986169855, +STORE, 47507986153472, 47507986161663, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986169855, +ERASE, 47507986153472, 47507986153472, +STORE, 47507986153472, 47507986161663, +STORE, 47507986161664, 47507986169855, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986169855, +STORE, 47507986169856, 47507986518015, +STORE, 47507986210816, 47507986518015, +STORE, 47507986169856, 47507986210815, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986493439, +STORE, 47507986493440, 47507986518015, +STORE, 47507986423808, 47507986493439, +STORE, 47507986210816, 47507986423807, +ERASE, 47507986210816, 47507986210816, +STORE, 47507986210816, 47507986423807, +STORE, 47507986489344, 47507986493439, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986423808, 47507986423808, +STORE, 47507986423808, 47507986489343, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986518015, +STORE, 47507986518016, 47507988779007, +STORE, 47507986669568, 47507988779007, +STORE, 47507986518016, 47507986669567, +ERASE, 47507986669568, 47507986669568, +STORE, 47507986669568, 47507988762623, +STORE, 47507988762624, 47507988779007, +STORE, 47507988770816, 47507988779007, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988770815, +ERASE, 47507988770816, 47507988770816, +STORE, 47507988770816, 47507988779007, +STORE, 47507988779008, 47507988914175, +ERASE, 47507988779008, 47507988779008, +STORE, 47507988779008, 47507988803583, +STORE, 47507988803584, 47507988914175, +STORE, 47507988865024, 47507988914175, +STORE, 47507988803584, 47507988865023, +ERASE, 47507988803584, 47507988803584, +STORE, 47507988803584, 47507988865023, +STORE, 47507988889600, 47507988914175, +STORE, 47507988865024, 47507988889599, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988914175, +ERASE, 47507988865024, 47507988865024, +STORE, 47507988865024, 47507988889599, +STORE, 47507988889600, 47507988914175, +STORE, 47507988897792, 47507988914175, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988897791, +ERASE, 47507988897792, 47507988897792, +STORE, 47507988897792, 47507988914175, +STORE, 47507988897792, 47507988922367, +STORE, 47507988922368, 47507989086207, +ERASE, 47507988922368, 47507988922368, +STORE, 47507988922368, 47507988934655, +STORE, 47507988934656, 47507989086207, +STORE, 47507989032960, 47507989086207, +STORE, 47507988934656, 47507989032959, +ERASE, 47507988934656, 47507988934656, +STORE, 47507988934656, 47507989032959, +STORE, 47507989078016, 47507989086207, +STORE, 47507989032960, 47507989078015, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989086207, +ERASE, 47507989032960, 47507989032960, +STORE, 47507989032960, 47507989078015, +STORE, 47507989078016, 47507989086207, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989086207, +STORE, 47507989086208, 47507989684223, +STORE, 47507989204992, 47507989684223, +STORE, 47507989086208, 47507989204991, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989630975, +STORE, 47507989630976, 47507989684223, +STORE, 47507989520384, 47507989630975, +STORE, 47507989204992, 47507989520383, +ERASE, 47507989204992, 47507989204992, +STORE, 47507989204992, 47507989520383, +STORE, 47507989626880, 47507989630975, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989520384, 47507989520384, +STORE, 47507989520384, 47507989626879, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989684223, +STORE, 47507989684224, 47507992735743, +STORE, 47507990228992, 47507992735743, +STORE, 47507989684224, 47507990228991, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507992514559, +STORE, 47507992514560, 47507992735743, +STORE, 47507991924736, 47507992514559, +STORE, 47507990228992, 47507991924735, +ERASE, 47507990228992, 47507990228992, +STORE, 47507990228992, 47507991924735, +STORE, 47507992510464, 47507992514559, +STORE, 47507991924736, 47507992510463, +ERASE, 47507991924736, 47507991924736, +STORE, 47507991924736, 47507992510463, +STORE, 47507992719360, 47507992735743, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992719359, +ERASE, 47507992719360, 47507992719360, +STORE, 47507992719360, 47507992735743, +STORE, 47507992735744, 47507992768511, +ERASE, 47507992735744, 47507992735744, +STORE, 47507992735744, 47507992743935, +STORE, 47507992743936, 47507992768511, +STORE, 47507992756224, 47507992768511, +STORE, 47507992743936, 47507992756223, +ERASE, 47507992743936, 47507992743936, +STORE, 47507992743936, 47507992756223, +STORE, 47507992760320, 47507992768511, +STORE, 47507992756224, 47507992760319, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992768511, +ERASE, 47507992756224, 47507992756224, +STORE, 47507992756224, 47507992760319, +STORE, 47507992760320, 47507992768511, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992768511, +STORE, 47507992768512, 47507992805375, +ERASE, 47507992768512, 47507992768512, +STORE, 47507992768512, 47507992776703, +STORE, 47507992776704, 47507992805375, +STORE, 47507992793088, 47507992805375, +STORE, 47507992776704, 47507992793087, +ERASE, 47507992776704, 47507992776704, +STORE, 47507992776704, 47507992793087, +STORE, 47507992797184, 47507992805375, +STORE, 47507992793088, 47507992797183, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992805375, +ERASE, 47507992793088, 47507992793088, +STORE, 47507992793088, 47507992797183, +STORE, 47507992797184, 47507992805375, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992805375, +STORE, 47507992805376, 47507993280511, +ERASE, 47507992805376, 47507992805376, +STORE, 47507992805376, 47507992813567, +STORE, 47507992813568, 47507993280511, +STORE, 47507993149440, 47507993280511, +STORE, 47507992813568, 47507993149439, +ERASE, 47507992813568, 47507992813568, +STORE, 47507992813568, 47507993149439, +STORE, 47507993272320, 47507993280511, +STORE, 47507993149440, 47507993272319, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993280511, +ERASE, 47507993149440, 47507993149440, +STORE, 47507993149440, 47507993272319, +STORE, 47507993272320, 47507993280511, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993280511, +STORE, 47507993280512, 47507993288703, +STORE, 47507993288704, 47507993309183, +ERASE, 47507993288704, 47507993288704, +STORE, 47507993288704, 47507993292799, +STORE, 47507993292800, 47507993309183, +STORE, 47507993296896, 47507993309183, +STORE, 47507993292800, 47507993296895, +ERASE, 47507993292800, 47507993292800, +STORE, 47507993292800, 47507993296895, +STORE, 47507993300992, 47507993309183, +STORE, 47507993296896, 47507993300991, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993309183, +ERASE, 47507993296896, 47507993296896, +STORE, 47507993296896, 47507993300991, +STORE, 47507993300992, 47507993309183, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993309183, +STORE, 47507993309184, 47507993317375, +ERASE, 47507985973248, 47507985973248, +STORE, 47507985973248, 47507985989631, +STORE, 47507985989632, 47507985997823, +ERASE, 47507993300992, 47507993300992, +STORE, 47507993300992, 47507993305087, +STORE, 47507993305088, 47507993309183, +ERASE, 47507988889600, 47507988889600, +STORE, 47507988889600, 47507988893695, +STORE, 47507988893696, 47507988897791, +ERASE, 47507993272320, 47507993272320, +STORE, 47507993272320, 47507993276415, +STORE, 47507993276416, 47507993280511, +ERASE, 47507992797184, 47507992797184, +STORE, 47507992797184, 47507992801279, +STORE, 47507992801280, 47507992805375, +ERASE, 47507992760320, 47507992760320, +STORE, 47507992760320, 47507992764415, +STORE, 47507992764416, 47507992768511, +ERASE, 47507992514560, 47507992514560, +STORE, 47507992514560, 47507992711167, +STORE, 47507992711168, 47507992719359, +ERASE, 47507989630976, 47507989630976, +STORE, 47507989630976, 47507989667839, +STORE, 47507989667840, 47507989684223, +ERASE, 47507989078016, 47507989078016, +STORE, 47507989078016, 47507989082111, +STORE, 47507989082112, 47507989086207, +ERASE, 47507988762624, 47507988762624, +STORE, 47507988762624, 47507988766719, +STORE, 47507988766720, 47507988770815, +ERASE, 47507986493440, 47507986493440, +STORE, 47507986493440, 47507986513919, +STORE, 47507986513920, 47507986518015, +ERASE, 47507986161664, 47507986161664, +STORE, 47507986161664, 47507986165759, +STORE, 47507986165760, 47507986169855, +ERASE, 47507986116608, 47507986116608, +STORE, 47507986116608, 47507986120703, +STORE, 47507986120704, 47507986124799, +ERASE, 94386579570688, 94386579570688, +STORE, 94386579570688, 94386579693567, +STORE, 94386579693568, 94386579697663, +ERASE, 140124810997760, 140124810997760, +STORE, 140124810997760, 140124811001855, +STORE, 140124811001856, 140124811005951, +ERASE, 47507984158720, 47507984158720, +STORE, 94386583982080, 94386584117247, +STORE, 94386583982080, 94386584256511, +ERASE, 94386583982080, 94386583982080, +STORE, 94386583982080, 94386584223743, +STORE, 94386584223744, 94386584256511, +ERASE, 94386584223744, 94386584223744, +STORE, 140737488347136, 140737488351231, +STORE, 140733763395584, 140737488351231, +ERASE, 140733763395584, 140733763395584, +STORE, 140733763395584, 140733763399679, +STORE, 94011546472448, 94011547152383, +ERASE, 94011546472448, 94011546472448, +STORE, 94011546472448, 94011546537983, +STORE, 94011546537984, 94011547152383, +ERASE, 94011546537984, 94011546537984, +STORE, 94011546537984, 94011546886143, +STORE, 94011546886144, 94011547025407, +STORE, 94011547025408, 94011547152383, +STORE, 139757597949952, 139757598121983, +ERASE, 139757597949952, 139757597949952, +STORE, 139757597949952, 139757597954047, +STORE, 139757597954048, 139757598121983, +ERASE, 139757597954048, 139757597954048, +STORE, 139757597954048, 139757598076927, +STORE, 139757598076928, 139757598109695, +STORE, 139757598109696, 139757598117887, +STORE, 139757598117888, 139757598121983, +STORE, 140733763596288, 140733763600383, +STORE, 140733763584000, 140733763596287, +STORE, 47875197046784, 47875197054975, +STORE, 47875197054976, 47875197063167, +STORE, 47875197063168, 47875198902271, +STORE, 47875197202432, 47875198902271, +STORE, 47875197063168, 47875197202431, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198861311, +STORE, 47875198861312, 47875198902271, +STORE, 47875198545920, 47875198861311, +STORE, 47875197202432, 47875198545919, +ERASE, 47875197202432, 47875197202432, +STORE, 47875197202432, 47875198545919, +STORE, 47875198857216, 47875198861311, +STORE, 47875198545920, 47875198857215, +ERASE, 47875198545920, 47875198545920, +STORE, 47875198545920, 47875198857215, +STORE, 47875198885888, 47875198902271, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198885887, +ERASE, 47875198885888, 47875198885888, +STORE, 47875198885888, 47875198902271, +STORE, 47875198902272, 47875199012863, +STORE, 47875198918656, 47875199012863, +STORE, 47875198902272, 47875198918655, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875199004671, +STORE, 47875199004672, 47875199012863, +STORE, 47875198980096, 47875199004671, +STORE, 47875198918656, 47875198980095, +ERASE, 47875198918656, 47875198918656, +STORE, 47875198918656, 47875198980095, +STORE, 47875199000576, 47875199004671, +STORE, 47875198980096, 47875199000575, +ERASE, 47875198980096, 47875198980096, +STORE, 47875198980096, 47875199000575, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199012863, +STORE, 47875199012864, 47875199057919, +ERASE, 47875199012864, 47875199012864, +STORE, 47875199012864, 47875199021055, +STORE, 47875199021056, 47875199057919, +STORE, 47875199041536, 47875199057919, +STORE, 47875199021056, 47875199041535, +ERASE, 47875199021056, 47875199021056, +STORE, 47875199021056, 47875199041535, +STORE, 47875199049728, 47875199057919, +STORE, 47875199041536, 47875199049727, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199057919, +ERASE, 47875199041536, 47875199041536, +STORE, 47875199041536, 47875199049727, +STORE, 47875199049728, 47875199057919, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199057919, +STORE, 47875199057920, 47875199406079, +STORE, 47875199098880, 47875199406079, +STORE, 47875199057920, 47875199098879, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199381503, +STORE, 47875199381504, 47875199406079, +STORE, 47875199311872, 47875199381503, +STORE, 47875199098880, 47875199311871, +ERASE, 47875199098880, 47875199098880, +STORE, 47875199098880, 47875199311871, +STORE, 47875199377408, 47875199381503, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199311872, 47875199311872, +STORE, 47875199311872, 47875199377407, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199406079, +STORE, 47875199406080, 47875201667071, +STORE, 47875199557632, 47875201667071, +STORE, 47875199406080, 47875199557631, +ERASE, 47875199557632, 47875199557632, +STORE, 47875199557632, 47875201650687, +STORE, 47875201650688, 47875201667071, +STORE, 47875201658880, 47875201667071, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201658879, +ERASE, 47875201658880, 47875201658880, +STORE, 47875201658880, 47875201667071, +STORE, 47875201667072, 47875201802239, +ERASE, 47875201667072, 47875201667072, +STORE, 47875201667072, 47875201691647, +STORE, 47875201691648, 47875201802239, +STORE, 47875201753088, 47875201802239, +STORE, 47875201691648, 47875201753087, +ERASE, 47875201691648, 47875201691648, +STORE, 47875201691648, 47875201753087, +STORE, 47875201777664, 47875201802239, +STORE, 47875201753088, 47875201777663, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201802239, +ERASE, 47875201753088, 47875201753088, +STORE, 47875201753088, 47875201777663, +STORE, 47875201777664, 47875201802239, +STORE, 47875201785856, 47875201802239, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201785855, +ERASE, 47875201785856, 47875201785856, +STORE, 47875201785856, 47875201802239, +STORE, 47875201785856, 47875201810431, +STORE, 47875201810432, 47875201974271, +ERASE, 47875201810432, 47875201810432, +STORE, 47875201810432, 47875201822719, +STORE, 47875201822720, 47875201974271, +STORE, 47875201921024, 47875201974271, +STORE, 47875201822720, 47875201921023, +ERASE, 47875201822720, 47875201822720, +STORE, 47875201822720, 47875201921023, +STORE, 47875201966080, 47875201974271, +STORE, 47875201921024, 47875201966079, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201974271, +ERASE, 47875201921024, 47875201921024, +STORE, 47875201921024, 47875201966079, +STORE, 47875201966080, 47875201974271, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201974271, +STORE, 47875201974272, 47875202572287, +STORE, 47875202093056, 47875202572287, +STORE, 47875201974272, 47875202093055, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202519039, +STORE, 47875202519040, 47875202572287, +STORE, 47875202408448, 47875202519039, +STORE, 47875202093056, 47875202408447, +ERASE, 47875202093056, 47875202093056, +STORE, 47875202093056, 47875202408447, +STORE, 47875202514944, 47875202519039, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202408448, 47875202408448, +STORE, 47875202408448, 47875202514943, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202572287, +STORE, 47875202572288, 47875205623807, +STORE, 47875203117056, 47875205623807, +STORE, 47875202572288, 47875203117055, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875205402623, +STORE, 47875205402624, 47875205623807, +STORE, 47875204812800, 47875205402623, +STORE, 47875203117056, 47875204812799, +ERASE, 47875203117056, 47875203117056, +STORE, 47875203117056, 47875204812799, +STORE, 47875205398528, 47875205402623, +STORE, 47875204812800, 47875205398527, +ERASE, 47875204812800, 47875204812800, +STORE, 47875204812800, 47875205398527, +STORE, 47875205607424, 47875205623807, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205607423, +ERASE, 47875205607424, 47875205607424, +STORE, 47875205607424, 47875205623807, +STORE, 47875205623808, 47875205656575, +ERASE, 47875205623808, 47875205623808, +STORE, 47875205623808, 47875205631999, +STORE, 47875205632000, 47875205656575, +STORE, 47875205644288, 47875205656575, +STORE, 47875205632000, 47875205644287, +ERASE, 47875205632000, 47875205632000, +STORE, 47875205632000, 47875205644287, +STORE, 47875205648384, 47875205656575, +STORE, 47875205644288, 47875205648383, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205656575, +ERASE, 47875205644288, 47875205644288, +STORE, 47875205644288, 47875205648383, +STORE, 47875205648384, 47875205656575, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205656575, +STORE, 47875205656576, 47875205693439, +ERASE, 47875205656576, 47875205656576, +STORE, 47875205656576, 47875205664767, +STORE, 47875205664768, 47875205693439, +STORE, 47875205681152, 47875205693439, +STORE, 47875205664768, 47875205681151, +ERASE, 47875205664768, 47875205664768, +STORE, 47875205664768, 47875205681151, +STORE, 47875205685248, 47875205693439, +STORE, 47875205681152, 47875205685247, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205693439, +ERASE, 47875205681152, 47875205681152, +STORE, 47875205681152, 47875205685247, +STORE, 47875205685248, 47875205693439, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205693439, +STORE, 47875205693440, 47875206168575, +ERASE, 47875205693440, 47875205693440, +STORE, 47875205693440, 47875205701631, +STORE, 47875205701632, 47875206168575, +STORE, 47875206037504, 47875206168575, +STORE, 47875205701632, 47875206037503, +ERASE, 47875205701632, 47875205701632, +STORE, 47875205701632, 47875206037503, +STORE, 47875206160384, 47875206168575, +STORE, 47875206037504, 47875206160383, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206168575, +ERASE, 47875206037504, 47875206037504, +STORE, 47875206037504, 47875206160383, +STORE, 47875206160384, 47875206168575, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206168575, +STORE, 47875206168576, 47875206176767, +STORE, 47875206176768, 47875206197247, +ERASE, 47875206176768, 47875206176768, +STORE, 47875206176768, 47875206180863, +STORE, 47875206180864, 47875206197247, +STORE, 47875206184960, 47875206197247, +STORE, 47875206180864, 47875206184959, +ERASE, 47875206180864, 47875206180864, +STORE, 47875206180864, 47875206184959, +STORE, 47875206189056, 47875206197247, +STORE, 47875206184960, 47875206189055, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206197247, +ERASE, 47875206184960, 47875206184960, +STORE, 47875206184960, 47875206189055, +STORE, 47875206189056, 47875206197247, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206197247, +STORE, 47875206197248, 47875206205439, +ERASE, 47875198861312, 47875198861312, +STORE, 47875198861312, 47875198877695, +STORE, 47875198877696, 47875198885887, +ERASE, 47875206189056, 47875206189056, +STORE, 47875206189056, 47875206193151, +STORE, 47875206193152, 47875206197247, +ERASE, 47875201777664, 47875201777664, +STORE, 47875201777664, 47875201781759, +STORE, 47875201781760, 47875201785855, +ERASE, 47875206160384, 47875206160384, +STORE, 47875206160384, 47875206164479, +STORE, 47875206164480, 47875206168575, +ERASE, 47875205685248, 47875205685248, +STORE, 47875205685248, 47875205689343, +STORE, 47875205689344, 47875205693439, +ERASE, 47875205648384, 47875205648384, +STORE, 47875205648384, 47875205652479, +STORE, 47875205652480, 47875205656575, +ERASE, 47875205402624, 47875205402624, +STORE, 47875205402624, 47875205599231, +STORE, 47875205599232, 47875205607423, +ERASE, 47875202519040, 47875202519040, +STORE, 47875202519040, 47875202555903, +STORE, 47875202555904, 47875202572287, +ERASE, 47875201966080, 47875201966080, +STORE, 47875201966080, 47875201970175, +STORE, 47875201970176, 47875201974271, +ERASE, 47875201650688, 47875201650688, +STORE, 47875201650688, 47875201654783, +STORE, 47875201654784, 47875201658879, +ERASE, 47875199381504, 47875199381504, +STORE, 47875199381504, 47875199401983, +STORE, 47875199401984, 47875199406079, +ERASE, 47875199049728, 47875199049728, +STORE, 47875199049728, 47875199053823, +STORE, 47875199053824, 47875199057919, +ERASE, 47875199004672, 47875199004672, +STORE, 47875199004672, 47875199008767, +STORE, 47875199008768, 47875199012863, +ERASE, 94011547025408, 94011547025408, +STORE, 94011547025408, 94011547148287, +STORE, 94011547148288, 94011547152383, +ERASE, 139757598109696, 139757598109696, +STORE, 139757598109696, 139757598113791, +STORE, 139757598113792, 139757598117887, +ERASE, 47875197046784, 47875197046784, +STORE, 94011557584896, 94011557720063, +STORE, 94011557584896, 94011557855231, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557851135, +STORE, 94011557851136, 94011557855231, +ERASE, 94011557851136, 94011557851136, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557847039, +STORE, 94011557847040, 94011557851135, +ERASE, 94011557847040, 94011557847040, +STORE, 94011557584896, 94011557982207, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557978111, +STORE, 94011557978112, 94011557982207, +ERASE, 94011557978112, 94011557978112, +ERASE, 94011557584896, 94011557584896, +STORE, 94011557584896, 94011557974015, +STORE, 94011557974016, 94011557978111, +ERASE, 94011557974016, 94011557974016, +STORE, 140737488347136, 140737488351231, +STORE, 140734130360320, 140737488351231, +ERASE, 140734130360320, 140734130360320, +STORE, 140734130360320, 140734130364415, +STORE, 94641232105472, 94641232785407, +ERASE, 94641232105472, 94641232105472, +STORE, 94641232105472, 94641232171007, +STORE, 94641232171008, 94641232785407, +ERASE, 94641232171008, 94641232171008, +STORE, 94641232171008, 94641232519167, +STORE, 94641232519168, 94641232658431, +STORE, 94641232658432, 94641232785407, +STORE, 139726599516160, 139726599688191, +ERASE, 139726599516160, 139726599516160, +STORE, 139726599516160, 139726599520255, +STORE, 139726599520256, 139726599688191, +ERASE, 139726599520256, 139726599520256, +STORE, 139726599520256, 139726599643135, +STORE, 139726599643136, 139726599675903, +STORE, 139726599675904, 139726599684095, +STORE, 139726599684096, 139726599688191, +STORE, 140734130446336, 140734130450431, +STORE, 140734130434048, 140734130446335, +STORE, 47906195480576, 47906195488767, +STORE, 47906195488768, 47906195496959, +STORE, 47906195496960, 47906197336063, +STORE, 47906195636224, 47906197336063, +STORE, 47906195496960, 47906195636223, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906197295103, +STORE, 47906197295104, 47906197336063, +STORE, 47906196979712, 47906197295103, +STORE, 47906195636224, 47906196979711, +ERASE, 47906195636224, 47906195636224, +STORE, 47906195636224, 47906196979711, +STORE, 47906197291008, 47906197295103, +STORE, 47906196979712, 47906197291007, +ERASE, 47906196979712, 47906196979712, +STORE, 47906196979712, 47906197291007, +STORE, 47906197319680, 47906197336063, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197319679, +ERASE, 47906197319680, 47906197319680, +STORE, 47906197319680, 47906197336063, +STORE, 47906197336064, 47906197446655, +STORE, 47906197352448, 47906197446655, +STORE, 47906197336064, 47906197352447, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197438463, +STORE, 47906197438464, 47906197446655, +STORE, 47906197413888, 47906197438463, +STORE, 47906197352448, 47906197413887, +ERASE, 47906197352448, 47906197352448, +STORE, 47906197352448, 47906197413887, +STORE, 47906197434368, 47906197438463, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197413888, 47906197413888, +STORE, 47906197413888, 47906197434367, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197446655, +STORE, 47906197446656, 47906197491711, +ERASE, 47906197446656, 47906197446656, +STORE, 47906197446656, 47906197454847, +STORE, 47906197454848, 47906197491711, +STORE, 47906197475328, 47906197491711, +STORE, 47906197454848, 47906197475327, +ERASE, 47906197454848, 47906197454848, +STORE, 47906197454848, 47906197475327, +STORE, 47906197483520, 47906197491711, +STORE, 47906197475328, 47906197483519, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197491711, +ERASE, 47906197475328, 47906197475328, +STORE, 47906197475328, 47906197483519, +STORE, 47906197483520, 47906197491711, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197491711, +STORE, 47906197491712, 47906197839871, +STORE, 47906197532672, 47906197839871, +STORE, 47906197491712, 47906197532671, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197815295, +STORE, 47906197815296, 47906197839871, +STORE, 47906197745664, 47906197815295, +STORE, 47906197532672, 47906197745663, +ERASE, 47906197532672, 47906197532672, +STORE, 47906197532672, 47906197745663, +STORE, 47906197811200, 47906197815295, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197745664, 47906197745664, +STORE, 47906197745664, 47906197811199, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197839871, +STORE, 47906197839872, 47906200100863, +STORE, 47906197991424, 47906200100863, +STORE, 47906197839872, 47906197991423, +ERASE, 47906197991424, 47906197991424, +STORE, 47906197991424, 47906200084479, +STORE, 47906200084480, 47906200100863, +STORE, 47906200092672, 47906200100863, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200092671, +ERASE, 47906200092672, 47906200092672, +STORE, 47906200092672, 47906200100863, +STORE, 47906200100864, 47906200236031, +ERASE, 47906200100864, 47906200100864, +STORE, 47906200100864, 47906200125439, +STORE, 47906200125440, 47906200236031, +STORE, 47906200186880, 47906200236031, +STORE, 47906200125440, 47906200186879, +ERASE, 47906200125440, 47906200125440, +STORE, 47906200125440, 47906200186879, +STORE, 47906200211456, 47906200236031, +STORE, 47906200186880, 47906200211455, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200236031, +ERASE, 47906200186880, 47906200186880, +STORE, 47906200186880, 47906200211455, +STORE, 47906200211456, 47906200236031, +STORE, 47906200219648, 47906200236031, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200219647, +ERASE, 47906200219648, 47906200219648, +STORE, 47906200219648, 47906200236031, +STORE, 47906200219648, 47906200244223, +STORE, 47906200244224, 47906200408063, +ERASE, 47906200244224, 47906200244224, +STORE, 47906200244224, 47906200256511, +STORE, 47906200256512, 47906200408063, +STORE, 47906200354816, 47906200408063, +STORE, 47906200256512, 47906200354815, +ERASE, 47906200256512, 47906200256512, +STORE, 47906200256512, 47906200354815, +STORE, 47906200399872, 47906200408063, +STORE, 47906200354816, 47906200399871, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200408063, +ERASE, 47906200354816, 47906200354816, +STORE, 47906200354816, 47906200399871, +STORE, 47906200399872, 47906200408063, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200408063, +STORE, 47906200408064, 47906201006079, +STORE, 47906200526848, 47906201006079, +STORE, 47906200408064, 47906200526847, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200952831, +STORE, 47906200952832, 47906201006079, +STORE, 47906200842240, 47906200952831, +STORE, 47906200526848, 47906200842239, +ERASE, 47906200526848, 47906200526848, +STORE, 47906200526848, 47906200842239, +STORE, 47906200948736, 47906200952831, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200842240, 47906200842240, +STORE, 47906200842240, 47906200948735, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906201006079, +STORE, 47906201006080, 47906204057599, +STORE, 47906201550848, 47906204057599, +STORE, 47906201006080, 47906201550847, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203836415, +STORE, 47906203836416, 47906204057599, +STORE, 47906203246592, 47906203836415, +STORE, 47906201550848, 47906203246591, +ERASE, 47906201550848, 47906201550848, +STORE, 47906201550848, 47906203246591, +STORE, 47906203832320, 47906203836415, +STORE, 47906203246592, 47906203832319, +ERASE, 47906203246592, 47906203246592, +STORE, 47906203246592, 47906203832319, +STORE, 47906204041216, 47906204057599, +STORE, 47906203836416, 47906204041215, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204041215, +ERASE, 47906204041216, 47906204041216, +STORE, 47906204041216, 47906204057599, +STORE, 47906204057600, 47906204090367, +ERASE, 47906204057600, 47906204057600, +STORE, 47906204057600, 47906204065791, +STORE, 47906204065792, 47906204090367, +STORE, 47906204078080, 47906204090367, +STORE, 47906204065792, 47906204078079, +ERASE, 47906204065792, 47906204065792, +STORE, 47906204065792, 47906204078079, +STORE, 47906204082176, 47906204090367, +STORE, 47906204078080, 47906204082175, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204090367, +ERASE, 47906204078080, 47906204078080, +STORE, 47906204078080, 47906204082175, +STORE, 47906204082176, 47906204090367, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204090367, +STORE, 47906204090368, 47906204127231, +ERASE, 47906204090368, 47906204090368, +STORE, 47906204090368, 47906204098559, +STORE, 47906204098560, 47906204127231, +STORE, 47906204114944, 47906204127231, +STORE, 47906204098560, 47906204114943, +ERASE, 47906204098560, 47906204098560, +STORE, 47906204098560, 47906204114943, +STORE, 47906204119040, 47906204127231, +STORE, 47906204114944, 47906204119039, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204127231, +ERASE, 47906204114944, 47906204114944, +STORE, 47906204114944, 47906204119039, +STORE, 47906204119040, 47906204127231, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204127231, +STORE, 47906204127232, 47906204602367, +ERASE, 47906204127232, 47906204127232, +STORE, 47906204127232, 47906204135423, +STORE, 47906204135424, 47906204602367, +STORE, 47906204471296, 47906204602367, +STORE, 47906204135424, 47906204471295, +ERASE, 47906204135424, 47906204135424, +STORE, 47906204135424, 47906204471295, +STORE, 47906204594176, 47906204602367, +STORE, 47906204471296, 47906204594175, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204602367, +ERASE, 47906204471296, 47906204471296, +STORE, 47906204471296, 47906204594175, +STORE, 47906204594176, 47906204602367, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204602367, +STORE, 47906204602368, 47906204610559, +STORE, 47906204610560, 47906204631039, +ERASE, 47906204610560, 47906204610560, +STORE, 47906204610560, 47906204614655, +STORE, 47906204614656, 47906204631039, +STORE, 47906204618752, 47906204631039, +STORE, 47906204614656, 47906204618751, +ERASE, 47906204614656, 47906204614656, +STORE, 47906204614656, 47906204618751, +STORE, 47906204622848, 47906204631039, +STORE, 47906204618752, 47906204622847, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204631039, +ERASE, 47906204618752, 47906204618752, +STORE, 47906204618752, 47906204622847, +STORE, 47906204622848, 47906204631039, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204631039, +STORE, 47906204631040, 47906204639231, +ERASE, 47906197295104, 47906197295104, +STORE, 47906197295104, 47906197311487, +STORE, 47906197311488, 47906197319679, +ERASE, 47906204622848, 47906204622848, +STORE, 47906204622848, 47906204626943, +STORE, 47906204626944, 47906204631039, +ERASE, 47906200211456, 47906200211456, +STORE, 47906200211456, 47906200215551, +STORE, 47906200215552, 47906200219647, +ERASE, 47906204594176, 47906204594176, +STORE, 47906204594176, 47906204598271, +STORE, 47906204598272, 47906204602367, +ERASE, 47906204119040, 47906204119040, +STORE, 47906204119040, 47906204123135, +STORE, 47906204123136, 47906204127231, +ERASE, 47906204082176, 47906204082176, +STORE, 47906204082176, 47906204086271, +STORE, 47906204086272, 47906204090367, +ERASE, 47906203836416, 47906203836416, +STORE, 47906203836416, 47906204033023, +STORE, 47906204033024, 47906204041215, +ERASE, 47906200952832, 47906200952832, +STORE, 47906200952832, 47906200989695, +STORE, 47906200989696, 47906201006079, +ERASE, 47906200399872, 47906200399872, +STORE, 47906200399872, 47906200403967, +STORE, 47906200403968, 47906200408063, +ERASE, 47906200084480, 47906200084480, +STORE, 47906200084480, 47906200088575, +STORE, 47906200088576, 47906200092671, +ERASE, 47906197815296, 47906197815296, +STORE, 47906197815296, 47906197835775, +STORE, 47906197835776, 47906197839871, +ERASE, 47906197483520, 47906197483520, +STORE, 47906197483520, 47906197487615, +STORE, 47906197487616, 47906197491711, +ERASE, 47906197438464, 47906197438464, +STORE, 47906197438464, 47906197442559, +STORE, 47906197442560, 47906197446655, +ERASE, 94641232658432, 94641232658432, +STORE, 94641232658432, 94641232781311, +STORE, 94641232781312, 94641232785407, +ERASE, 139726599675904, 139726599675904, +STORE, 139726599675904, 139726599679999, +STORE, 139726599680000, 139726599684095, +ERASE, 47906195480576, 47906195480576, +STORE, 94641242615808, 94641242750975, + }; + unsigned long set11[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140732658499584, 140737488351231, +ERASE, 140732658499584, 140732658499584, +STORE, 140732658499584, 140732658503679, +STORE, 94029856579584, 94029856751615, +ERASE, 94029856579584, 94029856579584, +STORE, 94029856579584, 94029856595967, +STORE, 94029856595968, 94029856751615, +ERASE, 94029856595968, 94029856595968, +STORE, 94029856595968, 94029856698367, +STORE, 94029856698368, 94029856739327, +STORE, 94029856739328, 94029856751615, +STORE, 140014592573440, 140014592745471, +ERASE, 140014592573440, 140014592573440, +STORE, 140014592573440, 140014592577535, +STORE, 140014592577536, 140014592745471, +ERASE, 140014592577536, 140014592577536, +STORE, 140014592577536, 140014592700415, +STORE, 140014592700416, 140014592733183, +STORE, 140014592733184, 140014592741375, +STORE, 140014592741376, 140014592745471, +STORE, 140732658565120, 140732658569215, +STORE, 140732658552832, 140732658565119, + }; + + unsigned long set12[] = { /* contains 12 values. */ +STORE, 140737488347136, 140737488351231, +STORE, 140732658499584, 140737488351231, +ERASE, 140732658499584, 140732658499584, +STORE, 140732658499584, 140732658503679, +STORE, 94029856579584, 94029856751615, +ERASE, 94029856579584, 94029856579584, +STORE, 94029856579584, 94029856595967, +STORE, 94029856595968, 94029856751615, +ERASE, 94029856595968, 94029856595968, +STORE, 94029856595968, 94029856698367, +STORE, 94029856698368, 94029856739327, +STORE, 94029856739328, 94029856751615, +STORE, 140014592573440, 140014592745471, +ERASE, 140014592573440, 140014592573440, +STORE, 140014592573440, 140014592577535, +STORE, 140014592577536, 140014592745471, +ERASE, 140014592577536, 140014592577536, +STORE, 140014592577536, 140014592700415, +STORE, 140014592700416, 140014592733183, +STORE, 140014592733184, 140014592741375, +STORE, 140014592741376, 140014592745471, +STORE, 140732658565120, 140732658569215, +STORE, 140732658552832, 140732658565119, +STORE, 140014592741375, 140014592741375, /* contrived */ +STORE, 140014592733184, 140014592741376, /* creates first entry retry. */ + }; + unsigned long set13[] = { +STORE, 140373516247040, 140373516251135,/*: ffffa2e7b0e10d80 */ +STORE, 140373516251136, 140373516255231,/*: ffffa2e7b1195d80 */ +STORE, 140373516255232, 140373516443647,/*: ffffa2e7b0e109c0 */ +STORE, 140373516443648, 140373516587007,/*: ffffa2e7b05fecc0 */ +STORE, 140373516963840, 140373518647295,/*: ffffa2e7bfbdcc00 */ +STORE, 140373518647296, 140373518663679,/*: ffffa2e7bf5d59c0 */ +STORE, 140373518663680, 140373518684159,/*: deleted (257) */ +STORE, 140373518680064, 140373518684159,/*: ffffa2e7b0e1cb40 */ +STORE, 140373518684160, 140373518688254,/*: ffffa2e7b05fec00 */ +STORE, 140373518688256, 140373518692351,/*: ffffa2e7bfbdcd80 */ +STORE, 140373518692352, 140373518696447,/*: ffffa2e7b0749e40 */ + }; + unsigned long set14[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140731667996672, 140737488351231, +SNULL, 140731668000767, 140737488351231, +STORE, 140731667996672, 140731668000767, +STORE, 140731667865600, 140731668000767, +STORE, 94077521272832, 94077521313791, +SNULL, 94077521301503, 94077521313791, +STORE, 94077521272832, 94077521301503, +STORE, 94077521301504, 94077521313791, +ERASE, 94077521301504, 94077521313791, +STORE, 94077521305600, 94077521313791, +STORE, 139826134630400, 139826136883199, +SNULL, 139826134773759, 139826136883199, +STORE, 139826134630400, 139826134773759, +STORE, 139826134773760, 139826136883199, +ERASE, 139826134773760, 139826136883199, +STORE, 139826136870912, 139826136879103, +STORE, 139826136879104, 139826136883199, +STORE, 140731668013056, 140731668017151, +STORE, 140731668000768, 140731668013055, +STORE, 139826136862720, 139826136870911, +STORE, 139826132406272, 139826134630399, +SNULL, 139826134056959, 139826134630399, +STORE, 139826132406272, 139826134056959, +STORE, 139826134056960, 139826134630399, +SNULL, 139826134056960, 139826134626303, +STORE, 139826134626304, 139826134630399, +STORE, 139826134056960, 139826134626303, +ERASE, 139826134056960, 139826134626303, +STORE, 139826134056960, 139826134626303, +ERASE, 139826134626304, 139826134630399, +STORE, 139826134626304, 139826134630399, +STORE, 139826136842240, 139826136862719, +STORE, 139826130022400, 139826132406271, +SNULL, 139826130022400, 139826130288639, +STORE, 139826130288640, 139826132406271, +STORE, 139826130022400, 139826130288639, +SNULL, 139826132381695, 139826132406271, +STORE, 139826130288640, 139826132381695, +STORE, 139826132381696, 139826132406271, +SNULL, 139826132381696, 139826132402175, +STORE, 139826132402176, 139826132406271, +STORE, 139826132381696, 139826132402175, +ERASE, 139826132381696, 139826132402175, +STORE, 139826132381696, 139826132402175, +ERASE, 139826132402176, 139826132406271, +STORE, 139826132402176, 139826132406271, +STORE, 139826127806464, 139826130022399, +SNULL, 139826127806464, 139826127904767, +STORE, 139826127904768, 139826130022399, +STORE, 139826127806464, 139826127904767, +SNULL, 139826129997823, 139826130022399, +STORE, 139826127904768, 139826129997823, +STORE, 139826129997824, 139826130022399, +SNULL, 139826129997824, 139826130006015, +STORE, 139826130006016, 139826130022399, +STORE, 139826129997824, 139826130006015, +ERASE, 139826129997824, 139826130006015, +STORE, 139826129997824, 139826130006015, +ERASE, 139826130006016, 139826130022399, +STORE, 139826130006016, 139826130022399, +STORE, 139826124009472, 139826127806463, +SNULL, 139826124009472, 139826125668351, +STORE, 139826125668352, 139826127806463, +STORE, 139826124009472, 139826125668351, +SNULL, 139826127765503, 139826127806463, +STORE, 139826125668352, 139826127765503, +STORE, 139826127765504, 139826127806463, +SNULL, 139826127765504, 139826127790079, +STORE, 139826127790080, 139826127806463, +STORE, 139826127765504, 139826127790079, +ERASE, 139826127765504, 139826127790079, +STORE, 139826127765504, 139826127790079, +ERASE, 139826127790080, 139826127806463, +STORE, 139826127790080, 139826127806463, +STORE, 139826121748480, 139826124009471, +SNULL, 139826121748480, 139826121900031, +STORE, 139826121900032, 139826124009471, +STORE, 139826121748480, 139826121900031, +SNULL, 139826123993087, 139826124009471, +STORE, 139826121900032, 139826123993087, +STORE, 139826123993088, 139826124009471, +SNULL, 139826123993088, 139826124001279, +STORE, 139826124001280, 139826124009471, +STORE, 139826123993088, 139826124001279, +ERASE, 139826123993088, 139826124001279, +STORE, 139826123993088, 139826124001279, +ERASE, 139826124001280, 139826124009471, +STORE, 139826124001280, 139826124009471, +STORE, 139826119626752, 139826121748479, +SNULL, 139826119626752, 139826119643135, +STORE, 139826119643136, 139826121748479, +STORE, 139826119626752, 139826119643135, +SNULL, 139826121740287, 139826121748479, +STORE, 139826119643136, 139826121740287, +STORE, 139826121740288, 139826121748479, +ERASE, 139826121740288, 139826121748479, +STORE, 139826121740288, 139826121748479, +STORE, 139826136834048, 139826136842239, +STORE, 139826117496832, 139826119626751, +SNULL, 139826117496832, 139826117525503, +STORE, 139826117525504, 139826119626751, +STORE, 139826117496832, 139826117525503, +SNULL, 139826119618559, 139826119626751, +STORE, 139826117525504, 139826119618559, +STORE, 139826119618560, 139826119626751, +ERASE, 139826119618560, 139826119626751, +STORE, 139826119618560, 139826119626751, +STORE, 139826115244032, 139826117496831, +SNULL, 139826115244032, 139826115395583, +STORE, 139826115395584, 139826117496831, +STORE, 139826115244032, 139826115395583, +SNULL, 139826117488639, 139826117496831, +STORE, 139826115395584, 139826117488639, +STORE, 139826117488640, 139826117496831, +ERASE, 139826117488640, 139826117496831, +STORE, 139826117488640, 139826117496831, +STORE, 139826113073152, 139826115244031, +SNULL, 139826113073152, 139826113142783, +STORE, 139826113142784, 139826115244031, +STORE, 139826113073152, 139826113142783, +SNULL, 139826115235839, 139826115244031, +STORE, 139826113142784, 139826115235839, +STORE, 139826115235840, 139826115244031, +ERASE, 139826115235840, 139826115244031, +STORE, 139826115235840, 139826115244031, +STORE, 139826109861888, 139826113073151, +SNULL, 139826109861888, 139826110939135, +STORE, 139826110939136, 139826113073151, +STORE, 139826109861888, 139826110939135, +SNULL, 139826113036287, 139826113073151, +STORE, 139826110939136, 139826113036287, +STORE, 139826113036288, 139826113073151, +ERASE, 139826113036288, 139826113073151, +STORE, 139826113036288, 139826113073151, +STORE, 139826107727872, 139826109861887, +SNULL, 139826107727872, 139826107756543, +STORE, 139826107756544, 139826109861887, +STORE, 139826107727872, 139826107756543, +SNULL, 139826109853695, 139826109861887, +STORE, 139826107756544, 139826109853695, +STORE, 139826109853696, 139826109861887, +ERASE, 139826109853696, 139826109861887, +STORE, 139826109853696, 139826109861887, +STORE, 139826105417728, 139826107727871, +SNULL, 139826105417728, 139826105622527, +STORE, 139826105622528, 139826107727871, +STORE, 139826105417728, 139826105622527, +SNULL, 139826107719679, 139826107727871, +STORE, 139826105622528, 139826107719679, +STORE, 139826107719680, 139826107727871, +ERASE, 139826107719680, 139826107727871, +STORE, 139826107719680, 139826107727871, +STORE, 139826136825856, 139826136842239, +STORE, 139826103033856, 139826105417727, +SNULL, 139826103033856, 139826103226367, +STORE, 139826103226368, 139826105417727, +STORE, 139826103033856, 139826103226367, +SNULL, 139826105319423, 139826105417727, +STORE, 139826103226368, 139826105319423, +STORE, 139826105319424, 139826105417727, +ERASE, 139826105319424, 139826105417727, +STORE, 139826105319424, 139826105417727, +STORE, 139826100916224, 139826103033855, +SNULL, 139826100916224, 139826100932607, +STORE, 139826100932608, 139826103033855, +STORE, 139826100916224, 139826100932607, +SNULL, 139826103025663, 139826103033855, +STORE, 139826100932608, 139826103025663, +STORE, 139826103025664, 139826103033855, +ERASE, 139826103025664, 139826103033855, +STORE, 139826103025664, 139826103033855, +STORE, 139826098348032, 139826100916223, +SNULL, 139826098348032, 139826098814975, +STORE, 139826098814976, 139826100916223, +STORE, 139826098348032, 139826098814975, +SNULL, 139826100908031, 139826100916223, +STORE, 139826098814976, 139826100908031, +STORE, 139826100908032, 139826100916223, +ERASE, 139826100908032, 139826100916223, +STORE, 139826100908032, 139826100916223, +STORE, 139826096234496, 139826098348031, +SNULL, 139826096234496, 139826096246783, +STORE, 139826096246784, 139826098348031, +STORE, 139826096234496, 139826096246783, +SNULL, 139826098339839, 139826098348031, +STORE, 139826096246784, 139826098339839, +STORE, 139826098339840, 139826098348031, +ERASE, 139826098339840, 139826098348031, +STORE, 139826098339840, 139826098348031, +STORE, 139826094055424, 139826096234495, +SNULL, 139826094055424, 139826094133247, +STORE, 139826094133248, 139826096234495, +STORE, 139826094055424, 139826094133247, +SNULL, 139826096226303, 139826096234495, +STORE, 139826094133248, 139826096226303, +STORE, 139826096226304, 139826096234495, +ERASE, 139826096226304, 139826096234495, +STORE, 139826096226304, 139826096234495, +STORE, 139826136817664, 139826136842239, +STORE, 139826091937792, 139826094055423, +SNULL, 139826091937792, 139826091954175, +STORE, 139826091954176, 139826094055423, +STORE, 139826091937792, 139826091954175, +SNULL, 139826094047231, 139826094055423, +STORE, 139826091954176, 139826094047231, +STORE, 139826094047232, 139826094055423, +ERASE, 139826094047232, 139826094055423, +STORE, 139826094047232, 139826094055423, +STORE, 139826136809472, 139826136842239, +SNULL, 139826127781887, 139826127790079, +STORE, 139826127765504, 139826127781887, +STORE, 139826127781888, 139826127790079, +SNULL, 139826094051327, 139826094055423, +STORE, 139826094047232, 139826094051327, +STORE, 139826094051328, 139826094055423, +SNULL, 139826096230399, 139826096234495, +STORE, 139826096226304, 139826096230399, +STORE, 139826096230400, 139826096234495, +SNULL, 139826098343935, 139826098348031, +STORE, 139826098339840, 139826098343935, +STORE, 139826098343936, 139826098348031, +SNULL, 139826130001919, 139826130006015, +STORE, 139826129997824, 139826130001919, +STORE, 139826130001920, 139826130006015, +SNULL, 139826100912127, 139826100916223, +STORE, 139826100908032, 139826100912127, +STORE, 139826100912128, 139826100916223, +SNULL, 139826103029759, 139826103033855, +STORE, 139826103025664, 139826103029759, +STORE, 139826103029760, 139826103033855, +SNULL, 139826105413631, 139826105417727, +STORE, 139826105319424, 139826105413631, +STORE, 139826105413632, 139826105417727, +SNULL, 139826107723775, 139826107727871, +STORE, 139826107719680, 139826107723775, +STORE, 139826107723776, 139826107727871, +SNULL, 139826109857791, 139826109861887, +STORE, 139826109853696, 139826109857791, +STORE, 139826109857792, 139826109861887, +SNULL, 139826113044479, 139826113073151, +STORE, 139826113036288, 139826113044479, +STORE, 139826113044480, 139826113073151, +SNULL, 139826115239935, 139826115244031, +STORE, 139826115235840, 139826115239935, +STORE, 139826115239936, 139826115244031, +SNULL, 139826117492735, 139826117496831, +STORE, 139826117488640, 139826117492735, +STORE, 139826117492736, 139826117496831, +SNULL, 139826119622655, 139826119626751, +STORE, 139826119618560, 139826119622655, +STORE, 139826119622656, 139826119626751, +SNULL, 139826121744383, 139826121748479, +STORE, 139826121740288, 139826121744383, +STORE, 139826121744384, 139826121748479, +SNULL, 139826123997183, 139826124001279, +STORE, 139826123993088, 139826123997183, +STORE, 139826123997184, 139826124001279, +SNULL, 139826132398079, 139826132402175, +STORE, 139826132381696, 139826132398079, +STORE, 139826132398080, 139826132402175, +SNULL, 139826134622207, 139826134626303, +STORE, 139826134056960, 139826134622207, +STORE, 139826134622208, 139826134626303, +SNULL, 94077521309695, 94077521313791, +STORE, 94077521305600, 94077521309695, +STORE, 94077521309696, 94077521313791, +SNULL, 139826136875007, 139826136879103, +STORE, 139826136870912, 139826136875007, +STORE, 139826136875008, 139826136879103, +ERASE, 139826136842240, 139826136862719, +STORE, 94077554049024, 94077554184191, +STORE, 139826136543232, 139826136842239, +STORE, 139826136276992, 139826136842239, +STORE, 139826136010752, 139826136842239, +STORE, 139826135744512, 139826136842239, +SNULL, 139826136543231, 139826136842239, +STORE, 139826135744512, 139826136543231, +STORE, 139826136543232, 139826136842239, +SNULL, 139826136543232, 139826136809471, +STORE, 139826136809472, 139826136842239, +STORE, 139826136543232, 139826136809471, + }; + unsigned long set15[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722061451264, 140737488351231, +SNULL, 140722061455359, 140737488351231, +STORE, 140722061451264, 140722061455359, +STORE, 140722061320192, 140722061455359, +STORE, 94728600248320, 94728600289279, +SNULL, 94728600276991, 94728600289279, +STORE, 94728600248320, 94728600276991, +STORE, 94728600276992, 94728600289279, +ERASE, 94728600276992, 94728600289279, +STORE, 94728600281088, 94728600289279, +STORE, 139906806779904, 139906809032703, +SNULL, 139906806923263, 139906809032703, +STORE, 139906806779904, 139906806923263, +STORE, 139906806923264, 139906809032703, +ERASE, 139906806923264, 139906809032703, +STORE, 139906809020416, 139906809028607, +STORE, 139906809028608, 139906809032703, +STORE, 140722061692928, 140722061697023, +STORE, 140722061680640, 140722061692927, +STORE, 139906809012224, 139906809020415, +STORE, 139906804555776, 139906806779903, +SNULL, 139906806206463, 139906806779903, +STORE, 139906804555776, 139906806206463, +STORE, 139906806206464, 139906806779903, +SNULL, 139906806206464, 139906806775807, +STORE, 139906806775808, 139906806779903, +STORE, 139906806206464, 139906806775807, +ERASE, 139906806206464, 139906806775807, +STORE, 139906806206464, 139906806775807, +ERASE, 139906806775808, 139906806779903, +STORE, 139906806775808, 139906806779903, +STORE, 139906808991744, 139906809012223, +STORE, 139906802171904, 139906804555775, +SNULL, 139906802171904, 139906802438143, +STORE, 139906802438144, 139906804555775, +STORE, 139906802171904, 139906802438143, +SNULL, 139906804531199, 139906804555775, +STORE, 139906802438144, 139906804531199, +STORE, 139906804531200, 139906804555775, +SNULL, 139906804531200, 139906804551679, +STORE, 139906804551680, 139906804555775, +STORE, 139906804531200, 139906804551679, +ERASE, 139906804531200, 139906804551679, +STORE, 139906804531200, 139906804551679, +ERASE, 139906804551680, 139906804555775, +STORE, 139906804551680, 139906804555775, +STORE, 139906799955968, 139906802171903, +SNULL, 139906799955968, 139906800054271, +STORE, 139906800054272, 139906802171903, +STORE, 139906799955968, 139906800054271, +SNULL, 139906802147327, 139906802171903, +STORE, 139906800054272, 139906802147327, +STORE, 139906802147328, 139906802171903, +SNULL, 139906802147328, 139906802155519, +STORE, 139906802155520, 139906802171903, +STORE, 139906802147328, 139906802155519, +ERASE, 139906802147328, 139906802155519, +STORE, 139906802147328, 139906802155519, +ERASE, 139906802155520, 139906802171903, +STORE, 139906802155520, 139906802171903, +STORE, 139906796158976, 139906799955967, +SNULL, 139906796158976, 139906797817855, +STORE, 139906797817856, 139906799955967, +STORE, 139906796158976, 139906797817855, +SNULL, 139906799915007, 139906799955967, +STORE, 139906797817856, 139906799915007, +STORE, 139906799915008, 139906799955967, +SNULL, 139906799915008, 139906799939583, +STORE, 139906799939584, 139906799955967, +STORE, 139906799915008, 139906799939583, +ERASE, 139906799915008, 139906799939583, +STORE, 139906799915008, 139906799939583, +ERASE, 139906799939584, 139906799955967, +STORE, 139906799939584, 139906799955967, +STORE, 139906793897984, 139906796158975, +SNULL, 139906793897984, 139906794049535, +STORE, 139906794049536, 139906796158975, +STORE, 139906793897984, 139906794049535, +SNULL, 139906796142591, 139906796158975, +STORE, 139906794049536, 139906796142591, +STORE, 139906796142592, 139906796158975, +SNULL, 139906796142592, 139906796150783, +STORE, 139906796150784, 139906796158975, +STORE, 139906796142592, 139906796150783, +ERASE, 139906796142592, 139906796150783, +STORE, 139906796142592, 139906796150783, +ERASE, 139906796150784, 139906796158975, +STORE, 139906796150784, 139906796158975, +STORE, 139906791776256, 139906793897983, +SNULL, 139906791776256, 139906791792639, +STORE, 139906791792640, 139906793897983, +STORE, 139906791776256, 139906791792639, +SNULL, 139906793889791, 139906793897983, +STORE, 139906791792640, 139906793889791, +STORE, 139906793889792, 139906793897983, +ERASE, 139906793889792, 139906793897983, +STORE, 139906793889792, 139906793897983, +STORE, 139906808983552, 139906808991743, +STORE, 139906789646336, 139906791776255, +SNULL, 139906789646336, 139906789675007, +STORE, 139906789675008, 139906791776255, +STORE, 139906789646336, 139906789675007, +SNULL, 139906791768063, 139906791776255, +STORE, 139906789675008, 139906791768063, +STORE, 139906791768064, 139906791776255, +ERASE, 139906791768064, 139906791776255, +STORE, 139906791768064, 139906791776255, +STORE, 139906787393536, 139906789646335, +SNULL, 139906787393536, 139906787545087, +STORE, 139906787545088, 139906789646335, +STORE, 139906787393536, 139906787545087, +SNULL, 139906789638143, 139906789646335, +STORE, 139906787545088, 139906789638143, +STORE, 139906789638144, 139906789646335, +ERASE, 139906789638144, 139906789646335, +STORE, 139906789638144, 139906789646335, +STORE, 139906785222656, 139906787393535, +SNULL, 139906785222656, 139906785292287, +STORE, 139906785292288, 139906787393535, +STORE, 139906785222656, 139906785292287, +SNULL, 139906787385343, 139906787393535, +STORE, 139906785292288, 139906787385343, +STORE, 139906787385344, 139906787393535, +ERASE, 139906787385344, 139906787393535, +STORE, 139906787385344, 139906787393535, +STORE, 139906782011392, 139906785222655, +SNULL, 139906782011392, 139906783088639, +STORE, 139906783088640, 139906785222655, +STORE, 139906782011392, 139906783088639, +SNULL, 139906785185791, 139906785222655, +STORE, 139906783088640, 139906785185791, +STORE, 139906785185792, 139906785222655, +ERASE, 139906785185792, 139906785222655, +STORE, 139906785185792, 139906785222655, +STORE, 139906779877376, 139906782011391, +SNULL, 139906779877376, 139906779906047, +STORE, 139906779906048, 139906782011391, +STORE, 139906779877376, 139906779906047, +SNULL, 139906782003199, 139906782011391, +STORE, 139906779906048, 139906782003199, +STORE, 139906782003200, 139906782011391, +ERASE, 139906782003200, 139906782011391, +STORE, 139906782003200, 139906782011391, +STORE, 139906777567232, 139906779877375, +SNULL, 139906777567232, 139906777772031, +STORE, 139906777772032, 139906779877375, +STORE, 139906777567232, 139906777772031, +SNULL, 139906779869183, 139906779877375, +STORE, 139906777772032, 139906779869183, +STORE, 139906779869184, 139906779877375, +ERASE, 139906779869184, 139906779877375, +STORE, 139906779869184, 139906779877375, +STORE, 139906808975360, 139906808991743, +STORE, 139906775183360, 139906777567231, +SNULL, 139906775183360, 139906775375871, +STORE, 139906775375872, 139906777567231, +STORE, 139906775183360, 139906775375871, +SNULL, 139906777468927, 139906777567231, +STORE, 139906775375872, 139906777468927, +STORE, 139906777468928, 139906777567231, +ERASE, 139906777468928, 139906777567231, +STORE, 139906777468928, 139906777567231, +STORE, 139906773065728, 139906775183359, +SNULL, 139906773065728, 139906773082111, +STORE, 139906773082112, 139906775183359, +STORE, 139906773065728, 139906773082111, +SNULL, 139906775175167, 139906775183359, +STORE, 139906773082112, 139906775175167, +STORE, 139906775175168, 139906775183359, +ERASE, 139906775175168, 139906775183359, +STORE, 139906775175168, 139906775183359, +STORE, 139906770497536, 139906773065727, +SNULL, 139906770497536, 139906770964479, +STORE, 139906770964480, 139906773065727, +STORE, 139906770497536, 139906770964479, +SNULL, 139906773057535, 139906773065727, +STORE, 139906770964480, 139906773057535, +STORE, 139906773057536, 139906773065727, +ERASE, 139906773057536, 139906773065727, +STORE, 139906773057536, 139906773065727, +STORE, 139906768384000, 139906770497535, +SNULL, 139906768384000, 139906768396287, +STORE, 139906768396288, 139906770497535, +STORE, 139906768384000, 139906768396287, +SNULL, 139906770489343, 139906770497535, +STORE, 139906768396288, 139906770489343, +STORE, 139906770489344, 139906770497535, +ERASE, 139906770489344, 139906770497535, +STORE, 139906770489344, 139906770497535, +STORE, 139906766204928, 139906768383999, +SNULL, 139906766204928, 139906766282751, +STORE, 139906766282752, 139906768383999, +STORE, 139906766204928, 139906766282751, +SNULL, 139906768375807, 139906768383999, +STORE, 139906766282752, 139906768375807, +STORE, 139906768375808, 139906768383999, +ERASE, 139906768375808, 139906768383999, +STORE, 139906768375808, 139906768383999, +STORE, 139906808967168, 139906808991743, +STORE, 139906764087296, 139906766204927, +SNULL, 139906764087296, 139906764103679, +STORE, 139906764103680, 139906766204927, +STORE, 139906764087296, 139906764103679, +SNULL, 139906766196735, 139906766204927, +STORE, 139906764103680, 139906766196735, +STORE, 139906766196736, 139906766204927, +ERASE, 139906766196736, 139906766204927, +STORE, 139906766196736, 139906766204927, +STORE, 139906808958976, 139906808991743, +SNULL, 139906799931391, 139906799939583, +STORE, 139906799915008, 139906799931391, +STORE, 139906799931392, 139906799939583, +SNULL, 139906766200831, 139906766204927, +STORE, 139906766196736, 139906766200831, +STORE, 139906766200832, 139906766204927, +SNULL, 139906768379903, 139906768383999, +STORE, 139906768375808, 139906768379903, +STORE, 139906768379904, 139906768383999, +SNULL, 139906770493439, 139906770497535, +STORE, 139906770489344, 139906770493439, +STORE, 139906770493440, 139906770497535, +SNULL, 139906802151423, 139906802155519, +STORE, 139906802147328, 139906802151423, +STORE, 139906802151424, 139906802155519, +SNULL, 139906773061631, 139906773065727, +STORE, 139906773057536, 139906773061631, +STORE, 139906773061632, 139906773065727, +SNULL, 139906775179263, 139906775183359, +STORE, 139906775175168, 139906775179263, +STORE, 139906775179264, 139906775183359, +SNULL, 139906777563135, 139906777567231, +STORE, 139906777468928, 139906777563135, +STORE, 139906777563136, 139906777567231, +SNULL, 139906779873279, 139906779877375, +STORE, 139906779869184, 139906779873279, +STORE, 139906779873280, 139906779877375, +SNULL, 139906782007295, 139906782011391, +STORE, 139906782003200, 139906782007295, +STORE, 139906782007296, 139906782011391, +SNULL, 139906785193983, 139906785222655, +STORE, 139906785185792, 139906785193983, +STORE, 139906785193984, 139906785222655, +SNULL, 139906787389439, 139906787393535, +STORE, 139906787385344, 139906787389439, +STORE, 139906787389440, 139906787393535, +SNULL, 139906789642239, 139906789646335, +STORE, 139906789638144, 139906789642239, +STORE, 139906789642240, 139906789646335, +SNULL, 139906791772159, 139906791776255, +STORE, 139906791768064, 139906791772159, +STORE, 139906791772160, 139906791776255, +SNULL, 139906793893887, 139906793897983, +STORE, 139906793889792, 139906793893887, +STORE, 139906793893888, 139906793897983, +SNULL, 139906796146687, 139906796150783, +STORE, 139906796142592, 139906796146687, +STORE, 139906796146688, 139906796150783, +SNULL, 139906804547583, 139906804551679, +STORE, 139906804531200, 139906804547583, +STORE, 139906804547584, 139906804551679, +SNULL, 139906806771711, 139906806775807, +STORE, 139906806206464, 139906806771711, +STORE, 139906806771712, 139906806775807, +SNULL, 94728600285183, 94728600289279, +STORE, 94728600281088, 94728600285183, +STORE, 94728600285184, 94728600289279, +SNULL, 139906809024511, 139906809028607, +STORE, 139906809020416, 139906809024511, +STORE, 139906809024512, 139906809028607, +ERASE, 139906808991744, 139906809012223, +STORE, 94728620138496, 94728620273663, +STORE, 139906808692736, 139906808991743, +STORE, 139906808426496, 139906808991743, +STORE, 139906808160256, 139906808991743, +STORE, 139906807894016, 139906808991743, +SNULL, 139906808692735, 139906808991743, +STORE, 139906807894016, 139906808692735, +STORE, 139906808692736, 139906808991743, +SNULL, 139906808692736, 139906808958975, +STORE, 139906808958976, 139906808991743, +STORE, 139906808692736, 139906808958975, + }; + + unsigned long set16[] = { +STORE, 94174808662016, 94174809321471, +STORE, 94174811414528, 94174811426815, +STORE, 94174811426816, 94174811430911, +STORE, 94174811430912, 94174811443199, +STORE, 94174841700352, 94174841835519, +STORE, 140173257838592, 140173259497471, +STORE, 140173259497472, 140173261594623, +STORE, 140173261594624, 140173261611007, +STORE, 140173261611008, 140173261619199, +STORE, 140173261619200, 140173261635583, +STORE, 140173261635584, 140173261778943, +STORE, 140173263863808, 140173263871999, +STORE, 140173263876096, 140173263880191, +STORE, 140173263880192, 140173263884287, +STORE, 140173263884288, 140173263888383, +STORE, 140729801007104, 140729801142271, +STORE, 140729801617408, 140729801629695, +STORE, 140729801629696, 140729801633791, +STORE, 140737488347136, 140737488351231, +STORE, 140728166858752, 140737488351231, +SNULL, 140728166862847, 140737488351231, +STORE, 140728166858752, 140728166862847, +STORE, 140728166727680, 140728166862847, +STORE, 93912949866496, 93912950337535, +SNULL, 93912950288383, 93912950337535, +STORE, 93912949866496, 93912950288383, +STORE, 93912950288384, 93912950337535, +ERASE, 93912950288384, 93912950337535, +STORE, 93912950292480, 93912950337535, +STORE, 139921863385088, 139921865637887, +SNULL, 139921863528447, 139921865637887, +STORE, 139921863385088, 139921863528447, +STORE, 139921863528448, 139921865637887, +ERASE, 139921863528448, 139921865637887, +STORE, 139921865625600, 139921865633791, +STORE, 139921865633792, 139921865637887, +STORE, 140728167899136, 140728167903231, +STORE, 140728167886848, 140728167899135, +STORE, 139921865601024, 139921865625599, +STORE, 139921865592832, 139921865601023, +STORE, 139921861251072, 139921863385087, +SNULL, 139921861251072, 139921861279743, +STORE, 139921861279744, 139921863385087, +STORE, 139921861251072, 139921861279743, +SNULL, 139921863376895, 139921863385087, +STORE, 139921861279744, 139921863376895, +STORE, 139921863376896, 139921863385087, +ERASE, 139921863376896, 139921863385087, +STORE, 139921863376896, 139921863385087, +STORE, 139921858867200, 139921861251071, +SNULL, 139921858867200, 139921859133439, +STORE, 139921859133440, 139921861251071, +STORE, 139921858867200, 139921859133439, +SNULL, 139921861226495, 139921861251071, +STORE, 139921859133440, 139921861226495, +STORE, 139921861226496, 139921861251071, +SNULL, 139921861226496, 139921861246975, +STORE, 139921861246976, 139921861251071, +STORE, 139921861226496, 139921861246975, +ERASE, 139921861226496, 139921861246975, +STORE, 139921861226496, 139921861246975, +ERASE, 139921861246976, 139921861251071, +STORE, 139921861246976, 139921861251071, +STORE, 139921856675840, 139921858867199, +SNULL, 139921856675840, 139921856765951, +STORE, 139921856765952, 139921858867199, +STORE, 139921856675840, 139921856765951, +SNULL, 139921858859007, 139921858867199, +STORE, 139921856765952, 139921858859007, +STORE, 139921858859008, 139921858867199, +ERASE, 139921858859008, 139921858867199, +STORE, 139921858859008, 139921858867199, +STORE, 139921854414848, 139921856675839, +SNULL, 139921854414848, 139921854566399, +STORE, 139921854566400, 139921856675839, +STORE, 139921854414848, 139921854566399, +SNULL, 139921856659455, 139921856675839, +STORE, 139921854566400, 139921856659455, +STORE, 139921856659456, 139921856675839, +SNULL, 139921856659456, 139921856667647, +STORE, 139921856667648, 139921856675839, +STORE, 139921856659456, 139921856667647, +ERASE, 139921856659456, 139921856667647, +STORE, 139921856659456, 139921856667647, +ERASE, 139921856667648, 139921856675839, +STORE, 139921856667648, 139921856675839, +STORE, 139921852284928, 139921854414847, +SNULL, 139921852284928, 139921852313599, +STORE, 139921852313600, 139921854414847, +STORE, 139921852284928, 139921852313599, +SNULL, 139921854406655, 139921854414847, +STORE, 139921852313600, 139921854406655, +STORE, 139921854406656, 139921854414847, +ERASE, 139921854406656, 139921854414847, +STORE, 139921854406656, 139921854414847, +STORE, 139921850068992, 139921852284927, +SNULL, 139921850068992, 139921850167295, +STORE, 139921850167296, 139921852284927, +STORE, 139921850068992, 139921850167295, +SNULL, 139921852260351, 139921852284927, +STORE, 139921850167296, 139921852260351, +STORE, 139921852260352, 139921852284927, +SNULL, 139921852260352, 139921852268543, +STORE, 139921852268544, 139921852284927, +STORE, 139921852260352, 139921852268543, +ERASE, 139921852260352, 139921852268543, +STORE, 139921852260352, 139921852268543, +ERASE, 139921852268544, 139921852284927, +STORE, 139921852268544, 139921852284927, +STORE, 139921865584640, 139921865601023, +STORE, 139921846272000, 139921850068991, +SNULL, 139921846272000, 139921847930879, +STORE, 139921847930880, 139921850068991, +STORE, 139921846272000, 139921847930879, +SNULL, 139921850028031, 139921850068991, +STORE, 139921847930880, 139921850028031, +STORE, 139921850028032, 139921850068991, +SNULL, 139921850028032, 139921850052607, +STORE, 139921850052608, 139921850068991, +STORE, 139921850028032, 139921850052607, +ERASE, 139921850028032, 139921850052607, +STORE, 139921850028032, 139921850052607, +ERASE, 139921850052608, 139921850068991, +STORE, 139921850052608, 139921850068991, +STORE, 139921844154368, 139921846271999, +SNULL, 139921844154368, 139921844170751, +STORE, 139921844170752, 139921846271999, +STORE, 139921844154368, 139921844170751, +SNULL, 139921846263807, 139921846271999, +STORE, 139921844170752, 139921846263807, +STORE, 139921846263808, 139921846271999, +ERASE, 139921846263808, 139921846271999, +STORE, 139921846263808, 139921846271999, +STORE, 139921842036736, 139921844154367, +SNULL, 139921842036736, 139921842053119, +STORE, 139921842053120, 139921844154367, +STORE, 139921842036736, 139921842053119, +SNULL, 139921844146175, 139921844154367, +STORE, 139921842053120, 139921844146175, +STORE, 139921844146176, 139921844154367, +ERASE, 139921844146176, 139921844154367, +STORE, 139921844146176, 139921844154367, +STORE, 139921839468544, 139921842036735, +SNULL, 139921839468544, 139921839935487, +STORE, 139921839935488, 139921842036735, +STORE, 139921839468544, 139921839935487, +SNULL, 139921842028543, 139921842036735, +STORE, 139921839935488, 139921842028543, +STORE, 139921842028544, 139921842036735, +ERASE, 139921842028544, 139921842036735, +STORE, 139921842028544, 139921842036735, +STORE, 139921837355008, 139921839468543, +SNULL, 139921837355008, 139921837367295, +STORE, 139921837367296, 139921839468543, +STORE, 139921837355008, 139921837367295, +SNULL, 139921839460351, 139921839468543, +STORE, 139921837367296, 139921839460351, +STORE, 139921839460352, 139921839468543, +ERASE, 139921839460352, 139921839468543, +STORE, 139921839460352, 139921839468543, +STORE, 139921865576448, 139921865601023, +STORE, 139921865564160, 139921865601023, +SNULL, 139921850044415, 139921850052607, +STORE, 139921850028032, 139921850044415, +STORE, 139921850044416, 139921850052607, +SNULL, 139921839464447, 139921839468543, +STORE, 139921839460352, 139921839464447, +STORE, 139921839464448, 139921839468543, +SNULL, 139921852264447, 139921852268543, +STORE, 139921852260352, 139921852264447, +STORE, 139921852264448, 139921852268543, +SNULL, 139921842032639, 139921842036735, +STORE, 139921842028544, 139921842032639, +STORE, 139921842032640, 139921842036735, +SNULL, 139921844150271, 139921844154367, +STORE, 139921844146176, 139921844150271, +STORE, 139921844150272, 139921844154367, +SNULL, 139921846267903, 139921846271999, +STORE, 139921846263808, 139921846267903, +STORE, 139921846267904, 139921846271999, +SNULL, 139921854410751, 139921854414847, +STORE, 139921854406656, 139921854410751, +STORE, 139921854410752, 139921854414847, +SNULL, 139921856663551, 139921856667647, +STORE, 139921856659456, 139921856663551, +STORE, 139921856663552, 139921856667647, +SNULL, 139921858863103, 139921858867199, +STORE, 139921858859008, 139921858863103, +STORE, 139921858863104, 139921858867199, +SNULL, 139921861242879, 139921861246975, +STORE, 139921861226496, 139921861242879, +STORE, 139921861242880, 139921861246975, +SNULL, 139921863380991, 139921863385087, +STORE, 139921863376896, 139921863380991, +STORE, 139921863380992, 139921863385087, +SNULL, 93912950333439, 93912950337535, +STORE, 93912950292480, 93912950333439, +STORE, 93912950333440, 93912950337535, +SNULL, 139921865629695, 139921865633791, +STORE, 139921865625600, 139921865629695, +STORE, 139921865629696, 139921865633791, +ERASE, 139921865601024, 139921865625599, +STORE, 93912968110080, 93912968245247, +STORE, 139921828913152, 139921837355007, +STORE, 139921865621504, 139921865625599, +STORE, 139921865617408, 139921865621503, +STORE, 139921865613312, 139921865617407, +STORE, 139921865547776, 139921865564159, + }; + + unsigned long set17[] = { +STORE, 94397057224704, 94397057646591, +STORE, 94397057650688, 94397057691647, +STORE, 94397057691648, 94397057695743, +STORE, 94397075271680, 94397075406847, +STORE, 139953169051648, 139953169063935, +STORE, 139953169063936, 139953171156991, +STORE, 139953171156992, 139953171161087, +STORE, 139953171161088, 139953171165183, +STORE, 139953171165184, 139953171632127, +STORE, 139953171632128, 139953173725183, +STORE, 139953173725184, 139953173729279, +STORE, 139953173729280, 139953173733375, +STORE, 139953173733376, 139953173749759, +STORE, 139953173749760, 139953175842815, +STORE, 139953175842816, 139953175846911, +STORE, 139953175846912, 139953175851007, +STORE, 139953175851008, 139953175867391, +STORE, 139953175867392, 139953177960447, +STORE, 139953177960448, 139953177964543, +STORE, 139953177964544, 139953177968639, +STORE, 139953177968640, 139953179627519, +STORE, 139953179627520, 139953181724671, +STORE, 139953181724672, 139953181741055, +STORE, 139953181741056, 139953181749247, +STORE, 139953181749248, 139953181765631, +STORE, 139953181765632, 139953181863935, +STORE, 139953181863936, 139953183956991, +STORE, 139953183956992, 139953183961087, +STORE, 139953183961088, 139953183965183, +STORE, 139953183965184, 139953183981567, +STORE, 139953183981568, 139953184010239, +STORE, 139953184010240, 139953186103295, +STORE, 139953186103296, 139953186107391, +STORE, 139953186107392, 139953186111487, +STORE, 139953186111488, 139953186263039, +STORE, 139953186263040, 139953188356095, +STORE, 139953188356096, 139953188360191, +STORE, 139953188360192, 139953188364287, +STORE, 139953188364288, 139953188372479, +STORE, 139953188372480, 139953188462591, +STORE, 139953188462592, 139953190555647, +STORE, 139953190555648, 139953190559743, +STORE, 139953190559744, 139953190563839, +STORE, 139953190563840, 139953190830079, +STORE, 139953190830080, 139953192923135, +STORE, 139953192923136, 139953192939519, +STORE, 139953192939520, 139953192943615, +STORE, 139953192943616, 139953192947711, +STORE, 139953192947712, 139953192976383, +STORE, 139953192976384, 139953195073535, +STORE, 139953195073536, 139953195077631, +STORE, 139953195077632, 139953195081727, +STORE, 139953195081728, 139953195225087, +STORE, 139953197281280, 139953197318143, +STORE, 139953197322240, 139953197326335, +STORE, 139953197326336, 139953197330431, +STORE, 139953197330432, 139953197334527, +STORE, 140720477511680, 140720477646847, +STORE, 140720478302208, 140720478314495, +STORE, 140720478314496, 140720478318591, + }; + unsigned long set18[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140724953673728, 140737488351231, +SNULL, 140724953677823, 140737488351231, +STORE, 140724953673728, 140724953677823, +STORE, 140724953542656, 140724953677823, +STORE, 94675199266816, 94675199311871, +SNULL, 94675199303679, 94675199311871, +STORE, 94675199266816, 94675199303679, +STORE, 94675199303680, 94675199311871, +ERASE, 94675199303680, 94675199311871, +STORE, 94675199303680, 94675199311871, +STORE, 140222970605568, 140222972858367, +SNULL, 140222970748927, 140222972858367, +STORE, 140222970605568, 140222970748927, +STORE, 140222970748928, 140222972858367, +ERASE, 140222970748928, 140222972858367, +STORE, 140222972846080, 140222972854271, +STORE, 140222972854272, 140222972858367, +STORE, 140724954365952, 140724954370047, +STORE, 140724954353664, 140724954365951, +STORE, 140222972841984, 140222972846079, +STORE, 140222972833792, 140222972841983, +STORE, 140222968475648, 140222970605567, +SNULL, 140222968475648, 140222968504319, +STORE, 140222968504320, 140222970605567, +STORE, 140222968475648, 140222968504319, +SNULL, 140222970597375, 140222970605567, +STORE, 140222968504320, 140222970597375, +STORE, 140222970597376, 140222970605567, +ERASE, 140222970597376, 140222970605567, +STORE, 140222970597376, 140222970605567, + }; + unsigned long set19[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140725182459904, 140737488351231, +SNULL, 140725182463999, 140737488351231, +STORE, 140725182459904, 140725182463999, +STORE, 140725182328832, 140725182463999, +STORE, 94730166636544, 94730166763519, +SNULL, 94730166747135, 94730166763519, +STORE, 94730166636544, 94730166747135, +STORE, 94730166747136, 94730166763519, +ERASE, 94730166747136, 94730166763519, +STORE, 94730166751232, 94730166763519, +STORE, 140656834555904, 140656836808703, +SNULL, 140656834699263, 140656836808703, +STORE, 140656834555904, 140656834699263, +STORE, 140656834699264, 140656836808703, +ERASE, 140656834699264, 140656836808703, +STORE, 140656836796416, 140656836804607, +STORE, 140656836804608, 140656836808703, +STORE, 140725183389696, 140725183393791, +STORE, 140725183377408, 140725183389695, +STORE, 140656836788224, 140656836796415, +STORE, 140656832331776, 140656834555903, +SNULL, 140656833982463, 140656834555903, +STORE, 140656832331776, 140656833982463, +STORE, 140656833982464, 140656834555903, +SNULL, 140656833982464, 140656834551807, +STORE, 140656834551808, 140656834555903, +STORE, 140656833982464, 140656834551807, +ERASE, 140656833982464, 140656834551807, +STORE, 140656833982464, 140656834551807, +ERASE, 140656834551808, 140656834555903, +STORE, 140656834551808, 140656834555903, +STORE, 140656836763648, 140656836788223, +STORE, 140656830070784, 140656832331775, +SNULL, 140656830070784, 140656830222335, +STORE, 140656830222336, 140656832331775, +STORE, 140656830070784, 140656830222335, +SNULL, 140656832315391, 140656832331775, +STORE, 140656830222336, 140656832315391, +STORE, 140656832315392, 140656832331775, +SNULL, 140656832315392, 140656832323583, +STORE, 140656832323584, 140656832331775, +STORE, 140656832315392, 140656832323583, +ERASE, 140656832315392, 140656832323583, +STORE, 140656832315392, 140656832323583, +ERASE, 140656832323584, 140656832331775, +STORE, 140656832323584, 140656832331775, +STORE, 140656827940864, 140656830070783, +SNULL, 140656827940864, 140656827969535, +STORE, 140656827969536, 140656830070783, +STORE, 140656827940864, 140656827969535, +SNULL, 140656830062591, 140656830070783, +STORE, 140656827969536, 140656830062591, +STORE, 140656830062592, 140656830070783, +ERASE, 140656830062592, 140656830070783, +STORE, 140656830062592, 140656830070783, +STORE, 140656825724928, 140656827940863, +SNULL, 140656825724928, 140656825823231, +STORE, 140656825823232, 140656827940863, +STORE, 140656825724928, 140656825823231, +SNULL, 140656827916287, 140656827940863, +STORE, 140656825823232, 140656827916287, +STORE, 140656827916288, 140656827940863, +SNULL, 140656827916288, 140656827924479, +STORE, 140656827924480, 140656827940863, +STORE, 140656827916288, 140656827924479, +ERASE, 140656827916288, 140656827924479, +STORE, 140656827916288, 140656827924479, +ERASE, 140656827924480, 140656827940863, +STORE, 140656827924480, 140656827940863, +STORE, 140656821927936, 140656825724927, +SNULL, 140656821927936, 140656823586815, +STORE, 140656823586816, 140656825724927, +STORE, 140656821927936, 140656823586815, +SNULL, 140656825683967, 140656825724927, +STORE, 140656823586816, 140656825683967, +STORE, 140656825683968, 140656825724927, +SNULL, 140656825683968, 140656825708543, +STORE, 140656825708544, 140656825724927, +STORE, 140656825683968, 140656825708543, +ERASE, 140656825683968, 140656825708543, +STORE, 140656825683968, 140656825708543, +ERASE, 140656825708544, 140656825724927, +STORE, 140656825708544, 140656825724927, +STORE, 140656819806208, 140656821927935, +SNULL, 140656819806208, 140656819822591, +STORE, 140656819822592, 140656821927935, +STORE, 140656819806208, 140656819822591, +SNULL, 140656821919743, 140656821927935, +STORE, 140656819822592, 140656821919743, +STORE, 140656821919744, 140656821927935, +ERASE, 140656821919744, 140656821927935, +STORE, 140656821919744, 140656821927935, +STORE, 140656836755456, 140656836763647, +STORE, 140656817553408, 140656819806207, +SNULL, 140656817553408, 140656817704959, +STORE, 140656817704960, 140656819806207, +STORE, 140656817553408, 140656817704959, +SNULL, 140656819798015, 140656819806207, +STORE, 140656817704960, 140656819798015, +STORE, 140656819798016, 140656819806207, +ERASE, 140656819798016, 140656819806207, +STORE, 140656819798016, 140656819806207, +STORE, 140656815382528, 140656817553407, +SNULL, 140656815382528, 140656815452159, +STORE, 140656815452160, 140656817553407, +STORE, 140656815382528, 140656815452159, +SNULL, 140656817545215, 140656817553407, +STORE, 140656815452160, 140656817545215, +STORE, 140656817545216, 140656817553407, +ERASE, 140656817545216, 140656817553407, +STORE, 140656817545216, 140656817553407, +STORE, 140656812171264, 140656815382527, +SNULL, 140656812171264, 140656813248511, +STORE, 140656813248512, 140656815382527, +STORE, 140656812171264, 140656813248511, +SNULL, 140656815345663, 140656815382527, +STORE, 140656813248512, 140656815345663, +STORE, 140656815345664, 140656815382527, +ERASE, 140656815345664, 140656815382527, +STORE, 140656815345664, 140656815382527, +STORE, 140656810037248, 140656812171263, +SNULL, 140656810037248, 140656810065919, +STORE, 140656810065920, 140656812171263, +STORE, 140656810037248, 140656810065919, +SNULL, 140656812163071, 140656812171263, +STORE, 140656810065920, 140656812163071, +STORE, 140656812163072, 140656812171263, +ERASE, 140656812163072, 140656812171263, +STORE, 140656812163072, 140656812171263, +STORE, 140656807727104, 140656810037247, +SNULL, 140656807727104, 140656807931903, +STORE, 140656807931904, 140656810037247, +STORE, 140656807727104, 140656807931903, +SNULL, 140656810029055, 140656810037247, +STORE, 140656807931904, 140656810029055, +STORE, 140656810029056, 140656810037247, +ERASE, 140656810029056, 140656810037247, +STORE, 140656810029056, 140656810037247, +STORE, 140656805343232, 140656807727103, +SNULL, 140656805343232, 140656805535743, +STORE, 140656805535744, 140656807727103, +STORE, 140656805343232, 140656805535743, +SNULL, 140656807628799, 140656807727103, +STORE, 140656805535744, 140656807628799, +STORE, 140656807628800, 140656807727103, +ERASE, 140656807628800, 140656807727103, +STORE, 140656807628800, 140656807727103, +STORE, 140656836747264, 140656836763647, +STORE, 140656802775040, 140656805343231, +SNULL, 140656802775040, 140656803241983, +STORE, 140656803241984, 140656805343231, +STORE, 140656802775040, 140656803241983, +SNULL, 140656805335039, 140656805343231, +STORE, 140656803241984, 140656805335039, +STORE, 140656805335040, 140656805343231, +ERASE, 140656805335040, 140656805343231, +STORE, 140656805335040, 140656805343231, +STORE, 140656800661504, 140656802775039, +SNULL, 140656800661504, 140656800673791, +STORE, 140656800673792, 140656802775039, +STORE, 140656800661504, 140656800673791, +SNULL, 140656802766847, 140656802775039, +STORE, 140656800673792, 140656802766847, +STORE, 140656802766848, 140656802775039, +ERASE, 140656802766848, 140656802775039, +STORE, 140656802766848, 140656802775039, +STORE, 140656798482432, 140656800661503, +SNULL, 140656798482432, 140656798560255, +STORE, 140656798560256, 140656800661503, +STORE, 140656798482432, 140656798560255, +SNULL, 140656800653311, 140656800661503, +STORE, 140656798560256, 140656800653311, +STORE, 140656800653312, 140656800661503, +ERASE, 140656800653312, 140656800661503, +STORE, 140656800653312, 140656800661503, +STORE, 140656796364800, 140656798482431, +SNULL, 140656796364800, 140656796381183, +STORE, 140656796381184, 140656798482431, +STORE, 140656796364800, 140656796381183, +SNULL, 140656798474239, 140656798482431, +STORE, 140656796381184, 140656798474239, +STORE, 140656798474240, 140656798482431, +ERASE, 140656798474240, 140656798482431, +STORE, 140656798474240, 140656798482431, +STORE, 140656836739072, 140656836763647, +STORE, 140656836726784, 140656836763647, +SNULL, 140656825700351, 140656825708543, +STORE, 140656825683968, 140656825700351, +STORE, 140656825700352, 140656825708543, +SNULL, 140656798478335, 140656798482431, +STORE, 140656798474240, 140656798478335, +STORE, 140656798478336, 140656798482431, +SNULL, 140656800657407, 140656800661503, +STORE, 140656800653312, 140656800657407, +STORE, 140656800657408, 140656800661503, +SNULL, 140656802770943, 140656802775039, +STORE, 140656802766848, 140656802770943, +STORE, 140656802770944, 140656802775039, +SNULL, 140656827920383, 140656827924479, +STORE, 140656827916288, 140656827920383, +STORE, 140656827920384, 140656827924479, +SNULL, 140656805339135, 140656805343231, +STORE, 140656805335040, 140656805339135, +STORE, 140656805339136, 140656805343231, +SNULL, 140656807723007, 140656807727103, +STORE, 140656807628800, 140656807723007, +STORE, 140656807723008, 140656807727103, +SNULL, 140656810033151, 140656810037247, +STORE, 140656810029056, 140656810033151, +STORE, 140656810033152, 140656810037247, +SNULL, 140656812167167, 140656812171263, +STORE, 140656812163072, 140656812167167, +STORE, 140656812167168, 140656812171263, +SNULL, 140656815353855, 140656815382527, +STORE, 140656815345664, 140656815353855, +STORE, 140656815353856, 140656815382527, +SNULL, 140656817549311, 140656817553407, +STORE, 140656817545216, 140656817549311, +STORE, 140656817549312, 140656817553407, +SNULL, 140656819802111, 140656819806207, +STORE, 140656819798016, 140656819802111, +STORE, 140656819802112, 140656819806207, +SNULL, 140656821923839, 140656821927935, +STORE, 140656821919744, 140656821923839, +STORE, 140656821923840, 140656821927935, +SNULL, 140656830066687, 140656830070783, +STORE, 140656830062592, 140656830066687, +STORE, 140656830066688, 140656830070783, +SNULL, 140656832319487, 140656832323583, +STORE, 140656832315392, 140656832319487, +STORE, 140656832319488, 140656832323583, +SNULL, 140656834547711, 140656834551807, +STORE, 140656833982464, 140656834547711, +STORE, 140656834547712, 140656834551807, +SNULL, 94730166759423, 94730166763519, +STORE, 94730166751232, 94730166759423, +STORE, 94730166759424, 94730166763519, +SNULL, 140656836800511, 140656836804607, +STORE, 140656836796416, 140656836800511, +STORE, 140656836800512, 140656836804607, +ERASE, 140656836763648, 140656836788223, +STORE, 94730171318272, 94730171453439, +STORE, 140656836784128, 140656836788223, +STORE, 140656836780032, 140656836784127, +STORE, 140656791920640, 140656796364799, +STORE, 140656836775936, 140656836780031, +STORE, 140656787476480, 140656791920639, +STORE, 140656779083776, 140656787476479, +SNULL, 140656779087871, 140656787476479, +STORE, 140656779083776, 140656779087871, +STORE, 140656779087872, 140656787476479, +STORE, 140656836771840, 140656836775935, +STORE, 140656774639616, 140656779083775, +STORE, 140656766246912, 140656774639615, +SNULL, 140656766251007, 140656774639615, +STORE, 140656766246912, 140656766251007, +STORE, 140656766251008, 140656774639615, +ERASE, 140656791920640, 140656796364799, +ERASE, 140656836780032, 140656836784127, +ERASE, 140656787476480, 140656791920639, +ERASE, 140656836775936, 140656836780031, +STORE, 140656836780032, 140656836784127, +STORE, 140656791920640, 140656796364799, +STORE, 140656836775936, 140656836780031, +STORE, 140656787476480, 140656791920639, +ERASE, 140656774639616, 140656779083775, + }; + unsigned long set20[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735952392192, 140737488351231, +SNULL, 140735952396287, 140737488351231, +STORE, 140735952392192, 140735952396287, +STORE, 140735952261120, 140735952396287, +STORE, 94849008947200, 94849009414143, +SNULL, 94849009364991, 94849009414143, +STORE, 94849008947200, 94849009364991, +STORE, 94849009364992, 94849009414143, +ERASE, 94849009364992, 94849009414143, +STORE, 94849009364992, 94849009414143, +STORE, 140590397943808, 140590400196607, +SNULL, 140590398087167, 140590400196607, +STORE, 140590397943808, 140590398087167, +STORE, 140590398087168, 140590400196607, +ERASE, 140590398087168, 140590400196607, +STORE, 140590400184320, 140590400192511, +STORE, 140590400192512, 140590400196607, +STORE, 140735952850944, 140735952855039, +STORE, 140735952838656, 140735952850943, +STORE, 140590400180224, 140590400184319, +STORE, 140590400172032, 140590400180223, +STORE, 140590395809792, 140590397943807, +SNULL, 140590395809792, 140590395838463, +STORE, 140590395838464, 140590397943807, +STORE, 140590395809792, 140590395838463, +SNULL, 140590397935615, 140590397943807, +STORE, 140590395838464, 140590397935615, +STORE, 140590397935616, 140590397943807, +ERASE, 140590397935616, 140590397943807, +STORE, 140590397935616, 140590397943807, +STORE, 140590393425920, 140590395809791, +SNULL, 140590393425920, 140590393692159, +STORE, 140590393692160, 140590395809791, +STORE, 140590393425920, 140590393692159, +SNULL, 140590395785215, 140590395809791, +STORE, 140590393692160, 140590395785215, +STORE, 140590395785216, 140590395809791, +SNULL, 140590395785216, 140590395805695, +STORE, 140590395805696, 140590395809791, +STORE, 140590395785216, 140590395805695, +ERASE, 140590395785216, 140590395805695, +STORE, 140590395785216, 140590395805695, +ERASE, 140590395805696, 140590395809791, +STORE, 140590395805696, 140590395809791, +STORE, 140590391234560, 140590393425919, +SNULL, 140590391234560, 140590391324671, +STORE, 140590391324672, 140590393425919, +STORE, 140590391234560, 140590391324671, +SNULL, 140590393417727, 140590393425919, +STORE, 140590391324672, 140590393417727, +STORE, 140590393417728, 140590393425919, +ERASE, 140590393417728, 140590393425919, +STORE, 140590393417728, 140590393425919, +STORE, 140590388973568, 140590391234559, +SNULL, 140590388973568, 140590389125119, +STORE, 140590389125120, 140590391234559, +STORE, 140590388973568, 140590389125119, +SNULL, 140590391218175, 140590391234559, +STORE, 140590389125120, 140590391218175, +STORE, 140590391218176, 140590391234559, +SNULL, 140590391218176, 140590391226367, +STORE, 140590391226368, 140590391234559, +STORE, 140590391218176, 140590391226367, +ERASE, 140590391218176, 140590391226367, +STORE, 140590391218176, 140590391226367, +ERASE, 140590391226368, 140590391234559, +STORE, 140590391226368, 140590391234559, +STORE, 140590386843648, 140590388973567, +SNULL, 140590386843648, 140590386872319, +STORE, 140590386872320, 140590388973567, +STORE, 140590386843648, 140590386872319, +SNULL, 140590388965375, 140590388973567, +STORE, 140590386872320, 140590388965375, +STORE, 140590388965376, 140590388973567, +ERASE, 140590388965376, 140590388973567, +STORE, 140590388965376, 140590388973567, +STORE, 140590384627712, 140590386843647, +SNULL, 140590384627712, 140590384726015, +STORE, 140590384726016, 140590386843647, +STORE, 140590384627712, 140590384726015, +SNULL, 140590386819071, 140590386843647, +STORE, 140590384726016, 140590386819071, +STORE, 140590386819072, 140590386843647, +SNULL, 140590386819072, 140590386827263, +STORE, 140590386827264, 140590386843647, +STORE, 140590386819072, 140590386827263, +ERASE, 140590386819072, 140590386827263, +STORE, 140590386819072, 140590386827263, +ERASE, 140590386827264, 140590386843647, +STORE, 140590386827264, 140590386843647, +STORE, 140590400163840, 140590400180223, +STORE, 140590380830720, 140590384627711, +SNULL, 140590380830720, 140590382489599, +STORE, 140590382489600, 140590384627711, +STORE, 140590380830720, 140590382489599, +SNULL, 140590384586751, 140590384627711, +STORE, 140590382489600, 140590384586751, +STORE, 140590384586752, 140590384627711, +SNULL, 140590384586752, 140590384611327, +STORE, 140590384611328, 140590384627711, +STORE, 140590384586752, 140590384611327, +ERASE, 140590384586752, 140590384611327, +STORE, 140590384586752, 140590384611327, +ERASE, 140590384611328, 140590384627711, +STORE, 140590384611328, 140590384627711, +STORE, 140590378713088, 140590380830719, +SNULL, 140590378713088, 140590378729471, +STORE, 140590378729472, 140590380830719, +STORE, 140590378713088, 140590378729471, +SNULL, 140590380822527, 140590380830719, +STORE, 140590378729472, 140590380822527, +STORE, 140590380822528, 140590380830719, +ERASE, 140590380822528, 140590380830719, +STORE, 140590380822528, 140590380830719, +STORE, 140590376595456, 140590378713087, +SNULL, 140590376595456, 140590376611839, +STORE, 140590376611840, 140590378713087, +STORE, 140590376595456, 140590376611839, +SNULL, 140590378704895, 140590378713087, +STORE, 140590376611840, 140590378704895, +STORE, 140590378704896, 140590378713087, +ERASE, 140590378704896, 140590378713087, +STORE, 140590378704896, 140590378713087, +STORE, 140590374027264, 140590376595455, +SNULL, 140590374027264, 140590374494207, +STORE, 140590374494208, 140590376595455, +STORE, 140590374027264, 140590374494207, +SNULL, 140590376587263, 140590376595455, +STORE, 140590374494208, 140590376587263, +STORE, 140590376587264, 140590376595455, +ERASE, 140590376587264, 140590376595455, +STORE, 140590376587264, 140590376595455, +STORE, 140590371913728, 140590374027263, +SNULL, 140590371913728, 140590371926015, +STORE, 140590371926016, 140590374027263, +STORE, 140590371913728, 140590371926015, +SNULL, 140590374019071, 140590374027263, +STORE, 140590371926016, 140590374019071, +STORE, 140590374019072, 140590374027263, +ERASE, 140590374019072, 140590374027263, +STORE, 140590374019072, 140590374027263, +STORE, 140590400155648, 140590400180223, +STORE, 140590400143360, 140590400180223, +SNULL, 140590384603135, 140590384611327, +STORE, 140590384586752, 140590384603135, +STORE, 140590384603136, 140590384611327, +SNULL, 140590374023167, 140590374027263, +STORE, 140590374019072, 140590374023167, +STORE, 140590374023168, 140590374027263, +SNULL, 140590386823167, 140590386827263, +STORE, 140590386819072, 140590386823167, +STORE, 140590386823168, 140590386827263, +SNULL, 140590376591359, 140590376595455, + }; + unsigned long set21[] = { +STORE, 93874710941696, 93874711363583, +STORE, 93874711367680, 93874711408639, +STORE, 93874711408640, 93874711412735, +STORE, 93874720989184, 93874721124351, +STORE, 140708365086720, 140708365099007, +STORE, 140708365099008, 140708367192063, +STORE, 140708367192064, 140708367196159, +STORE, 140708367196160, 140708367200255, +STORE, 140708367200256, 140708367667199, +STORE, 140708367667200, 140708369760255, +STORE, 140708369760256, 140708369764351, +STORE, 140708369764352, 140708369768447, +STORE, 140708369768448, 140708369784831, +STORE, 140708369784832, 140708371877887, +STORE, 140708371877888, 140708371881983, +STORE, 140708371881984, 140708371886079, +STORE, 140708371886080, 140708371902463, +STORE, 140708371902464, 140708373995519, +STORE, 140708373995520, 140708373999615, +STORE, 140708373999616, 140708374003711, +STORE, 140708374003712, 140708375662591, +STORE, 140708375662592, 140708377759743, +STORE, 140708377759744, 140708377776127, +STORE, 140708377776128, 140708377784319, +STORE, 140708377784320, 140708377800703, +STORE, 140708377800704, 140708377899007, +STORE, 140708377899008, 140708379992063, +STORE, 140708379992064, 140708379996159, +STORE, 140708379996160, 140708380000255, +STORE, 140708380000256, 140708380016639, +STORE, 140708380016640, 140708380045311, +STORE, 140708380045312, 140708382138367, +STORE, 140708382138368, 140708382142463, +STORE, 140708382142464, 140708382146559, +STORE, 140708382146560, 140708382298111, +STORE, 140708382298112, 140708384391167, +STORE, 140708384391168, 140708384395263, +STORE, 140708384395264, 140708384399359, +STORE, 140708384399360, 140708384407551, +STORE, 140708384407552, 140708384497663, +STORE, 140708384497664, 140708386590719, +STORE, 140708386590720, 140708386594815, +STORE, 140708386594816, 140708386598911, +STORE, 140708386598912, 140708386865151, +STORE, 140708386865152, 140708388958207, +STORE, 140708388958208, 140708388974591, +STORE, 140708388974592, 140708388978687, +STORE, 140708388978688, 140708388982783, +STORE, 140708388982784, 140708389011455, +STORE, 140708389011456, 140708391108607, +STORE, 140708391108608, 140708391112703, +STORE, 140708391112704, 140708391116799, +STORE, 140708391116800, 140708391260159, +STORE, 140708393291776, 140708393308159, +STORE, 140708393308160, 140708393312255, +STORE, 140708393312256, 140708393316351, +STORE, 140708393316352, 140708393353215, +STORE, 140708393353216, 140708393357311, +STORE, 140708393357312, 140708393361407, +STORE, 140708393361408, 140708393365503, +STORE, 140708393365504, 140708393369599, +STORE, 140730557042688, 140730557177855, +STORE, 140730557235200, 140730557247487, +STORE, 140730557247488, 140730557251583, +ERASE, 140708393353216, 140708393357311, +ERASE, 140708393312256, 140708393316351, +ERASE, 140708393308160, 140708393312255, +ERASE, 140708393291776, 140708393308159, + }; + unsigned long set22[] = { +STORE, 93951397134336, 93951397183487, +STORE, 93951397183488, 93951397728255, +STORE, 93951397728256, 93951397826559, +STORE, 93951397826560, 93951397842943, +STORE, 93951397842944, 93951397847039, +STORE, 93951425974272, 93951426109439, +STORE, 140685152665600, 140685152677887, +STORE, 140685152677888, 140685152829439, +STORE, 140685152829440, 140685154181119, +STORE, 140685154181120, 140685154484223, +STORE, 140685154484224, 140685154496511, +STORE, 140685154496512, 140685154508799, +STORE, 140685154508800, 140685154525183, +STORE, 140685154525184, 140685154541567, +STORE, 140685154541568, 140685154590719, +STORE, 140685154590720, 140685154603007, +STORE, 140685154603008, 140685154607103, +STORE, 140685154607104, 140685154611199, +STORE, 140685154611200, 140685154615295, +STORE, 140685154615296, 140685154631679, +STORE, 140685154639872, 140685154643967, +STORE, 140685154643968, 140685154766847, +STORE, 140685154766848, 140685154799615, +STORE, 140685154803712, 140685154807807, +STORE, 140685154807808, 140685154811903, +STORE, 140685154811904, 140685154815999, +STORE, 140722188902400, 140722189037567, +STORE, 140722189512704, 140722189524991, +STORE, 140722189524992, 140722189529087, +STORE, 140737488347136, 140737488351231, +STORE, 140733429354496, 140737488351231, +SNULL, 140733429358591, 140737488351231, +STORE, 140733429354496, 140733429358591, +STORE, 140733429223424, 140733429358591, +STORE, 94526683537408, 94526683660287, +SNULL, 94526683553791, 94526683660287, +STORE, 94526683537408, 94526683553791, +STORE, 94526683553792, 94526683660287, +ERASE, 94526683553792, 94526683660287, +STORE, 94526683553792, 94526683623423, +STORE, 94526683623424, 94526683647999, +STORE, 94526683652096, 94526683660287, +STORE, 140551363747840, 140551363923967, +SNULL, 140551363751935, 140551363923967, +STORE, 140551363747840, 140551363751935, +STORE, 140551363751936, 140551363923967, +ERASE, 140551363751936, 140551363923967, +STORE, 140551363751936, 140551363874815, +STORE, 140551363874816, 140551363907583, +STORE, 140551363911680, 140551363919871, +STORE, 140551363919872, 140551363923967, +STORE, 140733429690368, 140733429694463, +STORE, 140733429678080, 140733429690367, +STORE, 140551363739648, 140551363747839, +STORE, 140551363731456, 140551363739647, +STORE, 140551363379200, 140551363731455, +SNULL, 140551363379200, 140551363420159, +STORE, 140551363420160, 140551363731455, +STORE, 140551363379200, 140551363420159, +SNULL, 140551363706879, 140551363731455, +STORE, 140551363420160, 140551363706879, +STORE, 140551363706880, 140551363731455, +SNULL, 140551363420160, 140551363637247, +STORE, 140551363637248, 140551363706879, +STORE, 140551363420160, 140551363637247, +ERASE, 140551363420160, 140551363637247, +STORE, 140551363420160, 140551363637247, +SNULL, 140551363637248, 140551363702783, +STORE, 140551363702784, 140551363706879, +STORE, 140551363637248, 140551363702783, +ERASE, 140551363637248, 140551363702783, +STORE, 140551363637248, 140551363702783, +ERASE, 140551363706880, 140551363731455, +STORE, 140551363706880, 140551363731455, +STORE, 140551361531904, 140551363379199, +SNULL, 140551361683455, 140551363379199, +STORE, 140551361531904, 140551361683455, +STORE, 140551361683456, 140551363379199, +SNULL, 140551361683456, 140551363035135, +STORE, 140551363035136, 140551363379199, +STORE, 140551361683456, 140551363035135, +ERASE, 140551361683456, 140551363035135, +STORE, 140551361683456, 140551363035135, +SNULL, 140551363035136, 140551363338239, +STORE, 140551363338240, 140551363379199, +STORE, 140551363035136, 140551363338239, +ERASE, 140551363035136, 140551363338239, +STORE, 140551363035136, 140551363379199, +SNULL, 140551363338239, 140551363379199, +STORE, 140551363035136, 140551363338239, +STORE, 140551363338240, 140551363379199, +SNULL, 140551363338240, 140551363362815, +STORE, 140551363362816, 140551363379199, +STORE, 140551363338240, 140551363362815, +ERASE, 140551363338240, 140551363362815, +STORE, 140551363338240, 140551363362815, +ERASE, 140551363362816, 140551363379199, +STORE, 140551363362816, 140551363379199, +STORE, 140551361519616, 140551361531903, +SNULL, 140551363350527, 140551363362815, +STORE, 140551363338240, 140551363350527, +STORE, 140551363350528, 140551363362815, +SNULL, 140551363727359, 140551363731455, +STORE, 140551363706880, 140551363727359, +STORE, 140551363727360, 140551363731455, +SNULL, 94526683656191, 94526683660287, +STORE, 94526683652096, 94526683656191, +STORE, 94526683656192, 94526683660287, +SNULL, 140551363915775, 140551363919871, +STORE, 140551363911680, 140551363915775, +STORE, 140551363915776, 140551363919871, +ERASE, 140551363739648, 140551363747839, +STORE, 94526715490304, 94526715625471, +STORE, 140551361253376, 140551361531903, +STORE, 140551360987136, 140551361531903, +STORE, 140551360720896, 140551361531903, +STORE, 140551360454656, 140551361531903, +SNULL, 140551361253375, 140551361531903, +STORE, 140551360454656, 140551361253375, +STORE, 140551361253376, 140551361531903, +SNULL, 140551361253376, 140551361519615, +STORE, 140551361519616, 140551361531903, +STORE, 140551361253376, 140551361519615, +ERASE, 140551361253376, 140551361519615, + }; + + unsigned long set23[] = { +STORE, 94014447943680, 94014448156671, +STORE, 94014450253824, 94014450257919, +STORE, 94014450257920, 94014450266111, +STORE, 94014450266112, 94014450278399, +STORE, 94014464225280, 94014464630783, +STORE, 139761764306944, 139761765965823, +STORE, 139761765965824, 139761768062975, +STORE, 139761768062976, 139761768079359, +STORE, 139761768079360, 139761768087551, +STORE, 139761768087552, 139761768103935, +STORE, 139761768103936, 139761768116223, +STORE, 139761768116224, 139761770209279, +STORE, 139761770209280, 139761770213375, +STORE, 139761770213376, 139761770217471, +STORE, 139761770217472, 139761770360831, +STORE, 139761770729472, 139761772412927, +STORE, 139761772412928, 139761772429311, +STORE, 139761772457984, 139761772462079, +STORE, 139761772462080, 139761772466175, +STORE, 139761772466176, 139761772470271, +STORE, 140724336517120, 140724336652287, +STORE, 140724336955392, 140724336967679, +STORE, 140724336967680, 140724336971775, +STORE, 140737488347136, 140737488351231, +STORE, 140721840295936, 140737488351231, +SNULL, 140721840300031, 140737488351231, +STORE, 140721840295936, 140721840300031, +STORE, 140721840164864, 140721840300031, +STORE, 93937913667584, 93937915830271, +SNULL, 93937913729023, 93937915830271, +STORE, 93937913667584, 93937913729023, +STORE, 93937913729024, 93937915830271, +ERASE, 93937913729024, 93937915830271, +STORE, 93937915822080, 93937915830271, +STORE, 140598835335168, 140598837587967, +SNULL, 140598835478527, 140598837587967, +STORE, 140598835335168, 140598835478527, +STORE, 140598835478528, 140598837587967, +ERASE, 140598835478528, 140598837587967, +STORE, 140598837575680, 140598837583871, +STORE, 140598837583872, 140598837587967, +STORE, 140721841086464, 140721841090559, +STORE, 140721841074176, 140721841086463, +STORE, 140598837547008, 140598837575679, +STORE, 140598837538816, 140598837547007, +STORE, 140598831538176, 140598835335167, +SNULL, 140598831538176, 140598833197055, +STORE, 140598833197056, 140598835335167, +STORE, 140598831538176, 140598833197055, +SNULL, 140598835294207, 140598835335167, +STORE, 140598833197056, 140598835294207, +STORE, 140598835294208, 140598835335167, +SNULL, 140598835294208, 140598835318783, +STORE, 140598835318784, 140598835335167, +STORE, 140598835294208, 140598835318783, +ERASE, 140598835294208, 140598835318783, +STORE, 140598835294208, 140598835318783, +ERASE, 140598835318784, 140598835335167, +STORE, 140598835318784, 140598835335167, +SNULL, 140598835310591, 140598835318783, +STORE, 140598835294208, 140598835310591, +STORE, 140598835310592, 140598835318783, +SNULL, 93937915826175, 93937915830271, +STORE, 93937915822080, 93937915826175, +STORE, 93937915826176, 93937915830271, +SNULL, 140598837579775, 140598837583871, +STORE, 140598837575680, 140598837579775, +STORE, 140598837579776, 140598837583871, +ERASE, 140598837547008, 140598837575679, +STORE, 93937929179136, 93937929314303, +STORE, 140598835855360, 140598837538815, +STORE, 140737488347136, 140737488351231, +STORE, 140728187723776, 140737488351231, +SNULL, 140728187727871, 140737488351231, +STORE, 140728187723776, 140728187727871, +STORE, 140728187592704, 140728187727871, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140583951437824, 140583953690623, +SNULL, 140583951581183, 140583953690623, +STORE, 140583951437824, 140583951581183, +STORE, 140583951581184, 140583953690623, +ERASE, 140583951581184, 140583953690623, +STORE, 140583953678336, 140583953686527, +STORE, 140583953686528, 140583953690623, +STORE, 140728189116416, 140728189120511, +STORE, 140728189104128, 140728189116415, +STORE, 140583953649664, 140583953678335, +STORE, 140583953641472, 140583953649663, +STORE, 140583948275712, 140583951437823, +SNULL, 140583948275712, 140583949336575, +STORE, 140583949336576, 140583951437823, +STORE, 140583948275712, 140583949336575, +SNULL, 140583951429631, 140583951437823, +STORE, 140583949336576, 140583951429631, +STORE, 140583951429632, 140583951437823, +ERASE, 140583951429632, 140583951437823, +STORE, 140583951429632, 140583951437823, +STORE, 140583944478720, 140583948275711, +SNULL, 140583944478720, 140583946137599, +STORE, 140583946137600, 140583948275711, +STORE, 140583944478720, 140583946137599, +SNULL, 140583948234751, 140583948275711, +STORE, 140583946137600, 140583948234751, +STORE, 140583948234752, 140583948275711, +SNULL, 140583948234752, 140583948259327, +STORE, 140583948259328, 140583948275711, +STORE, 140583948234752, 140583948259327, +ERASE, 140583948234752, 140583948259327, +STORE, 140583948234752, 140583948259327, +ERASE, 140583948259328, 140583948275711, +STORE, 140583948259328, 140583948275711, +STORE, 140583953629184, 140583953649663, +SNULL, 140583948251135, 140583948259327, +STORE, 140583948234752, 140583948251135, +STORE, 140583948251136, 140583948259327, +SNULL, 140583951433727, 140583951437823, +STORE, 140583951429632, 140583951433727, +STORE, 140583951433728, 140583951437823, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140583953682431, 140583953686527, +STORE, 140583953678336, 140583953682431, +STORE, 140583953682432, 140583953686527, +ERASE, 140583953649664, 140583953678335, +STORE, 17821696, 17956863, +STORE, 17821696, 18104319, +STORE, 140583951945728, 140583953629183, +STORE, 94014447943680, 94014448156671, +STORE, 94014450253824, 94014450257919, +STORE, 94014450257920, 94014450266111, +STORE, 94014450266112, 94014450278399, +STORE, 94014464225280, 94014465196031, +STORE, 139761764306944, 139761765965823, +STORE, 139761765965824, 139761768062975, +STORE, 139761768062976, 139761768079359, +STORE, 139761768079360, 139761768087551, +STORE, 139761768087552, 139761768103935, +STORE, 139761768103936, 139761768116223, +STORE, 139761768116224, 139761770209279, +STORE, 139761770209280, 139761770213375, +STORE, 139761770213376, 139761770217471, +STORE, 139761770217472, 139761770360831, +STORE, 139761770729472, 139761772412927, +STORE, 139761772412928, 139761772429311, +STORE, 139761772457984, 139761772462079, +STORE, 139761772462080, 139761772466175, +STORE, 139761772466176, 139761772470271, +STORE, 140724336517120, 140724336652287, +STORE, 140724336955392, 140724336967679, +STORE, 140724336967680, 140724336971775, +STORE, 140737488347136, 140737488351231, +STORE, 140726063296512, 140737488351231, +SNULL, 140726063300607, 140737488351231, +STORE, 140726063296512, 140726063300607, +STORE, 140726063165440, 140726063300607, +STORE, 94016795934720, 94016798158847, +SNULL, 94016796045311, 94016798158847, +STORE, 94016795934720, 94016796045311, +STORE, 94016796045312, 94016798158847, +ERASE, 94016796045312, 94016798158847, +STORE, 94016798138368, 94016798150655, +STORE, 94016798150656, 94016798158847, +STORE, 139975915966464, 139975918219263, +SNULL, 139975916109823, 139975918219263, +STORE, 139975915966464, 139975916109823, +STORE, 139975916109824, 139975918219263, +ERASE, 139975916109824, 139975918219263, +STORE, 139975918206976, 139975918215167, +STORE, 139975918215168, 139975918219263, +STORE, 140726064541696, 140726064545791, +STORE, 140726064529408, 140726064541695, +STORE, 139975918178304, 139975918206975, +STORE, 139975918170112, 139975918178303, +STORE, 139975912169472, 139975915966463, +SNULL, 139975912169472, 139975913828351, +STORE, 139975913828352, 139975915966463, +STORE, 139975912169472, 139975913828351, +SNULL, 139975915925503, 139975915966463, +STORE, 139975913828352, 139975915925503, +STORE, 139975915925504, 139975915966463, +SNULL, 139975915925504, 139975915950079, +STORE, 139975915950080, 139975915966463, +STORE, 139975915925504, 139975915950079, +ERASE, 139975915925504, 139975915950079, +STORE, 139975915925504, 139975915950079, +ERASE, 139975915950080, 139975915966463, +STORE, 139975915950080, 139975915966463, +SNULL, 139975915941887, 139975915950079, +STORE, 139975915925504, 139975915941887, +STORE, 139975915941888, 139975915950079, +SNULL, 94016798146559, 94016798150655, +STORE, 94016798138368, 94016798146559, +STORE, 94016798146560, 94016798150655, +SNULL, 139975918211071, 139975918215167, +STORE, 139975918206976, 139975918211071, +STORE, 139975918211072, 139975918215167, +ERASE, 139975918178304, 139975918206975, +STORE, 94016804925440, 94016805060607, +STORE, 94596177661952, 94596177772543, +STORE, 94596179865600, 94596179873791, +STORE, 94596179873792, 94596179877887, +STORE, 94596179877888, 94596179886079, +STORE, 94596211597312, 94596211863551, +STORE, 140127351840768, 140127353499647, +STORE, 140127353499648, 140127355596799, +STORE, 140127355596800, 140127355613183, +STORE, 140127355613184, 140127355621375, +STORE, 140127355621376, 140127355637759, +STORE, 140127355637760, 140127355781119, +STORE, 140127357841408, 140127357849599, +STORE, 140127357878272, 140127357882367, +STORE, 140127357882368, 140127357886463, +STORE, 140127357886464, 140127357890559, +STORE, 140726167252992, 140726167392255, +STORE, 140726167838720, 140726167851007, +STORE, 140726167851008, 140726167855103, +STORE, 140737488347136, 140737488351231, +STORE, 140731874017280, 140737488351231, +SNULL, 140731874021375, 140737488351231, +STORE, 140731874017280, 140731874021375, +STORE, 140731873886208, 140731874021375, +STORE, 94178682265600, 94178684489727, +SNULL, 94178682376191, 94178684489727, +STORE, 94178682265600, 94178682376191, +STORE, 94178682376192, 94178684489727, +ERASE, 94178682376192, 94178684489727, +STORE, 94178684469248, 94178684481535, +STORE, 94178684481536, 94178684489727, +STORE, 140460853403648, 140460855656447, +SNULL, 140460853547007, 140460855656447, +STORE, 140460853403648, 140460853547007, +STORE, 140460853547008, 140460855656447, +ERASE, 140460853547008, 140460855656447, +STORE, 140460855644160, 140460855652351, +STORE, 140460855652352, 140460855656447, +STORE, 140731874103296, 140731874107391, +STORE, 140731874091008, 140731874103295, +STORE, 140460855615488, 140460855644159, +STORE, 140460855607296, 140460855615487, +STORE, 140460849606656, 140460853403647, +SNULL, 140460849606656, 140460851265535, +STORE, 140460851265536, 140460853403647, +STORE, 140460849606656, 140460851265535, +SNULL, 140460853362687, 140460853403647, +STORE, 140460851265536, 140460853362687, +STORE, 140460853362688, 140460853403647, +SNULL, 140460853362688, 140460853387263, +STORE, 140460853387264, 140460853403647, +STORE, 140460853362688, 140460853387263, +ERASE, 140460853362688, 140460853387263, +STORE, 140460853362688, 140460853387263, +ERASE, 140460853387264, 140460853403647, +STORE, 140460853387264, 140460853403647, +SNULL, 140460853379071, 140460853387263, +STORE, 140460853362688, 140460853379071, +STORE, 140460853379072, 140460853387263, +SNULL, 94178684477439, 94178684481535, +STORE, 94178684469248, 94178684477439, +STORE, 94178684477440, 94178684481535, +SNULL, 140460855648255, 140460855652351, +STORE, 140460855644160, 140460855648255, +STORE, 140460855648256, 140460855652351, +ERASE, 140460855615488, 140460855644159, +STORE, 94178692063232, 94178692198399, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733096603648, 140737488351231, +SNULL, 140733096611839, 140737488351231, +STORE, 140733096603648, 140733096611839, +STORE, 140733096472576, 140733096611839, +STORE, 94796716122112, 94796718325759, +SNULL, 94796716224511, 94796718325759, +STORE, 94796716122112, 94796716224511, +STORE, 94796716224512, 94796718325759, +ERASE, 94796716224512, 94796718325759, +STORE, 94796718317568, 94796718325759, +STORE, 139667892793344, 139667895046143, +SNULL, 139667892936703, 139667895046143, +STORE, 139667892793344, 139667892936703, +STORE, 139667892936704, 139667895046143, +ERASE, 139667892936704, 139667895046143, +STORE, 139667895033856, 139667895042047, +STORE, 139667895042048, 139667895046143, +STORE, 140733096857600, 140733096861695, +STORE, 140733096845312, 140733096857599, +STORE, 139667895005184, 139667895033855, +STORE, 139667894996992, 139667895005183, +STORE, 139667890532352, 139667892793343, +SNULL, 139667890532352, 139667890683903, +STORE, 139667890683904, 139667892793343, +STORE, 139667890532352, 139667890683903, +SNULL, 139667892776959, 139667892793343, +STORE, 139667890683904, 139667892776959, +STORE, 139667892776960, 139667892793343, +SNULL, 139667892776960, 139667892785151, +STORE, 139667892785152, 139667892793343, +STORE, 139667892776960, 139667892785151, +ERASE, 139667892776960, 139667892785151, +STORE, 139667892776960, 139667892785151, +ERASE, 139667892785152, 139667892793343, +STORE, 139667892785152, 139667892793343, +STORE, 139667886735360, 139667890532351, +SNULL, 139667886735360, 139667888394239, +STORE, 139667888394240, 139667890532351, +STORE, 139667886735360, 139667888394239, +SNULL, 139667890491391, 139667890532351, +STORE, 139667888394240, 139667890491391, +STORE, 139667890491392, 139667890532351, +SNULL, 139667890491392, 139667890515967, +STORE, 139667890515968, 139667890532351, +STORE, 139667890491392, 139667890515967, +ERASE, 139667890491392, 139667890515967, +STORE, 139667890491392, 139667890515967, +ERASE, 139667890515968, 139667890532351, +STORE, 139667890515968, 139667890532351, +STORE, 139667884167168, 139667886735359, +SNULL, 139667884167168, 139667884634111, +STORE, 139667884634112, 139667886735359, +STORE, 139667884167168, 139667884634111, +SNULL, 139667886727167, 139667886735359, +STORE, 139667884634112, 139667886727167, +STORE, 139667886727168, 139667886735359, +ERASE, 139667886727168, 139667886735359, +STORE, 139667886727168, 139667886735359, +STORE, 139667882053632, 139667884167167, +SNULL, 139667882053632, 139667882065919, +STORE, 139667882065920, 139667884167167, +STORE, 139667882053632, 139667882065919, +SNULL, 139667884158975, 139667884167167, +STORE, 139667882065920, 139667884158975, +STORE, 139667884158976, 139667884167167, +ERASE, 139667884158976, 139667884167167, +STORE, 139667884158976, 139667884167167, +STORE, 139667879837696, 139667882053631, +SNULL, 139667879837696, 139667879935999, +STORE, 139667879936000, 139667882053631, +STORE, 139667879837696, 139667879935999, +SNULL, 139667882029055, 139667882053631, +STORE, 139667879936000, 139667882029055, +STORE, 139667882029056, 139667882053631, +SNULL, 139667882029056, 139667882037247, +STORE, 139667882037248, 139667882053631, +STORE, 139667882029056, 139667882037247, +ERASE, 139667882029056, 139667882037247, +STORE, 139667882029056, 139667882037247, +ERASE, 139667882037248, 139667882053631, +STORE, 139667882037248, 139667882053631, +STORE, 139667894988800, 139667895005183, +SNULL, 139667890507775, 139667890515967, +STORE, 139667890491392, 139667890507775, +STORE, 139667890507776, 139667890515967, +SNULL, 139667882033151, 139667882037247, +STORE, 139667882029056, 139667882033151, +STORE, 139667882033152, 139667882037247, +SNULL, 139667884163071, 139667884167167, +STORE, 139667884158976, 139667884163071, +STORE, 139667884163072, 139667884167167, +SNULL, 139667886731263, 139667886735359, +STORE, 139667886727168, 139667886731263, +STORE, 139667886731264, 139667886735359, +SNULL, 139667892781055, 139667892785151, +STORE, 139667892776960, 139667892781055, +STORE, 139667892781056, 139667892785151, +SNULL, 94796718321663, 94796718325759, +STORE, 94796718317568, 94796718321663, +STORE, 94796718321664, 94796718325759, +SNULL, 139667895037951, 139667895042047, +STORE, 139667895033856, 139667895037951, +STORE, 139667895037952, 139667895042047, +ERASE, 139667895005184, 139667895033855, +STORE, 94796726063104, 94796726198271, +STORE, 139667893305344, 139667894988799, +STORE, 139667895005184, 139667895033855, +STORE, 94796726063104, 94796726333439, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722489507840, 140737488351231, +SNULL, 140722489516031, 140737488351231, +STORE, 140722489507840, 140722489516031, +STORE, 140722489376768, 140722489516031, +STORE, 93980993265664, 93980995489791, +SNULL, 93980993376255, 93980995489791, +STORE, 93980993265664, 93980993376255, +STORE, 93980993376256, 93980995489791, +ERASE, 93980993376256, 93980995489791, +STORE, 93980995469312, 93980995481599, +STORE, 93980995481600, 93980995489791, +STORE, 140261313593344, 140261315846143, +SNULL, 140261313736703, 140261315846143, +STORE, 140261313593344, 140261313736703, +STORE, 140261313736704, 140261315846143, +ERASE, 140261313736704, 140261315846143, +STORE, 140261315833856, 140261315842047, +STORE, 140261315842048, 140261315846143, +STORE, 140722489675776, 140722489679871, +STORE, 140722489663488, 140722489675775, +STORE, 140261315805184, 140261315833855, +STORE, 140261315796992, 140261315805183, +STORE, 140261309796352, 140261313593343, +SNULL, 140261309796352, 140261311455231, +STORE, 140261311455232, 140261313593343, +STORE, 140261309796352, 140261311455231, +SNULL, 140261313552383, 140261313593343, +STORE, 140261311455232, 140261313552383, +STORE, 140261313552384, 140261313593343, +SNULL, 140261313552384, 140261313576959, +STORE, 140261313576960, 140261313593343, +STORE, 140261313552384, 140261313576959, +ERASE, 140261313552384, 140261313576959, +STORE, 140261313552384, 140261313576959, +ERASE, 140261313576960, 140261313593343, +STORE, 140261313576960, 140261313593343, +SNULL, 140261313568767, 140261313576959, +STORE, 140261313552384, 140261313568767, +STORE, 140261313568768, 140261313576959, +SNULL, 93980995477503, 93980995481599, +STORE, 93980995469312, 93980995477503, +STORE, 93980995477504, 93980995481599, +SNULL, 140261315837951, 140261315842047, +STORE, 140261315833856, 140261315837951, +STORE, 140261315837952, 140261315842047, +ERASE, 140261315805184, 140261315833855, +STORE, 93980997443584, 93980997578751, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140734059450368, 140737488351231, +SNULL, 140734059462655, 140737488351231, +STORE, 140734059450368, 140734059462655, +STORE, 140734059319296, 140734059462655, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140307554983936, 140307557236735, +SNULL, 140307555127295, 140307557236735, +STORE, 140307554983936, 140307555127295, +STORE, 140307555127296, 140307557236735, +ERASE, 140307555127296, 140307557236735, +STORE, 140307557224448, 140307557232639, +STORE, 140307557232640, 140307557236735, +STORE, 140734059483136, 140734059487231, +STORE, 140734059470848, 140734059483135, +STORE, 140307557195776, 140307557224447, +STORE, 140307557187584, 140307557195775, +STORE, 140307551821824, 140307554983935, +SNULL, 140307551821824, 140307552882687, +STORE, 140307552882688, 140307554983935, +STORE, 140307551821824, 140307552882687, +SNULL, 140307554975743, 140307554983935, +STORE, 140307552882688, 140307554975743, +STORE, 140307554975744, 140307554983935, +ERASE, 140307554975744, 140307554983935, +STORE, 140307554975744, 140307554983935, +STORE, 140307548024832, 140307551821823, +SNULL, 140307548024832, 140307549683711, +STORE, 140307549683712, 140307551821823, +STORE, 140307548024832, 140307549683711, +SNULL, 140307551780863, 140307551821823, +STORE, 140307549683712, 140307551780863, +STORE, 140307551780864, 140307551821823, +SNULL, 140307551780864, 140307551805439, +STORE, 140307551805440, 140307551821823, +STORE, 140307551780864, 140307551805439, +ERASE, 140307551780864, 140307551805439, +STORE, 140307551780864, 140307551805439, +ERASE, 140307551805440, 140307551821823, +STORE, 140307551805440, 140307551821823, +STORE, 140307557175296, 140307557195775, +SNULL, 140307551797247, 140307551805439, +STORE, 140307551780864, 140307551797247, +STORE, 140307551797248, 140307551805439, +SNULL, 140307554979839, 140307554983935, +STORE, 140307554975744, 140307554979839, +STORE, 140307554979840, 140307554983935, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140307557228543, 140307557232639, +STORE, 140307557224448, 140307557228543, +STORE, 140307557228544, 140307557232639, +ERASE, 140307557195776, 140307557224447, +STORE, 39698432, 39833599, +STORE, 39698432, 39981055, +STORE, 94306485321728, 94306485432319, +STORE, 94306487525376, 94306487533567, +STORE, 94306487533568, 94306487537663, +STORE, 94306487537664, 94306487545855, +STORE, 94306488868864, 94306489004031, +STORE, 140497673998336, 140497675657215, +STORE, 140497675657216, 140497677754367, +STORE, 140497677754368, 140497677770751, +STORE, 140497677770752, 140497677778943, +STORE, 140497677778944, 140497677795327, +STORE, 140497677795328, 140497677938687, +STORE, 140497679998976, 140497680007167, +STORE, 140497680035840, 140497680039935, +STORE, 140497680039936, 140497680044031, +STORE, 140497680044032, 140497680048127, +STORE, 140732780462080, 140732780601343, +STORE, 140732782239744, 140732782252031, +STORE, 140732782252032, 140732782256127, +STORE, 94236915900416, 94236916011007, +STORE, 94236918104064, 94236918112255, +STORE, 94236918112256, 94236918116351, +STORE, 94236918116352, 94236918124543, +STORE, 94236939489280, 94236939624447, +STORE, 140046091743232, 140046093402111, +STORE, 140046093402112, 140046095499263, +STORE, 140046095499264, 140046095515647, +STORE, 140046095515648, 140046095523839, +STORE, 140046095523840, 140046095540223, +STORE, 140046095540224, 140046095683583, +STORE, 140046097743872, 140046097752063, +STORE, 140046097780736, 140046097784831, +STORE, 140046097784832, 140046097788927, +STORE, 140046097788928, 140046097793023, +STORE, 140726694449152, 140726694588415, +STORE, 140726695313408, 140726695325695, +STORE, 140726695325696, 140726695329791, +STORE, 94894582779904, 94894582992895, +STORE, 94894585090048, 94894585094143, +STORE, 94894585094144, 94894585102335, +STORE, 94894585102336, 94894585114623, +STORE, 94894592868352, 94894594293759, +STORE, 139733563842560, 139733565501439, +STORE, 139733565501440, 139733567598591, +STORE, 139733567598592, 139733567614975, +STORE, 139733567614976, 139733567623167, +STORE, 139733567623168, 139733567639551, +STORE, 139733567639552, 139733567651839, +STORE, 139733567651840, 139733569744895, +STORE, 139733569744896, 139733569748991, +STORE, 139733569748992, 139733569753087, +STORE, 139733569753088, 139733569896447, +STORE, 139733570265088, 139733571948543, +STORE, 139733571948544, 139733571964927, +STORE, 139733571993600, 139733571997695, +STORE, 139733571997696, 139733572001791, +STORE, 139733572001792, 139733572005887, +STORE, 140726369255424, 140726369394687, +STORE, 140726370402304, 140726370414591, +STORE, 140726370414592, 140726370418687, +STORE, 94899236483072, 94899236696063, +STORE, 94899238793216, 94899238797311, +STORE, 94899238797312, 94899238805503, +STORE, 94899238805504, 94899238817791, +STORE, 94899263045632, 94899263979519, +STORE, 140040959893504, 140040961552383, +STORE, 140040961552384, 140040963649535, +STORE, 140040963649536, 140040963665919, +STORE, 140040963665920, 140040963674111, +STORE, 140040963674112, 140040963690495, +STORE, 140040963690496, 140040963702783, +STORE, 140040963702784, 140040965795839, +STORE, 140040965795840, 140040965799935, +STORE, 140040965799936, 140040965804031, +STORE, 140040965804032, 140040965947391, +STORE, 140040966316032, 140040967999487, +STORE, 140040967999488, 140040968015871, +STORE, 140040968044544, 140040968048639, +STORE, 140040968048640, 140040968052735, +STORE, 140040968052736, 140040968056831, +STORE, 140729921359872, 140729921499135, +STORE, 140729921613824, 140729921626111, +STORE, 140729921626112, 140729921630207, +STORE, 94818265190400, 94818265403391, +STORE, 94818267500544, 94818267504639, +STORE, 94818267504640, 94818267512831, +STORE, 94818267512832, 94818267525119, +STORE, 94818283372544, 94818285858815, +STORE, 139818425675776, 139818427334655, +STORE, 139818427334656, 139818429431807, +STORE, 139818429431808, 139818429448191, +STORE, 139818429448192, 139818429456383, +STORE, 139818429456384, 139818429472767, +STORE, 139818429472768, 139818429485055, +STORE, 139818429485056, 139818431578111, +STORE, 139818431578112, 139818431582207, +STORE, 139818431582208, 139818431586303, +STORE, 139818431586304, 139818431729663, +STORE, 139818432098304, 139818433781759, +STORE, 139818433781760, 139818433798143, +STORE, 139818433826816, 139818433830911, +STORE, 139818433830912, 139818433835007, +STORE, 139818433835008, 139818433839103, +STORE, 140726170509312, 140726170648575, +STORE, 140726171824128, 140726171836415, +STORE, 140726171836416, 140726171840511, +STORE, 94611513188352, 94611513401343, +STORE, 94611515498496, 94611515502591, +STORE, 94611515502592, 94611515510783, +STORE, 94611515510784, 94611515523071, +STORE, 94611516502016, 94611516907519, +STORE, 140596246388736, 140596248047615, +STORE, 140596248047616, 140596250144767, +STORE, 140596250144768, 140596250161151, +STORE, 140596250161152, 140596250169343, +STORE, 140596250169344, 140596250185727, +STORE, 140596250185728, 140596250198015, +STORE, 140596250198016, 140596252291071, +STORE, 140596252291072, 140596252295167, +STORE, 140596252295168, 140596252299263, +STORE, 140596252299264, 140596252442623, +STORE, 140596252811264, 140596254494719, +STORE, 140596254494720, 140596254511103, +STORE, 140596254539776, 140596254543871, +STORE, 140596254543872, 140596254547967, +STORE, 140596254547968, 140596254552063, +STORE, 140731551338496, 140731551477759, +STORE, 140731551780864, 140731551793151, +STORE, 140731551793152, 140731551797247, +STORE, 94313835851776, 94313836064767, +STORE, 94313838161920, 94313838166015, +STORE, 94313838166016, 94313838174207, +STORE, 94313838174208, 94313838186495, +STORE, 94313858416640, 94313861906431, +STORE, 140693503918080, 140693505576959, +STORE, 140693505576960, 140693507674111, +STORE, 140693507674112, 140693507690495, +STORE, 140693507690496, 140693507698687, +STORE, 140693507698688, 140693507715071, +STORE, 140693507715072, 140693507727359, +STORE, 140693507727360, 140693509820415, +STORE, 140693509820416, 140693509824511, +STORE, 140693509824512, 140693509828607, +STORE, 140693509828608, 140693509971967, +STORE, 140693510340608, 140693512024063, +STORE, 140693512024064, 140693512040447, +STORE, 140693512069120, 140693512073215, +STORE, 140693512073216, 140693512077311, +STORE, 140693512077312, 140693512081407, +STORE, 140721116065792, 140721116205055, +STORE, 140721117831168, 140721117843455, +STORE, 140721117843456, 140721117847551, +STORE, 94843650150400, 94843650363391, +STORE, 94843652460544, 94843652464639, +STORE, 94843652464640, 94843652472831, +STORE, 94843652472832, 94843652485119, +STORE, 94843685388288, 94843686281215, +STORE, 140484193681408, 140484195340287, +STORE, 140484195340288, 140484197437439, +STORE, 140484197437440, 140484197453823, +STORE, 140484197453824, 140484197462015, +STORE, 140484197462016, 140484197478399, +STORE, 140484197478400, 140484197490687, +STORE, 140484197490688, 140484199583743, +STORE, 140484199583744, 140484199587839, +STORE, 140484199587840, 140484199591935, +STORE, 140484199591936, 140484199735295, +STORE, 140484200103936, 140484201787391, +STORE, 140484201787392, 140484201803775, +STORE, 140484201832448, 140484201836543, +STORE, 140484201836544, 140484201840639, +STORE, 140484201840640, 140484201844735, +STORE, 140726294315008, 140726294454271, +STORE, 140726295646208, 140726295658495, +STORE, 140726295658496, 140726295662591, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720422371328, 140737488351231, +SNULL, 140720422379519, 140737488351231, +STORE, 140720422371328, 140720422379519, +STORE, 140720422240256, 140720422379519, +STORE, 94417967845376, 94417970180095, +SNULL, 94417968058367, 94417970180095, +STORE, 94417967845376, 94417968058367, +STORE, 94417968058368, 94417970180095, +ERASE, 94417968058368, 94417970180095, +STORE, 94417970155520, 94417970167807, +STORE, 94417970167808, 94417970180095, +STORE, 140252450045952, 140252452298751, +SNULL, 140252450189311, 140252452298751, +STORE, 140252450045952, 140252450189311, +STORE, 140252450189312, 140252452298751, +ERASE, 140252450189312, 140252452298751, +STORE, 140252452286464, 140252452294655, +STORE, 140252452294656, 140252452298751, +STORE, 140720422416384, 140720422420479, +STORE, 140720422404096, 140720422416383, +STORE, 140252452257792, 140252452286463, +STORE, 140252452249600, 140252452257791, +STORE, 140252447932416, 140252450045951, +SNULL, 140252447932416, 140252447944703, +STORE, 140252447944704, 140252450045951, +STORE, 140252447932416, 140252447944703, +SNULL, 140252450037759, 140252450045951, +STORE, 140252447944704, 140252450037759, +STORE, 140252450037760, 140252450045951, +ERASE, 140252450037760, 140252450045951, +STORE, 140252450037760, 140252450045951, +STORE, 140252444135424, 140252447932415, +SNULL, 140252444135424, 140252445794303, +STORE, 140252445794304, 140252447932415, +STORE, 140252444135424, 140252445794303, +SNULL, 140252447891455, 140252447932415, +STORE, 140252445794304, 140252447891455, +STORE, 140252447891456, 140252447932415, +SNULL, 140252447891456, 140252447916031, +STORE, 140252447916032, 140252447932415, +STORE, 140252447891456, 140252447916031, +ERASE, 140252447891456, 140252447916031, +STORE, 140252447891456, 140252447916031, +ERASE, 140252447916032, 140252447932415, +STORE, 140252447916032, 140252447932415, +STORE, 140252452241408, 140252452257791, +SNULL, 140252447907839, 140252447916031, +STORE, 140252447891456, 140252447907839, +STORE, 140252447907840, 140252447916031, +SNULL, 140252450041855, 140252450045951, +STORE, 140252450037760, 140252450041855, +STORE, 140252450041856, 140252450045951, +SNULL, 94417970159615, 94417970167807, +STORE, 94417970155520, 94417970159615, +STORE, 94417970159616, 94417970167807, +SNULL, 140252452290559, 140252452294655, +STORE, 140252452286464, 140252452290559, +STORE, 140252452290560, 140252452294655, +ERASE, 140252452257792, 140252452286463, +STORE, 94417996333056, 94417996468223, +STORE, 140252450557952, 140252452241407, +STORE, 94417996333056, 94417996603391, +STORE, 94417996333056, 94417996738559, +STORE, 94417996333056, 94417996910591, +SNULL, 94417996881919, 94417996910591, +STORE, 94417996333056, 94417996881919, +STORE, 94417996881920, 94417996910591, +ERASE, 94417996881920, 94417996910591, +STORE, 94417996333056, 94417997017087, +STORE, 94417996333056, 94417997152255, +SNULL, 94417997135871, 94417997152255, +STORE, 94417996333056, 94417997135871, +STORE, 94417997135872, 94417997152255, +ERASE, 94417997135872, 94417997152255, +STORE, 94417996333056, 94417997291519, +SNULL, 94417997271039, 94417997291519, +STORE, 94417996333056, 94417997271039, +STORE, 94417997271040, 94417997291519, +ERASE, 94417997271040, 94417997291519, +STORE, 94417996333056, 94417997406207, +SNULL, 94417997381631, 94417997406207, +STORE, 94417996333056, 94417997381631, +STORE, 94417997381632, 94417997406207, +ERASE, 94417997381632, 94417997406207, +STORE, 94417996333056, 94417997516799, +SNULL, 94417997488127, 94417997516799, +STORE, 94417996333056, 94417997488127, +STORE, 94417997488128, 94417997516799, +ERASE, 94417997488128, 94417997516799, +STORE, 94417996333056, 94417997643775, +SNULL, 94417997631487, 94417997643775, +STORE, 94417996333056, 94417997631487, +STORE, 94417997631488, 94417997643775, +ERASE, 94417997631488, 94417997643775, +SNULL, 94417997590527, 94417997631487, +STORE, 94417996333056, 94417997590527, +STORE, 94417997590528, 94417997631487, +ERASE, 94417997590528, 94417997631487, +STORE, 94417996333056, 94417997733887, +STORE, 94417996333056, 94417997869055, +STORE, 94417996333056, 94417998004223, +SNULL, 94417998000127, 94417998004223, +STORE, 94417996333056, 94417998000127, +STORE, 94417998000128, 94417998004223, +ERASE, 94417998000128, 94417998004223, +STORE, 94049170993152, 94049171206143, +STORE, 94049173303296, 94049173307391, +STORE, 94049173307392, 94049173315583, +STORE, 94049173315584, 94049173327871, +STORE, 94049176236032, 94049183645695, +STORE, 139807795544064, 139807797202943, +STORE, 139807797202944, 139807799300095, +STORE, 139807799300096, 139807799316479, +STORE, 139807799316480, 139807799324671, +STORE, 139807799324672, 139807799341055, +STORE, 139807799341056, 139807799353343, +STORE, 139807799353344, 139807801446399, +STORE, 139807801446400, 139807801450495, +STORE, 139807801450496, 139807801454591, +STORE, 139807801454592, 139807801597951, +STORE, 139807801966592, 139807803650047, +STORE, 139807803650048, 139807803666431, +STORE, 139807803695104, 139807803699199, +STORE, 139807803699200, 139807803703295, +STORE, 139807803703296, 139807803707391, +STORE, 140727555538944, 140727555678207, +STORE, 140727555940352, 140727555952639, +STORE, 140727555952640, 140727555956735, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722483441664, 140737488351231, +SNULL, 140722483449855, 140737488351231, +STORE, 140722483441664, 140722483449855, +STORE, 140722483310592, 140722483449855, +STORE, 94416704921600, 94416707145727, +SNULL, 94416705032191, 94416707145727, +STORE, 94416704921600, 94416705032191, +STORE, 94416705032192, 94416707145727, +ERASE, 94416705032192, 94416707145727, +STORE, 94416707125248, 94416707137535, +STORE, 94416707137536, 94416707145727, +STORE, 140555439296512, 140555441549311, +SNULL, 140555439439871, 140555441549311, +STORE, 140555439296512, 140555439439871, +STORE, 140555439439872, 140555441549311, +ERASE, 140555439439872, 140555441549311, +STORE, 140555441537024, 140555441545215, +STORE, 140555441545216, 140555441549311, +STORE, 140722484781056, 140722484785151, +STORE, 140722484768768, 140722484781055, +STORE, 140555441508352, 140555441537023, +STORE, 140555441500160, 140555441508351, +STORE, 140555435499520, 140555439296511, +SNULL, 140555435499520, 140555437158399, +STORE, 140555437158400, 140555439296511, +STORE, 140555435499520, 140555437158399, +SNULL, 140555439255551, 140555439296511, +STORE, 140555437158400, 140555439255551, +STORE, 140555439255552, 140555439296511, +SNULL, 140555439255552, 140555439280127, +STORE, 140555439280128, 140555439296511, +STORE, 140555439255552, 140555439280127, +ERASE, 140555439255552, 140555439280127, +STORE, 140555439255552, 140555439280127, +ERASE, 140555439280128, 140555439296511, +STORE, 140555439280128, 140555439296511, +SNULL, 140555439271935, 140555439280127, +STORE, 140555439255552, 140555439271935, +STORE, 140555439271936, 140555439280127, +SNULL, 94416707133439, 94416707137535, +STORE, 94416707125248, 94416707133439, +STORE, 94416707133440, 94416707137535, +SNULL, 140555441541119, 140555441545215, +STORE, 140555441537024, 140555441541119, +STORE, 140555441541120, 140555441545215, +ERASE, 140555441508352, 140555441537023, +STORE, 94416724672512, 94416724807679, +STORE, 94686636953600, 94686637166591, +STORE, 94686639263744, 94686639267839, +STORE, 94686639267840, 94686639276031, +STORE, 94686639276032, 94686639288319, +STORE, 94686662193152, 94686663163903, +STORE, 140312944431104, 140312946089983, +STORE, 140312946089984, 140312948187135, +STORE, 140312948187136, 140312948203519, +STORE, 140312948203520, 140312948211711, +STORE, 140312948211712, 140312948228095, +STORE, 140312948228096, 140312948240383, +STORE, 140312948240384, 140312950333439, +STORE, 140312950333440, 140312950337535, +STORE, 140312950337536, 140312950341631, +STORE, 140312950341632, 140312950484991, +STORE, 140312950853632, 140312952537087, +STORE, 140312952537088, 140312952553471, +STORE, 140312952582144, 140312952586239, +STORE, 140312952586240, 140312952590335, +STORE, 140312952590336, 140312952594431, +STORE, 140730598920192, 140730599059455, +STORE, 140730599108608, 140730599120895, +STORE, 140730599120896, 140730599124991, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726234079232, 140737488351231, +SNULL, 140726234087423, 140737488351231, +STORE, 140726234079232, 140726234087423, +STORE, 140726233948160, 140726234087423, +STORE, 94589467578368, 94589469802495, +SNULL, 94589467688959, 94589469802495, +STORE, 94589467578368, 94589467688959, +STORE, 94589467688960, 94589469802495, +ERASE, 94589467688960, 94589469802495, +STORE, 94589469782016, 94589469794303, +STORE, 94589469794304, 94589469802495, +STORE, 140587082842112, 140587085094911, +SNULL, 140587082985471, 140587085094911, +STORE, 140587082842112, 140587082985471, +STORE, 140587082985472, 140587085094911, +ERASE, 140587082985472, 140587085094911, +STORE, 140587085082624, 140587085090815, +STORE, 140587085090816, 140587085094911, +STORE, 140726234103808, 140726234107903, +STORE, 140726234091520, 140726234103807, +STORE, 140587085053952, 140587085082623, +STORE, 140587085045760, 140587085053951, +STORE, 140587079045120, 140587082842111, +SNULL, 140587079045120, 140587080703999, +STORE, 140587080704000, 140587082842111, +STORE, 140587079045120, 140587080703999, +SNULL, 140587082801151, 140587082842111, +STORE, 140587080704000, 140587082801151, +STORE, 140587082801152, 140587082842111, +SNULL, 140587082801152, 140587082825727, +STORE, 140587082825728, 140587082842111, +STORE, 140587082801152, 140587082825727, +ERASE, 140587082801152, 140587082825727, +STORE, 140587082801152, 140587082825727, +ERASE, 140587082825728, 140587082842111, +STORE, 140587082825728, 140587082842111, +SNULL, 140587082817535, 140587082825727, +STORE, 140587082801152, 140587082817535, +STORE, 140587082817536, 140587082825727, +SNULL, 94589469790207, 94589469794303, +STORE, 94589469782016, 94589469790207, +STORE, 94589469790208, 94589469794303, +SNULL, 140587085086719, 140587085090815, +STORE, 140587085082624, 140587085086719, +STORE, 140587085086720, 140587085090815, +ERASE, 140587085053952, 140587085082623, +STORE, 94589477507072, 94589477642239, +STORE, 94225448325120, 94225448538111, +STORE, 94225450635264, 94225450639359, +STORE, 94225450639360, 94225450647551, +STORE, 94225450647552, 94225450659839, +STORE, 94225470246912, 94225473548287, +STORE, 140199245496320, 140199247155199, +STORE, 140199247155200, 140199249252351, +STORE, 140199249252352, 140199249268735, +STORE, 140199249268736, 140199249276927, +STORE, 140199249276928, 140199249293311, +STORE, 140199249293312, 140199249305599, +STORE, 140199249305600, 140199251398655, +STORE, 140199251398656, 140199251402751, +STORE, 140199251402752, 140199251406847, +STORE, 140199251406848, 140199251550207, +STORE, 140199251918848, 140199253602303, +STORE, 140199253602304, 140199253618687, +STORE, 140199253647360, 140199253651455, +STORE, 140199253651456, 140199253655551, +STORE, 140199253655552, 140199253659647, +STORE, 140726264414208, 140726264553471, +STORE, 140726265843712, 140726265855999, +STORE, 140726265856000, 140726265860095, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733508358144, 140737488351231, +SNULL, 140733508366335, 140737488351231, +STORE, 140733508358144, 140733508366335, +STORE, 140733508227072, 140733508366335, +STORE, 94766263947264, 94766266171391, +SNULL, 94766264057855, 94766266171391, +STORE, 94766263947264, 94766264057855, +STORE, 94766264057856, 94766266171391, +ERASE, 94766264057856, 94766266171391, +STORE, 94766266150912, 94766266163199, +STORE, 94766266163200, 94766266171391, +STORE, 140693985132544, 140693987385343, +SNULL, 140693985275903, 140693987385343, +STORE, 140693985132544, 140693985275903, +STORE, 140693985275904, 140693987385343, +ERASE, 140693985275904, 140693987385343, +STORE, 140693987373056, 140693987381247, +STORE, 140693987381248, 140693987385343, +STORE, 140733509939200, 140733509943295, +STORE, 140733509926912, 140733509939199, +STORE, 140693987344384, 140693987373055, +STORE, 140693987336192, 140693987344383, +STORE, 140693981335552, 140693985132543, +SNULL, 140693981335552, 140693982994431, +STORE, 140693982994432, 140693985132543, +STORE, 140693981335552, 140693982994431, +SNULL, 140693985091583, 140693985132543, +STORE, 140693982994432, 140693985091583, +STORE, 140693985091584, 140693985132543, +SNULL, 140693985091584, 140693985116159, +STORE, 140693985116160, 140693985132543, +STORE, 140693985091584, 140693985116159, +ERASE, 140693985091584, 140693985116159, +STORE, 140693985091584, 140693985116159, +ERASE, 140693985116160, 140693985132543, +STORE, 140693985116160, 140693985132543, +SNULL, 140693985107967, 140693985116159, +STORE, 140693985091584, 140693985107967, +STORE, 140693985107968, 140693985116159, +SNULL, 94766266159103, 94766266163199, +STORE, 94766266150912, 94766266159103, +STORE, 94766266159104, 94766266163199, +SNULL, 140693987377151, 140693987381247, +STORE, 140693987373056, 140693987377151, +STORE, 140693987377152, 140693987381247, +ERASE, 140693987344384, 140693987373055, +STORE, 94766282035200, 94766282170367, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724769353728, 140737488351231, +SNULL, 140724769361919, 140737488351231, +STORE, 140724769353728, 140724769361919, +STORE, 140724769222656, 140724769361919, +STORE, 94710460526592, 94710462750719, +SNULL, 94710460637183, 94710462750719, +STORE, 94710460526592, 94710460637183, +STORE, 94710460637184, 94710462750719, +ERASE, 94710460637184, 94710462750719, +STORE, 94710462730240, 94710462742527, +STORE, 94710462742528, 94710462750719, +STORE, 140469764395008, 140469766647807, +SNULL, 140469764538367, 140469766647807, +STORE, 140469764395008, 140469764538367, +STORE, 140469764538368, 140469766647807, +ERASE, 140469764538368, 140469766647807, +STORE, 140469766635520, 140469766643711, +STORE, 140469766643712, 140469766647807, +STORE, 140724770877440, 140724770881535, +STORE, 140724770865152, 140724770877439, +STORE, 140469766606848, 140469766635519, +STORE, 140469766598656, 140469766606847, +STORE, 140469760598016, 140469764395007, +SNULL, 140469760598016, 140469762256895, +STORE, 140469762256896, 140469764395007, +STORE, 140469760598016, 140469762256895, +SNULL, 140469764354047, 140469764395007, +STORE, 140469762256896, 140469764354047, +STORE, 140469764354048, 140469764395007, +SNULL, 140469764354048, 140469764378623, +STORE, 140469764378624, 140469764395007, +STORE, 140469764354048, 140469764378623, +ERASE, 140469764354048, 140469764378623, +STORE, 140469764354048, 140469764378623, +ERASE, 140469764378624, 140469764395007, +STORE, 140469764378624, 140469764395007, +SNULL, 140469764370431, 140469764378623, +STORE, 140469764354048, 140469764370431, +STORE, 140469764370432, 140469764378623, +SNULL, 94710462738431, 94710462742527, +STORE, 94710462730240, 94710462738431, +STORE, 94710462738432, 94710462742527, +SNULL, 140469766639615, 140469766643711, +STORE, 140469766635520, 140469766639615, +STORE, 140469766639616, 140469766643711, +ERASE, 140469766606848, 140469766635519, +STORE, 94710485581824, 94710485716991, +STORE, 94105755795456, 94105756008447, +STORE, 94105758105600, 94105758109695, +STORE, 94105758109696, 94105758117887, +STORE, 94105758117888, 94105758130175, +STORE, 94105788981248, 94105794871295, +STORE, 140641190031360, 140641191690239, +STORE, 140641191690240, 140641193787391, +STORE, 140641193787392, 140641193803775, +STORE, 140641193803776, 140641193811967, +STORE, 140641193811968, 140641193828351, +STORE, 140641193828352, 140641193840639, +STORE, 140641193840640, 140641195933695, +STORE, 140641195933696, 140641195937791, +STORE, 140641195937792, 140641195941887, +STORE, 140641195941888, 140641196085247, +STORE, 140641196453888, 140641198137343, +STORE, 140641198137344, 140641198153727, +STORE, 140641198182400, 140641198186495, +STORE, 140641198186496, 140641198190591, +STORE, 140641198190592, 140641198194687, +STORE, 140731980034048, 140731980173311, +STORE, 140731981078528, 140731981090815, +STORE, 140731981090816, 140731981094911, +STORE, 93828086431744, 93828086644735, +STORE, 93828088741888, 93828088745983, +STORE, 93828088745984, 93828088754175, +STORE, 93828088754176, 93828088766463, +STORE, 93828094193664, 93828096831487, +STORE, 139844717334528, 139844718993407, +STORE, 139844718993408, 139844721090559, +STORE, 139844721090560, 139844721106943, +STORE, 139844721106944, 139844721115135, +STORE, 139844721115136, 139844721131519, +STORE, 139844721131520, 139844721143807, +STORE, 139844721143808, 139844723236863, +STORE, 139844723236864, 139844723240959, +STORE, 139844723240960, 139844723245055, +STORE, 139844723245056, 139844723388415, +STORE, 139844723757056, 139844725440511, +STORE, 139844725440512, 139844725456895, +STORE, 139844725485568, 139844725489663, +STORE, 139844725489664, 139844725493759, +STORE, 139844725493760, 139844725497855, +STORE, 140729996185600, 140729996324863, +STORE, 140729996828672, 140729996840959, +STORE, 140729996840960, 140729996845055, +STORE, 140737488347136, 140737488351231, +STORE, 140722494771200, 140737488351231, +SNULL, 140722494775295, 140737488351231, +STORE, 140722494771200, 140722494775295, +STORE, 140722494640128, 140722494775295, +STORE, 94324011311104, 94324013535231, +SNULL, 94324011421695, 94324013535231, +STORE, 94324011311104, 94324011421695, +STORE, 94324011421696, 94324013535231, +ERASE, 94324011421696, 94324013535231, +STORE, 94324013514752, 94324013527039, +STORE, 94324013527040, 94324013535231, +STORE, 140151462309888, 140151464562687, +SNULL, 140151462453247, 140151464562687, +STORE, 140151462309888, 140151462453247, +STORE, 140151462453248, 140151464562687, +ERASE, 140151462453248, 140151464562687, +STORE, 140151464550400, 140151464558591, +STORE, 140151464558592, 140151464562687, +STORE, 140722495467520, 140722495471615, +STORE, 140722495455232, 140722495467519, +STORE, 140151464521728, 140151464550399, +STORE, 140151464513536, 140151464521727, +STORE, 140151458512896, 140151462309887, +SNULL, 140151458512896, 140151460171775, +STORE, 140151460171776, 140151462309887, +STORE, 140151458512896, 140151460171775, +SNULL, 140151462268927, 140151462309887, +STORE, 140151460171776, 140151462268927, +STORE, 140151462268928, 140151462309887, +SNULL, 140151462268928, 140151462293503, +STORE, 140151462293504, 140151462309887, +STORE, 140151462268928, 140151462293503, +ERASE, 140151462268928, 140151462293503, +STORE, 140151462268928, 140151462293503, +ERASE, 140151462293504, 140151462309887, +STORE, 140151462293504, 140151462309887, +SNULL, 140151462285311, 140151462293503, +STORE, 140151462268928, 140151462285311, +STORE, 140151462285312, 140151462293503, +SNULL, 94324013522943, 94324013527039, +STORE, 94324013514752, 94324013522943, +STORE, 94324013522944, 94324013527039, +SNULL, 140151464554495, 140151464558591, +STORE, 140151464550400, 140151464554495, +STORE, 140151464554496, 140151464558591, +ERASE, 140151464521728, 140151464550399, +STORE, 94324024778752, 94324024913919, +STORE, 94899262967808, 94899263180799, +STORE, 94899265277952, 94899265282047, +STORE, 94899265282048, 94899265290239, +STORE, 94899265290240, 94899265302527, +STORE, 94899295469568, 94899298689023, +STORE, 140434388418560, 140434390077439, +STORE, 140434390077440, 140434392174591, +STORE, 140434392174592, 140434392190975, +STORE, 140434392190976, 140434392199167, +STORE, 140434392199168, 140434392215551, +STORE, 140434392215552, 140434392227839, +STORE, 140434392227840, 140434394320895, +STORE, 140434394320896, 140434394324991, +STORE, 140434394324992, 140434394329087, +STORE, 140434394329088, 140434394472447, +STORE, 140434394841088, 140434396524543, +STORE, 140434396524544, 140434396540927, +STORE, 140434396569600, 140434396573695, +STORE, 140434396573696, 140434396577791, +STORE, 140434396577792, 140434396581887, +STORE, 140720618135552, 140720618274815, +STORE, 140720618418176, 140720618430463, +STORE, 140720618430464, 140720618434559, +STORE, 94425529798656, 94425530011647, +STORE, 94425532108800, 94425532112895, +STORE, 94425532112896, 94425532121087, +STORE, 94425532121088, 94425532133375, +STORE, 94425557753856, 94425566576639, +STORE, 140600528470016, 140600530128895, +STORE, 140600530128896, 140600532226047, +STORE, 140600532226048, 140600532242431, +STORE, 140600532242432, 140600532250623, +STORE, 140600532250624, 140600532267007, +STORE, 140600532267008, 140600532279295, +STORE, 140600532279296, 140600534372351, +STORE, 140600534372352, 140600534376447, +STORE, 140600534376448, 140600534380543, +STORE, 140600534380544, 140600534523903, +STORE, 140600534892544, 140600536575999, +STORE, 140600536576000, 140600536592383, +STORE, 140600536621056, 140600536625151, +STORE, 140600536625152, 140600536629247, +STORE, 140600536629248, 140600536633343, +STORE, 140721857785856, 140721857925119, +STORE, 140721858068480, 140721858080767, +STORE, 140721858080768, 140721858084863, +STORE, 94425529798656, 94425530011647, +STORE, 94425532108800, 94425532112895, +STORE, 94425532112896, 94425532121087, +STORE, 94425532121088, 94425532133375, +STORE, 94425557753856, 94425568772095, +STORE, 140600528470016, 140600530128895, +STORE, 140600530128896, 140600532226047, +STORE, 140600532226048, 140600532242431, +STORE, 140600532242432, 140600532250623, +STORE, 140600532250624, 140600532267007, +STORE, 140600532267008, 140600532279295, +STORE, 140600532279296, 140600534372351, +STORE, 140600534372352, 140600534376447, +STORE, 140600534376448, 140600534380543, +STORE, 140600534380544, 140600534523903, +STORE, 140600534892544, 140600536575999, +STORE, 140600536576000, 140600536592383, +STORE, 140600536621056, 140600536625151, +STORE, 140600536625152, 140600536629247, +STORE, 140600536629248, 140600536633343, +STORE, 140721857785856, 140721857925119, +STORE, 140721858068480, 140721858080767, +STORE, 140721858080768, 140721858084863, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735611645952, 140737488351231, +SNULL, 140735611654143, 140737488351231, +STORE, 140735611645952, 140735611654143, +STORE, 140735611514880, 140735611654143, +STORE, 94592137641984, 94592139866111, +SNULL, 94592137752575, 94592139866111, +STORE, 94592137641984, 94592137752575, +STORE, 94592137752576, 94592139866111, +ERASE, 94592137752576, 94592139866111, +STORE, 94592139845632, 94592139857919, +STORE, 94592139857920, 94592139866111, +STORE, 140350425030656, 140350427283455, +SNULL, 140350425174015, 140350427283455, +STORE, 140350425030656, 140350425174015, +STORE, 140350425174016, 140350427283455, +ERASE, 140350425174016, 140350427283455, +STORE, 140350427271168, 140350427279359, +STORE, 140350427279360, 140350427283455, +STORE, 140735612043264, 140735612047359, +STORE, 140735612030976, 140735612043263, +STORE, 140350427242496, 140350427271167, +STORE, 140350427234304, 140350427242495, +STORE, 140350421233664, 140350425030655, +SNULL, 140350421233664, 140350422892543, +STORE, 140350422892544, 140350425030655, +STORE, 140350421233664, 140350422892543, +SNULL, 140350424989695, 140350425030655, +STORE, 140350422892544, 140350424989695, +STORE, 140350424989696, 140350425030655, +SNULL, 140350424989696, 140350425014271, +STORE, 140350425014272, 140350425030655, +STORE, 140350424989696, 140350425014271, +ERASE, 140350424989696, 140350425014271, +STORE, 140350424989696, 140350425014271, +ERASE, 140350425014272, 140350425030655, +STORE, 140350425014272, 140350425030655, +SNULL, 140350425006079, 140350425014271, +STORE, 140350424989696, 140350425006079, +STORE, 140350425006080, 140350425014271, +SNULL, 94592139853823, 94592139857919, +STORE, 94592139845632, 94592139853823, +STORE, 94592139853824, 94592139857919, +SNULL, 140350427275263, 140350427279359, +STORE, 140350427271168, 140350427275263, +STORE, 140350427275264, 140350427279359, +ERASE, 140350427242496, 140350427271167, +STORE, 94592164823040, 94592164958207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723500535808, 140737488351231, +SNULL, 140723500543999, 140737488351231, +STORE, 140723500535808, 140723500543999, +STORE, 140723500404736, 140723500543999, +STORE, 94458379010048, 94458381234175, +SNULL, 94458379120639, 94458381234175, +STORE, 94458379010048, 94458379120639, +STORE, 94458379120640, 94458381234175, +ERASE, 94458379120640, 94458381234175, +STORE, 94458381213696, 94458381225983, +STORE, 94458381225984, 94458381234175, +STORE, 139771674230784, 139771676483583, +SNULL, 139771674374143, 139771676483583, +STORE, 139771674230784, 139771674374143, +STORE, 139771674374144, 139771676483583, +ERASE, 139771674374144, 139771676483583, +STORE, 139771676471296, 139771676479487, +STORE, 139771676479488, 139771676483583, +STORE, 140723500769280, 140723500773375, +STORE, 140723500756992, 140723500769279, +STORE, 139771676442624, 139771676471295, +STORE, 139771676434432, 139771676442623, +STORE, 139771670433792, 139771674230783, +SNULL, 139771670433792, 139771672092671, +STORE, 139771672092672, 139771674230783, +STORE, 139771670433792, 139771672092671, +SNULL, 139771674189823, 139771674230783, +STORE, 139771672092672, 139771674189823, +STORE, 139771674189824, 139771674230783, +SNULL, 139771674189824, 139771674214399, +STORE, 139771674214400, 139771674230783, +STORE, 139771674189824, 139771674214399, +ERASE, 139771674189824, 139771674214399, +STORE, 139771674189824, 139771674214399, +ERASE, 139771674214400, 139771674230783, +STORE, 139771674214400, 139771674230783, +SNULL, 139771674206207, 139771674214399, +STORE, 139771674189824, 139771674206207, +STORE, 139771674206208, 139771674214399, +SNULL, 94458381221887, 94458381225983, +STORE, 94458381213696, 94458381221887, +STORE, 94458381221888, 94458381225983, +SNULL, 139771676475391, 139771676479487, +STORE, 139771676471296, 139771676475391, +STORE, 139771676475392, 139771676479487, +ERASE, 139771676442624, 139771676471295, +STORE, 94458401873920, 94458402009087, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731316264960, 140737488351231, +SNULL, 140731316273151, 140737488351231, +STORE, 140731316264960, 140731316273151, +STORE, 140731316133888, 140731316273151, +STORE, 94437830881280, 94437833215999, +SNULL, 94437831094271, 94437833215999, +STORE, 94437830881280, 94437831094271, +STORE, 94437831094272, 94437833215999, +ERASE, 94437831094272, 94437833215999, +STORE, 94437833191424, 94437833203711, +STORE, 94437833203712, 94437833215999, +STORE, 140265986031616, 140265988284415, +SNULL, 140265986174975, 140265988284415, +STORE, 140265986031616, 140265986174975, +STORE, 140265986174976, 140265988284415, +ERASE, 140265986174976, 140265988284415, +STORE, 140265988272128, 140265988280319, +STORE, 140265988280320, 140265988284415, +STORE, 140731316318208, 140731316322303, +STORE, 140731316305920, 140731316318207, +STORE, 140265988243456, 140265988272127, +STORE, 140265988235264, 140265988243455, +STORE, 140265983918080, 140265986031615, +SNULL, 140265983918080, 140265983930367, +STORE, 140265983930368, 140265986031615, +STORE, 140265983918080, 140265983930367, +SNULL, 140265986023423, 140265986031615, +STORE, 140265983930368, 140265986023423, +STORE, 140265986023424, 140265986031615, +ERASE, 140265986023424, 140265986031615, +STORE, 140265986023424, 140265986031615, +STORE, 140265980121088, 140265983918079, +SNULL, 140265980121088, 140265981779967, +STORE, 140265981779968, 140265983918079, +STORE, 140265980121088, 140265981779967, +SNULL, 140265983877119, 140265983918079, +STORE, 140265981779968, 140265983877119, +STORE, 140265983877120, 140265983918079, +SNULL, 140265983877120, 140265983901695, +STORE, 140265983901696, 140265983918079, +STORE, 140265983877120, 140265983901695, +ERASE, 140265983877120, 140265983901695, +STORE, 140265983877120, 140265983901695, +ERASE, 140265983901696, 140265983918079, +STORE, 140265983901696, 140265983918079, +STORE, 140265988227072, 140265988243455, +SNULL, 140265983893503, 140265983901695, +STORE, 140265983877120, 140265983893503, +STORE, 140265983893504, 140265983901695, +SNULL, 140265986027519, 140265986031615, +STORE, 140265986023424, 140265986027519, +STORE, 140265986027520, 140265986031615, +SNULL, 94437833195519, 94437833203711, +STORE, 94437833191424, 94437833195519, +STORE, 94437833195520, 94437833203711, +SNULL, 140265988276223, 140265988280319, +STORE, 140265988272128, 140265988276223, +STORE, 140265988276224, 140265988280319, +ERASE, 140265988243456, 140265988272127, +STORE, 94437847638016, 94437847773183, +STORE, 140265986543616, 140265988227071, +STORE, 94437847638016, 94437847908351, +STORE, 94437847638016, 94437848043519, +STORE, 94437847638016, 94437848190975, +SNULL, 94437848178687, 94437848190975, +STORE, 94437847638016, 94437848178687, +STORE, 94437848178688, 94437848190975, +ERASE, 94437848178688, 94437848190975, +STORE, 94437847638016, 94437848330239, +STORE, 94437847638016, 94437848465407, +SNULL, 94437848444927, 94437848465407, +STORE, 94437847638016, 94437848444927, +STORE, 94437848444928, 94437848465407, +ERASE, 94437848444928, 94437848465407, +STORE, 94437847638016, 94437848584191, +STORE, 94437847638016, 94437848719359, +SNULL, 94437848678399, 94437848719359, +STORE, 94437847638016, 94437848678399, +STORE, 94437848678400, 94437848719359, +ERASE, 94437848678400, 94437848719359, +STORE, 94437847638016, 94437848842239, +SNULL, 94437848825855, 94437848842239, +STORE, 94437847638016, 94437848825855, +STORE, 94437848825856, 94437848842239, +ERASE, 94437848825856, 94437848842239, +STORE, 94437847638016, 94437848961023, +STORE, 94437847638016, 94437849096191, +STORE, 94661814710272, 94661814923263, +STORE, 94661817020416, 94661817024511, +STORE, 94661817024512, 94661817032703, +STORE, 94661817032704, 94661817044991, +STORE, 94661840424960, 94661841240063, +STORE, 140582259814400, 140582261473279, +STORE, 140582261473280, 140582263570431, +STORE, 140582263570432, 140582263586815, +STORE, 140582263586816, 140582263595007, +STORE, 140582263595008, 140582263611391, +STORE, 140582263611392, 140582263623679, +STORE, 140582263623680, 140582265716735, +STORE, 140582265716736, 140582265720831, +STORE, 140582265720832, 140582265724927, +STORE, 140582265724928, 140582265868287, +STORE, 140582266236928, 140582267920383, +STORE, 140582267920384, 140582267936767, +STORE, 140582267965440, 140582267969535, +STORE, 140582267969536, 140582267973631, +STORE, 140582267973632, 140582267977727, +STORE, 140735472508928, 140735472648191, +STORE, 140735472672768, 140735472685055, +STORE, 140735472685056, 140735472689151, +STORE, 94440069140480, 94440069353471, +STORE, 94440071450624, 94440071454719, +STORE, 94440071454720, 94440071462911, +STORE, 94440071462912, 94440071475199, +STORE, 94440072122368, 94440079048703, +STORE, 140112218095616, 140112219754495, +STORE, 140112219754496, 140112221851647, +STORE, 140112221851648, 140112221868031, +STORE, 140112221868032, 140112221876223, +STORE, 140112221876224, 140112221892607, +STORE, 140112221892608, 140112221904895, +STORE, 140112221904896, 140112223997951, +STORE, 140112223997952, 140112224002047, +STORE, 140112224002048, 140112224006143, +STORE, 140112224006144, 140112224149503, +STORE, 140112224518144, 140112226201599, +STORE, 140112226201600, 140112226217983, +STORE, 140112226246656, 140112226250751, +STORE, 140112226250752, 140112226254847, +STORE, 140112226254848, 140112226258943, +STORE, 140737460969472, 140737461108735, +STORE, 140737462083584, 140737462095871, +STORE, 140737462095872, 140737462099967, +STORE, 94257654345728, 94257654390783, +STORE, 94257656483840, 94257656487935, +STORE, 94257656487936, 94257656492031, +STORE, 94257656492032, 94257656496127, +STORE, 94257665859584, 94257665994751, +STORE, 140507070345216, 140507070386175, +STORE, 140507070386176, 140507072483327, +STORE, 140507072483328, 140507072487423, +STORE, 140507072487424, 140507072491519, +STORE, 140507072491520, 140507072516095, +STORE, 140507072516096, 140507072561151, +STORE, 140507072561152, 140507074654207, +STORE, 140507074654208, 140507074658303, +STORE, 140507074658304, 140507074662399, +STORE, 140507074662400, 140507074744319, +STORE, 140507074744320, 140507076841471, +STORE, 140507076841472, 140507076845567, +STORE, 140507076845568, 140507076849663, +STORE, 140507076849664, 140507076857855, +STORE, 140507076857856, 140507076886527, +STORE, 140507076886528, 140507078979583, +STORE, 140507078979584, 140507078983679, +STORE, 140507078983680, 140507078987775, +STORE, 140507078987776, 140507079086079, +STORE, 140507079086080, 140507081179135, +STORE, 140507081179136, 140507081183231, +STORE, 140507081183232, 140507081187327, +STORE, 140507081187328, 140507081203711, +STORE, 140507081203712, 140507081220095, +STORE, 140507081220096, 140507083317247, +STORE, 140507083317248, 140507083321343, +STORE, 140507083321344, 140507083325439, +STORE, 140507083325440, 140507083792383, +STORE, 140507083792384, 140507085885439, +STORE, 140507085885440, 140507085889535, +STORE, 140507085889536, 140507085893631, +STORE, 140507085893632, 140507085905919, +STORE, 140507085905920, 140507087998975, +STORE, 140507087998976, 140507088003071, +STORE, 140507088003072, 140507088007167, +STORE, 140507088007168, 140507088125951, +STORE, 140507088125952, 140507090219007, +STORE, 140507090219008, 140507090223103, +STORE, 140507090223104, 140507090227199, +STORE, 140507090227200, 140507090268159, +STORE, 140507090268160, 140507091927039, +STORE, 140507091927040, 140507094024191, +STORE, 140507094024192, 140507094040575, +STORE, 140507094040576, 140507094048767, +STORE, 140507094048768, 140507094065151, +STORE, 140507094065152, 140507094216703, +STORE, 140507094216704, 140507096309759, +STORE, 140507096309760, 140507096313855, +STORE, 140507096313856, 140507096317951, +STORE, 140507096317952, 140507096326143, +STORE, 140507096326144, 140507096379391, +STORE, 140507096379392, 140507098472447, +STORE, 140507098472448, 140507098476543, +STORE, 140507098476544, 140507098480639, +STORE, 140507098480640, 140507098623999, +STORE, 140507098980352, 140507100663807, +STORE, 140507100663808, 140507100692479, +STORE, 140507100721152, 140507100725247, +STORE, 140507100725248, 140507100729343, +STORE, 140507100729344, 140507100733439, +STORE, 140728152780800, 140728152915967, +STORE, 140728153698304, 140728153710591, +STORE, 140728153710592, 140728153714687, +STORE, 140507068137472, 140507070345215, +SNULL, 140507068137472, 140507068190719, +STORE, 140507068190720, 140507070345215, +STORE, 140507068137472, 140507068190719, +SNULL, 140507070287871, 140507070345215, +STORE, 140507068190720, 140507070287871, +STORE, 140507070287872, 140507070345215, +SNULL, 140507070287872, 140507070296063, +STORE, 140507070296064, 140507070345215, +STORE, 140507070287872, 140507070296063, +ERASE, 140507070287872, 140507070296063, +STORE, 140507070287872, 140507070296063, +ERASE, 140507070296064, 140507070345215, +STORE, 140507070296064, 140507070345215, +STORE, 140507100692480, 140507100721151, +STORE, 140507065810944, 140507068137471, +SNULL, 140507065810944, 140507065843711, +STORE, 140507065843712, 140507068137471, +STORE, 140507065810944, 140507065843711, +SNULL, 140507067940863, 140507068137471, +STORE, 140507065843712, 140507067940863, +STORE, 140507067940864, 140507068137471, +SNULL, 140507067940864, 140507067949055, +STORE, 140507067949056, 140507068137471, +STORE, 140507067940864, 140507067949055, +ERASE, 140507067940864, 140507067949055, +STORE, 140507067940864, 140507067949055, +ERASE, 140507067949056, 140507068137471, +STORE, 140507067949056, 140507068137471, +SNULL, 140507067944959, 140507067949055, +STORE, 140507067940864, 140507067944959, +STORE, 140507067944960, 140507067949055, +SNULL, 140507070291967, 140507070296063, +STORE, 140507070287872, 140507070291967, +STORE, 140507070291968, 140507070296063, +ERASE, 140507100692480, 140507100721151, +STORE, 140507063705600, 140507065810943, +SNULL, 140507063705600, 140507063709695, +STORE, 140507063709696, 140507065810943, +STORE, 140507063705600, 140507063709695, +SNULL, 140507065802751, 140507065810943, +STORE, 140507063709696, 140507065802751, +STORE, 140507065802752, 140507065810943, +ERASE, 140507065802752, 140507065810943, +STORE, 140507065802752, 140507065810943, +SNULL, 140507065806847, 140507065810943, +STORE, 140507065802752, 140507065806847, +STORE, 140507065806848, 140507065810943, +STORE, 140507061600256, 140507063705599, +SNULL, 140507061600256, 140507061604351, +STORE, 140507061604352, 140507063705599, +STORE, 140507061600256, 140507061604351, +SNULL, 140507063697407, 140507063705599, +STORE, 140507061604352, 140507063697407, +STORE, 140507063697408, 140507063705599, +ERASE, 140507063697408, 140507063705599, +STORE, 140507063697408, 140507063705599, +SNULL, 140507063701503, 140507063705599, +STORE, 140507063697408, 140507063701503, +STORE, 140507063701504, 140507063705599, +STORE, 140507059490816, 140507061600255, +SNULL, 140507059490816, 140507059499007, +STORE, 140507059499008, 140507061600255, +STORE, 140507059490816, 140507059499007, +SNULL, 140507061592063, 140507061600255, +STORE, 140507059499008, 140507061592063, +STORE, 140507061592064, 140507061600255, +ERASE, 140507061592064, 140507061600255, +STORE, 140507061592064, 140507061600255, +SNULL, 140507061596159, 140507061600255, +STORE, 140507061592064, 140507061596159, +STORE, 140507061596160, 140507061600255, +STORE, 140507057377280, 140507059490815, +SNULL, 140507057377280, 140507057389567, +STORE, 140507057389568, 140507059490815, +STORE, 140507057377280, 140507057389567, +SNULL, 140507059482623, 140507059490815, +STORE, 140507057389568, 140507059482623, +STORE, 140507059482624, 140507059490815, +ERASE, 140507059482624, 140507059490815, +STORE, 140507059482624, 140507059490815, +SNULL, 140507059486719, 140507059490815, +STORE, 140507059482624, 140507059486719, +STORE, 140507059486720, 140507059490815, +STORE, 140507055255552, 140507057377279, +SNULL, 140507055255552, 140507055276031, +STORE, 140507055276032, 140507057377279, +STORE, 140507055255552, 140507055276031, +SNULL, 140507057369087, 140507057377279, +STORE, 140507055276032, 140507057369087, +STORE, 140507057369088, 140507057377279, +ERASE, 140507057369088, 140507057377279, +STORE, 140507057369088, 140507057377279, +SNULL, 140507057373183, 140507057377279, +STORE, 140507057369088, 140507057373183, +STORE, 140507057373184, 140507057377279, +STORE, 140507098693632, 140507098980351, +SNULL, 140507098959871, 140507098980351, +STORE, 140507098693632, 140507098959871, +STORE, 140507098959872, 140507098980351, +SNULL, 140507098959872, 140507098976255, +STORE, 140507098976256, 140507098980351, +STORE, 140507098959872, 140507098976255, +ERASE, 140507098959872, 140507098976255, +STORE, 140507098959872, 140507098976255, +ERASE, 140507098976256, 140507098980351, +STORE, 140507098976256, 140507098980351, +STORE, 140507100692480, 140507100721151, +STORE, 140507053125632, 140507055255551, +SNULL, 140507053125632, 140507053154303, +STORE, 140507053154304, 140507055255551, +STORE, 140507053125632, 140507053154303, +SNULL, 140507055247359, 140507055255551, +STORE, 140507053154304, 140507055247359, +STORE, 140507055247360, 140507055255551, +ERASE, 140507055247360, 140507055255551, +STORE, 140507055247360, 140507055255551, +STORE, 140507051012096, 140507053125631, +SNULL, 140507051012096, 140507051024383, +STORE, 140507051024384, 140507053125631, +STORE, 140507051012096, 140507051024383, +SNULL, 140507053117439, 140507053125631, +STORE, 140507051024384, 140507053117439, +STORE, 140507053117440, 140507053125631, +ERASE, 140507053117440, 140507053125631, +STORE, 140507053117440, 140507053125631, +SNULL, 140507053121535, 140507053125631, +STORE, 140507053117440, 140507053121535, +STORE, 140507053121536, 140507053125631, +SNULL, 140507055251455, 140507055255551, +STORE, 140507055247360, 140507055251455, +STORE, 140507055251456, 140507055255551, +SNULL, 140507098972159, 140507098976255, +STORE, 140507098959872, 140507098972159, +STORE, 140507098972160, 140507098976255, +ERASE, 140507100692480, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100717056, 140507100721151, +ERASE, 140507100717056, 140507100721151, +STORE, 140507100692480, 140507100721151, +ERASE, 140507068137472, 140507068190719, +ERASE, 140507068190720, 140507070287871, +ERASE, 140507070287872, 140507070291967, +ERASE, 140507070291968, 140507070296063, +ERASE, 140507070296064, 140507070345215, +ERASE, 140507065810944, 140507065843711, +ERASE, 140507065843712, 140507067940863, +ERASE, 140507067940864, 140507067944959, +ERASE, 140507067944960, 140507067949055, +ERASE, 140507067949056, 140507068137471, +ERASE, 140507063705600, 140507063709695, +ERASE, 140507063709696, 140507065802751, +ERASE, 140507065802752, 140507065806847, +ERASE, 140507065806848, 140507065810943, +ERASE, 140507061600256, 140507061604351, +ERASE, 140507061604352, 140507063697407, +ERASE, 140507063697408, 140507063701503, +ERASE, 140507063701504, 140507063705599, +ERASE, 140507059490816, 140507059499007, +ERASE, 140507059499008, 140507061592063, +ERASE, 140507061592064, 140507061596159, +ERASE, 140507061596160, 140507061600255, +ERASE, 140507057377280, 140507057389567, +ERASE, 140507057389568, 140507059482623, +ERASE, 140507059482624, 140507059486719, +ERASE, 140507059486720, 140507059490815, +ERASE, 140507055255552, 140507055276031, +ERASE, 140507055276032, 140507057369087, +ERASE, 140507057369088, 140507057373183, +ERASE, 140507057373184, 140507057377279, +ERASE, 140507098693632, 140507098959871, +ERASE, 140507098959872, 140507098972159, +ERASE, 140507098972160, 140507098976255, +ERASE, 140507098976256, 140507098980351, +ERASE, 140507051012096, 140507051024383, +ERASE, 140507051024384, 140507053117439, +ERASE, 140507053117440, 140507053121535, +ERASE, 140507053121536, 140507053125631, +STORE, 94036448296960, 94036448509951, +STORE, 94036450607104, 94036450611199, +STORE, 94036450611200, 94036450619391, +STORE, 94036450619392, 94036450631679, +STORE, 94036482445312, 94036502376447, +STORE, 140469487013888, 140469488672767, +STORE, 140469488672768, 140469490769919, +STORE, 140469490769920, 140469490786303, +STORE, 140469490786304, 140469490794495, +STORE, 140469490794496, 140469490810879, +STORE, 140469490810880, 140469490823167, +STORE, 140469490823168, 140469492916223, +STORE, 140469492916224, 140469492920319, +STORE, 140469492920320, 140469492924415, +STORE, 140469492924416, 140469493067775, +STORE, 140469493436416, 140469495119871, +STORE, 140469495119872, 140469495136255, +STORE, 140469495164928, 140469495169023, +STORE, 140469495169024, 140469495173119, +STORE, 140469495173120, 140469495177215, +STORE, 140732281446400, 140732281585663, +STORE, 140732282736640, 140732282748927, +STORE, 140732282748928, 140732282753023, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723411931136, 140737488351231, +SNULL, 140723411939327, 140737488351231, +STORE, 140723411931136, 140723411939327, +STORE, 140723411800064, 140723411939327, +STORE, 93993768685568, 93993770909695, +SNULL, 93993768796159, 93993770909695, +STORE, 93993768685568, 93993768796159, +STORE, 93993768796160, 93993770909695, +ERASE, 93993768796160, 93993770909695, +STORE, 93993770889216, 93993770901503, +STORE, 93993770901504, 93993770909695, +STORE, 140508681740288, 140508683993087, +SNULL, 140508681883647, 140508683993087, +STORE, 140508681740288, 140508681883647, +STORE, 140508681883648, 140508683993087, +ERASE, 140508681883648, 140508683993087, +STORE, 140508683980800, 140508683988991, +STORE, 140508683988992, 140508683993087, +STORE, 140723412070400, 140723412074495, +STORE, 140723412058112, 140723412070399, +STORE, 140508683952128, 140508683980799, +STORE, 140508683943936, 140508683952127, +STORE, 140508677943296, 140508681740287, +SNULL, 140508677943296, 140508679602175, +STORE, 140508679602176, 140508681740287, +STORE, 140508677943296, 140508679602175, +SNULL, 140508681699327, 140508681740287, +STORE, 140508679602176, 140508681699327, +STORE, 140508681699328, 140508681740287, +SNULL, 140508681699328, 140508681723903, +STORE, 140508681723904, 140508681740287, +STORE, 140508681699328, 140508681723903, +ERASE, 140508681699328, 140508681723903, +STORE, 140508681699328, 140508681723903, +ERASE, 140508681723904, 140508681740287, +STORE, 140508681723904, 140508681740287, +SNULL, 140508681715711, 140508681723903, +STORE, 140508681699328, 140508681715711, +STORE, 140508681715712, 140508681723903, +SNULL, 93993770897407, 93993770901503, +STORE, 93993770889216, 93993770897407, +STORE, 93993770897408, 93993770901503, +SNULL, 140508683984895, 140508683988991, +STORE, 140508683980800, 140508683984895, +STORE, 140508683984896, 140508683988991, +ERASE, 140508683952128, 140508683980799, +STORE, 93993791582208, 93993791717375, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140734685458432, 140737488351231, +SNULL, 140734685466623, 140737488351231, +STORE, 140734685458432, 140734685466623, +STORE, 140734685327360, 140734685466623, +STORE, 93832321548288, 93832323772415, +SNULL, 93832321658879, 93832323772415, +STORE, 93832321548288, 93832321658879, +STORE, 93832321658880, 93832323772415, +ERASE, 93832321658880, 93832323772415, +STORE, 93832323751936, 93832323764223, +STORE, 93832323764224, 93832323772415, +STORE, 140650945118208, 140650947371007, +SNULL, 140650945261567, 140650947371007, +STORE, 140650945118208, 140650945261567, +STORE, 140650945261568, 140650947371007, +ERASE, 140650945261568, 140650947371007, +STORE, 140650947358720, 140650947366911, +STORE, 140650947366912, 140650947371007, +STORE, 140734686081024, 140734686085119, +STORE, 140734686068736, 140734686081023, +STORE, 140650947330048, 140650947358719, +STORE, 140650947321856, 140650947330047, +STORE, 140650941321216, 140650945118207, +SNULL, 140650941321216, 140650942980095, +STORE, 140650942980096, 140650945118207, +STORE, 140650941321216, 140650942980095, +SNULL, 140650945077247, 140650945118207, +STORE, 140650942980096, 140650945077247, +STORE, 140650945077248, 140650945118207, +SNULL, 140650945077248, 140650945101823, +STORE, 140650945101824, 140650945118207, +STORE, 140650945077248, 140650945101823, +ERASE, 140650945077248, 140650945101823, +STORE, 140650945077248, 140650945101823, +ERASE, 140650945101824, 140650945118207, +STORE, 140650945101824, 140650945118207, +SNULL, 140650945093631, 140650945101823, +STORE, 140650945077248, 140650945093631, +STORE, 140650945093632, 140650945101823, +SNULL, 93832323760127, 93832323764223, +STORE, 93832323751936, 93832323760127, +STORE, 93832323760128, 93832323764223, +SNULL, 140650947362815, 140650947366911, +STORE, 140650947358720, 140650947362815, +STORE, 140650947362816, 140650947366911, +ERASE, 140650947330048, 140650947358719, +STORE, 93832331890688, 93832332025855, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728333520896, 140737488351231, +SNULL, 140728333529087, 140737488351231, +STORE, 140728333520896, 140728333529087, +STORE, 140728333389824, 140728333529087, +STORE, 94872734732288, 94872736956415, +SNULL, 94872734842879, 94872736956415, +STORE, 94872734732288, 94872734842879, +STORE, 94872734842880, 94872736956415, +ERASE, 94872734842880, 94872736956415, +STORE, 94872736935936, 94872736948223, +STORE, 94872736948224, 94872736956415, +STORE, 139755193257984, 139755195510783, +SNULL, 139755193401343, 139755195510783, +STORE, 139755193257984, 139755193401343, +STORE, 139755193401344, 139755195510783, +ERASE, 139755193401344, 139755195510783, +STORE, 139755195498496, 139755195506687, +STORE, 139755195506688, 139755195510783, +STORE, 140728333926400, 140728333930495, +STORE, 140728333914112, 140728333926399, +STORE, 139755195469824, 139755195498495, +STORE, 139755195461632, 139755195469823, +STORE, 139755189460992, 139755193257983, +SNULL, 139755189460992, 139755191119871, +STORE, 139755191119872, 139755193257983, +STORE, 139755189460992, 139755191119871, +SNULL, 139755193217023, 139755193257983, +STORE, 139755191119872, 139755193217023, +STORE, 139755193217024, 139755193257983, +SNULL, 139755193217024, 139755193241599, +STORE, 139755193241600, 139755193257983, +STORE, 139755193217024, 139755193241599, +ERASE, 139755193217024, 139755193241599, +STORE, 139755193217024, 139755193241599, +ERASE, 139755193241600, 139755193257983, +STORE, 139755193241600, 139755193257983, +SNULL, 139755193233407, 139755193241599, +STORE, 139755193217024, 139755193233407, +STORE, 139755193233408, 139755193241599, +SNULL, 94872736944127, 94872736948223, +STORE, 94872736935936, 94872736944127, +STORE, 94872736944128, 94872736948223, +SNULL, 139755195502591, 139755195506687, +STORE, 139755195498496, 139755195502591, +STORE, 139755195502592, 139755195506687, +ERASE, 139755195469824, 139755195498495, +STORE, 94872749744128, 94872749879295, +STORE, 94720243642368, 94720243855359, +STORE, 94720245952512, 94720245956607, +STORE, 94720245956608, 94720245964799, +STORE, 94720245964800, 94720245977087, +STORE, 94720277745664, 94720278151167, +STORE, 140453174497280, 140453176156159, +STORE, 140453176156160, 140453178253311, +STORE, 140453178253312, 140453178269695, +STORE, 140453178269696, 140453178277887, +STORE, 140453178277888, 140453178294271, +STORE, 140453178294272, 140453178306559, +STORE, 140453178306560, 140453180399615, +STORE, 140453180399616, 140453180403711, +STORE, 140453180403712, 140453180407807, +STORE, 140453180407808, 140453180551167, +STORE, 140453180919808, 140453182603263, +STORE, 140453182603264, 140453182619647, +STORE, 140453182648320, 140453182652415, +STORE, 140453182652416, 140453182656511, +STORE, 140453182656512, 140453182660607, +STORE, 140733223923712, 140733224062975, +STORE, 140733224808448, 140733224820735, +STORE, 140733224820736, 140733224824831, +STORE, 94321091141632, 94321091354623, +STORE, 94321093451776, 94321093455871, +STORE, 94321093455872, 94321093464063, +STORE, 94321093464064, 94321093476351, +STORE, 94321115873280, 94321117229055, +STORE, 139695978840064, 139695980498943, +STORE, 139695980498944, 139695982596095, +STORE, 139695982596096, 139695982612479, +STORE, 139695982612480, 139695982620671, +STORE, 139695982620672, 139695982637055, +STORE, 139695982637056, 139695982649343, +STORE, 139695982649344, 139695984742399, +STORE, 139695984742400, 139695984746495, +STORE, 139695984746496, 139695984750591, +STORE, 139695984750592, 139695984893951, +STORE, 139695985262592, 139695986946047, +STORE, 139695986946048, 139695986962431, +STORE, 139695986991104, 139695986995199, +STORE, 139695986995200, 139695986999295, +STORE, 139695986999296, 139695987003391, +STORE, 140734650564608, 140734650703871, +STORE, 140734650785792, 140734650798079, +STORE, 140734650798080, 140734650802175, +STORE, 94523438456832, 94523438669823, +STORE, 94523440766976, 94523440771071, +STORE, 94523440771072, 94523440779263, +STORE, 94523440779264, 94523440791551, +STORE, 94523464544256, 94523465842687, +STORE, 140453231493120, 140453233151999, +STORE, 140453233152000, 140453235249151, +STORE, 140453235249152, 140453235265535, +STORE, 140453235265536, 140453235273727, +STORE, 140453235273728, 140453235290111, +STORE, 140453235290112, 140453235302399, +STORE, 140453235302400, 140453237395455, +STORE, 140453237395456, 140453237399551, +STORE, 140453237399552, 140453237403647, +STORE, 140453237403648, 140453237547007, +STORE, 140453237915648, 140453239599103, +STORE, 140453239599104, 140453239615487, +STORE, 140453239644160, 140453239648255, +STORE, 140453239648256, 140453239652351, +STORE, 140453239652352, 140453239656447, +STORE, 140734679445504, 140734679584767, +STORE, 140734680018944, 140734680031231, +STORE, 140734680031232, 140734680035327, +STORE, 94614776987648, 94614777200639, +STORE, 94614779297792, 94614779301887, +STORE, 94614779301888, 94614779310079, +STORE, 94614779310080, 94614779322367, +STORE, 94614798467072, 94614800699391, +STORE, 139677037182976, 139677038841855, +STORE, 139677038841856, 139677040939007, +STORE, 139677040939008, 139677040955391, +STORE, 139677040955392, 139677040963583, +STORE, 139677040963584, 139677040979967, +STORE, 139677040979968, 139677040992255, +STORE, 139677040992256, 139677043085311, +STORE, 139677043085312, 139677043089407, +STORE, 139677043089408, 139677043093503, +STORE, 139677043093504, 139677043236863, +STORE, 139677043605504, 139677045288959, +STORE, 139677045288960, 139677045305343, +STORE, 139677045334016, 139677045338111, +STORE, 139677045338112, 139677045342207, +STORE, 139677045342208, 139677045346303, +STORE, 140721604411392, 140721604550655, +STORE, 140721606135808, 140721606148095, +STORE, 140721606148096, 140721606152191, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140729280544768, 140737488351231, +SNULL, 140729280552959, 140737488351231, +STORE, 140729280544768, 140729280552959, +STORE, 140729280413696, 140729280552959, +STORE, 94863939334144, 94863941558271, +SNULL, 94863939444735, 94863941558271, +STORE, 94863939334144, 94863939444735, +STORE, 94863939444736, 94863941558271, +ERASE, 94863939444736, 94863941558271, +STORE, 94863941537792, 94863941550079, +STORE, 94863941550080, 94863941558271, +STORE, 139691047276544, 139691049529343, +SNULL, 139691047419903, 139691049529343, +STORE, 139691047276544, 139691047419903, +STORE, 139691047419904, 139691049529343, +ERASE, 139691047419904, 139691049529343, +STORE, 139691049517056, 139691049525247, +STORE, 139691049525248, 139691049529343, +STORE, 140729281679360, 140729281683455, +STORE, 140729281667072, 140729281679359, +STORE, 139691049488384, 139691049517055, +STORE, 139691049480192, 139691049488383, +STORE, 139691043479552, 139691047276543, +SNULL, 139691043479552, 139691045138431, +STORE, 139691045138432, 139691047276543, +STORE, 139691043479552, 139691045138431, +SNULL, 139691047235583, 139691047276543, +STORE, 139691045138432, 139691047235583, +STORE, 139691047235584, 139691047276543, +SNULL, 139691047235584, 139691047260159, +STORE, 139691047260160, 139691047276543, +STORE, 139691047235584, 139691047260159, +ERASE, 139691047235584, 139691047260159, +STORE, 139691047235584, 139691047260159, +ERASE, 139691047260160, 139691047276543, +STORE, 139691047260160, 139691047276543, +SNULL, 139691047251967, 139691047260159, +STORE, 139691047235584, 139691047251967, +STORE, 139691047251968, 139691047260159, +SNULL, 94863941545983, 94863941550079, +STORE, 94863941537792, 94863941545983, +STORE, 94863941545984, 94863941550079, +SNULL, 139691049521151, 139691049525247, +STORE, 139691049517056, 139691049521151, +STORE, 139691049521152, 139691049525247, +ERASE, 139691049488384, 139691049517055, +STORE, 94863951294464, 94863951429631, +STORE, 93998209294336, 93998209507327, +STORE, 93998211604480, 93998211608575, +STORE, 93998211608576, 93998211616767, +STORE, 93998211616768, 93998211629055, +STORE, 93998227210240, 93998227615743, +STORE, 140243029913600, 140243031572479, +STORE, 140243031572480, 140243033669631, +STORE, 140243033669632, 140243033686015, +STORE, 140243033686016, 140243033694207, +STORE, 140243033694208, 140243033710591, +STORE, 140243033710592, 140243033722879, +STORE, 140243033722880, 140243035815935, +STORE, 140243035815936, 140243035820031, +STORE, 140243035820032, 140243035824127, +STORE, 140243035824128, 140243035967487, +STORE, 140243036336128, 140243038019583, +STORE, 140243038019584, 140243038035967, +STORE, 140243038064640, 140243038068735, +STORE, 140243038068736, 140243038072831, +STORE, 140243038072832, 140243038076927, +STORE, 140734976479232, 140734976618495, +STORE, 140734977978368, 140734977990655, +STORE, 140734977990656, 140734977994751, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722742775808, 140737488351231, +SNULL, 140722742783999, 140737488351231, +STORE, 140722742775808, 140722742783999, +STORE, 140722742644736, 140722742783999, +STORE, 93857673662464, 93857675997183, +SNULL, 93857673875455, 93857675997183, +STORE, 93857673662464, 93857673875455, +STORE, 93857673875456, 93857675997183, +ERASE, 93857673875456, 93857675997183, +STORE, 93857675972608, 93857675984895, +STORE, 93857675984896, 93857675997183, +STORE, 140629677498368, 140629679751167, +SNULL, 140629677641727, 140629679751167, +STORE, 140629677498368, 140629677641727, +STORE, 140629677641728, 140629679751167, +ERASE, 140629677641728, 140629679751167, +STORE, 140629679738880, 140629679747071, +STORE, 140629679747072, 140629679751167, +STORE, 140722743222272, 140722743226367, +STORE, 140722743209984, 140722743222271, +STORE, 140629679710208, 140629679738879, +STORE, 140629679702016, 140629679710207, +STORE, 140629675384832, 140629677498367, +SNULL, 140629675384832, 140629675397119, +STORE, 140629675397120, 140629677498367, +STORE, 140629675384832, 140629675397119, +SNULL, 140629677490175, 140629677498367, +STORE, 140629675397120, 140629677490175, +STORE, 140629677490176, 140629677498367, +ERASE, 140629677490176, 140629677498367, +STORE, 140629677490176, 140629677498367, +STORE, 140629671587840, 140629675384831, +SNULL, 140629671587840, 140629673246719, +STORE, 140629673246720, 140629675384831, +STORE, 140629671587840, 140629673246719, +SNULL, 140629675343871, 140629675384831, +STORE, 140629673246720, 140629675343871, +STORE, 140629675343872, 140629675384831, +SNULL, 140629675343872, 140629675368447, +STORE, 140629675368448, 140629675384831, +STORE, 140629675343872, 140629675368447, +ERASE, 140629675343872, 140629675368447, +STORE, 140629675343872, 140629675368447, +ERASE, 140629675368448, 140629675384831, +STORE, 140629675368448, 140629675384831, +STORE, 140629679693824, 140629679710207, +SNULL, 140629675360255, 140629675368447, +STORE, 140629675343872, 140629675360255, +STORE, 140629675360256, 140629675368447, +SNULL, 140629677494271, 140629677498367, +STORE, 140629677490176, 140629677494271, +STORE, 140629677494272, 140629677498367, +SNULL, 93857675976703, 93857675984895, +STORE, 93857675972608, 93857675976703, +STORE, 93857675976704, 93857675984895, +SNULL, 140629679742975, 140629679747071, +STORE, 140629679738880, 140629679742975, +STORE, 140629679742976, 140629679747071, +ERASE, 140629679710208, 140629679738879, +STORE, 93857705832448, 93857705967615, +STORE, 140629678010368, 140629679693823, +STORE, 93857705832448, 93857706102783, +STORE, 93857705832448, 93857706237951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735922421760, 140737488351231, +SNULL, 140735922429951, 140737488351231, +STORE, 140735922421760, 140735922429951, +STORE, 140735922290688, 140735922429951, +STORE, 94651136139264, 94651138363391, +SNULL, 94651136249855, 94651138363391, +STORE, 94651136139264, 94651136249855, +STORE, 94651136249856, 94651138363391, +ERASE, 94651136249856, 94651138363391, +STORE, 94651138342912, 94651138355199, +STORE, 94651138355200, 94651138363391, +STORE, 140325788266496, 140325790519295, +SNULL, 140325788409855, 140325790519295, +STORE, 140325788266496, 140325788409855, +STORE, 140325788409856, 140325790519295, +ERASE, 140325788409856, 140325790519295, +STORE, 140325790507008, 140325790515199, +STORE, 140325790515200, 140325790519295, +STORE, 140735923572736, 140735923576831, +STORE, 140735923560448, 140735923572735, +STORE, 140325790478336, 140325790507007, +STORE, 140325790470144, 140325790478335, +STORE, 140325784469504, 140325788266495, +SNULL, 140325784469504, 140325786128383, +STORE, 140325786128384, 140325788266495, +STORE, 140325784469504, 140325786128383, +SNULL, 140325788225535, 140325788266495, +STORE, 140325786128384, 140325788225535, +STORE, 140325788225536, 140325788266495, +SNULL, 140325788225536, 140325788250111, +STORE, 140325788250112, 140325788266495, +STORE, 140325788225536, 140325788250111, +ERASE, 140325788225536, 140325788250111, +STORE, 140325788225536, 140325788250111, +ERASE, 140325788250112, 140325788266495, +STORE, 140325788250112, 140325788266495, +SNULL, 140325788241919, 140325788250111, +STORE, 140325788225536, 140325788241919, +STORE, 140325788241920, 140325788250111, +SNULL, 94651138351103, 94651138355199, +STORE, 94651138342912, 94651138351103, +STORE, 94651138351104, 94651138355199, +SNULL, 140325790511103, 140325790515199, +STORE, 140325790507008, 140325790511103, +STORE, 140325790511104, 140325790515199, +ERASE, 140325790478336, 140325790507007, +STORE, 94651146297344, 94651146432511, +STORE, 94212330168320, 94212330381311, +STORE, 94212332478464, 94212332482559, +STORE, 94212332482560, 94212332490751, +STORE, 94212332490752, 94212332503039, +STORE, 94212348891136, 94212349825023, +STORE, 140611630604288, 140611632263167, +STORE, 140611632263168, 140611634360319, +STORE, 140611634360320, 140611634376703, +STORE, 140611634376704, 140611634384895, +STORE, 140611634384896, 140611634401279, +STORE, 140611634401280, 140611634413567, +STORE, 140611634413568, 140611636506623, +STORE, 140611636506624, 140611636510719, +STORE, 140611636510720, 140611636514815, +STORE, 140611636514816, 140611636658175, +STORE, 140611637026816, 140611638710271, +STORE, 140611638710272, 140611638726655, +STORE, 140611638755328, 140611638759423, +STORE, 140611638759424, 140611638763519, +STORE, 140611638763520, 140611638767615, +STORE, 140726974533632, 140726974672895, +STORE, 140726974943232, 140726974955519, +STORE, 140726974955520, 140726974959615, +STORE, 94572463521792, 94572463734783, +STORE, 94572465831936, 94572465836031, +STORE, 94572465836032, 94572465844223, +STORE, 94572465844224, 94572465856511, +STORE, 94572491534336, 94572492865535, +STORE, 140644351492096, 140644353150975, +STORE, 140644353150976, 140644355248127, +STORE, 140644355248128, 140644355264511, +STORE, 140644355264512, 140644355272703, +STORE, 140644355272704, 140644355289087, +STORE, 140644355289088, 140644355301375, +STORE, 140644355301376, 140644357394431, +STORE, 140644357394432, 140644357398527, +STORE, 140644357398528, 140644357402623, +STORE, 140644357402624, 140644357545983, +STORE, 140644357914624, 140644359598079, +STORE, 140644359598080, 140644359614463, +STORE, 140644359643136, 140644359647231, +STORE, 140644359647232, 140644359651327, +STORE, 140644359651328, 140644359655423, +STORE, 140727841824768, 140727841964031, +STORE, 140727843188736, 140727843201023, +STORE, 140727843201024, 140727843205119, +STORE, 94144315457536, 94144315670527, +STORE, 94144317767680, 94144317771775, +STORE, 94144317771776, 94144317779967, +STORE, 94144317779968, 94144317792255, +STORE, 94144318369792, 94144320815103, +STORE, 140316717645824, 140316719304703, +STORE, 140316719304704, 140316721401855, +STORE, 140316721401856, 140316721418239, +STORE, 140316721418240, 140316721426431, +STORE, 140316721426432, 140316721442815, +STORE, 140316721442816, 140316721455103, +STORE, 140316721455104, 140316723548159, +STORE, 140316723548160, 140316723552255, +STORE, 140316723552256, 140316723556351, +STORE, 140316723556352, 140316723699711, +STORE, 140316724068352, 140316725751807, +STORE, 140316725751808, 140316725768191, +STORE, 140316725796864, 140316725800959, +STORE, 140316725800960, 140316725805055, +STORE, 140316725805056, 140316725809151, +STORE, 140725744283648, 140725744422911, +STORE, 140725745852416, 140725745864703, +STORE, 140725745864704, 140725745868799, +STORE, 94646858846208, 94646859059199, +STORE, 94646861156352, 94646861160447, +STORE, 94646861160448, 94646861168639, +STORE, 94646861168640, 94646861180927, +STORE, 94646879805440, 94646881894399, +STORE, 140435449745408, 140435451404287, +STORE, 140435451404288, 140435453501439, +STORE, 140435453501440, 140435453517823, +STORE, 140435453517824, 140435453526015, +STORE, 140435453526016, 140435453542399, +STORE, 140435453542400, 140435453554687, +STORE, 140435453554688, 140435455647743, +STORE, 140435455647744, 140435455651839, +STORE, 140435455651840, 140435455655935, +STORE, 140435455655936, 140435455799295, +STORE, 140435456167936, 140435457851391, +STORE, 140435457851392, 140435457867775, +STORE, 140435457896448, 140435457900543, +STORE, 140435457900544, 140435457904639, +STORE, 140435457904640, 140435457908735, +STORE, 140721033818112, 140721033957375, +STORE, 140721034018816, 140721034031103, +STORE, 140721034031104, 140721034035199, +STORE, 94872903438336, 94872903651327, +STORE, 94872905748480, 94872905752575, +STORE, 94872905752576, 94872905760767, +STORE, 94872905760768, 94872905773055, +STORE, 94872931246080, 94872931651583, +STORE, 139771607810048, 139771609468927, +STORE, 139771609468928, 139771611566079, +STORE, 139771611566080, 139771611582463, +STORE, 139771611582464, 139771611590655, +STORE, 139771611590656, 139771611607039, +STORE, 139771611607040, 139771611619327, +STORE, 139771611619328, 139771613712383, +STORE, 139771613712384, 139771613716479, +STORE, 139771613716480, 139771613720575, +STORE, 139771613720576, 139771613863935, +STORE, 139771614232576, 139771615916031, +STORE, 139771615916032, 139771615932415, +STORE, 139771615961088, 139771615965183, +STORE, 139771615965184, 139771615969279, +STORE, 139771615969280, 139771615973375, +STORE, 140725402931200, 140725403070463, +STORE, 140725403852800, 140725403865087, +STORE, 140725403865088, 140725403869183, +STORE, 94740737736704, 94740737949695, +STORE, 94740740046848, 94740740050943, +STORE, 94740740050944, 94740740059135, +STORE, 94740740059136, 94740740071423, +STORE, 94740743249920, 94740744724479, +STORE, 140640287010816, 140640288669695, +STORE, 140640288669696, 140640290766847, +STORE, 140640290766848, 140640290783231, +STORE, 140640290783232, 140640290791423, +STORE, 140640290791424, 140640290807807, +STORE, 140640290807808, 140640290820095, +STORE, 140640290820096, 140640292913151, +STORE, 140640292913152, 140640292917247, +STORE, 140640292917248, 140640292921343, +STORE, 140640292921344, 140640293064703, +STORE, 140640293433344, 140640295116799, +STORE, 140640295116800, 140640295133183, +STORE, 140640295161856, 140640295165951, +STORE, 140640295165952, 140640295170047, +STORE, 140640295170048, 140640295174143, +STORE, 140725133303808, 140725133443071, +STORE, 140725133684736, 140725133697023, +STORE, 140725133697024, 140725133701119, +STORE, 140737488347136, 140737488351231, +STORE, 140722826371072, 140737488351231, +SNULL, 140722826375167, 140737488351231, +STORE, 140722826371072, 140722826375167, +STORE, 140722826240000, 140722826375167, +STORE, 94113818611712, 94113820835839, +SNULL, 94113818722303, 94113820835839, +STORE, 94113818611712, 94113818722303, +STORE, 94113818722304, 94113820835839, +ERASE, 94113818722304, 94113820835839, +STORE, 94113820815360, 94113820827647, +STORE, 94113820827648, 94113820835839, +STORE, 139628194508800, 139628196761599, +SNULL, 139628194652159, 139628196761599, +STORE, 139628194508800, 139628194652159, +STORE, 139628194652160, 139628196761599, +ERASE, 139628194652160, 139628196761599, +STORE, 139628196749312, 139628196757503, +STORE, 139628196757504, 139628196761599, +STORE, 140722826727424, 140722826731519, +STORE, 140722826715136, 140722826727423, +STORE, 139628196720640, 139628196749311, +STORE, 139628196712448, 139628196720639, +STORE, 139628190711808, 139628194508799, +SNULL, 139628190711808, 139628192370687, +STORE, 139628192370688, 139628194508799, +STORE, 139628190711808, 139628192370687, +SNULL, 139628194467839, 139628194508799, +STORE, 139628192370688, 139628194467839, +STORE, 139628194467840, 139628194508799, +SNULL, 139628194467840, 139628194492415, +STORE, 139628194492416, 139628194508799, +STORE, 139628194467840, 139628194492415, +ERASE, 139628194467840, 139628194492415, +STORE, 139628194467840, 139628194492415, +ERASE, 139628194492416, 139628194508799, +STORE, 139628194492416, 139628194508799, +SNULL, 139628194484223, 139628194492415, +STORE, 139628194467840, 139628194484223, +STORE, 139628194484224, 139628194492415, +SNULL, 94113820823551, 94113820827647, +STORE, 94113820815360, 94113820823551, +STORE, 94113820823552, 94113820827647, +SNULL, 139628196753407, 139628196757503, +STORE, 139628196749312, 139628196753407, +STORE, 139628196753408, 139628196757503, +ERASE, 139628196720640, 139628196749311, +STORE, 94113830850560, 94113830985727, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731865833472, 140737488351231, +SNULL, 140731865841663, 140737488351231, +STORE, 140731865833472, 140731865841663, +STORE, 140731865702400, 140731865841663, +STORE, 94763339386880, 94763341611007, +SNULL, 94763339497471, 94763341611007, +STORE, 94763339386880, 94763339497471, +STORE, 94763339497472, 94763341611007, +ERASE, 94763339497472, 94763341611007, +STORE, 94763341590528, 94763341602815, +STORE, 94763341602816, 94763341611007, +STORE, 139778398486528, 139778400739327, +SNULL, 139778398629887, 139778400739327, +STORE, 139778398486528, 139778398629887, +STORE, 139778398629888, 139778400739327, +ERASE, 139778398629888, 139778400739327, +STORE, 139778400727040, 139778400735231, +STORE, 139778400735232, 139778400739327, +STORE, 140731865858048, 140731865862143, +STORE, 140731865845760, 140731865858047, +STORE, 139778400698368, 139778400727039, +STORE, 139778400690176, 139778400698367, +STORE, 139778394689536, 139778398486527, +SNULL, 139778394689536, 139778396348415, +STORE, 139778396348416, 139778398486527, +STORE, 139778394689536, 139778396348415, +SNULL, 139778398445567, 139778398486527, +STORE, 139778396348416, 139778398445567, +STORE, 139778398445568, 139778398486527, +SNULL, 139778398445568, 139778398470143, +STORE, 139778398470144, 139778398486527, +STORE, 139778398445568, 139778398470143, +ERASE, 139778398445568, 139778398470143, +STORE, 139778398445568, 139778398470143, +ERASE, 139778398470144, 139778398486527, +STORE, 139778398470144, 139778398486527, +SNULL, 139778398461951, 139778398470143, +STORE, 139778398445568, 139778398461951, +STORE, 139778398461952, 139778398470143, +SNULL, 94763341598719, 94763341602815, +STORE, 94763341590528, 94763341598719, +STORE, 94763341598720, 94763341602815, +SNULL, 139778400731135, 139778400735231, +STORE, 139778400727040, 139778400731135, +STORE, 139778400731136, 139778400735231, +ERASE, 139778400698368, 139778400727039, +STORE, 94763362197504, 94763362332671, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140732053192704, 140737488351231, +SNULL, 140732053204991, 140737488351231, +STORE, 140732053192704, 140732053204991, +STORE, 140732053061632, 140732053204991, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 140176018599936, 140176020852735, +SNULL, 140176018743295, 140176020852735, +STORE, 140176018599936, 140176018743295, +STORE, 140176018743296, 140176020852735, +ERASE, 140176018743296, 140176020852735, +STORE, 140176020840448, 140176020848639, +STORE, 140176020848640, 140176020852735, +STORE, 140732053381120, 140732053385215, +STORE, 140732053368832, 140732053381119, +STORE, 140176020811776, 140176020840447, +STORE, 140176020803584, 140176020811775, +STORE, 140176014766080, 140176018599935, +SNULL, 140176014766080, 140176016474111, +STORE, 140176016474112, 140176018599935, +STORE, 140176014766080, 140176016474111, +SNULL, 140176018567167, 140176018599935, +STORE, 140176016474112, 140176018567167, +STORE, 140176018567168, 140176018599935, +ERASE, 140176018567168, 140176018599935, +STORE, 140176018567168, 140176018599935, +STORE, 140176012570624, 140176014766079, +SNULL, 140176012570624, 140176012664831, +STORE, 140176012664832, 140176014766079, +STORE, 140176012570624, 140176012664831, +SNULL, 140176014757887, 140176014766079, +STORE, 140176012664832, 140176014757887, +STORE, 140176014757888, 140176014766079, +ERASE, 140176014757888, 140176014766079, +STORE, 140176014757888, 140176014766079, +STORE, 140176010051584, 140176012570623, +SNULL, 140176010051584, 140176010465279, +STORE, 140176010465280, 140176012570623, +STORE, 140176010051584, 140176010465279, +SNULL, 140176012558335, 140176012570623, +STORE, 140176010465280, 140176012558335, +STORE, 140176012558336, 140176012570623, +ERASE, 140176012558336, 140176012570623, +STORE, 140176012558336, 140176012570623, +STORE, 140176007417856, 140176010051583, +SNULL, 140176007417856, 140176007946239, +STORE, 140176007946240, 140176010051583, +STORE, 140176007417856, 140176007946239, +SNULL, 140176010043391, 140176010051583, +STORE, 140176007946240, 140176010043391, +STORE, 140176010043392, 140176010051583, +ERASE, 140176010043392, 140176010051583, +STORE, 140176010043392, 140176010051583, +STORE, 140176005304320, 140176007417855, +SNULL, 140176005304320, 140176005316607, +STORE, 140176005316608, 140176007417855, +STORE, 140176005304320, 140176005316607, +SNULL, 140176007409663, 140176007417855, +STORE, 140176005316608, 140176007409663, +STORE, 140176007409664, 140176007417855, +ERASE, 140176007409664, 140176007417855, +STORE, 140176007409664, 140176007417855, +STORE, 140176003100672, 140176005304319, +SNULL, 140176003100672, 140176003203071, +STORE, 140176003203072, 140176005304319, +STORE, 140176003100672, 140176003203071, +SNULL, 140176005296127, 140176005304319, +STORE, 140176003203072, 140176005296127, +STORE, 140176005296128, 140176005304319, +ERASE, 140176005296128, 140176005304319, +STORE, 140176005296128, 140176005304319, +STORE, 140176020795392, 140176020811775, +STORE, 140175999938560, 140176003100671, +SNULL, 140175999938560, 140176000999423, +STORE, 140176000999424, 140176003100671, +STORE, 140175999938560, 140176000999423, +SNULL, 140176003092479, 140176003100671, +STORE, 140176000999424, 140176003092479, +STORE, 140176003092480, 140176003100671, +ERASE, 140176003092480, 140176003100671, +STORE, 140176003092480, 140176003100671, +STORE, 140175996141568, 140175999938559, +SNULL, 140175996141568, 140175997800447, +STORE, 140175997800448, 140175999938559, +STORE, 140175996141568, 140175997800447, +SNULL, 140175999897599, 140175999938559, +STORE, 140175997800448, 140175999897599, +STORE, 140175999897600, 140175999938559, +SNULL, 140175999897600, 140175999922175, +STORE, 140175999922176, 140175999938559, +STORE, 140175999897600, 140175999922175, +ERASE, 140175999897600, 140175999922175, +STORE, 140175999897600, 140175999922175, +ERASE, 140175999922176, 140175999938559, +STORE, 140175999922176, 140175999938559, +STORE, 140176020783104, 140176020811775, +SNULL, 140175999913983, 140175999922175, +STORE, 140175999897600, 140175999913983, +STORE, 140175999913984, 140175999922175, +SNULL, 140176003096575, 140176003100671, +STORE, 140176003092480, 140176003096575, +STORE, 140176003096576, 140176003100671, +SNULL, 140176005300223, 140176005304319, +STORE, 140176005296128, 140176005300223, +STORE, 140176005300224, 140176005304319, +SNULL, 140176007413759, 140176007417855, +STORE, 140176007409664, 140176007413759, +STORE, 140176007413760, 140176007417855, +SNULL, 140176010047487, 140176010051583, +STORE, 140176010043392, 140176010047487, +STORE, 140176010047488, 140176010051583, +SNULL, 140176012566527, 140176012570623, +STORE, 140176012558336, 140176012566527, +STORE, 140176012566528, 140176012570623, +SNULL, 140176014761983, 140176014766079, +STORE, 140176014757888, 140176014761983, +STORE, 140176014761984, 140176014766079, +SNULL, 140176018571263, 140176018599935, +STORE, 140176018567168, 140176018571263, +STORE, 140176018571264, 140176018599935, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 140176020844543, 140176020848639, +STORE, 140176020840448, 140176020844543, +STORE, 140176020844544, 140176020848639, +ERASE, 140176020811776, 140176020840447, +STORE, 53080064, 53215231, +STORE, 140176019099648, 140176020783103, +STORE, 140176020836352, 140176020840447, +STORE, 140176018964480, 140176019099647, +STORE, 53080064, 53358591, +STORE, 140175994044416, 140175996141567, +STORE, 140176020828160, 140176020840447, +STORE, 140176020819968, 140176020840447, +STORE, 140176020783104, 140176020819967, +STORE, 140176018948096, 140176019099647, +STORE, 53080064, 53493759, +STORE, 53080064, 53649407, +STORE, 140176018939904, 140176019099647, +STORE, 140176018931712, 140176019099647, +STORE, 53080064, 53784575, +STORE, 53080064, 53919743, +STORE, 140176018915328, 140176019099647, +STORE, 140176018907136, 140176019099647, +STORE, 53080064, 54059007, +STORE, 140175993769984, 140175996141567, +STORE, 140176018747392, 140176019099647, +STORE, 53080064, 54198271, +SNULL, 54190079, 54198271, +STORE, 53080064, 54190079, +STORE, 54190080, 54198271, +ERASE, 54190080, 54198271, +SNULL, 54181887, 54190079, +STORE, 53080064, 54181887, +STORE, 54181888, 54190079, +ERASE, 54181888, 54190079, +SNULL, 54173695, 54181887, +STORE, 53080064, 54173695, +STORE, 54173696, 54181887, +ERASE, 54173696, 54181887, +SNULL, 54165503, 54173695, +STORE, 53080064, 54165503, +STORE, 54165504, 54173695, +ERASE, 54165504, 54173695, +STORE, 140175993753600, 140175996141567, +STORE, 140175993688064, 140175996141567, +STORE, 140175993655296, 140175996141567, +STORE, 140175991558144, 140175996141567, +STORE, 140175991492608, 140175996141567, +STORE, 53080064, 54312959, +STORE, 140175991361536, 140175996141567, +STORE, 140175991099392, 140175996141567, +STORE, 140175991091200, 140175996141567, +STORE, 140175991074816, 140175996141567, +STORE, 140175991066624, 140175996141567, +STORE, 140175991058432, 140175996141567, +STORE, 53080064, 54448127, +SNULL, 54439935, 54448127, +STORE, 53080064, 54439935, +STORE, 54439936, 54448127, +ERASE, 54439936, 54448127, +SNULL, 54431743, 54439935, +STORE, 53080064, 54431743, +STORE, 54431744, 54439935, +ERASE, 54431744, 54439935, +SNULL, 54419455, 54431743, +STORE, 53080064, 54419455, +STORE, 54419456, 54431743, +ERASE, 54419456, 54431743, +SNULL, 54403071, 54419455, +STORE, 53080064, 54403071, +STORE, 54403072, 54419455, +ERASE, 54403072, 54419455, +STORE, 140175991042048, 140175996141567, +STORE, 53080064, 54538239, +SNULL, 54534143, 54538239, +STORE, 53080064, 54534143, +STORE, 54534144, 54538239, +ERASE, 54534144, 54538239, +SNULL, 54530047, 54534143, +STORE, 53080064, 54530047, +STORE, 54530048, 54534143, +ERASE, 54530048, 54534143, +SNULL, 54525951, 54530047, +STORE, 53080064, 54525951, +STORE, 54525952, 54530047, +ERASE, 54525952, 54530047, +SNULL, 54521855, 54525951, +STORE, 53080064, 54521855, +STORE, 54521856, 54525951, +ERASE, 54521856, 54525951, +SNULL, 54517759, 54521855, +STORE, 53080064, 54517759, +STORE, 54517760, 54521855, +ERASE, 54517760, 54521855, +SNULL, 54513663, 54517759, +STORE, 53080064, 54513663, +STORE, 54513664, 54517759, +ERASE, 54513664, 54517759, +SNULL, 54509567, 54513663, +STORE, 53080064, 54509567, +STORE, 54509568, 54513663, +ERASE, 54509568, 54513663, +STORE, 140175991025664, 140175996141567, +STORE, 140175990992896, 140175996141567, +STORE, 53080064, 54644735, +SNULL, 54628351, 54644735, +STORE, 53080064, 54628351, +STORE, 54628352, 54644735, +ERASE, 54628352, 54644735, +SNULL, 54616063, 54628351, +STORE, 53080064, 54616063, +STORE, 54616064, 54628351, +ERASE, 54616064, 54628351, +STORE, 140175988895744, 140175996141567, +STORE, 53080064, 54767615, +STORE, 140175988879360, 140175996141567, +STORE, 140175988617216, 140175996141567, +STORE, 140175988609024, 140175996141567, +STORE, 140175988600832, 140175996141567, +STORE, 53080064, 54906879, +SNULL, 54898687, 54906879, +STORE, 53080064, 54898687, +STORE, 54898688, 54906879, +ERASE, 54898688, 54906879, +SNULL, 54853631, 54898687, +STORE, 53080064, 54853631, +STORE, 54853632, 54898687, +ERASE, 54853632, 54898687, +STORE, 140175986503680, 140175996141567, +STORE, 53080064, 54996991, +STORE, 140175986495488, 140175996141567, +STORE, 140175986487296, 140175996141567, +STORE, 140175985438720, 140175996141567, +STORE, 53080064, 55136255, +STORE, 140175985405952, 140175996141567, +STORE, 140175985139712, 140175996141567, +SNULL, 140176018964479, 140176019099647, +STORE, 140176018747392, 140176018964479, +STORE, 140176018964480, 140176019099647, +ERASE, 140176018964480, 140176019099647, +STORE, 140175983042560, 140175996141567, +STORE, 140175982518272, 140175996141567, +STORE, 140175980421120, 140175996141567, +STORE, 53080064, 55287807, +STORE, 53080064, 55427071, +STORE, 140176019091456, 140176019099647, +STORE, 140176019083264, 140176019099647, +STORE, 140176019075072, 140176019099647, +STORE, 140176019066880, 140176019099647, +STORE, 140176019058688, 140176019099647, +STORE, 140175980158976, 140175996141567, +STORE, 140176019050496, 140176019099647, +STORE, 140176019042304, 140176019099647, +STORE, 140176019034112, 140176019099647, +STORE, 140176019025920, 140176019099647, +STORE, 140176019017728, 140176019099647, +STORE, 140176019009536, 140176019099647, +STORE, 140176019001344, 140176019099647, +STORE, 140176018993152, 140176019099647, +STORE, 140176018984960, 140176019099647, +STORE, 140176018976768, 140176019099647, +STORE, 140176018968576, 140176019099647, +STORE, 140175978061824, 140175996141567, +STORE, 53080064, 55603199, +STORE, 140175978029056, 140175996141567, +STORE, 140175977996288, 140175996141567, +STORE, 53080064, 55738367, +STORE, 53080064, 55881727, +STORE, 140175977963520, 140175996141567, +STORE, 140175977930752, 140175996141567, +STORE, 53080064, 56041471, +STORE, 140175977897984, 140175996141567, +STORE, 140175977865216, 140175996141567, +SNULL, 55881727, 56041471, +STORE, 53080064, 55881727, +STORE, 55881728, 56041471, +ERASE, 55881728, 56041471, +SNULL, 55721983, 55881727, +STORE, 53080064, 55721983, +STORE, 55721984, 55881727, +ERASE, 55721984, 55881727, +SNULL, 55570431, 55721983, +STORE, 53080064, 55570431, +STORE, 55570432, 55721983, +ERASE, 55570432, 55721983, +STORE, 140175977857024, 140175996141567, +STORE, 140175975759872, 140175996141567, +STORE, 53080064, 55754751, +STORE, 53080064, 55943167, +STORE, 140175975751680, 140175996141567, +STORE, 140175975743488, 140175996141567, +STORE, 140175975735296, 140175996141567, +STORE, 140175975727104, 140175996141567, +STORE, 140175975718912, 140175996141567, +STORE, 140175975710720, 140175996141567, +STORE, 140175975702528, 140175996141567, +STORE, 140175975694336, 140175996141567, +STORE, 140175975686144, 140175996141567, +STORE, 140175975677952, 140175996141567, +STORE, 140175975669760, 140175996141567, +STORE, 140175974621184, 140175996141567, +STORE, 140175974612992, 140175996141567, +STORE, 53080064, 56139775, +STORE, 140175972515840, 140175996141567, +STORE, 53080064, 56401919, +STORE, 140175970418688, 140175996141567, +STORE, 140175970410496, 140175996141567, +STORE, 140175970402304, 140175996141567, +STORE, 140175970394112, 140175996141567, +STORE, 53080064, 56569855, +STORE, 140175969865728, 140175996141567, +SNULL, 140175985139711, 140175996141567, +STORE, 140175969865728, 140175985139711, +STORE, 140175985139712, 140175996141567, +SNULL, 140175985139712, 140175985405951, +STORE, 140175985405952, 140175996141567, +STORE, 140175985139712, 140175985405951, +ERASE, 140175985139712, 140175985405951, +STORE, 140175965671424, 140175985139711, +STORE, 140175985397760, 140175996141567, +STORE, 140175985389568, 140175996141567, +STORE, 140175985381376, 140175996141567, +STORE, 140175985373184, 140175996141567, +STORE, 140175985364992, 140175996141567, +STORE, 140175985356800, 140175996141567, +STORE, 140175985348608, 140175996141567, +STORE, 140175985340416, 140175996141567, +STORE, 140175985332224, 140175996141567, +STORE, 140175985324032, 140175996141567, +STORE, 140175985315840, 140175996141567, +STORE, 140175985307648, 140175996141567, +STORE, 140175985299456, 140175996141567, +STORE, 140175985291264, 140175996141567, +STORE, 140175985283072, 140175996141567, +STORE, 140175985274880, 140175996141567, +STORE, 140175963574272, 140175985139711, +STORE, 140175985266688, 140175996141567, +STORE, 140175961477120, 140175985139711, +STORE, 53080064, 56831999, +STORE, 140175959379968, 140175985139711, +STORE, 140175985258496, 140175996141567, +STORE, 140175957282816, 140175985139711, +STORE, 140175985250304, 140175996141567, +STORE, 140175985242112, 140175996141567, +STORE, 140175985233920, 140175996141567, +STORE, 140175985225728, 140175996141567, +STORE, 140175985217536, 140175996141567, +STORE, 140175957151744, 140175985139711, +STORE, 140175956627456, 140175985139711, +SNULL, 140175980158975, 140175985139711, +STORE, 140175956627456, 140175980158975, +STORE, 140175980158976, 140175985139711, +SNULL, 140175980158976, 140175980421119, +STORE, 140175980421120, 140175985139711, +STORE, 140175980158976, 140175980421119, +ERASE, 140175980158976, 140175980421119, +STORE, 140175954530304, 140175980158975, +STORE, 140175985209344, 140175996141567, +STORE, 53080064, 57094143, +STORE, 140175952433152, 140175980158975, +STORE, 140175985192960, 140175996141567, +STORE, 140175985184768, 140175996141567, +STORE, 140175985176576, 140175996141567, +STORE, 140175985168384, 140175996141567, +STORE, 140175985160192, 140175996141567, +STORE, 140175985152000, 140175996141567, +STORE, 140175985143808, 140175996141567, +STORE, 140175980412928, 140175985139711, +STORE, 140175980404736, 140175985139711, +STORE, 140175980396544, 140175985139711, +STORE, 140175980388352, 140175985139711, +STORE, 140175980380160, 140175985139711, +STORE, 140175980371968, 140175985139711, +STORE, 140175980363776, 140175985139711, +STORE, 140175980355584, 140175985139711, +STORE, 140175980347392, 140175985139711, +STORE, 140175980339200, 140175985139711, +STORE, 53080064, 57356287, +SNULL, 140176018747392, 140176018907135, +STORE, 140176018907136, 140176018964479, +STORE, 140176018747392, 140176018907135, +ERASE, 140176018747392, 140176018907135, +STORE, 140175952146432, 140175980158975, +STORE, 140175950049280, 140175980158975, +SNULL, 140175952146431, 140175980158975, +STORE, 140175950049280, 140175952146431, +STORE, 140175952146432, 140175980158975, +SNULL, 140175952146432, 140175952433151, +STORE, 140175952433152, 140175980158975, +STORE, 140175952146432, 140175952433151, +ERASE, 140175952146432, 140175952433151, +STORE, 140176018898944, 140176018964479, +STORE, 53080064, 57749503, +STORE, 140175949520896, 140175952146431, +STORE, 140175947423744, 140175952146431, +SNULL, 140175993769983, 140175996141567, +STORE, 140175985143808, 140175993769983, +STORE, 140175993769984, 140175996141567, +SNULL, 140175993769984, 140175994044415, +STORE, 140175994044416, 140175996141567, +STORE, 140175993769984, 140175994044415, +ERASE, 140175993769984, 140175994044415, +STORE, 140176018890752, 140176018964479, +STORE, 140176018882560, 140176018964479, +STORE, 140176018874368, 140176018964479, +STORE, 140176018866176, 140176018964479, +STORE, 140176018849792, 140176018964479, +STORE, 140176018841600, 140176018964479, +STORE, 140176018825216, 140176018964479, +STORE, 140176018817024, 140176018964479, +STORE, 140176018800640, 140176018964479, +STORE, 140176018792448, 140176018964479, +STORE, 140176018759680, 140176018964479, +STORE, 140176018751488, 140176018964479, +STORE, 140175994028032, 140175996141567, +STORE, 140176018743296, 140176018964479, +STORE, 140175994011648, 140175996141567, +STORE, 140175994003456, 140175996141567, +STORE, 140175993987072, 140175996141567, +STORE, 140175993978880, 140175996141567, +STORE, 140175993946112, 140175996141567, +STORE, 140175993937920, 140175996141567, +STORE, 140175993921536, 140175996141567, +STORE, 140175993913344, 140175996141567, +STORE, 140175993896960, 140175996141567, +STORE, 140175993888768, 140175996141567, +STORE, 140175993872384, 140175996141567, +STORE, 140175993864192, 140175996141567, +STORE, 140175993831424, 140175996141567, +STORE, 140175993823232, 140175996141567, +STORE, 140175993806848, 140175996141567, +STORE, 140175993798656, 140175996141567, +STORE, 140175993782272, 140175996141567, +STORE, 140175993774080, 140175996141567, +STORE, 140175980322816, 140175985139711, +STORE, 140175980314624, 140175985139711, +STORE, 140175980281856, 140175985139711, +STORE, 140175980273664, 140175985139711, +STORE, 140175980257280, 140175985139711, +STORE, 140175945326592, 140175952146431, +STORE, 140175980249088, 140175985139711, +STORE, 140175980232704, 140175985139711, +STORE, 140175980224512, 140175985139711, +STORE, 140175980208128, 140175985139711, +STORE, 140175980199936, 140175985139711, +STORE, 140175980167168, 140175985139711, +STORE, 140175952433152, 140175985139711, +STORE, 140175952416768, 140175985139711, +STORE, 140175952408576, 140175985139711, +STORE, 140175952392192, 140175985139711, +STORE, 140175952384000, 140175985139711, +STORE, 140175952367616, 140175985139711, +STORE, 140175943229440, 140175952146431, +STORE, 140175952359424, 140175985139711, +STORE, 140175952326656, 140175985139711, +STORE, 140175952318464, 140175985139711, +STORE, 140175952302080, 140175985139711, +STORE, 140175952293888, 140175985139711, +STORE, 140175952277504, 140175985139711, +STORE, 140175952269312, 140175985139711, +STORE, 140175952252928, 140175985139711, +STORE, 140175952244736, 140175985139711, +STORE, 140175952211968, 140175985139711, +STORE, 140175952203776, 140175985139711, +STORE, 140175952187392, 140175985139711, +STORE, 140175952179200, 140175985139711, +STORE, 140175952162816, 140175985139711, +STORE, 140175952154624, 140175985139711, +STORE, 140175943213056, 140175952146431, +STORE, 140175943213056, 140175985139711, +STORE, 140175943180288, 140175985139711, +STORE, 140175943172096, 140175985139711, +STORE, 140175943155712, 140175985139711, +STORE, 140175943147520, 140175985139711, +STORE, 140175943131136, 140175985139711, +STORE, 140175943122944, 140175985139711, +STORE, 140175943106560, 140175985139711, +STORE, 140175943098368, 140175985139711, +STORE, 140175943065600, 140175985139711, +STORE, 140175943057408, 140175985139711, +STORE, 140175943041024, 140175985139711, +STORE, 140175943032832, 140175985139711, +STORE, 140175943016448, 140175985139711, +STORE, 140175943008256, 140175985139711, +STORE, 140175942991872, 140175985139711, +STORE, 140175942983680, 140175985139711, +STORE, 140175942950912, 140175985139711, +STORE, 140175942942720, 140175985139711, +STORE, 140175942926336, 140175985139711, +STORE, 140175942918144, 140175985139711, +STORE, 140175942901760, 140175985139711, +STORE, 140175942893568, 140175985139711, +STORE, 140175942877184, 140175985139711, +STORE, 140175942868992, 140175985139711, +STORE, 140175942836224, 140175985139711, +STORE, 140175942828032, 140175985139711, +STORE, 140175942811648, 140175985139711, +STORE, 140175942803456, 140175985139711, +STORE, 140175942787072, 140175985139711, +STORE, 140175942778880, 140175985139711, +STORE, 140175942762496, 140175985139711, +STORE, 140175942754304, 140175985139711, +STORE, 140175942721536, 140175985139711, +STORE, 140175942713344, 140175985139711, +STORE, 140175942696960, 140175985139711, +STORE, 140175942688768, 140175985139711, +STORE, 140175942672384, 140175985139711, +STORE, 140175942664192, 140175985139711, +STORE, 140175942647808, 140175985139711, +STORE, 140175942639616, 140175985139711, +STORE, 140175942606848, 140175985139711, +STORE, 140175942598656, 140175985139711, +STORE, 140175942582272, 140175985139711, +STORE, 140175942574080, 140175985139711, +STORE, 140175942557696, 140175985139711, +STORE, 140175942549504, 140175985139711, +STORE, 140175942533120, 140175985139711, +STORE, 140175942524928, 140175985139711, +STORE, 140175942492160, 140175985139711, +STORE, 140175942483968, 140175985139711, +STORE, 140175942467584, 140175985139711, +STORE, 140175942459392, 140175985139711, +STORE, 140175942443008, 140175985139711, +STORE, 140175942434816, 140175985139711, +STORE, 140175942418432, 140175985139711, +STORE, 140175942410240, 140175985139711, +STORE, 140175942377472, 140175985139711, +STORE, 140175942369280, 140175985139711, +STORE, 140175942352896, 140175985139711, +STORE, 140175942344704, 140175985139711, +STORE, 140175942328320, 140175985139711, +STORE, 140175942320128, 140175985139711, +STORE, 140175942303744, 140175985139711, +STORE, 140175942295552, 140175985139711, +STORE, 140175942262784, 140175985139711, +STORE, 140175942254592, 140175985139711, +STORE, 140175942238208, 140175985139711, +STORE, 140175942230016, 140175985139711, +STORE, 140175942213632, 140175985139711, +STORE, 140175942205440, 140175985139711, +STORE, 140175942189056, 140175985139711, +STORE, 140175942180864, 140175985139711, +STORE, 140175942148096, 140175985139711, +STORE, 140175942139904, 140175985139711, +STORE, 140175942123520, 140175985139711, +STORE, 140175942115328, 140175985139711, +STORE, 140175942098944, 140175985139711, +STORE, 140175942090752, 140175985139711, +STORE, 140175942074368, 140175985139711, +STORE, 140175942066176, 140175985139711, +STORE, 140175942033408, 140175985139711, +STORE, 140175942025216, 140175985139711, +STORE, 140175942008832, 140175985139711, +STORE, 140175942000640, 140175985139711, +STORE, 140175941984256, 140175985139711, +STORE, 140175941976064, 140175985139711, +STORE, 140175941959680, 140175985139711, +STORE, 140175939862528, 140175985139711, +STORE, 140175939854336, 140175985139711, +STORE, 140175939821568, 140175985139711, +STORE, 140175939813376, 140175985139711, +STORE, 140175939796992, 140175985139711, +STORE, 140175939788800, 140175985139711, +STORE, 140175939772416, 140175985139711, +STORE, 140175939764224, 140175985139711, +STORE, 140175939747840, 140175985139711, +STORE, 140175939739648, 140175985139711, +STORE, 140175939706880, 140175985139711, +STORE, 140175939698688, 140175985139711, +STORE, 140175939682304, 140175985139711, +STORE, 140175939674112, 140175985139711, +STORE, 140175939657728, 140175985139711, +STORE, 140175939649536, 140175985139711, +STORE, 140175939633152, 140175985139711, +STORE, 140175939624960, 140175985139711, +STORE, 140175939592192, 140175985139711, +STORE, 140175939584000, 140175985139711, +STORE, 140175939567616, 140175985139711, +STORE, 140175939559424, 140175985139711, +STORE, 140175939543040, 140175985139711, +STORE, 140175939534848, 140175985139711, +STORE, 140175939518464, 140175985139711, +STORE, 140175939510272, 140175985139711, +STORE, 140175939477504, 140175985139711, +STORE, 140175939469312, 140175985139711, +STORE, 140175939452928, 140175985139711, +STORE, 140175939444736, 140175985139711, +STORE, 140175939428352, 140175985139711, +STORE, 140175939420160, 140175985139711, +STORE, 140175939403776, 140175985139711, +STORE, 140175939395584, 140175985139711, +STORE, 140175939362816, 140175985139711, +STORE, 140175939354624, 140175985139711, +STORE, 140175939338240, 140175985139711, +STORE, 140175939330048, 140175985139711, +STORE, 140175939313664, 140175985139711, +STORE, 140175939305472, 140175985139711, +STORE, 140175939289088, 140175985139711, +STORE, 140175939280896, 140175985139711, +STORE, 140175939248128, 140175985139711, +STORE, 140175939239936, 140175985139711, +STORE, 140175939223552, 140175985139711, +STORE, 140175939215360, 140175985139711, +STORE, 140175939198976, 140175985139711, +STORE, 140175939190784, 140175985139711, +STORE, 140175939174400, 140175985139711, +STORE, 140175939166208, 140175985139711, +STORE, 140175939133440, 140175985139711, +STORE, 140175939125248, 140175985139711, +STORE, 140175939108864, 140175985139711, +STORE, 140175939100672, 140175985139711, +STORE, 140175939084288, 140175985139711, +STORE, 140175939076096, 140175985139711, +STORE, 140175939059712, 140175985139711, +STORE, 140175939051520, 140175985139711, +STORE, 140175939018752, 140175985139711, +STORE, 140175939010560, 140175985139711, +STORE, 140175938994176, 140175985139711, +STORE, 140175938985984, 140175985139711, +STORE, 140175938969600, 140175985139711, +STORE, 140175938961408, 140175985139711, +STORE, 140175938945024, 140175985139711, +STORE, 140175938936832, 140175985139711, +STORE, 140175938904064, 140175985139711, +STORE, 140175938895872, 140175985139711, +STORE, 140175938879488, 140175985139711, +STORE, 140175938871296, 140175985139711, +STORE, 140175938854912, 140175985139711, +STORE, 140175938846720, 140175985139711, +STORE, 140175938830336, 140175985139711, +STORE, 140175938822144, 140175985139711, +STORE, 140175938789376, 140175985139711, +STORE, 140175938781184, 140175985139711, +STORE, 140175938764800, 140175985139711, +STORE, 140175938756608, 140175985139711, +STORE, 140175938740224, 140175985139711, +STORE, 140175938732032, 140175985139711, +STORE, 140175938715648, 140175985139711, +STORE, 140175938707456, 140175985139711, +STORE, 140175938674688, 140175985139711, +STORE, 140175938666496, 140175985139711, +STORE, 140175938650112, 140175985139711, +STORE, 140175938641920, 140175985139711, +STORE, 140175938625536, 140175985139711, +STORE, 140175938617344, 140175985139711, +STORE, 140175938600960, 140175985139711, +STORE, 140175938592768, 140175985139711, +STORE, 140175938560000, 140175985139711, +STORE, 140175938551808, 140175985139711, +STORE, 140175938535424, 140175985139711, +STORE, 140175938527232, 140175985139711, +STORE, 140175938510848, 140175985139711, +STORE, 140175938502656, 140175985139711, +STORE, 140175938486272, 140175985139711, +STORE, 140175938478080, 140175985139711, +STORE, 140175938445312, 140175985139711, +STORE, 140175938437120, 140175985139711, +STORE, 140175938420736, 140175985139711, +STORE, 140175938412544, 140175985139711, +STORE, 140175938396160, 140175985139711, +STORE, 140175938387968, 140175985139711, +STORE, 140175938371584, 140175985139711, +STORE, 140175938363392, 140175985139711, +STORE, 140175938330624, 140175985139711, +STORE, 140175938322432, 140175985139711, +STORE, 140175938306048, 140175985139711, +STORE, 140175938297856, 140175985139711, +STORE, 140175938281472, 140175985139711, +STORE, 140175938273280, 140175985139711, +STORE, 140175938256896, 140175985139711, +STORE, 140175938248704, 140175985139711, +STORE, 140175938215936, 140175985139711, +STORE, 140175938207744, 140175985139711, +STORE, 140175938191360, 140175985139711, +STORE, 140175938183168, 140175985139711, +STORE, 140175938166784, 140175985139711, +STORE, 140175938158592, 140175985139711, +STORE, 140175938142208, 140175985139711, +STORE, 140175936045056, 140175985139711, +STORE, 140175936036864, 140175985139711, +STORE, 140175936004096, 140175985139711, +STORE, 140175935995904, 140175985139711, +STORE, 140175935979520, 140175985139711, +STORE, 140175935971328, 140175985139711, +STORE, 140175935954944, 140175985139711, +STORE, 140175935946752, 140175985139711, +STORE, 140175935930368, 140175985139711, +STORE, 140175935922176, 140175985139711, +STORE, 140175935889408, 140175985139711, +STORE, 140175935881216, 140175985139711, +STORE, 140175935864832, 140175985139711, +STORE, 140175935856640, 140175985139711, +STORE, 140175935840256, 140175985139711, +STORE, 140175935832064, 140175985139711, +STORE, 140175935815680, 140175985139711, +STORE, 140175935807488, 140175985139711, +STORE, 140175935774720, 140175985139711, +STORE, 140175935766528, 140175985139711, +STORE, 140175935750144, 140175985139711, +STORE, 140175935741952, 140175985139711, +STORE, 140175935725568, 140175985139711, +STORE, 140175935717376, 140175985139711, +STORE, 140175935700992, 140175985139711, +STORE, 140175935692800, 140175985139711, +STORE, 140175935660032, 140175985139711, +STORE, 140175935651840, 140175985139711, +STORE, 140175935635456, 140175985139711, +STORE, 140175935627264, 140175985139711, +STORE, 140175935610880, 140175985139711, +STORE, 140175935602688, 140175985139711, +STORE, 140175935586304, 140175985139711, +STORE, 140175935578112, 140175985139711, +STORE, 140175935545344, 140175985139711, +STORE, 140175935537152, 140175985139711, +STORE, 140175935520768, 140175985139711, +STORE, 140175935512576, 140175985139711, +STORE, 140175935496192, 140175985139711, +STORE, 140175935488000, 140175985139711, +STORE, 140175935471616, 140175985139711, +STORE, 140175935463424, 140175985139711, +STORE, 140175935430656, 140175985139711, +STORE, 140175935422464, 140175985139711, +STORE, 140175935406080, 140175985139711, +STORE, 140175935397888, 140175985139711, +STORE, 140175935381504, 140175985139711, +STORE, 140175935373312, 140175985139711, +STORE, 140175935356928, 140175985139711, +STORE, 140175935348736, 140175985139711, +STORE, 140175935315968, 140175985139711, +STORE, 140175935307776, 140175985139711, +STORE, 140175935291392, 140175985139711, +STORE, 140175935283200, 140175985139711, +STORE, 140175935266816, 140175985139711, +STORE, 140175935258624, 140175985139711, +STORE, 140175935242240, 140175985139711, +STORE, 140175935234048, 140175985139711, +STORE, 140175935201280, 140175985139711, +STORE, 140175935193088, 140175985139711, +STORE, 140175935176704, 140175985139711, +STORE, 140175935168512, 140175985139711, +STORE, 140175935152128, 140175985139711, +STORE, 140175935143936, 140175985139711, +STORE, 140175935127552, 140175985139711, +STORE, 140175935119360, 140175985139711, +STORE, 140175935086592, 140175985139711, +STORE, 140175935078400, 140175985139711, +STORE, 140175935062016, 140175985139711, +STORE, 140175935053824, 140175985139711, +STORE, 140175935037440, 140175985139711, +STORE, 140175935029248, 140175985139711, +STORE, 140175935012864, 140175985139711, +STORE, 140175935004672, 140175985139711, +STORE, 140175934971904, 140175985139711, +STORE, 140175934963712, 140175985139711, +STORE, 140175934947328, 140175985139711, +STORE, 140175934939136, 140175985139711, +STORE, 140175934922752, 140175985139711, +STORE, 140175934914560, 140175985139711, +STORE, 140175934898176, 140175985139711, +STORE, 140175934889984, 140175985139711, +STORE, 140175934857216, 140175985139711, +STORE, 140175934849024, 140175985139711, +STORE, 140175934832640, 140175985139711, +STORE, 140175934824448, 140175985139711, +STORE, 140175934808064, 140175985139711, +STORE, 140175934799872, 140175985139711, +STORE, 140175934783488, 140175985139711, +STORE, 140175934775296, 140175985139711, +STORE, 140175934742528, 140175985139711, +STORE, 140175934734336, 140175985139711, +STORE, 140175934717952, 140175985139711, +STORE, 140175934709760, 140175985139711, +STORE, 140175934693376, 140175985139711, +STORE, 140175934685184, 140175985139711, +STORE, 140175934668800, 140175985139711, +STORE, 140175934660608, 140175985139711, +STORE, 140175934627840, 140175985139711, +STORE, 140175934619648, 140175985139711, +STORE, 140175934603264, 140175985139711, +STORE, 140175934595072, 140175985139711, +STORE, 140175934578688, 140175985139711, +STORE, 140175934570496, 140175985139711, +STORE, 140175934554112, 140175985139711, +STORE, 140175934545920, 140175985139711, +STORE, 140175934513152, 140175985139711, +STORE, 140175934504960, 140175985139711, +STORE, 140175934488576, 140175985139711, +STORE, 140175934480384, 140175985139711, +STORE, 140175934464000, 140175985139711, +STORE, 140175934455808, 140175985139711, +STORE, 140175934439424, 140175985139711, +STORE, 140175934431232, 140175985139711, +STORE, 140175934398464, 140175985139711, +STORE, 140175934390272, 140175985139711, +STORE, 140175934373888, 140175985139711, +STORE, 140175934365696, 140175985139711, +STORE, 140175934349312, 140175985139711, +STORE, 140175934341120, 140175985139711, +STORE, 140175934324736, 140175985139711, +STORE, 140175932227584, 140175985139711, +STORE, 140175932219392, 140175985139711, +STORE, 140175932186624, 140175985139711, +STORE, 140175932178432, 140175985139711, +STORE, 140175932162048, 140175985139711, +STORE, 140175932153856, 140175985139711, +STORE, 140175932137472, 140175985139711, +STORE, 53080064, 57884671, +STORE, 140175932129280, 140175985139711, +STORE, 140175932112896, 140175985139711, +STORE, 140175932104704, 140175985139711, +STORE, 140175932071936, 140175985139711, +STORE, 140175932063744, 140175985139711, +STORE, 140175932047360, 140175985139711, +STORE, 140175932039168, 140175985139711, +STORE, 140175932022784, 140175985139711, +STORE, 140175932014592, 140175985139711, +STORE, 140175931998208, 140175985139711, +STORE, 140175931990016, 140175985139711, +STORE, 140175931957248, 140175985139711, +STORE, 140175931949056, 140175985139711, +STORE, 140175931932672, 140175985139711, +STORE, 140175931924480, 140175985139711, +STORE, 140175931908096, 140175985139711, +STORE, 140175931899904, 140175985139711, +STORE, 140175931883520, 140175985139711, +STORE, 140175931875328, 140175985139711, +STORE, 140175931842560, 140175985139711, +STORE, 140175931834368, 140175985139711, +STORE, 140175931817984, 140175985139711, +STORE, 140175931809792, 140175985139711, +STORE, 140175931793408, 140175985139711, +STORE, 140175931785216, 140175985139711, +STORE, 140175931768832, 140175985139711, +STORE, 140175931760640, 140175985139711, +STORE, 140175931727872, 140175985139711, +STORE, 140175931719680, 140175985139711, +STORE, 140175931703296, 140175985139711, +STORE, 140175931695104, 140175985139711, +STORE, 140175931678720, 140175985139711, +STORE, 140175931670528, 140175985139711, +STORE, 140175931654144, 140175985139711, +STORE, 140175931645952, 140175985139711, +STORE, 140175931613184, 140175985139711, +STORE, 140175931604992, 140175985139711, +STORE, 140175931588608, 140175985139711, +STORE, 140175931580416, 140175985139711, +STORE, 140175931564032, 140175985139711, +STORE, 140175931555840, 140175985139711, +STORE, 140175931539456, 140175985139711, +STORE, 140175931531264, 140175985139711, +STORE, 140175931498496, 140175985139711, +STORE, 140175931490304, 140175985139711, +STORE, 140175931473920, 140175985139711, +STORE, 140175931465728, 140175985139711, +STORE, 140175931449344, 140175985139711, +STORE, 140175931441152, 140175985139711, +STORE, 140175931424768, 140175985139711, +STORE, 140175931416576, 140175985139711, +STORE, 140175931383808, 140175985139711, +STORE, 140175931375616, 140175985139711, +STORE, 140175931359232, 140175985139711, +STORE, 140175931351040, 140175985139711, +STORE, 140175931334656, 140175985139711, +STORE, 140175931326464, 140175985139711, +STORE, 140175931310080, 140175985139711, +STORE, 140175931301888, 140175985139711, +STORE, 140175931269120, 140175985139711, +STORE, 140175931260928, 140175985139711, +STORE, 140175931244544, 140175985139711, +STORE, 140175931236352, 140175985139711, +STORE, 140175931219968, 140175985139711, +STORE, 140175931211776, 140175985139711, +STORE, 140175931195392, 140175985139711, +STORE, 140175931187200, 140175985139711, +STORE, 140175931154432, 140175985139711, +STORE, 140175931146240, 140175985139711, +STORE, 140175931129856, 140175985139711, +STORE, 140175931121664, 140175985139711, +STORE, 140175931105280, 140175985139711, +STORE, 140175931097088, 140175985139711, +STORE, 140175931080704, 140175985139711, +STORE, 140175931072512, 140175985139711, +STORE, 140175931039744, 140175985139711, +STORE, 140175931031552, 140175985139711, +STORE, 140175931015168, 140175985139711, +STORE, 140175931006976, 140175985139711, +STORE, 140175930990592, 140175985139711, +STORE, 140175930982400, 140175985139711, +STORE, 140175930966016, 140175985139711, +STORE, 140175930957824, 140175985139711, +STORE, 140175930925056, 140175985139711, +STORE, 140175930916864, 140175985139711, +STORE, 140175930900480, 140175985139711, +STORE, 140175930892288, 140175985139711, +STORE, 140175930875904, 140175985139711, +STORE, 140175930867712, 140175985139711, +STORE, 140175930851328, 140175985139711, +STORE, 140175930843136, 140175985139711, +STORE, 140175930810368, 140175985139711, +STORE, 140175930802176, 140175985139711, +STORE, 140175930785792, 140175985139711, +STORE, 140175930777600, 140175985139711, +STORE, 140175930761216, 140175985139711, +STORE, 140175930753024, 140175985139711, +STORE, 140175930736640, 140175985139711, +STORE, 140175930728448, 140175985139711, +STORE, 140175930695680, 140175985139711, +STORE, 140175930687488, 140175985139711, +STORE, 140175930671104, 140175985139711, +STORE, 140175930662912, 140175985139711, +STORE, 140175930646528, 140175985139711, +STORE, 140175930638336, 140175985139711, +STORE, 140175930621952, 140175985139711, +STORE, 140175930613760, 140175985139711, +STORE, 140175930580992, 140175985139711, +STORE, 140175930572800, 140175985139711, +STORE, 140175930556416, 140175985139711, +STORE, 140175930548224, 140175985139711, +STORE, 140175930531840, 140175985139711, +STORE, 140175930523648, 140175985139711, +STORE, 140175930507264, 140175985139711, +STORE, 140175928410112, 140175985139711, +STORE, 140175928401920, 140175985139711, +STORE, 140175928369152, 140175985139711, +STORE, 140175928360960, 140175985139711, +STORE, 140175928344576, 140175985139711, +STORE, 140175928336384, 140175985139711, +STORE, 140175928320000, 140175985139711, +STORE, 140175928311808, 140175985139711, +STORE, 140175928295424, 140175985139711, +STORE, 140175927242752, 140175985139711, +SNULL, 140175956627455, 140175985139711, +STORE, 140175927242752, 140175956627455, +STORE, 140175956627456, 140175985139711, + }; + unsigned long set24[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140735281639424, 140737488351231, +SNULL, 140735281643519, 140737488351231, +STORE, 140735281639424, 140735281643519, +STORE, 140735281508352, 140735281643519, +STORE, 94717834911744, 94717834928127, +SNULL, 94717834915839, 94717834928127, +STORE, 94717834911744, 94717834915839, +STORE, 94717834915840, 94717834928127, +ERASE, 94717834915840, 94717834928127, +STORE, 94717834919936, 94717834928127, +STORE, 140428246065152, 140428248317951, +SNULL, 140428246208511, 140428248317951, +STORE, 140428246065152, 140428246208511, +STORE, 140428246208512, 140428248317951, +ERASE, 140428246208512, 140428248317951, +STORE, 140428248305664, 140428248313855, +STORE, 140428248313856, 140428248317951, +STORE, 140735281811456, 140735281815551, +STORE, 140735281799168, 140735281811455, +STORE, 140428248297472, 140428248305663, +STORE, 140428243841024, 140428246065151, +SNULL, 140428245491711, 140428246065151, +STORE, 140428243841024, 140428245491711, +STORE, 140428245491712, 140428246065151, +SNULL, 140428245491712, 140428246061055, +STORE, 140428246061056, 140428246065151, +STORE, 140428245491712, 140428246061055, +ERASE, 140428245491712, 140428246061055, +STORE, 140428245491712, 140428246061055, +ERASE, 140428246061056, 140428246065151, +STORE, 140428246061056, 140428246065151, +STORE, 140428248268800, 140428248297471, +STORE, 140428241625088, 140428243841023, +SNULL, 140428241625088, 140428241723391, +STORE, 140428241723392, 140428243841023, +STORE, 140428241625088, 140428241723391, +SNULL, 140428243816447, 140428243841023, +STORE, 140428241723392, 140428243816447, +STORE, 140428243816448, 140428243841023, +SNULL, 140428243816448, 140428243824639, +STORE, 140428243824640, 140428243841023, +STORE, 140428243816448, 140428243824639, +ERASE, 140428243816448, 140428243824639, +STORE, 140428243816448, 140428243824639, +ERASE, 140428243824640, 140428243841023, +STORE, 140428243824640, 140428243841023, +STORE, 140428237828096, 140428241625087, +SNULL, 140428237828096, 140428239486975, +STORE, 140428239486976, 140428241625087, +STORE, 140428237828096, 140428239486975, +SNULL, 140428241584127, 140428241625087, +STORE, 140428239486976, 140428241584127, +STORE, 140428241584128, 140428241625087, +SNULL, 140428241584128, 140428241608703, +STORE, 140428241608704, 140428241625087, +STORE, 140428241584128, 140428241608703, +ERASE, 140428241584128, 140428241608703, +STORE, 140428241584128, 140428241608703, +ERASE, 140428241608704, 140428241625087, +STORE, 140428241608704, 140428241625087, +STORE, 140428235567104, 140428237828095, +SNULL, 140428235567104, 140428235718655, +STORE, 140428235718656, 140428237828095, +STORE, 140428235567104, 140428235718655, +SNULL, 140428237811711, 140428237828095, +STORE, 140428235718656, 140428237811711, +STORE, 140428237811712, 140428237828095, +SNULL, 140428237811712, 140428237819903, +STORE, 140428237819904, 140428237828095, +STORE, 140428237811712, 140428237819903, +ERASE, 140428237811712, 140428237819903, +STORE, 140428237811712, 140428237819903, +ERASE, 140428237819904, 140428237828095, +STORE, 140428237819904, 140428237828095, +STORE, 140428233445376, 140428235567103, +SNULL, 140428233445376, 140428233461759, +STORE, 140428233461760, 140428235567103, +STORE, 140428233445376, 140428233461759, +SNULL, 140428235558911, 140428235567103, +STORE, 140428233461760, 140428235558911, +STORE, 140428235558912, 140428235567103, +ERASE, 140428235558912, 140428235567103, +STORE, 140428235558912, 140428235567103, +STORE, 140428231315456, 140428233445375, +SNULL, 140428231315456, 140428231344127, +STORE, 140428231344128, 140428233445375, +STORE, 140428231315456, 140428231344127, +SNULL, 140428233437183, 140428233445375, +STORE, 140428231344128, 140428233437183, +STORE, 140428233437184, 140428233445375, +ERASE, 140428233437184, 140428233445375, +STORE, 140428233437184, 140428233445375, +STORE, 140428248260608, 140428248268799, +STORE, 140428229062656, 140428231315455, +SNULL, 140428229062656, 140428229214207, +STORE, 140428229214208, 140428231315455, +STORE, 140428229062656, 140428229214207, +SNULL, 140428231307263, 140428231315455, +STORE, 140428229214208, 140428231307263, +STORE, 140428231307264, 140428231315455, +ERASE, 140428231307264, 140428231315455, +STORE, 140428231307264, 140428231315455, +STORE, 140428226891776, 140428229062655, +SNULL, 140428226891776, 140428226961407, +STORE, 140428226961408, 140428229062655, +STORE, 140428226891776, 140428226961407, +SNULL, 140428229054463, 140428229062655, +STORE, 140428226961408, 140428229054463, +STORE, 140428229054464, 140428229062655, +ERASE, 140428229054464, 140428229062655, +STORE, 140428229054464, 140428229062655, +STORE, 140428223680512, 140428226891775, +SNULL, 140428223680512, 140428224757759, +STORE, 140428224757760, 140428226891775, +STORE, 140428223680512, 140428224757759, +SNULL, 140428226854911, 140428226891775, +STORE, 140428224757760, 140428226854911, +STORE, 140428226854912, 140428226891775, +ERASE, 140428226854912, 140428226891775, +STORE, 140428226854912, 140428226891775, +STORE, 140428221546496, 140428223680511, +SNULL, 140428221546496, 140428221575167, +STORE, 140428221575168, 140428223680511, +STORE, 140428221546496, 140428221575167, +SNULL, 140428223672319, 140428223680511, +STORE, 140428221575168, 140428223672319, +STORE, 140428223672320, 140428223680511, +ERASE, 140428223672320, 140428223680511, +STORE, 140428223672320, 140428223680511, +STORE, 140428219236352, 140428221546495, +SNULL, 140428219236352, 140428219441151, +STORE, 140428219441152, 140428221546495, +STORE, 140428219236352, 140428219441151, +SNULL, 140428221538303, 140428221546495, +STORE, 140428219441152, 140428221538303, +STORE, 140428221538304, 140428221546495, +ERASE, 140428221538304, 140428221546495, +STORE, 140428221538304, 140428221546495, +STORE, 140428216852480, 140428219236351, +SNULL, 140428216852480, 140428217044991, +STORE, 140428217044992, 140428219236351, +STORE, 140428216852480, 140428217044991, +SNULL, 140428219138047, 140428219236351, +STORE, 140428217044992, 140428219138047, +STORE, 140428219138048, 140428219236351, +ERASE, 140428219138048, 140428219236351, +STORE, 140428219138048, 140428219236351, +STORE, 140428248252416, 140428248268799, +STORE, 140428214284288, 140428216852479, +SNULL, 140428214284288, 140428214751231, +STORE, 140428214751232, 140428216852479, +STORE, 140428214284288, 140428214751231, +SNULL, 140428216844287, 140428216852479, +STORE, 140428214751232, 140428216844287, +STORE, 140428216844288, 140428216852479, +ERASE, 140428216844288, 140428216852479, +STORE, 140428216844288, 140428216852479, +STORE, 140428212170752, 140428214284287, +SNULL, 140428212170752, 140428212183039, +STORE, 140428212183040, 140428214284287, +STORE, 140428212170752, 140428212183039, +SNULL, 140428214276095, 140428214284287, +STORE, 140428212183040, 140428214276095, +STORE, 140428214276096, 140428214284287, +ERASE, 140428214276096, 140428214284287, +STORE, 140428214276096, 140428214284287, +STORE, 140428209991680, 140428212170751, +SNULL, 140428209991680, 140428210069503, +STORE, 140428210069504, 140428212170751, +STORE, 140428209991680, 140428210069503, +SNULL, 140428212162559, 140428212170751, +STORE, 140428210069504, 140428212162559, +STORE, 140428212162560, 140428212170751, +ERASE, 140428212162560, 140428212170751, +STORE, 140428212162560, 140428212170751, +STORE, 140428207874048, 140428209991679, +SNULL, 140428207874048, 140428207890431, +STORE, 140428207890432, 140428209991679, +STORE, 140428207874048, 140428207890431, +SNULL, 140428209983487, 140428209991679, +STORE, 140428207890432, 140428209983487, +STORE, 140428209983488, 140428209991679, +ERASE, 140428209983488, 140428209991679, +STORE, 140428209983488, 140428209991679, +STORE, 140428248244224, 140428248268799, +STORE, 140428248231936, 140428248268799, +SNULL, 140428241600511, 140428241608703, +STORE, 140428241584128, 140428241600511, +STORE, 140428241600512, 140428241608703, +SNULL, 140428209987583, 140428209991679, +STORE, 140428209983488, 140428209987583, +STORE, 140428209987584, 140428209991679, +SNULL, 140428212166655, 140428212170751, +STORE, 140428212162560, 140428212166655, +STORE, 140428212166656, 140428212170751, +SNULL, 140428214280191, 140428214284287, +STORE, 140428214276096, 140428214280191, +STORE, 140428214280192, 140428214284287, +SNULL, 140428243820543, 140428243824639, +STORE, 140428243816448, 140428243820543, +STORE, 140428243820544, 140428243824639, +SNULL, 140428216848383, 140428216852479, +STORE, 140428216844288, 140428216848383, +STORE, 140428216848384, 140428216852479, +SNULL, 140428219232255, 140428219236351, +STORE, 140428219138048, 140428219232255, +STORE, 140428219232256, 140428219236351, +SNULL, 140428221542399, 140428221546495, +STORE, 140428221538304, 140428221542399, +STORE, 140428221542400, 140428221546495, +SNULL, 140428223676415, 140428223680511, +STORE, 140428223672320, 140428223676415, +STORE, 140428223676416, 140428223680511, +SNULL, 140428226863103, 140428226891775, +STORE, 140428226854912, 140428226863103, +STORE, 140428226863104, 140428226891775, +SNULL, 140428229058559, 140428229062655, +STORE, 140428229054464, 140428229058559, +STORE, 140428229058560, 140428229062655, +SNULL, 140428231311359, 140428231315455, +STORE, 140428231307264, 140428231311359, +STORE, 140428231311360, 140428231315455, +SNULL, 140428233441279, 140428233445375, +STORE, 140428233437184, 140428233441279, +STORE, 140428233441280, 140428233445375, +SNULL, 140428235563007, 140428235567103, +STORE, 140428235558912, 140428235563007, +STORE, 140428235563008, 140428235567103, +SNULL, 140428237815807, 140428237819903, +STORE, 140428237811712, 140428237815807, +STORE, 140428237815808, 140428237819903, +SNULL, 140428246056959, 140428246061055, +STORE, 140428245491712, 140428246056959, +STORE, 140428246056960, 140428246061055, +SNULL, 94717834924031, 94717834928127, +STORE, 94717834919936, 94717834924031, +STORE, 94717834924032, 94717834928127, +SNULL, 140428248309759, 140428248313855, +STORE, 140428248305664, 140428248309759, +STORE, 140428248309760, 140428248313855, +ERASE, 140428248268800, 140428248297471, +STORE, 94717843058688, 94717843193855, +STORE, 94749677137920, 94749677559807, +STORE, 94749677563904, 94749677604863, +STORE, 94749677604864, 94749677608959, +STORE, 94749710970880, 94749711241215, +STORE, 140490884894720, 140490884935679, +STORE, 140490884935680, 140490887032831, +STORE, 140490887032832, 140490887036927, +STORE, 140490887036928, 140490887041023, +STORE, 140490887041024, 140490887065599, +STORE, 140490887065600, 140490887110655, +STORE, 140490887110656, 140490889203711, +STORE, 140490889203712, 140490889207807, +STORE, 140490889207808, 140490889211903, +STORE, 140490889211904, 140490889293823, +STORE, 140490889293824, 140490891390975, +STORE, 140490891390976, 140490891395071, +STORE, 140490891395072, 140490891399167, +STORE, 140490891399168, 140490891407359, +STORE, 140490891407360, 140490891436031, +STORE, 140490891436032, 140490893529087, +STORE, 140490893529088, 140490893533183, +STORE, 140490893533184, 140490893537279, +STORE, 140490893537280, 140490901979135, +STORE, 140490901979136, 140490901991423, +STORE, 140490901991424, 140490904084479, +STORE, 140490904084480, 140490904088575, +STORE, 140490904088576, 140490904092671, +STORE, 140490904092672, 140490904559615, +STORE, 140490904559616, 140490906652671, +STORE, 140490906652672, 140490906656767, +STORE, 140490906656768, 140490906660863, +STORE, 140490906660864, 140490906677247, +STORE, 140490906677248, 140490908770303, +STORE, 140490908770304, 140490908774399, +STORE, 140490908774400, 140490908778495, +STORE, 140490908778496, 140490908794879, +STORE, 140490908794880, 140490910887935, +STORE, 140490910887936, 140490910892031, +STORE, 140490910892032, 140490910896127, +STORE, 140490910896128, 140490912555007, +STORE, 140490912555008, 140490914652159, +STORE, 140490914652160, 140490914668543, +STORE, 140490914668544, 140490914676735, +STORE, 140490914676736, 140490914693119, +STORE, 140490914693120, 140490914791423, +STORE, 140490914791424, 140490916884479, +STORE, 140490916884480, 140490916888575, +STORE, 140490916888576, 140490916892671, +STORE, 140490916892672, 140490916909055, +STORE, 140490916909056, 140490916937727, +STORE, 140490916937728, 140490919030783, +STORE, 140490919030784, 140490919034879, +STORE, 140490919034880, 140490919038975, +STORE, 140490919038976, 140490919190527, +STORE, 140490919190528, 140490921283583, +STORE, 140490921283584, 140490921287679, +STORE, 140490921287680, 140490921291775, +STORE, 140490921291776, 140490921299967, +STORE, 140490921299968, 140490921390079, +STORE, 140490921390080, 140490923483135, +STORE, 140490923483136, 140490923487231, +STORE, 140490923487232, 140490923491327, +STORE, 140490923491328, 140490923757567, +STORE, 140490923757568, 140490925850623, +STORE, 140490925850624, 140490925867007, +STORE, 140490925867008, 140490925871103, +STORE, 140490925871104, 140490925875199, +STORE, 140490925875200, 140490925903871, +STORE, 140490925903872, 140490928001023, +STORE, 140490928001024, 140490928005119, +STORE, 140490928005120, 140490928009215, +STORE, 140490928009216, 140490928152575, +STORE, 140490930184192, 140490930221055, +STORE, 140490930221056, 140490930237439, +STORE, 140490930237440, 140490930241535, +STORE, 140490930241536, 140490930245631, +STORE, 140490930245632, 140490930249727, +STORE, 140490930249728, 140490930253823, +STORE, 140490930253824, 140490930257919, +STORE, 140490930257920, 140490930262015, +STORE, 140724611694592, 140724611829759, +STORE, 140724612427776, 140724612440063, +STORE, 140724612440064, 140724612444159, +STORE, 94103163662336, 94103163772927, +STORE, 94103165865984, 94103165874175, +STORE, 94103165874176, 94103165878271, +STORE, 94103165878272, 94103165886463, +STORE, 94103182548992, 94103182684159, +STORE, 140092694708224, 140092696367103, +STORE, 140092696367104, 140092698464255, +STORE, 140092698464256, 140092698480639, +STORE, 140092698480640, 140092698488831, +STORE, 140092698488832, 140092698505215, +STORE, 140092698505216, 140092698648575, +STORE, 140092700708864, 140092700717055, +STORE, 140092700745728, 140092700749823, +STORE, 140092700749824, 140092700753919, +STORE, 140092700753920, 140092700758015, +STORE, 140736800911360, 140736801046527, +STORE, 140736802308096, 140736802320383, +STORE, 140736802320384, 140736802324479, +STORE, 93948802064384, 93948802174975, +STORE, 93948804268032, 93948804276223, +STORE, 93948804276224, 93948804280319, +STORE, 93948804280320, 93948804288511, +STORE, 93948806266880, 93948806402047, +STORE, 140222999113728, 140223000772607, +STORE, 140223000772608, 140223002869759, +STORE, 140223002869760, 140223002886143, +STORE, 140223002886144, 140223002894335, +STORE, 140223002894336, 140223002910719, +STORE, 140223002910720, 140223003054079, +STORE, 140223005114368, 140223005122559, +STORE, 140223005151232, 140223005155327, +STORE, 140223005155328, 140223005159423, +STORE, 140223005159424, 140223005163519, +STORE, 140720877506560, 140720877641727, +STORE, 140720878231552, 140720878243839, +STORE, 140720878243840, 140720878247935, +STORE, 140737488347136, 140737488351231, +STORE, 140733232087040, 140737488351231, +SNULL, 140733232091135, 140737488351231, +STORE, 140733232087040, 140733232091135, +STORE, 140733231955968, 140733232091135, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140161681321984, 140161683574783, +SNULL, 140161681465343, 140161683574783, +STORE, 140161681321984, 140161681465343, +STORE, 140161681465344, 140161683574783, +ERASE, 140161681465344, 140161683574783, +STORE, 140161683562496, 140161683570687, +STORE, 140161683570688, 140161683574783, +STORE, 140733232214016, 140733232218111, +STORE, 140733232201728, 140733232214015, +STORE, 140161683533824, 140161683562495, +STORE, 140161683525632, 140161683533823, +STORE, 140161678159872, 140161681321983, +SNULL, 140161678159872, 140161679220735, +STORE, 140161679220736, 140161681321983, +STORE, 140161678159872, 140161679220735, +SNULL, 140161681313791, 140161681321983, +STORE, 140161679220736, 140161681313791, +STORE, 140161681313792, 140161681321983, +ERASE, 140161681313792, 140161681321983, +STORE, 140161681313792, 140161681321983, +STORE, 140161674362880, 140161678159871, +SNULL, 140161674362880, 140161676021759, +STORE, 140161676021760, 140161678159871, +STORE, 140161674362880, 140161676021759, +SNULL, 140161678118911, 140161678159871, +STORE, 140161676021760, 140161678118911, +STORE, 140161678118912, 140161678159871, +SNULL, 140161678118912, 140161678143487, +STORE, 140161678143488, 140161678159871, +STORE, 140161678118912, 140161678143487, +ERASE, 140161678118912, 140161678143487, +STORE, 140161678118912, 140161678143487, +ERASE, 140161678143488, 140161678159871, +STORE, 140161678143488, 140161678159871, +STORE, 140161683513344, 140161683533823, +SNULL, 140161678135295, 140161678143487, +STORE, 140161678118912, 140161678135295, +STORE, 140161678135296, 140161678143487, +SNULL, 140161681317887, 140161681321983, +STORE, 140161681313792, 140161681317887, +STORE, 140161681317888, 140161681321983, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140161683566591, 140161683570687, +STORE, 140161683562496, 140161683566591, +STORE, 140161683566592, 140161683570687, +ERASE, 140161683533824, 140161683562495, +STORE, 25477120, 25612287, +STORE, 25477120, 25759743, +STORE, 140161681829888, 140161683513343, +STORE, 25477120, 25915391, +STORE, 25477120, 26054655, +SNULL, 25800703, 26054655, +STORE, 25477120, 25800703, +STORE, 25800704, 26054655, +ERASE, 25800704, 26054655, +STORE, 140737488347136, 140737488351231, +STORE, 140723218452480, 140737488351231, +SNULL, 140723218456575, 140737488351231, +STORE, 140723218452480, 140723218456575, +STORE, 140723218321408, 140723218456575, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 140398872264704, 140398874517503, +SNULL, 140398872408063, 140398874517503, +STORE, 140398872264704, 140398872408063, +STORE, 140398872408064, 140398874517503, +ERASE, 140398872408064, 140398874517503, +STORE, 140398874505216, 140398874513407, +STORE, 140398874513408, 140398874517503, +STORE, 140723219247104, 140723219251199, +STORE, 140723219234816, 140723219247103, +STORE, 140398874476544, 140398874505215, +STORE, 140398874468352, 140398874476543, +STORE, 140398868430848, 140398872264703, +SNULL, 140398868430848, 140398870138879, +STORE, 140398870138880, 140398872264703, +STORE, 140398868430848, 140398870138879, +SNULL, 140398872231935, 140398872264703, +STORE, 140398870138880, 140398872231935, +STORE, 140398872231936, 140398872264703, +ERASE, 140398872231936, 140398872264703, +STORE, 140398872231936, 140398872264703, +STORE, 140398866235392, 140398868430847, +SNULL, 140398866235392, 140398866329599, +STORE, 140398866329600, 140398868430847, +STORE, 140398866235392, 140398866329599, +SNULL, 140398868422655, 140398868430847, +STORE, 140398866329600, 140398868422655, +STORE, 140398868422656, 140398868430847, +ERASE, 140398868422656, 140398868430847, +STORE, 140398868422656, 140398868430847, +STORE, 140398863716352, 140398866235391, +SNULL, 140398863716352, 140398864130047, +STORE, 140398864130048, 140398866235391, +STORE, 140398863716352, 140398864130047, +SNULL, 140398866223103, 140398866235391, +STORE, 140398864130048, 140398866223103, +STORE, 140398866223104, 140398866235391, +ERASE, 140398866223104, 140398866235391, +STORE, 140398866223104, 140398866235391, +STORE, 140398861082624, 140398863716351, +SNULL, 140398861082624, 140398861611007, +STORE, 140398861611008, 140398863716351, +STORE, 140398861082624, 140398861611007, +SNULL, 140398863708159, 140398863716351, +STORE, 140398861611008, 140398863708159, +STORE, 140398863708160, 140398863716351, +ERASE, 140398863708160, 140398863716351, +STORE, 140398863708160, 140398863716351, +STORE, 140398858969088, 140398861082623, +SNULL, 140398858969088, 140398858981375, +STORE, 140398858981376, 140398861082623, +STORE, 140398858969088, 140398858981375, +SNULL, 140398861074431, 140398861082623, +STORE, 140398858981376, 140398861074431, +STORE, 140398861074432, 140398861082623, +ERASE, 140398861074432, 140398861082623, +STORE, 140398861074432, 140398861082623, +STORE, 140398856765440, 140398858969087, +SNULL, 140398856765440, 140398856867839, +STORE, 140398856867840, 140398858969087, +STORE, 140398856765440, 140398856867839, +SNULL, 140398858960895, 140398858969087, +STORE, 140398856867840, 140398858960895, +STORE, 140398858960896, 140398858969087, +ERASE, 140398858960896, 140398858969087, +STORE, 140398858960896, 140398858969087, +STORE, 140398874460160, 140398874476543, +STORE, 140398853603328, 140398856765439, +SNULL, 140398853603328, 140398854664191, +STORE, 140398854664192, 140398856765439, +STORE, 140398853603328, 140398854664191, +SNULL, 140398856757247, 140398856765439, +STORE, 140398854664192, 140398856757247, +STORE, 140398856757248, 140398856765439, +ERASE, 140398856757248, 140398856765439, +STORE, 140398856757248, 140398856765439, +STORE, 140398849806336, 140398853603327, +SNULL, 140398849806336, 140398851465215, +STORE, 140398851465216, 140398853603327, +STORE, 140398849806336, 140398851465215, +SNULL, 140398853562367, 140398853603327, +STORE, 140398851465216, 140398853562367, +STORE, 140398853562368, 140398853603327, +SNULL, 140398853562368, 140398853586943, +STORE, 140398853586944, 140398853603327, +STORE, 140398853562368, 140398853586943, +ERASE, 140398853562368, 140398853586943, +STORE, 140398853562368, 140398853586943, +ERASE, 140398853586944, 140398853603327, +STORE, 140398853586944, 140398853603327, +STORE, 140398874447872, 140398874476543, +SNULL, 140398853578751, 140398853586943, +STORE, 140398853562368, 140398853578751, +STORE, 140398853578752, 140398853586943, +SNULL, 140398856761343, 140398856765439, +STORE, 140398856757248, 140398856761343, +STORE, 140398856761344, 140398856765439, +SNULL, 140398858964991, 140398858969087, +STORE, 140398858960896, 140398858964991, +STORE, 140398858964992, 140398858969087, +SNULL, 140398861078527, 140398861082623, +STORE, 140398861074432, 140398861078527, +STORE, 140398861078528, 140398861082623, +SNULL, 140398863712255, 140398863716351, +STORE, 140398863708160, 140398863712255, +STORE, 140398863712256, 140398863716351, +SNULL, 140398866231295, 140398866235391, +STORE, 140398866223104, 140398866231295, +STORE, 140398866231296, 140398866235391, +SNULL, 140398868426751, 140398868430847, +STORE, 140398868422656, 140398868426751, +STORE, 140398868426752, 140398868430847, +SNULL, 140398872236031, 140398872264703, +STORE, 140398872231936, 140398872236031, +STORE, 140398872236032, 140398872264703, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 140398874509311, 140398874513407, +STORE, 140398874505216, 140398874509311, +STORE, 140398874509312, 140398874513407, +ERASE, 140398874476544, 140398874505215, +STORE, 43278336, 43413503, +STORE, 140398872764416, 140398874447871, +STORE, 140398874501120, 140398874505215, +STORE, 140398872629248, 140398872764415, +STORE, 43278336, 43556863, +STORE, 140398847709184, 140398849806335, +STORE, 140398874492928, 140398874505215, +STORE, 140398874484736, 140398874505215, +STORE, 140398874447872, 140398874484735, +STORE, 140398872612864, 140398872764415, +STORE, 43278336, 43692031, +STORE, 43278336, 43880447, +STORE, 140398872604672, 140398872764415, +STORE, 140398872596480, 140398872764415, +STORE, 43278336, 44044287, +STORE, 140398872580096, 140398872764415, +STORE, 140737488347136, 140737488351231, +STORE, 140734403092480, 140737488351231, +SNULL, 140734403096575, 140737488351231, +STORE, 140734403092480, 140734403096575, +STORE, 140734402961408, 140734403096575, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140240662380544, 140240664633343, +SNULL, 140240662523903, 140240664633343, +STORE, 140240662380544, 140240662523903, +STORE, 140240662523904, 140240664633343, +ERASE, 140240662523904, 140240664633343, +STORE, 140240664621056, 140240664629247, +STORE, 140240664629248, 140240664633343, +STORE, 140734403145728, 140734403149823, +STORE, 140734403133440, 140734403145727, +STORE, 140240664592384, 140240664621055, +STORE, 140240664584192, 140240664592383, +STORE, 140240659218432, 140240662380543, +SNULL, 140240659218432, 140240660279295, +STORE, 140240660279296, 140240662380543, +STORE, 140240659218432, 140240660279295, +SNULL, 140240662372351, 140240662380543, +STORE, 140240660279296, 140240662372351, +STORE, 140240662372352, 140240662380543, +ERASE, 140240662372352, 140240662380543, +STORE, 140240662372352, 140240662380543, +STORE, 140240655421440, 140240659218431, +SNULL, 140240655421440, 140240657080319, +STORE, 140240657080320, 140240659218431, +STORE, 140240655421440, 140240657080319, +SNULL, 140240659177471, 140240659218431, +STORE, 140240657080320, 140240659177471, +STORE, 140240659177472, 140240659218431, +SNULL, 140240659177472, 140240659202047, +STORE, 140240659202048, 140240659218431, +STORE, 140240659177472, 140240659202047, +ERASE, 140240659177472, 140240659202047, +STORE, 140240659177472, 140240659202047, +ERASE, 140240659202048, 140240659218431, +STORE, 140240659202048, 140240659218431, +STORE, 140240664571904, 140240664592383, +SNULL, 140240659193855, 140240659202047, +STORE, 140240659177472, 140240659193855, +STORE, 140240659193856, 140240659202047, +SNULL, 140240662376447, 140240662380543, +STORE, 140240662372352, 140240662376447, +STORE, 140240662376448, 140240662380543, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140240664625151, 140240664629247, +STORE, 140240664621056, 140240664625151, +STORE, 140240664625152, 140240664629247, +ERASE, 140240664592384, 140240664621055, +STORE, 30646272, 30781439, +STORE, 30646272, 30928895, +STORE, 140240662888448, 140240664571903, +STORE, 94256659468288, 94256659578879, +STORE, 94256661671936, 94256661680127, +STORE, 94256661680128, 94256661684223, +STORE, 94256661684224, 94256661692415, +STORE, 94256687980544, 94256688115711, +STORE, 139801712504832, 139801714163711, +STORE, 139801714163712, 139801716260863, +STORE, 139801716260864, 139801716277247, +STORE, 139801716277248, 139801716285439, +STORE, 139801716285440, 139801716301823, +STORE, 139801716301824, 139801716445183, +STORE, 139801718505472, 139801718513663, +STORE, 139801718542336, 139801718546431, +STORE, 139801718546432, 139801718550527, +STORE, 139801718550528, 139801718554623, +STORE, 140721575538688, 140721575673855, +STORE, 140721577013248, 140721577025535, +STORE, 140721577025536, 140721577029631, +STORE, 140737488347136, 140737488351231, +STORE, 140729259393024, 140737488351231, +SNULL, 140729259397119, 140737488351231, +STORE, 140729259393024, 140729259397119, +STORE, 140729259261952, 140729259397119, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 139682376638464, 139682378891263, +SNULL, 139682376781823, 139682378891263, +STORE, 139682376638464, 139682376781823, +STORE, 139682376781824, 139682378891263, +ERASE, 139682376781824, 139682378891263, +STORE, 139682378878976, 139682378887167, +STORE, 139682378887168, 139682378891263, +STORE, 140729260462080, 140729260466175, +STORE, 140729260449792, 140729260462079, +STORE, 139682378850304, 139682378878975, +STORE, 139682378842112, 139682378850303, +STORE, 139682373476352, 139682376638463, +SNULL, 139682373476352, 139682374537215, +STORE, 139682374537216, 139682376638463, +STORE, 139682373476352, 139682374537215, +SNULL, 139682376630271, 139682376638463, +STORE, 139682374537216, 139682376630271, +STORE, 139682376630272, 139682376638463, +ERASE, 139682376630272, 139682376638463, +STORE, 139682376630272, 139682376638463, +STORE, 139682369679360, 139682373476351, +SNULL, 139682369679360, 139682371338239, +STORE, 139682371338240, 139682373476351, +STORE, 139682369679360, 139682371338239, +SNULL, 139682373435391, 139682373476351, +STORE, 139682371338240, 139682373435391, +STORE, 139682373435392, 139682373476351, +SNULL, 139682373435392, 139682373459967, +STORE, 139682373459968, 139682373476351, +STORE, 139682373435392, 139682373459967, +ERASE, 139682373435392, 139682373459967, +STORE, 139682373435392, 139682373459967, +ERASE, 139682373459968, 139682373476351, +STORE, 139682373459968, 139682373476351, +STORE, 139682378829824, 139682378850303, +SNULL, 139682373451775, 139682373459967, +STORE, 139682373435392, 139682373451775, +STORE, 139682373451776, 139682373459967, +SNULL, 139682376634367, 139682376638463, +STORE, 139682376630272, 139682376634367, +STORE, 139682376634368, 139682376638463, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 139682378883071, 139682378887167, +STORE, 139682378878976, 139682378883071, +STORE, 139682378883072, 139682378887167, +ERASE, 139682378850304, 139682378878975, +STORE, 10022912, 10158079, +STORE, 10022912, 10305535, +STORE, 139682377146368, 139682378829823, +STORE, 140737488347136, 140737488351231, +STORE, 140731831926784, 140737488351231, +SNULL, 140731831930879, 140737488351231, +STORE, 140731831926784, 140731831930879, +STORE, 140731831795712, 140731831930879, +STORE, 94615305261056, 94615307485183, +SNULL, 94615305371647, 94615307485183, +STORE, 94615305261056, 94615305371647, +STORE, 94615305371648, 94615307485183, +ERASE, 94615305371648, 94615307485183, +STORE, 94615307464704, 94615307476991, +STORE, 94615307476992, 94615307485183, +STORE, 140163912994816, 140163915247615, +SNULL, 140163913138175, 140163915247615, +STORE, 140163912994816, 140163913138175, +STORE, 140163913138176, 140163915247615, +ERASE, 140163913138176, 140163915247615, +STORE, 140163915235328, 140163915243519, +STORE, 140163915243520, 140163915247615, +STORE, 140731832217600, 140731832221695, +STORE, 140731832205312, 140731832217599, +STORE, 140163915206656, 140163915235327, +STORE, 140163915198464, 140163915206655, +STORE, 140163909197824, 140163912994815, +SNULL, 140163909197824, 140163910856703, +STORE, 140163910856704, 140163912994815, +STORE, 140163909197824, 140163910856703, +SNULL, 140163912953855, 140163912994815, +STORE, 140163910856704, 140163912953855, +STORE, 140163912953856, 140163912994815, +SNULL, 140163912953856, 140163912978431, +STORE, 140163912978432, 140163912994815, +STORE, 140163912953856, 140163912978431, +ERASE, 140163912953856, 140163912978431, +STORE, 140163912953856, 140163912978431, +ERASE, 140163912978432, 140163912994815, +STORE, 140163912978432, 140163912994815, +SNULL, 140163912970239, 140163912978431, +STORE, 140163912953856, 140163912970239, +STORE, 140163912970240, 140163912978431, +SNULL, 94615307472895, 94615307476991, +STORE, 94615307464704, 94615307472895, +STORE, 94615307472896, 94615307476991, +SNULL, 140163915239423, 140163915243519, +STORE, 140163915235328, 140163915239423, +STORE, 140163915239424, 140163915243519, +ERASE, 140163915206656, 140163915235327, +STORE, 94615330672640, 94615330807807, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725254479872, 140737488351231, +SNULL, 140725254488063, 140737488351231, +STORE, 140725254479872, 140725254488063, +STORE, 140725254348800, 140725254488063, +STORE, 94572781277184, 94572785741823, +SNULL, 94572783312895, 94572785741823, +STORE, 94572781277184, 94572783312895, +STORE, 94572783312896, 94572785741823, +ERASE, 94572783312896, 94572785741823, +STORE, 94572785405952, 94572785455103, +STORE, 94572785455104, 94572785741823, +STORE, 139636001341440, 139636003594239, +SNULL, 139636001484799, 139636003594239, +STORE, 139636001341440, 139636001484799, +STORE, 139636001484800, 139636003594239, +ERASE, 139636001484800, 139636003594239, +STORE, 139636003581952, 139636003590143, +STORE, 139636003590144, 139636003594239, +STORE, 140725255557120, 140725255561215, +STORE, 140725255544832, 140725255557119, +STORE, 139636003553280, 139636003581951, +STORE, 139636003545088, 139636003553279, +STORE, 139635998773248, 139636001341439, +SNULL, 139635998773248, 139635999240191, +STORE, 139635999240192, 139636001341439, +STORE, 139635998773248, 139635999240191, +SNULL, 139636001333247, 139636001341439, +STORE, 139635999240192, 139636001333247, +STORE, 139636001333248, 139636001341439, +ERASE, 139636001333248, 139636001341439, +STORE, 139636001333248, 139636001341439, +STORE, 139635996569600, 139635998773247, +SNULL, 139635996569600, 139635996671999, +STORE, 139635996672000, 139635998773247, +STORE, 139635996569600, 139635996671999, +SNULL, 139635998765055, 139635998773247, +STORE, 139635996672000, 139635998765055, +STORE, 139635998765056, 139635998773247, +ERASE, 139635998765056, 139635998773247, +STORE, 139635998765056, 139635998773247, +STORE, 139635994353664, 139635996569599, +SNULL, 139635994353664, 139635994451967, +STORE, 139635994451968, 139635996569599, +STORE, 139635994353664, 139635994451967, +SNULL, 139635996545023, 139635996569599, +STORE, 139635994451968, 139635996545023, +STORE, 139635996545024, 139635996569599, +SNULL, 139635996545024, 139635996553215, +STORE, 139635996553216, 139635996569599, +STORE, 139635996545024, 139635996553215, +ERASE, 139635996545024, 139635996553215, +STORE, 139635996545024, 139635996553215, +ERASE, 139635996553216, 139635996569599, +STORE, 139635996553216, 139635996569599, +STORE, 139635992223744, 139635994353663, +SNULL, 139635992223744, 139635992252415, +STORE, 139635992252416, 139635994353663, +STORE, 139635992223744, 139635992252415, +SNULL, 139635994345471, 139635994353663, +STORE, 139635992252416, 139635994345471, +STORE, 139635994345472, 139635994353663, +ERASE, 139635994345472, 139635994353663, +STORE, 139635994345472, 139635994353663, +STORE, 139635988426752, 139635992223743, +SNULL, 139635988426752, 139635990085631, +STORE, 139635990085632, 139635992223743, +STORE, 139635988426752, 139635990085631, +SNULL, 139635992182783, 139635992223743, +STORE, 139635990085632, 139635992182783, +STORE, 139635992182784, 139635992223743, +SNULL, 139635992182784, 139635992207359, +STORE, 139635992207360, 139635992223743, +STORE, 139635992182784, 139635992207359, +ERASE, 139635992182784, 139635992207359, +STORE, 139635992182784, 139635992207359, +ERASE, 139635992207360, 139635992223743, +STORE, 139635992207360, 139635992223743, +STORE, 139636003536896, 139636003553279, +SNULL, 139635992199167, 139635992207359, +STORE, 139635992182784, 139635992199167, +STORE, 139635992199168, 139635992207359, +SNULL, 139635996549119, 139635996553215, +STORE, 139635996545024, 139635996549119, +STORE, 139635996549120, 139635996553215, +SNULL, 139635994349567, 139635994353663, +STORE, 139635994345472, 139635994349567, +STORE, 139635994349568, 139635994353663, +SNULL, 139635998769151, 139635998773247, +STORE, 139635998765056, 139635998769151, +STORE, 139635998769152, 139635998773247, +SNULL, 139636001337343, 139636001341439, +STORE, 139636001333248, 139636001337343, +STORE, 139636001337344, 139636001341439, +SNULL, 94572785418239, 94572785455103, +STORE, 94572785405952, 94572785418239, +STORE, 94572785418240, 94572785455103, +SNULL, 139636003586047, 139636003590143, +STORE, 139636003581952, 139636003586047, +STORE, 139636003586048, 139636003590143, +ERASE, 139636003553280, 139636003581951, +STORE, 94572798435328, 94572798570495, +STORE, 139636001853440, 139636003536895, +STORE, 139635981426688, 139635988426751, +STORE, 139635980615680, 139635981426687, +STORE, 94572798435328, 94572798705663, +STORE, 94572798435328, 94572798840831, +STORE, 94572798435328, 94572798975999, +STORE, 94572798435328, 94572799111167, +STORE, 94572798435328, 94572799246335, +STORE, 94572798435328, 94572799381503, +STORE, 94572798435328, 94572799516671, +STORE, 94572798435328, 94572799651839, +STORE, 94572798435328, 94572799787007, +STORE, 94572798435328, 94572799922175, +STORE, 94572798435328, 94572800057343, +STORE, 94572798435328, 94572800192511, +STORE, 94572798435328, 94572800327679, +STORE, 94572798435328, 94572800462847, +STORE, 94572798435328, 94572800598015, +STORE, 94572798435328, 94572800733183, +STORE, 94572798435328, 94572800868351, +STORE, 94572798435328, 94572801003519, +STORE, 94572798435328, 94572801138687, +STORE, 94572798435328, 94572801273855, +STORE, 94572798435328, 94572801409023, +STORE, 94572798435328, 94572801544191, +STORE, 94572798435328, 94572801679359, +STORE, 94572798435328, 94572801814527, +STORE, 94572798435328, 94572801949695, +STORE, 94572798435328, 94572802084863, +STORE, 94572798435328, 94572802220031, +STORE, 94572798435328, 94572802355199, +STORE, 94572798435328, 94572802490367, +STORE, 94572798435328, 94572802625535, +STORE, 94572798435328, 94572802760703, +STORE, 94572798435328, 94572802895871, +STORE, 94572798435328, 94572803031039, +STORE, 94572798435328, 94572803166207, +STORE, 94572798435328, 94572803301375, +STORE, 94572798435328, 94572803436543, +STORE, 94572798435328, 94572803571711, +STORE, 94572798435328, 94572803706879, +STORE, 94572798435328, 94572803842047, +STORE, 94572798435328, 94572803977215, +STORE, 94572798435328, 94572804112383, +STORE, 94572798435328, 94572804247551, +STORE, 94572798435328, 94572804382719, +STORE, 94572798435328, 94572804517887, +STORE, 94572798435328, 94572804653055, +STORE, 94572798435328, 94572804788223, +STORE, 94572798435328, 94572804923391, +STORE, 94572798435328, 94572805058559, +STORE, 94572798435328, 94572805193727, +STORE, 94572798435328, 94572805328895, +STORE, 94572798435328, 94572805464063, +STORE, 94572798435328, 94572805599231, +STORE, 94572798435328, 94572805734399, +STORE, 94572798435328, 94572805869567, +STORE, 94572798435328, 94572806004735, +STORE, 94572798435328, 94572806139903, +STORE, 94572798435328, 94572806275071, +STORE, 94572798435328, 94572806410239, +STORE, 94572798435328, 94572806545407, +STORE, 94572798435328, 94572806680575, +STORE, 94572798435328, 94572806815743, +STORE, 94572798435328, 94572806950911, +STORE, 94572798435328, 94572807086079, +STORE, 94572798435328, 94572807221247, +STORE, 94572798435328, 94572807356415, +STORE, 94572798435328, 94572807491583, +STORE, 94572798435328, 94572807626751, +STORE, 94572798435328, 94572807761919, +STORE, 94572798435328, 94572807897087, +STORE, 94572798435328, 94572808032255, +STORE, 94572798435328, 94572808167423, +STORE, 94572798435328, 94572808302591, +STORE, 94572798435328, 94572808437759, +STORE, 94572798435328, 94572808572927, +ERASE, 139635981426688, 139635988426751, +STORE, 139635985088512, 139635988426751, +STORE, 139635778273280, 139635980615679, +STORE, 139635567632384, 139635778273279, +STORE, 94572798435328, 94572808716287, +STORE, 139635984564224, 139635985088511, +STORE, 139635559239680, 139635567632383, +SNULL, 139635559243775, 139635567632383, +STORE, 139635559239680, 139635559243775, +STORE, 139635559243776, 139635567632383, +STORE, 139635550846976, 139635559239679, +SNULL, 139635550851071, 139635559239679, +STORE, 139635550846976, 139635550851071, +STORE, 139635550851072, 139635559239679, +STORE, 139635542454272, 139635550846975, +STORE, 139635408236544, 139635542454271, +SNULL, 139635408236544, 139635426590719, +STORE, 139635426590720, 139635542454271, +STORE, 139635408236544, 139635426590719, +ERASE, 139635408236544, 139635426590719, +STORE, 139635292372992, 139635542454271, +SNULL, 139635359481855, 139635542454271, +STORE, 139635292372992, 139635359481855, +STORE, 139635359481856, 139635542454271, +SNULL, 139635359481856, 139635426590719, +STORE, 139635426590720, 139635542454271, +STORE, 139635359481856, 139635426590719, +ERASE, 139635359481856, 139635426590719, +SNULL, 139635542458367, 139635550846975, +STORE, 139635542454272, 139635542458367, +STORE, 139635542458368, 139635550846975, +STORE, 139635418198016, 139635426590719, +SNULL, 139635493699583, 139635542454271, +STORE, 139635426590720, 139635493699583, +STORE, 139635493699584, 139635542454271, +ERASE, 139635493699584, 139635542454271, +SNULL, 139635426725887, 139635493699583, +STORE, 139635426590720, 139635426725887, +STORE, 139635426725888, 139635493699583, +SNULL, 139635292508159, 139635359481855, +STORE, 139635292372992, 139635292508159, +STORE, 139635292508160, 139635359481855, +SNULL, 139635418202111, 139635426590719, +STORE, 139635418198016, 139635418202111, +STORE, 139635418202112, 139635426590719, +STORE, 139635225264128, 139635292372991, +STORE, 139635534061568, 139635542454271, +SNULL, 139635534065663, 139635542454271, +STORE, 139635534061568, 139635534065663, +STORE, 139635534065664, 139635542454271, +STORE, 139635525668864, 139635534061567, +SNULL, 139635525672959, 139635534061567, +STORE, 139635525668864, 139635525672959, +STORE, 139635525672960, 139635534061567, +SNULL, 139635225399295, 139635292372991, +STORE, 139635225264128, 139635225399295, +STORE, 139635225399296, 139635292372991, +STORE, 139635091046400, 139635225264127, +SNULL, 139635158155263, 139635225264127, +STORE, 139635091046400, 139635158155263, +STORE, 139635158155264, 139635225264127, +ERASE, 139635158155264, 139635225264127, +STORE, 139634956828672, 139635158155263, +STORE, 139635517276160, 139635525668863, +SNULL, 139635517280255, 139635525668863, +STORE, 139635517276160, 139635517280255, +STORE, 139635517280256, 139635525668863, +SNULL, 139634956828672, 139635091046399, +STORE, 139635091046400, 139635158155263, +STORE, 139634956828672, 139635091046399, +SNULL, 139635091181567, 139635158155263, +STORE, 139635091046400, 139635091181567, +STORE, 139635091181568, 139635158155263, +SNULL, 139635023937535, 139635091046399, +STORE, 139634956828672, 139635023937535, +STORE, 139635023937536, 139635091046399, +ERASE, 139635023937536, 139635091046399, +STORE, 139634956828672, 139635091046399, +SNULL, 139634956828672, 139635023937535, +STORE, 139635023937536, 139635091046399, +STORE, 139634956828672, 139635023937535, +SNULL, 139635024072703, 139635091046399, +STORE, 139635023937536, 139635024072703, +STORE, 139635024072704, 139635091046399, +STORE, 139635508883456, 139635517276159, +SNULL, 139635508887551, 139635517276159, +STORE, 139635508883456, 139635508887551, +STORE, 139635508887552, 139635517276159, +STORE, 139634822610944, 139635023937535, +SNULL, 139634822610944, 139634956828671, +STORE, 139634956828672, 139635023937535, +STORE, 139634822610944, 139634956828671, +SNULL, 139634956963839, 139635023937535, +STORE, 139634956828672, 139634956963839, +STORE, 139634956963840, 139635023937535, +STORE, 139635500490752, 139635508883455, +SNULL, 139634889719807, 139634956828671, +STORE, 139634822610944, 139634889719807, +STORE, 139634889719808, 139634956828671, +ERASE, 139634889719808, 139634956828671, +SNULL, 139635500494847, 139635508883455, +STORE, 139635500490752, 139635500494847, +STORE, 139635500494848, 139635508883455, +SNULL, 139634822746111, 139634889719807, +STORE, 139634822610944, 139634822746111, +STORE, 139634822746112, 139634889719807, +STORE, 139635409805312, 139635418198015, +STORE, 139634822746112, 139634956828671, +SNULL, 139634822746112, 139634889719807, +STORE, 139634889719808, 139634956828671, +STORE, 139634822746112, 139634889719807, +SNULL, 139634889854975, 139634956828671, +STORE, 139634889719808, 139634889854975, +STORE, 139634889854976, 139634956828671, +SNULL, 139635409809407, 139635418198015, +STORE, 139635409805312, 139635409809407, +STORE, 139635409809408, 139635418198015, +STORE, 139635401412608, 139635409805311, +STORE, 139634688393216, 139634822610943, +SNULL, 139634755502079, 139634822610943, +STORE, 139634688393216, 139634755502079, +STORE, 139634755502080, 139634822610943, +ERASE, 139634755502080, 139634822610943, +SNULL, 139635401416703, 139635409805311, +STORE, 139635401412608, 139635401416703, +STORE, 139635401416704, 139635409805311, +STORE, 139634554175488, 139634755502079, +SNULL, 139634554175488, 139634688393215, +STORE, 139634688393216, 139634755502079, +STORE, 139634554175488, 139634688393215, +SNULL, 139634688528383, 139634755502079, +STORE, 139634688393216, 139634688528383, +STORE, 139634688528384, 139634755502079, +STORE, 139635393019904, 139635401412607, +SNULL, 139634621284351, 139634688393215, +STORE, 139634554175488, 139634621284351, +STORE, 139634621284352, 139634688393215, +ERASE, 139634621284352, 139634688393215, +SNULL, 139634554310655, 139634621284351, +STORE, 139634554175488, 139634554310655, +STORE, 139634554310656, 139634621284351, +STORE, 139634554310656, 139634688393215, +SNULL, 139635393023999, 139635401412607, +STORE, 139635393019904, 139635393023999, +STORE, 139635393024000, 139635401412607, +SNULL, 139634554310656, 139634621284351, +STORE, 139634621284352, 139634688393215, +STORE, 139634554310656, 139634621284351, +SNULL, 139634621419519, 139634688393215, +STORE, 139634621284352, 139634621419519, +STORE, 139634621419520, 139634688393215, +STORE, 139635384627200, 139635393019903, +SNULL, 139635384631295, 139635393019903, +STORE, 139635384627200, 139635384631295, +STORE, 139635384631296, 139635393019903, +STORE, 139635376234496, 139635384627199, +SNULL, 139635376238591, 139635384627199, +STORE, 139635376234496, 139635376238591, +STORE, 139635376238592, 139635384627199, +STORE, 139635367841792, 139635376234495, +SNULL, 139635367845887, 139635376234495, +STORE, 139635367841792, 139635367845887, +STORE, 139635367845888, 139635376234495, +STORE, 139634419957760, 139634554175487, +SNULL, 139634487066623, 139634554175487, +STORE, 139634419957760, 139634487066623, +STORE, 139634487066624, 139634554175487, +ERASE, 139634487066624, 139634554175487, +STORE, 139635216871424, 139635225264127, +SNULL, 139635216875519, 139635225264127, +STORE, 139635216871424, 139635216875519, +STORE, 139635216875520, 139635225264127, +SNULL, 139634420092927, 139634487066623, +STORE, 139634419957760, 139634420092927, +STORE, 139634420092928, 139634487066623, +STORE, 139635208478720, 139635216871423, +SNULL, 139635208482815, 139635216871423, +STORE, 139635208478720, 139635208482815, +STORE, 139635208482816, 139635216871423, +STORE, 139635200086016, 139635208478719, +SNULL, 139635200090111, 139635208478719, +STORE, 139635200086016, 139635200090111, +STORE, 139635200090112, 139635208478719, +STORE, 139635191693312, 139635200086015, +SNULL, 139635191697407, 139635200086015, +STORE, 139635191693312, 139635191697407, +STORE, 139635191697408, 139635200086015, +STORE, 139635183300608, 139635191693311, +SNULL, 139635183304703, 139635191693311, +STORE, 139635183300608, 139635183304703, +STORE, 139635183304704, 139635191693311, +STORE, 139634420092928, 139634554175487, +SNULL, 139634420092928, 139634487066623, +STORE, 139634487066624, 139634554175487, +STORE, 139634420092928, 139634487066623, +SNULL, 139634487201791, 139634554175487, +STORE, 139634487066624, 139634487201791, +STORE, 139634487201792, 139634554175487, +ERASE, 139635559239680, 139635559243775, +ERASE, 139635559243776, 139635567632383, +ERASE, 139635550846976, 139635550851071, +ERASE, 139635550851072, 139635559239679, +ERASE, 139635542454272, 139635542458367, +ERASE, 139635542458368, 139635550846975, +ERASE, 139635418198016, 139635418202111, +ERASE, 139635418202112, 139635426590719, +ERASE, 139635534061568, 139635534065663, +ERASE, 139635534065664, 139635542454271, +ERASE, 139635525668864, 139635525672959, +ERASE, 139635525672960, 139635534061567, +ERASE, 139635517276160, 139635517280255, +ERASE, 139635517280256, 139635525668863, +ERASE, 139635508883456, 139635508887551, +ERASE, 139635508887552, 139635517276159, +ERASE, 139635500490752, 139635500494847, +ERASE, 139635500494848, 139635508883455, +ERASE, 139635409805312, 139635409809407, +ERASE, 139635409809408, 139635418198015, +ERASE, 139635401412608, 139635401416703, +ERASE, 139635401416704, 139635409805311, +ERASE, 139635393019904, 139635393023999, +ERASE, 139635393024000, 139635401412607, +ERASE, 139635384627200, 139635384631295, +ERASE, 139635384631296, 139635393019903, + }; + unsigned long set25[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722547441664, 140737488351231, +SNULL, 140722547449855, 140737488351231, +STORE, 140722547441664, 140722547449855, +STORE, 140722547310592, 140722547449855, +STORE, 94827521732608, 94827523956735, +SNULL, 94827521843199, 94827523956735, +STORE, 94827521732608, 94827521843199, +STORE, 94827521843200, 94827523956735, +ERASE, 94827521843200, 94827523956735, +STORE, 94827523936256, 94827523948543, +STORE, 94827523948544, 94827523956735, +STORE, 139816136847360, 139816139100159, +SNULL, 139816136990719, 139816139100159, +STORE, 139816136847360, 139816136990719, +STORE, 139816136990720, 139816139100159, +ERASE, 139816136990720, 139816139100159, +STORE, 139816139087872, 139816139096063, +STORE, 139816139096064, 139816139100159, +STORE, 140722548142080, 140722548146175, +STORE, 140722548129792, 140722548142079, +STORE, 139816139059200, 139816139087871, +STORE, 139816139051008, 139816139059199, +STORE, 139816133050368, 139816136847359, +SNULL, 139816133050368, 139816134709247, +STORE, 139816134709248, 139816136847359, +STORE, 139816133050368, 139816134709247, +SNULL, 139816136806399, 139816136847359, +STORE, 139816134709248, 139816136806399, +STORE, 139816136806400, 139816136847359, +SNULL, 139816136806400, 139816136830975, +STORE, 139816136830976, 139816136847359, +STORE, 139816136806400, 139816136830975, +ERASE, 139816136806400, 139816136830975, +STORE, 139816136806400, 139816136830975, +ERASE, 139816136830976, 139816136847359, +STORE, 139816136830976, 139816136847359, +SNULL, 139816136822783, 139816136830975, +STORE, 139816136806400, 139816136822783, +STORE, 139816136822784, 139816136830975, +SNULL, 94827523944447, 94827523948543, +STORE, 94827523936256, 94827523944447, +STORE, 94827523944448, 94827523948543, +SNULL, 139816139091967, 139816139096063, +STORE, 139816139087872, 139816139091967, +STORE, 139816139091968, 139816139096063, +ERASE, 139816139059200, 139816139087871, +STORE, 94827534970880, 94827535106047, +STORE, 94114394132480, 94114394345471, +STORE, 94114396442624, 94114396446719, +STORE, 94114396446720, 94114396454911, +STORE, 94114396454912, 94114396467199, +STORE, 94114421575680, 94114427715583, +STORE, 139934313955328, 139934315614207, +STORE, 139934315614208, 139934317711359, +STORE, 139934317711360, 139934317727743, +STORE, 139934317727744, 139934317735935, +STORE, 139934317735936, 139934317752319, +STORE, 139934317752320, 139934317764607, +STORE, 139934317764608, 139934319857663, +STORE, 139934319857664, 139934319861759, +STORE, 139934319861760, 139934319865855, +STORE, 139934319865856, 139934320009215, +STORE, 139934320377856, 139934322061311, +STORE, 139934322061312, 139934322077695, +STORE, 139934322106368, 139934322110463, +STORE, 139934322110464, 139934322114559, +STORE, 139934322114560, 139934322118655, +STORE, 140731200376832, 140731200516095, +STORE, 140731200929792, 140731200942079, +STORE, 140731200942080, 140731200946175, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140734133174272, 140737488351231, +SNULL, 140734133182463, 140737488351231, +STORE, 140734133174272, 140734133182463, +STORE, 140734133043200, 140734133182463, +STORE, 94412675600384, 94412677824511, +SNULL, 94412675710975, 94412677824511, +STORE, 94412675600384, 94412675710975, +STORE, 94412675710976, 94412677824511, +ERASE, 94412675710976, 94412677824511, +STORE, 94412677804032, 94412677816319, +STORE, 94412677816320, 94412677824511, +STORE, 140320087945216, 140320090198015, +SNULL, 140320088088575, 140320090198015, +STORE, 140320087945216, 140320088088575, +STORE, 140320088088576, 140320090198015, +ERASE, 140320088088576, 140320090198015, +STORE, 140320090185728, 140320090193919, +STORE, 140320090193920, 140320090198015, +STORE, 140734134591488, 140734134595583, +STORE, 140734134579200, 140734134591487, +STORE, 140320090157056, 140320090185727, +STORE, 140320090148864, 140320090157055, +STORE, 140320084148224, 140320087945215, +SNULL, 140320084148224, 140320085807103, +STORE, 140320085807104, 140320087945215, +STORE, 140320084148224, 140320085807103, +SNULL, 140320087904255, 140320087945215, +STORE, 140320085807104, 140320087904255, +STORE, 140320087904256, 140320087945215, +SNULL, 140320087904256, 140320087928831, +STORE, 140320087928832, 140320087945215, +STORE, 140320087904256, 140320087928831, +ERASE, 140320087904256, 140320087928831, +STORE, 140320087904256, 140320087928831, +ERASE, 140320087928832, 140320087945215, +STORE, 140320087928832, 140320087945215, +SNULL, 140320087920639, 140320087928831, +STORE, 140320087904256, 140320087920639, +STORE, 140320087920640, 140320087928831, +SNULL, 94412677812223, 94412677816319, +STORE, 94412677804032, 94412677812223, +STORE, 94412677812224, 94412677816319, +SNULL, 140320090189823, 140320090193919, +STORE, 140320090185728, 140320090189823, +STORE, 140320090189824, 140320090193919, +ERASE, 140320090157056, 140320090185727, +STORE, 94412684546048, 94412684681215, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723005485056, 140737488351231, +SNULL, 140723005493247, 140737488351231, +STORE, 140723005485056, 140723005493247, +STORE, 140723005353984, 140723005493247, +STORE, 94387431936000, 94387434160127, +SNULL, 94387432046591, 94387434160127, +STORE, 94387431936000, 94387432046591, +STORE, 94387432046592, 94387434160127, +ERASE, 94387432046592, 94387434160127, +STORE, 94387434139648, 94387434151935, +STORE, 94387434151936, 94387434160127, +STORE, 140151675392000, 140151677644799, +SNULL, 140151675535359, 140151677644799, +STORE, 140151675392000, 140151675535359, +STORE, 140151675535360, 140151677644799, +ERASE, 140151675535360, 140151677644799, +STORE, 140151677632512, 140151677640703, +STORE, 140151677640704, 140151677644799, +STORE, 140723005784064, 140723005788159, +STORE, 140723005771776, 140723005784063, +STORE, 140151677603840, 140151677632511, +STORE, 140151677595648, 140151677603839, +STORE, 140151671595008, 140151675391999, +SNULL, 140151671595008, 140151673253887, +STORE, 140151673253888, 140151675391999, +STORE, 140151671595008, 140151673253887, +SNULL, 140151675351039, 140151675391999, +STORE, 140151673253888, 140151675351039, +STORE, 140151675351040, 140151675391999, +SNULL, 140151675351040, 140151675375615, +STORE, 140151675375616, 140151675391999, +STORE, 140151675351040, 140151675375615, +ERASE, 140151675351040, 140151675375615, +STORE, 140151675351040, 140151675375615, +ERASE, 140151675375616, 140151675391999, +STORE, 140151675375616, 140151675391999, +SNULL, 140151675367423, 140151675375615, +STORE, 140151675351040, 140151675367423, +STORE, 140151675367424, 140151675375615, +SNULL, 94387434147839, 94387434151935, +STORE, 94387434139648, 94387434147839, +STORE, 94387434147840, 94387434151935, +SNULL, 140151677636607, 140151677640703, +STORE, 140151677632512, 140151677636607, +STORE, 140151677636608, 140151677640703, +ERASE, 140151677603840, 140151677632511, +STORE, 94387458818048, 94387458953215, +STORE, 94909010997248, 94909011210239, +STORE, 94909013307392, 94909013311487, +STORE, 94909013311488, 94909013319679, +STORE, 94909013319680, 94909013331967, +STORE, 94909014827008, 94909023371263, +STORE, 140712411975680, 140712413634559, +STORE, 140712413634560, 140712415731711, +STORE, 140712415731712, 140712415748095, +STORE, 140712415748096, 140712415756287, +STORE, 140712415756288, 140712415772671, +STORE, 140712415772672, 140712415784959, +STORE, 140712415784960, 140712417878015, +STORE, 140712417878016, 140712417882111, +STORE, 140712417882112, 140712417886207, +STORE, 140712417886208, 140712418029567, +STORE, 140712418398208, 140712420081663, +STORE, 140712420081664, 140712420098047, +STORE, 140712420126720, 140712420130815, +STORE, 140712420130816, 140712420134911, +STORE, 140712420134912, 140712420139007, +STORE, 140729293111296, 140729293250559, +STORE, 140729293307904, 140729293320191, +STORE, 140729293320192, 140729293324287, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720541691904, 140737488351231, +SNULL, 140720541700095, 140737488351231, +STORE, 140720541691904, 140720541700095, +STORE, 140720541560832, 140720541700095, +STORE, 94203603419136, 94203605643263, +SNULL, 94203603529727, 94203605643263, +STORE, 94203603419136, 94203603529727, +STORE, 94203603529728, 94203605643263, +ERASE, 94203603529728, 94203605643263, +STORE, 94203605622784, 94203605635071, +STORE, 94203605635072, 94203605643263, +STORE, 139847623081984, 139847625334783, +SNULL, 139847623225343, 139847625334783, +STORE, 139847623081984, 139847623225343, +STORE, 139847623225344, 139847625334783, +ERASE, 139847623225344, 139847625334783, +STORE, 139847625322496, 139847625330687, +STORE, 139847625330688, 139847625334783, +STORE, 140720542547968, 140720542552063, +STORE, 140720542535680, 140720542547967, +STORE, 139847625293824, 139847625322495, +STORE, 139847625285632, 139847625293823, +STORE, 139847619284992, 139847623081983, +SNULL, 139847619284992, 139847620943871, +STORE, 139847620943872, 139847623081983, +STORE, 139847619284992, 139847620943871, +SNULL, 139847623041023, 139847623081983, +STORE, 139847620943872, 139847623041023, +STORE, 139847623041024, 139847623081983, +SNULL, 139847623041024, 139847623065599, +STORE, 139847623065600, 139847623081983, +STORE, 139847623041024, 139847623065599, +ERASE, 139847623041024, 139847623065599, +STORE, 139847623041024, 139847623065599, +ERASE, 139847623065600, 139847623081983, +STORE, 139847623065600, 139847623081983, +SNULL, 139847623057407, 139847623065599, +STORE, 139847623041024, 139847623057407, +STORE, 139847623057408, 139847623065599, +SNULL, 94203605630975, 94203605635071, +STORE, 94203605622784, 94203605630975, +STORE, 94203605630976, 94203605635071, +SNULL, 139847625326591, 139847625330687, +STORE, 139847625322496, 139847625326591, +STORE, 139847625326592, 139847625330687, +ERASE, 139847625293824, 139847625322495, +STORE, 94203634880512, 94203635015679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721428738048, 140737488351231, +SNULL, 140721428746239, 140737488351231, +STORE, 140721428738048, 140721428746239, +STORE, 140721428606976, 140721428746239, +STORE, 93968808378368, 93968810602495, +SNULL, 93968808488959, 93968810602495, +STORE, 93968808378368, 93968808488959, +STORE, 93968808488960, 93968810602495, +ERASE, 93968808488960, 93968810602495, +STORE, 93968810582016, 93968810594303, +STORE, 93968810594304, 93968810602495, +STORE, 140397757026304, 140397759279103, +SNULL, 140397757169663, 140397759279103, +STORE, 140397757026304, 140397757169663, +STORE, 140397757169664, 140397759279103, +ERASE, 140397757169664, 140397759279103, +STORE, 140397759266816, 140397759275007, +STORE, 140397759275008, 140397759279103, +STORE, 140721430368256, 140721430372351, +STORE, 140721430355968, 140721430368255, +STORE, 140397759238144, 140397759266815, +STORE, 140397759229952, 140397759238143, +STORE, 140397753229312, 140397757026303, +SNULL, 140397753229312, 140397754888191, +STORE, 140397754888192, 140397757026303, +STORE, 140397753229312, 140397754888191, +SNULL, 140397756985343, 140397757026303, +STORE, 140397754888192, 140397756985343, +STORE, 140397756985344, 140397757026303, +SNULL, 140397756985344, 140397757009919, +STORE, 140397757009920, 140397757026303, +STORE, 140397756985344, 140397757009919, +ERASE, 140397756985344, 140397757009919, +STORE, 140397756985344, 140397757009919, +ERASE, 140397757009920, 140397757026303, +STORE, 140397757009920, 140397757026303, +SNULL, 140397757001727, 140397757009919, +STORE, 140397756985344, 140397757001727, +STORE, 140397757001728, 140397757009919, +SNULL, 93968810590207, 93968810594303, +STORE, 93968810582016, 93968810590207, +STORE, 93968810590208, 93968810594303, +SNULL, 140397759270911, 140397759275007, +STORE, 140397759266816, 140397759270911, +STORE, 140397759270912, 140397759275007, +ERASE, 140397759238144, 140397759266815, +STORE, 93968837025792, 93968837160959, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721751044096, 140737488351231, +SNULL, 140721751052287, 140737488351231, +STORE, 140721751044096, 140721751052287, +STORE, 140721750913024, 140721751052287, +STORE, 94426051657728, 94426053881855, +SNULL, 94426051768319, 94426053881855, +STORE, 94426051657728, 94426051768319, +STORE, 94426051768320, 94426053881855, +ERASE, 94426051768320, 94426053881855, +STORE, 94426053861376, 94426053873663, +STORE, 94426053873664, 94426053881855, +STORE, 140228456181760, 140228458434559, +SNULL, 140228456325119, 140228458434559, +STORE, 140228456181760, 140228456325119, +STORE, 140228456325120, 140228458434559, +ERASE, 140228456325120, 140228458434559, +STORE, 140228458422272, 140228458430463, +STORE, 140228458430464, 140228458434559, +STORE, 140721751117824, 140721751121919, +STORE, 140721751105536, 140721751117823, +STORE, 140228458393600, 140228458422271, +STORE, 140228458385408, 140228458393599, +STORE, 140228452384768, 140228456181759, +SNULL, 140228452384768, 140228454043647, +STORE, 140228454043648, 140228456181759, +STORE, 140228452384768, 140228454043647, +SNULL, 140228456140799, 140228456181759, +STORE, 140228454043648, 140228456140799, +STORE, 140228456140800, 140228456181759, +SNULL, 140228456140800, 140228456165375, +STORE, 140228456165376, 140228456181759, +STORE, 140228456140800, 140228456165375, +ERASE, 140228456140800, 140228456165375, +STORE, 140228456140800, 140228456165375, +ERASE, 140228456165376, 140228456181759, +STORE, 140228456165376, 140228456181759, +SNULL, 140228456157183, 140228456165375, +STORE, 140228456140800, 140228456157183, +STORE, 140228456157184, 140228456165375, +SNULL, 94426053869567, 94426053873663, +STORE, 94426053861376, 94426053869567, +STORE, 94426053869568, 94426053873663, +SNULL, 140228458426367, 140228458430463, +STORE, 140228458422272, 140228458426367, +STORE, 140228458426368, 140228458430463, +ERASE, 140228458393600, 140228458422271, +STORE, 94426073681920, 94426073817087, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732727623680, 140737488351231, +SNULL, 140732727631871, 140737488351231, +STORE, 140732727623680, 140732727631871, +STORE, 140732727492608, 140732727631871, +STORE, 94537485996032, 94537488220159, +SNULL, 94537486106623, 94537488220159, +STORE, 94537485996032, 94537486106623, +STORE, 94537486106624, 94537488220159, +ERASE, 94537486106624, 94537488220159, +STORE, 94537488199680, 94537488211967, +STORE, 94537488211968, 94537488220159, +STORE, 140446578036736, 140446580289535, +SNULL, 140446578180095, 140446580289535, +STORE, 140446578036736, 140446578180095, +STORE, 140446578180096, 140446580289535, +ERASE, 140446578180096, 140446580289535, +STORE, 140446580277248, 140446580285439, +STORE, 140446580285440, 140446580289535, +STORE, 140732727758848, 140732727762943, +STORE, 140732727746560, 140732727758847, +STORE, 140446580248576, 140446580277247, +STORE, 140446580240384, 140446580248575, +STORE, 140446574239744, 140446578036735, +SNULL, 140446574239744, 140446575898623, +STORE, 140446575898624, 140446578036735, +STORE, 140446574239744, 140446575898623, +SNULL, 140446577995775, 140446578036735, +STORE, 140446575898624, 140446577995775, +STORE, 140446577995776, 140446578036735, +SNULL, 140446577995776, 140446578020351, +STORE, 140446578020352, 140446578036735, +STORE, 140446577995776, 140446578020351, +ERASE, 140446577995776, 140446578020351, +STORE, 140446577995776, 140446578020351, +ERASE, 140446578020352, 140446578036735, +STORE, 140446578020352, 140446578036735, +SNULL, 140446578012159, 140446578020351, +STORE, 140446577995776, 140446578012159, +STORE, 140446578012160, 140446578020351, +SNULL, 94537488207871, 94537488211967, +STORE, 94537488199680, 94537488207871, +STORE, 94537488207872, 94537488211967, +SNULL, 140446580281343, 140446580285439, +STORE, 140446580277248, 140446580281343, +STORE, 140446580281344, 140446580285439, +ERASE, 140446580248576, 140446580277247, +STORE, 94537489014784, 94537489149951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728766808064, 140737488351231, +SNULL, 140728766816255, 140737488351231, +STORE, 140728766808064, 140728766816255, +STORE, 140728766676992, 140728766816255, +STORE, 94418513866752, 94418516090879, +SNULL, 94418513977343, 94418516090879, +STORE, 94418513866752, 94418513977343, +STORE, 94418513977344, 94418516090879, +ERASE, 94418513977344, 94418516090879, +STORE, 94418516070400, 94418516082687, +STORE, 94418516082688, 94418516090879, +STORE, 140556479520768, 140556481773567, +SNULL, 140556479664127, 140556481773567, +STORE, 140556479520768, 140556479664127, +STORE, 140556479664128, 140556481773567, +ERASE, 140556479664128, 140556481773567, +STORE, 140556481761280, 140556481769471, +STORE, 140556481769472, 140556481773567, +STORE, 140728767148032, 140728767152127, +STORE, 140728767135744, 140728767148031, +STORE, 140556481732608, 140556481761279, +STORE, 140556481724416, 140556481732607, +STORE, 140556475723776, 140556479520767, +SNULL, 140556475723776, 140556477382655, +STORE, 140556477382656, 140556479520767, +STORE, 140556475723776, 140556477382655, +SNULL, 140556479479807, 140556479520767, +STORE, 140556477382656, 140556479479807, +STORE, 140556479479808, 140556479520767, +SNULL, 140556479479808, 140556479504383, +STORE, 140556479504384, 140556479520767, +STORE, 140556479479808, 140556479504383, +ERASE, 140556479479808, 140556479504383, +STORE, 140556479479808, 140556479504383, +ERASE, 140556479504384, 140556479520767, +STORE, 140556479504384, 140556479520767, +SNULL, 140556479496191, 140556479504383, +STORE, 140556479479808, 140556479496191, +STORE, 140556479496192, 140556479504383, +SNULL, 94418516078591, 94418516082687, +STORE, 94418516070400, 94418516078591, +STORE, 94418516078592, 94418516082687, +SNULL, 140556481765375, 140556481769471, +STORE, 140556481761280, 140556481765375, +STORE, 140556481765376, 140556481769471, +ERASE, 140556481732608, 140556481761279, +STORE, 94418541113344, 94418541248511, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723945873408, 140737488351231, +SNULL, 140723945881599, 140737488351231, +STORE, 140723945873408, 140723945881599, +STORE, 140723945742336, 140723945881599, +STORE, 94543169773568, 94543171997695, +SNULL, 94543169884159, 94543171997695, +STORE, 94543169773568, 94543169884159, +STORE, 94543169884160, 94543171997695, +ERASE, 94543169884160, 94543171997695, +STORE, 94543171977216, 94543171989503, +STORE, 94543171989504, 94543171997695, +STORE, 139890420883456, 139890423136255, +SNULL, 139890421026815, 139890423136255, +STORE, 139890420883456, 139890421026815, +STORE, 139890421026816, 139890423136255, +ERASE, 139890421026816, 139890423136255, +STORE, 139890423123968, 139890423132159, +STORE, 139890423132160, 139890423136255, +STORE, 140723946102784, 140723946106879, +STORE, 140723946090496, 140723946102783, +STORE, 139890423095296, 139890423123967, +STORE, 139890423087104, 139890423095295, +STORE, 139890417086464, 139890420883455, +SNULL, 139890417086464, 139890418745343, +STORE, 139890418745344, 139890420883455, +STORE, 139890417086464, 139890418745343, +SNULL, 139890420842495, 139890420883455, +STORE, 139890418745344, 139890420842495, +STORE, 139890420842496, 139890420883455, +SNULL, 139890420842496, 139890420867071, +STORE, 139890420867072, 139890420883455, +STORE, 139890420842496, 139890420867071, +ERASE, 139890420842496, 139890420867071, +STORE, 139890420842496, 139890420867071, +ERASE, 139890420867072, 139890420883455, +STORE, 139890420867072, 139890420883455, +SNULL, 139890420858879, 139890420867071, +STORE, 139890420842496, 139890420858879, +STORE, 139890420858880, 139890420867071, +SNULL, 94543171985407, 94543171989503, +STORE, 94543171977216, 94543171985407, +STORE, 94543171985408, 94543171989503, +SNULL, 139890423128063, 139890423132159, +STORE, 139890423123968, 139890423128063, +STORE, 139890423128064, 139890423132159, +ERASE, 139890423095296, 139890423123967, +STORE, 94543197097984, 94543197233151, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736205979648, 140737488351231, +SNULL, 140736205987839, 140737488351231, +STORE, 140736205979648, 140736205987839, +STORE, 140736205848576, 140736205987839, +STORE, 94913209913344, 94913212137471, +SNULL, 94913210023935, 94913212137471, +STORE, 94913209913344, 94913210023935, +STORE, 94913210023936, 94913212137471, +ERASE, 94913210023936, 94913212137471, +STORE, 94913212116992, 94913212129279, +STORE, 94913212129280, 94913212137471, +STORE, 140006323052544, 140006325305343, +SNULL, 140006323195903, 140006325305343, +STORE, 140006323052544, 140006323195903, +STORE, 140006323195904, 140006325305343, +ERASE, 140006323195904, 140006325305343, +STORE, 140006325293056, 140006325301247, +STORE, 140006325301248, 140006325305343, +STORE, 140736206716928, 140736206721023, +STORE, 140736206704640, 140736206716927, +STORE, 140006325264384, 140006325293055, +STORE, 140006325256192, 140006325264383, +STORE, 140006319255552, 140006323052543, +SNULL, 140006319255552, 140006320914431, +STORE, 140006320914432, 140006323052543, +STORE, 140006319255552, 140006320914431, +SNULL, 140006323011583, 140006323052543, +STORE, 140006320914432, 140006323011583, +STORE, 140006323011584, 140006323052543, +SNULL, 140006323011584, 140006323036159, +STORE, 140006323036160, 140006323052543, +STORE, 140006323011584, 140006323036159, +ERASE, 140006323011584, 140006323036159, +STORE, 140006323011584, 140006323036159, +ERASE, 140006323036160, 140006323052543, +STORE, 140006323036160, 140006323052543, +SNULL, 140006323027967, 140006323036159, +STORE, 140006323011584, 140006323027967, +STORE, 140006323027968, 140006323036159, +SNULL, 94913212125183, 94913212129279, +STORE, 94913212116992, 94913212125183, +STORE, 94913212125184, 94913212129279, +SNULL, 140006325297151, 140006325301247, +STORE, 140006325293056, 140006325297151, +STORE, 140006325297152, 140006325301247, +ERASE, 140006325264384, 140006325293055, +STORE, 94913239932928, 94913240068095, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726926897152, 140737488351231, +SNULL, 140726926905343, 140737488351231, +STORE, 140726926897152, 140726926905343, +STORE, 140726926766080, 140726926905343, +STORE, 94213246820352, 94213249044479, +SNULL, 94213246930943, 94213249044479, +STORE, 94213246820352, 94213246930943, +STORE, 94213246930944, 94213249044479, +ERASE, 94213246930944, 94213249044479, +STORE, 94213249024000, 94213249036287, +STORE, 94213249036288, 94213249044479, +STORE, 140368830242816, 140368832495615, +SNULL, 140368830386175, 140368832495615, +STORE, 140368830242816, 140368830386175, +STORE, 140368830386176, 140368832495615, +ERASE, 140368830386176, 140368832495615, +STORE, 140368832483328, 140368832491519, +STORE, 140368832491520, 140368832495615, +STORE, 140726926999552, 140726927003647, +STORE, 140726926987264, 140726926999551, +STORE, 140368832454656, 140368832483327, +STORE, 140368832446464, 140368832454655, +STORE, 140368826445824, 140368830242815, +SNULL, 140368826445824, 140368828104703, +STORE, 140368828104704, 140368830242815, +STORE, 140368826445824, 140368828104703, +SNULL, 140368830201855, 140368830242815, +STORE, 140368828104704, 140368830201855, +STORE, 140368830201856, 140368830242815, +SNULL, 140368830201856, 140368830226431, +STORE, 140368830226432, 140368830242815, +STORE, 140368830201856, 140368830226431, +ERASE, 140368830201856, 140368830226431, +STORE, 140368830201856, 140368830226431, +ERASE, 140368830226432, 140368830242815, +STORE, 140368830226432, 140368830242815, +SNULL, 140368830218239, 140368830226431, +STORE, 140368830201856, 140368830218239, +STORE, 140368830218240, 140368830226431, +SNULL, 94213249032191, 94213249036287, +STORE, 94213249024000, 94213249032191, +STORE, 94213249032192, 94213249036287, +SNULL, 140368832487423, 140368832491519, +STORE, 140368832483328, 140368832487423, +STORE, 140368832487424, 140368832491519, +ERASE, 140368832454656, 140368832483327, +STORE, 94213267435520, 94213267570687, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728954130432, 140737488351231, +SNULL, 140728954138623, 140737488351231, +STORE, 140728954130432, 140728954138623, +STORE, 140728953999360, 140728954138623, +STORE, 94672570966016, 94672573190143, +SNULL, 94672571076607, 94672573190143, +STORE, 94672570966016, 94672571076607, +STORE, 94672571076608, 94672573190143, +ERASE, 94672571076608, 94672573190143, +STORE, 94672573169664, 94672573181951, +STORE, 94672573181952, 94672573190143, +STORE, 140201696735232, 140201698988031, +SNULL, 140201696878591, 140201698988031, +STORE, 140201696735232, 140201696878591, +STORE, 140201696878592, 140201698988031, +ERASE, 140201696878592, 140201698988031, +STORE, 140201698975744, 140201698983935, +STORE, 140201698983936, 140201698988031, +STORE, 140728954163200, 140728954167295, +STORE, 140728954150912, 140728954163199, +STORE, 140201698947072, 140201698975743, +STORE, 140201698938880, 140201698947071, +STORE, 140201692938240, 140201696735231, +SNULL, 140201692938240, 140201694597119, +STORE, 140201694597120, 140201696735231, +STORE, 140201692938240, 140201694597119, +SNULL, 140201696694271, 140201696735231, +STORE, 140201694597120, 140201696694271, +STORE, 140201696694272, 140201696735231, +SNULL, 140201696694272, 140201696718847, +STORE, 140201696718848, 140201696735231, +STORE, 140201696694272, 140201696718847, +ERASE, 140201696694272, 140201696718847, +STORE, 140201696694272, 140201696718847, +ERASE, 140201696718848, 140201696735231, +STORE, 140201696718848, 140201696735231, +SNULL, 140201696710655, 140201696718847, +STORE, 140201696694272, 140201696710655, +STORE, 140201696710656, 140201696718847, +SNULL, 94672573177855, 94672573181951, +STORE, 94672573169664, 94672573177855, +STORE, 94672573177856, 94672573181951, +SNULL, 140201698979839, 140201698983935, +STORE, 140201698975744, 140201698979839, +STORE, 140201698979840, 140201698983935, +ERASE, 140201698947072, 140201698975743, +STORE, 94672595689472, 94672595824639, +STORE, 94114394132480, 94114394345471, +STORE, 94114396442624, 94114396446719, +STORE, 94114396446720, 94114396454911, +STORE, 94114396454912, 94114396467199, +STORE, 94114421575680, 94114428256255, +STORE, 139934313955328, 139934315614207, +STORE, 139934315614208, 139934317711359, +STORE, 139934317711360, 139934317727743, +STORE, 139934317727744, 139934317735935, +STORE, 139934317735936, 139934317752319, +STORE, 139934317752320, 139934317764607, +STORE, 139934317764608, 139934319857663, +STORE, 139934319857664, 139934319861759, +STORE, 139934319861760, 139934319865855, +STORE, 139934319865856, 139934320009215, +STORE, 139934320377856, 139934322061311, +STORE, 139934322061312, 139934322077695, +STORE, 139934322106368, 139934322110463, +STORE, 139934322110464, 139934322114559, +STORE, 139934322114560, 139934322118655, +STORE, 140731200376832, 140731200516095, +STORE, 140731200929792, 140731200942079, +STORE, 140731200942080, 140731200946175, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721532362752, 140737488351231, +SNULL, 140721532370943, 140737488351231, +STORE, 140721532362752, 140721532370943, +STORE, 140721532231680, 140721532370943, +STORE, 94467222597632, 94467224821759, +SNULL, 94467222708223, 94467224821759, +STORE, 94467222597632, 94467222708223, +STORE, 94467222708224, 94467224821759, +ERASE, 94467222708224, 94467224821759, +STORE, 94467224801280, 94467224813567, +STORE, 94467224813568, 94467224821759, +STORE, 140191433543680, 140191435796479, +SNULL, 140191433687039, 140191435796479, +STORE, 140191433543680, 140191433687039, +STORE, 140191433687040, 140191435796479, +ERASE, 140191433687040, 140191435796479, +STORE, 140191435784192, 140191435792383, +STORE, 140191435792384, 140191435796479, +STORE, 140721533034496, 140721533038591, +STORE, 140721533022208, 140721533034495, +STORE, 140191435755520, 140191435784191, +STORE, 140191435747328, 140191435755519, +STORE, 140191429746688, 140191433543679, +SNULL, 140191429746688, 140191431405567, +STORE, 140191431405568, 140191433543679, +STORE, 140191429746688, 140191431405567, +SNULL, 140191433502719, 140191433543679, +STORE, 140191431405568, 140191433502719, +STORE, 140191433502720, 140191433543679, +SNULL, 140191433502720, 140191433527295, +STORE, 140191433527296, 140191433543679, +STORE, 140191433502720, 140191433527295, +ERASE, 140191433502720, 140191433527295, +STORE, 140191433502720, 140191433527295, +ERASE, 140191433527296, 140191433543679, +STORE, 140191433527296, 140191433543679, +SNULL, 140191433519103, 140191433527295, +STORE, 140191433502720, 140191433519103, +STORE, 140191433519104, 140191433527295, +SNULL, 94467224809471, 94467224813567, +STORE, 94467224801280, 94467224809471, +STORE, 94467224809472, 94467224813567, +SNULL, 140191435788287, 140191435792383, +STORE, 140191435784192, 140191435788287, +STORE, 140191435788288, 140191435792383, +ERASE, 140191435755520, 140191435784191, +STORE, 94467251847168, 94467251982335, +STORE, 94367895400448, 94367895613439, +STORE, 94367897710592, 94367897714687, +STORE, 94367897714688, 94367897722879, +STORE, 94367897722880, 94367897735167, +STORE, 94367925264384, 94367926861823, +STORE, 139801317548032, 139801319206911, +STORE, 139801319206912, 139801321304063, +STORE, 139801321304064, 139801321320447, +STORE, 139801321320448, 139801321328639, +STORE, 139801321328640, 139801321345023, +STORE, 139801321345024, 139801321357311, +STORE, 139801321357312, 139801323450367, +STORE, 139801323450368, 139801323454463, +STORE, 139801323454464, 139801323458559, +STORE, 139801323458560, 139801323601919, +STORE, 139801323970560, 139801325654015, +STORE, 139801325654016, 139801325670399, +STORE, 139801325699072, 139801325703167, +STORE, 139801325703168, 139801325707263, +STORE, 139801325707264, 139801325711359, +STORE, 140724442861568, 140724443000831, +STORE, 140724443611136, 140724443623423, +STORE, 140724443623424, 140724443627519, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731353149440, 140737488351231, +SNULL, 140731353157631, 140737488351231, +STORE, 140731353149440, 140731353157631, +STORE, 140731353018368, 140731353157631, +STORE, 94310379503616, 94310381838335, +SNULL, 94310379716607, 94310381838335, +STORE, 94310379503616, 94310379716607, +STORE, 94310379716608, 94310381838335, +ERASE, 94310379716608, 94310381838335, +STORE, 94310381813760, 94310381826047, +STORE, 94310381826048, 94310381838335, +STORE, 140515434659840, 140515436912639, +SNULL, 140515434803199, 140515436912639, +STORE, 140515434659840, 140515434803199, +STORE, 140515434803200, 140515436912639, +ERASE, 140515434803200, 140515436912639, +STORE, 140515436900352, 140515436908543, +STORE, 140515436908544, 140515436912639, +STORE, 140731353886720, 140731353890815, +STORE, 140731353874432, 140731353886719, +STORE, 140515436871680, 140515436900351, +STORE, 140515436863488, 140515436871679, +STORE, 140515432546304, 140515434659839, +SNULL, 140515432546304, 140515432558591, +STORE, 140515432558592, 140515434659839, +STORE, 140515432546304, 140515432558591, +SNULL, 140515434651647, 140515434659839, +STORE, 140515432558592, 140515434651647, +STORE, 140515434651648, 140515434659839, +ERASE, 140515434651648, 140515434659839, +STORE, 140515434651648, 140515434659839, +STORE, 140515428749312, 140515432546303, +SNULL, 140515428749312, 140515430408191, +STORE, 140515430408192, 140515432546303, +STORE, 140515428749312, 140515430408191, +SNULL, 140515432505343, 140515432546303, +STORE, 140515430408192, 140515432505343, +STORE, 140515432505344, 140515432546303, +SNULL, 140515432505344, 140515432529919, +STORE, 140515432529920, 140515432546303, +STORE, 140515432505344, 140515432529919, +ERASE, 140515432505344, 140515432529919, +STORE, 140515432505344, 140515432529919, +ERASE, 140515432529920, 140515432546303, +STORE, 140515432529920, 140515432546303, +STORE, 140515436855296, 140515436871679, +SNULL, 140515432521727, 140515432529919, +STORE, 140515432505344, 140515432521727, +STORE, 140515432521728, 140515432529919, +SNULL, 140515434655743, 140515434659839, +STORE, 140515434651648, 140515434655743, +STORE, 140515434655744, 140515434659839, +SNULL, 94310381817855, 94310381826047, +STORE, 94310381813760, 94310381817855, +STORE, 94310381817856, 94310381826047, +SNULL, 140515436904447, 140515436908543, +STORE, 140515436900352, 140515436904447, +STORE, 140515436904448, 140515436908543, +ERASE, 140515436871680, 140515436900351, +STORE, 94310395457536, 94310395592703, +STORE, 140515435171840, 140515436855295, +STORE, 94310395457536, 94310395727871, +STORE, 94310395457536, 94310395863039, +STORE, 94310395457536, 94310396047359, +SNULL, 94310396022783, 94310396047359, +STORE, 94310395457536, 94310396022783, +STORE, 94310396022784, 94310396047359, +ERASE, 94310396022784, 94310396047359, +STORE, 94310395457536, 94310396157951, +STORE, 94310395457536, 94310396293119, +SNULL, 94310396276735, 94310396293119, +STORE, 94310395457536, 94310396276735, +STORE, 94310396276736, 94310396293119, +ERASE, 94310396276736, 94310396293119, +STORE, 94310395457536, 94310396411903, +SNULL, 94310396383231, 94310396411903, +STORE, 94310395457536, 94310396383231, +STORE, 94310396383232, 94310396411903, +ERASE, 94310396383232, 94310396411903, +STORE, 94310395457536, 94310396522495, +STORE, 94310395457536, 94310396674047, +SNULL, 94310396657663, 94310396674047, +STORE, 94310395457536, 94310396657663, +STORE, 94310396657664, 94310396674047, +ERASE, 94310396657664, 94310396674047, +SNULL, 94310396624895, 94310396657663, +STORE, 94310395457536, 94310396624895, +STORE, 94310396624896, 94310396657663, +ERASE, 94310396624896, 94310396657663, +STORE, 94310395457536, 94310396776447, +SNULL, 94310396764159, 94310396776447, +STORE, 94310395457536, 94310396764159, +STORE, 94310396764160, 94310396776447, +ERASE, 94310396764160, 94310396776447, +SNULL, 94310396739583, 94310396764159, +STORE, 94310395457536, 94310396739583, +STORE, 94310396739584, 94310396764159, +ERASE, 94310396739584, 94310396764159, +STORE, 94310395457536, 94310396882943, +STORE, 94310395457536, 94310397018111, +STORE, 94310395457536, 94310397161471, +STORE, 94310395457536, 94310397300735, +SNULL, 94310397292543, 94310397300735, +STORE, 94310395457536, 94310397292543, +STORE, 94310397292544, 94310397300735, +ERASE, 94310397292544, 94310397300735, +STORE, 94359222210560, 94359222423551, +STORE, 94359224520704, 94359224524799, +STORE, 94359224524800, 94359224532991, +STORE, 94359224532992, 94359224545279, +STORE, 94359238348800, 94359239385087, +STORE, 140675699838976, 140675701497855, +STORE, 140675701497856, 140675703595007, +STORE, 140675703595008, 140675703611391, +STORE, 140675703611392, 140675703619583, +STORE, 140675703619584, 140675703635967, +STORE, 140675703635968, 140675703648255, +STORE, 140675703648256, 140675705741311, +STORE, 140675705741312, 140675705745407, +STORE, 140675705745408, 140675705749503, +STORE, 140675705749504, 140675705892863, +STORE, 140675706261504, 140675707944959, +STORE, 140675707944960, 140675707961343, +STORE, 140675707990016, 140675707994111, +STORE, 140675707994112, 140675707998207, +STORE, 140675707998208, 140675708002303, +STORE, 140721324634112, 140721324773375, +STORE, 140721324810240, 140721324822527, +STORE, 140721324822528, 140721324826623, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724099678208, 140737488351231, +SNULL, 140724099686399, 140737488351231, +STORE, 140724099678208, 140724099686399, +STORE, 140724099547136, 140724099686399, +STORE, 94586638516224, 94586640850943, +SNULL, 94586638729215, 94586640850943, +STORE, 94586638516224, 94586638729215, +STORE, 94586638729216, 94586640850943, +ERASE, 94586638729216, 94586640850943, +STORE, 94586640826368, 94586640838655, +STORE, 94586640838656, 94586640850943, +STORE, 140371033796608, 140371036049407, +SNULL, 140371033939967, 140371036049407, +STORE, 140371033796608, 140371033939967, +STORE, 140371033939968, 140371036049407, +ERASE, 140371033939968, 140371036049407, +STORE, 140371036037120, 140371036045311, +STORE, 140371036045312, 140371036049407, +STORE, 140724100001792, 140724100005887, +STORE, 140724099989504, 140724100001791, +STORE, 140371036008448, 140371036037119, +STORE, 140371036000256, 140371036008447, +STORE, 140371031683072, 140371033796607, +SNULL, 140371031683072, 140371031695359, +STORE, 140371031695360, 140371033796607, +STORE, 140371031683072, 140371031695359, +SNULL, 140371033788415, 140371033796607, +STORE, 140371031695360, 140371033788415, +STORE, 140371033788416, 140371033796607, +ERASE, 140371033788416, 140371033796607, +STORE, 140371033788416, 140371033796607, +STORE, 140371027886080, 140371031683071, +SNULL, 140371027886080, 140371029544959, +STORE, 140371029544960, 140371031683071, +STORE, 140371027886080, 140371029544959, +SNULL, 140371031642111, 140371031683071, +STORE, 140371029544960, 140371031642111, +STORE, 140371031642112, 140371031683071, +SNULL, 140371031642112, 140371031666687, +STORE, 140371031666688, 140371031683071, +STORE, 140371031642112, 140371031666687, +ERASE, 140371031642112, 140371031666687, +STORE, 140371031642112, 140371031666687, +ERASE, 140371031666688, 140371031683071, +STORE, 140371031666688, 140371031683071, +STORE, 140371035992064, 140371036008447, +SNULL, 140371031658495, 140371031666687, +STORE, 140371031642112, 140371031658495, +STORE, 140371031658496, 140371031666687, +SNULL, 140371033792511, 140371033796607, +STORE, 140371033788416, 140371033792511, +STORE, 140371033792512, 140371033796607, +SNULL, 94586640830463, 94586640838655, +STORE, 94586640826368, 94586640830463, +STORE, 94586640830464, 94586640838655, +SNULL, 140371036041215, 140371036045311, +STORE, 140371036037120, 140371036041215, +STORE, 140371036041216, 140371036045311, +ERASE, 140371036008448, 140371036037119, +STORE, 94586663849984, 94586663985151, +STORE, 140371034308608, 140371035992063, +STORE, 94586663849984, 94586664120319, +STORE, 94586663849984, 94586664255487, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727532937216, 140737488351231, +SNULL, 140727532945407, 140737488351231, +STORE, 140727532937216, 140727532945407, +STORE, 140727532806144, 140727532945407, +STORE, 94849780191232, 94849782525951, +SNULL, 94849780404223, 94849782525951, +STORE, 94849780191232, 94849780404223, +STORE, 94849780404224, 94849782525951, +ERASE, 94849780404224, 94849782525951, +STORE, 94849782501376, 94849782513663, +STORE, 94849782513664, 94849782525951, +STORE, 140382070218752, 140382072471551, +SNULL, 140382070362111, 140382072471551, +STORE, 140382070218752, 140382070362111, +STORE, 140382070362112, 140382072471551, +ERASE, 140382070362112, 140382072471551, +STORE, 140382072459264, 140382072467455, +STORE, 140382072467456, 140382072471551, +STORE, 140727533092864, 140727533096959, +STORE, 140727533080576, 140727533092863, +STORE, 140382072430592, 140382072459263, +STORE, 140382072422400, 140382072430591, +STORE, 140382068105216, 140382070218751, +SNULL, 140382068105216, 140382068117503, +STORE, 140382068117504, 140382070218751, +STORE, 140382068105216, 140382068117503, +SNULL, 140382070210559, 140382070218751, +STORE, 140382068117504, 140382070210559, +STORE, 140382070210560, 140382070218751, +ERASE, 140382070210560, 140382070218751, +STORE, 140382070210560, 140382070218751, +STORE, 140382064308224, 140382068105215, +SNULL, 140382064308224, 140382065967103, +STORE, 140382065967104, 140382068105215, +STORE, 140382064308224, 140382065967103, +SNULL, 140382068064255, 140382068105215, +STORE, 140382065967104, 140382068064255, +STORE, 140382068064256, 140382068105215, +SNULL, 140382068064256, 140382068088831, +STORE, 140382068088832, 140382068105215, +STORE, 140382068064256, 140382068088831, +ERASE, 140382068064256, 140382068088831, +STORE, 140382068064256, 140382068088831, +ERASE, 140382068088832, 140382068105215, +STORE, 140382068088832, 140382068105215, +STORE, 140382072414208, 140382072430591, +SNULL, 140382068080639, 140382068088831, +STORE, 140382068064256, 140382068080639, +STORE, 140382068080640, 140382068088831, +SNULL, 140382070214655, 140382070218751, +STORE, 140382070210560, 140382070214655, +STORE, 140382070214656, 140382070218751, +SNULL, 94849782505471, 94849782513663, +STORE, 94849782501376, 94849782505471, +STORE, 94849782505472, 94849782513663, +SNULL, 140382072463359, 140382072467455, +STORE, 140382072459264, 140382072463359, +STORE, 140382072463360, 140382072467455, +ERASE, 140382072430592, 140382072459263, +STORE, 94849782845440, 94849782980607, +STORE, 140382070730752, 140382072414207, +STORE, 94849782845440, 94849783115775, +STORE, 94849782845440, 94849783250943, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722594377728, 140737488351231, +SNULL, 140722594385919, 140737488351231, +STORE, 140722594377728, 140722594385919, +STORE, 140722594246656, 140722594385919, +STORE, 94421466353664, 94421468577791, +SNULL, 94421466464255, 94421468577791, +STORE, 94421466353664, 94421466464255, +STORE, 94421466464256, 94421468577791, +ERASE, 94421466464256, 94421468577791, +STORE, 94421468557312, 94421468569599, +STORE, 94421468569600, 94421468577791, +STORE, 140345458057216, 140345460310015, +SNULL, 140345458200575, 140345460310015, +STORE, 140345458057216, 140345458200575, +STORE, 140345458200576, 140345460310015, +ERASE, 140345458200576, 140345460310015, +STORE, 140345460297728, 140345460305919, +STORE, 140345460305920, 140345460310015, +STORE, 140722595557376, 140722595561471, +STORE, 140722595545088, 140722595557375, +STORE, 140345460269056, 140345460297727, +STORE, 140345460260864, 140345460269055, +STORE, 140345454260224, 140345458057215, +SNULL, 140345454260224, 140345455919103, +STORE, 140345455919104, 140345458057215, +STORE, 140345454260224, 140345455919103, +SNULL, 140345458016255, 140345458057215, +STORE, 140345455919104, 140345458016255, +STORE, 140345458016256, 140345458057215, +SNULL, 140345458016256, 140345458040831, +STORE, 140345458040832, 140345458057215, +STORE, 140345458016256, 140345458040831, +ERASE, 140345458016256, 140345458040831, +STORE, 140345458016256, 140345458040831, +ERASE, 140345458040832, 140345458057215, +STORE, 140345458040832, 140345458057215, +SNULL, 140345458032639, 140345458040831, +STORE, 140345458016256, 140345458032639, +STORE, 140345458032640, 140345458040831, +SNULL, 94421468565503, 94421468569599, +STORE, 94421468557312, 94421468565503, +STORE, 94421468565504, 94421468569599, +SNULL, 140345460301823, 140345460305919, +STORE, 140345460297728, 140345460301823, +STORE, 140345460301824, 140345460305919, +ERASE, 140345460269056, 140345460297727, +STORE, 94421496004608, 94421496139775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726096302080, 140737488351231, +SNULL, 140726096310271, 140737488351231, +STORE, 140726096302080, 140726096310271, +STORE, 140726096171008, 140726096310271, +STORE, 94101992124416, 94101994459135, +SNULL, 94101992337407, 94101994459135, +STORE, 94101992124416, 94101992337407, +STORE, 94101992337408, 94101994459135, +ERASE, 94101992337408, 94101994459135, +STORE, 94101994434560, 94101994446847, +STORE, 94101994446848, 94101994459135, +STORE, 140192085594112, 140192087846911, +SNULL, 140192085737471, 140192087846911, +STORE, 140192085594112, 140192085737471, +STORE, 140192085737472, 140192087846911, +ERASE, 140192085737472, 140192087846911, +STORE, 140192087834624, 140192087842815, +STORE, 140192087842816, 140192087846911, +STORE, 140726096375808, 140726096379903, +STORE, 140726096363520, 140726096375807, +STORE, 140192087805952, 140192087834623, +STORE, 140192087797760, 140192087805951, +STORE, 140192083480576, 140192085594111, +SNULL, 140192083480576, 140192083492863, +STORE, 140192083492864, 140192085594111, +STORE, 140192083480576, 140192083492863, +SNULL, 140192085585919, 140192085594111, +STORE, 140192083492864, 140192085585919, +STORE, 140192085585920, 140192085594111, +ERASE, 140192085585920, 140192085594111, +STORE, 140192085585920, 140192085594111, +STORE, 140192079683584, 140192083480575, +SNULL, 140192079683584, 140192081342463, +STORE, 140192081342464, 140192083480575, +STORE, 140192079683584, 140192081342463, +SNULL, 140192083439615, 140192083480575, +STORE, 140192081342464, 140192083439615, +STORE, 140192083439616, 140192083480575, +SNULL, 140192083439616, 140192083464191, +STORE, 140192083464192, 140192083480575, +STORE, 140192083439616, 140192083464191, +ERASE, 140192083439616, 140192083464191, +STORE, 140192083439616, 140192083464191, +ERASE, 140192083464192, 140192083480575, +STORE, 140192083464192, 140192083480575, +STORE, 140192087789568, 140192087805951, +SNULL, 140192083455999, 140192083464191, +STORE, 140192083439616, 140192083455999, +STORE, 140192083456000, 140192083464191, +SNULL, 140192085590015, 140192085594111, +STORE, 140192085585920, 140192085590015, +STORE, 140192085590016, 140192085594111, +SNULL, 94101994438655, 94101994446847, +STORE, 94101994434560, 94101994438655, +STORE, 94101994438656, 94101994446847, +SNULL, 140192087838719, 140192087842815, +STORE, 140192087834624, 140192087838719, +STORE, 140192087838720, 140192087842815, +ERASE, 140192087805952, 140192087834623, +STORE, 94102011887616, 94102012022783, +STORE, 140192086106112, 140192087789567, +STORE, 94102011887616, 94102012157951, +STORE, 94102011887616, 94102012293119, +STORE, 94102011887616, 94102012440575, +SNULL, 94102012428287, 94102012440575, +STORE, 94102011887616, 94102012428287, +STORE, 94102012428288, 94102012440575, +ERASE, 94102012428288, 94102012440575, +STORE, 94102011887616, 94102012579839, +STORE, 94102011887616, 94102012715007, +SNULL, 94102012694527, 94102012715007, +STORE, 94102011887616, 94102012694527, +STORE, 94102012694528, 94102012715007, +ERASE, 94102012694528, 94102012715007, +STORE, 94102011887616, 94102012833791, +STORE, 94102011887616, 94102012968959, +SNULL, 94102012927999, 94102012968959, +STORE, 94102011887616, 94102012927999, +STORE, 94102012928000, 94102012968959, +ERASE, 94102012928000, 94102012968959, +STORE, 94102011887616, 94102013091839, +SNULL, 94102013075455, 94102013091839, +STORE, 94102011887616, 94102013075455, +STORE, 94102013075456, 94102013091839, +ERASE, 94102013075456, 94102013091839, +STORE, 94102011887616, 94102013210623, +STORE, 94102011887616, 94102013345791, +STORE, 93968727965696, 93968728178687, +STORE, 93968730275840, 93968730279935, +STORE, 93968730279936, 93968730288127, +STORE, 93968730288128, 93968730300415, +STORE, 93968731140096, 93968732704767, +STORE, 140588443168768, 140588444827647, +STORE, 140588444827648, 140588446924799, +STORE, 140588446924800, 140588446941183, +STORE, 140588446941184, 140588446949375, +STORE, 140588446949376, 140588446965759, +STORE, 140588446965760, 140588446978047, +STORE, 140588446978048, 140588449071103, +STORE, 140588449071104, 140588449075199, +STORE, 140588449075200, 140588449079295, +STORE, 140588449079296, 140588449222655, +STORE, 140588449591296, 140588451274751, +STORE, 140588451274752, 140588451291135, +STORE, 140588451319808, 140588451323903, +STORE, 140588451323904, 140588451327999, +STORE, 140588451328000, 140588451332095, +STORE, 140733877239808, 140733877379071, +STORE, 140733878702080, 140733878714367, +STORE, 140733878714368, 140733878718463, +STORE, 93968727965696, 93968728178687, +STORE, 93968730275840, 93968730279935, +STORE, 93968730279936, 93968730288127, +STORE, 93968730288128, 93968730300415, +STORE, 93968731140096, 93968732991487, +STORE, 140588443168768, 140588444827647, +STORE, 140588444827648, 140588446924799, +STORE, 140588446924800, 140588446941183, +STORE, 140588446941184, 140588446949375, +STORE, 140588446949376, 140588446965759, +STORE, 140588446965760, 140588446978047, +STORE, 140588446978048, 140588449071103, +STORE, 140588449071104, 140588449075199, +STORE, 140588449075200, 140588449079295, +STORE, 140588449079296, 140588449222655, +STORE, 140588449591296, 140588451274751, +STORE, 140588451274752, 140588451291135, +STORE, 140588451319808, 140588451323903, +STORE, 140588451323904, 140588451327999, +STORE, 140588451328000, 140588451332095, +STORE, 140733877239808, 140733877379071, +STORE, 140733878702080, 140733878714367, +STORE, 140733878714368, 140733878718463, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733054472192, 140737488351231, +SNULL, 140733054480383, 140737488351231, +STORE, 140733054472192, 140733054480383, +STORE, 140733054341120, 140733054480383, +STORE, 93992873623552, 93992875847679, +SNULL, 93992873734143, 93992875847679, +STORE, 93992873623552, 93992873734143, +STORE, 93992873734144, 93992875847679, +ERASE, 93992873734144, 93992875847679, +STORE, 93992875827200, 93992875839487, +STORE, 93992875839488, 93992875847679, +STORE, 139790881488896, 139790883741695, +SNULL, 139790881632255, 139790883741695, +STORE, 139790881488896, 139790881632255, +STORE, 139790881632256, 139790883741695, +ERASE, 139790881632256, 139790883741695, +STORE, 139790883729408, 139790883737599, +STORE, 139790883737600, 139790883741695, +STORE, 140733054754816, 140733054758911, +STORE, 140733054742528, 140733054754815, +STORE, 139790883700736, 139790883729407, +STORE, 139790883692544, 139790883700735, +STORE, 139790877691904, 139790881488895, +SNULL, 139790877691904, 139790879350783, +STORE, 139790879350784, 139790881488895, +STORE, 139790877691904, 139790879350783, +SNULL, 139790881447935, 139790881488895, +STORE, 139790879350784, 139790881447935, +STORE, 139790881447936, 139790881488895, +SNULL, 139790881447936, 139790881472511, +STORE, 139790881472512, 139790881488895, +STORE, 139790881447936, 139790881472511, +ERASE, 139790881447936, 139790881472511, +STORE, 139790881447936, 139790881472511, +ERASE, 139790881472512, 139790881488895, +STORE, 139790881472512, 139790881488895, +SNULL, 139790881464319, 139790881472511, +STORE, 139790881447936, 139790881464319, +STORE, 139790881464320, 139790881472511, +SNULL, 93992875835391, 93992875839487, +STORE, 93992875827200, 93992875835391, +STORE, 93992875835392, 93992875839487, +SNULL, 139790883733503, 139790883737599, +STORE, 139790883729408, 139790883733503, +STORE, 139790883733504, 139790883737599, +ERASE, 139790883700736, 139790883729407, +STORE, 93992877031424, 93992877166591, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728550887424, 140737488351231, +SNULL, 140728550895615, 140737488351231, +STORE, 140728550887424, 140728550895615, +STORE, 140728550756352, 140728550895615, +STORE, 94707634077696, 94707636301823, +SNULL, 94707634188287, 94707636301823, +STORE, 94707634077696, 94707634188287, +STORE, 94707634188288, 94707636301823, +ERASE, 94707634188288, 94707636301823, +STORE, 94707636281344, 94707636293631, +STORE, 94707636293632, 94707636301823, +STORE, 140553545666560, 140553547919359, +SNULL, 140553545809919, 140553547919359, +STORE, 140553545666560, 140553545809919, +STORE, 140553545809920, 140553547919359, +ERASE, 140553545809920, 140553547919359, +STORE, 140553547907072, 140553547915263, +STORE, 140553547915264, 140553547919359, +STORE, 140728552374272, 140728552378367, +STORE, 140728552361984, 140728552374271, +STORE, 140553547878400, 140553547907071, +STORE, 140553547870208, 140553547878399, +STORE, 140553541869568, 140553545666559, +SNULL, 140553541869568, 140553543528447, +STORE, 140553543528448, 140553545666559, +STORE, 140553541869568, 140553543528447, +SNULL, 140553545625599, 140553545666559, +STORE, 140553543528448, 140553545625599, +STORE, 140553545625600, 140553545666559, +SNULL, 140553545625600, 140553545650175, +STORE, 140553545650176, 140553545666559, +STORE, 140553545625600, 140553545650175, +ERASE, 140553545625600, 140553545650175, +STORE, 140553545625600, 140553545650175, +ERASE, 140553545650176, 140553545666559, +STORE, 140553545650176, 140553545666559, +SNULL, 140553545641983, 140553545650175, +STORE, 140553545625600, 140553545641983, +STORE, 140553545641984, 140553545650175, +SNULL, 94707636289535, 94707636293631, +STORE, 94707636281344, 94707636289535, +STORE, 94707636289536, 94707636293631, +SNULL, 140553547911167, 140553547915263, +STORE, 140553547907072, 140553547911167, +STORE, 140553547911168, 140553547915263, +ERASE, 140553547878400, 140553547907071, +STORE, 94707651411968, 94707651547135, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732168695808, 140737488351231, +SNULL, 140732168703999, 140737488351231, +STORE, 140732168695808, 140732168703999, +STORE, 140732168564736, 140732168703999, +STORE, 94454287859712, 94454290083839, +SNULL, 94454287970303, 94454290083839, +STORE, 94454287859712, 94454287970303, +STORE, 94454287970304, 94454290083839, +ERASE, 94454287970304, 94454290083839, +STORE, 94454290063360, 94454290075647, +STORE, 94454290075648, 94454290083839, +STORE, 140564947107840, 140564949360639, +SNULL, 140564947251199, 140564949360639, +STORE, 140564947107840, 140564947251199, +STORE, 140564947251200, 140564949360639, +ERASE, 140564947251200, 140564949360639, +STORE, 140564949348352, 140564949356543, +STORE, 140564949356544, 140564949360639, +STORE, 140732168843264, 140732168847359, +STORE, 140732168830976, 140732168843263, +STORE, 140564949319680, 140564949348351, +STORE, 140564949311488, 140564949319679, +STORE, 140564943310848, 140564947107839, +SNULL, 140564943310848, 140564944969727, +STORE, 140564944969728, 140564947107839, +STORE, 140564943310848, 140564944969727, +SNULL, 140564947066879, 140564947107839, +STORE, 140564944969728, 140564947066879, +STORE, 140564947066880, 140564947107839, +SNULL, 140564947066880, 140564947091455, +STORE, 140564947091456, 140564947107839, +STORE, 140564947066880, 140564947091455, +ERASE, 140564947066880, 140564947091455, +STORE, 140564947066880, 140564947091455, +ERASE, 140564947091456, 140564947107839, +STORE, 140564947091456, 140564947107839, +SNULL, 140564947083263, 140564947091455, +STORE, 140564947066880, 140564947083263, +STORE, 140564947083264, 140564947091455, +SNULL, 94454290071551, 94454290075647, +STORE, 94454290063360, 94454290071551, +STORE, 94454290071552, 94454290075647, +SNULL, 140564949352447, 140564949356543, +STORE, 140564949348352, 140564949352447, +STORE, 140564949352448, 140564949356543, +ERASE, 140564949319680, 140564949348351, +STORE, 94454316236800, 94454316371967, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735155617792, 140737488351231, +SNULL, 140735155625983, 140737488351231, +STORE, 140735155617792, 140735155625983, +STORE, 140735155486720, 140735155625983, +STORE, 93915969556480, 93915971780607, +SNULL, 93915969667071, 93915971780607, +STORE, 93915969556480, 93915969667071, +STORE, 93915969667072, 93915971780607, +ERASE, 93915969667072, 93915971780607, +STORE, 93915971760128, 93915971772415, +STORE, 93915971772416, 93915971780607, +STORE, 140141164605440, 140141166858239, +SNULL, 140141164748799, 140141166858239, +STORE, 140141164605440, 140141164748799, +STORE, 140141164748800, 140141166858239, +ERASE, 140141164748800, 140141166858239, +STORE, 140141166845952, 140141166854143, +STORE, 140141166854144, 140141166858239, +STORE, 140735155691520, 140735155695615, +STORE, 140735155679232, 140735155691519, +STORE, 140141166817280, 140141166845951, +STORE, 140141166809088, 140141166817279, +STORE, 140141160808448, 140141164605439, +SNULL, 140141160808448, 140141162467327, +STORE, 140141162467328, 140141164605439, +STORE, 140141160808448, 140141162467327, +SNULL, 140141164564479, 140141164605439, +STORE, 140141162467328, 140141164564479, +STORE, 140141164564480, 140141164605439, +SNULL, 140141164564480, 140141164589055, +STORE, 140141164589056, 140141164605439, +STORE, 140141164564480, 140141164589055, +ERASE, 140141164564480, 140141164589055, +STORE, 140141164564480, 140141164589055, +ERASE, 140141164589056, 140141164605439, +STORE, 140141164589056, 140141164605439, +SNULL, 140141164580863, 140141164589055, +STORE, 140141164564480, 140141164580863, +STORE, 140141164580864, 140141164589055, +SNULL, 93915971768319, 93915971772415, +STORE, 93915971760128, 93915971768319, +STORE, 93915971768320, 93915971772415, +SNULL, 140141166850047, 140141166854143, +STORE, 140141166845952, 140141166850047, +STORE, 140141166850048, 140141166854143, +ERASE, 140141166817280, 140141166845951, +STORE, 93916002775040, 93916002910207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728988409856, 140737488351231, +SNULL, 140728988418047, 140737488351231, +STORE, 140728988409856, 140728988418047, +STORE, 140728988278784, 140728988418047, +STORE, 94021634813952, 94021637038079, +SNULL, 94021634924543, 94021637038079, +STORE, 94021634813952, 94021634924543, +STORE, 94021634924544, 94021637038079, +ERASE, 94021634924544, 94021637038079, +STORE, 94021637017600, 94021637029887, +STORE, 94021637029888, 94021637038079, +STORE, 140638014038016, 140638016290815, +SNULL, 140638014181375, 140638016290815, +STORE, 140638014038016, 140638014181375, +STORE, 140638014181376, 140638016290815, +ERASE, 140638014181376, 140638016290815, +STORE, 140638016278528, 140638016286719, +STORE, 140638016286720, 140638016290815, +STORE, 140728988536832, 140728988540927, +STORE, 140728988524544, 140728988536831, +STORE, 140638016249856, 140638016278527, +STORE, 140638016241664, 140638016249855, +STORE, 140638010241024, 140638014038015, +SNULL, 140638010241024, 140638011899903, +STORE, 140638011899904, 140638014038015, +STORE, 140638010241024, 140638011899903, +SNULL, 140638013997055, 140638014038015, +STORE, 140638011899904, 140638013997055, +STORE, 140638013997056, 140638014038015, +SNULL, 140638013997056, 140638014021631, +STORE, 140638014021632, 140638014038015, +STORE, 140638013997056, 140638014021631, +ERASE, 140638013997056, 140638014021631, +STORE, 140638013997056, 140638014021631, +ERASE, 140638014021632, 140638014038015, +STORE, 140638014021632, 140638014038015, +SNULL, 140638014013439, 140638014021631, +STORE, 140638013997056, 140638014013439, +STORE, 140638014013440, 140638014021631, +SNULL, 94021637025791, 94021637029887, +STORE, 94021637017600, 94021637025791, +STORE, 94021637025792, 94021637029887, +SNULL, 140638016282623, 140638016286719, +STORE, 140638016278528, 140638016282623, +STORE, 140638016282624, 140638016286719, +ERASE, 140638016249856, 140638016278527, +STORE, 94021643124736, 94021643259903, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731219275776, 140737488351231, +SNULL, 140731219283967, 140737488351231, +STORE, 140731219275776, 140731219283967, +STORE, 140731219144704, 140731219283967, +STORE, 93888803647488, 93888805871615, +SNULL, 93888803758079, 93888805871615, +STORE, 93888803647488, 93888803758079, +STORE, 93888803758080, 93888805871615, +ERASE, 93888803758080, 93888805871615, +STORE, 93888805851136, 93888805863423, +STORE, 93888805863424, 93888805871615, +STORE, 139630576934912, 139630579187711, +SNULL, 139630577078271, 139630579187711, +STORE, 139630576934912, 139630577078271, +STORE, 139630577078272, 139630579187711, +ERASE, 139630577078272, 139630579187711, +STORE, 139630579175424, 139630579183615, +STORE, 139630579183616, 139630579187711, +STORE, 140731219718144, 140731219722239, +STORE, 140731219705856, 140731219718143, +STORE, 139630579146752, 139630579175423, +STORE, 139630579138560, 139630579146751, +STORE, 139630573137920, 139630576934911, +SNULL, 139630573137920, 139630574796799, +STORE, 139630574796800, 139630576934911, +STORE, 139630573137920, 139630574796799, +SNULL, 139630576893951, 139630576934911, +STORE, 139630574796800, 139630576893951, +STORE, 139630576893952, 139630576934911, +SNULL, 139630576893952, 139630576918527, +STORE, 139630576918528, 139630576934911, +STORE, 139630576893952, 139630576918527, +ERASE, 139630576893952, 139630576918527, +STORE, 139630576893952, 139630576918527, +ERASE, 139630576918528, 139630576934911, +STORE, 139630576918528, 139630576934911, +SNULL, 139630576910335, 139630576918527, +STORE, 139630576893952, 139630576910335, +STORE, 139630576910336, 139630576918527, +SNULL, 93888805859327, 93888805863423, +STORE, 93888805851136, 93888805859327, +STORE, 93888805859328, 93888805863423, +SNULL, 139630579179519, 139630579183615, +STORE, 139630579175424, 139630579179519, +STORE, 139630579179520, 139630579183615, +ERASE, 139630579146752, 139630579175423, +STORE, 93888822235136, 93888822370303, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733391151104, 140737488351231, +SNULL, 140733391159295, 140737488351231, +STORE, 140733391151104, 140733391159295, +STORE, 140733391020032, 140733391159295, +STORE, 94393875324928, 94393877549055, +SNULL, 94393875435519, 94393877549055, +STORE, 94393875324928, 94393875435519, +STORE, 94393875435520, 94393877549055, +ERASE, 94393875435520, 94393877549055, +STORE, 94393877528576, 94393877540863, +STORE, 94393877540864, 94393877549055, +STORE, 140292111740928, 140292113993727, +SNULL, 140292111884287, 140292113993727, +STORE, 140292111740928, 140292111884287, +STORE, 140292111884288, 140292113993727, +ERASE, 140292111884288, 140292113993727, +STORE, 140292113981440, 140292113989631, +STORE, 140292113989632, 140292113993727, +STORE, 140733391532032, 140733391536127, +STORE, 140733391519744, 140733391532031, +STORE, 140292113952768, 140292113981439, +STORE, 140292113944576, 140292113952767, +STORE, 140292107943936, 140292111740927, +SNULL, 140292107943936, 140292109602815, +STORE, 140292109602816, 140292111740927, +STORE, 140292107943936, 140292109602815, +SNULL, 140292111699967, 140292111740927, +STORE, 140292109602816, 140292111699967, +STORE, 140292111699968, 140292111740927, +SNULL, 140292111699968, 140292111724543, +STORE, 140292111724544, 140292111740927, +STORE, 140292111699968, 140292111724543, +ERASE, 140292111699968, 140292111724543, +STORE, 140292111699968, 140292111724543, +ERASE, 140292111724544, 140292111740927, +STORE, 140292111724544, 140292111740927, +SNULL, 140292111716351, 140292111724543, +STORE, 140292111699968, 140292111716351, +STORE, 140292111716352, 140292111724543, +SNULL, 94393877536767, 94393877540863, +STORE, 94393877528576, 94393877536767, +STORE, 94393877536768, 94393877540863, +SNULL, 140292113985535, 140292113989631, +STORE, 140292113981440, 140292113985535, +STORE, 140292113985536, 140292113989631, +ERASE, 140292113952768, 140292113981439, +STORE, 94393909342208, 94393909477375, +STORE, 94458367512576, 94458367725567, +STORE, 94458369822720, 94458369826815, +STORE, 94458369826816, 94458369835007, +STORE, 94458369835008, 94458369847295, +STORE, 94458393292800, 94458399666175, +STORE, 140619773841408, 140619775500287, +STORE, 140619775500288, 140619777597439, +STORE, 140619777597440, 140619777613823, +STORE, 140619777613824, 140619777622015, +STORE, 140619777622016, 140619777638399, +STORE, 140619777638400, 140619777650687, +STORE, 140619777650688, 140619779743743, +STORE, 140619779743744, 140619779747839, +STORE, 140619779747840, 140619779751935, +STORE, 140619779751936, 140619779895295, +STORE, 140619780263936, 140619781947391, +STORE, 140619781947392, 140619781963775, +STORE, 140619781992448, 140619781996543, +STORE, 140619781996544, 140619782000639, +STORE, 140619782000640, 140619782004735, +STORE, 140725811675136, 140725811814399, +STORE, 140725812813824, 140725812826111, +STORE, 140725812826112, 140725812830207, +STORE, 94458367512576, 94458367725567, +STORE, 94458369822720, 94458369826815, +STORE, 94458369826816, 94458369835007, +STORE, 94458369835008, 94458369847295, +STORE, 94458393292800, 94458400366591, +STORE, 140619773841408, 140619775500287, +STORE, 140619775500288, 140619777597439, +STORE, 140619777597440, 140619777613823, +STORE, 140619777613824, 140619777622015, +STORE, 140619777622016, 140619777638399, +STORE, 140619777638400, 140619777650687, +STORE, 140619777650688, 140619779743743, +STORE, 140619779743744, 140619779747839, +STORE, 140619779747840, 140619779751935, +STORE, 140619779751936, 140619779895295, +STORE, 140619780263936, 140619781947391, +STORE, 140619781947392, 140619781963775, +STORE, 140619781992448, 140619781996543, +STORE, 140619781996544, 140619782000639, +STORE, 140619782000640, 140619782004735, +STORE, 140725811675136, 140725811814399, +STORE, 140725812813824, 140725812826111, +STORE, 140725812826112, 140725812830207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728740679680, 140737488351231, +SNULL, 140728740687871, 140737488351231, +STORE, 140728740679680, 140728740687871, +STORE, 140728740548608, 140728740687871, +STORE, 94764075249664, 94764077473791, +SNULL, 94764075360255, 94764077473791, +STORE, 94764075249664, 94764075360255, +STORE, 94764075360256, 94764077473791, +ERASE, 94764075360256, 94764077473791, +STORE, 94764077453312, 94764077465599, +STORE, 94764077465600, 94764077473791, +STORE, 139766406791168, 139766409043967, +SNULL, 139766406934527, 139766409043967, +STORE, 139766406791168, 139766406934527, +STORE, 139766406934528, 139766409043967, +ERASE, 139766406934528, 139766409043967, +STORE, 139766409031680, 139766409039871, +STORE, 139766409039872, 139766409043967, +STORE, 140728740913152, 140728740917247, +STORE, 140728740900864, 140728740913151, +STORE, 139766409003008, 139766409031679, +STORE, 139766408994816, 139766409003007, +STORE, 139766402994176, 139766406791167, +SNULL, 139766402994176, 139766404653055, +STORE, 139766404653056, 139766406791167, +STORE, 139766402994176, 139766404653055, +SNULL, 139766406750207, 139766406791167, +STORE, 139766404653056, 139766406750207, +STORE, 139766406750208, 139766406791167, +SNULL, 139766406750208, 139766406774783, +STORE, 139766406774784, 139766406791167, +STORE, 139766406750208, 139766406774783, +ERASE, 139766406750208, 139766406774783, +STORE, 139766406750208, 139766406774783, +ERASE, 139766406774784, 139766406791167, +STORE, 139766406774784, 139766406791167, +SNULL, 139766406766591, 139766406774783, +STORE, 139766406750208, 139766406766591, +STORE, 139766406766592, 139766406774783, +SNULL, 94764077461503, 94764077465599, +STORE, 94764077453312, 94764077461503, +STORE, 94764077461504, 94764077465599, +SNULL, 139766409035775, 139766409039871, +STORE, 139766409031680, 139766409035775, +STORE, 139766409035776, 139766409039871, +ERASE, 139766409003008, 139766409031679, +STORE, 94764090458112, 94764090593279, +STORE, 94758057480192, 94758057590783, +STORE, 94758059683840, 94758059692031, +STORE, 94758059692032, 94758059696127, +STORE, 94758059696128, 94758059704319, +STORE, 94758083215360, 94758083350527, +STORE, 139951456772096, 139951458430975, +STORE, 139951458430976, 139951460528127, +STORE, 139951460528128, 139951460544511, +STORE, 139951460544512, 139951460552703, +STORE, 139951460552704, 139951460569087, +STORE, 139951460569088, 139951460712447, +STORE, 139951462772736, 139951462780927, +STORE, 139951462809600, 139951462813695, +STORE, 139951462813696, 139951462817791, +STORE, 139951462817792, 139951462821887, +STORE, 140734098313216, 140734098452479, +STORE, 140734098911232, 140734098923519, +STORE, 140734098923520, 140734098927615, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724904095744, 140737488351231, +SNULL, 140724904103935, 140737488351231, +STORE, 140724904095744, 140724904103935, +STORE, 140724903964672, 140724904103935, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140408497864704, 140408500117503, +SNULL, 140408498008063, 140408500117503, +STORE, 140408497864704, 140408498008063, +STORE, 140408498008064, 140408500117503, +ERASE, 140408498008064, 140408500117503, +STORE, 140408500105216, 140408500113407, +STORE, 140408500113408, 140408500117503, +STORE, 140724905369600, 140724905373695, +STORE, 140724905357312, 140724905369599, +STORE, 140408500076544, 140408500105215, +STORE, 140408500068352, 140408500076543, +STORE, 140408494702592, 140408497864703, +SNULL, 140408494702592, 140408495763455, +STORE, 140408495763456, 140408497864703, +STORE, 140408494702592, 140408495763455, +SNULL, 140408497856511, 140408497864703, +STORE, 140408495763456, 140408497856511, +STORE, 140408497856512, 140408497864703, +ERASE, 140408497856512, 140408497864703, +STORE, 140408497856512, 140408497864703, +STORE, 140408490905600, 140408494702591, +SNULL, 140408490905600, 140408492564479, +STORE, 140408492564480, 140408494702591, +STORE, 140408490905600, 140408492564479, +SNULL, 140408494661631, 140408494702591, +STORE, 140408492564480, 140408494661631, +STORE, 140408494661632, 140408494702591, +SNULL, 140408494661632, 140408494686207, +STORE, 140408494686208, 140408494702591, +STORE, 140408494661632, 140408494686207, +ERASE, 140408494661632, 140408494686207, +STORE, 140408494661632, 140408494686207, +ERASE, 140408494686208, 140408494702591, +STORE, 140408494686208, 140408494702591, +STORE, 140408500056064, 140408500076543, +SNULL, 140408494678015, 140408494686207, +STORE, 140408494661632, 140408494678015, +STORE, 140408494678016, 140408494686207, +SNULL, 140408497860607, 140408497864703, +STORE, 140408497856512, 140408497860607, +STORE, 140408497860608, 140408497864703, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140408500109311, 140408500113407, +STORE, 140408500105216, 140408500109311, +STORE, 140408500109312, 140408500113407, +ERASE, 140408500076544, 140408500105215, +STORE, 25235456, 25370623, +STORE, 25235456, 25518079, +STORE, 140408498372608, 140408500056063, +STORE, 94543937388544, 94543937499135, +STORE, 94543939592192, 94543939600383, +STORE, 94543939600384, 94543939604479, +STORE, 94543939604480, 94543939612671, +STORE, 94543941447680, 94543941582847, +STORE, 140282621947904, 140282623606783, +STORE, 140282623606784, 140282625703935, +STORE, 140282625703936, 140282625720319, +STORE, 140282625720320, 140282625728511, +STORE, 140282625728512, 140282625744895, +STORE, 140282625744896, 140282625888255, +STORE, 140282627948544, 140282627956735, +STORE, 140282627985408, 140282627989503, +STORE, 140282627989504, 140282627993599, +STORE, 140282627993600, 140282627997695, +STORE, 140728295723008, 140728295862271, +STORE, 140728296476672, 140728296488959, +STORE, 140728296488960, 140728296493055, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431510691839, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431510691839, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140736944451584, 140737488351231, +SNULL, 140736944463871, 140737488351231, +STORE, 140736944451584, 140736944463871, +STORE, 140736944320512, 140736944463871, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 139693609893888, 139693612146687, +SNULL, 139693610037247, 139693612146687, +STORE, 139693609893888, 139693610037247, +STORE, 139693610037248, 139693612146687, +ERASE, 139693610037248, 139693612146687, +STORE, 139693612134400, 139693612142591, +STORE, 139693612142592, 139693612146687, +STORE, 140736945152000, 140736945156095, +STORE, 140736945139712, 140736945151999, +STORE, 139693612105728, 139693612134399, +STORE, 139693612097536, 139693612105727, +STORE, 139693606060032, 139693609893887, +SNULL, 139693606060032, 139693607768063, +STORE, 139693607768064, 139693609893887, +STORE, 139693606060032, 139693607768063, +SNULL, 139693609861119, 139693609893887, +STORE, 139693607768064, 139693609861119, +STORE, 139693609861120, 139693609893887, +ERASE, 139693609861120, 139693609893887, +STORE, 139693609861120, 139693609893887, +STORE, 139693603864576, 139693606060031, +SNULL, 139693603864576, 139693603958783, +STORE, 139693603958784, 139693606060031, +STORE, 139693603864576, 139693603958783, +SNULL, 139693606051839, 139693606060031, +STORE, 139693603958784, 139693606051839, +STORE, 139693606051840, 139693606060031, +ERASE, 139693606051840, 139693606060031, +STORE, 139693606051840, 139693606060031, +STORE, 139693601345536, 139693603864575, +SNULL, 139693601345536, 139693601759231, +STORE, 139693601759232, 139693603864575, +STORE, 139693601345536, 139693601759231, +SNULL, 139693603852287, 139693603864575, +STORE, 139693601759232, 139693603852287, +STORE, 139693603852288, 139693603864575, +ERASE, 139693603852288, 139693603864575, +STORE, 139693603852288, 139693603864575, +STORE, 139693598711808, 139693601345535, +SNULL, 139693598711808, 139693599240191, +STORE, 139693599240192, 139693601345535, +STORE, 139693598711808, 139693599240191, +SNULL, 139693601337343, 139693601345535, +STORE, 139693599240192, 139693601337343, +STORE, 139693601337344, 139693601345535, +ERASE, 139693601337344, 139693601345535, +STORE, 139693601337344, 139693601345535, +STORE, 139693596598272, 139693598711807, +SNULL, 139693596598272, 139693596610559, +STORE, 139693596610560, 139693598711807, +STORE, 139693596598272, 139693596610559, +SNULL, 139693598703615, 139693598711807, +STORE, 139693596610560, 139693598703615, +STORE, 139693598703616, 139693598711807, +ERASE, 139693598703616, 139693598711807, +STORE, 139693598703616, 139693598711807, +STORE, 139693594394624, 139693596598271, +SNULL, 139693594394624, 139693594497023, +STORE, 139693594497024, 139693596598271, +STORE, 139693594394624, 139693594497023, +SNULL, 139693596590079, 139693596598271, +STORE, 139693594497024, 139693596590079, +STORE, 139693596590080, 139693596598271, +ERASE, 139693596590080, 139693596598271, +STORE, 139693596590080, 139693596598271, +STORE, 139693612089344, 139693612105727, +STORE, 139693591232512, 139693594394623, +SNULL, 139693591232512, 139693592293375, +STORE, 139693592293376, 139693594394623, +STORE, 139693591232512, 139693592293375, +SNULL, 139693594386431, 139693594394623, +STORE, 139693592293376, 139693594386431, +STORE, 139693594386432, 139693594394623, +ERASE, 139693594386432, 139693594394623, +STORE, 139693594386432, 139693594394623, +STORE, 139693587435520, 139693591232511, +SNULL, 139693587435520, 139693589094399, +STORE, 139693589094400, 139693591232511, +STORE, 139693587435520, 139693589094399, +SNULL, 139693591191551, 139693591232511, +STORE, 139693589094400, 139693591191551, +STORE, 139693591191552, 139693591232511, +SNULL, 139693591191552, 139693591216127, +STORE, 139693591216128, 139693591232511, +STORE, 139693591191552, 139693591216127, +ERASE, 139693591191552, 139693591216127, +STORE, 139693591191552, 139693591216127, +ERASE, 139693591216128, 139693591232511, +STORE, 139693591216128, 139693591232511, +STORE, 139693612077056, 139693612105727, +SNULL, 139693591207935, 139693591216127, +STORE, 139693591191552, 139693591207935, +STORE, 139693591207936, 139693591216127, +SNULL, 139693594390527, 139693594394623, +STORE, 139693594386432, 139693594390527, +STORE, 139693594390528, 139693594394623, +SNULL, 139693596594175, 139693596598271, +STORE, 139693596590080, 139693596594175, +STORE, 139693596594176, 139693596598271, +SNULL, 139693598707711, 139693598711807, +STORE, 139693598703616, 139693598707711, +STORE, 139693598707712, 139693598711807, +SNULL, 139693601341439, 139693601345535, +STORE, 139693601337344, 139693601341439, +STORE, 139693601341440, 139693601345535, +SNULL, 139693603860479, 139693603864575, +STORE, 139693603852288, 139693603860479, +STORE, 139693603860480, 139693603864575, +SNULL, 139693606055935, 139693606060031, +STORE, 139693606051840, 139693606055935, +STORE, 139693606055936, 139693606060031, +SNULL, 139693609865215, 139693609893887, +STORE, 139693609861120, 139693609865215, +STORE, 139693609865216, 139693609893887, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 139693612138495, 139693612142591, +STORE, 139693612134400, 139693612138495, +STORE, 139693612138496, 139693612142591, +ERASE, 139693612105728, 139693612134399, +STORE, 39976960, 40112127, +STORE, 139693610393600, 139693612077055, +STORE, 139693612130304, 139693612134399, +STORE, 139693610258432, 139693610393599, +STORE, 39976960, 40255487, +STORE, 139693585338368, 139693587435519, +STORE, 139693612122112, 139693612134399, +STORE, 139693612113920, 139693612134399, +STORE, 139693612077056, 139693612113919, +STORE, 139693610242048, 139693610393599, +STORE, 39976960, 40390655, +STORE, 39976960, 40546303, +STORE, 139693610233856, 139693610393599, +STORE, 139693610225664, 139693610393599, +STORE, 39976960, 40714239, +STORE, 139693610209280, 139693610393599, +STORE, 39976960, 40861695, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431528759295, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140729993904128, 140737488351231, +SNULL, 140729993912319, 140737488351231, +STORE, 140729993904128, 140729993912319, +STORE, 140729993773056, 140729993912319, +STORE, 93926271991808, 93926274215935, +SNULL, 93926272102399, 93926274215935, +STORE, 93926271991808, 93926272102399, +STORE, 93926272102400, 93926274215935, +ERASE, 93926272102400, 93926274215935, +STORE, 93926274195456, 93926274207743, +STORE, 93926274207744, 93926274215935, +STORE, 139962167296000, 139962169548799, +SNULL, 139962167439359, 139962169548799, +STORE, 139962167296000, 139962167439359, +STORE, 139962167439360, 139962169548799, +ERASE, 139962167439360, 139962169548799, +STORE, 139962169536512, 139962169544703, +STORE, 139962169544704, 139962169548799, +STORE, 140729995096064, 140729995100159, +STORE, 140729995083776, 140729995096063, +STORE, 139962169507840, 139962169536511, +STORE, 139962169499648, 139962169507839, +STORE, 139962163499008, 139962167295999, +SNULL, 139962163499008, 139962165157887, +STORE, 139962165157888, 139962167295999, +STORE, 139962163499008, 139962165157887, +SNULL, 139962167255039, 139962167295999, +STORE, 139962165157888, 139962167255039, +STORE, 139962167255040, 139962167295999, +SNULL, 139962167255040, 139962167279615, +STORE, 139962167279616, 139962167295999, +STORE, 139962167255040, 139962167279615, +ERASE, 139962167255040, 139962167279615, +STORE, 139962167255040, 139962167279615, +ERASE, 139962167279616, 139962167295999, +STORE, 139962167279616, 139962167295999, +SNULL, 139962167271423, 139962167279615, +STORE, 139962167255040, 139962167271423, +STORE, 139962167271424, 139962167279615, +SNULL, 93926274203647, 93926274207743, +STORE, 93926274195456, 93926274203647, +STORE, 93926274203648, 93926274207743, +SNULL, 139962169540607, 139962169544703, +STORE, 139962169536512, 139962169540607, +STORE, 139962169540608, 139962169544703, +ERASE, 139962169507840, 139962169536511, +STORE, 93926291120128, 93926291255295, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724960579584, 140737488351231, +SNULL, 140724960587775, 140737488351231, +STORE, 140724960579584, 140724960587775, +STORE, 140724960448512, 140724960587775, +STORE, 94246489489408, 94246491713535, +SNULL, 94246489599999, 94246491713535, +STORE, 94246489489408, 94246489599999, +STORE, 94246489600000, 94246491713535, +ERASE, 94246489600000, 94246491713535, +STORE, 94246491693056, 94246491705343, +STORE, 94246491705344, 94246491713535, +STORE, 140098174926848, 140098177179647, +SNULL, 140098175070207, 140098177179647, +STORE, 140098174926848, 140098175070207, +STORE, 140098175070208, 140098177179647, +ERASE, 140098175070208, 140098177179647, +STORE, 140098177167360, 140098177175551, +STORE, 140098177175552, 140098177179647, +STORE, 140724961439744, 140724961443839, +STORE, 140724961427456, 140724961439743, +STORE, 140098177138688, 140098177167359, +STORE, 140098177130496, 140098177138687, +STORE, 140098171129856, 140098174926847, +SNULL, 140098171129856, 140098172788735, +STORE, 140098172788736, 140098174926847, +STORE, 140098171129856, 140098172788735, +SNULL, 140098174885887, 140098174926847, +STORE, 140098172788736, 140098174885887, +STORE, 140098174885888, 140098174926847, +SNULL, 140098174885888, 140098174910463, +STORE, 140098174910464, 140098174926847, +STORE, 140098174885888, 140098174910463, +ERASE, 140098174885888, 140098174910463, +STORE, 140098174885888, 140098174910463, +ERASE, 140098174910464, 140098174926847, +STORE, 140098174910464, 140098174926847, +SNULL, 140098174902271, 140098174910463, +STORE, 140098174885888, 140098174902271, +STORE, 140098174902272, 140098174910463, +SNULL, 94246491701247, 94246491705343, +STORE, 94246491693056, 94246491701247, +STORE, 94246491701248, 94246491705343, +SNULL, 140098177171455, 140098177175551, +STORE, 140098177167360, 140098177171455, +STORE, 140098177171456, 140098177175551, +ERASE, 140098177138688, 140098177167359, +STORE, 94246516998144, 94246517133311, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730522918912, 140737488351231, +SNULL, 140730522927103, 140737488351231, +STORE, 140730522918912, 140730522927103, +STORE, 140730522787840, 140730522927103, +STORE, 94196043120640, 94196045344767, +SNULL, 94196043231231, 94196045344767, +STORE, 94196043120640, 94196043231231, +STORE, 94196043231232, 94196045344767, +ERASE, 94196043231232, 94196045344767, +STORE, 94196045324288, 94196045336575, +STORE, 94196045336576, 94196045344767, +STORE, 139815918940160, 139815921192959, +SNULL, 139815919083519, 139815921192959, +STORE, 139815918940160, 139815919083519, +STORE, 139815919083520, 139815921192959, +ERASE, 139815919083520, 139815921192959, +STORE, 139815921180672, 139815921188863, +STORE, 139815921188864, 139815921192959, +STORE, 140730523344896, 140730523348991, +STORE, 140730523332608, 140730523344895, +STORE, 139815921152000, 139815921180671, +STORE, 139815921143808, 139815921151999, +STORE, 139815915143168, 139815918940159, +SNULL, 139815915143168, 139815916802047, +STORE, 139815916802048, 139815918940159, +STORE, 139815915143168, 139815916802047, +SNULL, 139815918899199, 139815918940159, +STORE, 139815916802048, 139815918899199, +STORE, 139815918899200, 139815918940159, +SNULL, 139815918899200, 139815918923775, +STORE, 139815918923776, 139815918940159, +STORE, 139815918899200, 139815918923775, +ERASE, 139815918899200, 139815918923775, +STORE, 139815918899200, 139815918923775, +ERASE, 139815918923776, 139815918940159, +STORE, 139815918923776, 139815918940159, +SNULL, 139815918915583, 139815918923775, +STORE, 139815918899200, 139815918915583, +STORE, 139815918915584, 139815918923775, +SNULL, 94196045332479, 94196045336575, +STORE, 94196045324288, 94196045332479, +STORE, 94196045332480, 94196045336575, +SNULL, 139815921184767, 139815921188863, +STORE, 139815921180672, 139815921184767, +STORE, 139815921184768, 139815921188863, +ERASE, 139815921152000, 139815921180671, +STORE, 94196076183552, 94196076318719, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722460393472, 140737488351231, +SNULL, 140722460401663, 140737488351231, +STORE, 140722460393472, 140722460401663, +STORE, 140722460262400, 140722460401663, +STORE, 94569810399232, 94569812623359, +SNULL, 94569810509823, 94569812623359, +STORE, 94569810399232, 94569810509823, +STORE, 94569810509824, 94569812623359, +ERASE, 94569810509824, 94569812623359, +STORE, 94569812602880, 94569812615167, +STORE, 94569812615168, 94569812623359, +STORE, 139681565450240, 139681567703039, +SNULL, 139681565593599, 139681567703039, +STORE, 139681565450240, 139681565593599, +STORE, 139681565593600, 139681567703039, +ERASE, 139681565593600, 139681567703039, +STORE, 139681567690752, 139681567698943, +STORE, 139681567698944, 139681567703039, +STORE, 140722460569600, 140722460573695, +STORE, 140722460557312, 140722460569599, +STORE, 139681567662080, 139681567690751, +STORE, 139681567653888, 139681567662079, +STORE, 139681561653248, 139681565450239, +SNULL, 139681561653248, 139681563312127, +STORE, 139681563312128, 139681565450239, +STORE, 139681561653248, 139681563312127, +SNULL, 139681565409279, 139681565450239, +STORE, 139681563312128, 139681565409279, +STORE, 139681565409280, 139681565450239, +SNULL, 139681565409280, 139681565433855, +STORE, 139681565433856, 139681565450239, +STORE, 139681565409280, 139681565433855, +ERASE, 139681565409280, 139681565433855, +STORE, 139681565409280, 139681565433855, +ERASE, 139681565433856, 139681565450239, +STORE, 139681565433856, 139681565450239, +SNULL, 139681565425663, 139681565433855, +STORE, 139681565409280, 139681565425663, +STORE, 139681565425664, 139681565433855, +SNULL, 94569812611071, 94569812615167, +STORE, 94569812602880, 94569812611071, +STORE, 94569812611072, 94569812615167, +SNULL, 139681567694847, 139681567698943, +STORE, 139681567690752, 139681567694847, +STORE, 139681567694848, 139681567698943, +ERASE, 139681567662080, 139681567690751, +STORE, 94569818066944, 94569818202111, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431534280703, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725452365824, 140737488351231, +SNULL, 140725452374015, 140737488351231, +STORE, 140725452365824, 140725452374015, +STORE, 140725452234752, 140725452374015, +STORE, 94395067465728, 94395069689855, +SNULL, 94395067576319, 94395069689855, +STORE, 94395067465728, 94395067576319, +STORE, 94395067576320, 94395069689855, +ERASE, 94395067576320, 94395069689855, +STORE, 94395069669376, 94395069681663, +STORE, 94395069681664, 94395069689855, +STORE, 140269941211136, 140269943463935, +SNULL, 140269941354495, 140269943463935, +STORE, 140269941211136, 140269941354495, +STORE, 140269941354496, 140269943463935, +ERASE, 140269941354496, 140269943463935, +STORE, 140269943451648, 140269943459839, +STORE, 140269943459840, 140269943463935, +STORE, 140725452558336, 140725452562431, +STORE, 140725452546048, 140725452558335, +STORE, 140269943422976, 140269943451647, +STORE, 140269943414784, 140269943422975, +STORE, 140269937414144, 140269941211135, +SNULL, 140269937414144, 140269939073023, +STORE, 140269939073024, 140269941211135, +STORE, 140269937414144, 140269939073023, +SNULL, 140269941170175, 140269941211135, +STORE, 140269939073024, 140269941170175, +STORE, 140269941170176, 140269941211135, +SNULL, 140269941170176, 140269941194751, +STORE, 140269941194752, 140269941211135, +STORE, 140269941170176, 140269941194751, +ERASE, 140269941170176, 140269941194751, +STORE, 140269941170176, 140269941194751, +ERASE, 140269941194752, 140269941211135, +STORE, 140269941194752, 140269941211135, +SNULL, 140269941186559, 140269941194751, +STORE, 140269941170176, 140269941186559, +STORE, 140269941186560, 140269941194751, +SNULL, 94395069677567, 94395069681663, +STORE, 94395069669376, 94395069677567, +STORE, 94395069677568, 94395069681663, +SNULL, 140269943455743, 140269943459839, +STORE, 140269943451648, 140269943455743, +STORE, 140269943455744, 140269943459839, +ERASE, 140269943422976, 140269943451647, +STORE, 94395101691904, 94395101827071, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733860118528, 140737488351231, +SNULL, 140733860126719, 140737488351231, +STORE, 140733860118528, 140733860126719, +STORE, 140733859987456, 140733860126719, +STORE, 94484752990208, 94484755214335, +SNULL, 94484753100799, 94484755214335, +STORE, 94484752990208, 94484753100799, +STORE, 94484753100800, 94484755214335, +ERASE, 94484753100800, 94484755214335, +STORE, 94484755193856, 94484755206143, +STORE, 94484755206144, 94484755214335, +STORE, 139958922309632, 139958924562431, +SNULL, 139958922452991, 139958924562431, +STORE, 139958922309632, 139958922452991, +STORE, 139958922452992, 139958924562431, +ERASE, 139958922452992, 139958924562431, +STORE, 139958924550144, 139958924558335, +STORE, 139958924558336, 139958924562431, +STORE, 140733860253696, 140733860257791, +STORE, 140733860241408, 140733860253695, +STORE, 139958924521472, 139958924550143, +STORE, 139958924513280, 139958924521471, +STORE, 139958918512640, 139958922309631, +SNULL, 139958918512640, 139958920171519, +STORE, 139958920171520, 139958922309631, +STORE, 139958918512640, 139958920171519, +SNULL, 139958922268671, 139958922309631, +STORE, 139958920171520, 139958922268671, +STORE, 139958922268672, 139958922309631, +SNULL, 139958922268672, 139958922293247, +STORE, 139958922293248, 139958922309631, +STORE, 139958922268672, 139958922293247, +ERASE, 139958922268672, 139958922293247, +STORE, 139958922268672, 139958922293247, +ERASE, 139958922293248, 139958922309631, +STORE, 139958922293248, 139958922309631, +SNULL, 139958922285055, 139958922293247, +STORE, 139958922268672, 139958922285055, +STORE, 139958922285056, 139958922293247, +SNULL, 94484755202047, 94484755206143, +STORE, 94484755193856, 94484755202047, +STORE, 94484755202048, 94484755206143, +SNULL, 139958924554239, 139958924558335, +STORE, 139958924550144, 139958924554239, +STORE, 139958924554240, 139958924558335, +ERASE, 139958924521472, 139958924550143, +STORE, 94484777615360, 94484777750527, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731051036672, 140737488351231, +SNULL, 140731051044863, 140737488351231, +STORE, 140731051036672, 140731051044863, +STORE, 140731050905600, 140731051044863, +STORE, 93945822998528, 93945825222655, +SNULL, 93945823109119, 93945825222655, +STORE, 93945822998528, 93945823109119, +STORE, 93945823109120, 93945825222655, +ERASE, 93945823109120, 93945825222655, +STORE, 93945825202176, 93945825214463, +STORE, 93945825214464, 93945825222655, +STORE, 140153503997952, 140153506250751, +SNULL, 140153504141311, 140153506250751, +STORE, 140153503997952, 140153504141311, +STORE, 140153504141312, 140153506250751, +ERASE, 140153504141312, 140153506250751, +STORE, 140153506238464, 140153506246655, +STORE, 140153506246656, 140153506250751, +STORE, 140731051331584, 140731051335679, +STORE, 140731051319296, 140731051331583, +STORE, 140153506209792, 140153506238463, +STORE, 140153506201600, 140153506209791, +STORE, 140153500200960, 140153503997951, +SNULL, 140153500200960, 140153501859839, +STORE, 140153501859840, 140153503997951, +STORE, 140153500200960, 140153501859839, +SNULL, 140153503956991, 140153503997951, +STORE, 140153501859840, 140153503956991, +STORE, 140153503956992, 140153503997951, +SNULL, 140153503956992, 140153503981567, +STORE, 140153503981568, 140153503997951, +STORE, 140153503956992, 140153503981567, +ERASE, 140153503956992, 140153503981567, +STORE, 140153503956992, 140153503981567, +ERASE, 140153503981568, 140153503997951, +STORE, 140153503981568, 140153503997951, +SNULL, 140153503973375, 140153503981567, +STORE, 140153503956992, 140153503973375, +STORE, 140153503973376, 140153503981567, +SNULL, 93945825210367, 93945825214463, +STORE, 93945825202176, 93945825210367, +STORE, 93945825210368, 93945825214463, +SNULL, 140153506242559, 140153506246655, +STORE, 140153506238464, 140153506242559, +STORE, 140153506242560, 140153506246655, +ERASE, 140153506209792, 140153506238463, +STORE, 93945854537728, 93945854672895, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431537885183, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736025325568, 140737488351231, +SNULL, 140736025333759, 140737488351231, +STORE, 140736025325568, 140736025333759, +STORE, 140736025194496, 140736025333759, +STORE, 94809095172096, 94809097396223, +SNULL, 94809095282687, 94809097396223, +STORE, 94809095172096, 94809095282687, +STORE, 94809095282688, 94809097396223, +ERASE, 94809095282688, 94809097396223, +STORE, 94809097375744, 94809097388031, +STORE, 94809097388032, 94809097396223, +STORE, 140194992517120, 140194994769919, +SNULL, 140194992660479, 140194994769919, +STORE, 140194992517120, 140194992660479, +STORE, 140194992660480, 140194994769919, +ERASE, 140194992660480, 140194994769919, +STORE, 140194994757632, 140194994765823, +STORE, 140194994765824, 140194994769919, +STORE, 140736026173440, 140736026177535, +STORE, 140736026161152, 140736026173439, +STORE, 140194994728960, 140194994757631, +STORE, 140194994720768, 140194994728959, +STORE, 140194988720128, 140194992517119, +SNULL, 140194988720128, 140194990379007, +STORE, 140194990379008, 140194992517119, +STORE, 140194988720128, 140194990379007, +SNULL, 140194992476159, 140194992517119, +STORE, 140194990379008, 140194992476159, +STORE, 140194992476160, 140194992517119, +SNULL, 140194992476160, 140194992500735, +STORE, 140194992500736, 140194992517119, +STORE, 140194992476160, 140194992500735, +ERASE, 140194992476160, 140194992500735, +STORE, 140194992476160, 140194992500735, +ERASE, 140194992500736, 140194992517119, +STORE, 140194992500736, 140194992517119, +SNULL, 140194992492543, 140194992500735, +STORE, 140194992476160, 140194992492543, +STORE, 140194992492544, 140194992500735, +SNULL, 94809097383935, 94809097388031, +STORE, 94809097375744, 94809097383935, +STORE, 94809097383936, 94809097388031, +SNULL, 140194994761727, 140194994765823, +STORE, 140194994757632, 140194994761727, +STORE, 140194994761728, 140194994765823, +ERASE, 140194994728960, 140194994757631, +STORE, 94809124286464, 94809124421631, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140726342660096, 140737488351231, +SNULL, 140726342668287, 140737488351231, +STORE, 140726342660096, 140726342668287, +STORE, 140726342529024, 140726342668287, +STORE, 94140331462656, 94140333686783, +SNULL, 94140331573247, 94140333686783, +STORE, 94140331462656, 94140331573247, +STORE, 94140331573248, 94140333686783, +ERASE, 94140331573248, 94140333686783, +STORE, 94140333666304, 94140333678591, +STORE, 94140333678592, 94140333686783, +STORE, 140714077208576, 140714079461375, +SNULL, 140714077351935, 140714079461375, +STORE, 140714077208576, 140714077351935, +STORE, 140714077351936, 140714079461375, +ERASE, 140714077351936, 140714079461375, +STORE, 140714079449088, 140714079457279, +STORE, 140714079457280, 140714079461375, +STORE, 140726343933952, 140726343938047, +STORE, 140726343921664, 140726343933951, +STORE, 140714079420416, 140714079449087, +STORE, 140714079412224, 140714079420415, +STORE, 140714073411584, 140714077208575, +SNULL, 140714073411584, 140714075070463, +STORE, 140714075070464, 140714077208575, +STORE, 140714073411584, 140714075070463, +SNULL, 140714077167615, 140714077208575, +STORE, 140714075070464, 140714077167615, +STORE, 140714077167616, 140714077208575, +SNULL, 140714077167616, 140714077192191, +STORE, 140714077192192, 140714077208575, +STORE, 140714077167616, 140714077192191, +ERASE, 140714077167616, 140714077192191, +STORE, 140714077167616, 140714077192191, +ERASE, 140714077192192, 140714077208575, +STORE, 140714077192192, 140714077208575, +SNULL, 140714077183999, 140714077192191, +STORE, 140714077167616, 140714077183999, +STORE, 140714077184000, 140714077192191, +SNULL, 94140333674495, 94140333678591, +STORE, 94140333666304, 94140333674495, +STORE, 94140333674496, 94140333678591, +SNULL, 140714079453183, 140714079457279, +STORE, 140714079449088, 140714079453183, +STORE, 140714079453184, 140714079457279, +ERASE, 140714079420416, 140714079449087, +STORE, 94140341432320, 94140341567487, +STORE, 94431504838656, 94431505051647, +STORE, 94431507148800, 94431507152895, +STORE, 94431507152896, 94431507161087, +STORE, 94431507161088, 94431507173375, +STORE, 94431510286336, 94431539601407, +STORE, 139818797948928, 139818799607807, +STORE, 139818799607808, 139818801704959, +STORE, 139818801704960, 139818801721343, +STORE, 139818801721344, 139818801729535, +STORE, 139818801729536, 139818801745919, +STORE, 139818801745920, 139818801758207, +STORE, 139818801758208, 139818803851263, +STORE, 139818803851264, 139818803855359, +STORE, 139818803855360, 139818803859455, +STORE, 139818803859456, 139818804002815, +STORE, 139818804371456, 139818806054911, +STORE, 139818806054912, 139818806071295, +STORE, 139818806099968, 139818806104063, +STORE, 139818806104064, 139818806108159, +STORE, 139818806108160, 139818806112255, +STORE, 140731430457344, 140731430596607, +STORE, 140731431227392, 140731431239679, +STORE, 140731431239680, 140731431243775, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725843607552, 140737488351231, +SNULL, 140725843615743, 140737488351231, +STORE, 140725843607552, 140725843615743, +STORE, 140725843476480, 140725843615743, +STORE, 94889043505152, 94889045839871, +SNULL, 94889043718143, 94889045839871, +STORE, 94889043505152, 94889043718143, +STORE, 94889043718144, 94889045839871, +ERASE, 94889043718144, 94889045839871, +STORE, 94889045815296, 94889045827583, +STORE, 94889045827584, 94889045839871, +STORE, 140250965946368, 140250968199167, +SNULL, 140250966089727, 140250968199167, +STORE, 140250965946368, 140250966089727, +STORE, 140250966089728, 140250968199167, +ERASE, 140250966089728, 140250968199167, +STORE, 140250968186880, 140250968195071, +STORE, 140250968195072, 140250968199167, +STORE, 140725844500480, 140725844504575, +STORE, 140725844488192, 140725844500479, +STORE, 140250968158208, 140250968186879, +STORE, 140250968150016, 140250968158207, +STORE, 140250963832832, 140250965946367, +SNULL, 140250963832832, 140250963845119, +STORE, 140250963845120, 140250965946367, +STORE, 140250963832832, 140250963845119, +SNULL, 140250965938175, 140250965946367, +STORE, 140250963845120, 140250965938175, +STORE, 140250965938176, 140250965946367, +ERASE, 140250965938176, 140250965946367, +STORE, 140250965938176, 140250965946367, +STORE, 140250960035840, 140250963832831, +SNULL, 140250960035840, 140250961694719, +STORE, 140250961694720, 140250963832831, +STORE, 140250960035840, 140250961694719, +SNULL, 140250963791871, 140250963832831, +STORE, 140250961694720, 140250963791871, +STORE, 140250963791872, 140250963832831, +SNULL, 140250963791872, 140250963816447, +STORE, 140250963816448, 140250963832831, +STORE, 140250963791872, 140250963816447, +ERASE, 140250963791872, 140250963816447, +STORE, 140250963791872, 140250963816447, +ERASE, 140250963816448, 140250963832831, +STORE, 140250963816448, 140250963832831, +STORE, 140250968141824, 140250968158207, +SNULL, 140250963808255, 140250963816447, +STORE, 140250963791872, 140250963808255, +STORE, 140250963808256, 140250963816447, +SNULL, 140250965942271, 140250965946367, +STORE, 140250965938176, 140250965942271, +STORE, 140250965942272, 140250965946367, +SNULL, 94889045819391, 94889045827583, +STORE, 94889045815296, 94889045819391, +STORE, 94889045819392, 94889045827583, +SNULL, 140250968190975, 140250968195071, +STORE, 140250968186880, 140250968190975, +STORE, 140250968190976, 140250968195071, +ERASE, 140250968158208, 140250968186879, +STORE, 94889052213248, 94889052348415, +STORE, 140250966458368, 140250968141823, +STORE, 94889052213248, 94889052483583, +STORE, 94889052213248, 94889052618751, +STORE, 94170851819520, 94170852032511, +STORE, 94170854129664, 94170854133759, +STORE, 94170854133760, 94170854141951, +STORE, 94170854141952, 94170854154239, +STORE, 94170866515968, 94170867740671, +STORE, 140062030422016, 140062032080895, +STORE, 140062032080896, 140062034178047, +STORE, 140062034178048, 140062034194431, +STORE, 140062034194432, 140062034202623, +STORE, 140062034202624, 140062034219007, +STORE, 140062034219008, 140062034231295, +STORE, 140062034231296, 140062036324351, +STORE, 140062036324352, 140062036328447, +STORE, 140062036328448, 140062036332543, +STORE, 140062036332544, 140062036475903, +STORE, 140062036844544, 140062038527999, +STORE, 140062038528000, 140062038544383, +STORE, 140062038573056, 140062038577151, +STORE, 140062038577152, 140062038581247, +STORE, 140062038581248, 140062038585343, +STORE, 140736210550784, 140736210690047, +STORE, 140736210759680, 140736210771967, +STORE, 140736210771968, 140736210776063, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724272365568, 140737488351231, +SNULL, 140724272373759, 140737488351231, +STORE, 140724272365568, 140724272373759, +STORE, 140724272234496, 140724272373759, +STORE, 94607711965184, 94607714189311, +SNULL, 94607712075775, 94607714189311, +STORE, 94607711965184, 94607712075775, +STORE, 94607712075776, 94607714189311, +ERASE, 94607712075776, 94607714189311, +STORE, 94607714168832, 94607714181119, +STORE, 94607714181120, 94607714189311, +STORE, 140054949253120, 140054951505919, +SNULL, 140054949396479, 140054951505919, +STORE, 140054949253120, 140054949396479, +STORE, 140054949396480, 140054951505919, +ERASE, 140054949396480, 140054951505919, +STORE, 140054951493632, 140054951501823, +STORE, 140054951501824, 140054951505919, +STORE, 140724272992256, 140724272996351, +STORE, 140724272979968, 140724272992255, +STORE, 140054951464960, 140054951493631, +STORE, 140054951456768, 140054951464959, +STORE, 140054945456128, 140054949253119, +SNULL, 140054945456128, 140054947115007, +STORE, 140054947115008, 140054949253119, +STORE, 140054945456128, 140054947115007, +SNULL, 140054949212159, 140054949253119, +STORE, 140054947115008, 140054949212159, +STORE, 140054949212160, 140054949253119, +SNULL, 140054949212160, 140054949236735, +STORE, 140054949236736, 140054949253119, +STORE, 140054949212160, 140054949236735, +ERASE, 140054949212160, 140054949236735, +STORE, 140054949212160, 140054949236735, +ERASE, 140054949236736, 140054949253119, +STORE, 140054949236736, 140054949253119, +SNULL, 140054949228543, 140054949236735, +STORE, 140054949212160, 140054949228543, +STORE, 140054949228544, 140054949236735, +SNULL, 94607714177023, 94607714181119, +STORE, 94607714168832, 94607714177023, +STORE, 94607714177024, 94607714181119, +SNULL, 140054951497727, 140054951501823, +STORE, 140054951493632, 140054951497727, +STORE, 140054951497728, 140054951501823, +ERASE, 140054951464960, 140054951493631, +STORE, 94607733374976, 94607733510143, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733586923520, 140737488351231, +SNULL, 140733586931711, 140737488351231, +STORE, 140733586923520, 140733586931711, +STORE, 140733586792448, 140733586931711, +STORE, 93901634904064, 93901637128191, +SNULL, 93901635014655, 93901637128191, +STORE, 93901634904064, 93901635014655, +STORE, 93901635014656, 93901637128191, +ERASE, 93901635014656, 93901637128191, +STORE, 93901637107712, 93901637119999, +STORE, 93901637120000, 93901637128191, +STORE, 140086104784896, 140086107037695, +SNULL, 140086104928255, 140086107037695, +STORE, 140086104784896, 140086104928255, +STORE, 140086104928256, 140086107037695, +ERASE, 140086104928256, 140086107037695, +STORE, 140086107025408, 140086107033599, +STORE, 140086107033600, 140086107037695, +STORE, 140733587263488, 140733587267583, +STORE, 140733587251200, 140733587263487, +STORE, 140086106996736, 140086107025407, +STORE, 140086106988544, 140086106996735, +STORE, 140086100987904, 140086104784895, +SNULL, 140086100987904, 140086102646783, +STORE, 140086102646784, 140086104784895, +STORE, 140086100987904, 140086102646783, +SNULL, 140086104743935, 140086104784895, +STORE, 140086102646784, 140086104743935, +STORE, 140086104743936, 140086104784895, +SNULL, 140086104743936, 140086104768511, +STORE, 140086104768512, 140086104784895, +STORE, 140086104743936, 140086104768511, +ERASE, 140086104743936, 140086104768511, +STORE, 140086104743936, 140086104768511, +ERASE, 140086104768512, 140086104784895, +STORE, 140086104768512, 140086104784895, +SNULL, 140086104760319, 140086104768511, +STORE, 140086104743936, 140086104760319, +STORE, 140086104760320, 140086104768511, +SNULL, 93901637115903, 93901637119999, +STORE, 93901637107712, 93901637115903, +STORE, 93901637115904, 93901637119999, +SNULL, 140086107029503, 140086107033599, +STORE, 140086107025408, 140086107029503, +STORE, 140086107029504, 140086107033599, +ERASE, 140086106996736, 140086107025407, +STORE, 93901662715904, 93901662851071, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723365613568, 140737488351231, +SNULL, 140723365621759, 140737488351231, +STORE, 140723365613568, 140723365621759, +STORE, 140723365482496, 140723365621759, +STORE, 94759193546752, 94759195770879, +SNULL, 94759193657343, 94759195770879, +STORE, 94759193546752, 94759193657343, +STORE, 94759193657344, 94759195770879, +ERASE, 94759193657344, 94759195770879, +STORE, 94759195750400, 94759195762687, +STORE, 94759195762688, 94759195770879, +STORE, 140607636246528, 140607638499327, +SNULL, 140607636389887, 140607638499327, +STORE, 140607636246528, 140607636389887, +STORE, 140607636389888, 140607638499327, +ERASE, 140607636389888, 140607638499327, +STORE, 140607638487040, 140607638495231, +STORE, 140607638495232, 140607638499327, +STORE, 140723365900288, 140723365904383, +STORE, 140723365888000, 140723365900287, +STORE, 140607638458368, 140607638487039, +STORE, 140607638450176, 140607638458367, +STORE, 140607632449536, 140607636246527, +SNULL, 140607632449536, 140607634108415, +STORE, 140607634108416, 140607636246527, +STORE, 140607632449536, 140607634108415, +SNULL, 140607636205567, 140607636246527, +STORE, 140607634108416, 140607636205567, +STORE, 140607636205568, 140607636246527, +SNULL, 140607636205568, 140607636230143, +STORE, 140607636230144, 140607636246527, +STORE, 140607636205568, 140607636230143, +ERASE, 140607636205568, 140607636230143, +STORE, 140607636205568, 140607636230143, +ERASE, 140607636230144, 140607636246527, +STORE, 140607636230144, 140607636246527, +SNULL, 140607636221951, 140607636230143, +STORE, 140607636205568, 140607636221951, +STORE, 140607636221952, 140607636230143, +SNULL, 94759195758591, 94759195762687, +STORE, 94759195750400, 94759195758591, +STORE, 94759195758592, 94759195762687, +SNULL, 140607638491135, 140607638495231, +STORE, 140607638487040, 140607638491135, +STORE, 140607638491136, 140607638495231, +ERASE, 140607638458368, 140607638487039, +STORE, 94759204995072, 94759205130239, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732503789568, 140737488351231, +SNULL, 140732503797759, 140737488351231, +STORE, 140732503789568, 140732503797759, +STORE, 140732503658496, 140732503797759, +STORE, 94077792956416, 94077795180543, +SNULL, 94077793067007, 94077795180543, +STORE, 94077792956416, 94077793067007, +STORE, 94077793067008, 94077795180543, +ERASE, 94077793067008, 94077795180543, +STORE, 94077795160064, 94077795172351, +STORE, 94077795172352, 94077795180543, +STORE, 140359874252800, 140359876505599, +SNULL, 140359874396159, 140359876505599, +STORE, 140359874252800, 140359874396159, +STORE, 140359874396160, 140359876505599, +ERASE, 140359874396160, 140359876505599, +STORE, 140359876493312, 140359876501503, +STORE, 140359876501504, 140359876505599, +STORE, 140732504465408, 140732504469503, +STORE, 140732504453120, 140732504465407, +STORE, 140359876464640, 140359876493311, +STORE, 140359876456448, 140359876464639, +STORE, 140359870455808, 140359874252799, +SNULL, 140359870455808, 140359872114687, +STORE, 140359872114688, 140359874252799, +STORE, 140359870455808, 140359872114687, +SNULL, 140359874211839, 140359874252799, +STORE, 140359872114688, 140359874211839, +STORE, 140359874211840, 140359874252799, +SNULL, 140359874211840, 140359874236415, +STORE, 140359874236416, 140359874252799, +STORE, 140359874211840, 140359874236415, +ERASE, 140359874211840, 140359874236415, +STORE, 140359874211840, 140359874236415, +ERASE, 140359874236416, 140359874252799, +STORE, 140359874236416, 140359874252799, +SNULL, 140359874228223, 140359874236415, +STORE, 140359874211840, 140359874228223, +STORE, 140359874228224, 140359874236415, +SNULL, 94077795168255, 94077795172351, +STORE, 94077795160064, 94077795168255, +STORE, 94077795168256, 94077795172351, +SNULL, 140359876497407, 140359876501503, +STORE, 140359876493312, 140359876497407, +STORE, 140359876497408, 140359876501503, +ERASE, 140359876464640, 140359876493311, +STORE, 94077808717824, 94077808852991, +STORE, 94549486252032, 94549486465023, +STORE, 94549488562176, 94549488566271, +STORE, 94549488566272, 94549488574463, +STORE, 94549488574464, 94549488586751, +STORE, 94549503492096, 94549506121727, +STORE, 140085800894464, 140085802553343, +STORE, 140085802553344, 140085804650495, +STORE, 140085804650496, 140085804666879, +STORE, 140085804666880, 140085804675071, +STORE, 140085804675072, 140085804691455, +STORE, 140085804691456, 140085804703743, +STORE, 140085804703744, 140085806796799, +STORE, 140085806796800, 140085806800895, +STORE, 140085806800896, 140085806804991, +STORE, 140085806804992, 140085806948351, +STORE, 140085807316992, 140085809000447, +STORE, 140085809000448, 140085809016831, +STORE, 140085809045504, 140085809049599, +STORE, 140085809049600, 140085809053695, +STORE, 140085809053696, 140085809057791, +STORE, 140731810545664, 140731810684927, +STORE, 140731810967552, 140731810979839, +STORE, 140731810979840, 140731810983935, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724752330752, 140737488351231, +SNULL, 140724752338943, 140737488351231, +STORE, 140724752330752, 140724752338943, +STORE, 140724752199680, 140724752338943, +STORE, 94656357539840, 94656359874559, +SNULL, 94656357752831, 94656359874559, +STORE, 94656357539840, 94656357752831, +STORE, 94656357752832, 94656359874559, +ERASE, 94656357752832, 94656359874559, +STORE, 94656359849984, 94656359862271, +STORE, 94656359862272, 94656359874559, +STORE, 139632585203712, 139632587456511, +SNULL, 139632585347071, 139632587456511, +STORE, 139632585203712, 139632585347071, +STORE, 139632585347072, 139632587456511, +ERASE, 139632585347072, 139632587456511, +STORE, 139632587444224, 139632587452415, +STORE, 139632587452416, 139632587456511, +STORE, 139632587440128, 139632587444223, +STORE, 139632587427840, 139632587440127, +STORE, 139632587399168, 139632587427839, +STORE, 139632587390976, 139632587399167, +STORE, 139632583090176, 139632585203711, +SNULL, 139632583090176, 139632583102463, +STORE, 139632583102464, 139632585203711, +STORE, 139632583090176, 139632583102463, +SNULL, 139632585195519, 139632585203711, +STORE, 139632583102464, 139632585195519, +STORE, 139632585195520, 139632585203711, +ERASE, 139632585195520, 139632585203711, +STORE, 139632585195520, 139632585203711, +STORE, 139632579293184, 139632583090175, +SNULL, 139632579293184, 139632580952063, +STORE, 139632580952064, 139632583090175, +STORE, 139632579293184, 139632580952063, +SNULL, 139632583049215, 139632583090175, +STORE, 139632580952064, 139632583049215, +STORE, 139632583049216, 139632583090175, +SNULL, 139632583049216, 139632583073791, +STORE, 139632583073792, 139632583090175, +STORE, 139632583049216, 139632583073791, +ERASE, 139632583049216, 139632583073791, +STORE, 139632583049216, 139632583073791, +ERASE, 139632583073792, 139632583090175, +STORE, 139632583073792, 139632583090175, +STORE, 139632587382784, 139632587399167, +SNULL, 139632583065599, 139632583073791, +STORE, 139632583049216, 139632583065599, +STORE, 139632583065600, 139632583073791, +SNULL, 139632585199615, 139632585203711, +STORE, 139632585195520, 139632585199615, +STORE, 139632585199616, 139632585203711, +SNULL, 94656359854079, 94656359862271, +STORE, 94656359849984, 94656359854079, +STORE, 94656359854080, 94656359862271, +SNULL, 139632587448319, 139632587452415, +STORE, 139632587444224, 139632587448319, +STORE, 139632587448320, 139632587452415, +ERASE, 139632587399168, 139632587427839, +STORE, 94656378912768, 94656379047935, +STORE, 139632585699328, 139632587382783, +STORE, 94656378912768, 94656379183103, +STORE, 94656378912768, 94656379318271, +STORE, 94656378912768, 94656379494399, +SNULL, 94656379469823, 94656379494399, +STORE, 94656378912768, 94656379469823, +STORE, 94656379469824, 94656379494399, +ERASE, 94656379469824, 94656379494399, +STORE, 94656378912768, 94656379621375, +STORE, 94656378912768, 94656379756543, +STORE, 94656378912768, 94656379912191, +STORE, 94656378912768, 94656380055551, +STORE, 94656378912768, 94656380190719, +STORE, 94656378912768, 94656380338175, +SNULL, 94656380313599, 94656380338175, +STORE, 94656378912768, 94656380313599, +STORE, 94656380313600, 94656380338175, +ERASE, 94656380313600, 94656380338175, +STORE, 94656378912768, 94656380448767, +SNULL, 94656380432383, 94656380448767, +STORE, 94656378912768, 94656380432383, +STORE, 94656380432384, 94656380448767, +ERASE, 94656380432384, 94656380448767, +STORE, 94656378912768, 94656380567551, +STORE, 94656378912768, 94656380719103, +STORE, 94656378912768, 94656380858367, +STORE, 94656378912768, 94656380997631, +STORE, 94656378912768, 94656381132799, +SNULL, 94656381124607, 94656381132799, +STORE, 94656378912768, 94656381124607, +STORE, 94656381124608, 94656381132799, +ERASE, 94656381124608, 94656381132799, +STORE, 94656378912768, 94656381276159, +STORE, 94656378912768, 94656381427711, +STORE, 94604087611392, 94604087824383, +STORE, 94604089921536, 94604089925631, +STORE, 94604089925632, 94604089933823, +STORE, 94604089933824, 94604089946111, +STORE, 94604105125888, 94604106424319, +STORE, 140454937694208, 140454939353087, +STORE, 140454939353088, 140454941450239, +STORE, 140454941450240, 140454941466623, +STORE, 140454941466624, 140454941474815, +STORE, 140454941474816, 140454941491199, +STORE, 140454941491200, 140454941503487, +STORE, 140454941503488, 140454943596543, +STORE, 140454943596544, 140454943600639, +STORE, 140454943600640, 140454943604735, +STORE, 140454943604736, 140454943748095, +STORE, 140454944116736, 140454945800191, +STORE, 140454945800192, 140454945816575, +STORE, 140454945845248, 140454945849343, +STORE, 140454945849344, 140454945853439, +STORE, 140454945853440, 140454945857535, +STORE, 140728438214656, 140728438353919, +STORE, 140728439095296, 140728439107583, +STORE, 140728439107584, 140728439111679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727821099008, 140737488351231, +SNULL, 140727821107199, 140737488351231, +STORE, 140727821099008, 140727821107199, +STORE, 140727820967936, 140727821107199, +STORE, 94088457240576, 94088459575295, +SNULL, 94088457453567, 94088459575295, +STORE, 94088457240576, 94088457453567, +STORE, 94088457453568, 94088459575295, +ERASE, 94088457453568, 94088459575295, +STORE, 94088459550720, 94088459563007, +STORE, 94088459563008, 94088459575295, +STORE, 140234378989568, 140234381242367, +SNULL, 140234379132927, 140234381242367, +STORE, 140234378989568, 140234379132927, +STORE, 140234379132928, 140234381242367, +ERASE, 140234379132928, 140234381242367, +STORE, 140234381230080, 140234381238271, +STORE, 140234381238272, 140234381242367, +STORE, 140727822077952, 140727822082047, +STORE, 140727822065664, 140727822077951, +STORE, 140234381201408, 140234381230079, +STORE, 140234381193216, 140234381201407, +STORE, 140234376876032, 140234378989567, +SNULL, 140234376876032, 140234376888319, +STORE, 140234376888320, 140234378989567, +STORE, 140234376876032, 140234376888319, +SNULL, 140234378981375, 140234378989567, +STORE, 140234376888320, 140234378981375, +STORE, 140234378981376, 140234378989567, +ERASE, 140234378981376, 140234378989567, +STORE, 140234378981376, 140234378989567, +STORE, 140234373079040, 140234376876031, +SNULL, 140234373079040, 140234374737919, +STORE, 140234374737920, 140234376876031, +STORE, 140234373079040, 140234374737919, +SNULL, 140234376835071, 140234376876031, +STORE, 140234374737920, 140234376835071, +STORE, 140234376835072, 140234376876031, +SNULL, 140234376835072, 140234376859647, +STORE, 140234376859648, 140234376876031, +STORE, 140234376835072, 140234376859647, +ERASE, 140234376835072, 140234376859647, +STORE, 140234376835072, 140234376859647, +ERASE, 140234376859648, 140234376876031, +STORE, 140234376859648, 140234376876031, +STORE, 140234381185024, 140234381201407, +SNULL, 140234376851455, 140234376859647, +STORE, 140234376835072, 140234376851455, +STORE, 140234376851456, 140234376859647, +SNULL, 140234378985471, 140234378989567, +STORE, 140234378981376, 140234378985471, +STORE, 140234378985472, 140234378989567, +SNULL, 94088459554815, 94088459563007, +STORE, 94088459550720, 94088459554815, +STORE, 94088459554816, 94088459563007, +SNULL, 140234381234175, 140234381238271, +STORE, 140234381230080, 140234381234175, +STORE, 140234381234176, 140234381238271, +ERASE, 140234381201408, 140234381230079, +STORE, 94088468852736, 94088468987903, +STORE, 140234379501568, 140234381185023, +STORE, 94088468852736, 94088469123071, +STORE, 94088468852736, 94088469258239, +STORE, 94110050402304, 94110050615295, +STORE, 94110052712448, 94110052716543, +STORE, 94110052716544, 94110052724735, +STORE, 94110052724736, 94110052737023, +STORE, 94110061875200, 94110062415871, +STORE, 140139439357952, 140139441016831, +STORE, 140139441016832, 140139443113983, +STORE, 140139443113984, 140139443130367, +STORE, 140139443130368, 140139443138559, +STORE, 140139443138560, 140139443154943, +STORE, 140139443154944, 140139443167231, +STORE, 140139443167232, 140139445260287, +STORE, 140139445260288, 140139445264383, +STORE, 140139445264384, 140139445268479, +STORE, 140139445268480, 140139445411839, +STORE, 140139445780480, 140139447463935, +STORE, 140139447463936, 140139447480319, +STORE, 140139447508992, 140139447513087, +STORE, 140139447513088, 140139447517183, +STORE, 140139447517184, 140139447521279, +STORE, 140731901427712, 140731901566975, +STORE, 140731902259200, 140731902271487, +STORE, 140731902271488, 140731902275583, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727282622464, 140737488351231, +SNULL, 140727282630655, 140737488351231, +STORE, 140727282622464, 140727282630655, +STORE, 140727282491392, 140727282630655, +STORE, 94266649866240, 94266652200959, +SNULL, 94266650079231, 94266652200959, +STORE, 94266649866240, 94266650079231, +STORE, 94266650079232, 94266652200959, +ERASE, 94266650079232, 94266652200959, +STORE, 94266652176384, 94266652188671, +STORE, 94266652188672, 94266652200959, +STORE, 139888497991680, 139888500244479, +SNULL, 139888498135039, 139888500244479, +STORE, 139888497991680, 139888498135039, +STORE, 139888498135040, 139888500244479, +ERASE, 139888498135040, 139888500244479, +STORE, 139888500232192, 139888500240383, +STORE, 139888500240384, 139888500244479, +STORE, 140727283113984, 140727283118079, +STORE, 140727283101696, 140727283113983, +STORE, 139888500203520, 139888500232191, +STORE, 139888500195328, 139888500203519, +STORE, 139888495878144, 139888497991679, +SNULL, 139888495878144, 139888495890431, +STORE, 139888495890432, 139888497991679, +STORE, 139888495878144, 139888495890431, +SNULL, 139888497983487, 139888497991679, +STORE, 139888495890432, 139888497983487, +STORE, 139888497983488, 139888497991679, +ERASE, 139888497983488, 139888497991679, +STORE, 139888497983488, 139888497991679, +STORE, 139888492081152, 139888495878143, +SNULL, 139888492081152, 139888493740031, +STORE, 139888493740032, 139888495878143, +STORE, 139888492081152, 139888493740031, +SNULL, 139888495837183, 139888495878143, +STORE, 139888493740032, 139888495837183, +STORE, 139888495837184, 139888495878143, +SNULL, 139888495837184, 139888495861759, +STORE, 139888495861760, 139888495878143, +STORE, 139888495837184, 139888495861759, +ERASE, 139888495837184, 139888495861759, +STORE, 139888495837184, 139888495861759, +ERASE, 139888495861760, 139888495878143, +STORE, 139888495861760, 139888495878143, +STORE, 139888500187136, 139888500203519, +SNULL, 139888495853567, 139888495861759, +STORE, 139888495837184, 139888495853567, +STORE, 139888495853568, 139888495861759, +SNULL, 139888497987583, 139888497991679, +STORE, 139888497983488, 139888497987583, +STORE, 139888497987584, 139888497991679, +SNULL, 94266652180479, 94266652188671, +STORE, 94266652176384, 94266652180479, +STORE, 94266652180480, 94266652188671, +SNULL, 139888500236287, 139888500240383, +STORE, 139888500232192, 139888500236287, +STORE, 139888500236288, 139888500240383, +ERASE, 139888500203520, 139888500232191, +STORE, 94266678542336, 94266678677503, +STORE, 139888498503680, 139888500187135, +STORE, 94266678542336, 94266678812671, +STORE, 94266678542336, 94266678947839, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722507702272, 140737488351231, +SNULL, 140722507710463, 140737488351231, +STORE, 140722507702272, 140722507710463, +STORE, 140722507571200, 140722507710463, +STORE, 94313981394944, 94313983729663, +SNULL, 94313981607935, 94313983729663, +STORE, 94313981394944, 94313981607935, +STORE, 94313981607936, 94313983729663, +ERASE, 94313981607936, 94313983729663, +STORE, 94313983705088, 94313983717375, +STORE, 94313983717376, 94313983729663, +STORE, 140456286076928, 140456288329727, +SNULL, 140456286220287, 140456288329727, +STORE, 140456286076928, 140456286220287, +STORE, 140456286220288, 140456288329727, +ERASE, 140456286220288, 140456288329727, +STORE, 140456288317440, 140456288325631, +STORE, 140456288325632, 140456288329727, +STORE, 140722507997184, 140722508001279, +STORE, 140722507984896, 140722507997183, +STORE, 140456288288768, 140456288317439, +STORE, 140456288280576, 140456288288767, +STORE, 140456283963392, 140456286076927, +SNULL, 140456283963392, 140456283975679, +STORE, 140456283975680, 140456286076927, +STORE, 140456283963392, 140456283975679, +SNULL, 140456286068735, 140456286076927, +STORE, 140456283975680, 140456286068735, +STORE, 140456286068736, 140456286076927, +ERASE, 140456286068736, 140456286076927, +STORE, 140456286068736, 140456286076927, +STORE, 140456280166400, 140456283963391, +SNULL, 140456280166400, 140456281825279, +STORE, 140456281825280, 140456283963391, +STORE, 140456280166400, 140456281825279, +SNULL, 140456283922431, 140456283963391, +STORE, 140456281825280, 140456283922431, +STORE, 140456283922432, 140456283963391, +SNULL, 140456283922432, 140456283947007, +STORE, 140456283947008, 140456283963391, +STORE, 140456283922432, 140456283947007, +ERASE, 140456283922432, 140456283947007, +STORE, 140456283922432, 140456283947007, +ERASE, 140456283947008, 140456283963391, +STORE, 140456283947008, 140456283963391, +STORE, 140456288272384, 140456288288767, +SNULL, 140456283938815, 140456283947007, +STORE, 140456283922432, 140456283938815, +STORE, 140456283938816, 140456283947007, +SNULL, 140456286072831, 140456286076927, +STORE, 140456286068736, 140456286072831, +STORE, 140456286072832, 140456286076927, +SNULL, 94313983709183, 94313983717375, +STORE, 94313983705088, 94313983709183, +STORE, 94313983709184, 94313983717375, +SNULL, 140456288321535, 140456288325631, +STORE, 140456288317440, 140456288321535, +STORE, 140456288321536, 140456288325631, +ERASE, 140456288288768, 140456288317439, +STORE, 94314006716416, 94314006851583, +STORE, 140456286588928, 140456288272383, +STORE, 94314006716416, 94314006986751, +STORE, 94314006716416, 94314007121919, +STORE, 93948644454400, 93948644667391, +STORE, 93948646764544, 93948646768639, +STORE, 93948646768640, 93948646776831, +STORE, 93948646776832, 93948646789119, +STORE, 93948664999936, 93948667142143, +STORE, 140187350659072, 140187352317951, +STORE, 140187352317952, 140187354415103, +STORE, 140187354415104, 140187354431487, +STORE, 140187354431488, 140187354439679, +STORE, 140187354439680, 140187354456063, +STORE, 140187354456064, 140187354468351, +STORE, 140187354468352, 140187356561407, +STORE, 140187356561408, 140187356565503, +STORE, 140187356565504, 140187356569599, +STORE, 140187356569600, 140187356712959, +STORE, 140187357081600, 140187358765055, +STORE, 140187358765056, 140187358781439, +STORE, 140187358810112, 140187358814207, +STORE, 140187358814208, 140187358818303, +STORE, 140187358818304, 140187358822399, +STORE, 140730484518912, 140730484658175, +STORE, 140730485690368, 140730485702655, +STORE, 140730485702656, 140730485706751, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721211551744, 140737488351231, +SNULL, 140721211559935, 140737488351231, +STORE, 140721211551744, 140721211559935, +STORE, 140721211420672, 140721211559935, +STORE, 94105221423104, 94105223757823, +SNULL, 94105221636095, 94105223757823, +STORE, 94105221423104, 94105221636095, +STORE, 94105221636096, 94105223757823, +ERASE, 94105221636096, 94105223757823, +STORE, 94105223733248, 94105223745535, +STORE, 94105223745536, 94105223757823, +STORE, 140474453676032, 140474455928831, +SNULL, 140474453819391, 140474455928831, +STORE, 140474453676032, 140474453819391, +STORE, 140474453819392, 140474455928831, +ERASE, 140474453819392, 140474455928831, +STORE, 140474455916544, 140474455924735, +STORE, 140474455924736, 140474455928831, +STORE, 140721211703296, 140721211707391, +STORE, 140721211691008, 140721211703295, +STORE, 140474455887872, 140474455916543, +STORE, 140474455879680, 140474455887871, +STORE, 140474451562496, 140474453676031, +SNULL, 140474451562496, 140474451574783, +STORE, 140474451574784, 140474453676031, +STORE, 140474451562496, 140474451574783, +SNULL, 140474453667839, 140474453676031, +STORE, 140474451574784, 140474453667839, +STORE, 140474453667840, 140474453676031, +ERASE, 140474453667840, 140474453676031, +STORE, 140474453667840, 140474453676031, +STORE, 140474447765504, 140474451562495, +SNULL, 140474447765504, 140474449424383, +STORE, 140474449424384, 140474451562495, +STORE, 140474447765504, 140474449424383, +SNULL, 140474451521535, 140474451562495, +STORE, 140474449424384, 140474451521535, +STORE, 140474451521536, 140474451562495, +SNULL, 140474451521536, 140474451546111, +STORE, 140474451546112, 140474451562495, +STORE, 140474451521536, 140474451546111, +ERASE, 140474451521536, 140474451546111, +STORE, 140474451521536, 140474451546111, +ERASE, 140474451546112, 140474451562495, +STORE, 140474451546112, 140474451562495, +STORE, 140474455871488, 140474455887871, +SNULL, 140474451537919, 140474451546111, +STORE, 140474451521536, 140474451537919, +STORE, 140474451537920, 140474451546111, +SNULL, 140474453671935, 140474453676031, +STORE, 140474453667840, 140474453671935, +STORE, 140474453671936, 140474453676031, +SNULL, 94105223737343, 94105223745535, +STORE, 94105223733248, 94105223737343, +STORE, 94105223737344, 94105223745535, +SNULL, 140474455920639, 140474455924735, +STORE, 140474455916544, 140474455920639, +STORE, 140474455920640, 140474455924735, +ERASE, 140474455887872, 140474455916543, +STORE, 94105238712320, 94105238847487, +STORE, 140474454188032, 140474455871487, +STORE, 94105238712320, 94105238982655, +STORE, 94105238712320, 94105239117823, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732356354048, 140737488351231, +SNULL, 140732356362239, 140737488351231, +STORE, 140732356354048, 140732356362239, +STORE, 140732356222976, 140732356362239, +STORE, 94461165989888, 94461168324607, +SNULL, 94461166202879, 94461168324607, +STORE, 94461165989888, 94461166202879, +STORE, 94461166202880, 94461168324607, +ERASE, 94461166202880, 94461168324607, +STORE, 94461168300032, 94461168312319, +STORE, 94461168312320, 94461168324607, +STORE, 140317255110656, 140317257363455, +SNULL, 140317255254015, 140317257363455, +STORE, 140317255110656, 140317255254015, +STORE, 140317255254016, 140317257363455, +ERASE, 140317255254016, 140317257363455, +STORE, 140317257351168, 140317257359359, +STORE, 140317257359360, 140317257363455, +STORE, 140732356583424, 140732356587519, +STORE, 140732356571136, 140732356583423, +STORE, 140317257322496, 140317257351167, +STORE, 140317257314304, 140317257322495, +STORE, 140317252997120, 140317255110655, +SNULL, 140317252997120, 140317253009407, +STORE, 140317253009408, 140317255110655, +STORE, 140317252997120, 140317253009407, +SNULL, 140317255102463, 140317255110655, +STORE, 140317253009408, 140317255102463, +STORE, 140317255102464, 140317255110655, +ERASE, 140317255102464, 140317255110655, +STORE, 140317255102464, 140317255110655, +STORE, 140317249200128, 140317252997119, +SNULL, 140317249200128, 140317250859007, +STORE, 140317250859008, 140317252997119, +STORE, 140317249200128, 140317250859007, +SNULL, 140317252956159, 140317252997119, +STORE, 140317250859008, 140317252956159, +STORE, 140317252956160, 140317252997119, +SNULL, 140317252956160, 140317252980735, +STORE, 140317252980736, 140317252997119, +STORE, 140317252956160, 140317252980735, +ERASE, 140317252956160, 140317252980735, +STORE, 140317252956160, 140317252980735, +ERASE, 140317252980736, 140317252997119, +STORE, 140317252980736, 140317252997119, +STORE, 140317257306112, 140317257322495, +SNULL, 140317252972543, 140317252980735, +STORE, 140317252956160, 140317252972543, +STORE, 140317252972544, 140317252980735, +SNULL, 140317255106559, 140317255110655, +STORE, 140317255102464, 140317255106559, +STORE, 140317255106560, 140317255110655, +SNULL, 94461168304127, 94461168312319, +STORE, 94461168300032, 94461168304127, +STORE, 94461168304128, 94461168312319, +SNULL, 140317257355263, 140317257359359, +STORE, 140317257351168, 140317257355263, +STORE, 140317257355264, 140317257359359, +ERASE, 140317257322496, 140317257351167, +STORE, 94461195268096, 94461195403263, +STORE, 140317255622656, 140317257306111, +STORE, 94461195268096, 94461195538431, +STORE, 94461195268096, 94461195673599, +STORE, 94110050402304, 94110050615295, +STORE, 94110052712448, 94110052716543, +STORE, 94110052716544, 94110052724735, +STORE, 94110052724736, 94110052737023, +STORE, 94110061875200, 94110062415871, +STORE, 140139439357952, 140139441016831, +STORE, 140139441016832, 140139443113983, +STORE, 140139443113984, 140139443130367, +STORE, 140139443130368, 140139443138559, +STORE, 140139443138560, 140139443154943, +STORE, 140139443154944, 140139443167231, +STORE, 140139443167232, 140139445260287, +STORE, 140139445260288, 140139445264383, +STORE, 140139445264384, 140139445268479, +STORE, 140139445268480, 140139445411839, +STORE, 140139445780480, 140139447463935, +STORE, 140139447463936, 140139447480319, +STORE, 140139447508992, 140139447513087, +STORE, 140139447513088, 140139447517183, +STORE, 140139447517184, 140139447521279, +STORE, 140731901427712, 140731901566975, +STORE, 140731902259200, 140731902271487, +STORE, 140731902271488, 140731902275583, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720941613056, 140737488351231, +SNULL, 140720941621247, 140737488351231, +STORE, 140720941613056, 140720941621247, +STORE, 140720941481984, 140720941621247, +STORE, 93902377721856, 93902379945983, +SNULL, 93902377832447, 93902379945983, +STORE, 93902377721856, 93902377832447, +STORE, 93902377832448, 93902379945983, +ERASE, 93902377832448, 93902379945983, +STORE, 93902379925504, 93902379937791, +STORE, 93902379937792, 93902379945983, +STORE, 139836543635456, 139836545888255, +SNULL, 139836543778815, 139836545888255, +STORE, 139836543635456, 139836543778815, +STORE, 139836543778816, 139836545888255, +ERASE, 139836543778816, 139836545888255, +STORE, 139836545875968, 139836545884159, +STORE, 139836545884160, 139836545888255, +STORE, 140720941711360, 140720941715455, +STORE, 140720941699072, 140720941711359, +STORE, 139836545847296, 139836545875967, +STORE, 139836545839104, 139836545847295, +STORE, 139836539838464, 139836543635455, +SNULL, 139836539838464, 139836541497343, +STORE, 139836541497344, 139836543635455, +STORE, 139836539838464, 139836541497343, +SNULL, 139836543594495, 139836543635455, +STORE, 139836541497344, 139836543594495, +STORE, 139836543594496, 139836543635455, +SNULL, 139836543594496, 139836543619071, +STORE, 139836543619072, 139836543635455, +STORE, 139836543594496, 139836543619071, +ERASE, 139836543594496, 139836543619071, +STORE, 139836543594496, 139836543619071, +ERASE, 139836543619072, 139836543635455, +STORE, 139836543619072, 139836543635455, +SNULL, 139836543610879, 139836543619071, +STORE, 139836543594496, 139836543610879, +STORE, 139836543610880, 139836543619071, +SNULL, 93902379933695, 93902379937791, +STORE, 93902379925504, 93902379933695, +STORE, 93902379933696, 93902379937791, +SNULL, 139836545880063, 139836545884159, +STORE, 139836545875968, 139836545880063, +STORE, 139836545880064, 139836545884159, +ERASE, 139836545847296, 139836545875967, +STORE, 93902396891136, 93902397026303, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736538206208, 140737488351231, +SNULL, 140736538214399, 140737488351231, +STORE, 140736538206208, 140736538214399, +STORE, 140736538075136, 140736538214399, +STORE, 94173471399936, 94173473734655, +SNULL, 94173471612927, 94173473734655, +STORE, 94173471399936, 94173471612927, +STORE, 94173471612928, 94173473734655, +ERASE, 94173471612928, 94173473734655, +STORE, 94173473710080, 94173473722367, +STORE, 94173473722368, 94173473734655, +STORE, 140035513556992, 140035515809791, +SNULL, 140035513700351, 140035515809791, +STORE, 140035513556992, 140035513700351, +STORE, 140035513700352, 140035515809791, +ERASE, 140035513700352, 140035515809791, +STORE, 140035515797504, 140035515805695, +STORE, 140035515805696, 140035515809791, +STORE, 140736538329088, 140736538333183, +STORE, 140736538316800, 140736538329087, +STORE, 140035515768832, 140035515797503, +STORE, 140035515760640, 140035515768831, +STORE, 140035511443456, 140035513556991, +SNULL, 140035511443456, 140035511455743, +STORE, 140035511455744, 140035513556991, +STORE, 140035511443456, 140035511455743, +SNULL, 140035513548799, 140035513556991, +STORE, 140035511455744, 140035513548799, +STORE, 140035513548800, 140035513556991, +ERASE, 140035513548800, 140035513556991, +STORE, 140035513548800, 140035513556991, +STORE, 140035507646464, 140035511443455, +SNULL, 140035507646464, 140035509305343, +STORE, 140035509305344, 140035511443455, +STORE, 140035507646464, 140035509305343, +SNULL, 140035511402495, 140035511443455, +STORE, 140035509305344, 140035511402495, +STORE, 140035511402496, 140035511443455, +SNULL, 140035511402496, 140035511427071, +STORE, 140035511427072, 140035511443455, +STORE, 140035511402496, 140035511427071, +ERASE, 140035511402496, 140035511427071, +STORE, 140035511402496, 140035511427071, +ERASE, 140035511427072, 140035511443455, +STORE, 140035511427072, 140035511443455, +STORE, 140035515752448, 140035515768831, +SNULL, 140035511418879, 140035511427071, +STORE, 140035511402496, 140035511418879, +STORE, 140035511418880, 140035511427071, +SNULL, 140035513552895, 140035513556991, +STORE, 140035513548800, 140035513552895, +STORE, 140035513552896, 140035513556991, +SNULL, 94173473714175, 94173473722367, +STORE, 94173473710080, 94173473714175, +STORE, 94173473714176, 94173473722367, +SNULL, 140035515801599, 140035515805695, +STORE, 140035515797504, 140035515801599, +STORE, 140035515801600, 140035515805695, +ERASE, 140035515768832, 140035515797503, +STORE, 94173478645760, 94173478780927, +STORE, 140035514068992, 140035515752447, +STORE, 94173478645760, 94173478916095, +STORE, 94173478645760, 94173479051263, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140724216176640, 140737488351231, +SNULL, 140724216184831, 140737488351231, +STORE, 140724216176640, 140724216184831, +STORE, 140724216045568, 140724216184831, +STORE, 94870930628608, 94870932963327, +SNULL, 94870930841599, 94870932963327, +STORE, 94870930628608, 94870930841599, +STORE, 94870930841600, 94870932963327, +ERASE, 94870930841600, 94870932963327, +STORE, 94870932938752, 94870932951039, +STORE, 94870932951040, 94870932963327, +STORE, 140453683736576, 140453685989375, +SNULL, 140453683879935, 140453685989375, +STORE, 140453683736576, 140453683879935, +STORE, 140453683879936, 140453685989375, +ERASE, 140453683879936, 140453685989375, +STORE, 140453685977088, 140453685985279, +STORE, 140453685985280, 140453685989375, +STORE, 140724216832000, 140724216836095, +STORE, 140724216819712, 140724216831999, +STORE, 140453685948416, 140453685977087, +STORE, 140453685940224, 140453685948415, +STORE, 140453681623040, 140453683736575, +SNULL, 140453681623040, 140453681635327, +STORE, 140453681635328, 140453683736575, +STORE, 140453681623040, 140453681635327, +SNULL, 140453683728383, 140453683736575, +STORE, 140453681635328, 140453683728383, +STORE, 140453683728384, 140453683736575, +ERASE, 140453683728384, 140453683736575, +STORE, 140453683728384, 140453683736575, +STORE, 140453677826048, 140453681623039, +SNULL, 140453677826048, 140453679484927, +STORE, 140453679484928, 140453681623039, +STORE, 140453677826048, 140453679484927, +SNULL, 140453681582079, 140453681623039, +STORE, 140453679484928, 140453681582079, +STORE, 140453681582080, 140453681623039, +SNULL, 140453681582080, 140453681606655, +STORE, 140453681606656, 140453681623039, +STORE, 140453681582080, 140453681606655, +ERASE, 140453681582080, 140453681606655, +STORE, 140453681582080, 140453681606655, +ERASE, 140453681606656, 140453681623039, +STORE, 140453681606656, 140453681623039, +STORE, 140453685932032, 140453685948415, +SNULL, 140453681598463, 140453681606655, +STORE, 140453681582080, 140453681598463, +STORE, 140453681598464, 140453681606655, +SNULL, 140453683732479, 140453683736575, +STORE, 140453683728384, 140453683732479, +STORE, 140453683732480, 140453683736575, +SNULL, 94870932942847, 94870932951039, +STORE, 94870932938752, 94870932942847, +STORE, 94870932942848, 94870932951039, +SNULL, 140453685981183, 140453685985279, +STORE, 140453685977088, 140453685981183, +STORE, 140453685981184, 140453685985279, +ERASE, 140453685948416, 140453685977087, +STORE, 94870940565504, 94870940700671, +STORE, 140453684248576, 140453685932031, +STORE, 94870940565504, 94870940835839, +STORE, 94870940565504, 94870940971007, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731275661312, 140737488351231, +SNULL, 140731275669503, 140737488351231, +STORE, 140731275661312, 140731275669503, +STORE, 140731275530240, 140731275669503, +STORE, 94642788548608, 94642790883327, +SNULL, 94642788761599, 94642790883327, +STORE, 94642788548608, 94642788761599, +STORE, 94642788761600, 94642790883327, +ERASE, 94642788761600, 94642790883327, +STORE, 94642790858752, 94642790871039, +STORE, 94642790871040, 94642790883327, +STORE, 140228458749952, 140228461002751, +SNULL, 140228458893311, 140228461002751, +STORE, 140228458749952, 140228458893311, +STORE, 140228458893312, 140228461002751, +ERASE, 140228458893312, 140228461002751, +STORE, 140228460990464, 140228460998655, +STORE, 140228460998656, 140228461002751, +STORE, 140731276349440, 140731276353535, +STORE, 140731276337152, 140731276349439, +STORE, 140228460961792, 140228460990463, +STORE, 140228460953600, 140228460961791, +STORE, 140228456636416, 140228458749951, +SNULL, 140228456636416, 140228456648703, +STORE, 140228456648704, 140228458749951, +STORE, 140228456636416, 140228456648703, +SNULL, 140228458741759, 140228458749951, +STORE, 140228456648704, 140228458741759, +STORE, 140228458741760, 140228458749951, +ERASE, 140228458741760, 140228458749951, +STORE, 140228458741760, 140228458749951, +STORE, 140228452839424, 140228456636415, +SNULL, 140228452839424, 140228454498303, +STORE, 140228454498304, 140228456636415, +STORE, 140228452839424, 140228454498303, +SNULL, 140228456595455, 140228456636415, +STORE, 140228454498304, 140228456595455, +STORE, 140228456595456, 140228456636415, +SNULL, 140228456595456, 140228456620031, +STORE, 140228456620032, 140228456636415, +STORE, 140228456595456, 140228456620031, +ERASE, 140228456595456, 140228456620031, +STORE, 140228456595456, 140228456620031, +ERASE, 140228456620032, 140228456636415, +STORE, 140228456620032, 140228456636415, +STORE, 140228460945408, 140228460961791, +SNULL, 140228456611839, 140228456620031, +STORE, 140228456595456, 140228456611839, +STORE, 140228456611840, 140228456620031, +SNULL, 140228458745855, 140228458749951, +STORE, 140228458741760, 140228458745855, +STORE, 140228458745856, 140228458749951, +SNULL, 94642790862847, 94642790871039, +STORE, 94642790858752, 94642790862847, +STORE, 94642790862848, 94642790871039, +SNULL, 140228460994559, 140228460998655, +STORE, 140228460990464, 140228460994559, +STORE, 140228460994560, 140228460998655, +ERASE, 140228460961792, 140228460990463, +STORE, 94642801549312, 94642801684479, +STORE, 140228459261952, 140228460945407, +STORE, 94642801549312, 94642801819647, +STORE, 94642801549312, 94642801954815, +STORE, 94604087611392, 94604087824383, +STORE, 94604089921536, 94604089925631, +STORE, 94604089925632, 94604089933823, +STORE, 94604089933824, 94604089946111, +STORE, 94604105125888, 94604106424319, +STORE, 140454937694208, 140454939353087, +STORE, 140454939353088, 140454941450239, +STORE, 140454941450240, 140454941466623, +STORE, 140454941466624, 140454941474815, +STORE, 140454941474816, 140454941491199, +STORE, 140454941491200, 140454941503487, +STORE, 140454941503488, 140454943596543, +STORE, 140454943596544, 140454943600639, +STORE, 140454943600640, 140454943604735, +STORE, 140454943604736, 140454943748095, +STORE, 140454944116736, 140454945800191, +STORE, 140454945800192, 140454945816575, +STORE, 140454945845248, 140454945849343, +STORE, 140454945849344, 140454945853439, +STORE, 140454945853440, 140454945857535, +STORE, 140728438214656, 140728438353919, +STORE, 140728439095296, 140728439107583, +STORE, 140728439107584, 140728439111679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721843453952, 140737488351231, +SNULL, 140721843462143, 140737488351231, +STORE, 140721843453952, 140721843462143, +STORE, 140721843322880, 140721843462143, +STORE, 94465962455040, 94465964789759, +SNULL, 94465962668031, 94465964789759, +STORE, 94465962455040, 94465962668031, +STORE, 94465962668032, 94465964789759, +ERASE, 94465962668032, 94465964789759, +STORE, 94465964765184, 94465964777471, +STORE, 94465964777472, 94465964789759, +STORE, 139913488314368, 139913490567167, +SNULL, 139913488457727, 139913490567167, +STORE, 139913488314368, 139913488457727, +STORE, 139913488457728, 139913490567167, +ERASE, 139913488457728, 139913490567167, +STORE, 139913490554880, 139913490563071, +STORE, 139913490563072, 139913490567167, +STORE, 140721843503104, 140721843507199, +STORE, 140721843490816, 140721843503103, +STORE, 139913490526208, 139913490554879, +STORE, 139913490518016, 139913490526207, +STORE, 139913486200832, 139913488314367, +SNULL, 139913486200832, 139913486213119, +STORE, 139913486213120, 139913488314367, +STORE, 139913486200832, 139913486213119, +SNULL, 139913488306175, 139913488314367, +STORE, 139913486213120, 139913488306175, +STORE, 139913488306176, 139913488314367, +ERASE, 139913488306176, 139913488314367, +STORE, 139913488306176, 139913488314367, +STORE, 139913482403840, 139913486200831, +SNULL, 139913482403840, 139913484062719, +STORE, 139913484062720, 139913486200831, +STORE, 139913482403840, 139913484062719, +SNULL, 139913486159871, 139913486200831, +STORE, 139913484062720, 139913486159871, +STORE, 139913486159872, 139913486200831, +SNULL, 139913486159872, 139913486184447, +STORE, 139913486184448, 139913486200831, +STORE, 139913486159872, 139913486184447, +ERASE, 139913486159872, 139913486184447, +STORE, 139913486159872, 139913486184447, +ERASE, 139913486184448, 139913486200831, +STORE, 139913486184448, 139913486200831, +STORE, 139913490509824, 139913490526207, +SNULL, 139913486176255, 139913486184447, +STORE, 139913486159872, 139913486176255, +STORE, 139913486176256, 139913486184447, +SNULL, 139913488310271, 139913488314367, +STORE, 139913488306176, 139913488310271, +STORE, 139913488310272, 139913488314367, +SNULL, 94465964769279, 94465964777471, +STORE, 94465964765184, 94465964769279, +STORE, 94465964769280, 94465964777471, +SNULL, 139913490558975, 139913490563071, +STORE, 139913490554880, 139913490558975, +STORE, 139913490558976, 139913490563071, +ERASE, 139913490526208, 139913490554879, +STORE, 94465970024448, 94465970159615, +STORE, 139913488826368, 139913490509823, +STORE, 94465970024448, 94465970294783, +STORE, 94465970024448, 94465970429951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140720583307264, 140737488351231, +SNULL, 140720583315455, 140737488351231, +STORE, 140720583307264, 140720583315455, +STORE, 140720583176192, 140720583315455, +STORE, 94212322082816, 94212324417535, +SNULL, 94212322295807, 94212324417535, +STORE, 94212322082816, 94212322295807, +STORE, 94212322295808, 94212324417535, +ERASE, 94212322295808, 94212324417535, +STORE, 94212324392960, 94212324405247, +STORE, 94212324405248, 94212324417535, +STORE, 139659688538112, 139659690790911, +SNULL, 139659688681471, 139659690790911, +STORE, 139659688538112, 139659688681471, +STORE, 139659688681472, 139659690790911, +ERASE, 139659688681472, 139659690790911, +STORE, 139659690778624, 139659690786815, +STORE, 139659690786816, 139659690790911, +STORE, 140720584781824, 140720584785919, +STORE, 140720584769536, 140720584781823, +STORE, 139659690749952, 139659690778623, +STORE, 139659690741760, 139659690749951, +STORE, 139659686424576, 139659688538111, +SNULL, 139659686424576, 139659686436863, +STORE, 139659686436864, 139659688538111, +STORE, 139659686424576, 139659686436863, +SNULL, 139659688529919, 139659688538111, +STORE, 139659686436864, 139659688529919, +STORE, 139659688529920, 139659688538111, +ERASE, 139659688529920, 139659688538111, +STORE, 139659688529920, 139659688538111, +STORE, 139659682627584, 139659686424575, +SNULL, 139659682627584, 139659684286463, +STORE, 139659684286464, 139659686424575, +STORE, 139659682627584, 139659684286463, +SNULL, 139659686383615, 139659686424575, +STORE, 139659684286464, 139659686383615, +STORE, 139659686383616, 139659686424575, +SNULL, 139659686383616, 139659686408191, +STORE, 139659686408192, 139659686424575, +STORE, 139659686383616, 139659686408191, +ERASE, 139659686383616, 139659686408191, +STORE, 139659686383616, 139659686408191, +ERASE, 139659686408192, 139659686424575, +STORE, 139659686408192, 139659686424575, +STORE, 139659690733568, 139659690749951, +SNULL, 139659686399999, 139659686408191, +STORE, 139659686383616, 139659686399999, +STORE, 139659686400000, 139659686408191, +SNULL, 139659688534015, 139659688538111, +STORE, 139659688529920, 139659688534015, +STORE, 139659688534016, 139659688538111, +SNULL, 94212324397055, 94212324405247, +STORE, 94212324392960, 94212324397055, +STORE, 94212324397056, 94212324405247, +SNULL, 139659690782719, 139659690786815, +STORE, 139659690778624, 139659690782719, +STORE, 139659690782720, 139659690786815, +ERASE, 139659690749952, 139659690778623, +STORE, 94212355014656, 94212355149823, +STORE, 139659689050112, 139659690733567, +STORE, 94212355014656, 94212355284991, +STORE, 94212355014656, 94212355420159, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140727689830400, 140737488351231, +SNULL, 140727689838591, 140737488351231, +STORE, 140727689830400, 140727689838591, +STORE, 140727689699328, 140727689838591, +STORE, 94572390281216, 94572392615935, +SNULL, 94572390494207, 94572392615935, +STORE, 94572390281216, 94572390494207, +STORE, 94572390494208, 94572392615935, +ERASE, 94572390494208, 94572392615935, +STORE, 94572392591360, 94572392603647, +STORE, 94572392603648, 94572392615935, +STORE, 140575923769344, 140575926022143, +SNULL, 140575923912703, 140575926022143, +STORE, 140575923769344, 140575923912703, +STORE, 140575923912704, 140575926022143, +ERASE, 140575923912704, 140575926022143, +STORE, 140575926009856, 140575926018047, +STORE, 140575926018048, 140575926022143, +STORE, 140727689871360, 140727689875455, +STORE, 140727689859072, 140727689871359, +STORE, 140575925981184, 140575926009855, +STORE, 140575925972992, 140575925981183, +STORE, 140575921655808, 140575923769343, +SNULL, 140575921655808, 140575921668095, +STORE, 140575921668096, 140575923769343, +STORE, 140575921655808, 140575921668095, +SNULL, 140575923761151, 140575923769343, +STORE, 140575921668096, 140575923761151, +STORE, 140575923761152, 140575923769343, +ERASE, 140575923761152, 140575923769343, +STORE, 140575923761152, 140575923769343, +STORE, 140575917858816, 140575921655807, +SNULL, 140575917858816, 140575919517695, +STORE, 140575919517696, 140575921655807, +STORE, 140575917858816, 140575919517695, +SNULL, 140575921614847, 140575921655807, +STORE, 140575919517696, 140575921614847, +STORE, 140575921614848, 140575921655807, +SNULL, 140575921614848, 140575921639423, +STORE, 140575921639424, 140575921655807, +STORE, 140575921614848, 140575921639423, +ERASE, 140575921614848, 140575921639423, +STORE, 140575921614848, 140575921639423, +ERASE, 140575921639424, 140575921655807, +STORE, 140575921639424, 140575921655807, +STORE, 140575925964800, 140575925981183, +SNULL, 140575921631231, 140575921639423, +STORE, 140575921614848, 140575921631231, +STORE, 140575921631232, 140575921639423, +SNULL, 140575923765247, 140575923769343, +STORE, 140575923761152, 140575923765247, +STORE, 140575923765248, 140575923769343, +SNULL, 94572392595455, 94572392603647, +STORE, 94572392591360, 94572392595455, +STORE, 94572392595456, 94572392603647, +SNULL, 140575926013951, 140575926018047, +STORE, 140575926009856, 140575926013951, +STORE, 140575926013952, 140575926018047, +ERASE, 140575925981184, 140575926009855, +STORE, 94572402278400, 94572402413567, +STORE, 140575924281344, 140575925964799, +STORE, 94572402278400, 94572402548735, +STORE, 94572402278400, 94572402683903, +STORE, 94572402278400, 94572402851839, +SNULL, 94572402827263, 94572402851839, +STORE, 94572402278400, 94572402827263, +STORE, 94572402827264, 94572402851839, +ERASE, 94572402827264, 94572402851839, +STORE, 94572402278400, 94572402966527, +STORE, 94572402278400, 94572403109887, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725520506880, 140737488351231, +SNULL, 140725520515071, 140737488351231, +STORE, 140725520506880, 140725520515071, +STORE, 140725520375808, 140725520515071, +STORE, 93829948788736, 93829951012863, +SNULL, 93829948899327, 93829951012863, +STORE, 93829948788736, 93829948899327, +STORE, 93829948899328, 93829951012863, +ERASE, 93829948899328, 93829951012863, +STORE, 93829950992384, 93829951004671, +STORE, 93829951004672, 93829951012863, +STORE, 140133696794624, 140133699047423, +SNULL, 140133696937983, 140133699047423, +STORE, 140133696794624, 140133696937983, +STORE, 140133696937984, 140133699047423, +ERASE, 140133696937984, 140133699047423, +STORE, 140133699035136, 140133699043327, +STORE, 140133699043328, 140133699047423, +STORE, 140725520875520, 140725520879615, +STORE, 140725520863232, 140725520875519, +STORE, 140133699006464, 140133699035135, +STORE, 140133698998272, 140133699006463, +STORE, 140133692997632, 140133696794623, +SNULL, 140133692997632, 140133694656511, +STORE, 140133694656512, 140133696794623, +STORE, 140133692997632, 140133694656511, +SNULL, 140133696753663, 140133696794623, +STORE, 140133694656512, 140133696753663, +STORE, 140133696753664, 140133696794623, +SNULL, 140133696753664, 140133696778239, +STORE, 140133696778240, 140133696794623, +STORE, 140133696753664, 140133696778239, +ERASE, 140133696753664, 140133696778239, +STORE, 140133696753664, 140133696778239, +ERASE, 140133696778240, 140133696794623, +STORE, 140133696778240, 140133696794623, +SNULL, 140133696770047, 140133696778239, +STORE, 140133696753664, 140133696770047, +STORE, 140133696770048, 140133696778239, +SNULL, 93829951000575, 93829951004671, +STORE, 93829950992384, 93829951000575, +STORE, 93829951000576, 93829951004671, +SNULL, 140133699039231, 140133699043327, +STORE, 140133699035136, 140133699039231, +STORE, 140133699039232, 140133699043327, +ERASE, 140133699006464, 140133699035135, +STORE, 93829978693632, 93829978828799, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140736118022144, 140737488351231, +SNULL, 140736118030335, 140737488351231, +STORE, 140736118022144, 140736118030335, +STORE, 140736117891072, 140736118030335, +STORE, 94467663982592, 94467666206719, +SNULL, 94467664093183, 94467666206719, +STORE, 94467663982592, 94467664093183, +STORE, 94467664093184, 94467666206719, +ERASE, 94467664093184, 94467666206719, +STORE, 94467666186240, 94467666198527, +STORE, 94467666198528, 94467666206719, +STORE, 140525377327104, 140525379579903, +SNULL, 140525377470463, 140525379579903, +STORE, 140525377327104, 140525377470463, +STORE, 140525377470464, 140525379579903, +ERASE, 140525377470464, 140525379579903, +STORE, 140525379567616, 140525379575807, +STORE, 140525379575808, 140525379579903, +STORE, 140736118771712, 140736118775807, +STORE, 140736118759424, 140736118771711, +STORE, 140525379538944, 140525379567615, +STORE, 140525379530752, 140525379538943, +STORE, 140525373530112, 140525377327103, +SNULL, 140525373530112, 140525375188991, +STORE, 140525375188992, 140525377327103, +STORE, 140525373530112, 140525375188991, +SNULL, 140525377286143, 140525377327103, +STORE, 140525375188992, 140525377286143, +STORE, 140525377286144, 140525377327103, +SNULL, 140525377286144, 140525377310719, +STORE, 140525377310720, 140525377327103, +STORE, 140525377286144, 140525377310719, +ERASE, 140525377286144, 140525377310719, +STORE, 140525377286144, 140525377310719, +ERASE, 140525377310720, 140525377327103, +STORE, 140525377310720, 140525377327103, +SNULL, 140525377302527, 140525377310719, +STORE, 140525377286144, 140525377302527, +STORE, 140525377302528, 140525377310719, +SNULL, 94467666194431, 94467666198527, +STORE, 94467666186240, 94467666194431, +STORE, 94467666194432, 94467666198527, +SNULL, 140525379571711, 140525379575807, +STORE, 140525379567616, 140525379571711, +STORE, 140525379571712, 140525379575807, +ERASE, 140525379538944, 140525379567615, +STORE, 94467693379584, 94467693514751, +STORE, 94200172744704, 94200172957695, +STORE, 94200175054848, 94200175058943, +STORE, 94200175058944, 94200175067135, +STORE, 94200175067136, 94200175079423, +STORE, 94200196673536, 94200198905855, +STORE, 140053867720704, 140053869379583, +STORE, 140053869379584, 140053871476735, +STORE, 140053871476736, 140053871493119, +STORE, 140053871493120, 140053871501311, +STORE, 140053871501312, 140053871517695, +STORE, 140053871517696, 140053871529983, +STORE, 140053871529984, 140053873623039, +STORE, 140053873623040, 140053873627135, +STORE, 140053873627136, 140053873631231, +STORE, 140053873631232, 140053873774591, +STORE, 140053874143232, 140053875826687, +STORE, 140053875826688, 140053875843071, +STORE, 140053875871744, 140053875875839, +STORE, 140053875875840, 140053875879935, +STORE, 140053875879936, 140053875884031, +STORE, 140728538484736, 140728538623999, +STORE, 140728538652672, 140728538664959, +STORE, 140728538664960, 140728538669055, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732307775488, 140737488351231, +SNULL, 140732307783679, 140737488351231, +STORE, 140732307775488, 140732307783679, +STORE, 140732307644416, 140732307783679, +STORE, 93831417630720, 93831419965439, +SNULL, 93831417843711, 93831419965439, +STORE, 93831417630720, 93831417843711, +STORE, 93831417843712, 93831419965439, +ERASE, 93831417843712, 93831419965439, +STORE, 93831419940864, 93831419953151, +STORE, 93831419953152, 93831419965439, +STORE, 140241062088704, 140241064341503, +SNULL, 140241062232063, 140241064341503, +STORE, 140241062088704, 140241062232063, +STORE, 140241062232064, 140241064341503, +ERASE, 140241062232064, 140241064341503, +STORE, 140241064329216, 140241064337407, +STORE, 140241064337408, 140241064341503, +STORE, 140732308140032, 140732308144127, +STORE, 140732308127744, 140732308140031, +STORE, 140241064300544, 140241064329215, +STORE, 140241064292352, 140241064300543, +STORE, 140241059975168, 140241062088703, +SNULL, 140241059975168, 140241059987455, +STORE, 140241059987456, 140241062088703, +STORE, 140241059975168, 140241059987455, +SNULL, 140241062080511, 140241062088703, +STORE, 140241059987456, 140241062080511, +STORE, 140241062080512, 140241062088703, +ERASE, 140241062080512, 140241062088703, +STORE, 140241062080512, 140241062088703, +STORE, 140241056178176, 140241059975167, +SNULL, 140241056178176, 140241057837055, +STORE, 140241057837056, 140241059975167, +STORE, 140241056178176, 140241057837055, +SNULL, 140241059934207, 140241059975167, +STORE, 140241057837056, 140241059934207, +STORE, 140241059934208, 140241059975167, +SNULL, 140241059934208, 140241059958783, +STORE, 140241059958784, 140241059975167, +STORE, 140241059934208, 140241059958783, +ERASE, 140241059934208, 140241059958783, +STORE, 140241059934208, 140241059958783, +ERASE, 140241059958784, 140241059975167, +STORE, 140241059958784, 140241059975167, +STORE, 140241064284160, 140241064300543, +SNULL, 140241059950591, 140241059958783, +STORE, 140241059934208, 140241059950591, +STORE, 140241059950592, 140241059958783, +SNULL, 140241062084607, 140241062088703, +STORE, 140241062080512, 140241062084607, +STORE, 140241062084608, 140241062088703, +SNULL, 93831419944959, 93831419953151, +STORE, 93831419940864, 93831419944959, +STORE, 93831419944960, 93831419953151, +SNULL, 140241064333311, 140241064337407, +STORE, 140241064329216, 140241064333311, +STORE, 140241064333312, 140241064337407, +ERASE, 140241064300544, 140241064329215, +STORE, 93831435284480, 93831435419647, +STORE, 140241062600704, 140241064284159, +STORE, 93831435284480, 93831435554815, +STORE, 93831435284480, 93831435689983, +STORE, 93831435284480, 93831435862015, +SNULL, 93831435837439, 93831435862015, +STORE, 93831435284480, 93831435837439, +STORE, 93831435837440, 93831435862015, +ERASE, 93831435837440, 93831435862015, +STORE, 93831435284480, 93831435972607, +STORE, 93831435284480, 93831436107775, +SNULL, 93831436091391, 93831436107775, +STORE, 93831435284480, 93831436091391, +STORE, 93831436091392, 93831436107775, +ERASE, 93831436091392, 93831436107775, +STORE, 93831435284480, 93831436226559, +STORE, 93831435284480, 93831436361727, +STORE, 93831435284480, 93831436505087, +STORE, 93831435284480, 93831436652543, +STORE, 93831435284480, 93831436787711, +STORE, 93831435284480, 93831436926975, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140728546775040, 140737488351231, +SNULL, 140728546783231, 140737488351231, +STORE, 140728546775040, 140728546783231, +STORE, 140728546643968, 140728546783231, +STORE, 94456178786304, 94456181010431, +SNULL, 94456178896895, 94456181010431, +STORE, 94456178786304, 94456178896895, +STORE, 94456178896896, 94456181010431, +ERASE, 94456178896896, 94456181010431, +STORE, 94456180989952, 94456181002239, +STORE, 94456181002240, 94456181010431, +STORE, 140221893091328, 140221895344127, +SNULL, 140221893234687, 140221895344127, +STORE, 140221893091328, 140221893234687, +STORE, 140221893234688, 140221895344127, +ERASE, 140221893234688, 140221895344127, +STORE, 140221895331840, 140221895340031, +STORE, 140221895340032, 140221895344127, +STORE, 140728547803136, 140728547807231, +STORE, 140728547790848, 140728547803135, +STORE, 140221895303168, 140221895331839, +STORE, 140221895294976, 140221895303167, +STORE, 140221889294336, 140221893091327, +SNULL, 140221889294336, 140221890953215, +STORE, 140221890953216, 140221893091327, +STORE, 140221889294336, 140221890953215, +SNULL, 140221893050367, 140221893091327, +STORE, 140221890953216, 140221893050367, +STORE, 140221893050368, 140221893091327, +SNULL, 140221893050368, 140221893074943, +STORE, 140221893074944, 140221893091327, +STORE, 140221893050368, 140221893074943, +ERASE, 140221893050368, 140221893074943, +STORE, 140221893050368, 140221893074943, +ERASE, 140221893074944, 140221893091327, +STORE, 140221893074944, 140221893091327, +SNULL, 140221893066751, 140221893074943, +STORE, 140221893050368, 140221893066751, +STORE, 140221893066752, 140221893074943, +SNULL, 94456180998143, 94456181002239, +STORE, 94456180989952, 94456180998143, +STORE, 94456180998144, 94456181002239, +SNULL, 140221895335935, 140221895340031, +STORE, 140221895331840, 140221895335935, +STORE, 140221895335936, 140221895340031, +ERASE, 140221895303168, 140221895331839, +STORE, 94456203730944, 94456203866111, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140734438637568, 140737488351231, +SNULL, 140734438645759, 140737488351231, +STORE, 140734438637568, 140734438645759, +STORE, 140734438506496, 140734438645759, +STORE, 94652233351168, 94652235575295, +SNULL, 94652233461759, 94652235575295, +STORE, 94652233351168, 94652233461759, +STORE, 94652233461760, 94652235575295, +ERASE, 94652233461760, 94652235575295, +STORE, 94652235554816, 94652235567103, +STORE, 94652235567104, 94652235575295, +STORE, 140536493195264, 140536495448063, +SNULL, 140536493338623, 140536495448063, +STORE, 140536493195264, 140536493338623, +STORE, 140536493338624, 140536495448063, +ERASE, 140536493338624, 140536495448063, +STORE, 140536495435776, 140536495443967, +STORE, 140536495443968, 140536495448063, +STORE, 140734439002112, 140734439006207, +STORE, 140734438989824, 140734439002111, +STORE, 140536495407104, 140536495435775, +STORE, 140536495398912, 140536495407103, +STORE, 140536489398272, 140536493195263, +SNULL, 140536489398272, 140536491057151, +STORE, 140536491057152, 140536493195263, +STORE, 140536489398272, 140536491057151, +SNULL, 140536493154303, 140536493195263, +STORE, 140536491057152, 140536493154303, +STORE, 140536493154304, 140536493195263, +SNULL, 140536493154304, 140536493178879, +STORE, 140536493178880, 140536493195263, +STORE, 140536493154304, 140536493178879, +ERASE, 140536493154304, 140536493178879, +STORE, 140536493154304, 140536493178879, +ERASE, 140536493178880, 140536493195263, +STORE, 140536493178880, 140536493195263, +SNULL, 140536493170687, 140536493178879, +STORE, 140536493154304, 140536493170687, +STORE, 140536493170688, 140536493178879, +SNULL, 94652235563007, 94652235567103, +STORE, 94652235554816, 94652235563007, +STORE, 94652235563008, 94652235567103, +SNULL, 140536495439871, 140536495443967, +STORE, 140536495435776, 140536495439871, +STORE, 140536495439872, 140536495443967, +ERASE, 140536495407104, 140536495435775, +STORE, 94652265619456, 94652265754623, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140721814200320, 140737488351231, +SNULL, 140721814208511, 140737488351231, +STORE, 140721814200320, 140721814208511, +STORE, 140721814069248, 140721814208511, +STORE, 94062800691200, 94062802915327, +SNULL, 94062800801791, 94062802915327, +STORE, 94062800691200, 94062800801791, +STORE, 94062800801792, 94062802915327, +ERASE, 94062800801792, 94062802915327, +STORE, 94062802894848, 94062802907135, +STORE, 94062802907136, 94062802915327, +STORE, 139717739700224, 139717741953023, +SNULL, 139717739843583, 139717741953023, +STORE, 139717739700224, 139717739843583, +STORE, 139717739843584, 139717741953023, +ERASE, 139717739843584, 139717741953023, +STORE, 139717741940736, 139717741948927, +STORE, 139717741948928, 139717741953023, +STORE, 140721814224896, 140721814228991, +STORE, 140721814212608, 140721814224895, +STORE, 139717741912064, 139717741940735, +STORE, 139717741903872, 139717741912063, +STORE, 139717735903232, 139717739700223, +SNULL, 139717735903232, 139717737562111, +STORE, 139717737562112, 139717739700223, +STORE, 139717735903232, 139717737562111, +SNULL, 139717739659263, 139717739700223, +STORE, 139717737562112, 139717739659263, +STORE, 139717739659264, 139717739700223, +SNULL, 139717739659264, 139717739683839, +STORE, 139717739683840, 139717739700223, +STORE, 139717739659264, 139717739683839, +ERASE, 139717739659264, 139717739683839, +STORE, 139717739659264, 139717739683839, +ERASE, 139717739683840, 139717739700223, +STORE, 139717739683840, 139717739700223, +SNULL, 139717739675647, 139717739683839, +STORE, 139717739659264, 139717739675647, +STORE, 139717739675648, 139717739683839, +SNULL, 94062802903039, 94062802907135, +STORE, 94062802894848, 94062802903039, +STORE, 94062802903040, 94062802907135, +SNULL, 139717741944831, 139717741948927, +STORE, 139717741940736, 139717741944831, +STORE, 139717741944832, 139717741948927, +ERASE, 139717741912064, 139717741940735, +STORE, 94062814060544, 94062814195711, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723945754624, 140737488351231, +SNULL, 140723945762815, 140737488351231, +STORE, 140723945754624, 140723945762815, +STORE, 140723945623552, 140723945762815, +STORE, 94886119305216, 94886121639935, +SNULL, 94886119518207, 94886121639935, +STORE, 94886119305216, 94886119518207, +STORE, 94886119518208, 94886121639935, +ERASE, 94886119518208, 94886121639935, +STORE, 94886121615360, 94886121627647, +STORE, 94886121627648, 94886121639935, +STORE, 140152532131840, 140152534384639, +SNULL, 140152532275199, 140152534384639, +STORE, 140152532131840, 140152532275199, +STORE, 140152532275200, 140152534384639, +ERASE, 140152532275200, 140152534384639, +STORE, 140152534372352, 140152534380543, +STORE, 140152534380544, 140152534384639, +STORE, 140723946213376, 140723946217471, +STORE, 140723946201088, 140723946213375, +STORE, 140152534343680, 140152534372351, +STORE, 140152534335488, 140152534343679, +STORE, 140152530018304, 140152532131839, +SNULL, 140152530018304, 140152530030591, +STORE, 140152530030592, 140152532131839, +STORE, 140152530018304, 140152530030591, +SNULL, 140152532123647, 140152532131839, +STORE, 140152530030592, 140152532123647, +STORE, 140152532123648, 140152532131839, +ERASE, 140152532123648, 140152532131839, +STORE, 140152532123648, 140152532131839, +STORE, 140152526221312, 140152530018303, +SNULL, 140152526221312, 140152527880191, +STORE, 140152527880192, 140152530018303, +STORE, 140152526221312, 140152527880191, +SNULL, 140152529977343, 140152530018303, +STORE, 140152527880192, 140152529977343, +STORE, 140152529977344, 140152530018303, +SNULL, 140152529977344, 140152530001919, +STORE, 140152530001920, 140152530018303, +STORE, 140152529977344, 140152530001919, +ERASE, 140152529977344, 140152530001919, +STORE, 140152529977344, 140152530001919, +ERASE, 140152530001920, 140152530018303, +STORE, 140152530001920, 140152530018303, +STORE, 140152534327296, 140152534343679, +SNULL, 140152529993727, 140152530001919, +STORE, 140152529977344, 140152529993727, +STORE, 140152529993728, 140152530001919, +SNULL, 140152532127743, 140152532131839, +STORE, 140152532123648, 140152532127743, +STORE, 140152532127744, 140152532131839, +SNULL, 94886121619455, 94886121627647, +STORE, 94886121615360, 94886121619455, +STORE, 94886121619456, 94886121627647, +SNULL, 140152534376447, 140152534380543, +STORE, 140152534372352, 140152534376447, +STORE, 140152534376448, 140152534380543, +ERASE, 140152534343680, 140152534372351, +STORE, 94886129770496, 94886129905663, +STORE, 140152532643840, 140152534327295, +STORE, 94886129770496, 94886130040831, +STORE, 94886129770496, 94886130175999, +STORE, 94886129770496, 94886130348031, +SNULL, 94886130323455, 94886130348031, +STORE, 94886129770496, 94886130323455, +STORE, 94886130323456, 94886130348031, +ERASE, 94886130323456, 94886130348031, +STORE, 94886129770496, 94886130458623, +STORE, 94886129770496, 94886130606079, +SNULL, 94886130573311, 94886130606079, +STORE, 94886129770496, 94886130573311, +STORE, 94886130573312, 94886130606079, +ERASE, 94886130573312, 94886130606079, +STORE, 94886129770496, 94886130724863, +STORE, 94886129770496, 94886130876415, +STORE, 94886129770496, 94886131023871, +STORE, 94886129770496, 94886131175423, +STORE, 94886129770496, 94886131318783, +STORE, 94886129770496, 94886131453951, +SNULL, 94886131449855, 94886131453951, +STORE, 94886129770496, 94886131449855, +STORE, 94886131449856, 94886131453951, +ERASE, 94886131449856, 94886131453951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735450779648, 140737488351231, +SNULL, 140735450787839, 140737488351231, +STORE, 140735450779648, 140735450787839, +STORE, 140735450648576, 140735450787839, +STORE, 93947794079744, 93947796414463, +SNULL, 93947794292735, 93947796414463, +STORE, 93947794079744, 93947794292735, +STORE, 93947794292736, 93947796414463, +ERASE, 93947794292736, 93947796414463, +STORE, 93947796389888, 93947796402175, +STORE, 93947796402176, 93947796414463, +STORE, 139841993433088, 139841995685887, +SNULL, 139841993576447, 139841995685887, +STORE, 139841993433088, 139841993576447, +STORE, 139841993576448, 139841995685887, +ERASE, 139841993576448, 139841995685887, +STORE, 139841995673600, 139841995681791, +STORE, 139841995681792, 139841995685887, +STORE, 140735451308032, 140735451312127, +STORE, 140735451295744, 140735451308031, +STORE, 139841995644928, 139841995673599, +STORE, 139841995636736, 139841995644927, +STORE, 139841991319552, 139841993433087, +SNULL, 139841991319552, 139841991331839, +STORE, 139841991331840, 139841993433087, +STORE, 139841991319552, 139841991331839, +SNULL, 139841993424895, 139841993433087, +STORE, 139841991331840, 139841993424895, +STORE, 139841993424896, 139841993433087, +ERASE, 139841993424896, 139841993433087, +STORE, 139841993424896, 139841993433087, +STORE, 139841987522560, 139841991319551, +SNULL, 139841987522560, 139841989181439, +STORE, 139841989181440, 139841991319551, +STORE, 139841987522560, 139841989181439, +SNULL, 139841991278591, 139841991319551, +STORE, 139841989181440, 139841991278591, +STORE, 139841991278592, 139841991319551, +SNULL, 139841991278592, 139841991303167, +STORE, 139841991303168, 139841991319551, +STORE, 139841991278592, 139841991303167, +ERASE, 139841991278592, 139841991303167, +STORE, 139841991278592, 139841991303167, +ERASE, 139841991303168, 139841991319551, +STORE, 139841991303168, 139841991319551, +STORE, 139841995628544, 139841995644927, +SNULL, 139841991294975, 139841991303167, +STORE, 139841991278592, 139841991294975, +STORE, 139841991294976, 139841991303167, +SNULL, 139841993428991, 139841993433087, +STORE, 139841993424896, 139841993428991, +STORE, 139841993428992, 139841993433087, +SNULL, 93947796393983, 93947796402175, +STORE, 93947796389888, 93947796393983, +STORE, 93947796393984, 93947796402175, +SNULL, 139841995677695, 139841995681791, +STORE, 139841995673600, 139841995677695, +STORE, 139841995677696, 139841995681791, +ERASE, 139841995644928, 139841995673599, +STORE, 93947829739520, 93947829874687, +STORE, 139841993945088, 139841995628543, +STORE, 93947829739520, 93947830009855, +STORE, 93947829739520, 93947830145023, +STORE, 94659351814144, 94659352027135, +STORE, 94659354124288, 94659354128383, +STORE, 94659354128384, 94659354136575, +STORE, 94659354136576, 94659354148863, +STORE, 94659383476224, 94659385057279, +STORE, 139959054557184, 139959056216063, +STORE, 139959056216064, 139959058313215, +STORE, 139959058313216, 139959058329599, +STORE, 139959058329600, 139959058337791, +STORE, 139959058337792, 139959058354175, +STORE, 139959058354176, 139959058366463, +STORE, 139959058366464, 139959060459519, +STORE, 139959060459520, 139959060463615, +STORE, 139959060463616, 139959060467711, +STORE, 139959060467712, 139959060611071, +STORE, 139959060979712, 139959062663167, +STORE, 139959062663168, 139959062679551, +STORE, 139959062708224, 139959062712319, +STORE, 139959062712320, 139959062716415, +STORE, 139959062716416, 139959062720511, +STORE, 140735532539904, 140735532679167, +STORE, 140735532830720, 140735532843007, +STORE, 140735532843008, 140735532847103, +STORE, 93894361829376, 93894362042367, +STORE, 93894364139520, 93894364143615, +STORE, 93894364143616, 93894364151807, +STORE, 93894364151808, 93894364164095, +STORE, 93894396944384, 93894397624319, +STORE, 140075612573696, 140075614232575, +STORE, 140075614232576, 140075616329727, +STORE, 140075616329728, 140075616346111, +STORE, 140075616346112, 140075616354303, +STORE, 140075616354304, 140075616370687, +STORE, 140075616370688, 140075616382975, +STORE, 140075616382976, 140075618476031, +STORE, 140075618476032, 140075618480127, +STORE, 140075618480128, 140075618484223, +STORE, 140075618484224, 140075618627583, +STORE, 140075618996224, 140075620679679, +STORE, 140075620679680, 140075620696063, +STORE, 140075620724736, 140075620728831, +STORE, 140075620728832, 140075620732927, +STORE, 140075620732928, 140075620737023, +STORE, 140720830312448, 140720830451711, +STORE, 140720830631936, 140720830644223, +STORE, 140720830644224, 140720830648319, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735116226560, 140737488351231, +SNULL, 140735116234751, 140737488351231, +STORE, 140735116226560, 140735116234751, +STORE, 140735116095488, 140735116234751, +STORE, 94873398054912, 94873400279039, +SNULL, 94873398165503, 94873400279039, +STORE, 94873398054912, 94873398165503, +STORE, 94873398165504, 94873400279039, +ERASE, 94873398165504, 94873400279039, +STORE, 94873400258560, 94873400270847, +STORE, 94873400270848, 94873400279039, +STORE, 140303828606976, 140303830859775, +SNULL, 140303828750335, 140303830859775, +STORE, 140303828606976, 140303828750335, +STORE, 140303828750336, 140303830859775, +ERASE, 140303828750336, 140303830859775, +STORE, 140303830847488, 140303830855679, +STORE, 140303830855680, 140303830859775, +STORE, 140735116251136, 140735116255231, +STORE, 140735116238848, 140735116251135, +STORE, 140303830818816, 140303830847487, +STORE, 140303830810624, 140303830818815, +STORE, 140303824809984, 140303828606975, +SNULL, 140303824809984, 140303826468863, +STORE, 140303826468864, 140303828606975, +STORE, 140303824809984, 140303826468863, +SNULL, 140303828566015, 140303828606975, +STORE, 140303826468864, 140303828566015, +STORE, 140303828566016, 140303828606975, +SNULL, 140303828566016, 140303828590591, +STORE, 140303828590592, 140303828606975, +STORE, 140303828566016, 140303828590591, +ERASE, 140303828566016, 140303828590591, +STORE, 140303828566016, 140303828590591, +ERASE, 140303828590592, 140303828606975, +STORE, 140303828590592, 140303828606975, +SNULL, 140303828582399, 140303828590591, +STORE, 140303828566016, 140303828582399, +STORE, 140303828582400, 140303828590591, +SNULL, 94873400266751, 94873400270847, +STORE, 94873400258560, 94873400266751, +STORE, 94873400266752, 94873400270847, +SNULL, 140303830851583, 140303830855679, +STORE, 140303830847488, 140303830851583, +STORE, 140303830851584, 140303830855679, +ERASE, 140303830818816, 140303830847487, +STORE, 94873413713920, 94873413849087, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140732349956096, 140737488351231, +SNULL, 140732349964287, 140737488351231, +STORE, 140732349956096, 140732349964287, +STORE, 140732349825024, 140732349964287, +STORE, 94009652736000, 94009655070719, +SNULL, 94009652948991, 94009655070719, +STORE, 94009652736000, 94009652948991, +STORE, 94009652948992, 94009655070719, +ERASE, 94009652948992, 94009655070719, +STORE, 94009655046144, 94009655058431, +STORE, 94009655058432, 94009655070719, +STORE, 140295688531968, 140295690784767, +SNULL, 140295688675327, 140295690784767, +STORE, 140295688531968, 140295688675327, +STORE, 140295688675328, 140295690784767, +ERASE, 140295688675328, 140295690784767, +STORE, 140295690772480, 140295690780671, +STORE, 140295690780672, 140295690784767, +STORE, 140732350005248, 140732350009343, +STORE, 140732349992960, 140732350005247, +STORE, 140295690743808, 140295690772479, +STORE, 140295690735616, 140295690743807, +STORE, 140295686418432, 140295688531967, +SNULL, 140295686418432, 140295686430719, +STORE, 140295686430720, 140295688531967, +STORE, 140295686418432, 140295686430719, +SNULL, 140295688523775, 140295688531967, +STORE, 140295686430720, 140295688523775, +STORE, 140295688523776, 140295688531967, +ERASE, 140295688523776, 140295688531967, +STORE, 140295688523776, 140295688531967, +STORE, 140295682621440, 140295686418431, +SNULL, 140295682621440, 140295684280319, +STORE, 140295684280320, 140295686418431, +STORE, 140295682621440, 140295684280319, +SNULL, 140295686377471, 140295686418431, +STORE, 140295684280320, 140295686377471, +STORE, 140295686377472, 140295686418431, +SNULL, 140295686377472, 140295686402047, +STORE, 140295686402048, 140295686418431, +STORE, 140295686377472, 140295686402047, +ERASE, 140295686377472, 140295686402047, +STORE, 140295686377472, 140295686402047, +ERASE, 140295686402048, 140295686418431, +STORE, 140295686402048, 140295686418431, +STORE, 140295690727424, 140295690743807, +SNULL, 140295686393855, 140295686402047, +STORE, 140295686377472, 140295686393855, +STORE, 140295686393856, 140295686402047, +SNULL, 140295688527871, 140295688531967, +STORE, 140295688523776, 140295688527871, +STORE, 140295688527872, 140295688531967, +SNULL, 94009655050239, 94009655058431, +STORE, 94009655046144, 94009655050239, +STORE, 94009655050240, 94009655058431, +SNULL, 140295690776575, 140295690780671, +STORE, 140295690772480, 140295690776575, +STORE, 140295690776576, 140295690780671, +ERASE, 140295690743808, 140295690772479, +STORE, 94009672114176, 94009672249343, +STORE, 140295689043968, 140295690727423, +STORE, 94009672114176, 94009672384511, +STORE, 94009672114176, 94009672519679, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722376515584, 140737488351231, +SNULL, 140722376523775, 140737488351231, +STORE, 140722376515584, 140722376523775, +STORE, 140722376384512, 140722376523775, +STORE, 94089815773184, 94089818107903, +SNULL, 94089815986175, 94089818107903, +STORE, 94089815773184, 94089815986175, +STORE, 94089815986176, 94089818107903, +ERASE, 94089815986176, 94089818107903, +STORE, 94089818083328, 94089818095615, +STORE, 94089818095616, 94089818107903, +STORE, 140265595711488, 140265597964287, +SNULL, 140265595854847, 140265597964287, +STORE, 140265595711488, 140265595854847, +STORE, 140265595854848, 140265597964287, +ERASE, 140265595854848, 140265597964287, +STORE, 140265597952000, 140265597960191, +STORE, 140265597960192, 140265597964287, +STORE, 140722378297344, 140722378301439, +STORE, 140722378285056, 140722378297343, +STORE, 140265597923328, 140265597951999, +STORE, 140265597915136, 140265597923327, +STORE, 140265593597952, 140265595711487, +SNULL, 140265593597952, 140265593610239, +STORE, 140265593610240, 140265595711487, +STORE, 140265593597952, 140265593610239, +SNULL, 140265595703295, 140265595711487, +STORE, 140265593610240, 140265595703295, +STORE, 140265595703296, 140265595711487, +ERASE, 140265595703296, 140265595711487, +STORE, 140265595703296, 140265595711487, +STORE, 140265589800960, 140265593597951, +SNULL, 140265589800960, 140265591459839, +STORE, 140265591459840, 140265593597951, +STORE, 140265589800960, 140265591459839, +SNULL, 140265593556991, 140265593597951, +STORE, 140265591459840, 140265593556991, +STORE, 140265593556992, 140265593597951, +SNULL, 140265593556992, 140265593581567, +STORE, 140265593581568, 140265593597951, +STORE, 140265593556992, 140265593581567, +ERASE, 140265593556992, 140265593581567, +STORE, 140265593556992, 140265593581567, +ERASE, 140265593581568, 140265593597951, +STORE, 140265593581568, 140265593597951, +STORE, 140265597906944, 140265597923327, +SNULL, 140265593573375, 140265593581567, +STORE, 140265593556992, 140265593573375, +STORE, 140265593573376, 140265593581567, +SNULL, 140265595707391, 140265595711487, +STORE, 140265595703296, 140265595707391, +STORE, 140265595707392, 140265595711487, +SNULL, 94089818087423, 94089818095615, +STORE, 94089818083328, 94089818087423, +STORE, 94089818087424, 94089818095615, +SNULL, 140265597956095, 140265597960191, +STORE, 140265597952000, 140265597956095, +STORE, 140265597956096, 140265597960191, +ERASE, 140265597923328, 140265597951999, +STORE, 94089837146112, 94089837281279, +STORE, 140265596223488, 140265597906943, +STORE, 94089837146112, 94089837416447, +STORE, 94089837146112, 94089837551615, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735265218560, 140737488351231, +SNULL, 140735265226751, 140737488351231, +STORE, 140735265218560, 140735265226751, +STORE, 140735265087488, 140735265226751, +STORE, 94250422370304, 94250424705023, +SNULL, 94250422583295, 94250424705023, +STORE, 94250422370304, 94250422583295, +STORE, 94250422583296, 94250424705023, +ERASE, 94250422583296, 94250424705023, +STORE, 94250424680448, 94250424692735, +STORE, 94250424692736, 94250424705023, +STORE, 140344442474496, 140344444727295, +SNULL, 140344442617855, 140344444727295, +STORE, 140344442474496, 140344442617855, +STORE, 140344442617856, 140344444727295, +ERASE, 140344442617856, 140344444727295, +STORE, 140344444715008, 140344444723199, +STORE, 140344444723200, 140344444727295, +STORE, 140735265341440, 140735265345535, +STORE, 140735265329152, 140735265341439, +STORE, 140344444686336, 140344444715007, +STORE, 140344444678144, 140344444686335, +STORE, 140344440360960, 140344442474495, +SNULL, 140344440360960, 140344440373247, +STORE, 140344440373248, 140344442474495, +STORE, 140344440360960, 140344440373247, +SNULL, 140344442466303, 140344442474495, +STORE, 140344440373248, 140344442466303, +STORE, 140344442466304, 140344442474495, +ERASE, 140344442466304, 140344442474495, +STORE, 140344442466304, 140344442474495, +STORE, 140344436563968, 140344440360959, +SNULL, 140344436563968, 140344438222847, +STORE, 140344438222848, 140344440360959, +STORE, 140344436563968, 140344438222847, +SNULL, 140344440319999, 140344440360959, +STORE, 140344438222848, 140344440319999, +STORE, 140344440320000, 140344440360959, +SNULL, 140344440320000, 140344440344575, +STORE, 140344440344576, 140344440360959, +STORE, 140344440320000, 140344440344575, +ERASE, 140344440320000, 140344440344575, +STORE, 140344440320000, 140344440344575, +ERASE, 140344440344576, 140344440360959, +STORE, 140344440344576, 140344440360959, +STORE, 140344444669952, 140344444686335, +SNULL, 140344440336383, 140344440344575, +STORE, 140344440320000, 140344440336383, +STORE, 140344440336384, 140344440344575, +SNULL, 140344442470399, 140344442474495, +STORE, 140344442466304, 140344442470399, +STORE, 140344442470400, 140344442474495, +SNULL, 94250424684543, 94250424692735, +STORE, 94250424680448, 94250424684543, +STORE, 94250424684544, 94250424692735, +SNULL, 140344444719103, 140344444723199, +STORE, 140344444715008, 140344444719103, +STORE, 140344444719104, 140344444723199, +ERASE, 140344444686336, 140344444715007, +STORE, 94250445512704, 94250445647871, +STORE, 140344442986496, 140344444669951, +STORE, 94250445512704, 94250445783039, +STORE, 94250445512704, 94250445918207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725762719744, 140737488351231, +SNULL, 140725762727935, 140737488351231, +STORE, 140725762719744, 140725762727935, +STORE, 140725762588672, 140725762727935, +STORE, 94819009097728, 94819011432447, +SNULL, 94819009310719, 94819011432447, +STORE, 94819009097728, 94819009310719, +STORE, 94819009310720, 94819011432447, +ERASE, 94819009310720, 94819011432447, +STORE, 94819011407872, 94819011420159, +STORE, 94819011420160, 94819011432447, +STORE, 139987985596416, 139987987849215, +SNULL, 139987985739775, 139987987849215, +STORE, 139987985596416, 139987985739775, +STORE, 139987985739776, 139987987849215, +ERASE, 139987985739776, 139987987849215, +STORE, 139987987836928, 139987987845119, +STORE, 139987987845120, 139987987849215, +STORE, 140725763072000, 140725763076095, +STORE, 140725763059712, 140725763071999, +STORE, 139987987808256, 139987987836927, +STORE, 139987987800064, 139987987808255, +STORE, 139987983482880, 139987985596415, +SNULL, 139987983482880, 139987983495167, +STORE, 139987983495168, 139987985596415, +STORE, 139987983482880, 139987983495167, +SNULL, 139987985588223, 139987985596415, +STORE, 139987983495168, 139987985588223, +STORE, 139987985588224, 139987985596415, +ERASE, 139987985588224, 139987985596415, +STORE, 139987985588224, 139987985596415, +STORE, 139987979685888, 139987983482879, +SNULL, 139987979685888, 139987981344767, +STORE, 139987981344768, 139987983482879, +STORE, 139987979685888, 139987981344767, +SNULL, 139987983441919, 139987983482879, +STORE, 139987981344768, 139987983441919, +STORE, 139987983441920, 139987983482879, +SNULL, 139987983441920, 139987983466495, +STORE, 139987983466496, 139987983482879, +STORE, 139987983441920, 139987983466495, +ERASE, 139987983441920, 139987983466495, +STORE, 139987983441920, 139987983466495, +ERASE, 139987983466496, 139987983482879, +STORE, 139987983466496, 139987983482879, +STORE, 139987987791872, 139987987808255, +SNULL, 139987983458303, 139987983466495, +STORE, 139987983441920, 139987983458303, +STORE, 139987983458304, 139987983466495, +SNULL, 139987985592319, 139987985596415, +STORE, 139987985588224, 139987985592319, +STORE, 139987985592320, 139987985596415, +SNULL, 94819011411967, 94819011420159, +STORE, 94819011407872, 94819011411967, +STORE, 94819011411968, 94819011420159, +SNULL, 139987987841023, 139987987845119, +STORE, 139987987836928, 139987987841023, +STORE, 139987987841024, 139987987845119, +ERASE, 139987987808256, 139987987836927, +STORE, 94819028176896, 94819028312063, +STORE, 139987986108416, 139987987791871, +STORE, 94819028176896, 94819028447231, +STORE, 94819028176896, 94819028582399, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722475413504, 140737488351231, +SNULL, 140722475421695, 140737488351231, +STORE, 140722475413504, 140722475421695, +STORE, 140722475282432, 140722475421695, +STORE, 94620599119872, 94620601343999, +SNULL, 94620599230463, 94620601343999, +STORE, 94620599119872, 94620599230463, +STORE, 94620599230464, 94620601343999, +ERASE, 94620599230464, 94620601343999, +STORE, 94620601323520, 94620601335807, +STORE, 94620601335808, 94620601343999, +STORE, 139891763060736, 139891765313535, +SNULL, 139891763204095, 139891765313535, +STORE, 139891763060736, 139891763204095, +STORE, 139891763204096, 139891765313535, +ERASE, 139891763204096, 139891765313535, +STORE, 139891765301248, 139891765309439, +STORE, 139891765309440, 139891765313535, +STORE, 140722475700224, 140722475704319, +STORE, 140722475687936, 140722475700223, +STORE, 139891765272576, 139891765301247, +STORE, 139891765264384, 139891765272575, +STORE, 139891759263744, 139891763060735, +SNULL, 139891759263744, 139891760922623, +STORE, 139891760922624, 139891763060735, +STORE, 139891759263744, 139891760922623, +SNULL, 139891763019775, 139891763060735, +STORE, 139891760922624, 139891763019775, +STORE, 139891763019776, 139891763060735, +SNULL, 139891763019776, 139891763044351, +STORE, 139891763044352, 139891763060735, +STORE, 139891763019776, 139891763044351, +ERASE, 139891763019776, 139891763044351, +STORE, 139891763019776, 139891763044351, +ERASE, 139891763044352, 139891763060735, +STORE, 139891763044352, 139891763060735, +SNULL, 139891763036159, 139891763044351, +STORE, 139891763019776, 139891763036159, +STORE, 139891763036160, 139891763044351, +SNULL, 94620601331711, 94620601335807, +STORE, 94620601323520, 94620601331711, +STORE, 94620601331712, 94620601335807, +SNULL, 139891765305343, 139891765309439, +STORE, 139891765301248, 139891765305343, +STORE, 139891765305344, 139891765309439, +ERASE, 139891765272576, 139891765301247, +STORE, 94620610027520, 94620610162687, +STORE, 94031976210432, 94031976423423, +STORE, 94031978520576, 94031978524671, +STORE, 94031978524672, 94031978532863, +STORE, 94031978532864, 94031978545151, +STORE, 94031990398976, 94031992565759, +STORE, 140336240640000, 140336242298879, +STORE, 140336242298880, 140336244396031, +STORE, 140336244396032, 140336244412415, +STORE, 140336244412416, 140336244420607, +STORE, 140336244420608, 140336244436991, +STORE, 140336244436992, 140336244449279, +STORE, 140336244449280, 140336246542335, +STORE, 140336246542336, 140336246546431, +STORE, 140336246546432, 140336246550527, +STORE, 140336246550528, 140336246693887, +STORE, 140336247062528, 140336248745983, +STORE, 140336248745984, 140336248762367, +STORE, 140336248791040, 140336248795135, +STORE, 140336248795136, 140336248799231, +STORE, 140336248799232, 140336248803327, +STORE, 140728500064256, 140728500203519, +STORE, 140728501501952, 140728501514239, +STORE, 140728501514240, 140728501518335, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730503987200, 140737488351231, +SNULL, 140730503995391, 140737488351231, +STORE, 140730503987200, 140730503995391, +STORE, 140730503856128, 140730503995391, +STORE, 93866544205824, 93866546429951, +SNULL, 93866544316415, 93866546429951, +STORE, 93866544205824, 93866544316415, +STORE, 93866544316416, 93866546429951, +ERASE, 93866544316416, 93866546429951, +STORE, 93866546409472, 93866546421759, +STORE, 93866546421760, 93866546429951, +STORE, 140216311959552, 140216314212351, +SNULL, 140216312102911, 140216314212351, +STORE, 140216311959552, 140216312102911, +STORE, 140216312102912, 140216314212351, +ERASE, 140216312102912, 140216314212351, +STORE, 140216314200064, 140216314208255, +STORE, 140216314208256, 140216314212351, +STORE, 140730504626176, 140730504630271, +STORE, 140730504613888, 140730504626175, +STORE, 140216314171392, 140216314200063, +STORE, 140216314163200, 140216314171391, +STORE, 140216308162560, 140216311959551, +SNULL, 140216308162560, 140216309821439, +STORE, 140216309821440, 140216311959551, +STORE, 140216308162560, 140216309821439, +SNULL, 140216311918591, 140216311959551, +STORE, 140216309821440, 140216311918591, +STORE, 140216311918592, 140216311959551, +SNULL, 140216311918592, 140216311943167, +STORE, 140216311943168, 140216311959551, +STORE, 140216311918592, 140216311943167, +ERASE, 140216311918592, 140216311943167, +STORE, 140216311918592, 140216311943167, +ERASE, 140216311943168, 140216311959551, +STORE, 140216311943168, 140216311959551, +SNULL, 140216311934975, 140216311943167, +STORE, 140216311918592, 140216311934975, +STORE, 140216311934976, 140216311943167, +SNULL, 93866546417663, 93866546421759, +STORE, 93866546409472, 93866546417663, +STORE, 93866546417664, 93866546421759, +SNULL, 140216314204159, 140216314208255, +STORE, 140216314200064, 140216314204159, +STORE, 140216314204160, 140216314208255, +ERASE, 140216314171392, 140216314200063, +STORE, 93866550386688, 93866550521855, +STORE, 94074292674560, 94074292887551, +STORE, 94074294984704, 94074294988799, +STORE, 94074294988800, 94074294996991, +STORE, 94074294996992, 94074295009279, +STORE, 94074300219392, 94074301378559, +STORE, 139781563256832, 139781564915711, +STORE, 139781564915712, 139781567012863, +STORE, 139781567012864, 139781567029247, +STORE, 139781567029248, 139781567037439, +STORE, 139781567037440, 139781567053823, +STORE, 139781567053824, 139781567066111, +STORE, 139781567066112, 139781569159167, +STORE, 139781569159168, 139781569163263, +STORE, 139781569163264, 139781569167359, +STORE, 139781569167360, 139781569310719, +STORE, 139781569679360, 139781571362815, +STORE, 139781571362816, 139781571379199, +STORE, 139781571407872, 139781571411967, +STORE, 139781571411968, 139781571416063, +STORE, 139781571416064, 139781571420159, +STORE, 140723688488960, 140723688628223, +STORE, 140723689005056, 140723689017343, +STORE, 140723689017344, 140723689021439, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735189745664, 140737488351231, +SNULL, 140735189753855, 140737488351231, +STORE, 140735189745664, 140735189753855, +STORE, 140735189614592, 140735189753855, +STORE, 94172072177664, 94172074512383, +SNULL, 94172072390655, 94172074512383, +STORE, 94172072177664, 94172072390655, +STORE, 94172072390656, 94172074512383, +ERASE, 94172072390656, 94172074512383, +STORE, 94172074487808, 94172074500095, +STORE, 94172074500096, 94172074512383, +STORE, 140687827263488, 140687829516287, +SNULL, 140687827406847, 140687829516287, +STORE, 140687827263488, 140687827406847, +STORE, 140687827406848, 140687829516287, +ERASE, 140687827406848, 140687829516287, +STORE, 140687829504000, 140687829512191, +STORE, 140687829512192, 140687829516287, +STORE, 140735189766144, 140735189770239, +STORE, 140735189753856, 140735189766143, +STORE, 140687829475328, 140687829503999, +STORE, 140687829467136, 140687829475327, +STORE, 140687825149952, 140687827263487, +SNULL, 140687825149952, 140687825162239, +STORE, 140687825162240, 140687827263487, +STORE, 140687825149952, 140687825162239, +SNULL, 140687827255295, 140687827263487, +STORE, 140687825162240, 140687827255295, +STORE, 140687827255296, 140687827263487, +ERASE, 140687827255296, 140687827263487, +STORE, 140687827255296, 140687827263487, +STORE, 140687821352960, 140687825149951, +SNULL, 140687821352960, 140687823011839, +STORE, 140687823011840, 140687825149951, +STORE, 140687821352960, 140687823011839, +SNULL, 140687825108991, 140687825149951, +STORE, 140687823011840, 140687825108991, +STORE, 140687825108992, 140687825149951, +SNULL, 140687825108992, 140687825133567, +STORE, 140687825133568, 140687825149951, +STORE, 140687825108992, 140687825133567, +ERASE, 140687825108992, 140687825133567, +STORE, 140687825108992, 140687825133567, +ERASE, 140687825133568, 140687825149951, +STORE, 140687825133568, 140687825149951, +STORE, 140687829458944, 140687829475327, +SNULL, 140687825125375, 140687825133567, +STORE, 140687825108992, 140687825125375, +STORE, 140687825125376, 140687825133567, +SNULL, 140687827259391, 140687827263487, +STORE, 140687827255296, 140687827259391, +STORE, 140687827259392, 140687827263487, +SNULL, 94172074491903, 94172074500095, +STORE, 94172074487808, 94172074491903, +STORE, 94172074491904, 94172074500095, +SNULL, 140687829508095, 140687829512191, +STORE, 140687829504000, 140687829508095, +STORE, 140687829508096, 140687829512191, +ERASE, 140687829475328, 140687829503999, +STORE, 94172092432384, 94172092567551, +STORE, 140687827775488, 140687829458943, +STORE, 94172092432384, 94172092702719, +STORE, 94172092432384, 94172092837887, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737229504512, 140737488351231, +SNULL, 140737229512703, 140737488351231, +STORE, 140737229504512, 140737229512703, +STORE, 140737229373440, 140737229512703, +STORE, 94155246866432, 94155249090559, +SNULL, 94155246977023, 94155249090559, +STORE, 94155246866432, 94155246977023, +STORE, 94155246977024, 94155249090559, +ERASE, 94155246977024, 94155249090559, +STORE, 94155249070080, 94155249082367, +STORE, 94155249082368, 94155249090559, +STORE, 140640993693696, 140640995946495, +SNULL, 140640993837055, 140640995946495, +STORE, 140640993693696, 140640993837055, +STORE, 140640993837056, 140640995946495, +ERASE, 140640993837056, 140640995946495, +STORE, 140640995934208, 140640995942399, +STORE, 140640995942400, 140640995946495, +STORE, 140737230004224, 140737230008319, +STORE, 140737229991936, 140737230004223, +STORE, 140640995905536, 140640995934207, +STORE, 140640995897344, 140640995905535, +STORE, 140640989896704, 140640993693695, +SNULL, 140640989896704, 140640991555583, +STORE, 140640991555584, 140640993693695, +STORE, 140640989896704, 140640991555583, +SNULL, 140640993652735, 140640993693695, +STORE, 140640991555584, 140640993652735, +STORE, 140640993652736, 140640993693695, +SNULL, 140640993652736, 140640993677311, +STORE, 140640993677312, 140640993693695, +STORE, 140640993652736, 140640993677311, +ERASE, 140640993652736, 140640993677311, +STORE, 140640993652736, 140640993677311, +ERASE, 140640993677312, 140640993693695, +STORE, 140640993677312, 140640993693695, +SNULL, 140640993669119, 140640993677311, +STORE, 140640993652736, 140640993669119, +STORE, 140640993669120, 140640993677311, +SNULL, 94155249078271, 94155249082367, +STORE, 94155249070080, 94155249078271, +STORE, 94155249078272, 94155249082367, +SNULL, 140640995938303, 140640995942399, +STORE, 140640995934208, 140640995938303, +STORE, 140640995938304, 140640995942399, +ERASE, 140640995905536, 140640995934207, +STORE, 94155281035264, 94155281170431, +STORE, 94088066453504, 94088066564095, +STORE, 94088068657152, 94088068665343, +STORE, 94088068665344, 94088068669439, +STORE, 94088068669440, 94088068677631, +STORE, 94088090214400, 94088090349567, +STORE, 140503024627712, 140503026286591, +STORE, 140503026286592, 140503028383743, +STORE, 140503028383744, 140503028400127, +STORE, 140503028400128, 140503028408319, +STORE, 140503028408320, 140503028424703, +STORE, 140503028424704, 140503028568063, +STORE, 140503030628352, 140503030636543, +STORE, 140503030665216, 140503030669311, +STORE, 140503030669312, 140503030673407, +STORE, 140503030673408, 140503030677503, +STORE, 140730894725120, 140730894864383, +STORE, 140730894880768, 140730894893055, +STORE, 140730894893056, 140730894897151, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730434342912, 140737488351231, +SNULL, 140730434351103, 140737488351231, +STORE, 140730434342912, 140730434351103, +STORE, 140730434211840, 140730434351103, +STORE, 4194304, 5128191, +STORE, 7221248, 7241727, +STORE, 7241728, 7249919, +STORE, 140109041938432, 140109044191231, +SNULL, 140109042081791, 140109044191231, +STORE, 140109041938432, 140109042081791, +STORE, 140109042081792, 140109044191231, +ERASE, 140109042081792, 140109044191231, +STORE, 140109044178944, 140109044187135, +STORE, 140109044187136, 140109044191231, +STORE, 140730434850816, 140730434854911, +STORE, 140730434838528, 140730434850815, +STORE, 140109044150272, 140109044178943, +STORE, 140109044142080, 140109044150271, +STORE, 140109038776320, 140109041938431, +SNULL, 140109038776320, 140109039837183, +STORE, 140109039837184, 140109041938431, +STORE, 140109038776320, 140109039837183, +SNULL, 140109041930239, 140109041938431, +STORE, 140109039837184, 140109041930239, +STORE, 140109041930240, 140109041938431, +ERASE, 140109041930240, 140109041938431, +STORE, 140109041930240, 140109041938431, +STORE, 140109034979328, 140109038776319, +SNULL, 140109034979328, 140109036638207, +STORE, 140109036638208, 140109038776319, +STORE, 140109034979328, 140109036638207, +SNULL, 140109038735359, 140109038776319, +STORE, 140109036638208, 140109038735359, +STORE, 140109038735360, 140109038776319, +SNULL, 140109038735360, 140109038759935, +STORE, 140109038759936, 140109038776319, +STORE, 140109038735360, 140109038759935, +ERASE, 140109038735360, 140109038759935, +STORE, 140109038735360, 140109038759935, +ERASE, 140109038759936, 140109038776319, +STORE, 140109038759936, 140109038776319, +STORE, 140109044129792, 140109044150271, +SNULL, 140109038751743, 140109038759935, +STORE, 140109038735360, 140109038751743, +STORE, 140109038751744, 140109038759935, +SNULL, 140109041934335, 140109041938431, +STORE, 140109041930240, 140109041934335, +STORE, 140109041934336, 140109041938431, +SNULL, 7233535, 7241727, +STORE, 7221248, 7233535, +STORE, 7233536, 7241727, +SNULL, 140109044183039, 140109044187135, +STORE, 140109044178944, 140109044183039, +STORE, 140109044183040, 140109044187135, +ERASE, 140109044150272, 140109044178943, +STORE, 20000768, 20135935, +STORE, 20000768, 20283391, +STORE, 140109042446336, 140109044129791, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140730853408768, 140737488351231, +SNULL, 140730853416959, 140737488351231, +STORE, 140730853408768, 140730853416959, +STORE, 140730853277696, 140730853416959, +STORE, 94865902977024, 94865905311743, +SNULL, 94865903190015, 94865905311743, +STORE, 94865902977024, 94865903190015, +STORE, 94865903190016, 94865905311743, +ERASE, 94865903190016, 94865905311743, +STORE, 94865905287168, 94865905299455, +STORE, 94865905299456, 94865905311743, +STORE, 139768865738752, 139768867991551, +SNULL, 139768865882111, 139768867991551, +STORE, 139768865738752, 139768865882111, +STORE, 139768865882112, 139768867991551, +ERASE, 139768865882112, 139768867991551, +STORE, 139768867979264, 139768867987455, +STORE, 139768867987456, 139768867991551, +STORE, 140730853957632, 140730853961727, +STORE, 140730853945344, 140730853957631, +STORE, 139768867950592, 139768867979263, +STORE, 139768867942400, 139768867950591, +STORE, 139768863625216, 139768865738751, +SNULL, 139768863625216, 139768863637503, +STORE, 139768863637504, 139768865738751, +STORE, 139768863625216, 139768863637503, +SNULL, 139768865730559, 139768865738751, +STORE, 139768863637504, 139768865730559, +STORE, 139768865730560, 139768865738751, +ERASE, 139768865730560, 139768865738751, +STORE, 139768865730560, 139768865738751, +STORE, 139768859828224, 139768863625215, +SNULL, 139768859828224, 139768861487103, +STORE, 139768861487104, 139768863625215, +STORE, 139768859828224, 139768861487103, +SNULL, 139768863584255, 139768863625215, +STORE, 139768861487104, 139768863584255, +STORE, 139768863584256, 139768863625215, +SNULL, 139768863584256, 139768863608831, +STORE, 139768863608832, 139768863625215, +STORE, 139768863584256, 139768863608831, +ERASE, 139768863584256, 139768863608831, +STORE, 139768863584256, 139768863608831, +ERASE, 139768863608832, 139768863625215, +STORE, 139768863608832, 139768863625215, +STORE, 139768867934208, 139768867950591, +SNULL, 139768863600639, 139768863608831, +STORE, 139768863584256, 139768863600639, +STORE, 139768863600640, 139768863608831, +SNULL, 139768865734655, 139768865738751, +STORE, 139768865730560, 139768865734655, +STORE, 139768865734656, 139768865738751, +SNULL, 94865905291263, 94865905299455, +STORE, 94865905287168, 94865905291263, +STORE, 94865905291264, 94865905299455, +SNULL, 139768867983359, 139768867987455, +STORE, 139768867979264, 139768867983359, +STORE, 139768867983360, 139768867987455, +ERASE, 139768867950592, 139768867979263, +STORE, 94865923670016, 94865923805183, +STORE, 139768866250752, 139768867934207, +STORE, 94865923670016, 94865923940351, +STORE, 94865923670016, 94865924075519, +STORE, 94865923670016, 94865924222975, +SNULL, 94865924210687, 94865924222975, +STORE, 94865923670016, 94865924210687, +STORE, 94865924210688, 94865924222975, +ERASE, 94865924210688, 94865924222975, +STORE, 94865923670016, 94865924349951, +STORE, 94865923670016, 94865924493311, +STORE, 94865923670016, 94865924640767, +SNULL, 94865924603903, 94865924640767, +STORE, 94865923670016, 94865924603903, +STORE, 94865924603904, 94865924640767, +ERASE, 94865924603904, 94865924640767, +STORE, 94865923670016, 94865924747263, +STORE, 94865923670016, 94865924898815, +SNULL, 94865924874239, 94865924898815, +STORE, 94865923670016, 94865924874239, +STORE, 94865924874240, 94865924898815, +ERASE, 94865924874240, 94865924898815, +STORE, 94865923670016, 94865925025791, +SNULL, 94865925013503, 94865925025791, +STORE, 94865923670016, 94865925013503, +STORE, 94865925013504, 94865925025791, +ERASE, 94865925013504, 94865925025791, +SNULL, 94865924988927, 94865925013503, +STORE, 94865923670016, 94865924988927, +STORE, 94865924988928, 94865925013503, +ERASE, 94865924988928, 94865925013503, +STORE, 94865923670016, 94865925152767, +SNULL, 94865925136383, 94865925152767, +STORE, 94865923670016, 94865925136383, +STORE, 94865925136384, 94865925152767, +ERASE, 94865925136384, 94865925152767, +STORE, 94865923670016, 94865925292031, +SNULL, 94865925279743, 94865925292031, +STORE, 94865923670016, 94865925279743, +STORE, 94865925279744, 94865925292031, +ERASE, 94865925279744, 94865925292031, +SNULL, 94865925255167, 94865925279743, +STORE, 94865923670016, 94865925255167, +STORE, 94865925255168, 94865925279743, +ERASE, 94865925255168, 94865925279743, +STORE, 94865923670016, 94865925406719, +SNULL, 94865925394431, 94865925406719, +STORE, 94865923670016, 94865925394431, +STORE, 94865925394432, 94865925406719, +ERASE, 94865925394432, 94865925406719, +STORE, 94865923670016, 94865925545983, +SNULL, 94865925533695, 94865925545983, +STORE, 94865923670016, 94865925533695, +STORE, 94865925533696, 94865925545983, +ERASE, 94865925533696, 94865925545983, +SNULL, 94865925492735, 94865925533695, +STORE, 94865923670016, 94865925492735, +STORE, 94865925492736, 94865925533695, +ERASE, 94865925492736, 94865925533695, +STORE, 94865923670016, 94865925627903, +SNULL, 94865925599231, 94865925627903, +STORE, 94865923670016, 94865925599231, +STORE, 94865925599232, 94865925627903, +ERASE, 94865925599232, 94865925627903, +STORE, 94865923670016, 94865925738495, +SNULL, 94865925726207, 94865925738495, +STORE, 94865923670016, 94865925726207, +STORE, 94865925726208, 94865925738495, +ERASE, 94865925726208, 94865925738495, +STORE, 94865923670016, 94865925877759, +SNULL, 94865925865471, 94865925877759, +STORE, 94865923670016, 94865925865471, +STORE, 94865925865472, 94865925877759, +ERASE, 94865925865472, 94865925877759, +STORE, 94865923670016, 94865926021119, +SNULL, 94865926008831, 94865926021119, +STORE, 94865923670016, 94865926008831, +STORE, 94865926008832, 94865926021119, +ERASE, 94865926008832, 94865926021119, +SNULL, 94865925971967, 94865926008831, +STORE, 94865923670016, 94865925971967, +STORE, 94865925971968, 94865926008831, +ERASE, 94865925971968, 94865926008831, +STORE, 94865923670016, 94865926115327, +STORE, 94865923670016, 94865926254591, +SNULL, 94865926246399, 94865926254591, +STORE, 94865923670016, 94865926246399, +STORE, 94865926246400, 94865926254591, +ERASE, 94865926246400, 94865926254591, +STORE, 94865923670016, 94865926385663, +STORE, 94865923670016, 94865926537215, +STORE, 94865923670016, 94865926672383, +STORE, 94865923670016, 94865926815743, +STORE, 94865923670016, 94865926955007, +STORE, 94865923670016, 94865927094271, +STORE, 94865923670016, 94865927233535, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140731148435456, 140737488351231, +SNULL, 140731148443647, 140737488351231, +STORE, 140731148435456, 140731148443647, +STORE, 140731148304384, 140731148443647, +STORE, 94090775400448, 94090777735167, +SNULL, 94090775613439, 94090777735167, +STORE, 94090775400448, 94090775613439, +STORE, 94090775613440, 94090777735167, +ERASE, 94090775613440, 94090777735167, +STORE, 94090777710592, 94090777722879, +STORE, 94090777722880, 94090777735167, +STORE, 140301090283520, 140301092536319, +SNULL, 140301090426879, 140301092536319, +STORE, 140301090283520, 140301090426879, +STORE, 140301090426880, 140301092536319, +ERASE, 140301090426880, 140301092536319, +STORE, 140301092524032, 140301092532223, +STORE, 140301092532224, 140301092536319, +STORE, 140731148570624, 140731148574719, +STORE, 140731148558336, 140731148570623, +STORE, 140301092495360, 140301092524031, +STORE, 140301092487168, 140301092495359, +STORE, 140301088169984, 140301090283519, +SNULL, 140301088169984, 140301088182271, +STORE, 140301088182272, 140301090283519, +STORE, 140301088169984, 140301088182271, +SNULL, 140301090275327, 140301090283519, +STORE, 140301088182272, 140301090275327, +STORE, 140301090275328, 140301090283519, +ERASE, 140301090275328, 140301090283519, +STORE, 140301090275328, 140301090283519, +STORE, 140301084372992, 140301088169983, +SNULL, 140301084372992, 140301086031871, +STORE, 140301086031872, 140301088169983, +STORE, 140301084372992, 140301086031871, +SNULL, 140301088129023, 140301088169983, +STORE, 140301086031872, 140301088129023, +STORE, 140301088129024, 140301088169983, +SNULL, 140301088129024, 140301088153599, +STORE, 140301088153600, 140301088169983, +STORE, 140301088129024, 140301088153599, +ERASE, 140301088129024, 140301088153599, +STORE, 140301088129024, 140301088153599, +ERASE, 140301088153600, 140301088169983, +STORE, 140301088153600, 140301088169983, +STORE, 140301092478976, 140301092495359, +SNULL, 140301088145407, 140301088153599, +STORE, 140301088129024, 140301088145407, +STORE, 140301088145408, 140301088153599, +SNULL, 140301090279423, 140301090283519, +STORE, 140301090275328, 140301090279423, +STORE, 140301090279424, 140301090283519, +SNULL, 94090777714687, 94090777722879, +STORE, 94090777710592, 94090777714687, +STORE, 94090777714688, 94090777722879, +SNULL, 140301092528127, 140301092532223, +STORE, 140301092524032, 140301092528127, +STORE, 140301092528128, 140301092532223, +ERASE, 140301092495360, 140301092524031, +STORE, 94090794590208, 94090794725375, +STORE, 140301090795520, 140301092478975, +STORE, 94090794590208, 94090794860543, +STORE, 94090794590208, 94090794995711, +STORE, 94090794590208, 94090795163647, +SNULL, 94090795139071, 94090795163647, +STORE, 94090794590208, 94090795139071, +STORE, 94090795139072, 94090795163647, +ERASE, 94090795139072, 94090795163647, +STORE, 94090794590208, 94090795278335, +STORE, 94090794590208, 94090795425791, +SNULL, 94090795388927, 94090795425791, +STORE, 94090794590208, 94090795388927, +STORE, 94090795388928, 94090795425791, +ERASE, 94090795388928, 94090795425791, +STORE, 94090794590208, 94090795528191, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733084430336, 140737488351231, +SNULL, 140733084438527, 140737488351231, +STORE, 140733084430336, 140733084438527, +STORE, 140733084299264, 140733084438527, +STORE, 94116169183232, 94116171517951, +SNULL, 94116169396223, 94116171517951, +STORE, 94116169183232, 94116169396223, +STORE, 94116169396224, 94116171517951, +ERASE, 94116169396224, 94116171517951, +STORE, 94116171493376, 94116171505663, +STORE, 94116171505664, 94116171517951, +STORE, 139772214128640, 139772216381439, +SNULL, 139772214271999, 139772216381439, +STORE, 139772214128640, 139772214271999, +STORE, 139772214272000, 139772216381439, +ERASE, 139772214272000, 139772216381439, +STORE, 139772216369152, 139772216377343, +STORE, 139772216377344, 139772216381439, +STORE, 140733085270016, 140733085274111, +STORE, 140733085257728, 140733085270015, +STORE, 139772216340480, 139772216369151, +STORE, 139772216332288, 139772216340479, +STORE, 139772212015104, 139772214128639, +SNULL, 139772212015104, 139772212027391, +STORE, 139772212027392, 139772214128639, +STORE, 139772212015104, 139772212027391, +SNULL, 139772214120447, 139772214128639, +STORE, 139772212027392, 139772214120447, +STORE, 139772214120448, 139772214128639, +ERASE, 139772214120448, 139772214128639, +STORE, 139772214120448, 139772214128639, +STORE, 139772208218112, 139772212015103, +SNULL, 139772208218112, 139772209876991, +STORE, 139772209876992, 139772212015103, +STORE, 139772208218112, 139772209876991, +SNULL, 139772211974143, 139772212015103, +STORE, 139772209876992, 139772211974143, +STORE, 139772211974144, 139772212015103, +SNULL, 139772211974144, 139772211998719, +STORE, 139772211998720, 139772212015103, +STORE, 139772211974144, 139772211998719, +ERASE, 139772211974144, 139772211998719, +STORE, 139772211974144, 139772211998719, +ERASE, 139772211998720, 139772212015103, +STORE, 139772211998720, 139772212015103, +STORE, 139772216324096, 139772216340479, +SNULL, 139772211990527, 139772211998719, +STORE, 139772211974144, 139772211990527, +STORE, 139772211990528, 139772211998719, +SNULL, 139772214124543, 139772214128639, +STORE, 139772214120448, 139772214124543, +STORE, 139772214124544, 139772214128639, +SNULL, 94116171497471, 94116171505663, +STORE, 94116171493376, 94116171497471, +STORE, 94116171497472, 94116171505663, +SNULL, 139772216373247, 139772216377343, +STORE, 139772216369152, 139772216373247, +STORE, 139772216373248, 139772216377343, +ERASE, 139772216340480, 139772216369151, +STORE, 94116199383040, 94116199518207, +STORE, 139772214640640, 139772216324095, +STORE, 94116199383040, 94116199653375, +STORE, 94116199383040, 94116199788543, +STORE, 140737488347136, 140737488351231, +STORE, 140726067826688, 140737488351231, +SNULL, 140726067830783, 140737488351231, +STORE, 140726067826688, 140726067830783, +STORE, 140726067695616, 140726067830783, +STORE, 94535150673920, 94535152898047, +SNULL, 94535150784511, 94535152898047, +STORE, 94535150673920, 94535150784511, +STORE, 94535150784512, 94535152898047, +ERASE, 94535150784512, 94535152898047, +STORE, 94535152877568, 94535152889855, +STORE, 94535152889856, 94535152898047, +STORE, 140381257314304, 140381259567103, +SNULL, 140381257457663, 140381259567103, +STORE, 140381257314304, 140381257457663, +STORE, 140381257457664, 140381259567103, +ERASE, 140381257457664, 140381259567103, +STORE, 140381259554816, 140381259563007, +STORE, 140381259563008, 140381259567103, +STORE, 140726068060160, 140726068064255, +STORE, 140726068047872, 140726068060159, +STORE, 140381259526144, 140381259554815, +STORE, 140381259517952, 140381259526143, +STORE, 140381253517312, 140381257314303, +SNULL, 140381253517312, 140381255176191, +STORE, 140381255176192, 140381257314303, +STORE, 140381253517312, 140381255176191, +SNULL, 140381257273343, 140381257314303, +STORE, 140381255176192, 140381257273343, +STORE, 140381257273344, 140381257314303, +SNULL, 140381257273344, 140381257297919, +STORE, 140381257297920, 140381257314303, +STORE, 140381257273344, 140381257297919, +ERASE, 140381257273344, 140381257297919, +STORE, 140381257273344, 140381257297919, +ERASE, 140381257297920, 140381257314303, +STORE, 140381257297920, 140381257314303, +SNULL, 140381257289727, 140381257297919, +STORE, 140381257273344, 140381257289727, +STORE, 140381257289728, 140381257297919, +SNULL, 94535152885759, 94535152889855, +STORE, 94535152877568, 94535152885759, +STORE, 94535152885760, 94535152889855, +SNULL, 140381259558911, 140381259563007, +STORE, 140381259554816, 140381259558911, +STORE, 140381259558912, 140381259563007, +ERASE, 140381259526144, 140381259554815, +STORE, 94535186296832, 94535186431999, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140729189425152, 140737488351231, +SNULL, 140729189433343, 140737488351231, +STORE, 140729189425152, 140729189433343, +STORE, 140729189294080, 140729189433343, +STORE, 94428200128512, 94428202352639, +SNULL, 94428200239103, 94428202352639, +STORE, 94428200128512, 94428200239103, +STORE, 94428200239104, 94428202352639, +ERASE, 94428200239104, 94428202352639, +STORE, 94428202332160, 94428202344447, +STORE, 94428202344448, 94428202352639, +STORE, 139707216986112, 139707219238911, +SNULL, 139707217129471, 139707219238911, +STORE, 139707216986112, 139707217129471, +STORE, 139707217129472, 139707219238911, +ERASE, 139707217129472, 139707219238911, +STORE, 139707219226624, 139707219234815, +STORE, 139707219234816, 139707219238911, +STORE, 140729189785600, 140729189789695, +STORE, 140729189773312, 140729189785599, +STORE, 139707219197952, 139707219226623, +STORE, 139707219189760, 139707219197951, +STORE, 139707213189120, 139707216986111, +SNULL, 139707213189120, 139707214847999, +STORE, 139707214848000, 139707216986111, +STORE, 139707213189120, 139707214847999, +SNULL, 139707216945151, 139707216986111, +STORE, 139707214848000, 139707216945151, +STORE, 139707216945152, 139707216986111, +SNULL, 139707216945152, 139707216969727, +STORE, 139707216969728, 139707216986111, +STORE, 139707216945152, 139707216969727, +ERASE, 139707216945152, 139707216969727, +STORE, 139707216945152, 139707216969727, +ERASE, 139707216969728, 139707216986111, +STORE, 139707216969728, 139707216986111, +SNULL, 139707216961535, 139707216969727, +STORE, 139707216945152, 139707216961535, +STORE, 139707216961536, 139707216969727, +SNULL, 94428202340351, 94428202344447, +STORE, 94428202332160, 94428202340351, +STORE, 94428202340352, 94428202344447, +SNULL, 139707219230719, 139707219234815, +STORE, 139707219226624, 139707219230719, +STORE, 139707219230720, 139707219234815, +ERASE, 139707219197952, 139707219226623, +STORE, 94428208599040, 94428208734207, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722000953344, 140737488351231, +SNULL, 140722000961535, 140737488351231, +STORE, 140722000953344, 140722000961535, +STORE, 140722000822272, 140722000961535, +STORE, 94636494757888, 94636496982015, +SNULL, 94636494868479, 94636496982015, +STORE, 94636494757888, 94636494868479, +STORE, 94636494868480, 94636496982015, +ERASE, 94636494868480, 94636496982015, +STORE, 94636496961536, 94636496973823, +STORE, 94636496973824, 94636496982015, +STORE, 140142275100672, 140142277353471, +SNULL, 140142275244031, 140142277353471, +STORE, 140142275100672, 140142275244031, +STORE, 140142275244032, 140142277353471, +ERASE, 140142275244032, 140142277353471, +STORE, 140142277341184, 140142277349375, +STORE, 140142277349376, 140142277353471, +STORE, 140722002747392, 140722002751487, +STORE, 140722002735104, 140722002747391, +STORE, 140142277312512, 140142277341183, +STORE, 140142277304320, 140142277312511, +STORE, 140142271303680, 140142275100671, +SNULL, 140142271303680, 140142272962559, +STORE, 140142272962560, 140142275100671, +STORE, 140142271303680, 140142272962559, +SNULL, 140142275059711, 140142275100671, +STORE, 140142272962560, 140142275059711, +STORE, 140142275059712, 140142275100671, +SNULL, 140142275059712, 140142275084287, +STORE, 140142275084288, 140142275100671, +STORE, 140142275059712, 140142275084287, +ERASE, 140142275059712, 140142275084287, +STORE, 140142275059712, 140142275084287, +ERASE, 140142275084288, 140142275100671, +STORE, 140142275084288, 140142275100671, +SNULL, 140142275076095, 140142275084287, +STORE, 140142275059712, 140142275076095, +STORE, 140142275076096, 140142275084287, +SNULL, 94636496969727, 94636496973823, +STORE, 94636496961536, 94636496969727, +STORE, 94636496969728, 94636496973823, +SNULL, 140142277345279, 140142277349375, +STORE, 140142277341184, 140142277345279, +STORE, 140142277345280, 140142277349375, +ERASE, 140142277312512, 140142277341183, +STORE, 94636516286464, 94636516421631, +STORE, 94071103692800, 94071103905791, +STORE, 94071106002944, 94071106007039, +STORE, 94071106007040, 94071106015231, +STORE, 94071106015232, 94071106027519, +STORE, 94071138521088, 94071140368383, +STORE, 140145668190208, 140145669849087, +STORE, 140145669849088, 140145671946239, +STORE, 140145671946240, 140145671962623, +STORE, 140145671962624, 140145671970815, +STORE, 140145671970816, 140145671987199, +STORE, 140145671987200, 140145671999487, +STORE, 140145671999488, 140145674092543, +STORE, 140145674092544, 140145674096639, +STORE, 140145674096640, 140145674100735, +STORE, 140145674100736, 140145674244095, +STORE, 140145674612736, 140145676296191, +STORE, 140145676296192, 140145676312575, +STORE, 140145676341248, 140145676345343, +STORE, 140145676345344, 140145676349439, +STORE, 140145676349440, 140145676353535, +STORE, 140734927740928, 140734927880191, +STORE, 140734928842752, 140734928855039, +STORE, 140734928855040, 140734928859135, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722342535168, 140737488351231, +SNULL, 140722342543359, 140737488351231, +STORE, 140722342535168, 140722342543359, +STORE, 140722342404096, 140722342543359, +STORE, 94399699714048, 94399702048767, +SNULL, 94399699927039, 94399702048767, +STORE, 94399699714048, 94399699927039, +STORE, 94399699927040, 94399702048767, +ERASE, 94399699927040, 94399702048767, +STORE, 94399702024192, 94399702036479, +STORE, 94399702036480, 94399702048767, +STORE, 139811024748544, 139811027001343, +SNULL, 139811024891903, 139811027001343, +STORE, 139811024748544, 139811024891903, +STORE, 139811024891904, 139811027001343, +ERASE, 139811024891904, 139811027001343, +STORE, 139811026989056, 139811026997247, +STORE, 139811026997248, 139811027001343, +STORE, 140722342707200, 140722342711295, +STORE, 140722342694912, 140722342707199, +STORE, 139811026960384, 139811026989055, +STORE, 139811026952192, 139811026960383, +STORE, 139811022635008, 139811024748543, +SNULL, 139811022635008, 139811022647295, +STORE, 139811022647296, 139811024748543, +STORE, 139811022635008, 139811022647295, +SNULL, 139811024740351, 139811024748543, +STORE, 139811022647296, 139811024740351, +STORE, 139811024740352, 139811024748543, +ERASE, 139811024740352, 139811024748543, +STORE, 139811024740352, 139811024748543, +STORE, 139811018838016, 139811022635007, +SNULL, 139811018838016, 139811020496895, +STORE, 139811020496896, 139811022635007, +STORE, 139811018838016, 139811020496895, +SNULL, 139811022594047, 139811022635007, +STORE, 139811020496896, 139811022594047, +STORE, 139811022594048, 139811022635007, +SNULL, 139811022594048, 139811022618623, +STORE, 139811022618624, 139811022635007, +STORE, 139811022594048, 139811022618623, +ERASE, 139811022594048, 139811022618623, +STORE, 139811022594048, 139811022618623, +ERASE, 139811022618624, 139811022635007, +STORE, 139811022618624, 139811022635007, +STORE, 139811026944000, 139811026960383, +SNULL, 139811022610431, 139811022618623, +STORE, 139811022594048, 139811022610431, +STORE, 139811022610432, 139811022618623, +SNULL, 139811024744447, 139811024748543, +STORE, 139811024740352, 139811024744447, +STORE, 139811024744448, 139811024748543, +SNULL, 94399702028287, 94399702036479, +STORE, 94399702024192, 94399702028287, +STORE, 94399702028288, 94399702036479, +SNULL, 139811026993151, 139811026997247, +STORE, 139811026989056, 139811026993151, +STORE, 139811026993152, 139811026997247, +ERASE, 139811026960384, 139811026989055, +STORE, 94399723880448, 94399724015615, +STORE, 139811025260544, 139811026943999, +STORE, 94399723880448, 94399724150783, +STORE, 94399723880448, 94399724285951, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140735364939776, 140737488351231, +SNULL, 140735364947967, 140737488351231, +STORE, 140735364939776, 140735364947967, +STORE, 140735364808704, 140735364947967, +STORE, 94421528674304, 94421531009023, +SNULL, 94421528887295, 94421531009023, +STORE, 94421528674304, 94421528887295, +STORE, 94421528887296, 94421531009023, +ERASE, 94421528887296, 94421531009023, +STORE, 94421530984448, 94421530996735, +STORE, 94421530996736, 94421531009023, +STORE, 140162004742144, 140162006994943, +SNULL, 140162004885503, 140162006994943, +STORE, 140162004742144, 140162004885503, +STORE, 140162004885504, 140162006994943, +ERASE, 140162004885504, 140162006994943, +STORE, 140162006982656, 140162006990847, +STORE, 140162006990848, 140162006994943, +STORE, 140735365402624, 140735365406719, +STORE, 140735365390336, 140735365402623, +STORE, 140162006953984, 140162006982655, +STORE, 140162006945792, 140162006953983, +STORE, 140162002628608, 140162004742143, +SNULL, 140162002628608, 140162002640895, +STORE, 140162002640896, 140162004742143, +STORE, 140162002628608, 140162002640895, +SNULL, 140162004733951, 140162004742143, +STORE, 140162002640896, 140162004733951, +STORE, 140162004733952, 140162004742143, +ERASE, 140162004733952, 140162004742143, +STORE, 140162004733952, 140162004742143, +STORE, 140161998831616, 140162002628607, +SNULL, 140161998831616, 140162000490495, +STORE, 140162000490496, 140162002628607, +STORE, 140161998831616, 140162000490495, +SNULL, 140162002587647, 140162002628607, +STORE, 140162000490496, 140162002587647, +STORE, 140162002587648, 140162002628607, +SNULL, 140162002587648, 140162002612223, +STORE, 140162002612224, 140162002628607, +STORE, 140162002587648, 140162002612223, +ERASE, 140162002587648, 140162002612223, +STORE, 140162002587648, 140162002612223, +ERASE, 140162002612224, 140162002628607, +STORE, 140162002612224, 140162002628607, +STORE, 140162006937600, 140162006953983, +SNULL, 140162002604031, 140162002612223, +STORE, 140162002587648, 140162002604031, +STORE, 140162002604032, 140162002612223, +SNULL, 140162004738047, 140162004742143, +STORE, 140162004733952, 140162004738047, +STORE, 140162004738048, 140162004742143, +SNULL, 94421530988543, 94421530996735, +STORE, 94421530984448, 94421530988543, +STORE, 94421530988544, 94421530996735, +SNULL, 140162006986751, 140162006990847, +STORE, 140162006982656, 140162006986751, +STORE, 140162006986752, 140162006990847, +ERASE, 140162006953984, 140162006982655, +STORE, 94421551697920, 94421551833087, +STORE, 140162005254144, 140162006937599, +STORE, 94421551697920, 94421551968255, +STORE, 94421551697920, 94421552103423, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140733498486784, 140737488351231, +SNULL, 140733498494975, 140737488351231, +STORE, 140733498486784, 140733498494975, +STORE, 140733498355712, 140733498494975, +STORE, 94567985836032, 94567988170751, +SNULL, 94567986049023, 94567988170751, +STORE, 94567985836032, 94567986049023, +STORE, 94567986049024, 94567988170751, +ERASE, 94567986049024, 94567988170751, +STORE, 94567988146176, 94567988158463, +STORE, 94567988158464, 94567988170751, +STORE, 139634278572032, 139634280824831, +SNULL, 139634278715391, 139634280824831, +STORE, 139634278572032, 139634278715391, +STORE, 139634278715392, 139634280824831, +ERASE, 139634278715392, 139634280824831, +STORE, 139634280812544, 139634280820735, +STORE, 139634280820736, 139634280824831, +STORE, 140733498544128, 140733498548223, +STORE, 140733498531840, 140733498544127, +STORE, 139634280783872, 139634280812543, +STORE, 139634280775680, 139634280783871, +STORE, 139634276458496, 139634278572031, +SNULL, 139634276458496, 139634276470783, +STORE, 139634276470784, 139634278572031, +STORE, 139634276458496, 139634276470783, +SNULL, 139634278563839, 139634278572031, +STORE, 139634276470784, 139634278563839, +STORE, 139634278563840, 139634278572031, +ERASE, 139634278563840, 139634278572031, +STORE, 139634278563840, 139634278572031, +STORE, 139634272661504, 139634276458495, +SNULL, 139634272661504, 139634274320383, +STORE, 139634274320384, 139634276458495, +STORE, 139634272661504, 139634274320383, +SNULL, 139634276417535, 139634276458495, +STORE, 139634274320384, 139634276417535, +STORE, 139634276417536, 139634276458495, +SNULL, 139634276417536, 139634276442111, +STORE, 139634276442112, 139634276458495, +STORE, 139634276417536, 139634276442111, +ERASE, 139634276417536, 139634276442111, +STORE, 139634276417536, 139634276442111, +ERASE, 139634276442112, 139634276458495, +STORE, 139634276442112, 139634276458495, +STORE, 139634280767488, 139634280783871, +SNULL, 139634276433919, 139634276442111, +STORE, 139634276417536, 139634276433919, +STORE, 139634276433920, 139634276442111, +SNULL, 139634278567935, 139634278572031, +STORE, 139634278563840, 139634278567935, +STORE, 139634278567936, 139634278572031, +SNULL, 94567988150271, 94567988158463, +STORE, 94567988146176, 94567988150271, +STORE, 94567988150272, 94567988158463, +SNULL, 139634280816639, 139634280820735, +STORE, 139634280812544, 139634280816639, +STORE, 139634280816640, 139634280820735, +ERASE, 139634280783872, 139634280812543, +STORE, 94567996379136, 94567996514303, +STORE, 139634279084032, 139634280767487, +STORE, 94567996379136, 94567996649471, +STORE, 94567996379136, 94567996784639, +STORE, 94567996379136, 94567996960767, +SNULL, 94567996932095, 94567996960767, +STORE, 94567996379136, 94567996932095, +STORE, 94567996932096, 94567996960767, +ERASE, 94567996932096, 94567996960767, +STORE, 94567996379136, 94567997071359, +STORE, 94567996379136, 94567997206527, +SNULL, 94567997186047, 94567997206527, +STORE, 94567996379136, 94567997186047, +STORE, 94567997186048, 94567997206527, +ERASE, 94567997186048, 94567997206527, +STORE, 94567996379136, 94567997358079, +STORE, 94567996379136, 94567997493247, +SNULL, 94567997476863, 94567997493247, +STORE, 94567996379136, 94567997476863, +STORE, 94567997476864, 94567997493247, +ERASE, 94567997476864, 94567997493247, +STORE, 94567996379136, 94567997612031, +STORE, 94567996379136, 94567997767679, +SNULL, 94567997739007, 94567997767679, +STORE, 94567996379136, 94567997739007, +STORE, 94567997739008, 94567997767679, +ERASE, 94567997739008, 94567997767679, +SNULL, 94567997698047, 94567997739007, +STORE, 94567996379136, 94567997698047, +STORE, 94567997698048, 94567997739007, +ERASE, 94567997698048, 94567997739007, +STORE, 94567996379136, 94567997853695, +STORE, 94567996379136, 94567997988863, +STORE, 94567996379136, 94567998132223, +STORE, 94567996379136, 94567998275583, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140723667759104, 140737488351231, +SNULL, 140723667767295, 140737488351231, +STORE, 140723667759104, 140723667767295, +STORE, 140723667628032, 140723667767295, +STORE, 94231598800896, 94231601135615, +SNULL, 94231599013887, 94231601135615, +STORE, 94231598800896, 94231599013887, +STORE, 94231599013888, 94231601135615, +ERASE, 94231599013888, 94231601135615, +STORE, 94231601111040, 94231601123327, +STORE, 94231601123328, 94231601135615, +STORE, 140269472649216, 140269474902015, +SNULL, 140269472792575, 140269474902015, +STORE, 140269472649216, 140269472792575, +STORE, 140269472792576, 140269474902015, +ERASE, 140269472792576, 140269474902015, +STORE, 140269474889728, 140269474897919, +STORE, 140269474897920, 140269474902015, +STORE, 140723667836928, 140723667841023, +STORE, 140723667824640, 140723667836927, +STORE, 140269474861056, 140269474889727, +STORE, 140269474852864, 140269474861055, +STORE, 140269470535680, 140269472649215, +SNULL, 140269470535680, 140269470547967, +STORE, 140269470547968, 140269472649215, +STORE, 140269470535680, 140269470547967, +SNULL, 140269472641023, 140269472649215, +STORE, 140269470547968, 140269472641023, +STORE, 140269472641024, 140269472649215, +ERASE, 140269472641024, 140269472649215, +STORE, 140269472641024, 140269472649215, +STORE, 140269466738688, 140269470535679, +SNULL, 140269466738688, 140269468397567, +STORE, 140269468397568, 140269470535679, +STORE, 140269466738688, 140269468397567, +SNULL, 140269470494719, 140269470535679, +STORE, 140269468397568, 140269470494719, +STORE, 140269470494720, 140269470535679, +SNULL, 140269470494720, 140269470519295, +STORE, 140269470519296, 140269470535679, +STORE, 140269470494720, 140269470519295, +ERASE, 140269470494720, 140269470519295, +STORE, 140269470494720, 140269470519295, +ERASE, 140269470519296, 140269470535679, +STORE, 140269470519296, 140269470535679, +STORE, 140269474844672, 140269474861055, +SNULL, 140269470511103, 140269470519295, +STORE, 140269470494720, 140269470511103, +STORE, 140269470511104, 140269470519295, +SNULL, 140269472645119, 140269472649215, +STORE, 140269472641024, 140269472645119, +STORE, 140269472645120, 140269472649215, +SNULL, 94231601115135, 94231601123327, +STORE, 94231601111040, 94231601115135, +STORE, 94231601115136, 94231601123327, +SNULL, 140269474893823, 140269474897919, +STORE, 140269474889728, 140269474893823, +STORE, 140269474893824, 140269474897919, +ERASE, 140269474861056, 140269474889727, +STORE, 94231626592256, 94231626727423, +STORE, 140269473161216, 140269474844671, +STORE, 94231626592256, 94231626862591, +STORE, 94231626592256, 94231626997759, +STORE, 94327178862592, 94327179075583, +STORE, 94327181172736, 94327181176831, +STORE, 94327181176832, 94327181185023, +STORE, 94327181185024, 94327181197311, +STORE, 94327185715200, 94327186685951, +STORE, 140172071755776, 140172073414655, +STORE, 140172073414656, 140172075511807, +STORE, 140172075511808, 140172075528191, +STORE, 140172075528192, 140172075536383, +STORE, 140172075536384, 140172075552767, +STORE, 140172075552768, 140172075565055, +STORE, 140172075565056, 140172077658111, +STORE, 140172077658112, 140172077662207, +STORE, 140172077662208, 140172077666303, +STORE, 140172077666304, 140172077809663, +STORE, 140172078178304, 140172079861759, +STORE, 140172079861760, 140172079878143, +STORE, 140172079878144, 140172079906815, +STORE, 140172079906816, 140172079910911, +STORE, 140172079910912, 140172079915007, +STORE, 140172079915008, 140172079919103, +STORE, 140720358359040, 140720358494207, +STORE, 140720358498304, 140720358510591, +STORE, 140720358510592, 140720358514687, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140722548621312, 140737488351231, +SNULL, 140722548629503, 140737488351231, +STORE, 140722548621312, 140722548629503, +STORE, 140722548490240, 140722548629503, +STORE, 93949289504768, 93949291728895, +SNULL, 93949289615359, 93949291728895, +STORE, 93949289504768, 93949289615359, +STORE, 93949289615360, 93949291728895, +ERASE, 93949289615360, 93949291728895, +STORE, 93949291708416, 93949291720703, +STORE, 93949291720704, 93949291728895, +STORE, 140305861902336, 140305864155135, +SNULL, 140305862045695, 140305864155135, +STORE, 140305861902336, 140305862045695, +STORE, 140305862045696, 140305864155135, +ERASE, 140305862045696, 140305864155135, +STORE, 140305864142848, 140305864151039, +STORE, 140305864151040, 140305864155135, +STORE, 140722549821440, 140722549825535, +STORE, 140722549809152, 140722549821439, +STORE, 140305864114176, 140305864142847, +STORE, 140305864105984, 140305864114175, +STORE, 140305858105344, 140305861902335, +SNULL, 140305858105344, 140305859764223, +STORE, 140305859764224, 140305861902335, +STORE, 140305858105344, 140305859764223, +SNULL, 140305861861375, 140305861902335, +STORE, 140305859764224, 140305861861375, +STORE, 140305861861376, 140305861902335, +SNULL, 140305861861376, 140305861885951, +STORE, 140305861885952, 140305861902335, +STORE, 140305861861376, 140305861885951, +ERASE, 140305861861376, 140305861885951, +STORE, 140305861861376, 140305861885951, +ERASE, 140305861885952, 140305861902335, +STORE, 140305861885952, 140305861902335, +SNULL, 140305861877759, 140305861885951, +STORE, 140305861861376, 140305861877759, +STORE, 140305861877760, 140305861885951, +SNULL, 93949291716607, 93949291720703, +STORE, 93949291708416, 93949291716607, +STORE, 93949291716608, 93949291720703, +SNULL, 140305864146943, 140305864151039, +STORE, 140305864142848, 140305864146943, +STORE, 140305864146944, 140305864151039, +ERASE, 140305864114176, 140305864142847, +STORE, 93949324136448, 93949324271615, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140725754908672, 140737488351231, +SNULL, 140725754916863, 140737488351231, +STORE, 140725754908672, 140725754916863, +STORE, 140725754777600, 140725754916863, +STORE, 94831184375808, 94831186599935, +SNULL, 94831184486399, 94831186599935, +STORE, 94831184375808, 94831184486399, +STORE, 94831184486400, 94831186599935, +ERASE, 94831184486400, 94831186599935, +STORE, 94831186579456, 94831186591743, +STORE, 94831186591744, 94831186599935, +STORE, 140605482479616, 140605484732415, +SNULL, 140605482622975, 140605484732415, +STORE, 140605482479616, 140605482622975, +STORE, 140605482622976, 140605484732415, +ERASE, 140605482622976, 140605484732415, +STORE, 140605484720128, 140605484728319, +STORE, 140605484728320, 140605484732415, +STORE, 140725755670528, 140725755674623, +STORE, 140725755658240, 140725755670527, +STORE, 140605484691456, 140605484720127, +STORE, 140605484683264, 140605484691455, +STORE, 140605478682624, 140605482479615, +SNULL, 140605478682624, 140605480341503, +STORE, 140605480341504, 140605482479615, +STORE, 140605478682624, 140605480341503, +SNULL, 140605482438655, 140605482479615, +STORE, 140605480341504, 140605482438655, +STORE, 140605482438656, 140605482479615, +SNULL, 140605482438656, 140605482463231, +STORE, 140605482463232, 140605482479615, +STORE, 140605482438656, 140605482463231, +ERASE, 140605482438656, 140605482463231, +STORE, 140605482438656, 140605482463231, +ERASE, 140605482463232, 140605482479615, +STORE, 140605482463232, 140605482479615, +SNULL, 140605482455039, 140605482463231, +STORE, 140605482438656, 140605482455039, +STORE, 140605482455040, 140605482463231, +SNULL, 94831186587647, 94831186591743, +STORE, 94831186579456, 94831186587647, +STORE, 94831186587648, 94831186591743, +SNULL, 140605484724223, 140605484728319, +STORE, 140605484720128, 140605484724223, +STORE, 140605484724224, 140605484728319, +ERASE, 140605484691456, 140605484720127, +STORE, 94831217156096, 94831217291263, +STORE, 94327178862592, 94327179075583, +STORE, 94327181172736, 94327181176831, +STORE, 94327181176832, 94327181185023, +STORE, 94327181185024, 94327181197311, +STORE, 94327185715200, 94327186685951, +STORE, 140172071755776, 140172073414655, +STORE, 140172073414656, 140172075511807, +STORE, 140172075511808, 140172075528191, +STORE, 140172075528192, 140172075536383, +STORE, 140172075536384, 140172075552767, +STORE, 140172075552768, 140172075565055, +STORE, 140172075565056, 140172077658111, +STORE, 140172077658112, 140172077662207, +STORE, 140172077662208, 140172077666303, +STORE, 140172077666304, 140172077809663, +STORE, 140172078178304, 140172079861759, +STORE, 140172079861760, 140172079878143, +STORE, 140172079878144, 140172079906815, +STORE, 140172079906816, 140172079910911, +STORE, 140172079910912, 140172079915007, +STORE, 140172079915008, 140172079919103, +STORE, 140720358359040, 140720358494207, +STORE, 140720358498304, 140720358510591, +STORE, 140720358510592, 140720358514687, +STORE, 140737488347136, 140737488351231, +STORE, 140737488343040, 140737488351231, +STORE, 140737488338944, 140737488351231, +STORE, 140734529933312, 140737488351231, +SNULL, 140734529945599, 140737488351231, +STORE, 140734529933312, 140734529945599, +STORE, 140734529802240, 140734529945599, +STORE, 4194304, 26279935, +STORE, 28372992, 28454911, +STORE, 28454912, 29806591, +STORE, 140249744060416, 140249746313215, +SNULL, 140249744203775, 140249746313215, +STORE, 140249744060416, 140249744203775, +STORE, 140249744203776, 140249746313215, +ERASE, 140249744203776, 140249746313215, +STORE, 140249746300928, 140249746309119, +STORE, 140249746309120, 140249746313215, +STORE, 140734530174976, 140734530179071, +STORE, 140734530162688, 140734530174975, +STORE, 140249746272256, 140249746300927, +STORE, 140249746264064, 140249746272255, +STORE, 140249740226560, 140249744060415, +SNULL, 140249740226560, 140249741934591, +STORE, 140249741934592, 140249744060415, +STORE, 140249740226560, 140249741934591, +SNULL, 140249744027647, 140249744060415, +STORE, 140249741934592, 140249744027647, +STORE, 140249744027648, 140249744060415, +ERASE, 140249744027648, 140249744060415, +STORE, 140249744027648, 140249744060415, +STORE, 140249738031104, 140249740226559, +SNULL, 140249738031104, 140249738125311, +STORE, 140249738125312, 140249740226559, +STORE, 140249738031104, 140249738125311, +SNULL, 140249740218367, 140249740226559, +STORE, 140249738125312, 140249740218367, +STORE, 140249740218368, 140249740226559, +ERASE, 140249740218368, 140249740226559, +STORE, 140249740218368, 140249740226559, +STORE, 140249735512064, 140249738031103, +SNULL, 140249735512064, 140249735925759, +STORE, 140249735925760, 140249738031103, +STORE, 140249735512064, 140249735925759, +SNULL, 140249738018815, 140249738031103, +STORE, 140249735925760, 140249738018815, +STORE, 140249738018816, 140249738031103, +ERASE, 140249738018816, 140249738031103, +STORE, 140249738018816, 140249738031103, +STORE, 140249732878336, 140249735512063, +SNULL, 140249732878336, 140249733406719, +STORE, 140249733406720, 140249735512063, +STORE, 140249732878336, 140249733406719, +SNULL, 140249735503871, 140249735512063, +STORE, 140249733406720, 140249735503871, +STORE, 140249735503872, 140249735512063, +ERASE, 140249735503872, 140249735512063, +STORE, 140249735503872, 140249735512063, +STORE, 140249730764800, 140249732878335, +SNULL, 140249730764800, 140249730777087, +STORE, 140249730777088, 140249732878335, +STORE, 140249730764800, 140249730777087, +SNULL, 140249732870143, 140249732878335, +STORE, 140249730777088, 140249732870143, +STORE, 140249732870144, 140249732878335, +ERASE, 140249732870144, 140249732878335, +STORE, 140249732870144, 140249732878335, +STORE, 140249728561152, 140249730764799, +SNULL, 140249728561152, 140249728663551, +STORE, 140249728663552, 140249730764799, +STORE, 140249728561152, 140249728663551, +SNULL, 140249730756607, 140249730764799, +STORE, 140249728663552, 140249730756607, +STORE, 140249730756608, 140249730764799, +ERASE, 140249730756608, 140249730764799, +STORE, 140249730756608, 140249730764799, +STORE, 140249746255872, 140249746272255, +STORE, 140249725399040, 140249728561151, +SNULL, 140249725399040, 140249726459903, +STORE, 140249726459904, 140249728561151, +STORE, 140249725399040, 140249726459903, +SNULL, 140249728552959, 140249728561151, +STORE, 140249726459904, 140249728552959, +STORE, 140249728552960, 140249728561151, +ERASE, 140249728552960, 140249728561151, +STORE, 140249728552960, 140249728561151, +STORE, 140249721602048, 140249725399039, +SNULL, 140249721602048, 140249723260927, +STORE, 140249723260928, 140249725399039, +STORE, 140249721602048, 140249723260927, +SNULL, 140249725358079, 140249725399039, +STORE, 140249723260928, 140249725358079, +STORE, 140249725358080, 140249725399039, +SNULL, 140249725358080, 140249725382655, +STORE, 140249725382656, 140249725399039, +STORE, 140249725358080, 140249725382655, +ERASE, 140249725358080, 140249725382655, +STORE, 140249725358080, 140249725382655, +ERASE, 140249725382656, 140249725399039, +STORE, 140249725382656, 140249725399039, +STORE, 140249746243584, 140249746272255, +SNULL, 140249725374463, 140249725382655, +STORE, 140249725358080, 140249725374463, +STORE, 140249725374464, 140249725382655, +SNULL, 140249728557055, 140249728561151, +STORE, 140249728552960, 140249728557055, +STORE, 140249728557056, 140249728561151, +SNULL, 140249730760703, 140249730764799, +STORE, 140249730756608, 140249730760703, +STORE, 140249730760704, 140249730764799, +SNULL, 140249732874239, 140249732878335, +STORE, 140249732870144, 140249732874239, +STORE, 140249732874240, 140249732878335, +SNULL, 140249735507967, 140249735512063, +STORE, 140249735503872, 140249735507967, +STORE, 140249735507968, 140249735512063, +SNULL, 140249738027007, 140249738031103, +STORE, 140249738018816, 140249738027007, +STORE, 140249738027008, 140249738031103, +SNULL, 140249740222463, 140249740226559, +STORE, 140249740218368, 140249740222463, +STORE, 140249740222464, 140249740226559, +SNULL, 140249744031743, 140249744060415, +STORE, 140249744027648, 140249744031743, +STORE, 140249744031744, 140249744060415, +SNULL, 28405759, 28454911, +STORE, 28372992, 28405759, +STORE, 28405760, 28454911, +SNULL, 140249746305023, 140249746309119, +STORE, 140249746300928, 140249746305023, +STORE, 140249746305024, 140249746309119, +ERASE, 140249746272256, 140249746300927, +STORE, 33853440, 33988607, +STORE, 140249744560128, 140249746243583, +STORE, 140249746296832, 140249746300927, +STORE, 140249744424960, 140249744560127, +STORE, 33853440, 34131967, +STORE, 140249719504896, 140249721602047, +STORE, 140249746288640, 140249746300927, +STORE, 140249746280448, 140249746300927, +STORE, 140249746243584, 140249746280447, +STORE, 140249744408576, 140249744560127, +STORE, 33853440, 34267135, +STORE, 33853440, 34422783, +STORE, 140249744400384, 140249744560127, +STORE, 140249744392192, 140249744560127, +STORE, 33853440, 34557951, +STORE, 33853440, 34693119, +STORE, 140249744375808, 140249744560127, +STORE, 140249744367616, 140249744560127, +STORE, 33853440, 34832383, +STORE, 140249719230464, 140249721602047, +STORE, 140249744207872, 140249744560127, +STORE, 33853440, 34971647, +SNULL, 34963455, 34971647, +STORE, 33853440, 34963455, +STORE, 34963456, 34971647, +ERASE, 34963456, 34971647, +SNULL, 34955263, 34963455, +STORE, 33853440, 34955263, +STORE, 34955264, 34963455, +ERASE, 34955264, 34963455, +SNULL, 34947071, 34955263, +STORE, 33853440, 34947071, +STORE, 34947072, 34955263, +ERASE, 34947072, 34955263, +SNULL, 34938879, 34947071, +STORE, 33853440, 34938879, +STORE, 34938880, 34947071, +ERASE, 34938880, 34947071, +STORE, 140249719214080, 140249721602047, +STORE, 140249719148544, 140249721602047, +STORE, 140249719115776, 140249721602047, +STORE, 140249717018624, 140249721602047, +STORE, 140249716953088, 140249721602047, +STORE, 33853440, 35086335, +STORE, 140249716822016, 140249721602047, +STORE, 140249716559872, 140249721602047, +STORE, 140249716551680, 140249721602047, +STORE, 140249716535296, 140249721602047, +STORE, 140249716527104, 140249721602047, +STORE, 140249716518912, 140249721602047, +STORE, 33853440, 35221503, +SNULL, 35213311, 35221503, +STORE, 33853440, 35213311, +STORE, 35213312, 35221503, +ERASE, 35213312, 35221503, +SNULL, 35205119, 35213311, +STORE, 33853440, 35205119, +STORE, 35205120, 35213311, +ERASE, 35205120, 35213311, +SNULL, 35192831, 35205119, +STORE, 33853440, 35192831, +STORE, 35192832, 35205119, +ERASE, 35192832, 35205119, +SNULL, 35176447, 35192831, +STORE, 33853440, 35176447, +STORE, 35176448, 35192831, +ERASE, 35176448, 35192831, +STORE, 140249716502528, 140249721602047, +STORE, 33853440, 35311615, +SNULL, 35307519, 35311615, +STORE, 33853440, 35307519, +STORE, 35307520, 35311615, +ERASE, 35307520, 35311615, +SNULL, 35303423, 35307519, +STORE, 33853440, 35303423, +STORE, 35303424, 35307519, +ERASE, 35303424, 35307519, +SNULL, 35299327, 35303423, +STORE, 33853440, 35299327, +STORE, 35299328, 35303423, +ERASE, 35299328, 35303423, +SNULL, 35295231, 35299327, +STORE, 33853440, 35295231, +STORE, 35295232, 35299327, +ERASE, 35295232, 35299327, +SNULL, 35291135, 35295231, +STORE, 33853440, 35291135, +STORE, 35291136, 35295231, +ERASE, 35291136, 35295231, +SNULL, 35287039, 35291135, +STORE, 33853440, 35287039, +STORE, 35287040, 35291135, +ERASE, 35287040, 35291135, +SNULL, 35282943, 35287039, +STORE, 33853440, 35282943, +STORE, 35282944, 35287039, +ERASE, 35282944, 35287039, +STORE, 140249716486144, 140249721602047, +STORE, 140249716453376, 140249721602047, +STORE, 33853440, 35418111, +SNULL, 35401727, 35418111, +STORE, 33853440, 35401727, +STORE, 35401728, 35418111, +ERASE, 35401728, 35418111, +SNULL, 35389439, 35401727, +STORE, 33853440, 35389439, +STORE, 35389440, 35401727, +ERASE, 35389440, 35401727, +STORE, 140249714356224, 140249721602047, +STORE, 33853440, 35540991, +STORE, 140249714339840, 140249721602047, +STORE, 140249714077696, 140249721602047, +STORE, 140249714069504, 140249721602047, +STORE, 140249714061312, 140249721602047, +STORE, 33853440, 35680255, +SNULL, 35672063, 35680255, +STORE, 33853440, 35672063, +STORE, 35672064, 35680255, +ERASE, 35672064, 35680255, +SNULL, 35627007, 35672063, +STORE, 33853440, 35627007, +STORE, 35627008, 35672063, +ERASE, 35627008, 35672063, +STORE, 140249711964160, 140249721602047, +STORE, 33853440, 35762175, +SNULL, 35753983, 35762175, +STORE, 33853440, 35753983, +STORE, 35753984, 35762175, +ERASE, 35753984, 35762175, +SNULL, 35745791, 35753983, +STORE, 33853440, 35745791, +STORE, 35745792, 35753983, +ERASE, 35745792, 35753983, +STORE, 140249711955968, 140249721602047, +STORE, 140249711947776, 140249721602047, +STORE, 140249710899200, 140249721602047, +STORE, 140249710866432, 140249721602047, +STORE, 140249710600192, 140249721602047, +SNULL, 140249744424959, 140249744560127, +STORE, 140249744207872, 140249744424959, +STORE, 140249744424960, 140249744560127, +ERASE, 140249744424960, 140249744560127, +STORE, 140249708503040, 140249721602047, +STORE, 33853440, 35885055, +STORE, 140249707978752, 140249721602047, +STORE, 140249705881600, 140249721602047, +STORE, 33853440, 36036607, +STORE, 33853440, 36175871, +STORE, 140249744551936, 140249744560127, +STORE, 140249744543744, 140249744560127, +STORE, 140249744535552, 140249744560127, +STORE, 140249744527360, 140249744560127, +STORE, 140249744519168, 140249744560127, +STORE, 140249705619456, 140249721602047, +STORE, 140249744510976, 140249744560127, +STORE, 140249744502784, 140249744560127, +STORE, 140249744494592, 140249744560127, +STORE, 140249744486400, 140249744560127, +STORE, 140249744478208, 140249744560127, +STORE, 140249744470016, 140249744560127, +STORE, 140249744461824, 140249744560127, +STORE, 140249744453632, 140249744560127, +STORE, 140249744445440, 140249744560127, +STORE, 140249744437248, 140249744560127, +STORE, 140249744429056, 140249744560127, +STORE, 140249703522304, 140249721602047, +STORE, 33853440, 36311039, +STORE, 140249703489536, 140249721602047, +STORE, 33853440, 36474879, +STORE, 140249703456768, 140249721602047, +STORE, 33853440, 36622335, +STORE, 140249703424000, 140249721602047, +STORE, 140249703391232, 140249721602047, +STORE, 33853440, 36810751, +STORE, 140249703358464, 140249721602047, +STORE, 140249703325696, 140249721602047, +SNULL, 36655103, 36810751, +STORE, 33853440, 36655103, +STORE, 36655104, 36810751, +ERASE, 36655104, 36810751, +SNULL, 36438015, 36655103, +STORE, 33853440, 36438015, +STORE, 36438016, 36655103, +ERASE, 36438016, 36655103, +STORE, 140249703317504, 140249721602047, +STORE, 140249701220352, 140249721602047, +STORE, 33853440, 36585471, +STORE, 33853440, 36782079, +STORE, 140249701212160, 140249721602047, +STORE, 140249701203968, 140249721602047, +STORE, 140249701195776, 140249721602047, +STORE, 140249701187584, 140249721602047, +STORE, 140249701179392, 140249721602047, +STORE, 140249701171200, 140249721602047, +STORE, 140249701163008, 140249721602047, +STORE, 140249701154816, 140249721602047, +STORE, 140249701146624, 140249721602047, +STORE, 140249701138432, 140249721602047, +STORE, 140249701130240, 140249721602047, +STORE, 140249700081664, 140249721602047, +STORE, 140249700073472, 140249721602047, +STORE, 33853440, 36978687, +STORE, 140249697976320, 140249721602047, +STORE, 33853440, 37240831, +STORE, 140249695879168, 140249721602047, +STORE, 140249695870976, 140249721602047, +STORE, 140249695862784, 140249721602047, +STORE, 140249695854592, 140249721602047, +STORE, 140249695326208, 140249721602047, +SNULL, 140249710600191, 140249721602047, +STORE, 140249695326208, 140249710600191, +STORE, 140249710600192, 140249721602047, +SNULL, 140249710600192, 140249710866431, +STORE, 140249710866432, 140249721602047, +STORE, 140249710600192, 140249710866431, +ERASE, 140249710600192, 140249710866431, +STORE, 140249691131904, 140249710600191, +STORE, 33853440, 37474303, +STORE, 140249710858240, 140249721602047, +STORE, 140249710850048, 140249721602047, +STORE, 140249710841856, 140249721602047, +STORE, 140249710833664, 140249721602047, +STORE, 140249710825472, 140249721602047, +STORE, 140249710817280, 140249721602047, +STORE, 140249710809088, 140249721602047, +STORE, 140249710800896, 140249721602047, +STORE, 140249710792704, 140249721602047, +STORE, 140249710784512, 140249721602047, +STORE, 140249710776320, 140249721602047, +STORE, 140249710768128, 140249721602047, +STORE, 140249710759936, 140249721602047, +STORE, 140249710751744, 140249721602047, +STORE, 140249710743552, 140249721602047, +STORE, 140249710735360, 140249721602047, +STORE, 140249689034752, 140249710600191, +STORE, 140249710727168, 140249721602047, +STORE, 140249686937600, 140249710600191, +STORE, 33853440, 37867519, +STORE, 140249684840448, 140249710600191, +STORE, 140249710718976, 140249721602047, +STORE, 140249682743296, 140249710600191, +STORE, 140249710710784, 140249721602047, +STORE, 140249710702592, 140249721602047, +STORE, 140249710694400, 140249721602047, +STORE, 140249710686208, 140249721602047, +STORE, 140249710678016, 140249721602047, +STORE, 140249682612224, 140249710600191, +STORE, 140249682087936, 140249710600191, +SNULL, 140249705619455, 140249710600191, +STORE, 140249682087936, 140249705619455, +STORE, 140249705619456, 140249710600191, +SNULL, 140249705619456, 140249705881599, +STORE, 140249705881600, 140249710600191, +STORE, 140249705619456, 140249705881599, +ERASE, 140249705619456, 140249705881599, +STORE, 140249679990784, 140249705619455, +STORE, 140249710669824, 140249721602047, +STORE, 140249677893632, 140249705619455, +STORE, 140249710653440, 140249721602047, +STORE, 140249710645248, 140249721602047, +STORE, 140249710637056, 140249721602047, +STORE, 140249710628864, 140249721602047, +STORE, 140249710620672, 140249721602047, +STORE, 140249710612480, 140249721602047, +STORE, 140249710604288, 140249721602047, +STORE, 140249705873408, 140249710600191, +STORE, 140249705865216, 140249710600191, +STORE, 140249705857024, 140249710600191, +STORE, 140249705848832, 140249710600191, +STORE, 140249705840640, 140249710600191, +STORE, 140249705832448, 140249710600191, +STORE, 140249705824256, 140249710600191, +STORE, 140249705816064, 140249710600191, +STORE, 140249705807872, 140249710600191, +STORE, 140249705799680, 140249710600191, +STORE, 33853440, 38129663, +SNULL, 140249744207872, 140249744367615, +STORE, 140249744367616, 140249744424959, +STORE, 140249744207872, 140249744367615, +ERASE, 140249744207872, 140249744367615, +STORE, 140249677606912, 140249705619455, +STORE, 140249675509760, 140249705619455, +SNULL, 140249677606911, 140249705619455, +STORE, 140249675509760, 140249677606911, +STORE, 140249677606912, 140249705619455, +SNULL, 140249677606912, 140249677893631, +STORE, 140249677893632, 140249705619455, +STORE, 140249677606912, 140249677893631, +ERASE, 140249677606912, 140249677893631, +STORE, 140249744359424, 140249744424959, +STORE, 33853440, 38391807, +STORE, 140249674981376, 140249677606911, +STORE, 140249672884224, 140249677606911, +SNULL, 140249719230463, 140249721602047, +STORE, 140249710604288, 140249719230463, +STORE, 140249719230464, 140249721602047, +SNULL, 140249719230464, 140249719504895, +STORE, 140249719504896, 140249721602047, +STORE, 140249719230464, 140249719504895, +ERASE, 140249719230464, 140249719504895, +STORE, 140249744351232, 140249744424959, +STORE, 140249744343040, 140249744424959, +STORE, 140249744334848, 140249744424959, +STORE, 140249744326656, 140249744424959, +STORE, 140249744310272, 140249744424959, +STORE, 140249744302080, 140249744424959, +STORE, 140249744285696, 140249744424959, +STORE, 140249744277504, 140249744424959, +STORE, 140249744261120, 140249744424959, +STORE, 140249744252928, 140249744424959, +STORE, 140249744220160, 140249744424959, +STORE, 140249744211968, 140249744424959, +STORE, 140249719488512, 140249721602047, +STORE, 140249744203776, 140249744424959, +STORE, 140249719472128, 140249721602047, +STORE, 140249719463936, 140249721602047, +STORE, 140249719447552, 140249721602047, +STORE, 140249719439360, 140249721602047, +STORE, 140249719406592, 140249721602047, +STORE, 140249719398400, 140249721602047, +STORE, 140249719382016, 140249721602047, +STORE, 140249719373824, 140249721602047, +STORE, 140249719357440, 140249721602047, +STORE, 140249719349248, 140249721602047, +STORE, 140249719332864, 140249721602047, +STORE, 140249719324672, 140249721602047, +STORE, 140249719291904, 140249721602047, +STORE, 140249719283712, 140249721602047, +STORE, 140249719267328, 140249721602047, +STORE, 140249719259136, 140249721602047, +STORE, 140249719242752, 140249721602047, +STORE, 140249719234560, 140249721602047, +STORE, 140249705783296, 140249710600191, +STORE, 140249705775104, 140249710600191, +STORE, 140249705742336, 140249710600191, +STORE, 140249705734144, 140249710600191, +STORE, 140249705717760, 140249710600191, +STORE, 140249670787072, 140249677606911, +STORE, 140249705709568, 140249710600191, +STORE, 140249705693184, 140249710600191, +STORE, 140249705684992, 140249710600191, +STORE, 140249705668608, 140249710600191, +STORE, 140249705660416, 140249710600191, +STORE, 140249705627648, 140249710600191, +STORE, 140249677893632, 140249710600191, +STORE, 140249677877248, 140249710600191, +STORE, 140249677869056, 140249710600191, +STORE, 140249677852672, 140249710600191, +STORE, 140249677844480, 140249710600191, +STORE, 140249677828096, 140249710600191, +STORE, 140249668689920, 140249677606911, +STORE, 140249677819904, 140249710600191, +STORE, 140249677787136, 140249710600191, +STORE, 140249677778944, 140249710600191, +STORE, 140249677762560, 140249710600191, +STORE, 140249677754368, 140249710600191, +STORE, 140249677737984, 140249710600191, +STORE, 140249677729792, 140249710600191, +STORE, 140249677713408, 140249710600191, +STORE, 140249677705216, 140249710600191, +STORE, 140249677672448, 140249710600191, +STORE, 140249677664256, 140249710600191, +STORE, 140249677647872, 140249710600191, +STORE, 140249677639680, 140249710600191, +STORE, 140249677623296, 140249710600191, +STORE, 140249677615104, 140249710600191, +STORE, 140249668673536, 140249677606911, +STORE, 140249668673536, 140249710600191, +STORE, 140249668640768, 140249710600191, +STORE, 140249668632576, 140249710600191, +STORE, 140249668616192, 140249710600191, +STORE, 140249668608000, 140249710600191, +STORE, 140249668591616, 140249710600191, +STORE, 140249668583424, 140249710600191, +STORE, 140249668567040, 140249710600191, +STORE, 140249668558848, 140249710600191, +STORE, 140249668526080, 140249710600191, +STORE, 140249668517888, 140249710600191, +STORE, 140249668501504, 140249710600191, +STORE, 140249668493312, 140249710600191, +STORE, 140249668476928, 140249710600191, +STORE, 140249668468736, 140249710600191, +STORE, 140249668452352, 140249710600191, +STORE, 140249668444160, 140249710600191, +STORE, 140249668411392, 140249710600191, +STORE, 140249668403200, 140249710600191, +STORE, 140249668386816, 140249710600191, +STORE, 140249668378624, 140249710600191, +STORE, 140249668362240, 140249710600191, +STORE, 140249668354048, 140249710600191, +STORE, 140249668337664, 140249710600191, +STORE, 140249668329472, 140249710600191, +STORE, 140249668296704, 140249710600191, +STORE, 140249668288512, 140249710600191, +STORE, 140249668272128, 140249710600191, +STORE, 140249668263936, 140249710600191, +STORE, 140249668247552, 140249710600191, +STORE, 140249668239360, 140249710600191, +STORE, 140249668222976, 140249710600191, +STORE, 140249668214784, 140249710600191, +STORE, 140249668182016, 140249710600191, +STORE, 140249668173824, 140249710600191, +STORE, 140249668157440, 140249710600191, +STORE, 140249668149248, 140249710600191, +STORE, 140249668132864, 140249710600191, +STORE, 140249668124672, 140249710600191, +STORE, 140249668108288, 140249710600191, +STORE, 140249668100096, 140249710600191, +STORE, 140249668067328, 140249710600191, +STORE, 140249668059136, 140249710600191, +STORE, 140249668042752, 140249710600191, +STORE, 140249668034560, 140249710600191, +STORE, 140249668018176, 140249710600191, +STORE, 140249668009984, 140249710600191, +STORE, 140249667993600, 140249710600191, +STORE, 140249667985408, 140249710600191, +STORE, 140249667952640, 140249710600191, +STORE, 140249667944448, 140249710600191, +STORE, 140249667928064, 140249710600191, +STORE, 140249667919872, 140249710600191, +STORE, 140249667903488, 140249710600191, +STORE, 140249667895296, 140249710600191, +STORE, 140249667878912, 140249710600191, +STORE, 140249667870720, 140249710600191, +STORE, 140249667837952, 140249710600191, +STORE, 140249667829760, 140249710600191, +STORE, 140249667813376, 140249710600191, +STORE, 140249667805184, 140249710600191, +STORE, 140249667788800, 140249710600191, +STORE, 140249667780608, 140249710600191, +STORE, 140249667764224, 140249710600191, +STORE, 140249667756032, 140249710600191, +STORE, 140249667723264, 140249710600191, +STORE, 140249667715072, 140249710600191, +STORE, 140249667698688, 140249710600191, +STORE, 140249667690496, 140249710600191, +STORE, 140249667674112, 140249710600191, +STORE, 140249667665920, 140249710600191, +STORE, 140249667649536, 140249710600191, +STORE, 140249667641344, 140249710600191, +STORE, 140249667608576, 140249710600191, +STORE, 140249667600384, 140249710600191, +STORE, 140249667584000, 140249710600191, +STORE, 140249667575808, 140249710600191, +STORE, 140249667559424, 140249710600191, +STORE, 140249667551232, 140249710600191, +STORE, 140249667534848, 140249710600191, +STORE, 140249667526656, 140249710600191, +STORE, 140249667493888, 140249710600191, +STORE, 140249667485696, 140249710600191, +STORE, 140249667469312, 140249710600191, +STORE, 140249667461120, 140249710600191, +STORE, 140249667444736, 140249710600191, +STORE, 140249667436544, 140249710600191, +STORE, 140249667420160, 140249710600191, +STORE, 140249665323008, 140249710600191, +STORE, 140249665314816, 140249710600191, +STORE, 140249665282048, 140249710600191, +STORE, 140249665273856, 140249710600191, +STORE, 140249665257472, 140249710600191, +STORE, 140249665249280, 140249710600191, +STORE, 140249665232896, 140249710600191, +STORE, 140249665224704, 140249710600191, +STORE, 140249665208320, 140249710600191, +STORE, 140249665200128, 140249710600191, +STORE, 140249665167360, 140249710600191, +STORE, 140249665159168, 140249710600191, +STORE, 140249665142784, 140249710600191, +STORE, 140249665134592, 140249710600191, +STORE, 140249665118208, 140249710600191, +STORE, 140249665110016, 140249710600191, +STORE, 140249665093632, 140249710600191, +STORE, 140249665085440, 140249710600191, +STORE, 140249665052672, 140249710600191, +STORE, 140249665044480, 140249710600191, +STORE, 140249665028096, 140249710600191, +STORE, 140249665019904, 140249710600191, +STORE, 140249665003520, 140249710600191, +STORE, 140249664995328, 140249710600191, +STORE, 140249664978944, 140249710600191, +STORE, 140249664970752, 140249710600191, +STORE, 140249664937984, 140249710600191, +STORE, 140249664929792, 140249710600191, +STORE, 140249664913408, 140249710600191, +STORE, 140249664905216, 140249710600191, +STORE, 140249664888832, 140249710600191, +STORE, 140249664880640, 140249710600191, +STORE, 140249664864256, 140249710600191, +STORE, 140249664856064, 140249710600191, +STORE, 140249664823296, 140249710600191, +STORE, 140249664815104, 140249710600191, +STORE, 140249664798720, 140249710600191, +STORE, 140249664790528, 140249710600191, +STORE, 140249664774144, 140249710600191, +STORE, 140249664765952, 140249710600191, +STORE, 140249664749568, 140249710600191, +STORE, 140249664741376, 140249710600191, +STORE, 140249664708608, 140249710600191, +STORE, 140249664700416, 140249710600191, +STORE, 140249664684032, 140249710600191, +STORE, 140249664675840, 140249710600191, +STORE, 140249664659456, 140249710600191, +STORE, 140249664651264, 140249710600191, +STORE, 140249664634880, 140249710600191, +STORE, 140249664626688, 140249710600191, +STORE, 140249664593920, 140249710600191, +STORE, 140249664585728, 140249710600191, +STORE, 140249664569344, 140249710600191, +STORE, 140249664561152, 140249710600191, +STORE, 140249664544768, 140249710600191, +STORE, 140249664536576, 140249710600191, +STORE, 140249664520192, 140249710600191, +STORE, 140249664512000, 140249710600191, +STORE, 140249664479232, 140249710600191, +STORE, 140249664471040, 140249710600191, +STORE, 140249664454656, 140249710600191, +STORE, 140249664446464, 140249710600191, +STORE, 140249664430080, 140249710600191, +STORE, 140249664421888, 140249710600191, +STORE, 140249664405504, 140249710600191, +STORE, 140249664397312, 140249710600191, +STORE, 140249664364544, 140249710600191, +STORE, 140249664356352, 140249710600191, +STORE, 140249664339968, 140249710600191, +STORE, 140249664331776, 140249710600191, +STORE, 140249664315392, 140249710600191, +STORE, 140249664307200, 140249710600191, +STORE, 140249664290816, 140249710600191, +STORE, 140249664282624, 140249710600191, +STORE, 140249664249856, 140249710600191, +STORE, 140249664241664, 140249710600191, +STORE, 140249664225280, 140249710600191, +STORE, 140249664217088, 140249710600191, +STORE, 140249664200704, 140249710600191, +STORE, 140249664192512, 140249710600191, +STORE, 140249664176128, 140249710600191, +STORE, 140249664167936, 140249710600191, +STORE, 140249664135168, 140249710600191, +STORE, 140249664126976, 140249710600191, +STORE, 140249664110592, 140249710600191, +STORE, 140249664102400, 140249710600191, +STORE, 140249664086016, 140249710600191, +STORE, 140249664077824, 140249710600191, +STORE, 140249664061440, 140249710600191, +STORE, 140249664053248, 140249710600191, +STORE, 140249664020480, 140249710600191, +STORE, 140249664012288, 140249710600191, +STORE, 140249663995904, 140249710600191, +STORE, 140249663987712, 140249710600191, +STORE, 140249663971328, 140249710600191, +STORE, 140249663963136, 140249710600191, +STORE, 140249663946752, 140249710600191, +STORE, 140249663938560, 140249710600191, +STORE, 140249663905792, 140249710600191, +STORE, 140249663897600, 140249710600191, +STORE, 140249663881216, 140249710600191, +STORE, 140249663873024, 140249710600191, +STORE, 140249663856640, 140249710600191, +STORE, 140249663848448, 140249710600191, +STORE, 140249663832064, 140249710600191, +STORE, 140249663823872, 140249710600191, +STORE, 140249663791104, 140249710600191, +STORE, 140249663782912, 140249710600191, +STORE, 140249663766528, 140249710600191, +STORE, 140249663758336, 140249710600191, +STORE, 140249663741952, 140249710600191, +STORE, 140249663733760, 140249710600191, +STORE, 140249663717376, 140249710600191, +STORE, 140249663709184, 140249710600191, +STORE, 140249663676416, 140249710600191, +STORE, 140249663668224, 140249710600191, +STORE, 140249663651840, 140249710600191, +STORE, 140249663643648, 140249710600191, +STORE, 140249663627264, 140249710600191, +STORE, 33853440, 38526975, +STORE, 140249663619072, 140249710600191, +STORE, 140249663602688, 140249710600191, +STORE, 140249661505536, 140249710600191, +STORE, 140249661497344, 140249710600191, +STORE, 140249661464576, 140249710600191, +STORE, 140249661456384, 140249710600191, +STORE, 140249661440000, 140249710600191, +STORE, 140249661431808, 140249710600191, +STORE, 140249661415424, 140249710600191, +STORE, 140249661407232, 140249710600191, +STORE, 140249661390848, 140249710600191, +STORE, 140249661382656, 140249710600191, +STORE, 140249661349888, 140249710600191, +STORE, 140249661341696, 140249710600191, +STORE, 140249661325312, 140249710600191, +STORE, 140249661317120, 140249710600191, +STORE, 140249661300736, 140249710600191, +STORE, 140249661292544, 140249710600191, +STORE, 140249661276160, 140249710600191, +STORE, 140249661267968, 140249710600191, +STORE, 140249661235200, 140249710600191, +STORE, 140249661227008, 140249710600191, +STORE, 140249661210624, 140249710600191, +STORE, 140249661202432, 140249710600191, +STORE, 140249661186048, 140249710600191, +STORE, 140249661177856, 140249710600191, +STORE, 140249661161472, 140249710600191, +STORE, 140249661153280, 140249710600191, +STORE, 140249661120512, 140249710600191, +STORE, 140249661112320, 140249710600191, +STORE, 140249661095936, 140249710600191, +STORE, 140249661087744, 140249710600191, +STORE, 140249661071360, 140249710600191, +STORE, 140249661063168, 140249710600191, +STORE, 140249661046784, 140249710600191, +STORE, 140249661038592, 140249710600191, +STORE, 140249661005824, 140249710600191, +STORE, 140249660997632, 140249710600191, +STORE, 140249660981248, 140249710600191, +STORE, 140249660973056, 140249710600191, +STORE, 140249660956672, 140249710600191, +STORE, 140249660948480, 140249710600191, +STORE, 140249660932096, 140249710600191, +STORE, 140249660923904, 140249710600191, +STORE, 140249660891136, 140249710600191, +STORE, 140249660882944, 140249710600191, +STORE, 140249660866560, 140249710600191, +STORE, 140249660858368, 140249710600191, +STORE, 140249660841984, 140249710600191, +STORE, 140249660833792, 140249710600191, +STORE, 140249660817408, 140249710600191, +STORE, 140249660809216, 140249710600191, +STORE, 140249660776448, 140249710600191, +STORE, 140249660768256, 140249710600191, +STORE, 140249660751872, 140249710600191, +STORE, 140249660743680, 140249710600191, +STORE, 140249660727296, 140249710600191, +STORE, 140249660719104, 140249710600191, +STORE, 140249660702720, 140249710600191, +STORE, 140249660694528, 140249710600191, +STORE, 140249660661760, 140249710600191, +STORE, 140249660653568, 140249710600191, +STORE, 140249660637184, 140249710600191, +STORE, 140249660628992, 140249710600191, +STORE, 140249660612608, 140249710600191, +STORE, 140249660604416, 140249710600191, +STORE, 140249660588032, 140249710600191, +STORE, 140249660579840, 140249710600191, +STORE, 140249660547072, 140249710600191, +STORE, 140249660538880, 140249710600191, +STORE, 140249660522496, 140249710600191, +STORE, 140249660514304, 140249710600191, +STORE, 140249660497920, 140249710600191, +STORE, 140249660489728, 140249710600191, +STORE, 140249660473344, 140249710600191, +STORE, 140249660465152, 140249710600191, +STORE, 140249660432384, 140249710600191, +STORE, 140249660424192, 140249710600191, +STORE, 140249660407808, 140249710600191, +STORE, 140249660399616, 140249710600191, +STORE, 140249660383232, 140249710600191, +STORE, 140249660375040, 140249710600191, +STORE, 140249660358656, 140249710600191, +STORE, 140249660350464, 140249710600191, +STORE, 140249660317696, 140249710600191, +STORE, 140249660309504, 140249710600191, +STORE, 140249660293120, 140249710600191, +STORE, 140249660284928, 140249710600191, +STORE, 140249660268544, 140249710600191, +STORE, 140249660260352, 140249710600191, +STORE, 140249660243968, 140249710600191, +STORE, 140249660235776, 140249710600191, +STORE, 140249660203008, 140249710600191, +STORE, 140249660194816, 140249710600191, +STORE, 140249660178432, 140249710600191, +STORE, 140249660170240, 140249710600191, +STORE, 140249660153856, 140249710600191, +STORE, 140249660145664, 140249710600191, +STORE, 140249660129280, 140249710600191, +STORE, 140249660121088, 140249710600191, +STORE, 140249660088320, 140249710600191, +STORE, 140249660080128, 140249710600191, +STORE, 140249660063744, 140249710600191, +STORE, 140249660055552, 140249710600191, +STORE, 140249660039168, 140249710600191, +STORE, 140249660030976, 140249710600191, +STORE, 140249660014592, 140249710600191, +STORE, 140249660006400, 140249710600191, +STORE, 140249659973632, 140249710600191, +STORE, 140249659965440, 140249710600191, +STORE, 140249659949056, 140249710600191, +STORE, 140249659940864, 140249710600191, +STORE, 140249659924480, 140249710600191, +STORE, 140249659916288, 140249710600191, +STORE, 140249659899904, 140249710600191, +STORE, 140249659891712, 140249710600191, +STORE, 140249659858944, 140249710600191, +STORE, 140249659850752, 140249710600191, +STORE, 140249659834368, 140249710600191, +STORE, 140249659826176, 140249710600191, +STORE, 140249659809792, 140249710600191, +STORE, 140249659801600, 140249710600191, +STORE, 140249659785216, 140249710600191, +STORE, 140249657688064, 140249710600191, +STORE, 140249657679872, 140249710600191, +STORE, 140249657647104, 140249710600191, +STORE, 140249657638912, 140249710600191, +STORE, 140249657622528, 140249710600191, +STORE, 140249657614336, 140249710600191, +STORE, 140249657597952, 140249710600191, +STORE, 140249657589760, 140249710600191, +STORE, 140249657573376, 140249710600191, +STORE, 140249657565184, 140249710600191, +STORE, 140249657532416, 140249710600191, +STORE, 140249657524224, 140249710600191, +STORE, 140249657507840, 140249710600191, +STORE, 140249657499648, 140249710600191, +STORE, 140249657483264, 140249710600191, +STORE, 140249657475072, 140249710600191, +STORE, 140249657458688, 140249710600191, +STORE, 140249657450496, 140249710600191, +STORE, 140249657417728, 140249710600191, +STORE, 140249657409536, 140249710600191, +STORE, 140249657393152, 140249710600191, +STORE, 140249657384960, 140249710600191, +STORE, 140249657368576, 140249710600191, +STORE, 140249657360384, 140249710600191, +STORE, 140249657344000, 140249710600191, +STORE, 140249657335808, 140249710600191, +STORE, 140249657303040, 140249710600191, +STORE, 140249657294848, 140249710600191, +STORE, 140249657278464, 140249710600191, +STORE, 140249657270272, 140249710600191, +STORE, 140249657253888, 140249710600191, +STORE, 140249657245696, 140249710600191, +STORE, 140249657229312, 140249710600191, +STORE, 140249657221120, 140249710600191, +STORE, 140249657188352, 140249710600191, +STORE, 140249657180160, 140249710600191, +STORE, 140249657163776, 140249710600191, +STORE, 140249657155584, 140249710600191, +STORE, 140249657139200, 140249710600191, +STORE, 140249657131008, 140249710600191, +STORE, 140249657114624, 140249710600191, +STORE, 140249657106432, 140249710600191, +STORE, 140249657073664, 140249710600191, +STORE, 140249657065472, 140249710600191, +STORE, 140249657049088, 140249710600191, +STORE, 140249657040896, 140249710600191, +STORE, 140249657024512, 140249710600191, +STORE, 140249657016320, 140249710600191, +STORE, 140249656999936, 140249710600191, +STORE, 140249656991744, 140249710600191, +STORE, 140249656958976, 140249710600191, +STORE, 140249656950784, 140249710600191, +STORE, 140249656934400, 140249710600191, +STORE, 140249656926208, 140249710600191, +STORE, 140249656909824, 140249710600191, +STORE, 140249656901632, 140249710600191, +STORE, 140249656885248, 140249710600191, +STORE, 140249656877056, 140249710600191, +STORE, 140249656844288, 140249710600191, +STORE, 140249656836096, 140249710600191, +STORE, 140249656819712, 140249710600191, +STORE, 140249656811520, 140249710600191, +STORE, 140249656795136, 140249710600191, +STORE, 33853440, 38662143, +STORE, 140249656786944, 140249710600191, +STORE, 140249656770560, 140249710600191, +STORE, 140249656762368, 140249710600191, +STORE, 140249656729600, 140249710600191, +STORE, 140249656721408, 140249710600191, +STORE, 140249656705024, 140249710600191, +STORE, 140249656696832, 140249710600191, +STORE, 140249656680448, 140249710600191, +STORE, 140249656672256, 140249710600191, +STORE, 140249656655872, 140249710600191, +STORE, 140249656647680, 140249710600191, +STORE, 140249656614912, 140249710600191, +STORE, 140249656606720, 140249710600191, +STORE, 140249656590336, 140249710600191, +STORE, 140249656582144, 140249710600191, +STORE, 140249656565760, 140249710600191, +STORE, 140249656557568, 140249710600191, +STORE, 140249656541184, 140249710600191, +STORE, 140249656532992, 140249710600191, +STORE, 140249656500224, 140249710600191, +STORE, 140249656492032, 140249710600191, +STORE, 140249656475648, 140249710600191, +STORE, 140249656467456, 140249710600191, +STORE, 140249656451072, 140249710600191, +STORE, 140249656442880, 140249710600191, +STORE, 140249656426496, 140249710600191, +STORE, 140249656418304, 140249710600191, +STORE, 140249656385536, 140249710600191, +STORE, 140249656377344, 140249710600191, +STORE, 140249656360960, 140249710600191, +STORE, 140249656352768, 140249710600191, +STORE, 140249656336384, 140249710600191, +STORE, 140249656328192, 140249710600191, +STORE, 140249656311808, 140249710600191, +STORE, 140249656303616, 140249710600191, +STORE, 140249656270848, 140249710600191, +STORE, 140249656262656, 140249710600191, +STORE, 140249656246272, 140249710600191, +STORE, 140249656238080, 140249710600191, +STORE, 140249656221696, 140249710600191, +STORE, 140249656213504, 140249710600191, +STORE, 140249656197120, 140249710600191, +STORE, 140249656188928, 140249710600191, +STORE, 140249656156160, 140249710600191, +STORE, 140249656147968, 140249710600191, +STORE, 140249656131584, 140249710600191, +STORE, 140249656123392, 140249710600191, +STORE, 140249656107008, 140249710600191, +STORE, 140249656098816, 140249710600191, +STORE, 140249656082432, 140249710600191, +STORE, 140249656074240, 140249710600191, +STORE, 140249656041472, 140249710600191, +STORE, 140249656033280, 140249710600191, +STORE, 140249656016896, 140249710600191, +STORE, 140249656008704, 140249710600191, +STORE, 140249655992320, 140249710600191, +STORE, 140249655984128, 140249710600191, +STORE, 140249655967744, 140249710600191, +STORE, 140249653870592, 140249710600191, +STORE, 140249653862400, 140249710600191, +STORE, 140249653829632, 140249710600191, +STORE, 140249653821440, 140249710600191, +STORE, 140249653805056, 140249710600191, +STORE, 140249653796864, 140249710600191, +STORE, 140249653780480, 140249710600191, +STORE, 140249653772288, 140249710600191, +STORE, 140249653755904, 140249710600191, +STORE, 140249652703232, 140249710600191, +SNULL, 140249682087935, 140249710600191, +STORE, 140249652703232, 140249682087935, +STORE, 140249682087936, 140249710600191, + }; + + unsigned long set26[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140729464770560, 140737488351231, +SNULL, 140729464774655, 140737488351231, +STORE, 140729464770560, 140729464774655, +STORE, 140729464639488, 140729464774655, +STORE, 4194304, 5066751, +STORE, 7159808, 7172095, +STORE, 7172096, 7180287, +STORE, 140729465114624, 140729465118719, +STORE, 140729465102336, 140729465114623, +STORE, 30867456, 30875647, +STORE, 30867456, 31010815, +STORE, 140109040988160, 140109042671615, +STORE, 140109040959488, 140109040988159, +STORE, 140109040943104, 140109040959487, +ERASE, 140109040943104, 140109040959487, +STORE, 140109040840704, 140109040959487, +ERASE, 140109040840704, 140109040959487, +STORE, 140109040951296, 140109040959487, +ERASE, 140109040951296, 140109040959487, +STORE, 140109040955392, 140109040959487, +ERASE, 140109040955392, 140109040959487, + }; + unsigned long set27[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140726128070656, 140737488351231, +SNULL, 140726128074751, 140737488351231, +STORE, 140726128070656, 140726128074751, +STORE, 140726127939584, 140726128074751, +STORE, 94478497189888, 94478499303423, +SNULL, 94478497202175, 94478499303423, +STORE, 94478497189888, 94478497202175, +STORE, 94478497202176, 94478499303423, +ERASE, 94478497202176, 94478499303423, +STORE, 94478499295232, 94478499303423, +STORE, 140415605723136, 140415607975935, +SNULL, 140415605866495, 140415607975935, +STORE, 140415605723136, 140415605866495, +STORE, 140415605866496, 140415607975935, +ERASE, 140415605866496, 140415607975935, +STORE, 140415607963648, 140415607971839, +STORE, 140415607971840, 140415607975935, +STORE, 140726130024448, 140726130028543, +STORE, 140726130012160, 140726130024447, +STORE, 140415607934976, 140415607963647, +STORE, 140415607926784, 140415607934975, +STORE, 140415603245056, 140415605723135, +SNULL, 140415603245056, 140415603613695, +STORE, 140415603613696, 140415605723135, +STORE, 140415603245056, 140415603613695, +SNULL, 140415605710847, 140415605723135, +STORE, 140415603613696, 140415605710847, +STORE, 140415605710848, 140415605723135, +ERASE, 140415605710848, 140415605723135, +STORE, 140415605710848, 140415605723135, +STORE, 140415599370240, 140415603245055, +SNULL, 140415599370240, 140415601111039, +STORE, 140415601111040, 140415603245055, +STORE, 140415599370240, 140415601111039, +SNULL, 140415603208191, 140415603245055, +STORE, 140415601111040, 140415603208191, +STORE, 140415603208192, 140415603245055, +ERASE, 140415603208192, 140415603245055, +STORE, 140415603208192, 140415603245055, +STORE, 140415595692032, 140415599370239, +SNULL, 140415595692032, 140415597207551, +STORE, 140415597207552, 140415599370239, +STORE, 140415595692032, 140415597207551, +SNULL, 140415599304703, 140415599370239, +STORE, 140415597207552, 140415599304703, +STORE, 140415599304704, 140415599370239, +SNULL, 140415599304704, 140415599353855, +STORE, 140415599353856, 140415599370239, +STORE, 140415599304704, 140415599353855, +ERASE, 140415599304704, 140415599353855, +STORE, 140415599304704, 140415599353855, +ERASE, 140415599353856, 140415599370239, +STORE, 140415599353856, 140415599370239, +STORE, 140415593500672, 140415595692031, +SNULL, 140415593500672, 140415593590783, +STORE, 140415593590784, 140415595692031, +STORE, 140415593500672, 140415593590783, +SNULL, 140415595683839, 140415595692031, +STORE, 140415593590784, 140415595683839, +STORE, 140415595683840, 140415595692031, +ERASE, 140415595683840, 140415595692031, +STORE, 140415595683840, 140415595692031, +STORE, 140415589703680, 140415593500671, +SNULL, 140415589703680, 140415591362559, +STORE, 140415591362560, 140415593500671, +STORE, 140415589703680, 140415591362559, +SNULL, 140415593459711, 140415593500671, +STORE, 140415591362560, 140415593459711, +STORE, 140415593459712, 140415593500671, +SNULL, 140415593459712, 140415593484287, +STORE, 140415593484288, 140415593500671, +STORE, 140415593459712, 140415593484287, +ERASE, 140415593459712, 140415593484287, +STORE, 140415593459712, 140415593484287, +ERASE, 140415593484288, 140415593500671, +STORE, 140415593484288, 140415593500671, +STORE, 140415587590144, 140415589703679, +SNULL, 140415587590144, 140415587602431, +STORE, 140415587602432, 140415589703679, +STORE, 140415587590144, 140415587602431, +SNULL, 140415589695487, 140415589703679, +STORE, 140415587602432, 140415589695487, +STORE, 140415589695488, 140415589703679, +ERASE, 140415589695488, 140415589703679, +STORE, 140415589695488, 140415589703679, +STORE, 140415607918592, 140415607934975, +STORE, 140415585398784, 140415587590143, +SNULL, 140415585398784, 140415585480703, +STORE, 140415585480704, 140415587590143, +STORE, 140415585398784, 140415585480703, +SNULL, 140415587573759, 140415587590143, +STORE, 140415585480704, 140415587573759, +STORE, 140415587573760, 140415587590143, +SNULL, 140415587573760, 140415587581951, +STORE, 140415587581952, 140415587590143, +STORE, 140415587573760, 140415587581951, +ERASE, 140415587573760, 140415587581951, +STORE, 140415587573760, 140415587581951, +ERASE, 140415587581952, 140415587590143, +STORE, 140415587581952, 140415587590143, +STORE, 140415583182848, 140415585398783, +SNULL, 140415583182848, 140415583281151, +STORE, 140415583281152, 140415585398783, +STORE, 140415583182848, 140415583281151, +SNULL, 140415585374207, 140415585398783, +STORE, 140415583281152, 140415585374207, +STORE, 140415585374208, 140415585398783, +SNULL, 140415585374208, 140415585382399, +STORE, 140415585382400, 140415585398783, +STORE, 140415585374208, 140415585382399, +ERASE, 140415585374208, 140415585382399, +STORE, 140415585374208, 140415585382399, +ERASE, 140415585382400, 140415585398783, +STORE, 140415585382400, 140415585398783, +STORE, 140415580979200, 140415583182847, +SNULL, 140415580979200, 140415581081599, +STORE, 140415581081600, 140415583182847, +STORE, 140415580979200, 140415581081599, +SNULL, 140415583174655, 140415583182847, +STORE, 140415581081600, 140415583174655, +STORE, 140415583174656, 140415583182847, +ERASE, 140415583174656, 140415583182847, +STORE, 140415583174656, 140415583182847, +STORE, 140415578816512, 140415580979199, +SNULL, 140415578816512, 140415578877951, +STORE, 140415578877952, 140415580979199, +STORE, 140415578816512, 140415578877951, +SNULL, 140415580971007, 140415580979199, +STORE, 140415578877952, 140415580971007, +STORE, 140415580971008, 140415580979199, +ERASE, 140415580971008, 140415580979199, +STORE, 140415580971008, 140415580979199, +STORE, 140415576563712, 140415578816511, +SNULL, 140415576563712, 140415576715263, +STORE, 140415576715264, 140415578816511, +STORE, 140415576563712, 140415576715263, +SNULL, 140415578808319, 140415578816511, +STORE, 140415576715264, 140415578808319, +STORE, 140415578808320, 140415578816511, +ERASE, 140415578808320, 140415578816511, +STORE, 140415578808320, 140415578816511, +STORE, 140415574392832, 140415576563711, +SNULL, 140415574392832, 140415574462463, +STORE, 140415574462464, 140415576563711, +STORE, 140415574392832, 140415574462463, +SNULL, 140415576555519, 140415576563711, +STORE, 140415574462464, 140415576555519, +STORE, 140415576555520, 140415576563711, +ERASE, 140415576555520, 140415576563711, +STORE, 140415576555520, 140415576563711, +STORE, 140415607910400, 140415607934975, +STORE, 140415571230720, 140415574392831, +SNULL, 140415571230720, 140415572291583, +STORE, 140415572291584, 140415574392831, +STORE, 140415571230720, 140415572291583, +SNULL, 140415574384639, 140415574392831, +STORE, 140415572291584, 140415574384639, +STORE, 140415574384640, 140415574392831, +ERASE, 140415574384640, 140415574392831, +STORE, 140415574384640, 140415574392831, +STORE, 140415607902208, 140415607934975, +SNULL, 140415593476095, 140415593484287, +STORE, 140415593459712, 140415593476095, +STORE, 140415593476096, 140415593484287, +SNULL, 140415574388735, 140415574392831, +STORE, 140415574384640, 140415574388735, +STORE, 140415574388736, 140415574392831, +SNULL, 140415576559615, 140415576563711, +STORE, 140415576555520, 140415576559615, +STORE, 140415576559616, 140415576563711, +SNULL, 140415589699583, 140415589703679, +STORE, 140415589695488, 140415589699583, +STORE, 140415589699584, 140415589703679, +SNULL, 140415585378303, 140415585382399, +STORE, 140415585374208, 140415585378303, +STORE, 140415585378304, 140415585382399, +SNULL, 140415578812415, 140415578816511, +STORE, 140415578808320, 140415578812415, +STORE, 140415578812416, 140415578816511, +SNULL, 140415580975103, 140415580979199, +STORE, 140415580971008, 140415580975103, +STORE, 140415580975104, 140415580979199, +SNULL, 140415583178751, 140415583182847, +STORE, 140415583174656, 140415583178751, +STORE, 140415583178752, 140415583182847, +SNULL, 140415587577855, 140415587581951, +STORE, 140415587573760, 140415587577855, +STORE, 140415587577856, 140415587581951, +SNULL, 140415595687935, 140415595692031, +STORE, 140415595683840, 140415595687935, +STORE, 140415595687936, 140415595692031, +STORE, 140415607894016, 140415607934975, +SNULL, 140415599345663, 140415599353855, +STORE, 140415599304704, 140415599345663, +STORE, 140415599345664, 140415599353855, +SNULL, 140415603240959, 140415603245055, +STORE, 140415603208192, 140415603240959, +STORE, 140415603240960, 140415603245055, +SNULL, 140415605719039, 140415605723135, +STORE, 140415605710848, 140415605719039, +STORE, 140415605719040, 140415605723135, +SNULL, 94478499299327, 94478499303423, +STORE, 94478499295232, 94478499299327, +STORE, 94478499299328, 94478499303423, +SNULL, 140415607967743, 140415607971839, +STORE, 140415607963648, 140415607967743, +STORE, 140415607967744, 140415607971839, +ERASE, 140415607934976, 140415607963647, +STORE, 94478511173632, 94478511378431, +STORE, 140415606210560, 140415607894015, +STORE, 140415607934976, 140415607963647, +STORE, 94478511173632, 94478511513599, +STORE, 94478511173632, 94478511648767, +SNULL, 94478511615999, 94478511648767, +STORE, 94478511173632, 94478511615999, +STORE, 94478511616000, 94478511648767, +ERASE, 94478511616000, 94478511648767, +STORE, 94478511173632, 94478511751167, +SNULL, 94478511747071, 94478511751167, +STORE, 94478511173632, 94478511747071, +STORE, 94478511747072, 94478511751167, +ERASE, 94478511747072, 94478511751167, +STORE, 94478511173632, 94478511882239, +SNULL, 94478511878143, 94478511882239, +STORE, 94478511173632, 94478511878143, +STORE, 94478511878144, 94478511882239, +ERASE, 94478511878144, 94478511882239, +STORE, 94478511173632, 94478512013311, +SNULL, 94478512009215, 94478512013311, +STORE, 94478511173632, 94478512009215, +STORE, 94478512009216, 94478512013311, +ERASE, 94478512009216, 94478512013311, +STORE, 94478511173632, 94478512144383, +STORE, 94478511173632, 94478512279551, +STORE, 140415606181888, 140415606210559, +STORE, 140415569100800, 140415571230719, +SNULL, 140415569100800, 140415569129471, +STORE, 140415569129472, 140415571230719, +STORE, 140415569100800, 140415569129471, +SNULL, 140415571222527, 140415571230719, +STORE, 140415569129472, 140415571222527, +STORE, 140415571222528, 140415571230719, +ERASE, 140415571222528, 140415571230719, +STORE, 140415571222528, 140415571230719, +STORE, 140415566905344, 140415569100799, +SNULL, 140415566905344, 140415566987263, +STORE, 140415566987264, 140415569100799, +STORE, 140415566905344, 140415566987263, +SNULL, 140415569084415, 140415569100799, +STORE, 140415566987264, 140415569084415, +STORE, 140415569084416, 140415569100799, +SNULL, 140415569084416, 140415569092607, +STORE, 140415569092608, 140415569100799, +STORE, 140415569084416, 140415569092607, +ERASE, 140415569084416, 140415569092607, +STORE, 140415569084416, 140415569092607, +ERASE, 140415569092608, 140415569100799, +STORE, 140415569092608, 140415569100799, +SNULL, 140415569088511, 140415569092607, +STORE, 140415569084416, 140415569088511, +STORE, 140415569088512, 140415569092607, +SNULL, 140415571226623, 140415571230719, +STORE, 140415571222528, 140415571226623, +STORE, 140415571226624, 140415571230719, +ERASE, 140415606181888, 140415606210559, +STORE, 140415606181888, 140415606210559, +STORE, 140415564759040, 140415566905343, +SNULL, 140415564759040, 140415564804095, +STORE, 140415564804096, 140415566905343, +STORE, 140415564759040, 140415564804095, +SNULL, 140415566897151, 140415566905343, +STORE, 140415564804096, 140415566897151, +STORE, 140415566897152, 140415566905343, +ERASE, 140415566897152, 140415566905343, +STORE, 140415566897152, 140415566905343, +STORE, 140415562588160, 140415564759039, +SNULL, 140415562588160, 140415562629119, +STORE, 140415562629120, 140415564759039, +STORE, 140415562588160, 140415562629119, +SNULL, 140415564726271, 140415564759039, +STORE, 140415562629120, 140415564726271, +STORE, 140415564726272, 140415564759039, +SNULL, 140415564726272, 140415564734463, +STORE, 140415564734464, 140415564759039, +STORE, 140415564726272, 140415564734463, +ERASE, 140415564726272, 140415564734463, +STORE, 140415564726272, 140415564734463, +ERASE, 140415564734464, 140415564759039, +STORE, 140415564734464, 140415564759039, +SNULL, 140415564730367, 140415564734463, +STORE, 140415564726272, 140415564730367, +STORE, 140415564730368, 140415564734463, +SNULL, 140415566901247, 140415566905343, +STORE, 140415566897152, 140415566901247, +STORE, 140415566901248, 140415566905343, +ERASE, 140415606181888, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415605944320, 140415606210559, +ERASE, 140415605944320, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 94478511173632, 94478512414719, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 140415606206464, 140415606210559, +ERASE, 140415606206464, 140415606210559, +STORE, 94478511173632, 94478512652287, +STORE, 94478511173632, 94478512787455, +STORE, 94478511173632, 94478512922623, +STORE, 94478511173632, 94478513057791, +STORE, 140415537422336, 140415562588159, +STORE, 94478511173632, 94478513192959, +STORE, 94478511173632, 94478513356799, +STORE, 94478511173632, 94478513491967, +STORE, 94478511173632, 94478513627135, +STORE, 94478511173632, 94478513790975, +STORE, 94478511173632, 94478513926143, +STORE, 94478511173632, 94478514061311, +STORE, 94478511173632, 94478514196479, +STORE, 94478511173632, 94478514331647, +STORE, 94478511173632, 94478514606079, +STORE, 94478511173632, 94478514741247, +STORE, 94478511173632, 94478514876415, +STORE, 94478511173632, 94478515011583, +STORE, 94478511173632, 94478515146751, +STORE, 94478511173632, 94478515281919, +STORE, 94478511173632, 94478515474431, +STORE, 94478511173632, 94478515609599, +STORE, 94478511173632, 94478515744767, +STORE, 140415536922624, 140415562588159, +STORE, 94478511173632, 94478515879935, +STORE, 94478511173632, 94478516015103, +STORE, 94478511173632, 94478516150271, +STORE, 94478511173632, 94478516285439, +STORE, 94478511173632, 94478516420607, +STORE, 94478511173632, 94478516555775, +STORE, 94478511173632, 94478516690943, +STORE, 94478511173632, 94478516826111, +STORE, 94478511173632, 94478516961279, +STORE, 94478511173632, 94478517231615, +STORE, 94478511173632, 94478517366783, +STORE, 94478511173632, 94478517501951, +STORE, 94478511173632, 94478517637119, +STORE, 94478511173632, 94478517772287, +STORE, 94478511173632, 94478517907455, +STORE, 94478511173632, 94478518042623, +STORE, 94478511173632, 94478518177791, +STORE, 94478511173632, 94478518312959, +STORE, 94478511173632, 94478518448127, +STORE, 140415535910912, 140415562588159, +SNULL, 140415536922623, 140415562588159, +STORE, 140415535910912, 140415536922623, +STORE, 140415536922624, 140415562588159, +SNULL, 140415536922624, 140415537422335, +STORE, 140415537422336, 140415562588159, +STORE, 140415536922624, 140415537422335, +ERASE, 140415536922624, 140415537422335, +STORE, 94478511173632, 94478518583295, +STORE, 94478511173632, 94478518718463, +STORE, 94478511173632, 94478518853631, +STORE, 94478511173632, 94478518988799, +STORE, 94478511173632, 94478519123967, +STORE, 94478511173632, 94478519259135, +STORE, 140415509696512, 140415535910911, +ERASE, 140415537422336, 140415562588159, +STORE, 140415482433536, 140415509696511, + }; + unsigned long set28[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722475622400, 140737488351231, +SNULL, 140722475626495, 140737488351231, +STORE, 140722475622400, 140722475626495, +STORE, 140722475491328, 140722475626495, +STORE, 93865834291200, 93865836548095, +SNULL, 93865834422271, 93865836548095, +STORE, 93865834291200, 93865834422271, +STORE, 93865834422272, 93865836548095, +ERASE, 93865834422272, 93865836548095, +STORE, 93865836519424, 93865836527615, +STORE, 93865836527616, 93865836548095, +STORE, 139918411104256, 139918413357055, +SNULL, 139918411247615, 139918413357055, +STORE, 139918411104256, 139918411247615, +STORE, 139918411247616, 139918413357055, +ERASE, 139918411247616, 139918413357055, +STORE, 139918413344768, 139918413352959, +STORE, 139918413352960, 139918413357055, +STORE, 140722476642304, 140722476646399, +STORE, 140722476630016, 140722476642303, +STORE, 139918413316096, 139918413344767, +STORE, 139918413307904, 139918413316095, +STORE, 139918408888320, 139918411104255, +SNULL, 139918408888320, 139918408986623, +STORE, 139918408986624, 139918411104255, +STORE, 139918408888320, 139918408986623, +SNULL, 139918411079679, 139918411104255, +STORE, 139918408986624, 139918411079679, +STORE, 139918411079680, 139918411104255, +SNULL, 139918411079680, 139918411087871, +STORE, 139918411087872, 139918411104255, +STORE, 139918411079680, 139918411087871, +ERASE, 139918411079680, 139918411087871, +STORE, 139918411079680, 139918411087871, +ERASE, 139918411087872, 139918411104255, +STORE, 139918411087872, 139918411104255, +STORE, 139918405091328, 139918408888319, +SNULL, 139918405091328, 139918406750207, +STORE, 139918406750208, 139918408888319, +STORE, 139918405091328, 139918406750207, +SNULL, 139918408847359, 139918408888319, +STORE, 139918406750208, 139918408847359, +STORE, 139918408847360, 139918408888319, +SNULL, 139918408847360, 139918408871935, +STORE, 139918408871936, 139918408888319, +STORE, 139918408847360, 139918408871935, +ERASE, 139918408847360, 139918408871935, +STORE, 139918408847360, 139918408871935, +ERASE, 139918408871936, 139918408888319, +STORE, 139918408871936, 139918408888319, +STORE, 139918413299712, 139918413316095, +SNULL, 139918408863743, 139918408871935, +STORE, 139918408847360, 139918408863743, +STORE, 139918408863744, 139918408871935, +SNULL, 139918411083775, 139918411087871, +STORE, 139918411079680, 139918411083775, +STORE, 139918411083776, 139918411087871, +SNULL, 93865836523519, 93865836527615, +STORE, 93865836519424, 93865836523519, +STORE, 93865836523520, 93865836527615, +SNULL, 139918413348863, 139918413352959, +STORE, 139918413344768, 139918413348863, +STORE, 139918413348864, 139918413352959, +ERASE, 139918413316096, 139918413344767, +STORE, 93865848528896, 93865848664063, + }; + unsigned long set29[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734467944448, 140737488351231, +SNULL, 140734467948543, 140737488351231, +STORE, 140734467944448, 140734467948543, +STORE, 140734467813376, 140734467948543, +STORE, 94880407924736, 94880410177535, +SNULL, 94880408055807, 94880410177535, +STORE, 94880407924736, 94880408055807, +STORE, 94880408055808, 94880410177535, +ERASE, 94880408055808, 94880410177535, +STORE, 94880410148864, 94880410157055, +STORE, 94880410157056, 94880410177535, +STORE, 140143367815168, 140143370067967, +SNULL, 140143367958527, 140143370067967, +STORE, 140143367815168, 140143367958527, +STORE, 140143367958528, 140143370067967, +ERASE, 140143367958528, 140143370067967, +STORE, 140143370055680, 140143370063871, +STORE, 140143370063872, 140143370067967, +STORE, 140734468329472, 140734468333567, +STORE, 140734468317184, 140734468329471, +STORE, 140143370027008, 140143370055679, +STORE, 140143370018816, 140143370027007, +STORE, 140143365599232, 140143367815167, +SNULL, 140143365599232, 140143365697535, +STORE, 140143365697536, 140143367815167, +STORE, 140143365599232, 140143365697535, +SNULL, 140143367790591, 140143367815167, +STORE, 140143365697536, 140143367790591, +STORE, 140143367790592, 140143367815167, +SNULL, 140143367790592, 140143367798783, +STORE, 140143367798784, 140143367815167, +STORE, 140143367790592, 140143367798783, +ERASE, 140143367790592, 140143367798783, +STORE, 140143367790592, 140143367798783, +ERASE, 140143367798784, 140143367815167, +STORE, 140143367798784, 140143367815167, +STORE, 140143361802240, 140143365599231, +SNULL, 140143361802240, 140143363461119, +STORE, 140143363461120, 140143365599231, +STORE, 140143361802240, 140143363461119, +SNULL, 140143365558271, 140143365599231, +STORE, 140143363461120, 140143365558271, +STORE, 140143365558272, 140143365599231, +SNULL, 140143365558272, 140143365582847, +STORE, 140143365582848, 140143365599231, +STORE, 140143365558272, 140143365582847, +ERASE, 140143365558272, 140143365582847, +STORE, 140143365558272, 140143365582847, +ERASE, 140143365582848, 140143365599231, +STORE, 140143365582848, 140143365599231, +STORE, 140143370010624, 140143370027007, +SNULL, 140143365574655, 140143365582847, +STORE, 140143365558272, 140143365574655, +STORE, 140143365574656, 140143365582847, +SNULL, 140143367794687, 140143367798783, +STORE, 140143367790592, 140143367794687, +STORE, 140143367794688, 140143367798783, +SNULL, 94880410152959, 94880410157055, +STORE, 94880410148864, 94880410152959, +STORE, 94880410152960, 94880410157055, +SNULL, 140143370059775, 140143370063871, +STORE, 140143370055680, 140143370059775, +STORE, 140143370059776, 140143370063871, +ERASE, 140143370027008, 140143370055679, +STORE, 94880442400768, 94880442535935, +STORE, 140143353409536, 140143361802239, +SNULL, 140143353413631, 140143361802239, +STORE, 140143353409536, 140143353413631, +STORE, 140143353413632, 140143361802239, +STORE, 140143345016832, 140143353409535, +STORE, 140143210799104, 140143345016831, +SNULL, 140143210799104, 140143239364607, +STORE, 140143239364608, 140143345016831, +STORE, 140143210799104, 140143239364607, +ERASE, 140143210799104, 140143239364607, +SNULL, 140143306473471, 140143345016831, +STORE, 140143239364608, 140143306473471, +STORE, 140143306473472, 140143345016831, +ERASE, 140143306473472, 140143345016831, +SNULL, 140143239499775, 140143306473471, +STORE, 140143239364608, 140143239499775, +STORE, 140143239499776, 140143306473471, +SNULL, 140143345020927, 140143353409535, +STORE, 140143345016832, 140143345020927, +STORE, 140143345020928, 140143353409535, +STORE, 140143336624128, 140143345016831, +SNULL, 140143336628223, 140143345016831, +STORE, 140143336624128, 140143336628223, +STORE, 140143336628224, 140143345016831, +STORE, 140143328231424, 140143336624127, +SNULL, 140143328235519, 140143336624127, +STORE, 140143328231424, 140143328235519, +STORE, 140143328235520, 140143336624127, +STORE, 140143319838720, 140143328231423, +SNULL, 140143319842815, 140143328231423, +STORE, 140143319838720, 140143319842815, +STORE, 140143319842816, 140143328231423, +STORE, 140143311446016, 140143319838719, +STORE, 140143105146880, 140143239364607, +STORE, 140143096754176, 140143105146879, +STORE, 140143029645312, 140143096754175, +ERASE, 140143029645312, 140143096754175, +STORE, 140142962536448, 140143096754175, +SNULL, 140142962536448, 140142970929151, +STORE, 140142970929152, 140143096754175, +STORE, 140142962536448, 140142970929151, +ERASE, 140142962536448, 140142970929151, +STORE, 140142962536448, 140142970929151, +STORE, 140142828318720, 140142962536447, +STORE, 140142819926016, 140142828318719, +SNULL, 140142828318720, 140142836711423, +STORE, 140142836711424, 140142962536447, +STORE, 140142828318720, 140142836711423, +ERASE, 140142828318720, 140142836711423, +SNULL, 140143172255743, 140143239364607, +STORE, 140143105146880, 140143172255743, +STORE, 140143172255744, 140143239364607, +ERASE, 140143172255744, 140143239364607, +SNULL, 140143105282047, 140143172255743, +STORE, 140143105146880, 140143105282047, +STORE, 140143105282048, 140143172255743, +SNULL, 140143038038015, 140143096754175, +STORE, 140142970929152, 140143038038015, +STORE, 140143038038016, 140143096754175, +ERASE, 140143038038016, 140143096754175, +SNULL, 140142971064319, 140143038038015, +STORE, 140142970929152, 140142971064319, +STORE, 140142971064320, 140143038038015, +SNULL, 140142903820287, 140142962536447, +STORE, 140142836711424, 140142903820287, +STORE, 140142903820288, 140142962536447, +ERASE, 140142903820288, 140142962536447, +SNULL, 140142836846591, 140142903820287, +STORE, 140142836711424, 140142836846591, +STORE, 140142836846592, 140142903820287, +STORE, 140142685708288, 140142819926015, +SNULL, 140143311450111, 140143319838719, +STORE, 140143311446016, 140143311450111, +STORE, 140143311450112, 140143319838719, +SNULL, 140142962540543, 140142970929151, +STORE, 140142962536448, 140142962540543, +STORE, 140142962540544, 140142970929151, +SNULL, 140142685708288, 140142702493695, +STORE, 140142702493696, 140142819926015, +STORE, 140142685708288, 140142702493695, +ERASE, 140142685708288, 140142702493695, +SNULL, 140142769602559, 140142819926015, +STORE, 140142702493696, 140142769602559, +STORE, 140142769602560, 140142819926015, +ERASE, 140142769602560, 140142819926015, +SNULL, 140142702628863, 140142769602559, +STORE, 140142702493696, 140142702628863, +STORE, 140142702628864, 140142769602559, +STORE, 140143230971904, 140143239364607, +SNULL, 140143230975999, 140143239364607, +STORE, 140143230971904, 140143230975999, +STORE, 140143230976000, 140143239364607, +SNULL, 140143096758271, 140143105146879, +STORE, 140143096754176, 140143096758271, +STORE, 140143096758272, 140143105146879, +STORE, 140143222579200, 140143230971903, +SNULL, 140143222583295, 140143230971903, +STORE, 140143222579200, 140143222583295, +STORE, 140143222583296, 140143230971903, +STORE, 140143214186496, 140143222579199, +SNULL, 140142819930111, 140142828318719, +STORE, 140142819926016, 140142819930111, +STORE, 140142819930112, 140142828318719, +STORE, 140143205793792, 140143222579199, +SNULL, 140143205793792, 140143214186495, +STORE, 140143214186496, 140143222579199, +STORE, 140143205793792, 140143214186495, +SNULL, 140143214190591, 140143222579199, +STORE, 140143214186496, 140143214190591, +STORE, 140143214190592, 140143222579199, +SNULL, 140143205797887, 140143214186495, +STORE, 140143205793792, 140143205797887, +STORE, 140143205797888, 140143214186495, +STORE, 140143197401088, 140143205793791, +SNULL, 140143197405183, 140143205793791, +STORE, 140143197401088, 140143197405183, +STORE, 140143197405184, 140143205793791, +STORE, 140143189008384, 140143197401087, +STORE, 140143180615680, 140143197401087, +STORE, 140143088361472, 140143096754175, +SNULL, 140143180619775, 140143197401087, +STORE, 140143180615680, 140143180619775, +STORE, 140143180619776, 140143197401087, +SNULL, 140143180619776, 140143189008383, +STORE, 140143189008384, 140143197401087, +STORE, 140143180619776, 140143189008383, +SNULL, 140143189012479, 140143197401087, +STORE, 140143189008384, 140143189012479, +STORE, 140143189012480, 140143197401087, +SNULL, 140143088365567, 140143096754175, +STORE, 140143088361472, 140143088365567, +STORE, 140143088365568, 140143096754175, +STORE, 140143079968768, 140143088361471, +SNULL, 140143079972863, 140143088361471, +STORE, 140143079968768, 140143079972863, +STORE, 140143079972864, 140143088361471, +STORE, 140143071576064, 140143079968767, +SNULL, 140143071580159, 140143079968767, +STORE, 140143071576064, 140143071580159, +STORE, 140143071580160, 140143079968767, +STORE, 140143063183360, 140143071576063, +STORE, 140143054790656, 140143071576063, +SNULL, 140143054794751, 140143071576063, +STORE, 140143054790656, 140143054794751, +STORE, 140143054794752, 140143071576063, +SNULL, 140143054794752, 140143063183359, +STORE, 140143063183360, 140143071576063, +STORE, 140143054794752, 140143063183359, +SNULL, 140143063187455, 140143071576063, +STORE, 140143063183360, 140143063187455, +STORE, 140143063187456, 140143071576063, +STORE, 140143046397952, 140143054790655, +STORE, 140142954143744, 140142962536447, +STORE, 140142945751040, 140142962536447, +STORE, 140142937358336, 140142962536447, +STORE, 140142928965632, 140142962536447, +STORE, 140142568275968, 140142702493695, +SNULL, 140142635384831, 140142702493695, +STORE, 140142568275968, 140142635384831, +STORE, 140142635384832, 140142702493695, +ERASE, 140142635384832, 140142702493695, +STORE, 140142920572928, 140142962536447, +STORE, 140142912180224, 140142962536447, +STORE, 140142568275968, 140142702493695, +SNULL, 140142568275968, 140142635384831, +STORE, 140142635384832, 140142702493695, +STORE, 140142568275968, 140142635384831, +SNULL, 140142635519999, 140142702493695, +STORE, 140142635384832, 140142635519999, +STORE, 140142635520000, 140142702493695, +STORE, 140142819930112, 140142836711423, +STORE, 140142811533312, 140142819926015, +STORE, 140142434058240, 140142635384831, +SNULL, 140142501167103, 140142635384831, +STORE, 140142434058240, 140142501167103, +STORE, 140142501167104, 140142635384831, +SNULL, 140142501167104, 140142568275967, +STORE, 140142568275968, 140142635384831, +STORE, 140142501167104, 140142568275967, +ERASE, 140142501167104, 140142568275967, +STORE, 140142299840512, 140142501167103, +STORE, 140142803140608, 140142819926015, +SNULL, 140142366949375, 140142501167103, +STORE, 140142299840512, 140142366949375, +STORE, 140142366949376, 140142501167103, +SNULL, 140142366949376, 140142434058239, +STORE, 140142434058240, 140142501167103, +STORE, 140142366949376, 140142434058239, +ERASE, 140142366949376, 140142434058239, +STORE, 140142794747904, 140142819926015, +STORE, 140142786355200, 140142819926015, +STORE, 140142299840512, 140142501167103, +STORE, 140142777962496, 140142819926015, +STORE, 140142559883264, 140142568275967, +STORE, 140142232731648, 140142501167103, +STORE, 140142551490560, 140142568275967, +SNULL, 140142777962496, 140142803140607, +STORE, 140142803140608, 140142819926015, +STORE, 140142777962496, 140142803140607, +SNULL, 140142803144703, 140142819926015, +STORE, 140142803140608, 140142803144703, +STORE, 140142803144704, 140142819926015, +STORE, 140142543097856, 140142568275967, +STORE, 140142098513920, 140142501167103, +SNULL, 140142165622783, 140142501167103, +STORE, 140142098513920, 140142165622783, +STORE, 140142165622784, 140142501167103, +SNULL, 140142165622784, 140142232731647, +STORE, 140142232731648, 140142501167103, +STORE, 140142165622784, 140142232731647, +ERASE, 140142165622784, 140142232731647, +SNULL, 140142568411135, 140142635384831, +STORE, 140142568275968, 140142568411135, +STORE, 140142568411136, 140142635384831, +STORE, 140141964296192, 140142165622783, +SNULL, 140142912180224, 140142928965631, +STORE, 140142928965632, 140142962536447, +STORE, 140142912180224, 140142928965631, +SNULL, 140142928969727, 140142962536447, +STORE, 140142928965632, 140142928969727, +STORE, 140142928969728, 140142962536447, +STORE, 140141830078464, 140142165622783, +SNULL, 140142912184319, 140142928965631, +STORE, 140142912180224, 140142912184319, +STORE, 140142912184320, 140142928965631, +SNULL, 140142232731648, 140142434058239, +STORE, 140142434058240, 140142501167103, +STORE, 140142232731648, 140142434058239, +SNULL, 140142434193407, 140142501167103, +STORE, 140142434058240, 140142434193407, +STORE, 140142434193408, 140142501167103, +SNULL, 140142232731648, 140142299840511, +STORE, 140142299840512, 140142434058239, +STORE, 140142232731648, 140142299840511, +SNULL, 140142299975679, 140142434058239, +STORE, 140142299840512, 140142299975679, +STORE, 140142299975680, 140142434058239, +SNULL, 140142928969728, 140142954143743, +STORE, 140142954143744, 140142962536447, +STORE, 140142928969728, 140142954143743, +SNULL, 140142954147839, 140142962536447, +STORE, 140142954143744, 140142954147839, +STORE, 140142954147840, 140142962536447, +STORE, 140141830078464, 140142299840511, +SNULL, 140142543097856, 140142559883263, +STORE, 140142559883264, 140142568275967, +STORE, 140142543097856, 140142559883263, +SNULL, 140142559887359, 140142568275967, +STORE, 140142559883264, 140142559887359, +STORE, 140142559887360, 140142568275967, +STORE, 140142534705152, 140142559883263, +SNULL, 140142928969728, 140142945751039, +STORE, 140142945751040, 140142954143743, +STORE, 140142928969728, 140142945751039, +SNULL, 140142945755135, 140142954143743, +STORE, 140142945751040, 140142945755135, +STORE, 140142945755136, 140142954143743, +SNULL, 140142299975680, 140142366949375, +STORE, 140142366949376, 140142434058239, +STORE, 140142299975680, 140142366949375, +SNULL, 140142367084543, 140142434058239, +STORE, 140142366949376, 140142367084543, +STORE, 140142367084544, 140142434058239, +SNULL, 140142928969728, 140142937358335, +STORE, 140142937358336, 140142945751039, +STORE, 140142928969728, 140142937358335, +SNULL, 140142937362431, 140142945751039, +STORE, 140142937358336, 140142937362431, +STORE, 140142937362432, 140142945751039, +SNULL, 140141830078464, 140142232731647, +STORE, 140142232731648, 140142299840511, +STORE, 140141830078464, 140142232731647, +SNULL, 140142232866815, 140142299840511, +STORE, 140142232731648, 140142232866815, +STORE, 140142232866816, 140142299840511, +SNULL, 140142534705152, 140142543097855, +STORE, 140142543097856, 140142559883263, +STORE, 140142534705152, 140142543097855, +SNULL, 140142543101951, 140142559883263, +STORE, 140142543097856, 140142543101951, +STORE, 140142543101952, 140142559883263, +STORE, 140142526312448, 140142543097855, +STORE, 140142517919744, 140142543097855, +SNULL, 140141830078464, 140142098513919, +STORE, 140142098513920, 140142232731647, +STORE, 140141830078464, 140142098513919, +SNULL, 140142098649087, 140142232731647, +STORE, 140142098513920, 140142098649087, +STORE, 140142098649088, 140142232731647, +SNULL, 140142031405055, 140142098513919, +STORE, 140141830078464, 140142031405055, +STORE, 140142031405056, 140142098513919, +ERASE, 140142031405056, 140142098513919, +SNULL, 140141830078464, 140141964296191, +STORE, 140141964296192, 140142031405055, +STORE, 140141830078464, 140141964296191, +SNULL, 140141964431359, 140142031405055, +STORE, 140141964296192, 140141964431359, +STORE, 140141964431360, 140142031405055, +STORE, 140142509527040, 140142543097855, +SNULL, 140141897187327, 140141964296191, +STORE, 140141830078464, 140141897187327, +STORE, 140141897187328, 140141964296191, +ERASE, 140141897187328, 140141964296191, +SNULL, 140141830213631, 140141897187327, +STORE, 140141830078464, 140141830213631, +STORE, 140141830213632, 140141897187327, +SNULL, 140142803144704, 140142811533311, +STORE, 140142811533312, 140142819926015, +STORE, 140142803144704, 140142811533311, +SNULL, 140142811537407, 140142819926015, +STORE, 140142811533312, 140142811537407, +STORE, 140142811537408, 140142819926015, +SNULL, 140142098649088, 140142165622783, +STORE, 140142165622784, 140142232731647, +STORE, 140142098649088, 140142165622783, +SNULL, 140142165757951, 140142232731647, +STORE, 140142165622784, 140142165757951, +STORE, 140142165757952, 140142232731647, +STORE, 140142090121216, 140142098513919, +SNULL, 140142777962496, 140142786355199, +STORE, 140142786355200, 140142803140607, +STORE, 140142777962496, 140142786355199, +SNULL, 140142786359295, 140142803140607, +STORE, 140142786355200, 140142786359295, +STORE, 140142786359296, 140142803140607, +SNULL, 140142509527040, 140142534705151, +STORE, 140142534705152, 140142543097855, +STORE, 140142509527040, 140142534705151, +SNULL, 140142534709247, 140142543097855, +STORE, 140142534705152, 140142534709247, +STORE, 140142534709248, 140142543097855, +STORE, 140142081728512, 140142098513919, +SNULL, 140142786359296, 140142794747903, +STORE, 140142794747904, 140142803140607, +STORE, 140142786359296, 140142794747903, +SNULL, 140142794751999, 140142803140607, +STORE, 140142794747904, 140142794751999, +STORE, 140142794752000, 140142803140607, +STORE, 140142073335808, 140142098513919, +SNULL, 140142073339903, 140142098513919, +STORE, 140142073335808, 140142073339903, +STORE, 140142073339904, 140142098513919, +SNULL, 140142543101952, 140142551490559, +STORE, 140142551490560, 140142559883263, +STORE, 140142543101952, 140142551490559, +SNULL, 140142551494655, 140142559883263, +STORE, 140142551490560, 140142551494655, +STORE, 140142551494656, 140142559883263, +SNULL, 140142509527040, 140142517919743, +STORE, 140142517919744, 140142534705151, +STORE, 140142509527040, 140142517919743, +SNULL, 140142517923839, 140142534705151, +STORE, 140142517919744, 140142517923839, +STORE, 140142517923840, 140142534705151, +STORE, 140142064943104, 140142073335807, +SNULL, 140142073339904, 140142090121215, +STORE, 140142090121216, 140142098513919, +STORE, 140142073339904, 140142090121215, +SNULL, 140142090125311, 140142098513919, +STORE, 140142090121216, 140142090125311, +STORE, 140142090125312, 140142098513919, +STORE, 140142056550400, 140142073335807, +SNULL, 140142056554495, 140142073335807, +STORE, 140142056550400, 140142056554495, +STORE, 140142056554496, 140142073335807, +STORE, 140142048157696, 140142056550399, +SNULL, 140142509531135, 140142517919743, +STORE, 140142509527040, 140142509531135, +STORE, 140142509531136, 140142517919743, +SNULL, 140142777966591, 140142786355199, +STORE, 140142777962496, 140142777966591, +STORE, 140142777966592, 140142786355199, +SNULL, 140143046402047, 140143054790655, +STORE, 140143046397952, 140143046402047, +STORE, 140143046402048, 140143054790655, +SNULL, 140142912184320, 140142920572927, +STORE, 140142920572928, 140142928965631, +STORE, 140142912184320, 140142920572927, +SNULL, 140142920577023, 140142928965631, +STORE, 140142920572928, 140142920577023, +STORE, 140142920577024, 140142928965631, +STORE, 140142039764992, 140142056550399, +STORE, 140141955903488, 140141964296191, +SNULL, 140142819930112, 140142828318719, +STORE, 140142828318720, 140142836711423, +STORE, 140142819930112, 140142828318719, +SNULL, 140142828322815, 140142836711423, +STORE, 140142828318720, 140142828322815, +STORE, 140142828322816, 140142836711423, +SNULL, 140142517923840, 140142526312447, +STORE, 140142526312448, 140142534705151, +STORE, 140142517923840, 140142526312447, +SNULL, 140142526316543, 140142534705151, +STORE, 140142526312448, 140142526316543, +STORE, 140142526316544, 140142534705151, +STORE, 140141947510784, 140141964296191, +SNULL, 140142056554496, 140142064943103, +STORE, 140142064943104, 140142073335807, +STORE, 140142056554496, 140142064943103, +SNULL, 140142064947199, 140142073335807, +STORE, 140142064943104, 140142064947199, +STORE, 140142064947200, 140142073335807, +SNULL, 140142073339904, 140142081728511, +STORE, 140142081728512, 140142090121215, +STORE, 140142073339904, 140142081728511, +SNULL, 140142081732607, 140142090121215, +STORE, 140142081728512, 140142081732607, +STORE, 140142081732608, 140142090121215, +STORE, 140141939118080, 140141964296191, +STORE, 140141930725376, 140141964296191, +STORE, 140141922332672, 140141964296191, +STORE, 140141913939968, 140141964296191, +SNULL, 140141913939968, 140141922332671, +STORE, 140141922332672, 140141964296191, +STORE, 140141913939968, 140141922332671, +SNULL, 140141922336767, 140141964296191, +STORE, 140141922332672, 140141922336767, +STORE, 140141922336768, 140141964296191, +STORE, 140141905547264, 140141922332671, +SNULL, 140141905551359, 140141922332671, +STORE, 140141905547264, 140141905551359, +STORE, 140141905551360, 140141922332671, +STORE, 140141821685760, 140141830078463, +STORE, 140141813293056, 140141830078463, +STORE, 140141804900352, 140141830078463, +STORE, 140141796507648, 140141830078463, +SNULL, 140141796511743, 140141830078463, +STORE, 140141796507648, 140141796511743, +STORE, 140141796511744, 140141830078463, +SNULL, 140141922336768, 140141955903487, +STORE, 140141955903488, 140141964296191, +STORE, 140141922336768, 140141955903487, +SNULL, 140141955907583, 140141964296191, +STORE, 140141955903488, 140141955907583, +STORE, 140141955907584, 140141964296191, +STORE, 140141788114944, 140141796507647, +STORE, 140141779722240, 140141796507647, +SNULL, 140141779722240, 140141788114943, +STORE, 140141788114944, 140141796507647, +STORE, 140141779722240, 140141788114943, +SNULL, 140141788119039, 140141796507647, +STORE, 140141788114944, 140141788119039, +STORE, 140141788119040, 140141796507647, +SNULL, 140141922336768, 140141947510783, +STORE, 140141947510784, 140141955903487, +STORE, 140141922336768, 140141947510783, +SNULL, 140141947514879, 140141955903487, +STORE, 140141947510784, 140141947514879, +STORE, 140141947514880, 140141955903487, +SNULL, 140142039764992, 140142048157695, +STORE, 140142048157696, 140142056550399, +STORE, 140142039764992, 140142048157695, +SNULL, 140142048161791, 140142056550399, +STORE, 140142048157696, 140142048161791, +STORE, 140142048161792, 140142056550399, +SNULL, 140142039769087, 140142048157695, +STORE, 140142039764992, 140142039769087, +STORE, 140142039769088, 140142048157695, +SNULL, 140141796511744, 140141804900351, +STORE, 140141804900352, 140141830078463, +STORE, 140141796511744, 140141804900351, +SNULL, 140141804904447, 140141830078463, +STORE, 140141804900352, 140141804904447, +STORE, 140141804904448, 140141830078463, +STORE, 140141771329536, 140141788114943, +STORE, 140141762936832, 140141788114943, +STORE, 140141754544128, 140141788114943, +SNULL, 140141804904448, 140141821685759, +STORE, 140141821685760, 140141830078463, +STORE, 140141804904448, 140141821685759, +SNULL, 140141821689855, 140141830078463, +STORE, 140141821685760, 140141821689855, +STORE, 140141821689856, 140141830078463, +SNULL, 140141922336768, 140141939118079, +STORE, 140141939118080, 140141947510783, +STORE, 140141922336768, 140141939118079, +SNULL, 140141939122175, 140141947510783, +STORE, 140141939118080, 140141939122175, +STORE, 140141939122176, 140141947510783, +SNULL, 140141905551360, 140141913939967, +STORE, 140141913939968, 140141922332671, +STORE, 140141905551360, 140141913939967, +SNULL, 140141913944063, 140141922332671, +STORE, 140141913939968, 140141913944063, +STORE, 140141913944064, 140141922332671, +STORE, 140141746151424, 140141788114943, +STORE, 140141737758720, 140141788114943, +SNULL, 140141804904448, 140141813293055, +STORE, 140141813293056, 140141821685759, +STORE, 140141804904448, 140141813293055, +SNULL, 140141813297151, 140141821685759, +STORE, 140141813293056, 140141813297151, +STORE, 140141813297152, 140141821685759, +STORE, 140141729366016, 140141788114943, +STORE, 140141720973312, 140141788114943, +STORE, 140141712580608, 140141788114943, +SNULL, 140141712584703, 140141788114943, +STORE, 140141712580608, 140141712584703, +STORE, 140141712584704, 140141788114943, +SNULL, 140141922336768, 140141930725375, +STORE, 140141930725376, 140141939118079, +STORE, 140141922336768, 140141930725375, +SNULL, 140141930729471, 140141939118079, +STORE, 140141930725376, 140141930729471, +STORE, 140141930729472, 140141939118079, +STORE, 140141704187904, 140141712580607, +SNULL, 140141704191999, 140141712580607, +STORE, 140141704187904, 140141704191999, +STORE, 140141704192000, 140141712580607, +STORE, 140141695795200, 140141704187903, +STORE, 140141687402496, 140141704187903, +SNULL, 140141712584704, 140141771329535, +STORE, 140141771329536, 140141788114943, +STORE, 140141712584704, 140141771329535, +SNULL, 140141771333631, 140141788114943, +STORE, 140141771329536, 140141771333631, +STORE, 140141771333632, 140141788114943, +SNULL, 140141771333632, 140141779722239, +STORE, 140141779722240, 140141788114943, +STORE, 140141771333632, 140141779722239, +SNULL, 140141779726335, 140141788114943, +STORE, 140141779722240, 140141779726335, +STORE, 140141779726336, 140141788114943, +STORE, 140141679009792, 140141704187903, +SNULL, 140141679013887, 140141704187903, +STORE, 140141679009792, 140141679013887, +STORE, 140141679013888, 140141704187903, +STORE, 140141670617088, 140141679009791, +SNULL, 140141670621183, 140141679009791, +STORE, 140141670617088, 140141670621183, +STORE, 140141670621184, 140141679009791, +STORE, 140141662224384, 140141670617087, +SNULL, 140141712584704, 140141737758719, +STORE, 140141737758720, 140141771329535, +STORE, 140141712584704, 140141737758719, +SNULL, 140141737762815, 140141771329535, +STORE, 140141737758720, 140141737762815, +STORE, 140141737762816, 140141771329535, +SNULL, 140141712584704, 140141729366015, +STORE, 140141729366016, 140141737758719, +STORE, 140141712584704, 140141729366015, +SNULL, 140141729370111, 140141737758719, +STORE, 140141729366016, 140141729370111, +STORE, 140141729370112, 140141737758719, +SNULL, 140141737762816, 140141746151423, +STORE, 140141746151424, 140141771329535, +STORE, 140141737762816, 140141746151423, +SNULL, 140141746155519, 140141771329535, +STORE, 140141746151424, 140141746155519, +STORE, 140141746155520, 140141771329535, +STORE, 140141653831680, 140141670617087, +SNULL, 140141746155520, 140141762936831, +STORE, 140141762936832, 140141771329535, +STORE, 140141746155520, 140141762936831, +SNULL, 140141762940927, 140141771329535, +STORE, 140141762936832, 140141762940927, +STORE, 140141762940928, 140141771329535, +STORE, 140141645438976, 140141670617087, +SNULL, 140141645443071, 140141670617087, +STORE, 140141645438976, 140141645443071, +STORE, 140141645443072, 140141670617087, +SNULL, 140141712584704, 140141720973311, +STORE, 140141720973312, 140141729366015, +STORE, 140141712584704, 140141720973311, +SNULL, 140141720977407, 140141729366015, +STORE, 140141720973312, 140141720977407, +STORE, 140141720977408, 140141729366015, +STORE, 140141637046272, 140141645438975, +SNULL, 140141637050367, 140141645438975, +STORE, 140141637046272, 140141637050367, +STORE, 140141637050368, 140141645438975, +STORE, 140141628653568, 140141637046271, +SNULL, 140141628657663, 140141637046271, +STORE, 140141628653568, 140141628657663, +STORE, 140141628657664, 140141637046271, +STORE, 140141620260864, 140141628653567, +SNULL, 140141679013888, 140141687402495, +STORE, 140141687402496, 140141704187903, +STORE, 140141679013888, 140141687402495, +SNULL, 140141687406591, 140141704187903, +STORE, 140141687402496, 140141687406591, +STORE, 140141687406592, 140141704187903, +SNULL, 140141746155520, 140141754544127, +STORE, 140141754544128, 140141762936831, +STORE, 140141746155520, 140141754544127, +SNULL, 140141754548223, 140141762936831, +STORE, 140141754544128, 140141754548223, +STORE, 140141754548224, 140141762936831, +SNULL, 140141687406592, 140141695795199, +STORE, 140141695795200, 140141704187903, +STORE, 140141687406592, 140141695795199, +SNULL, 140141695799295, 140141704187903, +STORE, 140141695795200, 140141695799295, +STORE, 140141695799296, 140141704187903, +STORE, 140141611868160, 140141628653567, +SNULL, 140141611872255, 140141628653567, +STORE, 140141611868160, 140141611872255, +STORE, 140141611872256, 140141628653567, +SNULL, 140141645443072, 140141662224383, +STORE, 140141662224384, 140141670617087, +STORE, 140141645443072, 140141662224383, +SNULL, 140141662228479, 140141670617087, +STORE, 140141662224384, 140141662228479, +STORE, 140141662228480, 140141670617087, +STORE, 140141603475456, 140141611868159, +SNULL, 140141603479551, 140141611868159, +STORE, 140141603475456, 140141603479551, +STORE, 140141603479552, 140141611868159, +STORE, 140141595082752, 140141603475455, +SNULL, 140141645443072, 140141653831679, +STORE, 140141653831680, 140141662224383, +STORE, 140141645443072, 140141653831679, +SNULL, 140141653835775, 140141662224383, +STORE, 140141653831680, 140141653835775, +STORE, 140141653835776, 140141662224383, +STORE, 140141586690048, 140141603475455, +SNULL, 140141611872256, 140141620260863, +STORE, 140141620260864, 140141628653567, +STORE, 140141611872256, 140141620260863, +SNULL, 140141620264959, 140141628653567, +STORE, 140141620260864, 140141620264959, +STORE, 140141620264960, 140141628653567, +SNULL, 140141586690048, 140141595082751, +STORE, 140141595082752, 140141603475455, +STORE, 140141586690048, 140141595082751, +SNULL, 140141595086847, 140141603475455, +STORE, 140141595082752, 140141595086847, +STORE, 140141595086848, 140141603475455, +STORE, 140141578297344, 140141595082751, +SNULL, 140141578301439, 140141595082751, +STORE, 140141578297344, 140141578301439, +STORE, 140141578301440, 140141595082751, +SNULL, 140141578301440, 140141586690047, +STORE, 140141586690048, 140141595082751, +STORE, 140141578301440, 140141586690047, +SNULL, 140141586694143, 140141595082751, +STORE, 140141586690048, 140141586694143, +STORE, 140141586694144, 140141595082751, +STORE, 140143370027008, 140143370055679, +STORE, 140143309254656, 140143311446015, +SNULL, 140143309254656, 140143309344767, +STORE, 140143309344768, 140143311446015, +STORE, 140143309254656, 140143309344767, +SNULL, 140143311437823, 140143311446015, +STORE, 140143309344768, 140143311437823, +STORE, 140143311437824, 140143311446015, +ERASE, 140143311437824, 140143311446015, +STORE, 140143311437824, 140143311446015, +SNULL, 140143311441919, 140143311446015, +STORE, 140143311437824, 140143311441919, +STORE, 140143311441920, 140143311446015, +ERASE, 140143370027008, 140143370055679, +ERASE, 140142912180224, 140142912184319, +ERASE, 140142912184320, 140142920572927, +ERASE, 140142945751040, 140142945755135, +ERASE, 140142945755136, 140142954143743, +ERASE, 140142090121216, 140142090125311, +ERASE, 140142090125312, 140142098513919, +ERASE, 140142794747904, 140142794751999, +ERASE, 140142794752000, 140142803140607, +ERASE, 140141913939968, 140141913944063, +ERASE, 140141913944064, 140141922332671, +ERASE, 140141746151424, 140141746155519, +ERASE, 140141746155520, 140141754544127, +ERASE, 140142954143744, 140142954147839, +ERASE, 140142954147840, 140142962536447, +ERASE, 140142081728512, 140142081732607, +ERASE, 140142081732608, 140142090121215, +ERASE, 140141905547264, 140141905551359, +ERASE, 140141905551360, 140141913939967, +ERASE, 140141729366016, 140141729370111, +ERASE, 140141729370112, 140141737758719, +ERASE, 140142920572928, 140142920577023, +ERASE, 140142920577024, 140142928965631, +ERASE, 140142039764992, 140142039769087, +ERASE, 140142039769088, 140142048157695, +ERASE, 140141679009792, 140141679013887, +ERASE, 140141679013888, 140141687402495, +ERASE, 140142551490560, 140142551494655, +ERASE, 140142551494656, 140142559883263, +ERASE, 140141947510784, 140141947514879, +ERASE, 140141947514880, 140141955903487, +ERASE, 140141771329536, 140141771333631, +ERASE, 140141771333632, 140141779722239, +ERASE, 140142928965632, 140142928969727, +ERASE, 140142928969728, 140142937358335, +ERASE, 140142073335808, 140142073339903, +ERASE, 140142073339904, 140142081728511, +ERASE, 140142543097856, 140142543101951, +ERASE, 140142543101952, 140142551490559, +ERASE, 140141955903488, 140141955907583, +ERASE, 140141955907584, 140141964296191, +ERASE, 140141704187904, 140141704191999, +ERASE, 140141704192000, 140141712580607, +ERASE, 140142786355200, 140142786359295, +ERASE, 140142786359296, 140142794747903, +ERASE, 140142056550400, 140142056554495, +ERASE, 140142056554496, 140142064943103, +ERASE, 140142828318720, 140142828322815, +ERASE, 140142828322816, 140142836711423, +ERASE, 140141788114944, 140141788119039, +ERASE, 140141788119040, 140141796507647, +ERASE, 140141695795200, 140141695799295, +ERASE, 140141695799296, 140141704187903, +ERASE, 140141578297344, 140141578301439, +ERASE, 140141578301440, 140141586690047, +ERASE, 140141611868160, 140141611872255, +ERASE, 140141611872256, 140141620260863, +ERASE, 140142811533312, 140142811537407, +ERASE, 140142811537408, 140142819926015, +ERASE, 140142064943104, 140142064947199, +ERASE, 140142064947200, 140142073335807, +ERASE, 140141628653568, 140141628657663, +ERASE, 140141628657664, 140141637046271, +ERASE, 140143046397952, 140143046402047, +ERASE, 140143046402048, 140143054790655, +ERASE, 140141796507648, 140141796511743, +ERASE, 140141796511744, 140141804900351, +ERASE, 140142803140608, 140142803144703, +ERASE, 140142803144704, 140142811533311, +ERASE, 140142509527040, 140142509531135, +ERASE, 140142509531136, 140142517919743, +ERASE, 140141821685760, 140141821689855, +ERASE, 140141821689856, 140141830078463, +ERASE, 140142777962496, 140142777966591, +ERASE, 140142777966592, 140142786355199, +ERASE, 140141804900352, 140141804904447, +ERASE, 140141804904448, 140141813293055, +ERASE, 140141930725376, 140141930729471, +ERASE, 140141930729472, 140141939118079, +ERASE, 140142937358336, 140142937362431, +ERASE, 140142937362432, 140142945751039, +ERASE, 140142559883264, 140142559887359, +ERASE, 140142559887360, 140142568275967, +ERASE, 140142534705152, 140142534709247, +ERASE, 140142534709248, 140142543097855, +ERASE, 140142048157696, 140142048161791, +ERASE, 140142048161792, 140142056550399, +ERASE, 140141754544128, 140141754548223, +ERASE, 140141754548224, 140141762936831, +ERASE, 140141939118080, 140141939122175, +ERASE, 140141939122176, 140141947510783, +ERASE, 140141653831680, 140141653835775, +ERASE, 140141653835776, 140141662224383, +ERASE, 140141712580608, 140141712584703, +ERASE, 140141712584704, 140141720973311, +ERASE, 140141645438976, 140141645443071, +ERASE, 140141645443072, 140141653831679, +ERASE, 140141687402496, 140141687406591, +ERASE, 140141687406592, 140141695795199, +ERASE, 140141662224384, 140141662228479, +ERASE, 140141662228480, 140141670617087, +ERASE, 140141922332672, 140141922336767, +ERASE, 140141922336768, 140141930725375, +ERASE, 140141737758720, 140141737762815, +ERASE, 140141737762816, 140141746151423, +ERASE, 140141637046272, 140141637050367, +ERASE, 140141637050368, 140141645438975, +ERASE, 140142517919744, 140142517923839, +ERASE, 140142517923840, 140142526312447, +ERASE, 140143096754176, 140143096758271, +ERASE, 140143096758272, 140143105146879, +ERASE, 140141595082752, 140141595086847, +ERASE, 140141595086848, 140141603475455, +ERASE, 140141762936832, 140141762940927, +ERASE, 140141762940928, 140141771329535, +ERASE, 140143311446016, 140143311450111, +ERASE, 140143311450112, 140143319838719, +ERASE, 140142526312448, 140142526316543, +ERASE, 140142526316544, 140142534705151, +ERASE, 140142819926016, 140142819930111, +ERASE, 140142819930112, 140142828318719, +ERASE, 140143180615680, 140143180619775, +ERASE, 140143180619776, 140143189008383, +ERASE, 140142962536448, 140142962540543, +ERASE, 140142962540544, 140142970929151, +ERASE, 140143214186496, 140143214190591, +ERASE, 140143214190592, 140143222579199, +ERASE, 140143088361472, 140143088365567, +ERASE, 140143088365568, 140143096754175, +ERASE, 140141586690048, 140141586694143, +ERASE, 140141586694144, 140141595082751, +ERASE, 140143230971904, 140143230975999, +ERASE, 140143230976000, 140143239364607, +ERASE, 140141779722240, 140141779726335, +ERASE, 140141779726336, 140141788114943, +ERASE, 140141670617088, 140141670621183, +ERASE, 140141670621184, 140141679009791, +ERASE, 140141813293056, 140141813297151, +ERASE, 140141813297152, 140141821685759, +ERASE, 140143222579200, 140143222583295, +ERASE, 140143222583296, 140143230971903, +ERASE, 140143189008384, 140143189012479, +ERASE, 140143189012480, 140143197401087, +ERASE, 140143071576064, 140143071580159, +ERASE, 140143071580160, 140143079968767, +ERASE, 140141620260864, 140141620264959, +ERASE, 140141620264960, 140141628653567, +ERASE, 140141603475456, 140141603479551, +ERASE, 140141603479552, 140141611868159, +ERASE, 140141720973312, 140141720977407, +ERASE, 140141720977408, 140141729366015, +ERASE, 140143079968768, 140143079972863, +ERASE, 140143079972864, 140143088361471, +ERASE, 140143205793792, 140143205797887, +ERASE, 140143205797888, 140143214186495, + }; + unsigned long set30[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140733436743680, 140737488351231, +SNULL, 140733436747775, 140737488351231, +STORE, 140733436743680, 140733436747775, +STORE, 140733436612608, 140733436747775, +STORE, 94630728904704, 94630731157503, +SNULL, 94630729035775, 94630731157503, +STORE, 94630728904704, 94630729035775, +STORE, 94630729035776, 94630731157503, +ERASE, 94630729035776, 94630731157503, +STORE, 94630731128832, 94630731137023, +STORE, 94630731137024, 94630731157503, +STORE, 140165750841344, 140165753094143, +SNULL, 140165750984703, 140165753094143, +STORE, 140165750841344, 140165750984703, +STORE, 140165750984704, 140165753094143, +ERASE, 140165750984704, 140165753094143, +STORE, 140165753081856, 140165753090047, +STORE, 140165753090048, 140165753094143, +STORE, 140733436887040, 140733436891135, +STORE, 140733436874752, 140733436887039, +STORE, 140165753053184, 140165753081855, +STORE, 140165753044992, 140165753053183, +STORE, 140165748625408, 140165750841343, +SNULL, 140165748625408, 140165748723711, +STORE, 140165748723712, 140165750841343, +STORE, 140165748625408, 140165748723711, +SNULL, 140165750816767, 140165750841343, +STORE, 140165748723712, 140165750816767, +STORE, 140165750816768, 140165750841343, +SNULL, 140165750816768, 140165750824959, +STORE, 140165750824960, 140165750841343, +STORE, 140165750816768, 140165750824959, +ERASE, 140165750816768, 140165750824959, +STORE, 140165750816768, 140165750824959, +ERASE, 140165750824960, 140165750841343, +STORE, 140165750824960, 140165750841343, +STORE, 140165744828416, 140165748625407, +SNULL, 140165744828416, 140165746487295, +STORE, 140165746487296, 140165748625407, +STORE, 140165744828416, 140165746487295, +SNULL, 140165748584447, 140165748625407, +STORE, 140165746487296, 140165748584447, +STORE, 140165748584448, 140165748625407, +SNULL, 140165748584448, 140165748609023, +STORE, 140165748609024, 140165748625407, +STORE, 140165748584448, 140165748609023, +ERASE, 140165748584448, 140165748609023, +STORE, 140165748584448, 140165748609023, +ERASE, 140165748609024, 140165748625407, +STORE, 140165748609024, 140165748625407, +STORE, 140165753036800, 140165753053183, +SNULL, 140165748600831, 140165748609023, +STORE, 140165748584448, 140165748600831, +STORE, 140165748600832, 140165748609023, +SNULL, 140165750820863, 140165750824959, +STORE, 140165750816768, 140165750820863, +STORE, 140165750820864, 140165750824959, +SNULL, 94630731132927, 94630731137023, +STORE, 94630731128832, 94630731132927, +STORE, 94630731132928, 94630731137023, +SNULL, 140165753085951, 140165753090047, +STORE, 140165753081856, 140165753085951, +STORE, 140165753085952, 140165753090047, +ERASE, 140165753053184, 140165753081855, +STORE, 94630743547904, 94630743683071, +STORE, 140165736435712, 140165744828415, +SNULL, 140165736439807, 140165744828415, +STORE, 140165736435712, 140165736439807, +STORE, 140165736439808, 140165744828415, +STORE, 140165728043008, 140165736435711, +STORE, 140165593825280, 140165728043007, +SNULL, 140165593825280, 140165653725183, +STORE, 140165653725184, 140165728043007, +STORE, 140165593825280, 140165653725183, +ERASE, 140165593825280, 140165653725183, +SNULL, 140165720834047, 140165728043007, +STORE, 140165653725184, 140165720834047, +STORE, 140165720834048, 140165728043007, +ERASE, 140165720834048, 140165728043007, +SNULL, 140165653860351, 140165720834047, +STORE, 140165653725184, 140165653860351, +STORE, 140165653860352, 140165720834047, +SNULL, 140165728047103, 140165736435711, +STORE, 140165728043008, 140165728047103, +STORE, 140165728047104, 140165736435711, +STORE, 140165645332480, 140165653725183, +SNULL, 140165645336575, 140165653725183, +STORE, 140165645332480, 140165645336575, +STORE, 140165645336576, 140165653725183, +STORE, 140165636939776, 140165645332479, +SNULL, 140165636943871, 140165645332479, +STORE, 140165636939776, 140165636943871, +STORE, 140165636943872, 140165645332479, +STORE, 140165628547072, 140165636939775, +SNULL, 140165628551167, 140165636939775, +STORE, 140165628547072, 140165628551167, +STORE, 140165628551168, 140165636939775, +STORE, 140165620154368, 140165628547071, +STORE, 140165611761664, 140165628547071, +STORE, 140165603368960, 140165628547071, +STORE, 140165469151232, 140165603368959, +SNULL, 140165469151232, 140165519507455, +STORE, 140165519507456, 140165603368959, +STORE, 140165469151232, 140165519507455, +ERASE, 140165469151232, 140165519507455, +SNULL, 140165586616319, 140165603368959, +STORE, 140165519507456, 140165586616319, +STORE, 140165586616320, 140165603368959, +ERASE, 140165586616320, 140165603368959, +STORE, 140165594976256, 140165628547071, +STORE, 140165385289728, 140165586616319, +SNULL, 140165452398591, 140165586616319, +STORE, 140165385289728, 140165452398591, +STORE, 140165452398592, 140165586616319, +SNULL, 140165452398592, 140165519507455, +STORE, 140165519507456, 140165586616319, +STORE, 140165452398592, 140165519507455, +ERASE, 140165452398592, 140165519507455, +STORE, 140165251072000, 140165452398591, +SNULL, 140165318180863, 140165452398591, +STORE, 140165251072000, 140165318180863, +STORE, 140165318180864, 140165452398591, +SNULL, 140165318180864, 140165385289727, +STORE, 140165385289728, 140165452398591, +STORE, 140165318180864, 140165385289727, +ERASE, 140165318180864, 140165385289727, +SNULL, 140165519642623, 140165586616319, +STORE, 140165519507456, 140165519642623, +STORE, 140165519642624, 140165586616319, +SNULL, 140165594976256, 140165611761663, +STORE, 140165611761664, 140165628547071, +STORE, 140165594976256, 140165611761663, +SNULL, 140165611765759, 140165628547071, +STORE, 140165611761664, 140165611765759, +STORE, 140165611765760, 140165628547071, +STORE, 140165385289728, 140165519507455, +SNULL, 140165385424895, 140165519507455, +STORE, 140165385289728, 140165385424895, +STORE, 140165385424896, 140165519507455, +SNULL, 140165594976256, 140165603368959, +STORE, 140165603368960, 140165611761663, +STORE, 140165594976256, 140165603368959, +SNULL, 140165603373055, 140165611761663, +STORE, 140165603368960, 140165603373055, +STORE, 140165603373056, 140165611761663, +SNULL, 140165251207167, 140165318180863, +STORE, 140165251072000, 140165251207167, +STORE, 140165251207168, 140165318180863, +STORE, 140165376897024, 140165385289727, +SNULL, 140165376901119, 140165385289727, +STORE, 140165376897024, 140165376901119, +STORE, 140165376901120, 140165385289727, +SNULL, 140165385424896, 140165452398591, +STORE, 140165452398592, 140165519507455, +STORE, 140165385424896, 140165452398591, +SNULL, 140165452533759, 140165519507455, +STORE, 140165452398592, 140165452533759, +STORE, 140165452533760, 140165519507455, +STORE, 140165368504320, 140165376897023, +SNULL, 140165594980351, 140165603368959, +STORE, 140165594976256, 140165594980351, +STORE, 140165594980352, 140165603368959, +SNULL, 140165368508415, 140165376897023, +STORE, 140165368504320, 140165368508415, +STORE, 140165368508416, 140165376897023, +SNULL, 140165611765760, 140165620154367, +STORE, 140165620154368, 140165628547071, +STORE, 140165611765760, 140165620154367, +SNULL, 140165620158463, 140165628547071, +STORE, 140165620154368, 140165620158463, +STORE, 140165620158464, 140165628547071, +STORE, 140165360111616, 140165368504319, +STORE, 140165351718912, 140165368504319, +STORE, 140165343326208, 140165368504319, +SNULL, 140165343326208, 140165351718911, +STORE, 140165351718912, 140165368504319, +STORE, 140165343326208, 140165351718911, +SNULL, 140165351723007, 140165368504319, +STORE, 140165351718912, 140165351723007, +STORE, 140165351723008, 140165368504319, +SNULL, 140165343330303, 140165351718911, +STORE, 140165343326208, 140165343330303, +STORE, 140165343330304, 140165351718911, +SNULL, 140165351723008, 140165360111615, +STORE, 140165360111616, 140165368504319, +STORE, 140165351723008, 140165360111615, +SNULL, 140165360115711, 140165368504319, +STORE, 140165360111616, 140165360115711, +STORE, 140165360115712, 140165368504319, +STORE, 140165334933504, 140165343326207, +SNULL, 140165334937599, 140165343326207, +STORE, 140165334933504, 140165334937599, +STORE, 140165334937600, 140165343326207, +STORE, 140165326540800, 140165334933503, +STORE, 140165242679296, 140165251071999, +SNULL, 140165242683391, 140165251071999, +STORE, 140165242679296, 140165242683391, +STORE, 140165242683392, 140165251071999, +STORE, 140165234286592, 140165242679295, +STORE, 140165225893888, 140165242679295, +SNULL, 140165225897983, 140165242679295, +STORE, 140165225893888, 140165225897983, +STORE, 140165225897984, 140165242679295, +SNULL, 140165225897984, 140165234286591, +STORE, 140165234286592, 140165242679295, +STORE, 140165225897984, 140165234286591, +SNULL, 140165234290687, 140165242679295, +STORE, 140165234286592, 140165234290687, +STORE, 140165234290688, 140165242679295, +SNULL, 140165326544895, 140165334933503, +STORE, 140165326540800, 140165326544895, +STORE, 140165326544896, 140165334933503, +STORE, 140165217501184, 140165225893887, +STORE, 140165209108480, 140165225893887, +SNULL, 140165209108480, 140165217501183, +STORE, 140165217501184, 140165225893887, +STORE, 140165209108480, 140165217501183, +SNULL, 140165217505279, 140165225893887, +STORE, 140165217501184, 140165217505279, +STORE, 140165217505280, 140165225893887, +SNULL, 140165209112575, 140165217501183, +STORE, 140165209108480, 140165209112575, +STORE, 140165209112576, 140165217501183, +STORE, 140165200715776, 140165209108479, +STORE, 140165066498048, 140165200715775, +SNULL, 140165066498048, 140165116854271, +STORE, 140165116854272, 140165200715775, +STORE, 140165066498048, 140165116854271, +ERASE, 140165066498048, 140165116854271, +SNULL, 140165183963135, 140165200715775, +STORE, 140165116854272, 140165183963135, +STORE, 140165183963136, 140165200715775, +ERASE, 140165183963136, 140165200715775, +SNULL, 140165116989439, 140165183963135, +STORE, 140165116854272, 140165116989439, +STORE, 140165116989440, 140165183963135, +STORE, 140165192323072, 140165209108479, +STORE, 140165108461568, 140165116854271, +STORE, 140164974243840, 140165108461567, +STORE, 140164965851136, 140164974243839, +SNULL, 140164974243840, 140164982636543, +STORE, 140164982636544, 140165108461567, +STORE, 140164974243840, 140164982636543, +ERASE, 140164974243840, 140164982636543, +STORE, 140164965851136, 140164982636543, +STORE, 140164957458432, 140164982636543, +STORE, 140164949065728, 140164982636543, +STORE, 140164940673024, 140164982636543, +STORE, 140164806455296, 140164940673023, +STORE, 140164798062592, 140164806455295, +STORE, 140164789669888, 140164806455295, +STORE, 140164655452160, 140164789669887, +STORE, 140164647059456, 140164655452159, +STORE, 140164638666752, 140164655452159, +SNULL, 140164655452160, 140164714201087, +STORE, 140164714201088, 140164789669887, +STORE, 140164655452160, 140164714201087, +ERASE, 140164655452160, 140164714201087, +STORE, 140164705808384, 140164714201087, +STORE, 140164697415680, 140164714201087, +STORE, 140164504449024, 140164638666751, +SNULL, 140164504449024, 140164512874495, +STORE, 140164512874496, 140164638666751, +STORE, 140164504449024, 140164512874495, +ERASE, 140164504449024, 140164512874495, +STORE, 140164689022976, 140164714201087, +STORE, 140164680630272, 140164714201087, +SNULL, 140164680634367, 140164714201087, +STORE, 140164680630272, 140164680634367, +STORE, 140164680634368, 140164714201087, +STORE, 140164378656768, 140164638666751, +SNULL, 140165192323072, 140165200715775, +STORE, 140165200715776, 140165209108479, +STORE, 140165192323072, 140165200715775, +SNULL, 140165200719871, 140165209108479, +STORE, 140165200715776, 140165200719871, +STORE, 140165200719872, 140165209108479, +SNULL, 140165049745407, 140165108461567, +STORE, 140164982636544, 140165049745407, +STORE, 140165049745408, 140165108461567, +ERASE, 140165049745408, 140165108461567, +SNULL, 140164982771711, 140165049745407, +STORE, 140164982636544, 140164982771711, +STORE, 140164982771712, 140165049745407, +STORE, 140164244439040, 140164638666751, +SNULL, 140164311547903, 140164638666751, +STORE, 140164244439040, 140164311547903, +STORE, 140164311547904, 140164638666751, +SNULL, 140164311547904, 140164378656767, +STORE, 140164378656768, 140164638666751, +STORE, 140164311547904, 140164378656767, +ERASE, 140164311547904, 140164378656767, +SNULL, 140164806455296, 140164848418815, +STORE, 140164848418816, 140164940673023, +STORE, 140164806455296, 140164848418815, +ERASE, 140164806455296, 140164848418815, +SNULL, 140164915527679, 140164940673023, +STORE, 140164848418816, 140164915527679, +STORE, 140164915527680, 140164940673023, +ERASE, 140164915527680, 140164940673023, +STORE, 140164110221312, 140164311547903, +SNULL, 140164177330175, 140164311547903, +STORE, 140164110221312, 140164177330175, +STORE, 140164177330176, 140164311547903, +SNULL, 140164177330176, 140164244439039, +STORE, 140164244439040, 140164311547903, +STORE, 140164177330176, 140164244439039, +ERASE, 140164177330176, 140164244439039, +SNULL, 140164781309951, 140164789669887, +STORE, 140164714201088, 140164781309951, +STORE, 140164781309952, 140164789669887, +ERASE, 140164781309952, 140164789669887, +STORE, 140163976003584, 140164177330175, +SNULL, 140164043112447, 140164177330175, +STORE, 140163976003584, 140164043112447, +STORE, 140164043112448, 140164177330175, +SNULL, 140164043112448, 140164110221311, +STORE, 140164110221312, 140164177330175, +STORE, 140164043112448, 140164110221311, +ERASE, 140164043112448, 140164110221311, +SNULL, 140164579983359, 140164638666751, +STORE, 140164378656768, 140164579983359, +STORE, 140164579983360, 140164638666751, +ERASE, 140164579983360, 140164638666751, +STORE, 140163841785856, 140164043112447, +SNULL, 140163908894719, 140164043112447, +STORE, 140163841785856, 140163908894719, +STORE, 140163908894720, 140164043112447, +SNULL, 140163908894720, 140163976003583, +STORE, 140163976003584, 140164043112447, +STORE, 140163908894720, 140163976003583, +ERASE, 140163908894720, 140163976003583, +SNULL, 140164940673024, 140164965851135, +STORE, 140164965851136, 140164982636543, +STORE, 140164940673024, 140164965851135, +SNULL, 140164965855231, 140164982636543, +STORE, 140164965851136, 140164965855231, +STORE, 140164965855232, 140164982636543, +SNULL, 140164965855232, 140164974243839, +STORE, 140164974243840, 140164982636543, +STORE, 140164965855232, 140164974243839, +SNULL, 140164974247935, 140164982636543, +STORE, 140164974243840, 140164974247935, +STORE, 140164974247936, 140164982636543, +SNULL, 140164445765631, 140164579983359, +STORE, 140164378656768, 140164445765631, +STORE, 140164445765632, 140164579983359, +SNULL, 140164445765632, 140164512874495, +STORE, 140164512874496, 140164579983359, +STORE, 140164445765632, 140164512874495, +ERASE, 140164445765632, 140164512874495, +SNULL, 140164378791935, 140164445765631, +STORE, 140164378656768, 140164378791935, +STORE, 140164378791936, 140164445765631, +SNULL, 140164789673983, 140164806455295, +STORE, 140164789669888, 140164789673983, +STORE, 140164789673984, 140164806455295, +SNULL, 140164789673984, 140164798062591, +STORE, 140164798062592, 140164806455295, +STORE, 140164789673984, 140164798062591, +SNULL, 140164798066687, 140164806455295, +STORE, 140164798062592, 140164798066687, +STORE, 140164798066688, 140164806455295, +SNULL, 140164638670847, 140164655452159, +STORE, 140164638666752, 140164638670847, +STORE, 140164638670848, 140164655452159, +STORE, 140165100068864, 140165116854271, +STORE, 140165091676160, 140165116854271, +STORE, 140165083283456, 140165116854271, +SNULL, 140164244574207, 140164311547903, +STORE, 140164244439040, 140164244574207, +STORE, 140164244574208, 140164311547903, +SNULL, 140164848553983, 140164915527679, +STORE, 140164848418816, 140164848553983, +STORE, 140164848553984, 140164915527679, +SNULL, 140164110356479, 140164177330175, +STORE, 140164110221312, 140164110356479, +STORE, 140164110356480, 140164177330175, +SNULL, 140164714336255, 140164781309951, +STORE, 140164714201088, 140164714336255, +STORE, 140164714336256, 140164781309951, +SNULL, 140163976138751, 140164043112447, +STORE, 140163976003584, 140163976138751, +STORE, 140163976138752, 140164043112447, +SNULL, 140164513009663, 140164579983359, +STORE, 140164512874496, 140164513009663, +STORE, 140164513009664, 140164579983359, +SNULL, 140163841921023, 140163908894719, +STORE, 140163841785856, 140163841921023, +STORE, 140163841921024, 140163908894719, +SNULL, 140165083283456, 140165100068863, +STORE, 140165100068864, 140165116854271, +STORE, 140165083283456, 140165100068863, +SNULL, 140165100072959, 140165116854271, +STORE, 140165100068864, 140165100072959, +STORE, 140165100072960, 140165116854271, +SNULL, 140165100072960, 140165108461567, +STORE, 140165108461568, 140165116854271, +STORE, 140165100072960, 140165108461567, +SNULL, 140165108465663, 140165116854271, +STORE, 140165108461568, 140165108465663, +STORE, 140165108465664, 140165116854271, +STORE, 140165074890752, 140165100068863, +SNULL, 140165074894847, 140165100068863, +STORE, 140165074890752, 140165074894847, +STORE, 140165074894848, 140165100068863, +STORE, 140165066498048, 140165074890751, +STORE, 140165058105344, 140165074890751, +STORE, 140164932280320, 140164965851135, +SNULL, 140165192327167, 140165200715775, +STORE, 140165192323072, 140165192327167, +STORE, 140165192327168, 140165200715775, +STORE, 140164923887616, 140164965851135, +SNULL, 140164923891711, 140164965851135, +STORE, 140164923887616, 140164923891711, +STORE, 140164923891712, 140164965851135, +SNULL, 140164680634368, 140164705808383, +STORE, 140164705808384, 140164714201087, +STORE, 140164680634368, 140164705808383, +SNULL, 140164705812479, 140164714201087, +STORE, 140164705808384, 140164705812479, +STORE, 140164705812480, 140164714201087, +SNULL, 140164680634368, 140164697415679, +STORE, 140164697415680, 140164705808383, +STORE, 140164680634368, 140164697415679, +SNULL, 140164697419775, 140164705808383, +STORE, 140164697415680, 140164697419775, +STORE, 140164697419776, 140164705808383, +STORE, 140164840026112, 140164848418815, +STORE, 140164831633408, 140164848418815, +STORE, 140164823240704, 140164848418815, +SNULL, 140165074894848, 140165083283455, +STORE, 140165083283456, 140165100068863, +STORE, 140165074894848, 140165083283455, +SNULL, 140165083287551, 140165100068863, +STORE, 140165083283456, 140165083287551, +STORE, 140165083287552, 140165100068863, +SNULL, 140165083287552, 140165091676159, +STORE, 140165091676160, 140165100068863, +STORE, 140165083287552, 140165091676159, +SNULL, 140165091680255, 140165100068863, +STORE, 140165091676160, 140165091680255, +STORE, 140165091680256, 140165100068863, +SNULL, 140164638670848, 140164647059455, +STORE, 140164647059456, 140164655452159, +STORE, 140164638670848, 140164647059455, +SNULL, 140164647063551, 140164655452159, +STORE, 140164647059456, 140164647063551, +STORE, 140164647063552, 140164655452159, +SNULL, 140164923891712, 140164940673023, +STORE, 140164940673024, 140164965851135, +STORE, 140164923891712, 140164940673023, +SNULL, 140164940677119, 140164965851135, +STORE, 140164940673024, 140164940677119, +STORE, 140164940677120, 140164965851135, +SNULL, 140164940677120, 140164949065727, +STORE, 140164949065728, 140164965851135, +STORE, 140164940677120, 140164949065727, +SNULL, 140164949069823, 140164965851135, +STORE, 140164949065728, 140164949069823, +STORE, 140164949069824, 140164965851135, +SNULL, 140164949069824, 140164957458431, +STORE, 140164957458432, 140164965851135, +STORE, 140164949069824, 140164957458431, +SNULL, 140164957462527, 140164965851135, +STORE, 140164957458432, 140164957462527, +STORE, 140164957462528, 140164965851135, +SNULL, 140164680634368, 140164689022975, +STORE, 140164689022976, 140164697415679, +STORE, 140164680634368, 140164689022975, +SNULL, 140164689027071, 140164697415679, +STORE, 140164689022976, 140164689027071, +STORE, 140164689027072, 140164697415679, +STORE, 140164814848000, 140164848418815, +SNULL, 140165058105344, 140165066498047, +STORE, 140165066498048, 140165074890751, +STORE, 140165058105344, 140165066498047, +SNULL, 140165066502143, 140165074890751, +STORE, 140165066498048, 140165066502143, +STORE, 140165066502144, 140165074890751, +SNULL, 140165058109439, 140165066498047, +STORE, 140165058105344, 140165058109439, +STORE, 140165058109440, 140165066498047, +STORE, 140164798066688, 140164814847999, +SNULL, 140164798066688, 140164806455295, +STORE, 140164806455296, 140164814847999, +STORE, 140164798066688, 140164806455295, +SNULL, 140164806459391, 140164814847999, +STORE, 140164806455296, 140164806459391, +STORE, 140164806459392, 140164814847999, +SNULL, 140164923891712, 140164932280319, +STORE, 140164932280320, 140164940673023, +STORE, 140164923891712, 140164932280319, +SNULL, 140164932284415, 140164940673023, +STORE, 140164932280320, 140164932284415, +STORE, 140164932284416, 140164940673023, +STORE, 140164672237568, 140164680630271, +STORE, 140164663844864, 140164680630271, +STORE, 140164647063552, 140164680630271, +SNULL, 140164647063552, 140164655452159, +STORE, 140164655452160, 140164680630271, +STORE, 140164647063552, 140164655452159, +SNULL, 140164655456255, 140164680630271, +STORE, 140164655452160, 140164655456255, +STORE, 140164655456256, 140164680630271, +STORE, 140164630274048, 140164638666751, +SNULL, 140164814852095, 140164848418815, +STORE, 140164814848000, 140164814852095, +STORE, 140164814852096, 140164848418815, +SNULL, 140164814852096, 140164831633407, +STORE, 140164831633408, 140164848418815, +STORE, 140164814852096, 140164831633407, +SNULL, 140164831637503, 140164848418815, +STORE, 140164831633408, 140164831637503, +STORE, 140164831637504, 140164848418815, +STORE, 140164621881344, 140164638666751, +SNULL, 140164831637504, 140164840026111, +STORE, 140164840026112, 140164848418815, +STORE, 140164831637504, 140164840026111, +SNULL, 140164840030207, 140164848418815, +STORE, 140164840026112, 140164840030207, +STORE, 140164840030208, 140164848418815, +STORE, 140164613488640, 140164638666751, +SNULL, 140164613492735, 140164638666751, +STORE, 140164613488640, 140164613492735, +STORE, 140164613492736, 140164638666751, +STORE, 140164605095936, 140164613488639, +SNULL, 140164605100031, 140164613488639, +STORE, 140164605095936, 140164605100031, +STORE, 140164605100032, 140164613488639, +STORE, 140164596703232, 140164605095935, +STORE, 140164588310528, 140164605095935, +SNULL, 140164588314623, 140164605095935, +STORE, 140164588310528, 140164588314623, +STORE, 140164588314624, 140164605095935, +STORE, 140164504481792, 140164512874495, +STORE, 140164496089088, 140164512874495, +SNULL, 140164496089088, 140164504481791, +STORE, 140164504481792, 140164512874495, +STORE, 140164496089088, 140164504481791, +SNULL, 140164504485887, 140164512874495, +STORE, 140164504481792, 140164504485887, +STORE, 140164504485888, 140164512874495, +SNULL, 140164613492736, 140164630274047, +STORE, 140164630274048, 140164638666751, +STORE, 140164613492736, 140164630274047, +SNULL, 140164630278143, 140164638666751, +STORE, 140164630274048, 140164630278143, +STORE, 140164630278144, 140164638666751, +STORE, 140164487696384, 140164504481791, +STORE, 140164479303680, 140164504481791, +SNULL, 140164814852096, 140164823240703, +STORE, 140164823240704, 140164831633407, +STORE, 140164814852096, 140164823240703, +SNULL, 140164823244799, 140164831633407, +STORE, 140164823240704, 140164823244799, +STORE, 140164823244800, 140164831633407, +STORE, 140164470910976, 140164504481791, +SNULL, 140164470910976, 140164496089087, +STORE, 140164496089088, 140164504481791, +STORE, 140164470910976, 140164496089087, +SNULL, 140164496093183, 140164504481791, +STORE, 140164496089088, 140164496093183, +STORE, 140164496093184, 140164504481791, +SNULL, 140164655456256, 140164672237567, +STORE, 140164672237568, 140164680630271, +STORE, 140164655456256, 140164672237567, +SNULL, 140164672241663, 140164680630271, +STORE, 140164672237568, 140164672241663, +STORE, 140164672241664, 140164680630271, +STORE, 140164462518272, 140164496089087, +STORE, 140164454125568, 140164496089087, +SNULL, 140164655456256, 140164663844863, +STORE, 140164663844864, 140164672237567, +STORE, 140164655456256, 140164663844863, +SNULL, 140164663848959, 140164672237567, +STORE, 140164663844864, 140164663848959, +STORE, 140164663848960, 140164672237567, +STORE, 140164370264064, 140164378656767, +STORE, 140164361871360, 140164378656767, +STORE, 140164353478656, 140164378656767, +STORE, 140164345085952, 140164378656767, +SNULL, 140164345085952, 140164353478655, +STORE, 140164353478656, 140164378656767, +STORE, 140164345085952, 140164353478655, +SNULL, 140164353482751, 140164378656767, +STORE, 140164353478656, 140164353482751, +STORE, 140164353482752, 140164378656767, +SNULL, 140164454125568, 140164487696383, +STORE, 140164487696384, 140164496089087, +STORE, 140164454125568, 140164487696383, +SNULL, 140164487700479, 140164496089087, +STORE, 140164487696384, 140164487700479, +STORE, 140164487700480, 140164496089087, +STORE, 140164336693248, 140164353478655, +SNULL, 140164336697343, 140164353478655, +STORE, 140164336693248, 140164336697343, +STORE, 140164336697344, 140164353478655, +STORE, 140164328300544, 140164336693247, +SNULL, 140164454125568, 140164479303679, +STORE, 140164479303680, 140164487696383, +STORE, 140164454125568, 140164479303679, +SNULL, 140164479307775, 140164487696383, +STORE, 140164479303680, 140164479307775, +STORE, 140164479307776, 140164487696383, +STORE, 140164319907840, 140164336693247, +STORE, 140164236046336, 140164244439039, +SNULL, 140164588314624, 140164596703231, +STORE, 140164596703232, 140164605095935, +STORE, 140164588314624, 140164596703231, +SNULL, 140164596707327, 140164605095935, +STORE, 140164596703232, 140164596707327, +STORE, 140164596707328, 140164605095935, +SNULL, 140164454125568, 140164462518271, +STORE, 140164462518272, 140164479303679, +STORE, 140164454125568, 140164462518271, +SNULL, 140164462522367, 140164479303679, +STORE, 140164462518272, 140164462522367, +STORE, 140164462522368, 140164479303679, +STORE, 140164227653632, 140164244439039, +SNULL, 140164227657727, 140164244439039, +STORE, 140164227653632, 140164227657727, +STORE, 140164227657728, 140164244439039, +SNULL, 140164462522368, 140164470910975, +STORE, 140164470910976, 140164479303679, +STORE, 140164462522368, 140164470910975, +SNULL, 140164470915071, 140164479303679, +STORE, 140164470910976, 140164470915071, +STORE, 140164470915072, 140164479303679, +SNULL, 140164613492736, 140164621881343, +STORE, 140164621881344, 140164630274047, +STORE, 140164613492736, 140164621881343, +SNULL, 140164621885439, 140164630274047, +STORE, 140164621881344, 140164621885439, +STORE, 140164621885440, 140164630274047, +SNULL, 140164353482752, 140164370264063, +STORE, 140164370264064, 140164378656767, +STORE, 140164353482752, 140164370264063, +SNULL, 140164370268159, 140164378656767, +STORE, 140164370264064, 140164370268159, +STORE, 140164370268160, 140164378656767, +STORE, 140164219260928, 140164227653631, +SNULL, 140164319911935, 140164336693247, +STORE, 140164319907840, 140164319911935, +STORE, 140164319911936, 140164336693247, +SNULL, 140164336697344, 140164345085951, +STORE, 140164345085952, 140164353478655, +STORE, 140164336697344, 140164345085951, +SNULL, 140164345090047, 140164353478655, +STORE, 140164345085952, 140164345090047, +STORE, 140164345090048, 140164353478655, +SNULL, 140164319911936, 140164328300543, +STORE, 140164328300544, 140164336693247, +STORE, 140164319911936, 140164328300543, +SNULL, 140164328304639, 140164336693247, +STORE, 140164328300544, 140164328304639, +STORE, 140164328304640, 140164336693247, +SNULL, 140164454129663, 140164462518271, +STORE, 140164454125568, 140164454129663, +STORE, 140164454129664, 140164462518271, +STORE, 140164210868224, 140164227653631, +STORE, 140164202475520, 140164227653631, +STORE, 140164194082816, 140164227653631, +SNULL, 140164194086911, 140164227653631, +STORE, 140164194082816, 140164194086911, +STORE, 140164194086912, 140164227653631, +SNULL, 140164353482752, 140164361871359, +STORE, 140164361871360, 140164370264063, +STORE, 140164353482752, 140164361871359, +SNULL, 140164361875455, 140164370264063, +STORE, 140164361871360, 140164361875455, +STORE, 140164361875456, 140164370264063, +SNULL, 140164227657728, 140164236046335, +STORE, 140164236046336, 140164244439039, +STORE, 140164227657728, 140164236046335, +SNULL, 140164236050431, 140164244439039, +STORE, 140164236046336, 140164236050431, +STORE, 140164236050432, 140164244439039, +STORE, 140164185690112, 140164194082815, +SNULL, 140164194086912, 140164219260927, +STORE, 140164219260928, 140164227653631, +STORE, 140164194086912, 140164219260927, +SNULL, 140164219265023, 140164227653631, +STORE, 140164219260928, 140164219265023, +STORE, 140164219265024, 140164227653631, +STORE, 140164101828608, 140164110221311, +STORE, 140164093435904, 140164110221311, +STORE, 140164085043200, 140164110221311, +SNULL, 140164085047295, 140164110221311, +STORE, 140164085043200, 140164085047295, +STORE, 140164085047296, 140164110221311, +STORE, 140164076650496, 140164085043199, +SNULL, 140164185694207, 140164194082815, +STORE, 140164185690112, 140164185694207, +STORE, 140164185694208, 140164194082815, +SNULL, 140164085047296, 140164101828607, +STORE, 140164101828608, 140164110221311, +STORE, 140164085047296, 140164101828607, +SNULL, 140164101832703, 140164110221311, +STORE, 140164101828608, 140164101832703, +STORE, 140164101832704, 140164110221311, +SNULL, 140164085047296, 140164093435903, +STORE, 140164093435904, 140164101828607, +STORE, 140164085047296, 140164093435903, +SNULL, 140164093439999, 140164101828607, +STORE, 140164093435904, 140164093439999, +STORE, 140164093440000, 140164101828607, +SNULL, 140164194086912, 140164202475519, +STORE, 140164202475520, 140164219260927, +STORE, 140164194086912, 140164202475519, +SNULL, 140164202479615, 140164219260927, +STORE, 140164202475520, 140164202479615, +STORE, 140164202479616, 140164219260927, +SNULL, 140164202479616, 140164210868223, +STORE, 140164210868224, 140164219260927, +STORE, 140164202479616, 140164210868223, +SNULL, 140164210872319, 140164219260927, +STORE, 140164210868224, 140164210872319, +STORE, 140164210872320, 140164219260927, +SNULL, 140164076654591, 140164085043199, +STORE, 140164076650496, 140164076654591, +STORE, 140164076654592, 140164085043199, +STORE, 140164068257792, 140164076650495, +SNULL, 140164068261887, 140164076650495, +STORE, 140164068257792, 140164068261887, +STORE, 140164068261888, 140164076650495, +STORE, 140165753053184, 140165753081855, +STORE, 140165725851648, 140165728043007, +SNULL, 140165725851648, 140165725941759, +STORE, 140165725941760, 140165728043007, +STORE, 140165725851648, 140165725941759, +SNULL, 140165728034815, 140165728043007, +STORE, 140165725941760, 140165728034815, +STORE, 140165728034816, 140165728043007, +ERASE, 140165728034816, 140165728043007, +STORE, 140165728034816, 140165728043007, +SNULL, 140165728038911, 140165728043007, +STORE, 140165728034816, 140165728038911, +STORE, 140165728038912, 140165728043007, +ERASE, 140165753053184, 140165753081855, +ERASE, 140164638666752, 140164638670847, +ERASE, 140164638670848, 140164647059455, +ERASE, 140165091676160, 140165091680255, +ERASE, 140165091680256, 140165100068863, +ERASE, 140164613488640, 140164613492735, +ERASE, 140164613492736, 140164621881343, +ERASE, 140164319907840, 140164319911935, +ERASE, 140164319911936, 140164328300543, +ERASE, 140165620154368, 140165620158463, +ERASE, 140165620158464, 140165628547071, +ERASE, 140164798062592, 140164798066687, +ERASE, 140164798066688, 140164806455295, +ERASE, 140164789669888, 140164789673983, +ERASE, 140164789673984, 140164798062591, +ERASE, 140164965851136, 140164965855231, +ERASE, 140164965855232, 140164974243839, +ERASE, 140165074890752, 140165074894847, +ERASE, 140165074894848, 140165083283455, +ERASE, 140164672237568, 140164672241663, +ERASE, 140164672241664, 140164680630271, +ERASE, 140164454125568, 140164454129663, +ERASE, 140164454129664, 140164462518271, +ERASE, 140165200715776, 140165200719871, +ERASE, 140165200719872, 140165209108479, +ERASE, 140164932280320, 140164932284415, +ERASE, 140164932284416, 140164940673023, +ERASE, 140164663844864, 140164663848959, +ERASE, 140164663848960, 140164672237567, +ERASE, 140164697415680, 140164697419775, +ERASE, 140164697419776, 140164705808383, +ERASE, 140164831633408, 140164831637503, +ERASE, 140164831637504, 140164840026111, +ERASE, 140165192323072, 140165192327167, +ERASE, 140165192327168, 140165200715775, +ERASE, 140165108461568, 140165108465663, +ERASE, 140165108465664, 140165116854271, +ERASE, 140164840026112, 140164840030207, +ERASE, 140164840030208, 140164848418815, +ERASE, 140164647059456, 140164647063551, +ERASE, 140164647063552, 140164655452159, +ERASE, 140165083283456, 140165083287551, +ERASE, 140165083287552, 140165091676159, +ERASE, 140164923887616, 140164923891711, +ERASE, 140164923891712, 140164932280319, +ERASE, 140164823240704, 140164823244799, +ERASE, 140164823244800, 140164831633407, +ERASE, 140164227653632, 140164227657727, +ERASE, 140164227657728, 140164236046335, +ERASE, 140164957458432, 140164957462527, +ERASE, 140164957462528, 140164965851135, +ERASE, 140164680630272, 140164680634367, +ERASE, 140164680634368, 140164689022975, +ERASE, 140164974243840, 140164974247935, +ERASE, 140164974247936, 140164982636543, +ERASE, 140165066498048, 140165066502143, +ERASE, 140165066502144, 140165074890751, +ERASE, 140164621881344, 140164621885439, +ERASE, 140164621885440, 140164630274047, +ERASE, 140164949065728, 140164949069823, +ERASE, 140164949069824, 140164957458431, +ERASE, 140164588310528, 140164588314623, +ERASE, 140164588314624, 140164596703231, +ERASE, 140164806455296, 140164806459391, +ERASE, 140164806459392, 140164814847999, +ERASE, 140164940673024, 140164940677119, +ERASE, 140164940677120, 140164949065727, +ERASE, 140164596703232, 140164596707327, +ERASE, 140164596707328, 140164605095935, +ERASE, 140164605095936, 140164605100031, +ERASE, 140164605100032, 140164613488639, +ERASE, 140164655452160, 140164655456255, +ERASE, 140164655456256, 140164663844863, +ERASE, 140164705808384, 140164705812479, +ERASE, 140164705812480, 140164714201087, +ERASE, 140164689022976, 140164689027071, +ERASE, 140164689027072, 140164697415679, +ERASE, 140164630274048, 140164630278143, +ERASE, 140164630278144, 140164638666751, +ERASE, 140164479303680, 140164479307775, +ERASE, 140164479307776, 140164487696383, +ERASE, 140164236046336, 140164236050431, +ERASE, 140164236050432, 140164244439039, +ERASE, 140164085043200, 140164085047295, +ERASE, 140164085047296, 140164093435903, +ERASE, 140164345085952, 140164345090047, +ERASE, 140164345090048, 140164353478655, +ERASE, 140164101828608, 140164101832703, +ERASE, 140164101832704, 140164110221311, +ERASE, 140164370264064, 140164370268159, +ERASE, 140164370268160, 140164378656767, +ERASE, 140164336693248, 140164336697343, +ERASE, 140164336697344, 140164345085951, +ERASE, 140164194082816, 140164194086911, +ERASE, 140164194086912, 140164202475519, +ERASE, 140164353478656, 140164353482751, +ERASE, 140164353482752, 140164361871359, +ERASE, 140164210868224, 140164210872319, +ERASE, 140164210872320, 140164219260927, +ERASE, 140164814848000, 140164814852095, +ERASE, 140164814852096, 140164823240703, +ERASE, 140164504481792, 140164504485887, +ERASE, 140164504485888, 140164512874495, +ERASE, 140165100068864, 140165100072959, +ERASE, 140165100072960, 140165108461567, +ERASE, 140164361871360, 140164361875455, +ERASE, 140164361875456, 140164370264063, +ERASE, 140164470910976, 140164470915071, +ERASE, 140164470915072, 140164479303679, +ERASE, 140164076650496, 140164076654591, +ERASE, 140164076654592, 140164085043199, +ERASE, 140164202475520, 140164202479615, +ERASE, 140164202479616, 140164210868223, +ERASE, 140164462518272, 140164462522367, +ERASE, 140164462522368, 140164470910975, +ERASE, 140165351718912, 140165351723007, +ERASE, 140165351723008, 140165360111615, +ERASE, 140164328300544, 140164328304639, +ERASE, 140164328304640, 140164336693247, +ERASE, 140164093435904, 140164093439999, +ERASE, 140164093440000, 140164101828607, +ERASE, 140165603368960, 140165603373055, +ERASE, 140165603373056, 140165611761663, +ERASE, 140165368504320, 140165368508415, +ERASE, 140165368508416, 140165376897023, +ERASE, 140165334933504, 140165334937599, +ERASE, 140165334937600, 140165343326207, +ERASE, 140165594976256, 140165594980351, +ERASE, 140165594980352, 140165603368959, +ERASE, 140164487696384, 140164487700479, +ERASE, 140164487700480, 140164496089087, +ERASE, 140164219260928, 140164219265023, +ERASE, 140164219265024, 140164227653631, +ERASE, 140164185690112, 140164185694207, +ERASE, 140164185694208, 140164194082815, +ERASE, 140164068257792, 140164068261887, +ERASE, 140164068261888, 140164076650495, +ERASE, 140165225893888, 140165225897983, +ERASE, 140165225897984, 140165234286591, +ERASE, 140165058105344, 140165058109439, + }; + unsigned long set31[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140730890784768, 140737488351231, +SNULL, 140730890788863, 140737488351231, +STORE, 140730890784768, 140730890788863, +STORE, 140730890653696, 140730890788863, +STORE, 94577123659776, 94577125912575, +SNULL, 94577123790847, 94577125912575, +STORE, 94577123659776, 94577123790847, +STORE, 94577123790848, 94577125912575, +ERASE, 94577123790848, 94577125912575, +STORE, 94577125883904, 94577125892095, +STORE, 94577125892096, 94577125912575, +STORE, 140624060407808, 140624062660607, +SNULL, 140624060551167, 140624062660607, +STORE, 140624060407808, 140624060551167, +STORE, 140624060551168, 140624062660607, +ERASE, 140624060551168, 140624062660607, +STORE, 140624062648320, 140624062656511, +STORE, 140624062656512, 140624062660607, +STORE, 140730892140544, 140730892144639, +STORE, 140730892128256, 140730892140543, +STORE, 140624062619648, 140624062648319, +STORE, 140624062611456, 140624062619647, +STORE, 140624058191872, 140624060407807, +SNULL, 140624058191872, 140624058290175, +STORE, 140624058290176, 140624060407807, +STORE, 140624058191872, 140624058290175, +SNULL, 140624060383231, 140624060407807, +STORE, 140624058290176, 140624060383231, +STORE, 140624060383232, 140624060407807, +SNULL, 140624060383232, 140624060391423, +STORE, 140624060391424, 140624060407807, +STORE, 140624060383232, 140624060391423, +ERASE, 140624060383232, 140624060391423, +STORE, 140624060383232, 140624060391423, +ERASE, 140624060391424, 140624060407807, +STORE, 140624060391424, 140624060407807, +STORE, 140624054394880, 140624058191871, +SNULL, 140624054394880, 140624056053759, +STORE, 140624056053760, 140624058191871, +STORE, 140624054394880, 140624056053759, +SNULL, 140624058150911, 140624058191871, +STORE, 140624056053760, 140624058150911, +STORE, 140624058150912, 140624058191871, +SNULL, 140624058150912, 140624058175487, +STORE, 140624058175488, 140624058191871, +STORE, 140624058150912, 140624058175487, +ERASE, 140624058150912, 140624058175487, +STORE, 140624058150912, 140624058175487, +ERASE, 140624058175488, 140624058191871, +STORE, 140624058175488, 140624058191871, +STORE, 140624062603264, 140624062619647, +SNULL, 140624058167295, 140624058175487, +STORE, 140624058150912, 140624058167295, +STORE, 140624058167296, 140624058175487, +SNULL, 140624060387327, 140624060391423, +STORE, 140624060383232, 140624060387327, +STORE, 140624060387328, 140624060391423, +SNULL, 94577125887999, 94577125892095, +STORE, 94577125883904, 94577125887999, +STORE, 94577125888000, 94577125892095, +SNULL, 140624062652415, 140624062656511, +STORE, 140624062648320, 140624062652415, +STORE, 140624062652416, 140624062656511, +ERASE, 140624062619648, 140624062648319, +STORE, 94577157709824, 94577157844991, +STORE, 140624046002176, 140624054394879, +SNULL, 140624046006271, 140624054394879, +STORE, 140624046002176, 140624046006271, +STORE, 140624046006272, 140624054394879, +STORE, 140624037609472, 140624046002175, +STORE, 140623903391744, 140624037609471, +SNULL, 140623903391744, 140623940157439, +STORE, 140623940157440, 140624037609471, +STORE, 140623903391744, 140623940157439, +ERASE, 140623903391744, 140623940157439, +SNULL, 140624007266303, 140624037609471, +STORE, 140623940157440, 140624007266303, +STORE, 140624007266304, 140624037609471, +ERASE, 140624007266304, 140624037609471, +SNULL, 140623940292607, 140624007266303, +STORE, 140623940157440, 140623940292607, +STORE, 140623940292608, 140624007266303, +SNULL, 140624037613567, 140624046002175, +STORE, 140624037609472, 140624037613567, +STORE, 140624037613568, 140624046002175, +STORE, 140624029216768, 140624037609471, +SNULL, 140624029220863, 140624037609471, +STORE, 140624029216768, 140624029220863, +STORE, 140624029220864, 140624037609471, +STORE, 140624020824064, 140624029216767, +SNULL, 140624020828159, 140624029216767, +STORE, 140624020824064, 140624020828159, +STORE, 140624020828160, 140624029216767, +STORE, 140624012431360, 140624020824063, +SNULL, 140624012435455, 140624020824063, +STORE, 140624012431360, 140624012435455, +STORE, 140624012435456, 140624020824063, +STORE, 140623931764736, 140623940157439, +STORE, 140623797547008, 140623931764735, +SNULL, 140623797547008, 140623805939711, +STORE, 140623805939712, 140623931764735, +STORE, 140623797547008, 140623805939711, +ERASE, 140623797547008, 140623805939711, +SNULL, 140623873048575, 140623931764735, +STORE, 140623805939712, 140623873048575, +STORE, 140623873048576, 140623931764735, +ERASE, 140623873048576, 140623931764735, +STORE, 140623923372032, 140623940157439, +STORE, 140623914979328, 140623940157439, +STORE, 140623906586624, 140623940157439, +STORE, 140623671721984, 140623873048575, +SNULL, 140623738830847, 140623873048575, +STORE, 140623671721984, 140623738830847, +STORE, 140623738830848, 140623873048575, +SNULL, 140623738830848, 140623805939711, +STORE, 140623805939712, 140623873048575, +STORE, 140623738830848, 140623805939711, +ERASE, 140623738830848, 140623805939711, +SNULL, 140623806074879, 140623873048575, +STORE, 140623805939712, 140623806074879, +STORE, 140623806074880, 140623873048575, +SNULL, 140623906586624, 140623931764735, +STORE, 140623931764736, 140623940157439, +STORE, 140623906586624, 140623931764735, +SNULL, 140623931768831, 140623940157439, +STORE, 140623931764736, 140623931768831, +STORE, 140623931768832, 140623940157439, +STORE, 140623537504256, 140623738830847, +SNULL, 140623537504256, 140623671721983, +STORE, 140623671721984, 140623738830847, +STORE, 140623537504256, 140623671721983, +SNULL, 140623671857151, 140623738830847, +STORE, 140623671721984, 140623671857151, +STORE, 140623671857152, 140623738830847, +SNULL, 140623604613119, 140623671721983, +STORE, 140623537504256, 140623604613119, +STORE, 140623604613120, 140623671721983, +ERASE, 140623604613120, 140623671721983, +SNULL, 140623537639423, 140623604613119, +STORE, 140623537504256, 140623537639423, +STORE, 140623537639424, 140623604613119, +STORE, 140623537639424, 140623671721983, +SNULL, 140623537639424, 140623604613119, +STORE, 140623604613120, 140623671721983, +STORE, 140623537639424, 140623604613119, +SNULL, 140623604748287, 140623671721983, +STORE, 140623604613120, 140623604748287, +STORE, 140623604748288, 140623671721983, +STORE, 140623898193920, 140623931764735, +SNULL, 140623898193920, 140623923372031, +STORE, 140623923372032, 140623931764735, +STORE, 140623898193920, 140623923372031, +SNULL, 140623923376127, 140623931764735, +STORE, 140623923372032, 140623923376127, +STORE, 140623923376128, 140623931764735, +STORE, 140623889801216, 140623923372031, +SNULL, 140623889801216, 140623898193919, +STORE, 140623898193920, 140623923372031, +STORE, 140623889801216, 140623898193919, +SNULL, 140623898198015, 140623923372031, +STORE, 140623898193920, 140623898198015, +STORE, 140623898198016, 140623923372031, +SNULL, 140623889805311, 140623898193919, +STORE, 140623889801216, 140623889805311, +STORE, 140623889805312, 140623898193919, +SNULL, 140623898198016, 140623906586623, +STORE, 140623906586624, 140623923372031, +STORE, 140623898198016, 140623906586623, +SNULL, 140623906590719, 140623923372031, +STORE, 140623906586624, 140623906590719, +STORE, 140623906590720, 140623923372031, +STORE, 140623881408512, 140623889801215, +SNULL, 140623906590720, 140623914979327, +STORE, 140623914979328, 140623923372031, +STORE, 140623906590720, 140623914979327, +SNULL, 140623914983423, 140623923372031, +STORE, 140623914979328, 140623914983423, +STORE, 140623914983424, 140623923372031, +SNULL, 140623881412607, 140623889801215, +STORE, 140623881408512, 140623881412607, +STORE, 140623881412608, 140623889801215, +STORE, 140623797547008, 140623805939711, +STORE, 140623789154304, 140623805939711, +STORE, 140623780761600, 140623805939711, +SNULL, 140623780761600, 140623789154303, +STORE, 140623789154304, 140623805939711, +STORE, 140623780761600, 140623789154303, +SNULL, 140623789158399, 140623805939711, +STORE, 140623789154304, 140623789158399, +STORE, 140623789158400, 140623805939711, +STORE, 140623772368896, 140623789154303, +STORE, 140623763976192, 140623789154303, +SNULL, 140623763976192, 140623780761599, +STORE, 140623780761600, 140623789154303, +STORE, 140623763976192, 140623780761599, +SNULL, 140623780765695, 140623789154303, +STORE, 140623780761600, 140623780765695, +STORE, 140623780765696, 140623789154303, +SNULL, 140623789158400, 140623797547007, +STORE, 140623797547008, 140623805939711, +STORE, 140623789158400, 140623797547007, +SNULL, 140623797551103, 140623805939711, +STORE, 140623797547008, 140623797551103, +STORE, 140623797551104, 140623805939711, +SNULL, 140623763976192, 140623772368895, +STORE, 140623772368896, 140623780761599, +STORE, 140623763976192, 140623772368895, +SNULL, 140623772372991, 140623780761599, +STORE, 140623772368896, 140623772372991, +STORE, 140623772372992, 140623780761599, +SNULL, 140623763980287, 140623772368895, +STORE, 140623763976192, 140623763980287, +STORE, 140623763980288, 140623772368895, +STORE, 140623755583488, 140623763976191, +STORE, 140623747190784, 140623763976191, +SNULL, 140623747190784, 140623755583487, +STORE, 140623755583488, 140623763976191, +STORE, 140623747190784, 140623755583487, +SNULL, 140623755587583, 140623763976191, +STORE, 140623755583488, 140623755587583, +STORE, 140623755587584, 140623763976191, +STORE, 140623529111552, 140623537504255, +SNULL, 140623747194879, 140623755583487, +STORE, 140623747190784, 140623747194879, +STORE, 140623747194880, 140623755583487, +SNULL, 140623529115647, 140623537504255, +STORE, 140623529111552, 140623529115647, +STORE, 140623529115648, 140623537504255, +STORE, 140623520718848, 140623529111551, +SNULL, 140623520722943, 140623529111551, +STORE, 140623520718848, 140623520722943, +STORE, 140623520722944, 140623529111551, +STORE, 140623512326144, 140623520718847, +STORE, 140623503933440, 140623520718847, +STORE, 140623495540736, 140623520718847, +STORE, 140623361323008, 140623495540735, +STORE, 140623227105280, 140623495540735, +STORE, 140623218712576, 140623227105279, +STORE, 140623084494848, 140623218712575, +STORE, 140623076102144, 140623084494847, +STORE, 140622941884416, 140623076102143, +SNULL, 140622941884416, 140623000633343, +STORE, 140623000633344, 140623076102143, +STORE, 140622941884416, 140623000633343, +ERASE, 140622941884416, 140623000633343, +STORE, 140622992240640, 140623000633343, +STORE, 140622983847936, 140623000633343, +STORE, 140622849630208, 140622983847935, +STORE, 140622841237504, 140622849630207, +SNULL, 140622849630208, 140622866415615, +STORE, 140622866415616, 140622983847935, +STORE, 140622849630208, 140622866415615, +ERASE, 140622849630208, 140622866415615, +STORE, 140622858022912, 140622866415615, +SNULL, 140622933524479, 140622983847935, +STORE, 140622866415616, 140622933524479, +STORE, 140622933524480, 140622983847935, +ERASE, 140622933524480, 140622983847935, +STORE, 140622975455232, 140623000633343, +STORE, 140622707019776, 140622841237503, +STORE, 140622967062528, 140623000633343, +STORE, 140622572802048, 140622841237503, +STORE, 140622958669824, 140623000633343, +STORE, 140622438584320, 140622841237503, +STORE, 140622950277120, 140623000633343, +SNULL, 140622858027007, 140622866415615, +STORE, 140622858022912, 140622858027007, +STORE, 140622858027008, 140622866415615, +STORE, 140622941884416, 140623000633343, +STORE, 140622841237504, 140622858022911, +SNULL, 140622841237504, 140622849630207, +STORE, 140622849630208, 140622858022911, +STORE, 140622841237504, 140622849630207, +SNULL, 140622849634303, 140622858022911, +STORE, 140622849630208, 140622849634303, +STORE, 140622849634304, 140622858022911, +STORE, 140622430191616, 140622438584319, +SNULL, 140622430195711, 140622438584319, +STORE, 140622430191616, 140622430195711, +STORE, 140622430195712, 140622438584319, +SNULL, 140623361323007, 140623495540735, +STORE, 140623227105280, 140623361323007, +STORE, 140623361323008, 140623495540735, +SNULL, 140623361323008, 140623403286527, +STORE, 140623403286528, 140623495540735, +STORE, 140623361323008, 140623403286527, +ERASE, 140623361323008, 140623403286527, +SNULL, 140623470395391, 140623495540735, +STORE, 140623403286528, 140623470395391, +STORE, 140623470395392, 140623495540735, +ERASE, 140623470395392, 140623495540735, +SNULL, 140623227105280, 140623269068799, +STORE, 140623269068800, 140623361323007, +STORE, 140623227105280, 140623269068799, +ERASE, 140623227105280, 140623269068799, +SNULL, 140623084494848, 140623134851071, +STORE, 140623134851072, 140623218712575, +STORE, 140623084494848, 140623134851071, +ERASE, 140623084494848, 140623134851071, +SNULL, 140623201959935, 140623218712575, +STORE, 140623134851072, 140623201959935, +STORE, 140623201959936, 140623218712575, +ERASE, 140623201959936, 140623218712575, +SNULL, 140623067742207, 140623076102143, +STORE, 140623000633344, 140623067742207, +STORE, 140623067742208, 140623076102143, +ERASE, 140623067742208, 140623076102143, +STORE, 140622295973888, 140622430191615, +SNULL, 140622295973888, 140622329544703, +STORE, 140622329544704, 140622430191615, +STORE, 140622295973888, 140622329544703, +ERASE, 140622295973888, 140622329544703, +SNULL, 140622866550783, 140622933524479, +STORE, 140622866415616, 140622866550783, +STORE, 140622866550784, 140622933524479, +SNULL, 140622707019775, 140622841237503, +STORE, 140622438584320, 140622707019775, +STORE, 140622707019776, 140622841237503, +SNULL, 140622707019776, 140622732197887, +STORE, 140622732197888, 140622841237503, +STORE, 140622707019776, 140622732197887, +ERASE, 140622707019776, 140622732197887, +SNULL, 140622799306751, 140622841237503, +STORE, 140622732197888, 140622799306751, +STORE, 140622799306752, 140622841237503, +ERASE, 140622799306752, 140622841237503, +SNULL, 140622572802047, 140622707019775, +STORE, 140622438584320, 140622572802047, +STORE, 140622572802048, 140622707019775, +SNULL, 140622572802048, 140622597980159, +STORE, 140622597980160, 140622707019775, +STORE, 140622572802048, 140622597980159, +ERASE, 140622572802048, 140622597980159, +SNULL, 140622438584320, 140622463762431, +STORE, 140622463762432, 140622572802047, +STORE, 140622438584320, 140622463762431, +ERASE, 140622438584320, 140622463762431, +SNULL, 140622530871295, 140622572802047, +STORE, 140622463762432, 140622530871295, +STORE, 140622530871296, 140622572802047, +ERASE, 140622530871296, 140622572802047, +STORE, 140622195326976, 140622430191615, +SNULL, 140622262435839, 140622430191615, +STORE, 140622195326976, 140622262435839, +STORE, 140622262435840, 140622430191615, +SNULL, 140622262435840, 140622329544703, +STORE, 140622329544704, 140622430191615, +STORE, 140622262435840, 140622329544703, +ERASE, 140622262435840, 140622329544703, +SNULL, 140622841241599, 140622849630207, +STORE, 140622841237504, 140622841241599, +STORE, 140622841241600, 140622849630207, +STORE, 140623487148032, 140623520718847, +STORE, 140623478755328, 140623520718847, +SNULL, 140622941884416, 140622983847935, +STORE, 140622983847936, 140623000633343, +STORE, 140622941884416, 140622983847935, +SNULL, 140622983852031, 140623000633343, +STORE, 140622983847936, 140622983852031, +STORE, 140622983852032, 140623000633343, +STORE, 140623394893824, 140623403286527, +SNULL, 140623394897919, 140623403286527, +STORE, 140623394893824, 140623394897919, +STORE, 140623394897920, 140623403286527, +SNULL, 140623403421695, 140623470395391, +STORE, 140623403286528, 140623403421695, +STORE, 140623403421696, 140623470395391, +SNULL, 140623478755328, 140623503933439, +STORE, 140623503933440, 140623520718847, +STORE, 140623478755328, 140623503933439, +SNULL, 140623503937535, 140623520718847, +STORE, 140623503933440, 140623503937535, +STORE, 140623503937536, 140623520718847, +SNULL, 140623336177663, 140623361323007, +STORE, 140623269068800, 140623336177663, +STORE, 140623336177664, 140623361323007, +ERASE, 140623336177664, 140623361323007, +SNULL, 140623269203967, 140623336177663, +STORE, 140623269068800, 140623269203967, +STORE, 140623269203968, 140623336177663, +SNULL, 140623134986239, 140623201959935, +STORE, 140623134851072, 140623134986239, +STORE, 140623134986240, 140623201959935, +SNULL, 140623000768511, 140623067742207, +STORE, 140623000633344, 140623000768511, +STORE, 140623000768512, 140623067742207, +SNULL, 140622396653567, 140622430191615, +STORE, 140622329544704, 140622396653567, +STORE, 140622396653568, 140622430191615, +ERASE, 140622396653568, 140622430191615, +SNULL, 140622732333055, 140622799306751, +STORE, 140622732197888, 140622732333055, +STORE, 140622732333056, 140622799306751, +SNULL, 140622941884416, 140622975455231, +STORE, 140622975455232, 140622983847935, +STORE, 140622941884416, 140622975455231, +SNULL, 140622975459327, 140622983847935, +STORE, 140622975455232, 140622975459327, +STORE, 140622975459328, 140622983847935, +SNULL, 140622665089023, 140622707019775, +STORE, 140622597980160, 140622665089023, +STORE, 140622665089024, 140622707019775, +ERASE, 140622665089024, 140622707019775, +SNULL, 140622598115327, 140622665089023, +STORE, 140622597980160, 140622598115327, +STORE, 140622598115328, 140622665089023, +SNULL, 140622463897599, 140622530871295, +STORE, 140622463762432, 140622463897599, +STORE, 140622463897600, 140622530871295, +SNULL, 140622195462143, 140622262435839, +STORE, 140622195326976, 140622195462143, +STORE, 140622195462144, 140622262435839, +STORE, 140623386501120, 140623394893823, +SNULL, 140622941884416, 140622950277119, +STORE, 140622950277120, 140622975455231, +STORE, 140622941884416, 140622950277119, +SNULL, 140622950281215, 140622975455231, +STORE, 140622950277120, 140622950281215, +STORE, 140622950281216, 140622975455231, +SNULL, 140622941888511, 140622950277119, +STORE, 140622941884416, 140622941888511, +STORE, 140622941888512, 140622950277119, +STORE, 140623378108416, 140623394893823, +SNULL, 140623478755328, 140623495540735, +STORE, 140623495540736, 140623503933439, +STORE, 140623478755328, 140623495540735, +SNULL, 140623495544831, 140623503933439, +STORE, 140623495540736, 140623495544831, +STORE, 140623495544832, 140623503933439, +SNULL, 140623478755328, 140623487148031, +STORE, 140623487148032, 140623495540735, +STORE, 140623478755328, 140623487148031, +SNULL, 140623487152127, 140623495540735, +STORE, 140623487148032, 140623487152127, +STORE, 140623487152128, 140623495540735, +SNULL, 140623218716671, 140623227105279, +STORE, 140623218712576, 140623218716671, +STORE, 140623218716672, 140623227105279, +SNULL, 140623076106239, 140623084494847, +STORE, 140623076102144, 140623076106239, +STORE, 140623076106240, 140623084494847, +SNULL, 140622329679871, 140622396653567, +STORE, 140622329544704, 140622329679871, +STORE, 140622329679872, 140622396653567, +SNULL, 140622950281216, 140622958669823, +STORE, 140622958669824, 140622975455231, +STORE, 140622950281216, 140622958669823, +SNULL, 140622958673919, 140622975455231, +STORE, 140622958669824, 140622958673919, +STORE, 140622958673920, 140622975455231, +SNULL, 140623503937536, 140623512326143, +STORE, 140623512326144, 140623520718847, +STORE, 140623503937536, 140623512326143, +SNULL, 140623512330239, 140623520718847, +STORE, 140623512326144, 140623512330239, +STORE, 140623512330240, 140623520718847, +SNULL, 140623378108416, 140623386501119, +STORE, 140623386501120, 140623394893823, +STORE, 140623378108416, 140623386501119, +SNULL, 140623386505215, 140623394893823, +STORE, 140623386501120, 140623386505215, +STORE, 140623386505216, 140623394893823, +STORE, 140623369715712, 140623386501119, +STORE, 140623361323008, 140623386501119, +STORE, 140623352930304, 140623386501119, +SNULL, 140623352930304, 140623361323007, +STORE, 140623361323008, 140623386501119, +STORE, 140623352930304, 140623361323007, +SNULL, 140623361327103, 140623386501119, +STORE, 140623361323008, 140623361327103, +STORE, 140623361327104, 140623386501119, +SNULL, 140623478759423, 140623487148031, +STORE, 140623478755328, 140623478759423, +STORE, 140623478759424, 140623487148031, +STORE, 140623344537600, 140623361323007, +STORE, 140623260676096, 140623269068799, +SNULL, 140622958673920, 140622967062527, +STORE, 140622967062528, 140622975455231, +STORE, 140622958673920, 140622967062527, +SNULL, 140622967066623, 140622975455231, +STORE, 140622967062528, 140622967066623, +STORE, 140622967066624, 140622975455231, +STORE, 140623252283392, 140623269068799, +STORE, 140623243890688, 140623269068799, +SNULL, 140622983852032, 140622992240639, +STORE, 140622992240640, 140623000633343, +STORE, 140622983852032, 140622992240639, +SNULL, 140622992244735, 140623000633343, +STORE, 140622992240640, 140622992244735, +STORE, 140622992244736, 140623000633343, +STORE, 140623235497984, 140623269068799, +STORE, 140623218716672, 140623235497983, +STORE, 140623210319872, 140623218712575, +STORE, 140623126458368, 140623134851071, +SNULL, 140623210323967, 140623218712575, +STORE, 140623210319872, 140623210323967, +STORE, 140623210323968, 140623218712575, +SNULL, 140623218716672, 140623227105279, +STORE, 140623227105280, 140623235497983, +STORE, 140623218716672, 140623227105279, +SNULL, 140623227109375, 140623235497983, +STORE, 140623227105280, 140623227109375, +STORE, 140623227109376, 140623235497983, +STORE, 140623118065664, 140623134851071, +STORE, 140623109672960, 140623134851071, +SNULL, 140623109677055, 140623134851071, +STORE, 140623109672960, 140623109677055, +STORE, 140623109677056, 140623134851071, +STORE, 140623101280256, 140623109672959, +STORE, 140623092887552, 140623109672959, +SNULL, 140623092887552, 140623101280255, +STORE, 140623101280256, 140623109672959, +STORE, 140623092887552, 140623101280255, +SNULL, 140623101284351, 140623109672959, +STORE, 140623101280256, 140623101284351, +STORE, 140623101284352, 140623109672959, +SNULL, 140623361327104, 140623378108415, +STORE, 140623378108416, 140623386501119, +STORE, 140623361327104, 140623378108415, +SNULL, 140623378112511, 140623386501119, +STORE, 140623378108416, 140623378112511, +STORE, 140623378112512, 140623386501119, +SNULL, 140623235497984, 140623243890687, +STORE, 140623243890688, 140623269068799, +STORE, 140623235497984, 140623243890687, +SNULL, 140623243894783, 140623269068799, +STORE, 140623243890688, 140623243894783, +STORE, 140623243894784, 140623269068799, +SNULL, 140623361327104, 140623369715711, +STORE, 140623369715712, 140623378108415, +STORE, 140623361327104, 140623369715711, +SNULL, 140623369719807, 140623378108415, +STORE, 140623369715712, 140623369719807, +STORE, 140623369719808, 140623378108415, +SNULL, 140623243894784, 140623252283391, +STORE, 140623252283392, 140623269068799, +STORE, 140623243894784, 140623252283391, +SNULL, 140623252287487, 140623269068799, +STORE, 140623252283392, 140623252287487, +STORE, 140623252287488, 140623269068799, +SNULL, 140623235502079, 140623243890687, +STORE, 140623235497984, 140623235502079, +STORE, 140623235502080, 140623243890687, +SNULL, 140623344541695, 140623361323007, +STORE, 140623344537600, 140623344541695, +STORE, 140623344541696, 140623361323007, +STORE, 140623076106240, 140623092887551, +SNULL, 140623076106240, 140623084494847, +STORE, 140623084494848, 140623092887551, +STORE, 140623076106240, 140623084494847, +SNULL, 140623084498943, 140623092887551, +STORE, 140623084494848, 140623084498943, +STORE, 140623084498944, 140623092887551, +SNULL, 140623344541696, 140623352930303, +STORE, 140623352930304, 140623361323007, +STORE, 140623344541696, 140623352930303, +SNULL, 140623352934399, 140623361323007, +STORE, 140623352930304, 140623352934399, +STORE, 140623352934400, 140623361323007, +SNULL, 140623109677056, 140623118065663, +STORE, 140623118065664, 140623134851071, +STORE, 140623109677056, 140623118065663, +SNULL, 140623118069759, 140623134851071, +STORE, 140623118065664, 140623118069759, +STORE, 140623118069760, 140623134851071, +STORE, 140622832844800, 140622841237503, +STORE, 140622824452096, 140622841237503, +SNULL, 140622824452096, 140622832844799, +STORE, 140622832844800, 140622841237503, +STORE, 140622824452096, 140622832844799, +SNULL, 140622832848895, 140622841237503, +STORE, 140622832844800, 140622832848895, +STORE, 140622832848896, 140622841237503, +STORE, 140622816059392, 140622832844799, +SNULL, 140623092891647, 140623101280255, +STORE, 140623092887552, 140623092891647, +STORE, 140623092891648, 140623101280255, +SNULL, 140623118069760, 140623126458367, +STORE, 140623126458368, 140623134851071, +STORE, 140623118069760, 140623126458367, +SNULL, 140623126462463, 140623134851071, +STORE, 140623126458368, 140623126462463, +STORE, 140623126462464, 140623134851071, +SNULL, 140623252287488, 140623260676095, +STORE, 140623260676096, 140623269068799, +STORE, 140623252287488, 140623260676095, +SNULL, 140623260680191, 140623269068799, +STORE, 140623260676096, 140623260680191, +STORE, 140623260680192, 140623269068799, +STORE, 140622807666688, 140622832844799, +STORE, 140622723805184, 140622732197887, +STORE, 140622715412480, 140622732197887, +STORE, 140622707019776, 140622732197887, +SNULL, 140622707023871, 140622732197887, +STORE, 140622707019776, 140622707023871, +STORE, 140622707023872, 140622732197887, +STORE, 140622698627072, 140622707019775, +STORE, 140622690234368, 140622707019775, +SNULL, 140622690238463, 140622707019775, +STORE, 140622690234368, 140622690238463, +STORE, 140622690238464, 140622707019775, +SNULL, 140622807666688, 140622816059391, +STORE, 140622816059392, 140622832844799, +STORE, 140622807666688, 140622816059391, +SNULL, 140622816063487, 140622832844799, +STORE, 140622816059392, 140622816063487, +STORE, 140622816063488, 140622832844799, +STORE, 140622681841664, 140622690234367, +STORE, 140622673448960, 140622690234367, +SNULL, 140622673453055, 140622690234367, +STORE, 140622673448960, 140622673453055, +STORE, 140622673453056, 140622690234367, +STORE, 140622589587456, 140622597980159, +SNULL, 140622807670783, 140622816059391, +STORE, 140622807666688, 140622807670783, +STORE, 140622807670784, 140622816059391, +STORE, 140622581194752, 140622597980159, +SNULL, 140622581198847, 140622597980159, +STORE, 140622581194752, 140622581198847, +STORE, 140622581198848, 140622597980159, +SNULL, 140622816063488, 140622824452095, +STORE, 140622824452096, 140622832844799, +STORE, 140622816063488, 140622824452095, +SNULL, 140622824456191, 140622832844799, +STORE, 140622824452096, 140622824456191, +STORE, 140622824456192, 140622832844799, +STORE, 140622572802048, 140622581194751, +SNULL, 140622572806143, 140622581194751, +STORE, 140622572802048, 140622572806143, +STORE, 140622572806144, 140622581194751, +STORE, 140622564409344, 140622572802047, +STORE, 140622556016640, 140622572802047, +SNULL, 140622556016640, 140622564409343, +STORE, 140622564409344, 140622572802047, +STORE, 140622556016640, 140622564409343, +SNULL, 140622564413439, 140622572802047, +STORE, 140622564409344, 140622564413439, +STORE, 140622564413440, 140622572802047, +SNULL, 140622690238464, 140622698627071, +STORE, 140622698627072, 140622707019775, +STORE, 140622690238464, 140622698627071, +SNULL, 140622698631167, 140622707019775, +STORE, 140622698627072, 140622698631167, +STORE, 140622698631168, 140622707019775, +SNULL, 140622707023872, 140622723805183, +STORE, 140622723805184, 140622732197887, +STORE, 140622707023872, 140622723805183, +SNULL, 140622723809279, 140622732197887, +STORE, 140622723805184, 140622723809279, +STORE, 140622723809280, 140622732197887, +SNULL, 140622707023872, 140622715412479, +STORE, 140622715412480, 140622723805183, +STORE, 140622707023872, 140622715412479, +SNULL, 140622715416575, 140622723805183, +STORE, 140622715412480, 140622715416575, +STORE, 140622715416576, 140622723805183, +STORE, 140622547623936, 140622564409343, +SNULL, 140622547628031, 140622564409343, +STORE, 140622547623936, 140622547628031, +STORE, 140622547628032, 140622564409343, +STORE, 140622539231232, 140622547623935, +SNULL, 140622539235327, 140622547623935, +STORE, 140622539231232, 140622539235327, +STORE, 140622539235328, 140622547623935, +SNULL, 140622581198848, 140622589587455, +STORE, 140622589587456, 140622597980159, +STORE, 140622581198848, 140622589587455, +SNULL, 140622589591551, 140622597980159, +STORE, 140622589587456, 140622589591551, +STORE, 140622589591552, 140622597980159, +STORE, 140622455369728, 140622463762431, +SNULL, 140622455373823, 140622463762431, +STORE, 140622455369728, 140622455373823, +STORE, 140622455373824, 140622463762431, +STORE, 140622446977024, 140622455369727, +SNULL, 140622446981119, 140622455369727, +STORE, 140622446977024, 140622446981119, +STORE, 140622446981120, 140622455369727, +SNULL, 140622547628032, 140622556016639, +STORE, 140622556016640, 140622564409343, +STORE, 140622547628032, 140622556016639, +SNULL, 140622556020735, 140622564409343, +STORE, 140622556016640, 140622556020735, +STORE, 140622556020736, 140622564409343, +STORE, 140622430195712, 140622446977023, +STORE, 140622421798912, 140622430191615, +SNULL, 140622430195712, 140622438584319, +STORE, 140622438584320, 140622446977023, +STORE, 140622430195712, 140622438584319, +SNULL, 140622438588415, 140622446977023, +STORE, 140622438584320, 140622438588415, +STORE, 140622438588416, 140622446977023, +STORE, 140622413406208, 140622430191615, +STORE, 140622405013504, 140622430191615, +SNULL, 140622405013504, 140622413406207, +STORE, 140622413406208, 140622430191615, +STORE, 140622405013504, 140622413406207, +SNULL, 140622413410303, 140622430191615, +STORE, 140622413406208, 140622413410303, +STORE, 140622413410304, 140622430191615, +SNULL, 140622673453056, 140622681841663, +STORE, 140622681841664, 140622690234367, +STORE, 140622673453056, 140622681841663, +SNULL, 140622681845759, 140622690234367, +STORE, 140622681841664, 140622681845759, +STORE, 140622681845760, 140622690234367, +STORE, 140622321152000, 140622329544703, +SNULL, 140622413410304, 140622421798911, +STORE, 140622421798912, 140622430191615, +STORE, 140622413410304, 140622421798911, +SNULL, 140622421803007, 140622430191615, +STORE, 140622421798912, 140622421803007, +STORE, 140622421803008, 140622430191615, +STORE, 140622312759296, 140622329544703, +SNULL, 140622312763391, 140622329544703, +STORE, 140622312759296, 140622312763391, +STORE, 140622312763392, 140622329544703, +SNULL, 140622405017599, 140622413406207, +STORE, 140622405013504, 140622405017599, +STORE, 140622405017600, 140622413406207, +STORE, 140622304366592, 140622312759295, +SNULL, 140622304370687, 140622312759295, +STORE, 140622304366592, 140622304370687, +STORE, 140622304370688, 140622312759295, +SNULL, 140622312763392, 140622321151999, +STORE, 140622321152000, 140622329544703, +STORE, 140622312763392, 140622321151999, +SNULL, 140622321156095, 140622329544703, +STORE, 140622321152000, 140622321156095, +STORE, 140622321156096, 140622329544703, +STORE, 140624062619648, 140624062648319, +STORE, 140624010240000, 140624012431359, +SNULL, 140624010240000, 140624010330111, +STORE, 140624010330112, 140624012431359, +STORE, 140624010240000, 140624010330111, +SNULL, 140624012423167, 140624012431359, +STORE, 140624010330112, 140624012423167, +STORE, 140624012423168, 140624012431359, +ERASE, 140624012423168, 140624012431359, +STORE, 140624012423168, 140624012431359, +SNULL, 140624012427263, 140624012431359, +STORE, 140624012423168, 140624012427263, +STORE, 140624012427264, 140624012431359, +ERASE, 140624062619648, 140624062648319, +ERASE, 140622849630208, 140622849634303, +ERASE, 140622849634304, 140622858022911, +ERASE, 140623394893824, 140623394897919, +ERASE, 140623394897920, 140623403286527, +ERASE, 140623361323008, 140623361327103, +ERASE, 140623361327104, 140623369715711, +ERASE, 140623084494848, 140623084498943, +ERASE, 140623084498944, 140623092887551, +ERASE, 140623931764736, 140623931768831, +ERASE, 140623931768832, 140623940157439, +ERASE, 140622841237504, 140622841241599, +ERASE, 140622841241600, 140622849630207, +ERASE, 140623487148032, 140623487152127, +ERASE, 140623487152128, 140623495540735, +ERASE, 140623109672960, 140623109677055, +ERASE, 140623109677056, 140623118065663, +ERASE, 140622983847936, 140622983852031, +ERASE, 140622983852032, 140622992240639, +ERASE, 140623352930304, 140623352934399, +ERASE, 140623352934400, 140623361323007, +ERASE, 140622564409344, 140622564413439, +ERASE, 140622564413440, 140622572802047, +ERASE, 140622430191616, 140622430195711, +ERASE, 140622430195712, 140622438584319, +ERASE, 140622958669824, 140622958673919, +ERASE, 140622958673920, 140622967062527, +ERASE, 140622992240640, 140622992244735, +ERASE, 140622992244736, 140623000633343, +ERASE, 140623227105280, 140623227109375, +ERASE, 140623227109376, 140623235497983, +ERASE, 140622321152000, 140622321156095, +ERASE, 140622321156096, 140622329544703, +ERASE, 140622858022912, 140622858027007, +ERASE, 140622858027008, 140622866415615, +ERASE, 140622975455232, 140622975459327, +ERASE, 140622975459328, 140622983847935, +ERASE, 140623378108416, 140623378112511, +ERASE, 140623378112512, 140623386501119, +ERASE, 140623495540736, 140623495544831, +ERASE, 140623495544832, 140623503933439, +ERASE, 140623118065664, 140623118069759, +ERASE, 140623118069760, 140623126458367, +ERASE, 140622572802048, 140622572806143, +ERASE, 140622572806144, 140622581194751, +ERASE, 140622421798912, 140622421803007, +ERASE, 140622421803008, 140622430191615, +ERASE, 140622967062528, 140622967066623, +ERASE, 140622967066624, 140622975455231, +ERASE, 140623252283392, 140623252287487, +ERASE, 140623252287488, 140623260676095, +ERASE, 140622673448960, 140622673453055, +ERASE, 140622673453056, 140622681841663, +ERASE, 140623076102144, 140623076106239, +ERASE, 140623076106240, 140623084494847, +ERASE, 140623101280256, 140623101284351, +ERASE, 140623101284352, 140623109672959, +ERASE, 140622715412480, 140622715416575, +ERASE, 140622715416576, 140622723805183, +ERASE, 140622405013504, 140622405017599, +ERASE, 140622405017600, 140622413406207, +ERASE, 140623478755328, 140623478759423, +ERASE, 140623478759424, 140623487148031, +ERASE, 140623906586624, 140623906590719, +ERASE, 140623906590720, 140623914979327, +ERASE, 140622950277120, 140622950281215, +ERASE, 140622950281216, 140622958669823, + }; + unsigned long set32[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140731244212224, 140737488351231, +SNULL, 140731244216319, 140737488351231, +STORE, 140731244212224, 140731244216319, +STORE, 140731244081152, 140731244216319, +STORE, 94427773984768, 94427776237567, +SNULL, 94427774115839, 94427776237567, +STORE, 94427773984768, 94427774115839, +STORE, 94427774115840, 94427776237567, +ERASE, 94427774115840, 94427776237567, +STORE, 94427776208896, 94427776217087, +STORE, 94427776217088, 94427776237567, +STORE, 140401464893440, 140401467146239, +SNULL, 140401465036799, 140401467146239, +STORE, 140401464893440, 140401465036799, +STORE, 140401465036800, 140401467146239, +ERASE, 140401465036800, 140401467146239, +STORE, 140401467133952, 140401467142143, +STORE, 140401467142144, 140401467146239, +STORE, 140731244507136, 140731244511231, +STORE, 140731244494848, 140731244507135, +STORE, 140401467105280, 140401467133951, +STORE, 140401467097088, 140401467105279, +STORE, 140401462677504, 140401464893439, +SNULL, 140401462677504, 140401462775807, +STORE, 140401462775808, 140401464893439, +STORE, 140401462677504, 140401462775807, +SNULL, 140401464868863, 140401464893439, +STORE, 140401462775808, 140401464868863, +STORE, 140401464868864, 140401464893439, +SNULL, 140401464868864, 140401464877055, +STORE, 140401464877056, 140401464893439, +STORE, 140401464868864, 140401464877055, +ERASE, 140401464868864, 140401464877055, +STORE, 140401464868864, 140401464877055, +ERASE, 140401464877056, 140401464893439, +STORE, 140401464877056, 140401464893439, +STORE, 140401458880512, 140401462677503, +SNULL, 140401458880512, 140401460539391, +STORE, 140401460539392, 140401462677503, +STORE, 140401458880512, 140401460539391, +SNULL, 140401462636543, 140401462677503, +STORE, 140401460539392, 140401462636543, +STORE, 140401462636544, 140401462677503, +SNULL, 140401462636544, 140401462661119, +STORE, 140401462661120, 140401462677503, +STORE, 140401462636544, 140401462661119, +ERASE, 140401462636544, 140401462661119, +STORE, 140401462636544, 140401462661119, +ERASE, 140401462661120, 140401462677503, +STORE, 140401462661120, 140401462677503, +STORE, 140401467088896, 140401467105279, +SNULL, 140401462652927, 140401462661119, +STORE, 140401462636544, 140401462652927, +STORE, 140401462652928, 140401462661119, +SNULL, 140401464872959, 140401464877055, +STORE, 140401464868864, 140401464872959, +STORE, 140401464872960, 140401464877055, +SNULL, 94427776212991, 94427776217087, +STORE, 94427776208896, 94427776212991, +STORE, 94427776212992, 94427776217087, +SNULL, 140401467138047, 140401467142143, +STORE, 140401467133952, 140401467138047, +STORE, 140401467138048, 140401467142143, +ERASE, 140401467105280, 140401467133951, +STORE, 94427784683520, 94427784818687, +STORE, 140401450487808, 140401458880511, +SNULL, 140401450491903, 140401458880511, +STORE, 140401450487808, 140401450491903, +STORE, 140401450491904, 140401458880511, +STORE, 140401442095104, 140401450487807, +STORE, 140401307877376, 140401442095103, +SNULL, 140401307877376, 140401340055551, +STORE, 140401340055552, 140401442095103, +STORE, 140401307877376, 140401340055551, +ERASE, 140401307877376, 140401340055551, +SNULL, 140401407164415, 140401442095103, +STORE, 140401340055552, 140401407164415, +STORE, 140401407164416, 140401442095103, +ERASE, 140401407164416, 140401442095103, +SNULL, 140401340190719, 140401407164415, +STORE, 140401340055552, 140401340190719, +STORE, 140401340190720, 140401407164415, +SNULL, 140401442099199, 140401450487807, +STORE, 140401442095104, 140401442099199, +STORE, 140401442099200, 140401450487807, +STORE, 140401433702400, 140401442095103, +SNULL, 140401433706495, 140401442095103, +STORE, 140401433702400, 140401433706495, +STORE, 140401433706496, 140401442095103, +STORE, 140401425309696, 140401433702399, +SNULL, 140401425313791, 140401433702399, +STORE, 140401425309696, 140401425313791, +STORE, 140401425313792, 140401433702399, +STORE, 140401416916992, 140401425309695, +SNULL, 140401416921087, 140401425309695, +STORE, 140401416916992, 140401416921087, +STORE, 140401416921088, 140401425309695, +STORE, 140401408524288, 140401416916991, +STORE, 140401205837824, 140401340055551, +SNULL, 140401272946687, 140401340055551, +STORE, 140401205837824, 140401272946687, +STORE, 140401272946688, 140401340055551, +ERASE, 140401272946688, 140401340055551, +SNULL, 140401205972991, 140401272946687, +STORE, 140401205837824, 140401205972991, +STORE, 140401205972992, 140401272946687, +STORE, 140401331662848, 140401340055551, +STORE, 140401323270144, 140401340055551, +STORE, 140401138728960, 140401205837823, +STORE, 140401314877440, 140401340055551, +SNULL, 140401408528383, 140401416916991, +STORE, 140401408524288, 140401408528383, +STORE, 140401408528384, 140401416916991, +SNULL, 140401138864127, 140401205837823, +STORE, 140401138728960, 140401138864127, +STORE, 140401138864128, 140401205837823, +STORE, 140401004511232, 140401138728959, +SNULL, 140401071620095, 140401138728959, +STORE, 140401004511232, 140401071620095, +STORE, 140401071620096, 140401138728959, +ERASE, 140401071620096, 140401138728959, +STORE, 140400870293504, 140401071620095, +SNULL, 140400937402367, 140401071620095, +STORE, 140400870293504, 140400937402367, +STORE, 140400937402368, 140401071620095, +SNULL, 140400937402368, 140401004511231, +STORE, 140401004511232, 140401071620095, +STORE, 140400937402368, 140401004511231, +ERASE, 140400937402368, 140401004511231, +STORE, 140401306484736, 140401340055551, +SNULL, 140401306484736, 140401323270143, +STORE, 140401323270144, 140401340055551, +STORE, 140401306484736, 140401323270143, +SNULL, 140401323274239, 140401340055551, +STORE, 140401323270144, 140401323274239, +STORE, 140401323274240, 140401340055551, +SNULL, 140401004646399, 140401071620095, +STORE, 140401004511232, 140401004646399, +STORE, 140401004646400, 140401071620095, +SNULL, 140400870428671, 140400937402367, +STORE, 140400870293504, 140400870428671, +STORE, 140400870428672, 140400937402367, +SNULL, 140401306488831, 140401323270143, +STORE, 140401306484736, 140401306488831, +STORE, 140401306488832, 140401323270143, +STORE, 140401298092032, 140401306484735, +SNULL, 140401306488832, 140401314877439, +STORE, 140401314877440, 140401323270143, +STORE, 140401306488832, 140401314877439, +SNULL, 140401314881535, 140401323270143, +STORE, 140401314877440, 140401314881535, +STORE, 140401314881536, 140401323270143, +SNULL, 140401323274240, 140401331662847, +STORE, 140401331662848, 140401340055551, +STORE, 140401323274240, 140401331662847, +SNULL, 140401331666943, 140401340055551, +STORE, 140401331662848, 140401331666943, +STORE, 140401331666944, 140401340055551, +SNULL, 140401298096127, 140401306484735, +STORE, 140401298092032, 140401298096127, +STORE, 140401298096128, 140401306484735, +STORE, 140401289699328, 140401298092031, +STORE, 140401281306624, 140401298092031, +STORE, 140401130336256, 140401138728959, +SNULL, 140401281306624, 140401289699327, +STORE, 140401289699328, 140401298092031, +STORE, 140401281306624, 140401289699327, +SNULL, 140401289703423, 140401298092031, +STORE, 140401289699328, 140401289703423, +STORE, 140401289703424, 140401298092031, +STORE, 140401121943552, 140401138728959, +STORE, 140401113550848, 140401138728959, +SNULL, 140401281310719, 140401289699327, +STORE, 140401281306624, 140401281310719, +STORE, 140401281310720, 140401289699327, +SNULL, 140401113550848, 140401121943551, +STORE, 140401121943552, 140401138728959, +STORE, 140401113550848, 140401121943551, +SNULL, 140401121947647, 140401138728959, +STORE, 140401121943552, 140401121947647, +STORE, 140401121947648, 140401138728959, +STORE, 140401105158144, 140401121943551, +SNULL, 140401121947648, 140401130336255, +STORE, 140401130336256, 140401138728959, +STORE, 140401121947648, 140401130336255, +SNULL, 140401130340351, 140401138728959, +STORE, 140401130336256, 140401130340351, +STORE, 140401130340352, 140401138728959, +STORE, 140401096765440, 140401121943551, +SNULL, 140401096765440, 140401113550847, +STORE, 140401113550848, 140401121943551, +STORE, 140401096765440, 140401113550847, +SNULL, 140401113554943, 140401121943551, +STORE, 140401113550848, 140401113554943, +STORE, 140401113554944, 140401121943551, +STORE, 140401088372736, 140401113550847, +SNULL, 140401088372736, 140401096765439, +STORE, 140401096765440, 140401113550847, +STORE, 140401088372736, 140401096765439, +SNULL, 140401096769535, 140401113550847, +STORE, 140401096765440, 140401096769535, +STORE, 140401096769536, 140401113550847, +SNULL, 140401096769536, 140401105158143, +STORE, 140401105158144, 140401113550847, +STORE, 140401096769536, 140401105158143, +SNULL, 140401105162239, 140401113550847, +STORE, 140401105158144, 140401105162239, +STORE, 140401105162240, 140401113550847, +SNULL, 140401088376831, 140401096765439, +STORE, 140401088372736, 140401088376831, +STORE, 140401088376832, 140401096765439, +STORE, 140401079980032, 140401088372735, +STORE, 140400996118528, 140401004511231, +SNULL, 140401079984127, 140401088372735, +STORE, 140401079980032, 140401079984127, +STORE, 140401079984128, 140401088372735, +SNULL, 140400996122623, 140401004511231, +STORE, 140400996118528, 140400996122623, +STORE, 140400996122624, 140401004511231, +STORE, 140400987725824, 140400996118527, +STORE, 140400979333120, 140400996118527, +STORE, 140400803184640, 140400870293503, +SNULL, 140400803319807, 140400870293503, +STORE, 140400803184640, 140400803319807, +STORE, 140400803319808, 140400870293503, +SNULL, 140400979333120, 140400987725823, +STORE, 140400987725824, 140400996118527, +STORE, 140400979333120, 140400987725823, +SNULL, 140400987729919, 140400996118527, +STORE, 140400987725824, 140400987729919, +STORE, 140400987729920, 140400996118527, +STORE, 140400970940416, 140400987725823, +STORE, 140400962547712, 140400987725823, +STORE, 140400668966912, 140400803184639, +STORE, 140400954155008, 140400987725823, +STORE, 140400945762304, 140400987725823, +STORE, 140400660574208, 140400668966911, +STORE, 140400593465344, 140400660574207, +STORE, 140400585072640, 140400593465343, +STORE, 140400450854912, 140400585072639, +STORE, 140400442462208, 140400450854911, +STORE, 140400434069504, 140400450854911, +STORE, 140400299851776, 140400434069503, +STORE, 140400291459072, 140400299851775, +SNULL, 140400299851776, 140400333422591, +STORE, 140400333422592, 140400434069503, +STORE, 140400299851776, 140400333422591, +ERASE, 140400299851776, 140400333422591, +STORE, 140400325029888, 140400333422591, +STORE, 140400157241344, 140400291459071, +STORE, 140400316637184, 140400333422591, +STORE, 140400308244480, 140400333422591, +STORE, 140400023023616, 140400291459071, +STORE, 140400291459072, 140400333422591, +SNULL, 140400023023616, 140400064987135, +STORE, 140400064987136, 140400291459071, +STORE, 140400023023616, 140400064987135, +ERASE, 140400023023616, 140400064987135, +STORE, 140400056594432, 140400064987135, +SNULL, 140400056598527, 140400064987135, +STORE, 140400056594432, 140400056598527, +STORE, 140400056598528, 140400064987135, +STORE, 140399989485568, 140400056594431, +SNULL, 140400291459072, 140400316637183, +STORE, 140400316637184, 140400333422591, +STORE, 140400291459072, 140400316637183, +SNULL, 140400316641279, 140400333422591, +STORE, 140400316637184, 140400316641279, +STORE, 140400316641280, 140400333422591, +STORE, 140399855267840, 140400056594431, +SNULL, 140399855267840, 140399863660543, +STORE, 140399863660544, 140400056594431, +STORE, 140399855267840, 140399863660543, +ERASE, 140399855267840, 140399863660543, +SNULL, 140400736075775, 140400803184639, +STORE, 140400668966912, 140400736075775, +STORE, 140400736075776, 140400803184639, +ERASE, 140400736075776, 140400803184639, +SNULL, 140400669102079, 140400736075775, +STORE, 140400668966912, 140400669102079, +STORE, 140400669102080, 140400736075775, +STORE, 140400669102080, 140400803184639, +SNULL, 140400669102080, 140400736075775, +STORE, 140400736075776, 140400803184639, +STORE, 140400669102080, 140400736075775, +SNULL, 140400736210943, 140400803184639, +STORE, 140400736075776, 140400736210943, +STORE, 140400736210944, 140400803184639, +ERASE, 140400593465344, 140400660574207, +SNULL, 140400450854912, 140400467640319, +STORE, 140400467640320, 140400585072639, +STORE, 140400450854912, 140400467640319, +ERASE, 140400450854912, 140400467640319, +STORE, 140399729442816, 140400056594431, +SNULL, 140400400531455, 140400434069503, +STORE, 140400333422592, 140400400531455, +STORE, 140400400531456, 140400434069503, +ERASE, 140400400531456, 140400434069503, +SNULL, 140400333557759, 140400400531455, +STORE, 140400333422592, 140400333557759, +STORE, 140400333557760, 140400400531455, +SNULL, 140400157241343, 140400291459071, +STORE, 140400064987136, 140400157241343, +STORE, 140400157241344, 140400291459071, +SNULL, 140400157241344, 140400199204863, +STORE, 140400199204864, 140400291459071, +STORE, 140400157241344, 140400199204863, +ERASE, 140400157241344, 140400199204863, +SNULL, 140400266313727, 140400291459071, +STORE, 140400199204864, 140400266313727, +STORE, 140400266313728, 140400291459071, +ERASE, 140400266313728, 140400291459071, +SNULL, 140400132095999, 140400157241343, +STORE, 140400064987136, 140400132095999, +STORE, 140400132096000, 140400157241343, +ERASE, 140400132096000, 140400157241343, +SNULL, 140400065122303, 140400132095999, +STORE, 140400064987136, 140400065122303, +STORE, 140400065122304, 140400132095999, +SNULL, 140400945762304, 140400954155007, +STORE, 140400954155008, 140400987725823, +STORE, 140400945762304, 140400954155007, +SNULL, 140400954159103, 140400987725823, +STORE, 140400954155008, 140400954159103, +STORE, 140400954159104, 140400987725823, +SNULL, 140400434069504, 140400442462207, +STORE, 140400442462208, 140400450854911, +STORE, 140400434069504, 140400442462207, +SNULL, 140400442466303, 140400450854911, +STORE, 140400442462208, 140400442466303, +STORE, 140400442466304, 140400450854911, +SNULL, 140400291463167, 140400316637183, +STORE, 140400291459072, 140400291463167, +STORE, 140400291463168, 140400316637183, +STORE, 140400652181504, 140400668966911, +STORE, 140400643788800, 140400668966911, +SNULL, 140400291463168, 140400299851775, +STORE, 140400299851776, 140400316637183, +STORE, 140400291463168, 140400299851775, +SNULL, 140400299855871, 140400316637183, +STORE, 140400299851776, 140400299855871, +STORE, 140400299855872, 140400316637183, +STORE, 140400635396096, 140400668966911, +SNULL, 140400635396096, 140400643788799, +STORE, 140400643788800, 140400668966911, +STORE, 140400635396096, 140400643788799, +SNULL, 140400643792895, 140400668966911, +STORE, 140400643788800, 140400643792895, +STORE, 140400643792896, 140400668966911, +SNULL, 140399989485567, 140400056594431, +STORE, 140399729442816, 140399989485567, +STORE, 140399989485568, 140400056594431, +ERASE, 140399989485568, 140400056594431, +SNULL, 140399930769407, 140399989485567, +STORE, 140399729442816, 140399930769407, +STORE, 140399930769408, 140399989485567, +ERASE, 140399930769408, 140399989485567, +SNULL, 140400945766399, 140400954155007, +STORE, 140400945762304, 140400945766399, +STORE, 140400945766400, 140400954155007, +SNULL, 140400534749183, 140400585072639, +STORE, 140400467640320, 140400534749183, +STORE, 140400534749184, 140400585072639, +ERASE, 140400534749184, 140400585072639, +SNULL, 140399796551679, 140399930769407, +STORE, 140399729442816, 140399796551679, +STORE, 140399796551680, 140399930769407, +SNULL, 140399796551680, 140399863660543, +STORE, 140399863660544, 140399930769407, +STORE, 140399796551680, 140399863660543, +ERASE, 140399796551680, 140399863660543, +SNULL, 140400199340031, 140400266313727, +STORE, 140400199204864, 140400199340031, +STORE, 140400199340032, 140400266313727, +STORE, 140400627003392, 140400643788799, +SNULL, 140400316641280, 140400325029887, +STORE, 140400325029888, 140400333422591, +STORE, 140400316641280, 140400325029887, +SNULL, 140400325033983, 140400333422591, +STORE, 140400325029888, 140400325033983, +STORE, 140400325033984, 140400333422591, +SNULL, 140400627003392, 140400635396095, +STORE, 140400635396096, 140400643788799, +STORE, 140400627003392, 140400635396095, +SNULL, 140400635400191, 140400643788799, +STORE, 140400635396096, 140400635400191, +STORE, 140400635400192, 140400643788799, +SNULL, 140400434073599, 140400442462207, +STORE, 140400434069504, 140400434073599, +STORE, 140400434073600, 140400442462207, +STORE, 140400618610688, 140400635396095, +STORE, 140400610217984, 140400635396095, +SNULL, 140400954159104, 140400962547711, +STORE, 140400962547712, 140400987725823, +STORE, 140400954159104, 140400962547711, +SNULL, 140400962551807, 140400987725823, +STORE, 140400962547712, 140400962551807, +STORE, 140400962551808, 140400987725823, +SNULL, 140400299855872, 140400308244479, +STORE, 140400308244480, 140400316637183, +STORE, 140400299855872, 140400308244479, +SNULL, 140400308248575, 140400316637183, +STORE, 140400308244480, 140400308248575, +STORE, 140400308248576, 140400316637183, +STORE, 140400601825280, 140400635396095, +SNULL, 140400601829375, 140400635396095, +STORE, 140400601825280, 140400601829375, +STORE, 140400601829376, 140400635396095, +STORE, 140400576679936, 140400593465343, +SNULL, 140400576684031, 140400593465343, +STORE, 140400576679936, 140400576684031, +STORE, 140400576684032, 140400593465343, +SNULL, 140400643792896, 140400652181503, +STORE, 140400652181504, 140400668966911, +STORE, 140400643792896, 140400652181503, +SNULL, 140400652185599, 140400668966911, +STORE, 140400652181504, 140400652185599, +STORE, 140400652185600, 140400668966911, +STORE, 140399595225088, 140399796551679, +SNULL, 140399662333951, 140399796551679, +STORE, 140399595225088, 140399662333951, +STORE, 140399662333952, 140399796551679, +SNULL, 140399662333952, 140399729442815, +STORE, 140399729442816, 140399796551679, +STORE, 140399662333952, 140399729442815, +ERASE, 140399662333952, 140399729442815, +SNULL, 140399863795711, 140399930769407, +STORE, 140399863660544, 140399863795711, +STORE, 140399863795712, 140399930769407, +STORE, 140400568287232, 140400576679935, +SNULL, 140400568291327, 140400576679935, +STORE, 140400568287232, 140400568291327, +STORE, 140400568291328, 140400576679935, +SNULL, 140400467775487, 140400534749183, +STORE, 140400467640320, 140400467775487, +STORE, 140400467775488, 140400534749183, +SNULL, 140399729577983, 140399796551679, +STORE, 140399729442816, 140399729577983, +STORE, 140399729577984, 140399796551679, +SNULL, 140400601829376, 140400627003391, +STORE, 140400627003392, 140400635396095, +STORE, 140400601829376, 140400627003391, +SNULL, 140400627007487, 140400635396095, +STORE, 140400627003392, 140400627007487, +STORE, 140400627007488, 140400635396095, +STORE, 140400559894528, 140400568287231, +STORE, 140400551501824, 140400568287231, +STORE, 140400543109120, 140400568287231, +STORE, 140400459247616, 140400467640319, +STORE, 140400442466304, 140400467640319, +SNULL, 140399595360255, 140399662333951, +STORE, 140399595225088, 140399595360255, +STORE, 140399595360256, 140399662333951, +SNULL, 140400962551808, 140400970940415, +STORE, 140400970940416, 140400987725823, +STORE, 140400962551808, 140400970940415, +SNULL, 140400970944511, 140400987725823, +STORE, 140400970940416, 140400970944511, +STORE, 140400970944512, 140400987725823, +SNULL, 140400652185600, 140400660574207, +STORE, 140400660574208, 140400668966911, +STORE, 140400652185600, 140400660574207, +SNULL, 140400660578303, 140400668966911, +STORE, 140400660574208, 140400660578303, +STORE, 140400660578304, 140400668966911, +SNULL, 140400576684032, 140400585072639, +STORE, 140400585072640, 140400593465343, +STORE, 140400576684032, 140400585072639, +SNULL, 140400585076735, 140400593465343, +STORE, 140400585072640, 140400585076735, +STORE, 140400585076736, 140400593465343, +STORE, 140400425676800, 140400434069503, +STORE, 140400417284096, 140400434069503, +STORE, 140400408891392, 140400434069503, +SNULL, 140400408891392, 140400417284095, +STORE, 140400417284096, 140400434069503, +STORE, 140400408891392, 140400417284095, +SNULL, 140400417288191, 140400434069503, +STORE, 140400417284096, 140400417288191, +STORE, 140400417288192, 140400434069503, +STORE, 140400283066368, 140400291459071, +SNULL, 140400601829376, 140400618610687, +STORE, 140400618610688, 140400627003391, +STORE, 140400601829376, 140400618610687, +SNULL, 140400618614783, 140400627003391, +STORE, 140400618610688, 140400618614783, +STORE, 140400618614784, 140400627003391, +SNULL, 140400601829376, 140400610217983, +STORE, 140400610217984, 140400618610687, +STORE, 140400601829376, 140400610217983, +SNULL, 140400610222079, 140400618610687, +STORE, 140400610217984, 140400610222079, +STORE, 140400610222080, 140400618610687, +STORE, 140400274673664, 140400291459071, +STORE, 140400190812160, 140400199204863, +STORE, 140400182419456, 140400199204863, +SNULL, 140400442466304, 140400450854911, +STORE, 140400450854912, 140400467640319, +STORE, 140400442466304, 140400450854911, +SNULL, 140400450859007, 140400467640319, +STORE, 140400450854912, 140400450859007, +STORE, 140400450859008, 140400467640319, +SNULL, 140400543109120, 140400559894527, +STORE, 140400559894528, 140400568287231, +STORE, 140400543109120, 140400559894527, +SNULL, 140400559898623, 140400568287231, +STORE, 140400559894528, 140400559898623, +STORE, 140400559898624, 140400568287231, +SNULL, 140400450859008, 140400459247615, +STORE, 140400459247616, 140400467640319, +STORE, 140400450859008, 140400459247615, +SNULL, 140400459251711, 140400467640319, +STORE, 140400459247616, 140400459251711, +STORE, 140400459251712, 140400467640319, +SNULL, 140400543113215, 140400559894527, +STORE, 140400543109120, 140400543113215, +STORE, 140400543113216, 140400559894527, +SNULL, 140400970944512, 140400979333119, +STORE, 140400979333120, 140400987725823, +STORE, 140400970944512, 140400979333119, +SNULL, 140400979337215, 140400987725823, +STORE, 140400979333120, 140400979337215, +STORE, 140400979337216, 140400987725823, +STORE, 140400174026752, 140400199204863, +SNULL, 140400174030847, 140400199204863, +STORE, 140400174026752, 140400174030847, +STORE, 140400174030848, 140400199204863, +SNULL, 140400274673664, 140400283066367, +STORE, 140400283066368, 140400291459071, +STORE, 140400274673664, 140400283066367, +SNULL, 140400283070463, 140400291459071, +STORE, 140400283066368, 140400283070463, +STORE, 140400283070464, 140400291459071, +STORE, 140400165634048, 140400174026751, +SNULL, 140400165638143, 140400174026751, +STORE, 140400165634048, 140400165638143, +STORE, 140400165638144, 140400174026751, +SNULL, 140400174030848, 140400182419455, +STORE, 140400182419456, 140400199204863, +STORE, 140400174030848, 140400182419455, +SNULL, 140400182423551, 140400199204863, +STORE, 140400182419456, 140400182423551, +STORE, 140400182423552, 140400199204863, +SNULL, 140400182423552, 140400190812159, +STORE, 140400190812160, 140400199204863, +STORE, 140400182423552, 140400190812159, +SNULL, 140400190816255, 140400199204863, +STORE, 140400190812160, 140400190816255, +STORE, 140400190816256, 140400199204863, +STORE, 140400157241344, 140400165634047, +SNULL, 140400157245439, 140400165634047, +STORE, 140400157241344, 140400157245439, +STORE, 140400157245440, 140400165634047, +SNULL, 140400408895487, 140400417284095, +STORE, 140400408891392, 140400408895487, +STORE, 140400408895488, 140400417284095, +SNULL, 140400417288192, 140400425676799, +STORE, 140400425676800, 140400434069503, +STORE, 140400417288192, 140400425676799, +SNULL, 140400425680895, 140400434069503, +STORE, 140400425676800, 140400425680895, +STORE, 140400425680896, 140400434069503, +STORE, 140400148848640, 140400157241343, +SNULL, 140400148852735, 140400157241343, +STORE, 140400148848640, 140400148852735, +STORE, 140400148852736, 140400157241343, +SNULL, 140400543113216, 140400551501823, +STORE, 140400551501824, 140400559894527, +STORE, 140400543113216, 140400551501823, +SNULL, 140400551505919, 140400559894527, +STORE, 140400551501824, 140400551505919, +STORE, 140400551505920, 140400559894527, +STORE, 140400140455936, 140400148848639, +STORE, 140400048201728, 140400056594431, +SNULL, 140400140460031, 140400148848639, +STORE, 140400140455936, 140400140460031, +STORE, 140400140460032, 140400148848639, +STORE, 140400039809024, 140400056594431, +SNULL, 140400039813119, 140400056594431, +STORE, 140400039809024, 140400039813119, +STORE, 140400039813120, 140400056594431, +STORE, 140400031416320, 140400039809023, +STORE, 140400023023616, 140400039809023, +SNULL, 140400274677759, 140400283066367, +STORE, 140400274673664, 140400274677759, +STORE, 140400274677760, 140400283066367, +STORE, 140400014630912, 140400039809023, +STORE, 140400006238208, 140400039809023, +STORE, 140399997845504, 140400039809023, +SNULL, 140399997849599, 140400039809023, +STORE, 140399997845504, 140399997849599, +STORE, 140399997849600, 140400039809023, +STORE, 140399989452800, 140399997845503, +SNULL, 140399989456895, 140399997845503, +STORE, 140399989452800, 140399989456895, +STORE, 140399989456896, 140399997845503, +STORE, 140399981060096, 140399989452799, +SNULL, 140399981064191, 140399989452799, +STORE, 140399981060096, 140399981064191, +STORE, 140399981064192, 140399989452799, +STORE, 140399972667392, 140399981060095, +STORE, 140399964274688, 140399981060095, +SNULL, 140399964278783, 140399981060095, +STORE, 140399964274688, 140399964278783, +STORE, 140399964278784, 140399981060095, +SNULL, 140400039813120, 140400048201727, +STORE, 140400048201728, 140400056594431, +STORE, 140400039813120, 140400048201727, +SNULL, 140400048205823, 140400056594431, +STORE, 140400048201728, 140400048205823, +STORE, 140400048205824, 140400056594431, +SNULL, 140399997849600, 140400031416319, +STORE, 140400031416320, 140400039809023, +STORE, 140399997849600, 140400031416319, +SNULL, 140400031420415, 140400039809023, +STORE, 140400031416320, 140400031420415, +STORE, 140400031420416, 140400039809023, +STORE, 140399955881984, 140399964274687, +SNULL, 140399955886079, 140399964274687, +STORE, 140399955881984, 140399955886079, +STORE, 140399955886080, 140399964274687, +STORE, 140399947489280, 140399955881983, +STORE, 140399939096576, 140399955881983, +STORE, 140399855267840, 140399863660543, +SNULL, 140399939100671, 140399955881983, +STORE, 140399939096576, 140399939100671, +STORE, 140399939100672, 140399955881983, +SNULL, 140399997849600, 140400014630911, +STORE, 140400014630912, 140400031416319, +STORE, 140399997849600, 140400014630911, +SNULL, 140400014635007, 140400031416319, +STORE, 140400014630912, 140400014635007, +STORE, 140400014635008, 140400031416319, +SNULL, 140400014635008, 140400023023615, +STORE, 140400023023616, 140400031416319, +STORE, 140400014635008, 140400023023615, +SNULL, 140400023027711, 140400031416319, +STORE, 140400023023616, 140400023027711, +STORE, 140400023027712, 140400031416319, +SNULL, 140399997849600, 140400006238207, +STORE, 140400006238208, 140400014630911, +STORE, 140399997849600, 140400006238207, +SNULL, 140400006242303, 140400014630911, +STORE, 140400006238208, 140400006242303, +STORE, 140400006242304, 140400014630911, +STORE, 140399846875136, 140399863660543, +STORE, 140399838482432, 140399863660543, +SNULL, 140399838486527, 140399863660543, +STORE, 140399838482432, 140399838486527, +STORE, 140399838486528, 140399863660543, +SNULL, 140399939100672, 140399947489279, +STORE, 140399947489280, 140399955881983, +STORE, 140399939100672, 140399947489279, +SNULL, 140399947493375, 140399955881983, +STORE, 140399947489280, 140399947493375, +STORE, 140399947493376, 140399955881983, +SNULL, 140399964278784, 140399972667391, +STORE, 140399972667392, 140399981060095, +STORE, 140399964278784, 140399972667391, +SNULL, 140399972671487, 140399981060095, +STORE, 140399972667392, 140399972671487, +STORE, 140399972671488, 140399981060095, +SNULL, 140399838486528, 140399855267839, +STORE, 140399855267840, 140399863660543, +STORE, 140399838486528, 140399855267839, +SNULL, 140399855271935, 140399863660543, +STORE, 140399855267840, 140399855271935, +STORE, 140399855271936, 140399863660543, +STORE, 140399830089728, 140399838482431, +SNULL, 140399830093823, 140399838482431, +STORE, 140399830089728, 140399830093823, +STORE, 140399830093824, 140399838482431, +STORE, 140399821697024, 140399830089727, +SNULL, 140399821701119, 140399830089727, +STORE, 140399821697024, 140399821701119, +STORE, 140399821701120, 140399830089727, +SNULL, 140399838486528, 140399846875135, +STORE, 140399846875136, 140399855267839, +STORE, 140399838486528, 140399846875135, +SNULL, 140399846879231, 140399855267839, +STORE, 140399846875136, 140399846879231, +STORE, 140399846879232, 140399855267839, +STORE, 140399813304320, 140399821697023, +STORE, 140399804911616, 140399821697023, +SNULL, 140399804915711, 140399821697023, +STORE, 140399804911616, 140399804915711, +STORE, 140399804915712, 140399821697023, +STORE, 140399721050112, 140399729442815, +SNULL, 140399804915712, 140399813304319, +STORE, 140399813304320, 140399821697023, +STORE, 140399804915712, 140399813304319, +SNULL, 140399813308415, 140399821697023, +STORE, 140399813304320, 140399813308415, +STORE, 140399813308416, 140399821697023, +SNULL, 140399721054207, 140399729442815, +STORE, 140399721050112, 140399721054207, +STORE, 140399721054208, 140399729442815, +STORE, 140401467105280, 140401467133951, +STORE, 140401279115264, 140401281306623, +SNULL, 140401279115264, 140401279205375, +STORE, 140401279205376, 140401281306623, +STORE, 140401279115264, 140401279205375, +SNULL, 140401281298431, 140401281306623, +STORE, 140401279205376, 140401281298431, +STORE, 140401281298432, 140401281306623, +ERASE, 140401281298432, 140401281306623, +STORE, 140401281298432, 140401281306623, +SNULL, 140401281302527, 140401281306623, +STORE, 140401281298432, 140401281302527, +STORE, 140401281302528, 140401281306623, +ERASE, 140401467105280, 140401467133951, +ERASE, 140400056594432, 140400056598527, +ERASE, 140400056598528, 140400064987135, +ERASE, 140400635396096, 140400635400191, +ERASE, 140400635400192, 140400643788799, +ERASE, 140400408891392, 140400408895487, +ERASE, 140400408895488, 140400417284095, +ERASE, 140400299851776, 140400299855871, +ERASE, 140400299855872, 140400308244479, +ERASE, 140400627003392, 140400627007487, +ERASE, 140400627007488, 140400635396095, +ERASE, 140400954155008, 140400954159103, +ERASE, 140400954159104, 140400962547711, +ERASE, 140400291459072, 140400291463167, +ERASE, 140400291463168, 140400299851775, +ERASE, 140400643788800, 140400643792895, +ERASE, 140400643792896, 140400652181503, +ERASE, 140400325029888, 140400325033983, +ERASE, 140400325033984, 140400333422591, +ERASE, 140400610217984, 140400610222079, +ERASE, 140400610222080, 140400618610687, +ERASE, 140400190812160, 140400190816255, +ERASE, 140400190816256, 140400199204863, +ERASE, 140399964274688, 140399964278783, +ERASE, 140399964278784, 140399972667391, +ERASE, 140400945762304, 140400945766399, +ERASE, 140400945766400, 140400954155007, +ERASE, 140400568287232, 140400568291327, +ERASE, 140400568291328, 140400576679935, +ERASE, 140399972667392, 140399972671487, +ERASE, 140399972671488, 140399981060095, +ERASE, 140400962547712, 140400962551807, +ERASE, 140400962551808, 140400970940415, +ERASE, 140400987725824, 140400987729919, +ERASE, 140400987729920, 140400996118527, +ERASE, 140400652181504, 140400652185599, +ERASE, 140400652185600, 140400660574207, +ERASE, 140400450854912, 140400450859007, +ERASE, 140400450859008, 140400459247615, +ERASE, 140400031416320, 140400031420415, +ERASE, 140400031420416, 140400039809023, +ERASE, 140400308244480, 140400308248575, +ERASE, 140400308248576, 140400316637183, +ERASE, 140400434069504, 140400434073599, +ERASE, 140400434073600, 140400442462207, +ERASE, 140400543109120, 140400543113215, +ERASE, 140400543113216, 140400551501823, +ERASE, 140400023023616, 140400023027711, +ERASE, 140400023027712, 140400031416319, +ERASE, 140399813304320, 140399813308415, +ERASE, 140399813308416, 140399821697023, +ERASE, 140400316637184, 140400316641279, +ERASE, 140400316641280, 140400325029887, +ERASE, 140400585072640, 140400585076735, +ERASE, 140400585076736, 140400593465343, +ERASE, 140400148848640, 140400148852735, +ERASE, 140400148852736, 140400157241343, +ERASE, 140399955881984, 140399955886079, +ERASE, 140399955886080, 140399964274687, +ERASE, 140399821697024, 140399821701119, +ERASE, 140399821701120, 140399830089727, +ERASE, 140400601825280, 140400601829375, +ERASE, 140400601829376, 140400610217983, +ERASE, 140400979333120, 140400979337215, +ERASE, 140400979337216, 140400987725823, +ERASE, 140399997845504, 140399997849599, +ERASE, 140399997849600, 140400006238207, +ERASE, 140400459247616, 140400459251711, +ERASE, 140400459251712, 140400467640319, +ERASE, 140400551501824, 140400551505919, +ERASE, 140400551505920, 140400559894527, +ERASE, 140399939096576, 140399939100671, +ERASE, 140399939100672, 140399947489279, +ERASE, 140400442462208, 140400442466303, +ERASE, 140400442466304, 140400450854911, +ERASE, 140400576679936, 140400576684031, +ERASE, 140400576684032, 140400585072639, +ERASE, 140400559894528, 140400559898623, +ERASE, 140400559898624, 140400568287231, +ERASE, 140400417284096, 140400417288191, +ERASE, 140400417288192, 140400425676799, +ERASE, 140400283066368, 140400283070463, +ERASE, 140400283070464, 140400291459071, + }; + unsigned long set33[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734562918400, 140737488351231, +SNULL, 140734562922495, 140737488351231, +STORE, 140734562918400, 140734562922495, +STORE, 140734562787328, 140734562922495, +STORE, 94133878984704, 94133881237503, +SNULL, 94133879115775, 94133881237503, +STORE, 94133878984704, 94133879115775, +STORE, 94133879115776, 94133881237503, +ERASE, 94133879115776, 94133881237503, +STORE, 94133881208832, 94133881217023, +STORE, 94133881217024, 94133881237503, +STORE, 140583654043648, 140583656296447, +SNULL, 140583654187007, 140583656296447, +STORE, 140583654043648, 140583654187007, +STORE, 140583654187008, 140583656296447, +ERASE, 140583654187008, 140583656296447, +STORE, 140583656284160, 140583656292351, +STORE, 140583656292352, 140583656296447, +STORE, 140734564319232, 140734564323327, +STORE, 140734564306944, 140734564319231, +STORE, 140583656255488, 140583656284159, +STORE, 140583656247296, 140583656255487, +STORE, 140583651827712, 140583654043647, +SNULL, 140583651827712, 140583651926015, +STORE, 140583651926016, 140583654043647, +STORE, 140583651827712, 140583651926015, +SNULL, 140583654019071, 140583654043647, +STORE, 140583651926016, 140583654019071, +STORE, 140583654019072, 140583654043647, +SNULL, 140583654019072, 140583654027263, +STORE, 140583654027264, 140583654043647, +STORE, 140583654019072, 140583654027263, +ERASE, 140583654019072, 140583654027263, +STORE, 140583654019072, 140583654027263, +ERASE, 140583654027264, 140583654043647, +STORE, 140583654027264, 140583654043647, +STORE, 140583648030720, 140583651827711, +SNULL, 140583648030720, 140583649689599, +STORE, 140583649689600, 140583651827711, +STORE, 140583648030720, 140583649689599, +SNULL, 140583651786751, 140583651827711, +STORE, 140583649689600, 140583651786751, +STORE, 140583651786752, 140583651827711, +SNULL, 140583651786752, 140583651811327, +STORE, 140583651811328, 140583651827711, +STORE, 140583651786752, 140583651811327, +ERASE, 140583651786752, 140583651811327, +STORE, 140583651786752, 140583651811327, +ERASE, 140583651811328, 140583651827711, +STORE, 140583651811328, 140583651827711, +STORE, 140583656239104, 140583656255487, +SNULL, 140583651803135, 140583651811327, +STORE, 140583651786752, 140583651803135, +STORE, 140583651803136, 140583651811327, +SNULL, 140583654023167, 140583654027263, +STORE, 140583654019072, 140583654023167, +STORE, 140583654023168, 140583654027263, +SNULL, 94133881212927, 94133881217023, +STORE, 94133881208832, 94133881212927, +STORE, 94133881212928, 94133881217023, +SNULL, 140583656288255, 140583656292351, +STORE, 140583656284160, 140583656288255, +STORE, 140583656288256, 140583656292351, +ERASE, 140583656255488, 140583656284159, +STORE, 94133881733120, 94133881868287, +STORE, 140583639638016, 140583648030719, +SNULL, 140583639642111, 140583648030719, +STORE, 140583639638016, 140583639642111, +STORE, 140583639642112, 140583648030719, +STORE, 140583631245312, 140583639638015, +STORE, 140583497027584, 140583631245311, +SNULL, 140583497027584, 140583540621311, +STORE, 140583540621312, 140583631245311, +STORE, 140583497027584, 140583540621311, +ERASE, 140583497027584, 140583540621311, +SNULL, 140583607730175, 140583631245311, +STORE, 140583540621312, 140583607730175, +STORE, 140583607730176, 140583631245311, +ERASE, 140583607730176, 140583631245311, +SNULL, 140583540756479, 140583607730175, +STORE, 140583540621312, 140583540756479, +STORE, 140583540756480, 140583607730175, +SNULL, 140583631249407, 140583639638015, +STORE, 140583631245312, 140583631249407, +STORE, 140583631249408, 140583639638015, +STORE, 140583622852608, 140583631245311, +SNULL, 140583622856703, 140583631245311, +STORE, 140583622852608, 140583622856703, +STORE, 140583622856704, 140583631245311, +STORE, 140583614459904, 140583622852607, +SNULL, 140583614463999, 140583622852607, +STORE, 140583614459904, 140583614463999, +STORE, 140583614464000, 140583622852607, +STORE, 140583532228608, 140583540621311, +SNULL, 140583532232703, 140583540621311, +STORE, 140583532228608, 140583532232703, +STORE, 140583532232704, 140583540621311, +STORE, 140583523835904, 140583532228607, +STORE, 140583515443200, 140583532228607, +STORE, 140583507050496, 140583532228607, +STORE, 140583372832768, 140583507050495, +STORE, 140583364440064, 140583372832767, +STORE, 140583230222336, 140583364440063, +STORE, 140583096004608, 140583364440063, +SNULL, 140583230222335, 140583364440063, +STORE, 140583096004608, 140583230222335, +STORE, 140583230222336, 140583364440063, +SNULL, 140583230222336, 140583272185855, +STORE, 140583272185856, 140583364440063, +STORE, 140583230222336, 140583272185855, +ERASE, 140583230222336, 140583272185855, +STORE, 140582961786880, 140583230222335, +SNULL, 140583372832768, 140583406403583, +STORE, 140583406403584, 140583507050495, +STORE, 140583372832768, 140583406403583, +ERASE, 140583372832768, 140583406403583, +SNULL, 140583473512447, 140583507050495, +STORE, 140583406403584, 140583473512447, +STORE, 140583473512448, 140583507050495, +ERASE, 140583473512448, 140583507050495, +SNULL, 140583096004607, 140583230222335, +STORE, 140582961786880, 140583096004607, +STORE, 140583096004608, 140583230222335, +SNULL, 140583096004608, 140583137968127, +STORE, 140583137968128, 140583230222335, +STORE, 140583096004608, 140583137968127, +ERASE, 140583096004608, 140583137968127, +SNULL, 140583339294719, 140583364440063, +STORE, 140583272185856, 140583339294719, +STORE, 140583339294720, 140583364440063, +ERASE, 140583339294720, 140583364440063, +SNULL, 140583272321023, 140583339294719, +STORE, 140583272185856, 140583272321023, +STORE, 140583272321024, 140583339294719, +SNULL, 140582961786880, 140583003750399, +STORE, 140583003750400, 140583096004607, +STORE, 140582961786880, 140583003750399, +ERASE, 140582961786880, 140583003750399, + }; + + unsigned long set34[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140731327180800, 140737488351231, +SNULL, 140731327184895, 140737488351231, +STORE, 140731327180800, 140731327184895, +STORE, 140731327049728, 140731327184895, +STORE, 94632924487680, 94632926740479, +SNULL, 94632924618751, 94632926740479, +STORE, 94632924487680, 94632924618751, +STORE, 94632924618752, 94632926740479, +ERASE, 94632924618752, 94632926740479, +STORE, 94632926711808, 94632926719999, +STORE, 94632926720000, 94632926740479, +STORE, 140012544888832, 140012547141631, +SNULL, 140012545032191, 140012547141631, +STORE, 140012544888832, 140012545032191, +STORE, 140012545032192, 140012547141631, +ERASE, 140012545032192, 140012547141631, +STORE, 140012547129344, 140012547137535, +STORE, 140012547137536, 140012547141631, +STORE, 140731327725568, 140731327729663, +STORE, 140731327713280, 140731327725567, +STORE, 140012547100672, 140012547129343, +STORE, 140012547092480, 140012547100671, +STORE, 140012542672896, 140012544888831, +SNULL, 140012542672896, 140012542771199, +STORE, 140012542771200, 140012544888831, +STORE, 140012542672896, 140012542771199, +SNULL, 140012544864255, 140012544888831, +STORE, 140012542771200, 140012544864255, +STORE, 140012544864256, 140012544888831, +SNULL, 140012544864256, 140012544872447, +STORE, 140012544872448, 140012544888831, +STORE, 140012544864256, 140012544872447, +ERASE, 140012544864256, 140012544872447, +STORE, 140012544864256, 140012544872447, +ERASE, 140012544872448, 140012544888831, +STORE, 140012544872448, 140012544888831, +STORE, 140012538875904, 140012542672895, +SNULL, 140012538875904, 140012540534783, +STORE, 140012540534784, 140012542672895, +STORE, 140012538875904, 140012540534783, +SNULL, 140012542631935, 140012542672895, +STORE, 140012540534784, 140012542631935, +STORE, 140012542631936, 140012542672895, +SNULL, 140012542631936, 140012542656511, +STORE, 140012542656512, 140012542672895, +STORE, 140012542631936, 140012542656511, +ERASE, 140012542631936, 140012542656511, +STORE, 140012542631936, 140012542656511, +ERASE, 140012542656512, 140012542672895, +STORE, 140012542656512, 140012542672895, +STORE, 140012547084288, 140012547100671, +SNULL, 140012542648319, 140012542656511, +STORE, 140012542631936, 140012542648319, +STORE, 140012542648320, 140012542656511, +SNULL, 140012544868351, 140012544872447, +STORE, 140012544864256, 140012544868351, +STORE, 140012544868352, 140012544872447, +SNULL, 94632926715903, 94632926719999, +STORE, 94632926711808, 94632926715903, +STORE, 94632926715904, 94632926719999, +SNULL, 140012547133439, 140012547137535, +STORE, 140012547129344, 140012547133439, +STORE, 140012547133440, 140012547137535, +ERASE, 140012547100672, 140012547129343, +STORE, 94632939606016, 94632939741183, +STORE, 140012530483200, 140012538875903, +SNULL, 140012530487295, 140012538875903, +STORE, 140012530483200, 140012530487295, +STORE, 140012530487296, 140012538875903, +STORE, 140012522090496, 140012530483199, +STORE, 140012387872768, 140012522090495, +SNULL, 140012387872768, 140012444188671, +STORE, 140012444188672, 140012522090495, +STORE, 140012387872768, 140012444188671, +ERASE, 140012387872768, 140012444188671, +SNULL, 140012511297535, 140012522090495, +STORE, 140012444188672, 140012511297535, +STORE, 140012511297536, 140012522090495, +ERASE, 140012511297536, 140012522090495, +SNULL, 140012444323839, 140012511297535, +STORE, 140012444188672, 140012444323839, +STORE, 140012444323840, 140012511297535, +SNULL, 140012522094591, 140012530483199, +STORE, 140012522090496, 140012522094591, +STORE, 140012522094592, 140012530483199, +STORE, 140012513697792, 140012522090495, +SNULL, 140012513701887, 140012522090495, +STORE, 140012513697792, 140012513701887, +STORE, 140012513701888, 140012522090495, +STORE, 140012435795968, 140012444188671, +SNULL, 140012435800063, 140012444188671, +STORE, 140012435795968, 140012435800063, +STORE, 140012435800064, 140012444188671, +STORE, 140012427403264, 140012435795967, +SNULL, 140012427407359, 140012435795967, +STORE, 140012427403264, 140012427407359, +STORE, 140012427407360, 140012435795967, +STORE, 140012419010560, 140012427403263, +STORE, 140012410617856, 140012427403263, +STORE, 140012276400128, 140012410617855, +STORE, 140012268007424, 140012276400127, +STORE, 140012133789696, 140012268007423, +SNULL, 140012133789696, 140012175753215, +STORE, 140012175753216, 140012268007423, +STORE, 140012133789696, 140012175753215, +ERASE, 140012133789696, 140012175753215, +STORE, 140012041535488, 140012268007423, +SNULL, 140012108644351, 140012268007423, +STORE, 140012041535488, 140012108644351, +STORE, 140012108644352, 140012268007423, +SNULL, 140012108644352, 140012175753215, +STORE, 140012175753216, 140012268007423, +STORE, 140012108644352, 140012175753215, +ERASE, 140012108644352, 140012175753215, +SNULL, 140012276400128, 140012309970943, +STORE, 140012309970944, 140012410617855, +STORE, 140012276400128, 140012309970943, +ERASE, 140012276400128, 140012309970943, +STORE, 140012301578240, 140012309970943, +STORE, 140012041535488, 140012268007423, +SNULL, 140012242862079, 140012268007423, +STORE, 140012041535488, 140012242862079, +STORE, 140012242862080, 140012268007423, +ERASE, 140012242862080, 140012268007423, +SNULL, 140012041670655, 140012242862079, +STORE, 140012041535488, 140012041670655, +STORE, 140012041670656, 140012242862079, +SNULL, 140012041670656, 140012108644351, +STORE, 140012108644352, 140012242862079, +STORE, 140012041670656, 140012108644351, +SNULL, 140012108779519, 140012242862079, +STORE, 140012108644352, 140012108779519, +STORE, 140012108779520, 140012242862079, +SNULL, 140012377079807, 140012410617855, +STORE, 140012309970944, 140012377079807, +STORE, 140012377079808, 140012410617855, +ERASE, 140012377079808, 140012410617855, +SNULL, 140012310106111, 140012377079807, +STORE, 140012309970944, 140012310106111, +STORE, 140012310106112, 140012377079807, +SNULL, 140012410621951, 140012427403263, +STORE, 140012410617856, 140012410621951, +STORE, 140012410621952, 140012427403263, +SNULL, 140012108779520, 140012175753215, +STORE, 140012175753216, 140012242862079, +STORE, 140012108779520, 140012175753215, +SNULL, 140012175888383, 140012242862079, +STORE, 140012175753216, 140012175888383, +STORE, 140012175888384, 140012242862079, +SNULL, 140012301582335, 140012309970943, +STORE, 140012301578240, 140012301582335, +STORE, 140012301582336, 140012309970943, +SNULL, 140012410621952, 140012419010559, +STORE, 140012419010560, 140012427403263, +STORE, 140012410621952, 140012419010559, +SNULL, 140012419014655, 140012427403263, +STORE, 140012419010560, 140012419014655, +STORE, 140012419014656, 140012427403263, +SNULL, 140012268011519, 140012276400127, +STORE, 140012268007424, 140012268011519, +STORE, 140012268011520, 140012276400127, +STORE, 140012402225152, 140012410617855, +STORE, 140012393832448, 140012410617855, +SNULL, 140012393832448, 140012402225151, +STORE, 140012402225152, 140012410617855, +STORE, 140012393832448, 140012402225151, +SNULL, 140012402229247, 140012410617855, +STORE, 140012402225152, 140012402229247, +STORE, 140012402229248, 140012410617855, +STORE, 140012385439744, 140012402225151, +SNULL, 140012385439744, 140012393832447, +STORE, 140012393832448, 140012402225151, +STORE, 140012385439744, 140012393832447, +SNULL, 140012393836543, 140012402225151, +STORE, 140012393832448, 140012393836543, +STORE, 140012393836544, 140012402225151, +STORE, 140012293185536, 140012301578239, +STORE, 140012284792832, 140012301578239, +SNULL, 140012284792832, 140012293185535, +STORE, 140012293185536, 140012301578239, +STORE, 140012284792832, 140012293185535, +SNULL, 140012293189631, 140012301578239, +STORE, 140012293185536, 140012293189631, +STORE, 140012293189632, 140012301578239, +STORE, 140012268011520, 140012284792831, +SNULL, 140012385443839, 140012393832447, +STORE, 140012385439744, 140012385443839, +STORE, 140012385443840, 140012393832447, +STORE, 140012259614720, 140012268007423, +SNULL, 140012259618815, 140012268007423, +STORE, 140012259614720, 140012259618815, +STORE, 140012259618816, 140012268007423, +STORE, 140012251222016, 140012259614719, +SNULL, 140012251226111, 140012259614719, +STORE, 140012251222016, 140012251226111, +STORE, 140012251226112, 140012259614719, +SNULL, 140012284796927, 140012293185535, +STORE, 140012284792832, 140012284796927, +STORE, 140012284796928, 140012293185535, +SNULL, 140012268011520, 140012276400127, +STORE, 140012276400128, 140012284792831, +STORE, 140012268011520, 140012276400127, +SNULL, 140012276404223, 140012284792831, +STORE, 140012276400128, 140012276404223, +STORE, 140012276404224, 140012284792831, +STORE, 140012033142784, 140012041535487, +SNULL, 140012033146879, 140012041535487, +STORE, 140012033142784, 140012033146879, +STORE, 140012033146880, 140012041535487, +STORE, 140012024750080, 140012033142783, +STORE, 140012016357376, 140012033142783, +SNULL, 140012016357376, 140012024750079, +STORE, 140012024750080, 140012033142783, +STORE, 140012016357376, 140012024750079, +SNULL, 140012024754175, 140012033142783, +STORE, 140012024750080, 140012024754175, +STORE, 140012024754176, 140012033142783, +SNULL, 140012016361471, 140012024750079, +STORE, 140012016357376, 140012016361471, +STORE, 140012016361472, 140012024750079, +STORE, 140012007964672, 140012016357375, +SNULL, 140012007968767, 140012016357375, +STORE, 140012007964672, 140012007968767, +STORE, 140012007968768, 140012016357375, +STORE, 140011999571968, 140012007964671, +STORE, 140011991179264, 140012007964671, +STORE, 140011856961536, 140011991179263, +STORE, 140011848568832, 140011856961535, +STORE, 140011714351104, 140011848568831, +SNULL, 140011714351104, 140011773100031, +STORE, 140011773100032, 140011848568831, +STORE, 140011714351104, 140011773100031, +ERASE, 140011714351104, 140011773100031, +STORE, 140011764707328, 140011773100031, +STORE, 140011756314624, 140011773100031, +STORE, 140011622096896, 140011756314623, +STORE, 140011613704192, 140011622096895, +STORE, 140011479486464, 140011613704191, +STORE, 140011471093760, 140011479486463, +SNULL, 140011479486464, 140011504664575, +STORE, 140011504664576, 140011613704191, +STORE, 140011479486464, 140011504664575, +ERASE, 140011479486464, 140011504664575, +STORE, 140011496271872, 140011504664575, +STORE, 140011487879168, 140011504664575, +STORE, 140011336876032, 140011471093759, +SNULL, 140011336876032, 140011370446847, +STORE, 140011370446848, 140011471093759, +STORE, 140011336876032, 140011370446847, +ERASE, 140011336876032, 140011370446847, +STORE, 140011471093760, 140011487879167, +STORE, 140011362054144, 140011370446847, +SNULL, 140011362058239, 140011370446847, +STORE, 140011362054144, 140011362058239, +STORE, 140011362058240, 140011370446847, +STORE, 140011353661440, 140011362054143, +STORE, 140011345268736, 140011362054143, +SNULL, 140011345272831, 140011362054143, +STORE, 140011345268736, 140011345272831, +STORE, 140011345272832, 140011362054143, +STORE, 140011336876032, 140011345268735, +STORE, 140011328483328, 140011345268735, +SNULL, 140011328487423, 140011345268735, +STORE, 140011328483328, 140011328487423, +STORE, 140011328487424, 140011345268735, +STORE, 140011320090624, 140011328483327, +STORE, 140011185872896, 140011320090623, +SNULL, 140011185872896, 140011236229119, +STORE, 140011236229120, 140011320090623, +STORE, 140011185872896, 140011236229119, +ERASE, 140011185872896, 140011236229119, +SNULL, 140011856961536, 140011907317759, +STORE, 140011907317760, 140011991179263, +STORE, 140011856961536, 140011907317759, +ERASE, 140011856961536, 140011907317759, +SNULL, 140011974426623, 140011991179263, +STORE, 140011907317760, 140011974426623, +STORE, 140011974426624, 140011991179263, +ERASE, 140011974426624, 140011991179263, +SNULL, 140011840208895, 140011848568831, +STORE, 140011773100032, 140011840208895, +STORE, 140011840208896, 140011848568831, +ERASE, 140011840208896, 140011848568831, +SNULL, 140011773235199, 140011840208895, +STORE, 140011773100032, 140011773235199, +STORE, 140011773235200, 140011840208895, +STORE, 140011102011392, 140011320090623, +SNULL, 140011169120255, 140011320090623, +STORE, 140011102011392, 140011169120255, +STORE, 140011169120256, 140011320090623, +SNULL, 140011169120256, 140011236229119, +STORE, 140011236229120, 140011320090623, +STORE, 140011169120256, 140011236229119, +ERASE, 140011169120256, 140011236229119, +SNULL, 140011622096896, 140011638882303, +STORE, 140011638882304, 140011756314623, +STORE, 140011622096896, 140011638882303, +ERASE, 140011622096896, 140011638882303, +SNULL, 140011705991167, 140011756314623, +STORE, 140011638882304, 140011705991167, +STORE, 140011705991168, 140011756314623, +ERASE, 140011705991168, 140011756314623, +SNULL, 140011571773439, 140011613704191, +STORE, 140011504664576, 140011571773439, +STORE, 140011571773440, 140011613704191, +ERASE, 140011571773440, 140011613704191, +STORE, 140010967793664, 140011169120255, +SNULL, 140011034902527, 140011169120255, +STORE, 140010967793664, 140011034902527, +STORE, 140011034902528, 140011169120255, +SNULL, 140011034902528, 140011102011391, +STORE, 140011102011392, 140011169120255, +STORE, 140011034902528, 140011102011391, +ERASE, 140011034902528, 140011102011391, +STORE, 140010833575936, 140011034902527, +SNULL, 140011437555711, 140011471093759, +STORE, 140011370446848, 140011437555711, +STORE, 140011437555712, 140011471093759, +ERASE, 140011437555712, 140011471093759, +SNULL, 140011370582015, 140011437555711, +STORE, 140011370446848, 140011370582015, +STORE, 140011370582016, 140011437555711, +STORE, 140010699358208, 140011034902527, +SNULL, 140011487883263, 140011504664575, +STORE, 140011487879168, 140011487883263, +STORE, 140011487883264, 140011504664575, +SNULL, 140011345272832, 140011353661439, +STORE, 140011353661440, 140011362054143, +STORE, 140011345272832, 140011353661439, +SNULL, 140011353665535, 140011362054143, +STORE, 140011353661440, 140011353665535, +STORE, 140011353665536, 140011362054143, +SNULL, 140011328487424, 140011336876031, +STORE, 140011336876032, 140011345268735, +STORE, 140011328487424, 140011336876031, +SNULL, 140011336880127, 140011345268735, +STORE, 140011336876032, 140011336880127, +STORE, 140011336880128, 140011345268735, +SNULL, 140011303337983, 140011320090623, +STORE, 140011236229120, 140011303337983, +STORE, 140011303337984, 140011320090623, +ERASE, 140011303337984, 140011320090623, +SNULL, 140011907452927, 140011974426623, +STORE, 140011907317760, 140011907452927, +STORE, 140011907452928, 140011974426623, +SNULL, 140011102146559, 140011169120255, +STORE, 140011102011392, 140011102146559, +STORE, 140011102146560, 140011169120255, +SNULL, 140011639017471, 140011705991167, +STORE, 140011638882304, 140011639017471, +STORE, 140011639017472, 140011705991167, +SNULL, 140011504799743, 140011571773439, +STORE, 140011504664576, 140011504799743, +STORE, 140011504799744, 140011571773439, +SNULL, 140011613708287, 140011622096895, +STORE, 140011613704192, 140011613708287, +STORE, 140011613708288, 140011622096895, +SNULL, 140010699358208, 140010967793663, +STORE, 140010967793664, 140011034902527, +STORE, 140010699358208, 140010967793663, +SNULL, 140010967928831, 140011034902527, +STORE, 140010967793664, 140010967928831, +STORE, 140010967928832, 140011034902527, +SNULL, 140010900684799, 140010967793663, +STORE, 140010699358208, 140010900684799, +STORE, 140010900684800, 140010967793663, +ERASE, 140010900684800, 140010967793663, +SNULL, 140010766467071, 140010900684799, +STORE, 140010699358208, 140010766467071, +STORE, 140010766467072, 140010900684799, +SNULL, 140010766467072, 140010833575935, +STORE, 140010833575936, 140010900684799, +STORE, 140010766467072, 140010833575935, +ERASE, 140010766467072, 140010833575935, +SNULL, 140010699493375, 140010766467071, +STORE, 140010699358208, 140010699493375, +STORE, 140010699493376, 140010766467071, +SNULL, 140011848572927, 140011856961535, +STORE, 140011848568832, 140011848572927, +STORE, 140011848572928, 140011856961535, +STORE, 140011982786560, 140012007964671, +STORE, 140011898925056, 140011907317759, +SNULL, 140011898929151, 140011907317759, +STORE, 140011898925056, 140011898929151, +STORE, 140011898929152, 140011907317759, +SNULL, 140011320094719, 140011328483327, +STORE, 140011320090624, 140011320094719, +STORE, 140011320094720, 140011328483327, +STORE, 140011890532352, 140011898925055, +STORE, 140011882139648, 140011898925055, +SNULL, 140011882143743, 140011898925055, +STORE, 140011882139648, 140011882143743, +STORE, 140011882143744, 140011898925055, +STORE, 140011873746944, 140011882139647, +SNULL, 140011873751039, 140011882139647, +STORE, 140011873746944, 140011873751039, +STORE, 140011873751040, 140011882139647, +SNULL, 140011236364287, 140011303337983, +STORE, 140011236229120, 140011236364287, +STORE, 140011236364288, 140011303337983, +SNULL, 140011756318719, 140011773100031, +STORE, 140011756314624, 140011756318719, +STORE, 140011756318720, 140011773100031, +SNULL, 140011756318720, 140011764707327, +STORE, 140011764707328, 140011773100031, +STORE, 140011756318720, 140011764707327, +SNULL, 140011764711423, 140011773100031, +STORE, 140011764707328, 140011764711423, +STORE, 140011764711424, 140011773100031, +SNULL, 140011471097855, 140011487879167, +STORE, 140011471093760, 140011471097855, +STORE, 140011471097856, 140011487879167, +SNULL, 140010833711103, 140010900684799, +STORE, 140010833575936, 140010833711103, +STORE, 140010833711104, 140010900684799, +SNULL, 140011982790655, 140012007964671, +STORE, 140011982786560, 140011982790655, +STORE, 140011982790656, 140012007964671, +STORE, 140011865354240, 140011873746943, +STORE, 140011848572928, 140011865354239, +SNULL, 140011848572928, 140011856961535, +STORE, 140011856961536, 140011865354239, +STORE, 140011848572928, 140011856961535, +SNULL, 140011856965631, 140011865354239, +STORE, 140011856961536, 140011856965631, +STORE, 140011856965632, 140011865354239, +STORE, 140011747921920, 140011756314623, +STORE, 140011739529216, 140011756314623, +SNULL, 140011471097856, 140011479486463, +STORE, 140011479486464, 140011487879167, +STORE, 140011471097856, 140011479486463, +SNULL, 140011479490559, 140011487879167, +STORE, 140011479486464, 140011479490559, +STORE, 140011479490560, 140011487879167, +STORE, 140011731136512, 140011756314623, +STORE, 140011722743808, 140011756314623, +SNULL, 140011982790656, 140011999571967, +STORE, 140011999571968, 140012007964671, +STORE, 140011982790656, 140011999571967, +SNULL, 140011999576063, 140012007964671, +STORE, 140011999571968, 140011999576063, +STORE, 140011999576064, 140012007964671, +STORE, 140011714351104, 140011756314623, +SNULL, 140011882143744, 140011890532351, +STORE, 140011890532352, 140011898925055, +STORE, 140011882143744, 140011890532351, +SNULL, 140011890536447, 140011898925055, +STORE, 140011890532352, 140011890536447, +STORE, 140011890536448, 140011898925055, +STORE, 140011630489600, 140011638882303, +STORE, 140011613708288, 140011638882303, +STORE, 140011605311488, 140011613704191, +STORE, 140011596918784, 140011613704191, +STORE, 140011588526080, 140011613704191, +SNULL, 140011487883264, 140011496271871, +STORE, 140011496271872, 140011504664575, +STORE, 140011487883264, 140011496271871, +SNULL, 140011496275967, 140011504664575, +STORE, 140011496271872, 140011496275967, +STORE, 140011496275968, 140011504664575, +STORE, 140011580133376, 140011613704191, +SNULL, 140011580137471, 140011613704191, +STORE, 140011580133376, 140011580137471, +STORE, 140011580137472, 140011613704191, +SNULL, 140011982790656, 140011991179263, +STORE, 140011991179264, 140011999571967, +STORE, 140011982790656, 140011991179263, +SNULL, 140011991183359, 140011999571967, +STORE, 140011991179264, 140011991183359, +STORE, 140011991183360, 140011999571967, +SNULL, 140011865358335, 140011873746943, +STORE, 140011865354240, 140011865358335, +STORE, 140011865358336, 140011873746943, +STORE, 140011462701056, 140011471093759, +SNULL, 140011714351104, 140011739529215, +STORE, 140011739529216, 140011756314623, +STORE, 140011714351104, 140011739529215, +SNULL, 140011739533311, 140011756314623, +STORE, 140011739529216, 140011739533311, +STORE, 140011739533312, 140011756314623, +SNULL, 140011739533312, 140011747921919, +STORE, 140011747921920, 140011756314623, +STORE, 140011739533312, 140011747921919, +SNULL, 140011747926015, 140011756314623, +STORE, 140011747921920, 140011747926015, +STORE, 140011747926016, 140011756314623, +SNULL, 140011613708288, 140011630489599, +STORE, 140011630489600, 140011638882303, +STORE, 140011613708288, 140011630489599, +SNULL, 140011630493695, 140011638882303, +STORE, 140011630489600, 140011630493695, +STORE, 140011630493696, 140011638882303, +SNULL, 140011714351104, 140011722743807, +STORE, 140011722743808, 140011739529215, +STORE, 140011714351104, 140011722743807, +SNULL, 140011722747903, 140011739529215, +STORE, 140011722743808, 140011722747903, +STORE, 140011722747904, 140011739529215, +SNULL, 140011714355199, 140011722743807, +STORE, 140011714351104, 140011714355199, +STORE, 140011714355200, 140011722743807, +SNULL, 140011722747904, 140011731136511, +STORE, 140011731136512, 140011739529215, +STORE, 140011722747904, 140011731136511, +SNULL, 140011731140607, 140011739529215, +STORE, 140011731136512, 140011731140607, +STORE, 140011731140608, 140011739529215, +STORE, 140011454308352, 140011471093759, +STORE, 140011445915648, 140011471093759, +SNULL, 140011580137472, 140011588526079, +STORE, 140011588526080, 140011613704191, +STORE, 140011580137472, 140011588526079, +SNULL, 140011588530175, 140011613704191, +STORE, 140011588526080, 140011588530175, +STORE, 140011588530176, 140011613704191, +SNULL, 140011445915648, 140011462701055, +STORE, 140011462701056, 140011471093759, +STORE, 140011445915648, 140011462701055, +SNULL, 140011462705151, 140011471093759, +STORE, 140011462701056, 140011462705151, +STORE, 140011462705152, 140011471093759, +SNULL, 140011588530176, 140011596918783, +STORE, 140011596918784, 140011613704191, +STORE, 140011588530176, 140011596918783, +SNULL, 140011596922879, 140011613704191, +STORE, 140011596918784, 140011596922879, +STORE, 140011596922880, 140011613704191, +SNULL, 140011596922880, 140011605311487, +STORE, 140011605311488, 140011613704191, +STORE, 140011596922880, 140011605311487, +SNULL, 140011605315583, 140011613704191, +STORE, 140011605311488, 140011605315583, +STORE, 140011605315584, 140011613704191, +SNULL, 140011613708288, 140011622096895, +STORE, 140011622096896, 140011630489599, +STORE, 140011613708288, 140011622096895, +SNULL, 140011622100991, 140011630489599, +STORE, 140011622096896, 140011622100991, +STORE, 140011622100992, 140011630489599, +STORE, 140011311697920, 140011320090623, +STORE, 140011227836416, 140011236229119, +STORE, 140011219443712, 140011236229119, +SNULL, 140011219447807, 140011236229119, +STORE, 140011219443712, 140011219447807, +STORE, 140011219447808, 140011236229119, +STORE, 140011211051008, 140011219443711, +STORE, 140011202658304, 140011219443711, +SNULL, 140011202662399, 140011219443711, +STORE, 140011202658304, 140011202662399, +STORE, 140011202662400, 140011219443711, +STORE, 140011194265600, 140011202658303, +STORE, 140011185872896, 140011202658303, +STORE, 140011177480192, 140011202658303, +STORE, 140011093618688, 140011102011391, +SNULL, 140011445915648, 140011454308351, +STORE, 140011454308352, 140011462701055, +STORE, 140011445915648, 140011454308351, +SNULL, 140011454312447, 140011462701055, +STORE, 140011454308352, 140011454312447, +STORE, 140011454312448, 140011462701055, +STORE, 140011085225984, 140011102011391, +SNULL, 140011085230079, 140011102011391, +STORE, 140011085225984, 140011085230079, +STORE, 140011085230080, 140011102011391, +SNULL, 140011177484287, 140011202658303, +STORE, 140011177480192, 140011177484287, +STORE, 140011177484288, 140011202658303, +SNULL, 140011445919743, 140011454308351, +STORE, 140011445915648, 140011445919743, +STORE, 140011445919744, 140011454308351, +SNULL, 140011177484288, 140011185872895, +STORE, 140011185872896, 140011202658303, +STORE, 140011177484288, 140011185872895, +SNULL, 140011185876991, 140011202658303, +STORE, 140011185872896, 140011185876991, +STORE, 140011185876992, 140011202658303, +STORE, 140011076833280, 140011085225983, +SNULL, 140011202662400, 140011211051007, +STORE, 140011211051008, 140011219443711, +STORE, 140011202662400, 140011211051007, +SNULL, 140011211055103, 140011219443711, +STORE, 140011211051008, 140011211055103, +STORE, 140011211055104, 140011219443711, +SNULL, 140011185876992, 140011194265599, +STORE, 140011194265600, 140011202658303, +STORE, 140011185876992, 140011194265599, +SNULL, 140011194269695, 140011202658303, +STORE, 140011194265600, 140011194269695, +STORE, 140011194269696, 140011202658303, +STORE, 140011068440576, 140011085225983, +SNULL, 140011311702015, 140011320090623, +STORE, 140011311697920, 140011311702015, +STORE, 140011311702016, 140011320090623, +STORE, 140011060047872, 140011085225983, +SNULL, 140011060051967, 140011085225983, +STORE, 140011060047872, 140011060051967, +STORE, 140011060051968, 140011085225983, +STORE, 140011051655168, 140011060047871, +STORE, 140011043262464, 140011060047871, +SNULL, 140011043266559, 140011060047871, +STORE, 140011043262464, 140011043266559, +STORE, 140011043266560, 140011060047871, +SNULL, 140011219447808, 140011227836415, +STORE, 140011227836416, 140011236229119, +STORE, 140011219447808, 140011227836415, +SNULL, 140011227840511, 140011236229119, +STORE, 140011227836416, 140011227840511, +STORE, 140011227840512, 140011236229119, +SNULL, 140011085230080, 140011093618687, +STORE, 140011093618688, 140011102011391, +STORE, 140011085230080, 140011093618687, +SNULL, 140011093622783, 140011102011391, +STORE, 140011093618688, 140011093622783, +STORE, 140011093622784, 140011102011391, +STORE, 140010959400960, 140010967793663, +STORE, 140010951008256, 140010967793663, +SNULL, 140010951008256, 140010959400959, +STORE, 140010959400960, 140010967793663, +STORE, 140010951008256, 140010959400959, +SNULL, 140010959405055, 140010967793663, +STORE, 140010959400960, 140010959405055, +STORE, 140010959405056, 140010967793663, +STORE, 140010942615552, 140010959400959, +STORE, 140010934222848, 140010959400959, +SNULL, 140011060051968, 140011076833279, +STORE, 140011076833280, 140011085225983, +STORE, 140011060051968, 140011076833279, +SNULL, 140011076837375, 140011085225983, +STORE, 140011076833280, 140011076837375, +STORE, 140011076837376, 140011085225983, +SNULL, 140011043266560, 140011051655167, +STORE, 140011051655168, 140011060047871, +STORE, 140011043266560, 140011051655167, +SNULL, 140011051659263, 140011060047871, +STORE, 140011051655168, 140011051659263, +STORE, 140011051659264, 140011060047871, +STORE, 140010925830144, 140010959400959, +SNULL, 140011060051968, 140011068440575, +STORE, 140011068440576, 140011076833279, +STORE, 140011060051968, 140011068440575, +SNULL, 140011068444671, 140011076833279, +STORE, 140011068440576, 140011068444671, +STORE, 140011068444672, 140011076833279, +STORE, 140010917437440, 140010959400959, +STORE, 140010909044736, 140010959400959, +STORE, 140010825183232, 140010833575935, +SNULL, 140010909044736, 140010942615551, +STORE, 140010942615552, 140010959400959, +STORE, 140010909044736, 140010942615551, +SNULL, 140010942619647, 140010959400959, +STORE, 140010942615552, 140010942619647, +STORE, 140010942619648, 140010959400959, +SNULL, 140010909044736, 140010934222847, +STORE, 140010934222848, 140010942615551, +STORE, 140010909044736, 140010934222847, +SNULL, 140010934226943, 140010942615551, +STORE, 140010934222848, 140010934226943, +STORE, 140010934226944, 140010942615551, +SNULL, 140010909048831, 140010934222847, +STORE, 140010909044736, 140010909048831, +STORE, 140010909048832, 140010934222847, +STORE, 140010816790528, 140010833575935, +SNULL, 140010816794623, 140010833575935, +STORE, 140010816790528, 140010816794623, +STORE, 140010816794624, 140010833575935, +STORE, 140010808397824, 140010816790527, +SNULL, 140010942619648, 140010951008255, +STORE, 140010951008256, 140010959400959, +STORE, 140010942619648, 140010951008255, +SNULL, 140010951012351, 140010959400959, +STORE, 140010951008256, 140010951012351, +STORE, 140010951012352, 140010959400959, +STORE, 140010800005120, 140010816790527, +SNULL, 140010800009215, 140010816790527, +STORE, 140010800005120, 140010800009215, +STORE, 140010800009216, 140010816790527, +SNULL, 140010909048832, 140010925830143, +STORE, 140010925830144, 140010934222847, +STORE, 140010909048832, 140010925830143, +SNULL, 140010925834239, 140010934222847, +STORE, 140010925830144, 140010925834239, +STORE, 140010925834240, 140010934222847, +SNULL, 140010816794624, 140010825183231, +STORE, 140010825183232, 140010833575935, +STORE, 140010816794624, 140010825183231, +SNULL, 140010825187327, 140010833575935, +STORE, 140010825183232, 140010825187327, +STORE, 140010825187328, 140010833575935, +SNULL, 140010909048832, 140010917437439, +STORE, 140010917437440, 140010925830143, +STORE, 140010909048832, 140010917437439, +SNULL, 140010917441535, 140010925830143, +STORE, 140010917437440, 140010917441535, +STORE, 140010917441536, 140010925830143, +SNULL, 140010800009216, 140010808397823, +STORE, 140010808397824, 140010816790527, +STORE, 140010800009216, 140010808397823, +SNULL, 140010808401919, 140010816790527, +STORE, 140010808397824, 140010808401919, +STORE, 140010808401920, 140010816790527, +STORE, 140010791612416, 140010800005119, +SNULL, 140010791616511, 140010800005119, +STORE, 140010791612416, 140010791616511, +STORE, 140010791616512, 140010800005119, +STORE, 140012547100672, 140012547129343, +STORE, 140012511506432, 140012513697791, +SNULL, 140012511506432, 140012511596543, +STORE, 140012511596544, 140012513697791, +STORE, 140012511506432, 140012511596543, +SNULL, 140012513689599, 140012513697791, +STORE, 140012511596544, 140012513689599, +STORE, 140012513689600, 140012513697791, +ERASE, 140012513689600, 140012513697791, +STORE, 140012513689600, 140012513697791, +SNULL, 140012513693695, 140012513697791, +STORE, 140012513689600, 140012513693695, +STORE, 140012513693696, 140012513697791, +ERASE, 140012547100672, 140012547129343, +ERASE, 140011362054144, 140011362058239, +ERASE, 140011362058240, 140011370446847, +ERASE, 140011882139648, 140011882143743, +ERASE, 140011882143744, 140011890532351, +ERASE, 140011873746944, 140011873751039, +ERASE, 140011873751040, 140011882139647, +ERASE, 140011588526080, 140011588530175, +ERASE, 140011588530176, 140011596918783, +ERASE, 140011328483328, 140011328487423, +ERASE, 140011328487424, 140011336876031, +ERASE, 140011898925056, 140011898929151, +ERASE, 140011898929152, 140011907317759, +ERASE, 140011353661440, 140011353665535, +ERASE, 140011353665536, 140011362054143, +ERASE, 140011336876032, 140011336880127, +ERASE, 140011336880128, 140011345268735, +ERASE, 140011731136512, 140011731140607, +ERASE, 140011731140608, 140011739529215, +ERASE, 140011479486464, 140011479490559, +ERASE, 140011479490560, 140011487879167, +ERASE, 140011756314624, 140011756318719, +ERASE, 140011756318720, 140011764707327, +ERASE, 140011580133376, 140011580137471, +ERASE, 140011580137472, 140011588526079, +ERASE, 140011219443712, 140011219447807, +ERASE, 140011219447808, 140011227836415, +ERASE, 140011051655168, 140011051659263, +ERASE, 140011051659264, 140011060047871, +ERASE, 140011999571968, 140011999576063, +ERASE, 140011999576064, 140012007964671, +ERASE, 140011714351104, 140011714355199, +ERASE, 140011714355200, 140011722743807, +ERASE, 140011739529216, 140011739533311, +ERASE, 140011739533312, 140011747921919, +ERASE, 140011320090624, 140011320094719, +ERASE, 140011320094720, 140011328483327, +ERASE, 140011630489600, 140011630493695, +ERASE, 140011630493696, 140011638882303, +ERASE, 140011345268736, 140011345272831, +ERASE, 140011345272832, 140011353661439, +ERASE, 140011496271872, 140011496275967, +ERASE, 140011496275968, 140011504664575, +ERASE, 140011194265600, 140011194269695, +ERASE, 140011194269696, 140011202658303, +ERASE, 140011068440576, 140011068444671, +ERASE, 140011068444672, 140011076833279, +ERASE, 140010909044736, 140010909048831, +ERASE, 140010909048832, 140010917437439, +ERASE, 140011764707328, 140011764711423, +ERASE, 140011764711424, 140011773100031, +ERASE, 140011462701056, 140011462705151, +ERASE, 140011462705152, 140011471093759, +ERASE, 140011076833280, 140011076837375, +ERASE, 140011076837376, 140011085225983, +ERASE, 140011991179264, 140011991183359, +ERASE, 140011991183360, 140011999571967, +ERASE, 140011211051008, 140011211055103, +ERASE, 140011211055104, 140011219443711, +ERASE, 140010917437440, 140010917441535, +ERASE, 140010917441536, 140010925830143, +ERASE, 140011085225984, 140011085230079, +ERASE, 140011085230080, 140011093618687, +ERASE, 140011487879168, 140011487883263, +ERASE, 140011487883264, 140011496271871, +ERASE, 140011856961536, 140011856965631, +ERASE, 140011856965632, 140011865354239, +ERASE, 140011982786560, 140011982790655, +ERASE, 140011982790656, 140011991179263, +ERASE, 140011722743808, 140011722747903, +ERASE, 140011722747904, 140011731136511, +ERASE, 140011177480192, 140011177484287, +ERASE, 140011177484288, 140011185872895, +ERASE, 140011848568832, 140011848572927, +ERASE, 140011848572928, 140011856961535, +ERASE, 140011890532352, 140011890536447, +ERASE, 140011890536448, 140011898925055, +ERASE, 140011622096896, 140011622100991, +ERASE, 140011622100992, 140011630489599, +ERASE, 140011311697920, 140011311702015, +ERASE, 140011311702016, 140011320090623, +ERASE, 140011471093760, 140011471097855, +ERASE, 140011471097856, 140011479486463, +ERASE, 140011605311488, 140011605315583, +ERASE, 140011605315584, 140011613704191, +ERASE, 140010791612416, 140010791616511, +ERASE, 140010791616512, 140010800005119, +ERASE, 140010959400960, 140010959405055, +ERASE, 140010959405056, 140010967793663, +ERASE, 140011185872896, 140011185876991, +ERASE, 140011185876992, 140011194265599, +ERASE, 140011454308352, 140011454312447, +ERASE, 140011454312448, 140011462701055, +ERASE, 140011596918784, 140011596922879, +ERASE, 140011596922880, 140011605311487, +ERASE, 140011060047872, 140011060051967, +ERASE, 140011060051968, 140011068440575, +ERASE, 140010925830144, 140010925834239, +ERASE, 140010925834240, 140010934222847, +ERASE, 140011747921920, 140011747926015, +ERASE, 140011747926016, 140011756314623, +ERASE, 140011202658304, 140011202662399, +ERASE, 140011202662400, 140011211051007, +ERASE, 140010800005120, 140010800009215, +ERASE, 140010800009216, 140010808397823, +ERASE, 140011093618688, 140011093622783, +ERASE, 140011093622784, 140011102011391, +ERASE, 140010808397824, 140010808401919, +ERASE, 140010808401920, 140010816790527, +ERASE, 140012419010560, 140012419014655, +ERASE, 140012419014656, 140012427403263, +ERASE, 140010934222848, 140010934226943, +ERASE, 140010934226944, 140010942615551, +ERASE, 140010942615552, 140010942619647, +ERASE, 140010942619648, 140010951008255, +ERASE, 140011613704192, 140011613708287, +ERASE, 140011613708288, 140011622096895, +ERASE, 140011865354240, 140011865358335, +ERASE, 140011865358336, 140011873746943, +ERASE, 140012301578240, 140012301582335, +ERASE, 140012301582336, 140012309970943, +ERASE, 140012393832448, 140012393836543, +ERASE, 140012393836544, 140012402225151, +ERASE, 140012410617856, 140012410621951, +ERASE, 140012410621952, 140012419010559, +ERASE, 140012402225152, 140012402229247, +ERASE, 140012402229248, 140012410617855, +ERASE, 140012259614720, 140012259618815, +ERASE, 140012259618816, 140012268007423, +ERASE, 140012251222016, 140012251226111, +ERASE, 140012251226112, 140012259614719, +ERASE, 140012284792832, 140012284796927, +ERASE, 140012284796928, 140012293185535, +ERASE, 140011445915648, 140011445919743, +ERASE, 140011445919744, 140011454308351, +ERASE, 140010951008256, 140010951012351, +ERASE, 140010951012352, 140010959400959, +ERASE, 140011043262464, 140011043266559, +ERASE, 140011043266560, 140011051655167, +ERASE, 140010825183232, 140010825187327, +ERASE, 140010825187328, 140010833575935, +ERASE, 140012293185536, 140012293189631, +ERASE, 140012293189632, 140012301578239, +ERASE, 140012276400128, 140012276404223, +ERASE, 140012276404224, 140012284792831, +ERASE, 140012016357376, 140012016361471, +ERASE, 140012016361472, 140012024750079, +ERASE, 140012024750080, 140012024754175, +ERASE, 140012024754176, 140012033142783, +ERASE, 140011227836416, 140011227840511, +ERASE, 140011227840512, 140011236229119, +ERASE, 140010816790528, 140010816794623, +ERASE, 140010816794624, 140010825183231, +ERASE, 140012268007424, 140012268011519, +ERASE, 140012268011520, 140012276400127, +ERASE, 140012385439744, 140012385443839, +ERASE, 140012385443840, 140012393832447, +ERASE, 140012522090496, 140012522094591, +ERASE, 140012522094592, 140012530483199, +ERASE, 140012033142784, 140012033146879, +ERASE, 140012033146880, 140012041535487, + }; + unsigned long set35[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140730536939520, 140737488351231, +SNULL, 140730536943615, 140737488351231, +STORE, 140730536939520, 140730536943615, +STORE, 140730536808448, 140730536943615, +STORE, 94245239877632, 94245242130431, +SNULL, 94245240008703, 94245242130431, +STORE, 94245239877632, 94245240008703, +STORE, 94245240008704, 94245242130431, +ERASE, 94245240008704, 94245242130431, +STORE, 94245242101760, 94245242109951, +STORE, 94245242109952, 94245242130431, +STORE, 140475575263232, 140475577516031, +SNULL, 140475575406591, 140475577516031, +STORE, 140475575263232, 140475575406591, +STORE, 140475575406592, 140475577516031, +ERASE, 140475575406592, 140475577516031, +STORE, 140475577503744, 140475577511935, +STORE, 140475577511936, 140475577516031, +STORE, 140730538164224, 140730538168319, +STORE, 140730538151936, 140730538164223, +STORE, 140475577475072, 140475577503743, +STORE, 140475577466880, 140475577475071, +STORE, 140475573047296, 140475575263231, +SNULL, 140475573047296, 140475573145599, +STORE, 140475573145600, 140475575263231, +STORE, 140475573047296, 140475573145599, +SNULL, 140475575238655, 140475575263231, +STORE, 140475573145600, 140475575238655, +STORE, 140475575238656, 140475575263231, +SNULL, 140475575238656, 140475575246847, +STORE, 140475575246848, 140475575263231, +STORE, 140475575238656, 140475575246847, +ERASE, 140475575238656, 140475575246847, +STORE, 140475575238656, 140475575246847, +ERASE, 140475575246848, 140475575263231, +STORE, 140475575246848, 140475575263231, +STORE, 140475569250304, 140475573047295, +SNULL, 140475569250304, 140475570909183, +STORE, 140475570909184, 140475573047295, +STORE, 140475569250304, 140475570909183, +SNULL, 140475573006335, 140475573047295, +STORE, 140475570909184, 140475573006335, +STORE, 140475573006336, 140475573047295, +SNULL, 140475573006336, 140475573030911, +STORE, 140475573030912, 140475573047295, +STORE, 140475573006336, 140475573030911, +ERASE, 140475573006336, 140475573030911, +STORE, 140475573006336, 140475573030911, +ERASE, 140475573030912, 140475573047295, +STORE, 140475573030912, 140475573047295, +STORE, 140475577458688, 140475577475071, +SNULL, 140475573022719, 140475573030911, +STORE, 140475573006336, 140475573022719, +STORE, 140475573022720, 140475573030911, +SNULL, 140475575242751, 140475575246847, +STORE, 140475575238656, 140475575242751, +STORE, 140475575242752, 140475575246847, +SNULL, 94245242105855, 94245242109951, +STORE, 94245242101760, 94245242105855, +STORE, 94245242105856, 94245242109951, +SNULL, 140475577507839, 140475577511935, +STORE, 140475577503744, 140475577507839, +STORE, 140475577507840, 140475577511935, +ERASE, 140475577475072, 140475577503743, +STORE, 94245271216128, 94245271351295, +STORE, 140475560857600, 140475569250303, +SNULL, 140475560861695, 140475569250303, +STORE, 140475560857600, 140475560861695, +STORE, 140475560861696, 140475569250303, +STORE, 140475552464896, 140475560857599, +STORE, 140475418247168, 140475552464895, +SNULL, 140475418247168, 140475428241407, +STORE, 140475428241408, 140475552464895, +STORE, 140475418247168, 140475428241407, +ERASE, 140475418247168, 140475428241407, +SNULL, 140475495350271, 140475552464895, +STORE, 140475428241408, 140475495350271, +STORE, 140475495350272, 140475552464895, +ERASE, 140475495350272, 140475552464895, +SNULL, 140475428376575, 140475495350271, +STORE, 140475428241408, 140475428376575, +STORE, 140475428376576, 140475495350271, +SNULL, 140475552468991, 140475560857599, +STORE, 140475552464896, 140475552468991, +STORE, 140475552468992, 140475560857599, +STORE, 140475544072192, 140475552464895, +SNULL, 140475544076287, 140475552464895, +STORE, 140475544072192, 140475544076287, +STORE, 140475544076288, 140475552464895, +STORE, 140475535679488, 140475544072191, +SNULL, 140475535683583, 140475544072191, +STORE, 140475535679488, 140475535683583, +STORE, 140475535683584, 140475544072191, +STORE, 140475527286784, 140475535679487, +SNULL, 140475527290879, 140475535679487, +STORE, 140475527286784, 140475527290879, +STORE, 140475527290880, 140475535679487, +STORE, 140475518894080, 140475527286783, +STORE, 140475510501376, 140475527286783, +STORE, 140475502108672, 140475527286783, +STORE, 140475419848704, 140475428241407, +STORE, 140475285630976, 140475419848703, +SNULL, 140475285630976, 140475294023679, +STORE, 140475294023680, 140475419848703, +STORE, 140475285630976, 140475294023679, +ERASE, 140475285630976, 140475294023679, +STORE, 140475159805952, 140475419848703, +STORE, 140475025588224, 140475419848703, +SNULL, 140475092697087, 140475419848703, +STORE, 140475025588224, 140475092697087, +STORE, 140475092697088, 140475419848703, +SNULL, 140475092697088, 140475159805951, +STORE, 140475159805952, 140475419848703, +STORE, 140475092697088, 140475159805951, +ERASE, 140475092697088, 140475159805951, +STORE, 140474891370496, 140475092697087, +SNULL, 140474958479359, 140475092697087, +STORE, 140474891370496, 140474958479359, +STORE, 140474958479360, 140475092697087, +SNULL, 140474958479360, 140475025588223, +STORE, 140475025588224, 140475092697087, +STORE, 140474958479360, 140475025588223, +ERASE, 140474958479360, 140475025588223, +SNULL, 140475361132543, 140475419848703, +STORE, 140475159805952, 140475361132543, +STORE, 140475361132544, 140475419848703, +ERASE, 140475361132544, 140475419848703, +SNULL, 140475159805952, 140475294023679, +STORE, 140475294023680, 140475361132543, +STORE, 140475159805952, 140475294023679, +SNULL, 140475294158847, 140475361132543, +STORE, 140475294023680, 140475294158847, +STORE, 140475294158848, 140475361132543, +SNULL, 140475226914815, 140475294023679, +STORE, 140475159805952, 140475226914815, +STORE, 140475226914816, 140475294023679, +ERASE, 140475226914816, 140475294023679, +SNULL, 140475025723391, 140475092697087, +STORE, 140475025588224, 140475025723391, +STORE, 140475025723392, 140475092697087, +SNULL, 140475159941119, 140475226914815, +STORE, 140475159805952, 140475159941119, +STORE, 140475159941120, 140475226914815, +SNULL, 140474891505663, 140474958479359, +STORE, 140474891370496, 140474891505663, +STORE, 140474891505664, 140474958479359, +SNULL, 140475502108672, 140475518894079, +STORE, 140475518894080, 140475527286783, +STORE, 140475502108672, 140475518894079, +SNULL, 140475518898175, 140475527286783, +STORE, 140475518894080, 140475518898175, +STORE, 140475518898176, 140475527286783, +STORE, 140475411456000, 140475428241407, +SNULL, 140475502112767, 140475518894079, +STORE, 140475502108672, 140475502112767, +STORE, 140475502112768, 140475518894079, +SNULL, 140475411460095, 140475428241407, +STORE, 140475411456000, 140475411460095, +STORE, 140475411460096, 140475428241407, +SNULL, 140475411460096, 140475419848703, +STORE, 140475419848704, 140475428241407, +STORE, 140475411460096, 140475419848703, +SNULL, 140475419852799, 140475428241407, +STORE, 140475419848704, 140475419852799, +STORE, 140475419852800, 140475428241407, +STORE, 140475403063296, 140475411455999, +SNULL, 140475502112768, 140475510501375, +STORE, 140475510501376, 140475518894079, +STORE, 140475502112768, 140475510501375, +SNULL, 140475510505471, 140475518894079, +STORE, 140475510501376, 140475510505471, +STORE, 140475510505472, 140475518894079, +SNULL, 140475403067391, 140475411455999, +STORE, 140475403063296, 140475403067391, +STORE, 140475403067392, 140475411455999, +STORE, 140475394670592, 140475403063295, +SNULL, 140475394674687, 140475403063295, +STORE, 140475394670592, 140475394674687, +STORE, 140475394674688, 140475403063295, +STORE, 140475386277888, 140475394670591, +STORE, 140475377885184, 140475394670591, +STORE, 140475369492480, 140475394670591, +SNULL, 140475369496575, 140475394670591, +STORE, 140475369492480, 140475369496575, +STORE, 140475369496576, 140475394670591, +SNULL, 140475369496576, 140475377885183, +STORE, 140475377885184, 140475394670591, +STORE, 140475369496576, 140475377885183, +SNULL, 140475377889279, 140475394670591, +STORE, 140475377885184, 140475377889279, +STORE, 140475377889280, 140475394670591, +STORE, 140475285630976, 140475294023679, +SNULL, 140475377889280, 140475386277887, +STORE, 140475386277888, 140475394670591, +STORE, 140475377889280, 140475386277887, +SNULL, 140475386281983, 140475394670591, +STORE, 140475386277888, 140475386281983, +STORE, 140475386281984, 140475394670591, +SNULL, 140475285635071, 140475294023679, +STORE, 140475285630976, 140475285635071, +STORE, 140475285635072, 140475294023679, +STORE, 140475277238272, 140475285630975, +STORE, 140475268845568, 140475285630975, +SNULL, 140475268845568, 140475277238271, +STORE, 140475277238272, 140475285630975, +STORE, 140475268845568, 140475277238271, +SNULL, 140475277242367, 140475285630975, +STORE, 140475277238272, 140475277242367, +STORE, 140475277242368, 140475285630975, +STORE, 140475260452864, 140475277238271, +SNULL, 140475260452864, 140475268845567, +STORE, 140475268845568, 140475277238271, +STORE, 140475260452864, 140475268845567, +SNULL, 140475268849663, 140475277238271, +STORE, 140475268845568, 140475268849663, +STORE, 140475268849664, 140475277238271, +SNULL, 140475260456959, 140475268845567, +STORE, 140475260452864, 140475260456959, +STORE, 140475260456960, 140475268845567, +STORE, 140475252060160, 140475260452863, +SNULL, 140475252064255, 140475260452863, +STORE, 140475252060160, 140475252064255, +STORE, 140475252064256, 140475260452863, +STORE, 140475243667456, 140475252060159, +SNULL, 140475243671551, 140475252060159, +STORE, 140475243667456, 140475243671551, +STORE, 140475243671552, 140475252060159, +STORE, 140475235274752, 140475243667455, +STORE, 140475151413248, 140475159805951, +STORE, 140474891505664, 140475025588223, +STORE, 140475143020544, 140475159805951, +SNULL, 140474891505664, 140474958479359, +STORE, 140474958479360, 140475025588223, +STORE, 140474891505664, 140474958479359, +SNULL, 140474958614527, 140475025588223, +STORE, 140474958479360, 140474958614527, +STORE, 140474958614528, 140475025588223, +STORE, 140474824261632, 140474891370495, +SNULL, 140474824396799, 140474891370495, +STORE, 140474824261632, 140474824396799, +STORE, 140474824396800, 140474891370495, +STORE, 140475134627840, 140475159805951, +STORE, 140474690043904, 140474824261631, +STORE, 140475126235136, 140475159805951, +STORE, 140475117842432, 140475159805951, +STORE, 140474622935040, 140474824261631, +STORE, 140475109449728, 140475159805951, +STORE, 140474488717312, 140474824261631, +STORE, 140475101057024, 140475159805951, +STORE, 140474480324608, 140474488717311, +STORE, 140474413215744, 140474480324607, +STORE, 140474404823040, 140474413215743, +ERASE, 140474413215744, 140474480324607, +STORE, 140474471931904, 140474488717311, +STORE, 140474270605312, 140474404823039, +SNULL, 140475101057024, 140475126235135, +STORE, 140475126235136, 140475159805951, +STORE, 140475101057024, 140475126235135, +SNULL, 140475126239231, 140475159805951, +STORE, 140475126235136, 140475126239231, +STORE, 140475126239232, 140475159805951, +STORE, 140474463539200, 140474488717311, +STORE, 140474455146496, 140474488717311, +SNULL, 140474455150591, 140474488717311, +STORE, 140474455146496, 140474455150591, +STORE, 140474455150592, 140474488717311, +STORE, 140474446753792, 140474455146495, +SNULL, 140474446757887, 140474455146495, +STORE, 140474446753792, 140474446757887, +STORE, 140474446757888, 140474455146495, +STORE, 140474438361088, 140474446753791, +STORE, 140474429968384, 140474446753791, +SNULL, 140474429972479, 140474446753791, +STORE, 140474429968384, 140474429972479, +STORE, 140474429972480, 140474446753791, +SNULL, 140475235278847, 140475243667455, +STORE, 140475235274752, 140475235278847, +STORE, 140475235278848, 140475243667455, +SNULL, 140474757152767, 140474824261631, +STORE, 140474488717312, 140474757152767, +STORE, 140474757152768, 140474824261631, +ERASE, 140474757152768, 140474824261631, +SNULL, 140474488717312, 140474690043903, +STORE, 140474690043904, 140474757152767, +STORE, 140474488717312, 140474690043903, +SNULL, 140474690179071, 140474757152767, +STORE, 140474690043904, 140474690179071, +STORE, 140474690179072, 140474757152767, +SNULL, 140474488717312, 140474622935039, +STORE, 140474622935040, 140474690043903, +STORE, 140474488717312, 140474622935039, +SNULL, 140474623070207, 140474690043903, +STORE, 140474622935040, 140474623070207, +STORE, 140474623070208, 140474690043903, +SNULL, 140475101057024, 140475117842431, +STORE, 140475117842432, 140475126235135, +STORE, 140475101057024, 140475117842431, +SNULL, 140475117846527, 140475126235135, +STORE, 140475117842432, 140475117846527, +STORE, 140475117846528, 140475126235135, +SNULL, 140474555826175, 140474622935039, +STORE, 140474488717312, 140474555826175, +STORE, 140474555826176, 140474622935039, +ERASE, 140474555826176, 140474622935039, +STORE, 140474136387584, 140474404823039, +SNULL, 140474136387584, 140474153172991, +STORE, 140474153172992, 140474404823039, +STORE, 140474136387584, 140474153172991, +ERASE, 140474136387584, 140474153172991, +STORE, 140474018955264, 140474404823039, +STORE, 140473884737536, 140474404823039, +SNULL, 140474086064127, 140474404823039, +STORE, 140473884737536, 140474086064127, +STORE, 140474086064128, 140474404823039, +SNULL, 140474086064128, 140474153172991, +STORE, 140474153172992, 140474404823039, +STORE, 140474086064128, 140474153172991, +ERASE, 140474086064128, 140474153172991, +STORE, 140473750519808, 140474086064127, +SNULL, 140473817628671, 140474086064127, +STORE, 140473750519808, 140473817628671, +STORE, 140473817628672, 140474086064127, +SNULL, 140473817628672, 140473884737535, +STORE, 140473884737536, 140474086064127, +STORE, 140473817628672, 140473884737535, +ERASE, 140473817628672, 140473884737535, +SNULL, 140475126239232, 140475151413247, +STORE, 140475151413248, 140475159805951, +STORE, 140475126239232, 140475151413247, +SNULL, 140475151417343, 140475159805951, +STORE, 140475151413248, 140475151417343, +STORE, 140475151417344, 140475159805951, +SNULL, 140474270605311, 140474404823039, +STORE, 140474153172992, 140474270605311, +STORE, 140474270605312, 140474404823039, +SNULL, 140474270605312, 140474287390719, +STORE, 140474287390720, 140474404823039, +STORE, 140474270605312, 140474287390719, +ERASE, 140474270605312, 140474287390719, +SNULL, 140474429972480, 140474438361087, +STORE, 140474438361088, 140474446753791, +STORE, 140474429972480, 140474438361087, +SNULL, 140474438365183, 140474446753791, +STORE, 140474438361088, 140474438365183, +STORE, 140474438365184, 140474446753791, +STORE, 140474815868928, 140474824261631, +SNULL, 140474815873023, 140474824261631, +STORE, 140474815868928, 140474815873023, +STORE, 140474815873024, 140474824261631, +SNULL, 140474220281855, 140474270605311, +STORE, 140474153172992, 140474220281855, +STORE, 140474220281856, 140474270605311, +ERASE, 140474220281856, 140474270605311, +SNULL, 140474488852479, 140474555826175, +STORE, 140474488717312, 140474488852479, +STORE, 140474488852480, 140474555826175, +SNULL, 140475101057024, 140475109449727, +STORE, 140475109449728, 140475117842431, +STORE, 140475101057024, 140475109449727, +SNULL, 140475109453823, 140475117842431, +STORE, 140475109449728, 140475109453823, +STORE, 140475109453824, 140475117842431, +SNULL, 140473951846399, 140474086064127, +STORE, 140473884737536, 140473951846399, +STORE, 140473951846400, 140474086064127, +SNULL, 140473951846400, 140474018955263, +STORE, 140474018955264, 140474086064127, +STORE, 140473951846400, 140474018955263, +ERASE, 140473951846400, 140474018955263, +SNULL, 140473884872703, 140473951846399, +STORE, 140473884737536, 140473884872703, +STORE, 140473884872704, 140473951846399, +SNULL, 140474019090431, 140474086064127, +STORE, 140474018955264, 140474019090431, +STORE, 140474019090432, 140474086064127, +SNULL, 140473750654975, 140473817628671, +STORE, 140473750519808, 140473750654975, +STORE, 140473750654976, 140473817628671, +SNULL, 140474455150592, 140474463539199, +STORE, 140474463539200, 140474488717311, +STORE, 140474455150592, 140474463539199, +SNULL, 140474463543295, 140474488717311, +STORE, 140474463539200, 140474463543295, +STORE, 140474463543296, 140474488717311, +STORE, 140474807476224, 140474815868927, +SNULL, 140474463543296, 140474471931903, +STORE, 140474471931904, 140474488717311, +STORE, 140474463543296, 140474471931903, +SNULL, 140474471935999, 140474488717311, +STORE, 140474471931904, 140474471935999, +STORE, 140474471936000, 140474488717311, +STORE, 140474799083520, 140474815868927, +STORE, 140474790690816, 140474815868927, +SNULL, 140474790690816, 140474799083519, +STORE, 140474799083520, 140474815868927, +STORE, 140474790690816, 140474799083519, +SNULL, 140474799087615, 140474815868927, +STORE, 140474799083520, 140474799087615, +STORE, 140474799087616, 140474815868927, +SNULL, 140474354499583, 140474404823039, +STORE, 140474287390720, 140474354499583, +STORE, 140474354499584, 140474404823039, +ERASE, 140474354499584, 140474404823039, +SNULL, 140474287525887, 140474354499583, +STORE, 140474287390720, 140474287525887, +STORE, 140474287525888, 140474354499583, +STORE, 140474782298112, 140474799083519, +STORE, 140474773905408, 140474799083519, +SNULL, 140474773909503, 140474799083519, +STORE, 140474773905408, 140474773909503, +STORE, 140474773909504, 140474799083519, +SNULL, 140475126239232, 140475134627839, +STORE, 140475134627840, 140475151413247, +STORE, 140475126239232, 140475134627839, +SNULL, 140475134631935, 140475151413247, +STORE, 140475134627840, 140475134631935, +STORE, 140475134631936, 140475151413247, +STORE, 140474765512704, 140474773905407, +STORE, 140474614542336, 140474622935039, +SNULL, 140474153308159, 140474220281855, +STORE, 140474153172992, 140474153308159, +STORE, 140474153308160, 140474220281855, +SNULL, 140474404827135, 140474413215743, +STORE, 140474404823040, 140474404827135, +STORE, 140474404827136, 140474413215743, +STORE, 140474606149632, 140474622935039, +SNULL, 140474606153727, 140474622935039, +STORE, 140474606149632, 140474606153727, +STORE, 140474606153728, 140474622935039, +STORE, 140474597756928, 140474606149631, +SNULL, 140474597761023, 140474606149631, +STORE, 140474597756928, 140474597761023, +STORE, 140474597761024, 140474606149631, +SNULL, 140475134631936, 140475143020543, +STORE, 140475143020544, 140475151413247, +STORE, 140475134631936, 140475143020543, +SNULL, 140475143024639, 140475151413247, +STORE, 140475143020544, 140475143024639, +STORE, 140475143024640, 140475151413247, +STORE, 140474589364224, 140474597756927, +SNULL, 140474606153728, 140474614542335, +STORE, 140474614542336, 140474622935039, +STORE, 140474606153728, 140474614542335, +SNULL, 140474614546431, 140474622935039, +STORE, 140474614542336, 140474614546431, +STORE, 140474614546432, 140474622935039, +SNULL, 140474765516799, 140474773905407, +STORE, 140474765512704, 140474765516799, +STORE, 140474765516800, 140474773905407, +STORE, 140474580971520, 140474597756927, +SNULL, 140474773909504, 140474782298111, +STORE, 140474782298112, 140474799083519, +STORE, 140474773909504, 140474782298111, +SNULL, 140474782302207, 140474799083519, +STORE, 140474782298112, 140474782302207, +STORE, 140474782302208, 140474799083519, +SNULL, 140474471936000, 140474480324607, +STORE, 140474480324608, 140474488717311, +STORE, 140474471936000, 140474480324607, +SNULL, 140474480328703, 140474488717311, +STORE, 140474480324608, 140474480328703, +STORE, 140474480328704, 140474488717311, +STORE, 140474572578816, 140474597756927, +SNULL, 140474572582911, 140474597756927, +STORE, 140474572578816, 140474572582911, +STORE, 140474572582912, 140474597756927, +SNULL, 140474782302208, 140474790690815, +STORE, 140474790690816, 140474799083519, +STORE, 140474782302208, 140474790690815, +SNULL, 140474790694911, 140474799083519, +STORE, 140474790690816, 140474790694911, +STORE, 140474790694912, 140474799083519, +STORE, 140474564186112, 140474572578815, +STORE, 140474421575680, 140474429968383, +STORE, 140474396430336, 140474404823039, +SNULL, 140474396434431, 140474404823039, +STORE, 140474396430336, 140474396434431, +STORE, 140474396434432, 140474404823039, +STORE, 140474388037632, 140474396430335, +SNULL, 140474799087616, 140474807476223, +STORE, 140474807476224, 140474815868927, +STORE, 140474799087616, 140474807476223, +SNULL, 140474807480319, 140474815868927, +STORE, 140474807476224, 140474807480319, +STORE, 140474807480320, 140474815868927, +SNULL, 140475101061119, 140475109449727, +STORE, 140475101057024, 140475101061119, +STORE, 140475101061120, 140475109449727, +STORE, 140474379644928, 140474396430335, +SNULL, 140474572582912, 140474589364223, +STORE, 140474589364224, 140474597756927, +STORE, 140474572582912, 140474589364223, +SNULL, 140474589368319, 140474597756927, +STORE, 140474589364224, 140474589368319, +STORE, 140474589368320, 140474597756927, +STORE, 140474371252224, 140474396430335, +STORE, 140474362859520, 140474396430335, +STORE, 140474278998016, 140474287390719, +STORE, 140474270605312, 140474287390719, +STORE, 140474262212608, 140474287390719, +SNULL, 140474262216703, 140474287390719, +STORE, 140474262212608, 140474262216703, +STORE, 140474262216704, 140474287390719, +STORE, 140474253819904, 140474262212607, +SNULL, 140474253823999, 140474262212607, +STORE, 140474253819904, 140474253823999, +STORE, 140474253824000, 140474262212607, +SNULL, 140474362859520, 140474388037631, +STORE, 140474388037632, 140474396430335, +STORE, 140474362859520, 140474388037631, +SNULL, 140474388041727, 140474396430335, +STORE, 140474388037632, 140474388041727, +STORE, 140474388041728, 140474396430335, +SNULL, 140474362859520, 140474379644927, +STORE, 140474379644928, 140474388037631, +STORE, 140474362859520, 140474379644927, +SNULL, 140474379649023, 140474388037631, +STORE, 140474379644928, 140474379649023, +STORE, 140474379649024, 140474388037631, +STORE, 140474245427200, 140474253819903, +STORE, 140474237034496, 140474253819903, +STORE, 140474228641792, 140474253819903, +STORE, 140474144780288, 140474153172991, +SNULL, 140474228645887, 140474253819903, +STORE, 140474228641792, 140474228645887, +STORE, 140474228645888, 140474253819903, +SNULL, 140474564190207, 140474572578815, +STORE, 140474564186112, 140474564190207, +STORE, 140474564190208, 140474572578815, +STORE, 140474136387584, 140474153172991, +SNULL, 140474362859520, 140474371252223, +STORE, 140474371252224, 140474379644927, +STORE, 140474362859520, 140474371252223, +SNULL, 140474371256319, 140474379644927, +STORE, 140474371252224, 140474371256319, +STORE, 140474371256320, 140474379644927, +STORE, 140474127994880, 140474153172991, +STORE, 140474119602176, 140474153172991, +SNULL, 140474421579775, 140474429968383, +STORE, 140474421575680, 140474421579775, +STORE, 140474421579776, 140474429968383, +STORE, 140474111209472, 140474153172991, +SNULL, 140474111213567, 140474153172991, +STORE, 140474111209472, 140474111213567, +STORE, 140474111213568, 140474153172991, +SNULL, 140474262216704, 140474270605311, +STORE, 140474270605312, 140474287390719, +STORE, 140474262216704, 140474270605311, +SNULL, 140474270609407, 140474287390719, +STORE, 140474270605312, 140474270609407, +STORE, 140474270609408, 140474287390719, +STORE, 140474102816768, 140474111209471, +SNULL, 140474102820863, 140474111209471, +STORE, 140474102816768, 140474102820863, +STORE, 140474102820864, 140474111209471, +SNULL, 140474270609408, 140474278998015, +STORE, 140474278998016, 140474287390719, +STORE, 140474270609408, 140474278998015, +SNULL, 140474279002111, 140474287390719, +STORE, 140474278998016, 140474279002111, +STORE, 140474279002112, 140474287390719, +STORE, 140474094424064, 140474102816767, +SNULL, 140474572582912, 140474580971519, +STORE, 140474580971520, 140474589364223, +STORE, 140474572582912, 140474580971519, +SNULL, 140474580975615, 140474589364223, +STORE, 140474580971520, 140474580975615, +STORE, 140474580975616, 140474589364223, +SNULL, 140474362863615, 140474371252223, +STORE, 140474362859520, 140474362863615, +STORE, 140474362863616, 140474371252223, +STORE, 140474010562560, 140474018955263, +SNULL, 140474228645888, 140474245427199, +STORE, 140474245427200, 140474253819903, +STORE, 140474228645888, 140474245427199, +SNULL, 140474245431295, 140474253819903, +STORE, 140474245427200, 140474245431295, +STORE, 140474245431296, 140474253819903, +SNULL, 140474111213568, 140474136387583, +STORE, 140474136387584, 140474153172991, +STORE, 140474111213568, 140474136387583, +SNULL, 140474136391679, 140474153172991, +STORE, 140474136387584, 140474136391679, +STORE, 140474136391680, 140474153172991, +STORE, 140474002169856, 140474018955263, +STORE, 140473993777152, 140474018955263, +SNULL, 140474111213568, 140474127994879, +STORE, 140474127994880, 140474136387583, +STORE, 140474111213568, 140474127994879, +SNULL, 140474127998975, 140474136387583, +STORE, 140474127994880, 140474127998975, +STORE, 140474127998976, 140474136387583, +SNULL, 140474228645888, 140474237034495, +STORE, 140474237034496, 140474245427199, +STORE, 140474228645888, 140474237034495, +SNULL, 140474237038591, 140474245427199, +STORE, 140474237034496, 140474237038591, +STORE, 140474237038592, 140474245427199, +SNULL, 140474136391680, 140474144780287, +STORE, 140474144780288, 140474153172991, +STORE, 140474136391680, 140474144780287, +SNULL, 140474144784383, 140474153172991, +STORE, 140474144780288, 140474144784383, +STORE, 140474144784384, 140474153172991, +STORE, 140473985384448, 140474018955263, +STORE, 140473976991744, 140474018955263, +STORE, 140473968599040, 140474018955263, +SNULL, 140473968603135, 140474018955263, +STORE, 140473968599040, 140473968603135, +STORE, 140473968603136, 140474018955263, +SNULL, 140474111213568, 140474119602175, +STORE, 140474119602176, 140474127994879, +STORE, 140474111213568, 140474119602175, +SNULL, 140474119606271, 140474127994879, +STORE, 140474119602176, 140474119606271, +STORE, 140474119606272, 140474127994879, +STORE, 140473960206336, 140473968599039, +SNULL, 140474094428159, 140474102816767, +STORE, 140474094424064, 140474094428159, +STORE, 140474094428160, 140474102816767, +STORE, 140473876344832, 140473884737535, +STORE, 140473867952128, 140473884737535, +STORE, 140473859559424, 140473884737535, +SNULL, 140473859563519, 140473884737535, +STORE, 140473859559424, 140473859563519, +STORE, 140473859563520, 140473884737535, +SNULL, 140473968603136, 140473993777151, +STORE, 140473993777152, 140474018955263, +STORE, 140473968603136, 140473993777151, +SNULL, 140473993781247, 140474018955263, +STORE, 140473993777152, 140473993781247, +STORE, 140473993781248, 140474018955263, +SNULL, 140473960210431, 140473968599039, +STORE, 140473960206336, 140473960210431, +STORE, 140473960210432, 140473968599039, +SNULL, 140473993781248, 140474010562559, +STORE, 140474010562560, 140474018955263, +STORE, 140473993781248, 140474010562559, +SNULL, 140474010566655, 140474018955263, +STORE, 140474010562560, 140474010566655, +STORE, 140474010566656, 140474018955263, +SNULL, 140473968603136, 140473985384447, +STORE, 140473985384448, 140473993777151, +STORE, 140473968603136, 140473985384447, +SNULL, 140473985388543, 140473993777151, +STORE, 140473985384448, 140473985388543, +STORE, 140473985388544, 140473993777151, +SNULL, 140473993781248, 140474002169855, +STORE, 140474002169856, 140474010562559, +STORE, 140473993781248, 140474002169855, +SNULL, 140474002173951, 140474010562559, +STORE, 140474002169856, 140474002173951, +STORE, 140474002173952, 140474010562559, +STORE, 140473851166720, 140473859559423, +SNULL, 140473851170815, 140473859559423, +STORE, 140473851166720, 140473851170815, +STORE, 140473851170816, 140473859559423, +SNULL, 140473968603136, 140473976991743, +STORE, 140473976991744, 140473985384447, +STORE, 140473968603136, 140473976991743, +SNULL, 140473976995839, 140473985384447, +STORE, 140473976991744, 140473976995839, +STORE, 140473976995840, 140473985384447, +STORE, 140473842774016, 140473851166719, +SNULL, 140473859563520, 140473867952127, +STORE, 140473867952128, 140473884737535, +STORE, 140473859563520, 140473867952127, +SNULL, 140473867956223, 140473884737535, +STORE, 140473867952128, 140473867956223, +STORE, 140473867956224, 140473884737535, +SNULL, 140473867956224, 140473876344831, +STORE, 140473876344832, 140473884737535, +STORE, 140473867956224, 140473876344831, +SNULL, 140473876348927, 140473884737535, +STORE, 140473876344832, 140473876348927, +STORE, 140473876348928, 140473884737535, +STORE, 140473834381312, 140473851166719, +SNULL, 140473834385407, 140473851166719, +STORE, 140473834381312, 140473834385407, +STORE, 140473834385408, 140473851166719, +SNULL, 140473834385408, 140473842774015, +STORE, 140473842774016, 140473851166719, +STORE, 140473834385408, 140473842774015, +SNULL, 140473842778111, 140473851166719, +STORE, 140473842774016, 140473842778111, +STORE, 140473842778112, 140473851166719, +STORE, 140473825988608, 140473834381311, +SNULL, 140473825992703, 140473834381311, +STORE, 140473825988608, 140473825992703, +STORE, 140473825992704, 140473834381311, +STORE, 140475577475072, 140475577503743, +STORE, 140475499917312, 140475502108671, +SNULL, 140475499917312, 140475500007423, +STORE, 140475500007424, 140475502108671, +STORE, 140475499917312, 140475500007423, +SNULL, 140475502100479, 140475502108671, +STORE, 140475500007424, 140475502100479, +STORE, 140475502100480, 140475502108671, +ERASE, 140475502100480, 140475502108671, +STORE, 140475502100480, 140475502108671, +SNULL, 140475502104575, 140475502108671, +STORE, 140475502100480, 140475502104575, +STORE, 140475502104576, 140475502108671, +ERASE, 140475577475072, 140475577503743, +ERASE, 140475235274752, 140475235278847, +ERASE, 140475235278848, 140475243667455, +ERASE, 140474815868928, 140474815873023, +ERASE, 140474815873024, 140474824261631, +ERASE, 140474606149632, 140474606153727, +ERASE, 140474606153728, 140474614542335, +ERASE, 140474270605312, 140474270609407, +ERASE, 140474270609408, 140474278998015, +ERASE, 140474438361088, 140474438365183, +ERASE, 140474438365184, 140474446753791, +ERASE, 140474597756928, 140474597761023, +ERASE, 140474597761024, 140474606149631, +ERASE, 140475126235136, 140475126239231, +ERASE, 140475126239232, 140475134627839, +ERASE, 140474463539200, 140474463543295, +ERASE, 140474463543296, 140474471931903, +ERASE, 140474388037632, 140474388041727, +ERASE, 140474388041728, 140474396430335, +ERASE, 140474404823040, 140474404827135, +ERASE, 140474404827136, 140474413215743, +ERASE, 140474278998016, 140474279002111, +ERASE, 140474279002112, 140474287390719, +ERASE, 140474094424064, 140474094428159, +ERASE, 140474094428160, 140474102816767, +ERASE, 140473867952128, 140473867956223, +ERASE, 140473867956224, 140473876344831, +ERASE, 140475151413248, 140475151417343, +ERASE, 140475151417344, 140475159805951, +ERASE, 140474455146496, 140474455150591, +ERASE, 140474455150592, 140474463539199, +ERASE, 140474807476224, 140474807480319, +ERASE, 140474807480320, 140474815868927, +ERASE, 140475117842432, 140475117846527, +ERASE, 140475117846528, 140475126235135, +ERASE, 140474446753792, 140474446757887, +ERASE, 140474446757888, 140474455146495, +ERASE, 140474429968384, 140474429972479, +ERASE, 140474429972480, 140474438361087, +ERASE, 140474782298112, 140474782302207, +ERASE, 140474782302208, 140474790690815, +ERASE, 140474136387584, 140474136391679, +ERASE, 140474136391680, 140474144780287, +ERASE, 140474002169856, 140474002173951, +ERASE, 140474002173952, 140474010562559, +ERASE, 140475134627840, 140475134631935, +ERASE, 140475134631936, 140475143020543, +ERASE, 140474471931904, 140474471935999, +ERASE, 140474471936000, 140474480324607, +ERASE, 140474396430336, 140474396434431, +ERASE, 140474396434432, 140474404823039, + }; + unsigned long set36[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140723893125120, 140737488351231, +SNULL, 140723893129215, 140737488351231, +STORE, 140723893125120, 140723893129215, +STORE, 140723892994048, 140723893129215, +STORE, 94076829786112, 94076832038911, +SNULL, 94076829917183, 94076832038911, +STORE, 94076829786112, 94076829917183, +STORE, 94076829917184, 94076832038911, +ERASE, 94076829917184, 94076832038911, +STORE, 94076832010240, 94076832018431, +STORE, 94076832018432, 94076832038911, +STORE, 140122444345344, 140122446598143, +SNULL, 140122444488703, 140122446598143, +STORE, 140122444345344, 140122444488703, +STORE, 140122444488704, 140122446598143, +ERASE, 140122444488704, 140122446598143, +STORE, 140122446585856, 140122446594047, +STORE, 140122446594048, 140122446598143, +STORE, 140723893538816, 140723893542911, +STORE, 140723893526528, 140723893538815, +STORE, 140122446557184, 140122446585855, +STORE, 140122446548992, 140122446557183, +STORE, 140122442129408, 140122444345343, +SNULL, 140122442129408, 140122442227711, +STORE, 140122442227712, 140122444345343, +STORE, 140122442129408, 140122442227711, +SNULL, 140122444320767, 140122444345343, +STORE, 140122442227712, 140122444320767, +STORE, 140122444320768, 140122444345343, +SNULL, 140122444320768, 140122444328959, +STORE, 140122444328960, 140122444345343, +STORE, 140122444320768, 140122444328959, +ERASE, 140122444320768, 140122444328959, +STORE, 140122444320768, 140122444328959, +ERASE, 140122444328960, 140122444345343, +STORE, 140122444328960, 140122444345343, +STORE, 140122438332416, 140122442129407, +SNULL, 140122438332416, 140122439991295, +STORE, 140122439991296, 140122442129407, +STORE, 140122438332416, 140122439991295, +SNULL, 140122442088447, 140122442129407, +STORE, 140122439991296, 140122442088447, +STORE, 140122442088448, 140122442129407, +SNULL, 140122442088448, 140122442113023, +STORE, 140122442113024, 140122442129407, +STORE, 140122442088448, 140122442113023, +ERASE, 140122442088448, 140122442113023, +STORE, 140122442088448, 140122442113023, +ERASE, 140122442113024, 140122442129407, +STORE, 140122442113024, 140122442129407, +STORE, 140122446540800, 140122446557183, +SNULL, 140122442104831, 140122442113023, +STORE, 140122442088448, 140122442104831, +STORE, 140122442104832, 140122442113023, +SNULL, 140122444324863, 140122444328959, +STORE, 140122444320768, 140122444324863, +STORE, 140122444324864, 140122444328959, +SNULL, 94076832014335, 94076832018431, +STORE, 94076832010240, 94076832014335, +STORE, 94076832014336, 94076832018431, +SNULL, 140122446589951, 140122446594047, +STORE, 140122446585856, 140122446589951, +STORE, 140122446589952, 140122446594047, +ERASE, 140122446557184, 140122446585855, +STORE, 94076845723648, 94076845858815, +STORE, 140122429939712, 140122438332415, +SNULL, 140122429943807, 140122438332415, +STORE, 140122429939712, 140122429943807, +STORE, 140122429943808, 140122438332415, +STORE, 140122421547008, 140122429939711, +STORE, 140122287329280, 140122421547007, +SNULL, 140122287329280, 140122301399039, +STORE, 140122301399040, 140122421547007, +STORE, 140122287329280, 140122301399039, +ERASE, 140122287329280, 140122301399039, +SNULL, 140122368507903, 140122421547007, +STORE, 140122301399040, 140122368507903, +STORE, 140122368507904, 140122421547007, +ERASE, 140122368507904, 140122421547007, +SNULL, 140122301534207, 140122368507903, +STORE, 140122301399040, 140122301534207, +STORE, 140122301534208, 140122368507903, +SNULL, 140122421551103, 140122429939711, +STORE, 140122421547008, 140122421551103, +STORE, 140122421551104, 140122429939711, +STORE, 140122413154304, 140122421547007, +SNULL, 140122413158399, 140122421547007, +STORE, 140122413154304, 140122413158399, +STORE, 140122413158400, 140122421547007, +STORE, 140122404761600, 140122413154303, +SNULL, 140122404765695, 140122413154303, +STORE, 140122404761600, 140122404765695, +STORE, 140122404765696, 140122413154303, +STORE, 140122396368896, 140122404761599, +SNULL, 140122396372991, 140122404761599, +STORE, 140122396368896, 140122396372991, +STORE, 140122396372992, 140122404761599, +STORE, 140122387976192, 140122396368895, +STORE, 140122167181312, 140122301399039, +SNULL, 140122234290175, 140122301399039, +STORE, 140122167181312, 140122234290175, +STORE, 140122234290176, 140122301399039, +ERASE, 140122234290176, 140122301399039, +SNULL, 140122167316479, 140122234290175, +STORE, 140122167181312, 140122167316479, +STORE, 140122167316480, 140122234290175, +STORE, 140122379583488, 140122396368895, +STORE, 140122371190784, 140122396368895, +STORE, 140122167316480, 140122301399039, +STORE, 140122158788608, 140122167181311, +SNULL, 140122371190784, 140122387976191, +STORE, 140122387976192, 140122396368895, +STORE, 140122371190784, 140122387976191, +SNULL, 140122387980287, 140122396368895, +STORE, 140122387976192, 140122387980287, +STORE, 140122387980288, 140122396368895, +SNULL, 140122167316480, 140122234290175, +STORE, 140122234290176, 140122301399039, +STORE, 140122167316480, 140122234290175, +SNULL, 140122234425343, 140122301399039, +STORE, 140122234290176, 140122234425343, +STORE, 140122234425344, 140122301399039, +STORE, 140122024570880, 140122158788607, +SNULL, 140122024570880, 140122032963583, +STORE, 140122032963584, 140122158788607, +STORE, 140122024570880, 140122032963583, +ERASE, 140122024570880, 140122032963583, +STORE, 140121898745856, 140122158788607, +STORE, 140121890353152, 140121898745855, +SNULL, 140122100072447, 140122158788607, +STORE, 140121898745856, 140122100072447, +STORE, 140122100072448, 140122158788607, +ERASE, 140122100072448, 140122158788607, +SNULL, 140121965854719, 140122100072447, +STORE, 140121898745856, 140121965854719, +STORE, 140121965854720, 140122100072447, +SNULL, 140121965854720, 140122032963583, +STORE, 140122032963584, 140122100072447, +STORE, 140121965854720, 140122032963583, +ERASE, 140121965854720, 140122032963583, +SNULL, 140121898881023, 140121965854719, +STORE, 140121898745856, 140121898881023, +STORE, 140121898881024, 140121965854719, +SNULL, 140121890357247, 140121898745855, +STORE, 140121890353152, 140121890357247, +STORE, 140121890357248, 140121898745855, +SNULL, 140122371190784, 140122379583487, +STORE, 140122379583488, 140122387976191, +STORE, 140122371190784, 140122379583487, +SNULL, 140122379587583, 140122387976191, +STORE, 140122379583488, 140122379587583, +STORE, 140122379587584, 140122387976191, +SNULL, 140122033098751, 140122100072447, +STORE, 140122032963584, 140122033098751, +STORE, 140122033098752, 140122100072447, +SNULL, 140122158792703, 140122167181311, +STORE, 140122158788608, 140122158792703, +STORE, 140122158792704, 140122167181311, +STORE, 140122150395904, 140122158788607, +STORE, 140122142003200, 140122158788607, +SNULL, 140122142007295, 140122158788607, +STORE, 140122142003200, 140122142007295, +STORE, 140122142007296, 140122158788607, +SNULL, 140122371194879, 140122379583487, +STORE, 140122371190784, 140122371194879, +STORE, 140122371194880, 140122379583487, +SNULL, 140122142007296, 140122150395903, +STORE, 140122150395904, 140122158788607, +STORE, 140122142007296, 140122150395903, +SNULL, 140122150399999, 140122158788607, +STORE, 140122150395904, 140122150399999, +STORE, 140122150400000, 140122158788607, +STORE, 140122133610496, 140122142003199, +STORE, 140122125217792, 140122142003199, +STORE, 140122116825088, 140122142003199, +SNULL, 140122116829183, 140122142003199, +STORE, 140122116825088, 140122116829183, +STORE, 140122116829184, 140122142003199, +SNULL, 140122116829184, 140122133610495, +STORE, 140122133610496, 140122142003199, +STORE, 140122116829184, 140122133610495, +SNULL, 140122133614591, 140122142003199, +STORE, 140122133610496, 140122133614591, +STORE, 140122133614592, 140122142003199, +SNULL, 140122116829184, 140122125217791, +STORE, 140122125217792, 140122133610495, +STORE, 140122116829184, 140122125217791, +SNULL, 140122125221887, 140122133610495, +STORE, 140122125217792, 140122125221887, +STORE, 140122125221888, 140122133610495, +STORE, 140122108432384, 140122116825087, +SNULL, 140122108436479, 140122116825087, +STORE, 140122108432384, 140122108436479, +STORE, 140122108436480, 140122116825087, +STORE, 140122024570880, 140122032963583, +STORE, 140122016178176, 140122032963583, +SNULL, 140122016182271, 140122032963583, +STORE, 140122016178176, 140122016182271, +STORE, 140122016182272, 140122032963583, +SNULL, 140122016182272, 140122024570879, +STORE, 140122024570880, 140122032963583, +STORE, 140122016182272, 140122024570879, +SNULL, 140122024574975, 140122032963583, +STORE, 140122024570880, 140122024574975, +STORE, 140122024574976, 140122032963583, +STORE, 140122007785472, 140122016178175, +SNULL, 140122007789567, 140122016178175, +STORE, 140122007785472, 140122007789567, +STORE, 140122007789568, 140122016178175, +STORE, 140121999392768, 140122007785471, +STORE, 140121991000064, 140122007785471, +SNULL, 140121991004159, 140122007785471, +STORE, 140121991000064, 140121991004159, +STORE, 140121991004160, 140122007785471, +SNULL, 140121991004160, 140121999392767, +STORE, 140121999392768, 140122007785471, +STORE, 140121991004160, 140121999392767, +SNULL, 140121999396863, 140122007785471, +STORE, 140121999392768, 140121999396863, +STORE, 140121999396864, 140122007785471, +STORE, 140121982607360, 140121991000063, +STORE, 140121823244288, 140121890353151, +ERASE, 140121823244288, 140121890353151, +STORE, 140121756135424, 140121890353151, +SNULL, 140121756135424, 140121764528127, +STORE, 140121764528128, 140121890353151, +STORE, 140121756135424, 140121764528127, +ERASE, 140121756135424, 140121764528127, +SNULL, 140121831636991, 140121890353151, +STORE, 140121764528128, 140121831636991, +STORE, 140121831636992, 140121890353151, +ERASE, 140121831636992, 140121890353151, +STORE, 140121974214656, 140121991000063, +STORE, 140121630310400, 140121831636991, +SNULL, 140121697419263, 140121831636991, +STORE, 140121630310400, 140121697419263, +STORE, 140121697419264, 140121831636991, +SNULL, 140121697419264, 140121764528127, +STORE, 140121764528128, 140121831636991, +STORE, 140121697419264, 140121764528127, +ERASE, 140121697419264, 140121764528127, +STORE, 140121881960448, 140121890353151, +STORE, 140121630310400, 140121831636991, +STORE, 140121873567744, 140121890353151, +SNULL, 140121630310400, 140121697419263, +STORE, 140121697419264, 140121831636991, +STORE, 140121630310400, 140121697419263, +SNULL, 140121697554431, 140121831636991, +STORE, 140121697419264, 140121697554431, +STORE, 140121697554432, 140121831636991, +STORE, 140121865175040, 140121890353151, +STORE, 140121856782336, 140121890353151, +STORE, 140121848389632, 140121890353151, +STORE, 140121839996928, 140121890353151, +STORE, 140121496092672, 140121697419263, +STORE, 140121487699968, 140121496092671, +STORE, 140121420591104, 140121487699967, +STORE, 140121412198400, 140121420591103, +ERASE, 140121420591104, 140121487699967, +STORE, 140121479307264, 140121496092671, +STORE, 140121277980672, 140121412198399, +SNULL, 140121277980672, 140121294766079, +STORE, 140121294766080, 140121412198399, +STORE, 140121277980672, 140121294766079, +ERASE, 140121277980672, 140121294766079, +STORE, 140121470914560, 140121496092671, +STORE, 140121462521856, 140121496092671, +STORE, 140121160548352, 140121412198399, +STORE, 140121454129152, 140121496092671, +SNULL, 140121227657215, 140121412198399, +STORE, 140121160548352, 140121227657215, +STORE, 140121227657216, 140121412198399, +SNULL, 140121227657216, 140121294766079, +STORE, 140121294766080, 140121412198399, +STORE, 140121227657216, 140121294766079, +ERASE, 140121227657216, 140121294766079, +STORE, 140121445736448, 140121496092671, +STORE, 140121437343744, 140121496092671, +SNULL, 140121437343744, 140121445736447, +STORE, 140121445736448, 140121496092671, +STORE, 140121437343744, 140121445736447, +SNULL, 140121445740543, 140121496092671, +STORE, 140121445736448, 140121445740543, +STORE, 140121445740544, 140121496092671, +SNULL, 140121697554432, 140121764528127, +STORE, 140121764528128, 140121831636991, +STORE, 140121697554432, 140121764528127, +SNULL, 140121764663295, 140121831636991, +STORE, 140121764528128, 140121764663295, +STORE, 140121764663296, 140121831636991, +SNULL, 140121496092672, 140121630310399, +STORE, 140121630310400, 140121697419263, +STORE, 140121496092672, 140121630310399, +SNULL, 140121630445567, 140121697419263, +STORE, 140121630310400, 140121630445567, +STORE, 140121630445568, 140121697419263, +SNULL, 140121445740544, 140121454129151, +STORE, 140121454129152, 140121496092671, +STORE, 140121445740544, 140121454129151, +SNULL, 140121454133247, 140121496092671, +STORE, 140121454129152, 140121454133247, +STORE, 140121454133248, 140121496092671, +STORE, 140121026330624, 140121227657215, +SNULL, 140121093439487, 140121227657215, +STORE, 140121026330624, 140121093439487, +STORE, 140121093439488, 140121227657215, +SNULL, 140121093439488, 140121160548351, +STORE, 140121160548352, 140121227657215, +STORE, 140121093439488, 140121160548351, +ERASE, 140121093439488, 140121160548351, +SNULL, 140121563201535, 140121630310399, +STORE, 140121496092672, 140121563201535, +STORE, 140121563201536, 140121630310399, +ERASE, 140121563201536, 140121630310399, +STORE, 140120892112896, 140121093439487, +SNULL, 140120959221759, 140121093439487, +STORE, 140120892112896, 140120959221759, +STORE, 140120959221760, 140121093439487, +SNULL, 140120959221760, 140121026330623, +STORE, 140121026330624, 140121093439487, +STORE, 140120959221760, 140121026330623, +ERASE, 140120959221760, 140121026330623, +STORE, 140120757895168, 140120959221759, +SNULL, 140121361874943, 140121412198399, +STORE, 140121294766080, 140121361874943, +STORE, 140121361874944, 140121412198399, +ERASE, 140121361874944, 140121412198399, +SNULL, 140121294901247, 140121361874943, +STORE, 140121294766080, 140121294901247, +STORE, 140121294901248, 140121361874943, +STORE, 140120623677440, 140120959221759, +SNULL, 140120690786303, 140120959221759, +STORE, 140120623677440, 140120690786303, +STORE, 140120690786304, 140120959221759, +SNULL, 140120690786304, 140120757895167, +STORE, 140120757895168, 140120959221759, +STORE, 140120690786304, 140120757895167, +ERASE, 140120690786304, 140120757895167, +SNULL, 140121160683519, 140121227657215, +STORE, 140121160548352, 140121160683519, +STORE, 140121160683520, 140121227657215, +SNULL, 140121974214656, 140121982607359, +STORE, 140121982607360, 140121991000063, +STORE, 140121974214656, 140121982607359, +SNULL, 140121982611455, 140121991000063, +STORE, 140121982607360, 140121982611455, +STORE, 140121982611456, 140121991000063, +SNULL, 140121839996928, 140121873567743, +STORE, 140121873567744, 140121890353151, +STORE, 140121839996928, 140121873567743, +SNULL, 140121873571839, 140121890353151, +STORE, 140121873567744, 140121873571839, +STORE, 140121873571840, 140121890353151, +SNULL, 140121873571840, 140121881960447, +STORE, 140121881960448, 140121890353151, +STORE, 140121873571840, 140121881960447, +SNULL, 140121881964543, 140121890353151, +STORE, 140121881960448, 140121881964543, +STORE, 140121881964544, 140121890353151, +SNULL, 140121840001023, 140121873567743, +STORE, 140121839996928, 140121840001023, +STORE, 140121840001024, 140121873567743, +SNULL, 140121840001024, 140121865175039, +STORE, 140121865175040, 140121873567743, +STORE, 140121840001024, 140121865175039, +SNULL, 140121865179135, 140121873567743, +STORE, 140121865175040, 140121865179135, +STORE, 140121865179136, 140121873567743, +SNULL, 140121437347839, 140121445736447, +STORE, 140121437343744, 140121437347839, +STORE, 140121437347840, 140121445736447, +STORE, 140121621917696, 140121630310399, +STORE, 140121613524992, 140121630310399, +SNULL, 140121026465791, 140121093439487, +STORE, 140121026330624, 140121026465791, +STORE, 140121026465792, 140121093439487, +SNULL, 140121496227839, 140121563201535, +STORE, 140121496092672, 140121496227839, +STORE, 140121496227840, 140121563201535, +SNULL, 140120757895168, 140120892112895, +STORE, 140120892112896, 140120959221759, +STORE, 140120757895168, 140120892112895, +SNULL, 140120892248063, 140120959221759, +STORE, 140120892112896, 140120892248063, +STORE, 140120892248064, 140120959221759, +SNULL, 140120825004031, 140120892112895, +STORE, 140120757895168, 140120825004031, +STORE, 140120825004032, 140120892112895, +ERASE, 140120825004032, 140120892112895, +SNULL, 140120623812607, 140120690786303, +STORE, 140120623677440, 140120623812607, +STORE, 140120623812608, 140120690786303, +SNULL, 140120758030335, 140120825004031, +STORE, 140120757895168, 140120758030335, +STORE, 140120758030336, 140120825004031, +SNULL, 140121454133248, 140121462521855, +STORE, 140121462521856, 140121496092671, +STORE, 140121454133248, 140121462521855, +SNULL, 140121462525951, 140121496092671, +STORE, 140121462521856, 140121462525951, +STORE, 140121462525952, 140121496092671, +STORE, 140121605132288, 140121630310399, +SNULL, 140121605136383, 140121630310399, +STORE, 140121605132288, 140121605136383, +STORE, 140121605136384, 140121630310399, +STORE, 140121596739584, 140121605132287, +SNULL, 140121605136384, 140121621917695, +STORE, 140121621917696, 140121630310399, +STORE, 140121605136384, 140121621917695, +SNULL, 140121621921791, 140121630310399, +STORE, 140121621917696, 140121621921791, +STORE, 140121621921792, 140121630310399, +STORE, 140121588346880, 140121605132287, +STORE, 140121579954176, 140121605132287, +SNULL, 140121412202495, 140121420591103, +STORE, 140121412198400, 140121412202495, +STORE, 140121412202496, 140121420591103, +SNULL, 140121974218751, 140121982607359, +STORE, 140121974214656, 140121974218751, +STORE, 140121974218752, 140121982607359, +SNULL, 140121462525952, 140121479307263, +STORE, 140121479307264, 140121496092671, +STORE, 140121462525952, 140121479307263, +SNULL, 140121479311359, 140121496092671, +STORE, 140121479307264, 140121479311359, +STORE, 140121479311360, 140121496092671, +STORE, 140121571561472, 140121605132287, +SNULL, 140121571565567, 140121605132287, +STORE, 140121571561472, 140121571565567, +STORE, 140121571565568, 140121605132287, +STORE, 140121428951040, 140121437343743, +SNULL, 140121428955135, 140121437343743, +STORE, 140121428951040, 140121428955135, +STORE, 140121428955136, 140121437343743, +SNULL, 140121840001024, 140121856782335, +STORE, 140121856782336, 140121865175039, +STORE, 140121840001024, 140121856782335, +SNULL, 140121856786431, 140121865175039, +STORE, 140121856782336, 140121856786431, +STORE, 140121856786432, 140121865175039, +STORE, 140121403805696, 140121412198399, +SNULL, 140121840001024, 140121848389631, +STORE, 140121848389632, 140121856782335, +STORE, 140121840001024, 140121848389631, +SNULL, 140121848393727, 140121856782335, +STORE, 140121848389632, 140121848393727, +STORE, 140121848393728, 140121856782335, +SNULL, 140121479311360, 140121487699967, +STORE, 140121487699968, 140121496092671, +STORE, 140121479311360, 140121487699967, +SNULL, 140121487704063, 140121496092671, +STORE, 140121487699968, 140121487704063, +STORE, 140121487704064, 140121496092671, +STORE, 140121395412992, 140121412198399, +STORE, 140121387020288, 140121412198399, +SNULL, 140121387024383, 140121412198399, +STORE, 140121387020288, 140121387024383, +STORE, 140121387024384, 140121412198399, +SNULL, 140121605136384, 140121613524991, +STORE, 140121613524992, 140121621917695, +STORE, 140121605136384, 140121613524991, +SNULL, 140121613529087, 140121621917695, +STORE, 140121613524992, 140121613529087, +STORE, 140121613529088, 140121621917695, +SNULL, 140121462525952, 140121470914559, +STORE, 140121470914560, 140121479307263, +STORE, 140121462525952, 140121470914559, +SNULL, 140121470918655, 140121479307263, +STORE, 140121470914560, 140121470918655, +STORE, 140121470918656, 140121479307263, +STORE, 140121378627584, 140121387020287, +SNULL, 140121378631679, 140121387020287, +STORE, 140121378627584, 140121378631679, +STORE, 140121378631680, 140121387020287, +SNULL, 140121571565568, 140121596739583, +STORE, 140121596739584, 140121605132287, +STORE, 140121571565568, 140121596739583, +SNULL, 140121596743679, 140121605132287, +STORE, 140121596739584, 140121596743679, +STORE, 140121596743680, 140121605132287, +SNULL, 140121387024384, 140121403805695, +STORE, 140121403805696, 140121412198399, +STORE, 140121387024384, 140121403805695, +SNULL, 140121403809791, 140121412198399, +STORE, 140121403805696, 140121403809791, +STORE, 140121403809792, 140121412198399, +STORE, 140121370234880, 140121378627583, +SNULL, 140121387024384, 140121395412991, +STORE, 140121395412992, 140121403805695, +STORE, 140121387024384, 140121395412991, +SNULL, 140121395417087, 140121403805695, +STORE, 140121395412992, 140121395417087, +STORE, 140121395417088, 140121403805695, +SNULL, 140121571565568, 140121588346879, +STORE, 140121588346880, 140121596739583, +STORE, 140121571565568, 140121588346879, +SNULL, 140121588350975, 140121596739583, +STORE, 140121588346880, 140121588350975, +STORE, 140121588350976, 140121596739583, +SNULL, 140121571565568, 140121579954175, +STORE, 140121579954176, 140121588346879, +STORE, 140121571565568, 140121579954175, +SNULL, 140121579958271, 140121588346879, +STORE, 140121579954176, 140121579958271, +STORE, 140121579958272, 140121588346879, +STORE, 140121286373376, 140121294766079, +STORE, 140121277980672, 140121294766079, +SNULL, 140121277980672, 140121286373375, +STORE, 140121286373376, 140121294766079, +STORE, 140121277980672, 140121286373375, +SNULL, 140121286377471, 140121294766079, +STORE, 140121286373376, 140121286377471, +STORE, 140121286377472, 140121294766079, +STORE, 140121269587968, 140121286373375, +STORE, 140121261195264, 140121286373375, +SNULL, 140121261195264, 140121269587967, +STORE, 140121269587968, 140121286373375, +STORE, 140121261195264, 140121269587967, +SNULL, 140121269592063, 140121286373375, +STORE, 140121269587968, 140121269592063, +STORE, 140121269592064, 140121286373375, +STORE, 140121252802560, 140121269587967, +SNULL, 140121252806655, 140121269587967, +STORE, 140121252802560, 140121252806655, +STORE, 140121252806656, 140121269587967, +STORE, 140121244409856, 140121252802559, +STORE, 140121236017152, 140121252802559, +SNULL, 140121236017152, 140121244409855, +STORE, 140121244409856, 140121252802559, +STORE, 140121236017152, 140121244409855, +SNULL, 140121244413951, 140121252802559, +STORE, 140121244409856, 140121244413951, +STORE, 140121244413952, 140121252802559, +SNULL, 140121370238975, 140121378627583, +STORE, 140121370234880, 140121370238975, +STORE, 140121370238976, 140121378627583, +STORE, 140121152155648, 140121160548351, +STORE, 140121143762944, 140121160548351, +STORE, 140121135370240, 140121160548351, +SNULL, 140121135374335, 140121160548351, +STORE, 140121135370240, 140121135374335, +STORE, 140121135374336, 140121160548351, +STORE, 140121126977536, 140121135370239, +STORE, 140121118584832, 140121135370239, +STORE, 140121110192128, 140121135370239, +SNULL, 140121110192128, 140121118584831, +STORE, 140121118584832, 140121135370239, +STORE, 140121110192128, 140121118584831, +SNULL, 140121118588927, 140121135370239, +STORE, 140121118584832, 140121118588927, +STORE, 140121118588928, 140121135370239, +STORE, 140121101799424, 140121118584831, +STORE, 140121017937920, 140121026330623, +STORE, 140121009545216, 140121026330623, +SNULL, 140121009545216, 140121017937919, +STORE, 140121017937920, 140121026330623, +STORE, 140121009545216, 140121017937919, +SNULL, 140121017942015, 140121026330623, +STORE, 140121017937920, 140121017942015, +STORE, 140121017942016, 140121026330623, +SNULL, 140121269592064, 140121277980671, +STORE, 140121277980672, 140121286373375, +STORE, 140121269592064, 140121277980671, +SNULL, 140121277984767, 140121286373375, +STORE, 140121277980672, 140121277984767, +STORE, 140121277984768, 140121286373375, +STORE, 140121001152512, 140121017937919, +SNULL, 140121252806656, 140121261195263, +STORE, 140121261195264, 140121269587967, +STORE, 140121252806656, 140121261195263, +SNULL, 140121261199359, 140121269587967, +STORE, 140121261195264, 140121261199359, +STORE, 140121261199360, 140121269587967, +SNULL, 140121135374336, 140121152155647, +STORE, 140121152155648, 140121160548351, +STORE, 140121135374336, 140121152155647, +SNULL, 140121152159743, 140121160548351, +STORE, 140121152155648, 140121152159743, +STORE, 140121152159744, 140121160548351, +STORE, 140120992759808, 140121017937919, +STORE, 140120984367104, 140121017937919, +STORE, 140120975974400, 140121017937919, +SNULL, 140121101799424, 140121110192127, +STORE, 140121110192128, 140121118584831, +STORE, 140121101799424, 140121110192127, +SNULL, 140121110196223, 140121118584831, +STORE, 140121110192128, 140121110196223, +STORE, 140121110196224, 140121118584831, +SNULL, 140121118588928, 140121126977535, +STORE, 140121126977536, 140121135370239, +STORE, 140121118588928, 140121126977535, +SNULL, 140121126981631, 140121135370239, +STORE, 140121126977536, 140121126981631, +STORE, 140121126981632, 140121135370239, +STORE, 140120967581696, 140121017937919, +STORE, 140120883720192, 140120892112895, +SNULL, 140120883724287, 140120892112895, +STORE, 140120883720192, 140120883724287, +STORE, 140120883724288, 140120892112895, +STORE, 140120875327488, 140120883720191, +SNULL, 140121101803519, 140121110192127, +STORE, 140121101799424, 140121101803519, +STORE, 140121101803520, 140121110192127, +SNULL, 140121135374336, 140121143762943, +STORE, 140121143762944, 140121152155647, +STORE, 140121135374336, 140121143762943, +SNULL, 140121143767039, 140121152155647, +STORE, 140121143762944, 140121143767039, +STORE, 140121143767040, 140121152155647, +STORE, 140120866934784, 140120883720191, +SNULL, 140120967581696, 140120984367103, +STORE, 140120984367104, 140121017937919, +STORE, 140120967581696, 140120984367103, +SNULL, 140120984371199, 140121017937919, +STORE, 140120984367104, 140120984371199, +STORE, 140120984371200, 140121017937919, +STORE, 140120858542080, 140120883720191, +SNULL, 140121236021247, 140121244409855, +STORE, 140121236017152, 140121236021247, +STORE, 140121236021248, 140121244409855, +SNULL, 140120984371200, 140121009545215, +STORE, 140121009545216, 140121017937919, +STORE, 140120984371200, 140121009545215, +SNULL, 140121009549311, 140121017937919, +STORE, 140121009545216, 140121009549311, +STORE, 140121009549312, 140121017937919, +SNULL, 140120984371200, 140120992759807, +STORE, 140120992759808, 140121009545215, +STORE, 140120984371200, 140120992759807, +SNULL, 140120992763903, 140121009545215, +STORE, 140120992759808, 140120992763903, +STORE, 140120992763904, 140121009545215, +SNULL, 140120992763904, 140121001152511, +STORE, 140121001152512, 140121009545215, +STORE, 140120992763904, 140121001152511, +SNULL, 140121001156607, 140121009545215, +STORE, 140121001152512, 140121001156607, +STORE, 140121001156608, 140121009545215, +STORE, 140120850149376, 140120883720191, +SNULL, 140120850153471, 140120883720191, +STORE, 140120850149376, 140120850153471, +STORE, 140120850153472, 140120883720191, +SNULL, 140120967585791, 140120984367103, +STORE, 140120967581696, 140120967585791, +STORE, 140120967585792, 140120984367103, +SNULL, 140120850153472, 140120866934783, +STORE, 140120866934784, 140120883720191, +STORE, 140120850153472, 140120866934783, +SNULL, 140120866938879, 140120883720191, +STORE, 140120866934784, 140120866938879, +STORE, 140120866938880, 140120883720191, +STORE, 140120841756672, 140120850149375, +SNULL, 140120967585792, 140120975974399, +STORE, 140120975974400, 140120984367103, +STORE, 140120967585792, 140120975974399, +SNULL, 140120975978495, 140120984367103, +STORE, 140120975974400, 140120975978495, +STORE, 140120975978496, 140120984367103, +SNULL, 140120866938880, 140120875327487, +STORE, 140120875327488, 140120883720191, +STORE, 140120866938880, 140120875327487, +SNULL, 140120875331583, 140120883720191, +STORE, 140120875327488, 140120875331583, +STORE, 140120875331584, 140120883720191, +STORE, 140120833363968, 140120850149375, +STORE, 140120749502464, 140120757895167, +STORE, 140120741109760, 140120757895167, +STORE, 140120732717056, 140120757895167, +STORE, 140120724324352, 140120757895167, +SNULL, 140120724324352, 140120732717055, +STORE, 140120732717056, 140120757895167, +STORE, 140120724324352, 140120732717055, +SNULL, 140120732721151, 140120757895167, +STORE, 140120732717056, 140120732721151, +STORE, 140120732721152, 140120757895167, +STORE, 140120715931648, 140120732717055, +SNULL, 140120715935743, 140120732717055, +STORE, 140120715931648, 140120715935743, +STORE, 140120715935744, 140120732717055, +SNULL, 140120850153472, 140120858542079, +STORE, 140120858542080, 140120866934783, +STORE, 140120850153472, 140120858542079, +SNULL, 140120858546175, 140120866934783, +STORE, 140120858542080, 140120858546175, +STORE, 140120858546176, 140120866934783, +STORE, 140120707538944, 140120715931647, +SNULL, 140120707543039, 140120715931647, +STORE, 140120707538944, 140120707543039, +STORE, 140120707543040, 140120715931647, +SNULL, 140120833368063, 140120850149375, +STORE, 140120833363968, 140120833368063, +STORE, 140120833368064, 140120850149375, +SNULL, 140120833368064, 140120841756671, +STORE, 140120841756672, 140120850149375, +STORE, 140120833368064, 140120841756671, +SNULL, 140120841760767, 140120850149375, +STORE, 140120841756672, 140120841760767, +STORE, 140120841760768, 140120850149375, +STORE, 140120699146240, 140120707538943, +SNULL, 140120715935744, 140120724324351, +STORE, 140120724324352, 140120732717055, +STORE, 140120715935744, 140120724324351, +SNULL, 140120724328447, 140120732717055, +STORE, 140120724324352, 140120724328447, +STORE, 140120724328448, 140120732717055, +SNULL, 140120732721152, 140120741109759, +STORE, 140120741109760, 140120757895167, +STORE, 140120732721152, 140120741109759, +SNULL, 140120741113855, 140120757895167, +STORE, 140120741109760, 140120741113855, +STORE, 140120741113856, 140120757895167, +SNULL, 140120741113856, 140120749502463, +STORE, 140120749502464, 140120757895167, +STORE, 140120741113856, 140120749502463, +SNULL, 140120749506559, 140120757895167, +STORE, 140120749502464, 140120749506559, +STORE, 140120749506560, 140120757895167, +SNULL, 140120699150335, 140120707538943, +STORE, 140120699146240, 140120699150335, +STORE, 140120699150336, 140120707538943, +STORE, 140122446557184, 140122446585855, +STORE, 140122368999424, 140122371190783, +SNULL, 140122368999424, 140122369089535, +STORE, 140122369089536, 140122371190783, +STORE, 140122368999424, 140122369089535, +SNULL, 140122371182591, 140122371190783, +STORE, 140122369089536, 140122371182591, +STORE, 140122371182592, 140122371190783, +ERASE, 140122371182592, 140122371190783, +STORE, 140122371182592, 140122371190783, +SNULL, 140122371186687, 140122371190783, +STORE, 140122371182592, 140122371186687, +STORE, 140122371186688, 140122371190783, +ERASE, 140122446557184, 140122446585855, +ERASE, 140121445736448, 140121445740543, +ERASE, 140121445740544, 140121454129151, +ERASE, 140121621917696, 140121621921791, +ERASE, 140121621921792, 140121630310399, +ERASE, 140121579954176, 140121579958271, +ERASE, 140121579958272, 140121588346879, +ERASE, 140121261195264, 140121261199359, +ERASE, 140121261199360, 140121269587967, +ERASE, 140121454129152, 140121454133247, +ERASE, 140121454133248, 140121462521855, +ERASE, 140121588346880, 140121588350975, +ERASE, 140121588350976, 140121596739583, +ERASE, 140121135370240, 140121135374335, +ERASE, 140121135374336, 140121143762943, +ERASE, 140121881960448, 140121881964543, +ERASE, 140121881964544, 140121890353151, +ERASE, 140121428951040, 140121428955135, +ERASE, 140121428955136, 140121437343743, +ERASE, 140121387020288, 140121387024383, +ERASE, 140121387024384, 140121395412991, +ERASE, 140121487699968, 140121487704063, +ERASE, 140121487704064, 140121496092671, +ERASE, 140121437343744, 140121437347839, +ERASE, 140121437347840, 140121445736447, +ERASE, 140121613524992, 140121613529087, +ERASE, 140121613529088, 140121621917695, +ERASE, 140121856782336, 140121856786431, +ERASE, 140121856786432, 140121865175039, +ERASE, 140121252802560, 140121252806655, +ERASE, 140121252806656, 140121261195263, +ERASE, 140121839996928, 140121840001023, +ERASE, 140121840001024, 140121848389631, +ERASE, 140121596739584, 140121596743679, +ERASE, 140121596743680, 140121605132287, +ERASE, 140121009545216, 140121009549311, +ERASE, 140121009549312, 140121017937919, +ERASE, 140120724324352, 140120724328447, +ERASE, 140120724328448, 140120732717055, +ERASE, 140120883720192, 140120883724287, +ERASE, 140120883724288, 140120892112895, +ERASE, 140121982607360, 140121982611455, +ERASE, 140121982611456, 140121991000063, +ERASE, 140121571561472, 140121571565567, +ERASE, 140121571565568, 140121579954175, +ERASE, 140121286373376, 140121286377471, +ERASE, 140121286377472, 140121294766079, +ERASE, 140120875327488, 140120875331583, +ERASE, 140120875331584, 140120883720191, +ERASE, 140121848389632, 140121848393727, +ERASE, 140121848393728, 140121856782335, +ERASE, 140121370234880, 140121370238975, +ERASE, 140121370238976, 140121378627583, +ERASE, 140121143762944, 140121143767039, +ERASE, 140121143767040, 140121152155647, +ERASE, 140121118584832, 140121118588927, +ERASE, 140121118588928, 140121126977535, +ERASE, 140120866934784, 140120866938879, +ERASE, 140120866938880, 140120875327487, +ERASE, 140120741109760, 140120741113855, +ERASE, 140120741113856, 140120749502463, +ERASE, 140121865175040, 140121865179135, +ERASE, 140121865179136, 140121873567743, +ERASE, 140121403805696, 140121403809791, +ERASE, 140121403809792, 140121412198399, +ERASE, 140121236017152, 140121236021247, +ERASE, 140121236021248, 140121244409855, +ERASE, 140120732717056, 140120732721151, +ERASE, 140120732721152, 140120741109759, +ERASE, 140121017937920, 140121017942015, +ERASE, 140121017942016, 140121026330623, +ERASE, 140121873567744, 140121873571839, +ERASE, 140121873571840, 140121881960447, +ERASE, 140121470914560, 140121470918655, +ERASE, 140121470918656, 140121479307263, +ERASE, 140121126977536, 140121126981631, +ERASE, 140121126981632, 140121135370239, +ERASE, 140120850149376, 140120850153471, +ERASE, 140120850153472, 140120858542079, +ERASE, 140120707538944, 140120707543039, +ERASE, 140120707543040, 140120715931647, +ERASE, 140121479307264, 140121479311359, +ERASE, 140121479311360, 140121487699967, +ERASE, 140120967581696, 140120967585791, +ERASE, 140120967585792, 140120975974399, +ERASE, 140120841756672, 140120841760767, +ERASE, 140120841760768, 140120850149375, +ERASE, 140121412198400, 140121412202495, +ERASE, 140121412202496, 140121420591103, +ERASE, 140122158788608, 140122158792703, +ERASE, 140122158792704, 140122167181311, +ERASE, 140122142003200, 140122142007295, +ERASE, 140122142007296, 140122150395903, +ERASE, 140121101799424, 140121101803519, +ERASE, 140121101803520, 140121110192127, +ERASE, 140120858542080, 140120858546175, +ERASE, 140120858546176, 140120866934783, +ERASE, 140120833363968, 140120833368063, +ERASE, 140120833368064, 140120841756671, +ERASE, 140121277980672, 140121277984767, +ERASE, 140121277984768, 140121286373375, +ERASE, 140121001152512, 140121001156607, +ERASE, 140121001156608, 140121009545215, +ERASE, 140120749502464, 140120749506559, +ERASE, 140120749506560, 140120757895167, +ERASE, 140121605132288, 140121605136383, +ERASE, 140121605136384, 140121613524991, +ERASE, 140121378627584, 140121378631679, +ERASE, 140121378631680, 140121387020287, +ERASE, 140121110192128, 140121110196223, +ERASE, 140121110196224, 140121118584831, +ERASE, 140121462521856, 140121462525951, +ERASE, 140121462525952, 140121470914559, +ERASE, 140121395412992, 140121395417087, +ERASE, 140121395417088, 140121403805695, +ERASE, 140121152155648, 140121152159743, +ERASE, 140121152159744, 140121160548351, +ERASE, 140120992759808, 140120992763903, +ERASE, 140120992763904, 140121001152511, +ERASE, 140122387976192, 140122387980287, +ERASE, 140122387980288, 140122396368895, +ERASE, 140121890353152, 140121890357247, +ERASE, 140121890357248, 140121898745855, +ERASE, 140121269587968, 140121269592063, +ERASE, 140121269592064, 140121277980671, + }; + unsigned long set37[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140722404016128, 140737488351231, +SNULL, 140722404020223, 140737488351231, +STORE, 140722404016128, 140722404020223, +STORE, 140722403885056, 140722404020223, +STORE, 94637010001920, 94637012254719, +SNULL, 94637010132991, 94637012254719, +STORE, 94637010001920, 94637010132991, +STORE, 94637010132992, 94637012254719, +ERASE, 94637010132992, 94637012254719, +STORE, 94637012226048, 94637012234239, +STORE, 94637012234240, 94637012254719, +STORE, 139760240594944, 139760242847743, +SNULL, 139760240738303, 139760242847743, +STORE, 139760240594944, 139760240738303, +STORE, 139760240738304, 139760242847743, +ERASE, 139760240738304, 139760242847743, +STORE, 139760242835456, 139760242843647, +STORE, 139760242843648, 139760242847743, +STORE, 140722405232640, 140722405236735, +STORE, 140722405220352, 140722405232639, +STORE, 139760242806784, 139760242835455, +STORE, 139760242798592, 139760242806783, +STORE, 139760238379008, 139760240594943, +SNULL, 139760238379008, 139760238477311, +STORE, 139760238477312, 139760240594943, +STORE, 139760238379008, 139760238477311, +SNULL, 139760240570367, 139760240594943, +STORE, 139760238477312, 139760240570367, +STORE, 139760240570368, 139760240594943, +SNULL, 139760240570368, 139760240578559, +STORE, 139760240578560, 139760240594943, +STORE, 139760240570368, 139760240578559, +ERASE, 139760240570368, 139760240578559, +STORE, 139760240570368, 139760240578559, +ERASE, 139760240578560, 139760240594943, +STORE, 139760240578560, 139760240594943, +STORE, 139760234582016, 139760238379007, +SNULL, 139760234582016, 139760236240895, +STORE, 139760236240896, 139760238379007, +STORE, 139760234582016, 139760236240895, +SNULL, 139760238338047, 139760238379007, +STORE, 139760236240896, 139760238338047, +STORE, 139760238338048, 139760238379007, +SNULL, 139760238338048, 139760238362623, +STORE, 139760238362624, 139760238379007, +STORE, 139760238338048, 139760238362623, +ERASE, 139760238338048, 139760238362623, +STORE, 139760238338048, 139760238362623, +ERASE, 139760238362624, 139760238379007, +STORE, 139760238362624, 139760238379007, +STORE, 139760242790400, 139760242806783, +SNULL, 139760238354431, 139760238362623, +STORE, 139760238338048, 139760238354431, +STORE, 139760238354432, 139760238362623, +SNULL, 139760240574463, 139760240578559, +STORE, 139760240570368, 139760240574463, +STORE, 139760240574464, 139760240578559, +SNULL, 94637012230143, 94637012234239, +STORE, 94637012226048, 94637012230143, +STORE, 94637012230144, 94637012234239, +SNULL, 139760242839551, 139760242843647, +STORE, 139760242835456, 139760242839551, +STORE, 139760242839552, 139760242843647, +ERASE, 139760242806784, 139760242835455, +STORE, 94637033324544, 94637033459711, +STORE, 139760226189312, 139760234582015, +SNULL, 139760226193407, 139760234582015, +STORE, 139760226189312, 139760226193407, +STORE, 139760226193408, 139760234582015, +STORE, 139760217796608, 139760226189311, +STORE, 139760083578880, 139760217796607, +SNULL, 139760083578880, 139760114860031, +STORE, 139760114860032, 139760217796607, +STORE, 139760083578880, 139760114860031, +ERASE, 139760083578880, 139760114860031, +SNULL, 139760181968895, 139760217796607, +STORE, 139760114860032, 139760181968895, +STORE, 139760181968896, 139760217796607, +ERASE, 139760181968896, 139760217796607, +SNULL, 139760114995199, 139760181968895, +STORE, 139760114860032, 139760114995199, +STORE, 139760114995200, 139760181968895, +SNULL, 139760217800703, 139760226189311, +STORE, 139760217796608, 139760217800703, +STORE, 139760217800704, 139760226189311, +STORE, 139760209403904, 139760217796607, +SNULL, 139760209407999, 139760217796607, +STORE, 139760209403904, 139760209407999, +STORE, 139760209408000, 139760217796607, +STORE, 139760201011200, 139760209403903, +SNULL, 139760201015295, 139760209403903, +STORE, 139760201011200, 139760201015295, +STORE, 139760201015296, 139760209403903, +STORE, 139760192618496, 139760201011199, +SNULL, 139760192622591, 139760201011199, +STORE, 139760192618496, 139760192622591, +STORE, 139760192622592, 139760201011199, +STORE, 139760184225792, 139760192618495, +STORE, 139759980642304, 139760114860031, +STORE, 139759972249600, 139759980642303, +STORE, 139759963856896, 139759980642303, +STORE, 139759955464192, 139759980642303, +STORE, 139759888355328, 139759955464191, +SNULL, 139760047751167, 139760114860031, +STORE, 139759980642304, 139760047751167, +STORE, 139760047751168, 139760114860031, +ERASE, 139760047751168, 139760114860031, +SNULL, 139759980777471, 139760047751167, +STORE, 139759980642304, 139759980777471, +STORE, 139759980777472, 139760047751167, +STORE, 139759980777472, 139760114860031, +SNULL, 139759980777472, 139760047751167, +STORE, 139760047751168, 139760114860031, +STORE, 139759980777472, 139760047751167, +SNULL, 139760047886335, 139760114860031, +STORE, 139760047751168, 139760047886335, +STORE, 139760047886336, 139760114860031, +STORE, 139759821246464, 139759955464191, +SNULL, 139759821246464, 139759888355327, +STORE, 139759888355328, 139759955464191, +STORE, 139759821246464, 139759888355327, +ERASE, 139759821246464, 139759888355327, +ERASE, 139759888355328, 139759955464191, + }; + unsigned long set38[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140730666221568, 140737488351231, +SNULL, 140730666225663, 140737488351231, +STORE, 140730666221568, 140730666225663, +STORE, 140730666090496, 140730666225663, +STORE, 94177584803840, 94177587056639, +SNULL, 94177584934911, 94177587056639, +STORE, 94177584803840, 94177584934911, +STORE, 94177584934912, 94177587056639, +ERASE, 94177584934912, 94177587056639, +STORE, 94177587027968, 94177587036159, +STORE, 94177587036160, 94177587056639, +STORE, 140614382714880, 140614384967679, +SNULL, 140614382858239, 140614384967679, +STORE, 140614382714880, 140614382858239, +STORE, 140614382858240, 140614384967679, +ERASE, 140614382858240, 140614384967679, +STORE, 140614384955392, 140614384963583, +STORE, 140614384963584, 140614384967679, +STORE, 140730666315776, 140730666319871, +STORE, 140730666303488, 140730666315775, +STORE, 140614384926720, 140614384955391, +STORE, 140614384918528, 140614384926719, +STORE, 140614380498944, 140614382714879, +SNULL, 140614380498944, 140614380597247, +STORE, 140614380597248, 140614382714879, +STORE, 140614380498944, 140614380597247, +SNULL, 140614382690303, 140614382714879, +STORE, 140614380597248, 140614382690303, +STORE, 140614382690304, 140614382714879, +SNULL, 140614382690304, 140614382698495, +STORE, 140614382698496, 140614382714879, +STORE, 140614382690304, 140614382698495, +ERASE, 140614382690304, 140614382698495, +STORE, 140614382690304, 140614382698495, +ERASE, 140614382698496, 140614382714879, +STORE, 140614382698496, 140614382714879, +STORE, 140614376701952, 140614380498943, +SNULL, 140614376701952, 140614378360831, +STORE, 140614378360832, 140614380498943, +STORE, 140614376701952, 140614378360831, +SNULL, 140614380457983, 140614380498943, +STORE, 140614378360832, 140614380457983, +STORE, 140614380457984, 140614380498943, +SNULL, 140614380457984, 140614380482559, +STORE, 140614380482560, 140614380498943, +STORE, 140614380457984, 140614380482559, +ERASE, 140614380457984, 140614380482559, +STORE, 140614380457984, 140614380482559, +ERASE, 140614380482560, 140614380498943, +STORE, 140614380482560, 140614380498943, +STORE, 140614384910336, 140614384926719, +SNULL, 140614380474367, 140614380482559, +STORE, 140614380457984, 140614380474367, +STORE, 140614380474368, 140614380482559, +SNULL, 140614382694399, 140614382698495, +STORE, 140614382690304, 140614382694399, +STORE, 140614382694400, 140614382698495, +SNULL, 94177587032063, 94177587036159, +STORE, 94177587027968, 94177587032063, +STORE, 94177587032064, 94177587036159, +SNULL, 140614384959487, 140614384963583, +STORE, 140614384955392, 140614384959487, +STORE, 140614384959488, 140614384963583, +ERASE, 140614384926720, 140614384955391, +STORE, 94177619791872, 94177619927039, +STORE, 140614368309248, 140614376701951, +SNULL, 140614368313343, 140614376701951, +STORE, 140614368309248, 140614368313343, +STORE, 140614368313344, 140614376701951, +STORE, 140614359916544, 140614368309247, +STORE, 140614225698816, 140614359916543, +SNULL, 140614225698816, 140614276481023, +STORE, 140614276481024, 140614359916543, +STORE, 140614225698816, 140614276481023, +ERASE, 140614225698816, 140614276481023, +SNULL, 140614343589887, 140614359916543, +STORE, 140614276481024, 140614343589887, +STORE, 140614343589888, 140614359916543, +ERASE, 140614343589888, 140614359916543, +SNULL, 140614276616191, 140614343589887, +STORE, 140614276481024, 140614276616191, +STORE, 140614276616192, 140614343589887, +SNULL, 140614359920639, 140614368309247, +STORE, 140614359916544, 140614359920639, +STORE, 140614359920640, 140614368309247, +STORE, 140614351523840, 140614359916543, +SNULL, 140614351527935, 140614359916543, +STORE, 140614351523840, 140614351527935, +STORE, 140614351527936, 140614359916543, +STORE, 140614268088320, 140614276481023, +SNULL, 140614268092415, 140614276481023, +STORE, 140614268088320, 140614268092415, +STORE, 140614268092416, 140614276481023, +STORE, 140614259695616, 140614268088319, +SNULL, 140614259699711, 140614268088319, +STORE, 140614259695616, 140614259699711, +STORE, 140614259699712, 140614268088319, +STORE, 140614251302912, 140614259695615, +STORE, 140614242910208, 140614259695615, +STORE, 140614108692480, 140614242910207, +SNULL, 140614108692480, 140614142263295, +STORE, 140614142263296, 140614242910207, +STORE, 140614108692480, 140614142263295, +ERASE, 140614108692480, 140614142263295, +STORE, 140614133870592, 140614142263295, +STORE, 140613999652864, 140614133870591, +SNULL, 140613999652864, 140614008045567, +STORE, 140614008045568, 140614133870591, +STORE, 140613999652864, 140614008045567, +ERASE, 140613999652864, 140614008045567, +STORE, 140613999652864, 140614008045567, +STORE, 140613865435136, 140613999652863, +SNULL, 140613865435136, 140613873827839, +STORE, 140613873827840, 140613999652863, +STORE, 140613865435136, 140613873827839, +ERASE, 140613865435136, 140613873827839, +SNULL, 140614209372159, 140614242910207, +STORE, 140614142263296, 140614209372159, +STORE, 140614209372160, 140614242910207, +ERASE, 140614209372160, 140614242910207, +SNULL, 140614142398463, 140614209372159, +STORE, 140614142263296, 140614142398463, +STORE, 140614142398464, 140614209372159, +SNULL, 140614075154431, 140614133870591, +STORE, 140614008045568, 140614075154431, +STORE, 140614075154432, 140614133870591, +ERASE, 140614075154432, 140614133870591, +SNULL, 140614008180735, 140614075154431, +STORE, 140614008045568, 140614008180735, +STORE, 140614008180736, 140614075154431, +SNULL, 140613940936703, 140613999652863, +STORE, 140613873827840, 140613940936703, +STORE, 140613940936704, 140613999652863, +ERASE, 140613940936704, 140613999652863, +SNULL, 140614242914303, 140614259695615, +STORE, 140614242910208, 140614242914303, +STORE, 140614242914304, 140614259695615, +STORE, 140613739610112, 140613940936703, +STORE, 140614234517504, 140614242910207, +SNULL, 140614242914304, 140614251302911, +STORE, 140614251302912, 140614259695615, +STORE, 140614242914304, 140614251302911, +SNULL, 140614251307007, 140614259695615, +STORE, 140614251302912, 140614251307007, +STORE, 140614251307008, 140614259695615, +SNULL, 140613739610112, 140613873827839, +STORE, 140613873827840, 140613940936703, +STORE, 140613739610112, 140613873827839, +SNULL, 140613873963007, 140613940936703, +STORE, 140613873827840, 140613873963007, +STORE, 140613873963008, 140613940936703, +SNULL, 140614133874687, 140614142263295, +STORE, 140614133870592, 140614133874687, +STORE, 140614133874688, 140614142263295, +SNULL, 140613806718975, 140613873827839, +STORE, 140613739610112, 140613806718975, +STORE, 140613806718976, 140613873827839, +ERASE, 140613806718976, 140613873827839, +STORE, 140614226124800, 140614242910207, +SNULL, 140613739745279, 140613806718975, +STORE, 140613739610112, 140613739745279, +STORE, 140613739745280, 140613806718975, +SNULL, 140613999656959, 140614008045567, +STORE, 140613999652864, 140613999656959, +STORE, 140613999656960, 140614008045567, +SNULL, 140614226124800, 140614234517503, +STORE, 140614234517504, 140614242910207, +STORE, 140614226124800, 140614234517503, +SNULL, 140614234521599, 140614242910207, +STORE, 140614234517504, 140614234521599, +STORE, 140614234521600, 140614242910207, +STORE, 140614217732096, 140614234517503, +STORE, 140614125477888, 140614133870591, +SNULL, 140614125481983, 140614133870591, +STORE, 140614125477888, 140614125481983, +STORE, 140614125481984, 140614133870591, +STORE, 140614117085184, 140614125477887, +SNULL, 140614217736191, 140614234517503, +STORE, 140614217732096, 140614217736191, +STORE, 140614217736192, 140614234517503, +SNULL, 140614117089279, 140614125477887, +STORE, 140614117085184, 140614117089279, +STORE, 140614117089280, 140614125477887, +SNULL, 140614217736192, 140614226124799, +STORE, 140614226124800, 140614234517503, +STORE, 140614217736192, 140614226124799, +SNULL, 140614226128895, 140614234517503, +STORE, 140614226124800, 140614226128895, +STORE, 140614226128896, 140614234517503, +STORE, 140614108692480, 140614117085183, +STORE, 140614100299776, 140614117085183, +STORE, 140614091907072, 140614117085183, +SNULL, 140614091907072, 140614108692479, +STORE, 140614108692480, 140614117085183, +STORE, 140614091907072, 140614108692479, +SNULL, 140614108696575, 140614117085183, +STORE, 140614108692480, 140614108696575, +STORE, 140614108696576, 140614117085183, +SNULL, 140614091907072, 140614100299775, +STORE, 140614100299776, 140614108692479, +STORE, 140614091907072, 140614100299775, +SNULL, 140614100303871, 140614108692479, +STORE, 140614100299776, 140614100303871, +STORE, 140614100303872, 140614108692479, +STORE, 140614083514368, 140614100299775, +SNULL, 140614083518463, 140614100299775, +STORE, 140614083514368, 140614083518463, +STORE, 140614083518464, 140614100299775, +STORE, 140613991260160, 140613999652863, +SNULL, 140614083518464, 140614091907071, +STORE, 140614091907072, 140614100299775, +STORE, 140614083518464, 140614091907071, +SNULL, 140614091911167, 140614100299775, +STORE, 140614091907072, 140614091911167, +STORE, 140614091911168, 140614100299775, +SNULL, 140613991264255, 140613999652863, +STORE, 140613991260160, 140613991264255, +STORE, 140613991264256, 140613999652863, +STORE, 140613982867456, 140613991260159, +SNULL, 140613982871551, 140613991260159, +STORE, 140613982867456, 140613982871551, +STORE, 140613982871552, 140613991260159, +STORE, 140613974474752, 140613982867455, +SNULL, 140613974478847, 140613982867455, +STORE, 140613974474752, 140613974478847, +STORE, 140613974478848, 140613982867455, +STORE, 140613966082048, 140613974474751, +STORE, 140613739745280, 140613873827839, +SNULL, 140613739745280, 140613806718975, +STORE, 140613806718976, 140613873827839, +STORE, 140613739745280, 140613806718975, +SNULL, 140613806854143, 140613873827839, +STORE, 140613806718976, 140613806854143, +STORE, 140613806854144, 140613873827839, +SNULL, 140613966086143, 140613974474751, +STORE, 140613966082048, 140613966086143, +STORE, 140613966086144, 140613974474751, +STORE, 140613957689344, 140613966082047, +STORE, 140613605392384, 140613739610111, +STORE, 140613949296640, 140613966082047, +STORE, 140613596999680, 140613605392383, +STORE, 140613529890816, 140613596999679, +STORE, 140613521498112, 140613529890815, +STORE, 140613513105408, 140613529890815, +STORE, 140613378887680, 140613513105407, +SNULL, 140613378887680, 140613404065791, +STORE, 140613404065792, 140613513105407, +STORE, 140613378887680, 140613404065791, +ERASE, 140613378887680, 140613404065791, +STORE, 140613395673088, 140613404065791, +STORE, 140613261455360, 140613395673087, +SNULL, 140613261455360, 140613269848063, +STORE, 140613269848064, 140613395673087, +STORE, 140613261455360, 140613269848063, +ERASE, 140613261455360, 140613269848063, +STORE, 140613261455360, 140613269848063, +STORE, 140613253062656, 140613269848063, +STORE, 140613118844928, 140613253062655, +STORE, 140613110452224, 140613118844927, +SNULL, 140613118844928, 140613135630335, +STORE, 140613135630336, 140613253062655, +STORE, 140613118844928, 140613135630335, +ERASE, 140613118844928, 140613135630335, +STORE, 140613127237632, 140613135630335, +STORE, 140613110452224, 140613135630335, +STORE, 140612976234496, 140613110452223, +STORE, 140612967841792, 140612976234495, +STORE, 140612833624064, 140612967841791, +STORE, 140612825231360, 140612833624063, +STORE, 140612816838656, 140612833624063, +STORE, 140612682620928, 140612816838655, +STORE, 140612674228224, 140612682620927, +SNULL, 140612682620928, 140612732977151, +STORE, 140612732977152, 140612816838655, +STORE, 140612682620928, 140612732977151, +ERASE, 140612682620928, 140612732977151, +SNULL, 140613672501247, 140613739610111, +STORE, 140613605392384, 140613672501247, +STORE, 140613672501248, 140613739610111, +ERASE, 140613672501248, 140613739610111, +SNULL, 140613605527551, 140613672501247, +STORE, 140613605392384, 140613605527551, +STORE, 140613605527552, 140613672501247, +ERASE, 140613529890816, 140613596999679, +STORE, 140612540010496, 140612674228223, +SNULL, 140612540010496, 140612598759423, +STORE, 140612598759424, 140612674228223, +STORE, 140612540010496, 140612598759423, +ERASE, 140612540010496, 140612598759423, +SNULL, 140613471174655, 140613513105407, +STORE, 140613404065792, 140613471174655, +STORE, 140613471174656, 140613513105407, +ERASE, 140613471174656, 140613513105407, +SNULL, 140613404200959, 140613471174655, +STORE, 140613404065792, 140613404200959, +STORE, 140613404200960, 140613471174655, +SNULL, 140613336956927, 140613395673087, +STORE, 140613269848064, 140613336956927, +STORE, 140613336956928, 140613395673087, +ERASE, 140613336956928, 140613395673087, +SNULL, 140612833624064, 140612867194879, +STORE, 140612867194880, 140612967841791, +STORE, 140612833624064, 140612867194879, +ERASE, 140612833624064, 140612867194879, +SNULL, 140612976234496, 140613001412607, +STORE, 140613001412608, 140613110452223, +STORE, 140612976234496, 140613001412607, +ERASE, 140612976234496, 140613001412607, +SNULL, 140613202739199, 140613253062655, +STORE, 140613135630336, 140613202739199, +STORE, 140613202739200, 140613253062655, +ERASE, 140613202739200, 140613253062655, +SNULL, 140613135765503, 140613202739199, +STORE, 140613135630336, 140613135765503, +STORE, 140613135765504, 140613202739199, +SNULL, 140612816842751, 140612833624063, +STORE, 140612816838656, 140612816842751, +STORE, 140612816842752, 140612833624063, +SNULL, 140613110456319, 140613135630335, +STORE, 140613110452224, 140613110456319, +STORE, 140613110456320, 140613135630335, +SNULL, 140613949300735, 140613966082047, +STORE, 140613949296640, 140613949300735, +STORE, 140613949300736, 140613966082047, +SNULL, 140613110456320, 140613118844927, +STORE, 140613118844928, 140613135630335, +STORE, 140613110456320, 140613118844927, +SNULL, 140613118849023, 140613135630335, +STORE, 140613118844928, 140613118849023, +STORE, 140613118849024, 140613135630335, +SNULL, 140612800086015, 140612816838655, +STORE, 140612732977152, 140612800086015, +STORE, 140612800086016, 140612816838655, +ERASE, 140612800086016, 140612816838655, +SNULL, 140613253062656, 140613261455359, +STORE, 140613261455360, 140613269848063, +STORE, 140613253062656, 140613261455359, +SNULL, 140613261459455, 140613269848063, +STORE, 140613261455360, 140613261459455, +STORE, 140613261459456, 140613269848063, +SNULL, 140612674232319, 140612682620927, +STORE, 140612674228224, 140612674232319, +STORE, 140612674232320, 140612682620927, +STORE, 140613731217408, 140613739610111, +STORE, 140613722824704, 140613739610111, +SNULL, 140613949300736, 140613957689343, +STORE, 140613957689344, 140613966082047, +STORE, 140613949300736, 140613957689343, +SNULL, 140613957693439, 140613966082047, +STORE, 140613957689344, 140613957693439, +STORE, 140613957693440, 140613966082047, +STORE, 140612464541696, 140612674228223, +SNULL, 140612531650559, 140612674228223, +STORE, 140612464541696, 140612531650559, +STORE, 140612531650560, 140612674228223, +SNULL, 140612531650560, 140612598759423, +STORE, 140612598759424, 140612674228223, +STORE, 140612531650560, 140612598759423, +ERASE, 140612531650560, 140612598759423, +SNULL, 140612665868287, 140612674228223, +STORE, 140612598759424, 140612665868287, +STORE, 140612665868288, 140612674228223, +ERASE, 140612665868288, 140612674228223, +SNULL, 140613269983231, 140613336956927, +STORE, 140613269848064, 140613269983231, +STORE, 140613269983232, 140613336956927, +SNULL, 140612934303743, 140612967841791, +STORE, 140612867194880, 140612934303743, +STORE, 140612934303744, 140612967841791, +ERASE, 140612934303744, 140612967841791, +SNULL, 140613068521471, 140613110452223, +STORE, 140613001412608, 140613068521471, +STORE, 140613068521472, 140613110452223, +ERASE, 140613068521472, 140613110452223, +STORE, 140613714432000, 140613739610111, +SNULL, 140613001547775, 140613068521471, +STORE, 140613001412608, 140613001547775, +STORE, 140613001547776, 140613068521471, +SNULL, 140612733112319, 140612800086015, +STORE, 140612732977152, 140612733112319, +STORE, 140612733112320, 140612800086015, +SNULL, 140613513109503, 140613529890815, +STORE, 140613513105408, 140613513109503, +STORE, 140613513109504, 140613529890815, +STORE, 140613706039296, 140613739610111, +STORE, 140613697646592, 140613739610111, +STORE, 140613689253888, 140613739610111, +SNULL, 140613689257983, 140613739610111, +STORE, 140613689253888, 140613689257983, +STORE, 140613689257984, 140613739610111, +SNULL, 140613253066751, 140613261455359, +STORE, 140613253062656, 140613253066751, +STORE, 140613253066752, 140613261455359, +STORE, 140613680861184, 140613689253887, +STORE, 140613588606976, 140613605392383, +SNULL, 140613689257984, 140613731217407, +STORE, 140613731217408, 140613739610111, +STORE, 140613689257984, 140613731217407, +SNULL, 140613731221503, 140613739610111, +STORE, 140613731217408, 140613731221503, +STORE, 140613731221504, 140613739610111, +STORE, 140613580214272, 140613605392383, +SNULL, 140612464676863, 140612531650559, +STORE, 140612464541696, 140612464676863, +STORE, 140612464676864, 140612531650559, +SNULL, 140612598894591, 140612665868287, +STORE, 140612598759424, 140612598894591, +STORE, 140612598894592, 140612665868287, +SNULL, 140612867330047, 140612934303743, +STORE, 140612867194880, 140612867330047, +STORE, 140612867330048, 140612934303743, +STORE, 140613571821568, 140613605392383, +SNULL, 140613571825663, 140613605392383, +STORE, 140613571821568, 140613571825663, +STORE, 140613571825664, 140613605392383, +SNULL, 140613689257984, 140613722824703, +STORE, 140613722824704, 140613731217407, +STORE, 140613689257984, 140613722824703, +SNULL, 140613722828799, 140613731217407, +STORE, 140613722824704, 140613722828799, +STORE, 140613722828800, 140613731217407, +SNULL, 140613689257984, 140613714431999, +STORE, 140613714432000, 140613722824703, +STORE, 140613689257984, 140613714431999, +SNULL, 140613714436095, 140613722824703, +STORE, 140613714432000, 140613714436095, +STORE, 140613714436096, 140613722824703, +SNULL, 140612816842752, 140612825231359, +STORE, 140612825231360, 140612833624063, +STORE, 140612816842752, 140612825231359, +SNULL, 140612825235455, 140612833624063, +STORE, 140612825231360, 140612825235455, +STORE, 140612825235456, 140612833624063, +SNULL, 140613395677183, 140613404065791, +STORE, 140613395673088, 140613395677183, +STORE, 140613395677184, 140613404065791, +SNULL, 140613689257984, 140613706039295, +STORE, 140613706039296, 140613714431999, +STORE, 140613689257984, 140613706039295, +SNULL, 140613706043391, 140613714431999, +STORE, 140613706039296, 140613706043391, +STORE, 140613706043392, 140613714431999, +SNULL, 140613118849024, 140613127237631, +STORE, 140613127237632, 140613135630335, +STORE, 140613118849024, 140613127237631, +SNULL, 140613127241727, 140613135630335, +STORE, 140613127237632, 140613127241727, +STORE, 140613127241728, 140613135630335, +SNULL, 140613571825664, 140613580214271, +STORE, 140613580214272, 140613605392383, +STORE, 140613571825664, 140613580214271, +SNULL, 140613580218367, 140613605392383, +STORE, 140613580214272, 140613580218367, +STORE, 140613580218368, 140613605392383, +SNULL, 140613689257984, 140613697646591, +STORE, 140613697646592, 140613706039295, +STORE, 140613689257984, 140613697646591, +SNULL, 140613697650687, 140613706039295, +STORE, 140613697646592, 140613697650687, +STORE, 140613697650688, 140613706039295, +SNULL, 140613680865279, 140613689253887, +STORE, 140613680861184, 140613680865279, +STORE, 140613680865280, 140613689253887, +STORE, 140613563428864, 140613571821567, +SNULL, 140613563432959, 140613571821567, +STORE, 140613563428864, 140613563432959, +STORE, 140613563432960, 140613571821567, +SNULL, 140613580218368, 140613588606975, +STORE, 140613588606976, 140613605392383, +STORE, 140613580218368, 140613588606975, +SNULL, 140613588611071, 140613605392383, +STORE, 140613588606976, 140613588611071, +STORE, 140613588611072, 140613605392383, +SNULL, 140613513109504, 140613521498111, +STORE, 140613521498112, 140613529890815, +STORE, 140613513109504, 140613521498111, +SNULL, 140613521502207, 140613529890815, +STORE, 140613521498112, 140613521502207, +STORE, 140613521502208, 140613529890815, +SNULL, 140613588611072, 140613596999679, +STORE, 140613596999680, 140613605392383, +STORE, 140613588611072, 140613596999679, +SNULL, 140613597003775, 140613605392383, +STORE, 140613596999680, 140613597003775, +STORE, 140613597003776, 140613605392383, +STORE, 140613555036160, 140613563428863, +SNULL, 140613555040255, 140613563428863, +STORE, 140613555036160, 140613555040255, +STORE, 140613555040256, 140613563428863, +STORE, 140613546643456, 140613555036159, +STORE, 140613538250752, 140613555036159, +SNULL, 140613538250752, 140613546643455, +STORE, 140613546643456, 140613555036159, +STORE, 140613538250752, 140613546643455, +SNULL, 140613546647551, 140613555036159, +STORE, 140613546643456, 140613546647551, +STORE, 140613546647552, 140613555036159, +STORE, 140613504712704, 140613513105407, +STORE, 140613496320000, 140613513105407, +SNULL, 140613496324095, 140613513105407, +STORE, 140613496320000, 140613496324095, +STORE, 140613496324096, 140613513105407, +STORE, 140613487927296, 140613496319999, +SNULL, 140613487931391, 140613496319999, +STORE, 140613487927296, 140613487931391, +STORE, 140613487931392, 140613496319999, +STORE, 140613479534592, 140613487927295, +SNULL, 140612967845887, 140612976234495, +STORE, 140612967841792, 140612967845887, +STORE, 140612967845888, 140612976234495, +STORE, 140613387280384, 140613395673087, +STORE, 140613378887680, 140613395673087, +SNULL, 140613378887680, 140613387280383, +STORE, 140613387280384, 140613395673087, +STORE, 140613378887680, 140613387280383, +SNULL, 140613387284479, 140613395673087, +STORE, 140613387280384, 140613387284479, +STORE, 140613387284480, 140613395673087, +STORE, 140613370494976, 140613387280383, +STORE, 140613362102272, 140613387280383, +SNULL, 140613479538687, 140613487927295, +STORE, 140613479534592, 140613479538687, +STORE, 140613479538688, 140613487927295, +STORE, 140613353709568, 140613387280383, +STORE, 140613345316864, 140613387280383, +STORE, 140613244669952, 140613253062655, +SNULL, 140613345320959, 140613387280383, +STORE, 140613345316864, 140613345320959, +STORE, 140613345320960, 140613387280383, +SNULL, 140613538254847, 140613546643455, +STORE, 140613538250752, 140613538254847, +STORE, 140613538254848, 140613546643455, +STORE, 140613236277248, 140613253062655, +STORE, 140613227884544, 140613253062655, +STORE, 140613219491840, 140613253062655, +STORE, 140613211099136, 140613253062655, +SNULL, 140613211103231, 140613253062655, +STORE, 140613211099136, 140613211103231, +STORE, 140613211103232, 140613253062655, +STORE, 140613102059520, 140613110452223, +STORE, 140613093666816, 140613110452223, +SNULL, 140613093670911, 140613110452223, +STORE, 140613093666816, 140613093670911, +STORE, 140613093670912, 140613110452223, +STORE, 140613085274112, 140613093666815, +SNULL, 140613496324096, 140613504712703, +STORE, 140613504712704, 140613513105407, +STORE, 140613496324096, 140613504712703, +SNULL, 140613504716799, 140613513105407, +STORE, 140613504712704, 140613504716799, +STORE, 140613504716800, 140613513105407, +SNULL, 140613345320960, 140613378887679, +STORE, 140613378887680, 140613387280383, +STORE, 140613345320960, 140613378887679, +SNULL, 140613378891775, 140613387280383, +STORE, 140613378887680, 140613378891775, +STORE, 140613378891776, 140613387280383, +SNULL, 140613345320960, 140613362102271, +STORE, 140613362102272, 140613378887679, +STORE, 140613345320960, 140613362102271, +SNULL, 140613362106367, 140613378887679, +STORE, 140613362102272, 140613362106367, +STORE, 140613362106368, 140613378887679, +SNULL, 140613362106368, 140613370494975, +STORE, 140613370494976, 140613378887679, +STORE, 140613362106368, 140613370494975, +SNULL, 140613370499071, 140613378887679, +STORE, 140613370494976, 140613370499071, +STORE, 140613370499072, 140613378887679, +STORE, 140613076881408, 140613093666815, +STORE, 140612993019904, 140613001412607, +SNULL, 140613076885503, 140613093666815, +STORE, 140613076881408, 140613076885503, +STORE, 140613076885504, 140613093666815, +SNULL, 140613093670912, 140613102059519, +STORE, 140613102059520, 140613110452223, +STORE, 140613093670912, 140613102059519, +SNULL, 140613102063615, 140613110452223, +STORE, 140613102059520, 140613102063615, +STORE, 140613102063616, 140613110452223, +SNULL, 140613076885504, 140613085274111, +STORE, 140613085274112, 140613093666815, +STORE, 140613076885504, 140613085274111, +SNULL, 140613085278207, 140613093666815, +STORE, 140613085274112, 140613085278207, +STORE, 140613085278208, 140613093666815, +STORE, 140612984627200, 140613001412607, +STORE, 140612967845888, 140612984627199, +SNULL, 140613211103232, 140613219491839, +STORE, 140613219491840, 140613253062655, +STORE, 140613211103232, 140613219491839, +SNULL, 140613219495935, 140613253062655, +STORE, 140613219491840, 140613219495935, +STORE, 140613219495936, 140613253062655, +STORE, 140612959449088, 140612967841791, +STORE, 140612951056384, 140612967841791, +SNULL, 140612951060479, 140612967841791, +STORE, 140612951056384, 140612951060479, +STORE, 140612951060480, 140612967841791, +SNULL, 140613345320960, 140613353709567, +STORE, 140613353709568, 140613362102271, +STORE, 140613345320960, 140613353709567, +SNULL, 140613353713663, 140613362102271, +STORE, 140613353709568, 140613353713663, +STORE, 140613353713664, 140613362102271, +SNULL, 140613219495936, 140613244669951, +STORE, 140613244669952, 140613253062655, +STORE, 140613219495936, 140613244669951, +SNULL, 140613244674047, 140613253062655, +STORE, 140613244669952, 140613244674047, +STORE, 140613244674048, 140613253062655, +STORE, 140612942663680, 140612951056383, +SNULL, 140613219495936, 140613236277247, +STORE, 140613236277248, 140613244669951, +STORE, 140613219495936, 140613236277247, +SNULL, 140613236281343, 140613244669951, +STORE, 140613236277248, 140613236281343, +STORE, 140613236281344, 140613244669951, +SNULL, 140613219495936, 140613227884543, +STORE, 140613227884544, 140613236277247, +STORE, 140613219495936, 140613227884543, +SNULL, 140613227888639, 140613236277247, +STORE, 140613227884544, 140613227888639, +STORE, 140613227888640, 140613236277247, +SNULL, 140612984627200, 140612993019903, +STORE, 140612993019904, 140613001412607, +STORE, 140612984627200, 140612993019903, +SNULL, 140612993023999, 140613001412607, +STORE, 140612993019904, 140612993023999, +STORE, 140612993024000, 140613001412607, +STORE, 140612858802176, 140612867194879, +STORE, 140612850409472, 140612867194879, +SNULL, 140612951060480, 140612959449087, +STORE, 140612959449088, 140612967841791, +STORE, 140612951060480, 140612959449087, +SNULL, 140612959453183, 140612967841791, +STORE, 140612959449088, 140612959453183, +STORE, 140612959453184, 140612967841791, +SNULL, 140612967845888, 140612976234495, +STORE, 140612976234496, 140612984627199, +STORE, 140612967845888, 140612976234495, +SNULL, 140612976238591, 140612984627199, +STORE, 140612976234496, 140612976238591, +STORE, 140612976238592, 140612984627199, +STORE, 140612842016768, 140612867194879, +SNULL, 140612842020863, 140612867194879, +STORE, 140612842016768, 140612842020863, +STORE, 140612842020864, 140612867194879, +SNULL, 140612984631295, 140612993019903, +STORE, 140612984627200, 140612984631295, +STORE, 140612984631296, 140612993019903, +STORE, 140612825235456, 140612842016767, +STORE, 140612808445952, 140612816838655, +SNULL, 140612942667775, 140612951056383, +STORE, 140612942663680, 140612942667775, +STORE, 140612942667776, 140612951056383, +STORE, 140612724584448, 140612732977151, +SNULL, 140612724588543, 140612732977151, +STORE, 140612724584448, 140612724588543, +STORE, 140612724588544, 140612732977151, +STORE, 140612716191744, 140612724584447, +SNULL, 140612842020864, 140612850409471, +STORE, 140612850409472, 140612867194879, +STORE, 140612842020864, 140612850409471, +SNULL, 140612850413567, 140612867194879, +STORE, 140612850409472, 140612850413567, +STORE, 140612850413568, 140612867194879, +SNULL, 140612850413568, 140612858802175, +STORE, 140612858802176, 140612867194879, +STORE, 140612850413568, 140612858802175, +SNULL, 140612858806271, 140612867194879, +STORE, 140612858802176, 140612858806271, +STORE, 140612858806272, 140612867194879, +STORE, 140612707799040, 140612724584447, +SNULL, 140612707803135, 140612724584447, +STORE, 140612707799040, 140612707803135, +STORE, 140612707803136, 140612724584447, +SNULL, 140612707803136, 140612716191743, +STORE, 140612716191744, 140612724584447, +STORE, 140612707803136, 140612716191743, +SNULL, 140612716195839, 140612724584447, +STORE, 140612716191744, 140612716195839, +STORE, 140612716195840, 140612724584447, +SNULL, 140612808450047, 140612816838655, +STORE, 140612808445952, 140612808450047, +STORE, 140612808450048, 140612816838655, +SNULL, 140612825235456, 140612833624063, +STORE, 140612833624064, 140612842016767, +STORE, 140612825235456, 140612833624063, +SNULL, 140612833628159, 140612842016767, +STORE, 140612833624064, 140612833628159, +STORE, 140612833628160, 140612842016767, +STORE, 140612699406336, 140612707799039, +SNULL, 140612699410431, 140612707799039, +STORE, 140612699406336, 140612699410431, +STORE, 140612699410432, 140612707799039, +STORE, 140614384926720, 140614384955391, +STORE, 140614349332480, 140614351523839, +SNULL, 140614349332480, 140614349422591, +STORE, 140614349422592, 140614351523839, +STORE, 140614349332480, 140614349422591, +SNULL, 140614351515647, 140614351523839, +STORE, 140614349422592, 140614351515647, +STORE, 140614351515648, 140614351523839, +ERASE, 140614351515648, 140614351523839, +STORE, 140614351515648, 140614351523839, +SNULL, 140614351519743, 140614351523839, +STORE, 140614351515648, 140614351519743, +STORE, 140614351519744, 140614351523839, +ERASE, 140614384926720, 140614384955391, +ERASE, 140613949296640, 140613949300735, +ERASE, 140613949300736, 140613957689343, +ERASE, 140613689253888, 140613689257983, +ERASE, 140613689257984, 140613697646591, +ERASE, 140613563428864, 140613563432959, +ERASE, 140613563432960, 140613571821567, +ERASE, 140613211099136, 140613211103231, +ERASE, 140613211103232, 140613219491839, +ERASE, 140614133870592, 140614133874687, +ERASE, 140614133874688, 140614142263295, +ERASE, 140612967841792, 140612967845887, +ERASE, 140612967845888, 140612976234495, +ERASE, 140613076881408, 140613076885503, +ERASE, 140613076885504, 140613085274111, +ERASE, 140612850409472, 140612850413567, +ERASE, 140612850413568, 140612858802175, +ERASE, 140613110452224, 140613110456319, +ERASE, 140613110456320, 140613118844927, +ERASE, 140613706039296, 140613706043391, +ERASE, 140613706043392, 140613714431999, +ERASE, 140613521498112, 140613521502207, +ERASE, 140613521502208, 140613529890815, +ERASE, 140613362102272, 140613362106367, +ERASE, 140613362106368, 140613370494975, +ERASE, 140613253062656, 140613253066751, +ERASE, 140613253066752, 140613261455359, +ERASE, 140612816838656, 140612816842751, +ERASE, 140612816842752, 140612825231359, +ERASE, 140613261455360, 140613261459455, +ERASE, 140613261459456, 140613269848063, +ERASE, 140613118844928, 140613118849023, +ERASE, 140613118849024, 140613127237631, +ERASE, 140613714432000, 140613714436095, +ERASE, 140613714436096, 140613722824703, +ERASE, 140613496320000, 140613496324095, +ERASE, 140613496324096, 140613504712703, +ERASE, 140613513105408, 140613513109503, +ERASE, 140613513109504, 140613521498111, +ERASE, 140613697646592, 140613697650687, +ERASE, 140613697650688, 140613706039295, +ERASE, 140613093666816, 140613093670911, +ERASE, 140613093670912, 140613102059519, +ERASE, 140612993019904, 140612993023999, +ERASE, 140612993024000, 140613001412607, +ERASE, 140613127237632, 140613127241727, +ERASE, 140613127241728, 140613135630335, +ERASE, 140613957689344, 140613957693439, +ERASE, 140613957693440, 140613966082047, +ERASE, 140613571821568, 140613571825663, +ERASE, 140613571825664, 140613580214271, +ERASE, 140613479534592, 140613479538687, +ERASE, 140613479538688, 140613487927295, +ERASE, 140612984627200, 140612984631295, +ERASE, 140612984631296, 140612993019903, +ERASE, 140613588606976, 140613588611071, +ERASE, 140613588611072, 140613596999679, +ERASE, 140613680861184, 140613680865279, +ERASE, 140613680865280, 140613689253887, +ERASE, 140613345316864, 140613345320959, +ERASE, 140613345320960, 140613353709567, +ERASE, 140613596999680, 140613597003775, +ERASE, 140613597003776, 140613605392383, +ERASE, 140613966082048, 140613966086143, +ERASE, 140613966086144, 140613974474751, +ERASE, 140613731217408, 140613731221503, +ERASE, 140613731221504, 140613739610111, +ERASE, 140613395673088, 140613395677183, +ERASE, 140613395677184, 140613404065791, +ERASE, 140612825231360, 140612825235455, +ERASE, 140612825235456, 140612833624063, +ERASE, 140612674228224, 140612674232319, +ERASE, 140612674232320, 140612682620927, +ERASE, 140613722824704, 140613722828799, +ERASE, 140613722828800, 140613731217407, +ERASE, 140613487927296, 140613487931391, +ERASE, 140613487931392, 140613496319999, +ERASE, 140613102059520, 140613102063615, +ERASE, 140613102063616, 140613110452223, +ERASE, 140614242910208, 140614242914303, +ERASE, 140614242914304, 140614251302911, +ERASE, 140612808445952, 140612808450047, +ERASE, 140612808450048, 140612816838655, +ERASE, 140613236277248, 140613236281343, +ERASE, 140613236281344, 140613244669951, +ERASE, 140613580214272, 140613580218367, +ERASE, 140613580218368, 140613588606975, +ERASE, 140613370494976, 140613370499071, +ERASE, 140613370499072, 140613378887679, +ERASE, 140613244669952, 140613244674047, +ERASE, 140613244674048, 140613253062655, +ERASE, 140612724584448, 140612724588543, +ERASE, 140612724588544, 140612732977151, +ERASE, 140612707799040, 140612707803135, +ERASE, 140612707803136, 140612716191743, +ERASE, 140613504712704, 140613504716799, +ERASE, 140613504716800, 140613513105407, + }; + + unsigned long set39[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140736271417344, 140737488351231, +SNULL, 140736271421439, 140737488351231, +STORE, 140736271417344, 140736271421439, +STORE, 140736271286272, 140736271421439, +STORE, 94412930822144, 94412933074943, +SNULL, 94412930953215, 94412933074943, +STORE, 94412930822144, 94412930953215, +STORE, 94412930953216, 94412933074943, +ERASE, 94412930953216, 94412933074943, +STORE, 94412933046272, 94412933054463, +STORE, 94412933054464, 94412933074943, +STORE, 140326136901632, 140326139154431, +SNULL, 140326137044991, 140326139154431, +STORE, 140326136901632, 140326137044991, +STORE, 140326137044992, 140326139154431, +ERASE, 140326137044992, 140326139154431, +STORE, 140326139142144, 140326139150335, +STORE, 140326139150336, 140326139154431, +STORE, 140736271585280, 140736271589375, +STORE, 140736271572992, 140736271585279, +STORE, 140326139113472, 140326139142143, +STORE, 140326139105280, 140326139113471, +STORE, 140326134685696, 140326136901631, +SNULL, 140326134685696, 140326134783999, +STORE, 140326134784000, 140326136901631, +STORE, 140326134685696, 140326134783999, +SNULL, 140326136877055, 140326136901631, +STORE, 140326134784000, 140326136877055, +STORE, 140326136877056, 140326136901631, +SNULL, 140326136877056, 140326136885247, +STORE, 140326136885248, 140326136901631, +STORE, 140326136877056, 140326136885247, +ERASE, 140326136877056, 140326136885247, +STORE, 140326136877056, 140326136885247, +ERASE, 140326136885248, 140326136901631, +STORE, 140326136885248, 140326136901631, +STORE, 140326130888704, 140326134685695, +SNULL, 140326130888704, 140326132547583, +STORE, 140326132547584, 140326134685695, +STORE, 140326130888704, 140326132547583, +SNULL, 140326134644735, 140326134685695, +STORE, 140326132547584, 140326134644735, +STORE, 140326134644736, 140326134685695, +SNULL, 140326134644736, 140326134669311, +STORE, 140326134669312, 140326134685695, +STORE, 140326134644736, 140326134669311, +ERASE, 140326134644736, 140326134669311, +STORE, 140326134644736, 140326134669311, +ERASE, 140326134669312, 140326134685695, +STORE, 140326134669312, 140326134685695, +STORE, 140326139097088, 140326139113471, +SNULL, 140326134661119, 140326134669311, +STORE, 140326134644736, 140326134661119, +STORE, 140326134661120, 140326134669311, +SNULL, 140326136881151, 140326136885247, +STORE, 140326136877056, 140326136881151, +STORE, 140326136881152, 140326136885247, +SNULL, 94412933050367, 94412933054463, +STORE, 94412933046272, 94412933050367, +STORE, 94412933050368, 94412933054463, +SNULL, 140326139146239, 140326139150335, +STORE, 140326139142144, 140326139146239, +STORE, 140326139146240, 140326139150335, +ERASE, 140326139113472, 140326139142143, +STORE, 94412939493376, 94412939628543, +STORE, 140326122496000, 140326130888703, +SNULL, 140326122500095, 140326130888703, +STORE, 140326122496000, 140326122500095, +STORE, 140326122500096, 140326130888703, +STORE, 140326114103296, 140326122495999, +STORE, 140325979885568, 140326114103295, +SNULL, 140325979885568, 140326043910143, +STORE, 140326043910144, 140326114103295, +STORE, 140325979885568, 140326043910143, +ERASE, 140325979885568, 140326043910143, +SNULL, 140326111019007, 140326114103295, +STORE, 140326043910144, 140326111019007, +STORE, 140326111019008, 140326114103295, +ERASE, 140326111019008, 140326114103295, +SNULL, 140326044045311, 140326111019007, +STORE, 140326043910144, 140326044045311, +STORE, 140326044045312, 140326111019007, +SNULL, 140326114107391, 140326122495999, +STORE, 140326114103296, 140326114107391, +STORE, 140326114107392, 140326122495999, +STORE, 140326035517440, 140326043910143, +SNULL, 140326035521535, 140326043910143, +STORE, 140326035517440, 140326035521535, +STORE, 140326035521536, 140326043910143, +STORE, 140326027124736, 140326035517439, +SNULL, 140326027128831, 140326035517439, +STORE, 140326027124736, 140326027128831, +STORE, 140326027128832, 140326035517439, +STORE, 140326018732032, 140326027124735, +SNULL, 140326018736127, 140326027124735, +STORE, 140326018732032, 140326018736127, +STORE, 140326018736128, 140326027124735, +STORE, 140326010339328, 140326018732031, +STORE, 140326001946624, 140326018732031, +STORE, 140325993553920, 140326018732031, +STORE, 140325859336192, 140325993553919, +SNULL, 140325859336192, 140325909692415, +STORE, 140325909692416, 140325993553919, +STORE, 140325859336192, 140325909692415, +ERASE, 140325859336192, 140325909692415, +SNULL, 140325976801279, 140325993553919, +STORE, 140325909692416, 140325976801279, +STORE, 140325976801280, 140325993553919, +ERASE, 140325976801280, 140325993553919, +STORE, 140325985161216, 140326018732031, +STORE, 140325775474688, 140325976801279, +STORE, 140325708365824, 140325976801279, +SNULL, 140325708500991, 140325976801279, +STORE, 140325708365824, 140325708500991, +STORE, 140325708500992, 140325976801279, +SNULL, 140325708500992, 140325909692415, +STORE, 140325909692416, 140325976801279, +STORE, 140325708500992, 140325909692415, +SNULL, 140325909827583, 140325976801279, +STORE, 140325909692416, 140325909827583, +STORE, 140325909827584, 140325976801279, +SNULL, 140325842583551, 140325909692415, +STORE, 140325708500992, 140325842583551, +STORE, 140325842583552, 140325909692415, +ERASE, 140325842583552, 140325909692415, +SNULL, 140325708500992, 140325775474687, +STORE, 140325775474688, 140325842583551, +STORE, 140325708500992, 140325775474687, +SNULL, 140325775609855, 140325842583551, +STORE, 140325775474688, 140325775609855, +STORE, 140325775609856, 140325842583551, +STORE, 140325775609856, 140325909692415, +SNULL, 140325775609856, 140325842583551, +STORE, 140325842583552, 140325909692415, +STORE, 140325775609856, 140325842583551, +SNULL, 140325842718719, 140325909692415, +STORE, 140325842583552, 140325842718719, +STORE, 140325842718720, 140325909692415, +SNULL, 140325985161216, 140325993553919, +STORE, 140325993553920, 140326018732031, +STORE, 140325985161216, 140325993553919, +SNULL, 140325993558015, 140326018732031, +STORE, 140325993553920, 140325993558015, +STORE, 140325993558016, 140326018732031, +SNULL, 140325985165311, 140325993553919, +STORE, 140325985161216, 140325985165311, +STORE, 140325985165312, 140325993553919, +SNULL, 140325993558016, 140326001946623, +STORE, 140326001946624, 140326018732031, +STORE, 140325993558016, 140326001946623, +SNULL, 140326001950719, 140326018732031, +STORE, 140326001946624, 140326001950719, +STORE, 140326001950720, 140326018732031, +SNULL, 140326001950720, 140326010339327, +STORE, 140326010339328, 140326018732031, +STORE, 140326001950720, 140326010339327, +SNULL, 140326010343423, 140326018732031, +STORE, 140326010339328, 140326010343423, +STORE, 140326010343424, 140326018732031, +STORE, 140325699973120, 140325708365823, +STORE, 140325691580416, 140325708365823, +STORE, 140325683187712, 140325708365823, +SNULL, 140325683191807, 140325708365823, +STORE, 140325683187712, 140325683191807, +STORE, 140325683191808, 140325708365823, +SNULL, 140325683191808, 140325699973119, +STORE, 140325699973120, 140325708365823, +STORE, 140325683191808, 140325699973119, +SNULL, 140325699977215, 140325708365823, +STORE, 140325699973120, 140325699977215, +STORE, 140325699977216, 140325708365823, +STORE, 140325674795008, 140325683187711, +STORE, 140325666402304, 140325683187711, +STORE, 140325658009600, 140325683187711, +SNULL, 140325658009600, 140325666402303, +STORE, 140325666402304, 140325683187711, +STORE, 140325658009600, 140325666402303, +SNULL, 140325666406399, 140325683187711, +STORE, 140325666402304, 140325666406399, +STORE, 140325666406400, 140325683187711, +SNULL, 140325683191808, 140325691580415, +STORE, 140325691580416, 140325699973119, +STORE, 140325683191808, 140325691580415, +SNULL, 140325691584511, 140325699973119, +STORE, 140325691580416, 140325691584511, +STORE, 140325691584512, 140325699973119, +SNULL, 140325666406400, 140325674795007, +STORE, 140325674795008, 140325683187711, +STORE, 140325666406400, 140325674795007, +SNULL, 140325674799103, 140325683187711, +STORE, 140325674795008, 140325674799103, +STORE, 140325674799104, 140325683187711, +STORE, 140325649616896, 140325666402303, +SNULL, 140325649616896, 140325658009599, +STORE, 140325658009600, 140325666402303, +STORE, 140325649616896, 140325658009599, +SNULL, 140325658013695, 140325666402303, +STORE, 140325658009600, 140325658013695, +STORE, 140325658013696, 140325666402303, +SNULL, 140325649620991, 140325658009599, +STORE, 140325649616896, 140325649620991, +STORE, 140325649620992, 140325658009599, +STORE, 140325641224192, 140325649616895, +STORE, 140325632831488, 140325649616895, +SNULL, 140325632835583, 140325649616895, +STORE, 140325632831488, 140325632835583, +STORE, 140325632835584, 140325649616895, +STORE, 140325624438784, 140325632831487, +SNULL, 140325624442879, 140325632831487, +STORE, 140325624438784, 140325624442879, +STORE, 140325624442880, 140325632831487, +SNULL, 140325632835584, 140325641224191, +STORE, 140325641224192, 140325649616895, +STORE, 140325632835584, 140325641224191, +SNULL, 140325641228287, 140325649616895, +STORE, 140325641224192, 140325641228287, +STORE, 140325641228288, 140325649616895, +STORE, 140325616046080, 140325624438783, +SNULL, 140325616050175, 140325624438783, +STORE, 140325616046080, 140325616050175, +STORE, 140325616050176, 140325624438783, +STORE, 140325607653376, 140325616046079, +SNULL, 140325607657471, 140325616046079, +STORE, 140325607653376, 140325607657471, +STORE, 140325607657472, 140325616046079, +STORE, 140325599260672, 140325607653375, +STORE, 140325590867968, 140325607653375, +STORE, 140325456650240, 140325590867967, +SNULL, 140325456650240, 140325507039231, +STORE, 140325507039232, 140325590867967, +STORE, 140325456650240, 140325507039231, +ERASE, 140325456650240, 140325507039231, +STORE, 140325498646528, 140325507039231, +STORE, 140325364428800, 140325498646527, +SNULL, 140325364428800, 140325372821503, +STORE, 140325372821504, 140325498646527, +STORE, 140325364428800, 140325372821503, +ERASE, 140325364428800, 140325372821503, +STORE, 140325364428800, 140325372821503, +STORE, 140325356036096, 140325372821503, +STORE, 140325221818368, 140325356036095, +SNULL, 140325221818368, 140325238603775, +STORE, 140325238603776, 140325356036095, +STORE, 140325221818368, 140325238603775, +ERASE, 140325221818368, 140325238603775, +STORE, 140325230211072, 140325238603775, +STORE, 140325221818368, 140325238603775, +STORE, 140325087600640, 140325221818367, +STORE, 140325079207936, 140325087600639, +SNULL, 140325087600640, 140325104386047, +STORE, 140325104386048, 140325221818367, +STORE, 140325087600640, 140325104386047, +ERASE, 140325087600640, 140325104386047, +STORE, 140325095993344, 140325104386047, +STORE, 140325079207936, 140325104386047, +STORE, 140324944990208, 140325079207935, +SNULL, 140324944990208, 140324970168319, +STORE, 140324970168320, 140325079207935, +STORE, 140324944990208, 140324970168319, +ERASE, 140324944990208, 140324970168319, +STORE, 140324961775616, 140324970168319, +STORE, 140324953382912, 140324970168319, +STORE, 140324819165184, 140324953382911, +STORE, 140324684947456, 140324953382911, +STORE, 140324676554752, 140324684947455, +STORE, 140324668162048, 140324684947455, +STORE, 140324533944320, 140324668162047, +STORE, 140324525551616, 140324533944319, +SNULL, 140324533944320, 140324567515135, +STORE, 140324567515136, 140324668162047, +STORE, 140324533944320, 140324567515135, +ERASE, 140324533944320, 140324567515135, +STORE, 140324559122432, 140324567515135, +STORE, 140324391333888, 140324525551615, +SNULL, 140325574148095, 140325590867967, +STORE, 140325507039232, 140325574148095, +STORE, 140325574148096, 140325590867967, +ERASE, 140325574148096, 140325590867967, +SNULL, 140325439930367, 140325498646527, +STORE, 140325372821504, 140325439930367, +STORE, 140325439930368, 140325498646527, +ERASE, 140325439930368, 140325498646527, +SNULL, 140325305712639, 140325356036095, +STORE, 140325238603776, 140325305712639, +STORE, 140325305712640, 140325356036095, +ERASE, 140325305712640, 140325356036095, +SNULL, 140325171494911, 140325221818367, +STORE, 140325104386048, 140325171494911, +STORE, 140325171494912, 140325221818367, +ERASE, 140325171494912, 140325221818367, +SNULL, 140325104521215, 140325171494911, +STORE, 140325104386048, 140325104521215, +STORE, 140325104521216, 140325171494911, +STORE, 140324257116160, 140324525551615, +SNULL, 140324257116160, 140324299079679, +STORE, 140324299079680, 140324525551615, +STORE, 140324257116160, 140324299079679, +ERASE, 140324257116160, 140324299079679, +SNULL, 140325037277183, 140325079207935, +STORE, 140324970168320, 140325037277183, +STORE, 140325037277184, 140325079207935, +ERASE, 140325037277184, 140325079207935, +SNULL, 140324819165183, 140324953382911, +STORE, 140324684947456, 140324819165183, +STORE, 140324819165184, 140324953382911, +SNULL, 140324819165184, 140324835950591, +STORE, 140324835950592, 140324953382911, +STORE, 140324819165184, 140324835950591, +ERASE, 140324819165184, 140324835950591, +SNULL, 140324903059455, 140324953382911, +STORE, 140324835950592, 140324903059455, +STORE, 140324903059456, 140324953382911, +ERASE, 140324903059456, 140324953382911, +SNULL, 140324684947456, 140324701732863, +STORE, 140324701732864, 140324819165183, +STORE, 140324684947456, 140324701732863, +ERASE, 140324684947456, 140324701732863, +SNULL, 140324768841727, 140324819165183, +STORE, 140324701732864, 140324768841727, +STORE, 140324768841728, 140324819165183, +ERASE, 140324768841728, 140324819165183, +SNULL, 140324634623999, 140324668162047, +STORE, 140324567515136, 140324634623999, +STORE, 140324634624000, 140324668162047, +ERASE, 140324634624000, 140324668162047, +SNULL, 140324391333887, 140324525551615, +STORE, 140324299079680, 140324391333887, +STORE, 140324391333888, 140324525551615, +SNULL, 140324391333888, 140324433297407, +STORE, 140324433297408, 140324525551615, +STORE, 140324391333888, 140324433297407, +ERASE, 140324391333888, 140324433297407, +SNULL, 140325507174399, 140325574148095, +STORE, 140325507039232, 140325507174399, +STORE, 140325507174400, 140325574148095, +SNULL, 140325590867968, 140325599260671, +STORE, 140325599260672, 140325607653375, +STORE, 140325590867968, 140325599260671, +SNULL, 140325599264767, 140325607653375, +STORE, 140325599260672, 140325599264767, +STORE, 140325599264768, 140325607653375, +SNULL, 140325372956671, 140325439930367, +STORE, 140325372821504, 140325372956671, +STORE, 140325372956672, 140325439930367, +SNULL, 140324668166143, 140324684947455, +STORE, 140324668162048, 140324668166143, +STORE, 140324668166144, 140324684947455, +SNULL, 140324525555711, 140324533944319, +STORE, 140324525551616, 140324525555711, +STORE, 140324525555712, 140324533944319, +SNULL, 140324953382912, 140324961775615, +STORE, 140324961775616, 140324970168319, +STORE, 140324953382912, 140324961775615, +SNULL, 140324961779711, 140324970168319, +STORE, 140324961775616, 140324961779711, +STORE, 140324961779712, 140324970168319, +SNULL, 140325079212031, 140325104386047, +STORE, 140325079207936, 140325079212031, +STORE, 140325079212032, 140325104386047, +SNULL, 140325221818368, 140325230211071, +STORE, 140325230211072, 140325238603775, +STORE, 140325221818368, 140325230211071, +SNULL, 140325230215167, 140325238603775, +STORE, 140325230211072, 140325230215167, +STORE, 140325230215168, 140325238603775, +SNULL, 140325356036096, 140325364428799, +STORE, 140325364428800, 140325372821503, +STORE, 140325356036096, 140325364428799, +SNULL, 140325364432895, 140325372821503, + }; + unsigned long set40[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734309167104, 140737488351231, +SNULL, 140734309171199, 140737488351231, +STORE, 140734309167104, 140734309171199, +STORE, 140734309036032, 140734309171199, +STORE, 94270500081664, 94270502334463, +SNULL, 94270500212735, 94270502334463, +STORE, 94270500081664, 94270500212735, +STORE, 94270500212736, 94270502334463, +ERASE, 94270500212736, 94270502334463, +STORE, 94270502305792, 94270502313983, +STORE, 94270502313984, 94270502334463, +STORE, 140321935110144, 140321937362943, +SNULL, 140321935253503, 140321937362943, +STORE, 140321935110144, 140321935253503, +STORE, 140321935253504, 140321937362943, +ERASE, 140321935253504, 140321937362943, +STORE, 140321937350656, 140321937358847, +STORE, 140321937358848, 140321937362943, +STORE, 140734309625856, 140734309629951, +STORE, 140734309613568, 140734309625855, +STORE, 140321937321984, 140321937350655, +STORE, 140321937313792, 140321937321983, +STORE, 140321932894208, 140321935110143, +SNULL, 140321932894208, 140321932992511, +STORE, 140321932992512, 140321935110143, +STORE, 140321932894208, 140321932992511, +SNULL, 140321935085567, 140321935110143, +STORE, 140321932992512, 140321935085567, +STORE, 140321935085568, 140321935110143, +SNULL, 140321935085568, 140321935093759, +STORE, 140321935093760, 140321935110143, +STORE, 140321935085568, 140321935093759, +ERASE, 140321935085568, 140321935093759, +STORE, 140321935085568, 140321935093759, +ERASE, 140321935093760, 140321935110143, +STORE, 140321935093760, 140321935110143, +STORE, 140321929097216, 140321932894207, +SNULL, 140321929097216, 140321930756095, +STORE, 140321930756096, 140321932894207, +STORE, 140321929097216, 140321930756095, +SNULL, 140321932853247, 140321932894207, +STORE, 140321930756096, 140321932853247, +STORE, 140321932853248, 140321932894207, +SNULL, 140321932853248, 140321932877823, +STORE, 140321932877824, 140321932894207, +STORE, 140321932853248, 140321932877823, +ERASE, 140321932853248, 140321932877823, +STORE, 140321932853248, 140321932877823, +ERASE, 140321932877824, 140321932894207, +STORE, 140321932877824, 140321932894207, +STORE, 140321937305600, 140321937321983, +SNULL, 140321932869631, 140321932877823, +STORE, 140321932853248, 140321932869631, +STORE, 140321932869632, 140321932877823, +SNULL, 140321935089663, 140321935093759, +STORE, 140321935085568, 140321935089663, +STORE, 140321935089664, 140321935093759, +SNULL, 94270502309887, 94270502313983, +STORE, 94270502305792, 94270502309887, +STORE, 94270502309888, 94270502313983, +SNULL, 140321937354751, 140321937358847, +STORE, 140321937350656, 140321937354751, +STORE, 140321937354752, 140321937358847, +ERASE, 140321937321984, 140321937350655, +STORE, 94270507364352, 94270507499519, +STORE, 140321920704512, 140321929097215, +SNULL, 140321920708607, 140321929097215, +STORE, 140321920704512, 140321920708607, +STORE, 140321920708608, 140321929097215, +STORE, 140321912311808, 140321920704511, +STORE, 140321778094080, 140321912311807, +SNULL, 140321778094080, 140321816051711, +STORE, 140321816051712, 140321912311807, +STORE, 140321778094080, 140321816051711, +ERASE, 140321778094080, 140321816051711, +SNULL, 140321883160575, 140321912311807, +STORE, 140321816051712, 140321883160575, +STORE, 140321883160576, 140321912311807, +ERASE, 140321883160576, 140321912311807, +SNULL, 140321816186879, 140321883160575, +STORE, 140321816051712, 140321816186879, +STORE, 140321816186880, 140321883160575, +SNULL, 140321912315903, 140321920704511, +STORE, 140321912311808, 140321912315903, +STORE, 140321912315904, 140321920704511, +STORE, 140321903919104, 140321912311807, +SNULL, 140321903923199, 140321912311807, +STORE, 140321903919104, 140321903923199, +STORE, 140321903923200, 140321912311807, +STORE, 140321895526400, 140321903919103, +SNULL, 140321895530495, 140321903919103, +STORE, 140321895526400, 140321895530495, +STORE, 140321895530496, 140321903919103, +STORE, 140321887133696, 140321895526399, +SNULL, 140321887137791, 140321895526399, +STORE, 140321887133696, 140321887137791, +STORE, 140321887137792, 140321895526399, +STORE, 140321807659008, 140321816051711, +STORE, 140321673441280, 140321807659007, +SNULL, 140321673441280, 140321681833983, +STORE, 140321681833984, 140321807659007, +STORE, 140321673441280, 140321681833983, +ERASE, 140321673441280, 140321681833983, +SNULL, 140321748942847, 140321807659007, +STORE, 140321681833984, 140321748942847, +STORE, 140321748942848, 140321807659007, +ERASE, 140321748942848, 140321807659007, +STORE, 140321799266304, 140321816051711, +STORE, 140321790873600, 140321816051711, +STORE, 140321782480896, 140321816051711, +STORE, 140321547616256, 140321748942847, +SNULL, 140321614725119, 140321748942847, +STORE, 140321547616256, 140321614725119, +STORE, 140321614725120, 140321748942847, +SNULL, 140321614725120, 140321681833983, +STORE, 140321681833984, 140321748942847, +STORE, 140321614725120, 140321681833983, +ERASE, 140321614725120, 140321681833983, +SNULL, 140321681969151, 140321748942847, +STORE, 140321681833984, 140321681969151, +STORE, 140321681969152, 140321748942847, +STORE, 140321547616256, 140321681833983, +SNULL, 140321547616256, 140321614725119, +STORE, 140321614725120, 140321681833983, +STORE, 140321547616256, 140321614725119, +SNULL, 140321614860287, 140321681833983, +STORE, 140321614725120, 140321614860287, +STORE, 140321614860288, 140321681833983, +SNULL, 140321547751423, 140321614725119, +STORE, 140321547616256, 140321547751423, +STORE, 140321547751424, 140321614725119, +STORE, 140321480507392, 140321547616255, +SNULL, 140321782480896, 140321799266303, +STORE, 140321799266304, 140321816051711, +STORE, 140321782480896, 140321799266303, +SNULL, 140321799270399, 140321816051711, +STORE, 140321799266304, 140321799270399, +STORE, 140321799270400, 140321816051711, +STORE, 140321774088192, 140321799266303, +SNULL, 140321774088192, 140321790873599, +STORE, 140321790873600, 140321799266303, +STORE, 140321774088192, 140321790873599, +SNULL, 140321790877695, 140321799266303, +STORE, 140321790873600, 140321790877695, +STORE, 140321790877696, 140321799266303, +SNULL, 140321480642559, 140321547616255, +STORE, 140321480507392, 140321480642559, +STORE, 140321480642560, 140321547616255, +SNULL, 140321774088192, 140321782480895, +STORE, 140321782480896, 140321790873599, +STORE, 140321774088192, 140321782480895, +SNULL, 140321782484991, 140321790873599, +STORE, 140321782480896, 140321782484991, +STORE, 140321782484992, 140321790873599, +SNULL, 140321799270400, 140321807659007, +STORE, 140321807659008, 140321816051711, +STORE, 140321799270400, 140321807659007, +SNULL, 140321807663103, 140321816051711, +STORE, 140321807659008, 140321807663103, +STORE, 140321807663104, 140321816051711, +STORE, 140321765695488, 140321782480895, +STORE, 140321757302784, 140321782480895, +SNULL, 140321757306879, 140321782480895, +STORE, 140321757302784, 140321757306879, +STORE, 140321757306880, 140321782480895, +STORE, 140321472114688, 140321480507391, +STORE, 140321463721984, 140321480507391, +SNULL, 140321463726079, 140321480507391, +STORE, 140321463721984, 140321463726079, +STORE, 140321463726080, 140321480507391, +SNULL, 140321757306880, 140321774088191, +STORE, 140321774088192, 140321782480895, +STORE, 140321757306880, 140321774088191, +SNULL, 140321774092287, 140321782480895, +STORE, 140321774088192, 140321774092287, +STORE, 140321774092288, 140321782480895, +SNULL, 140321463726080, 140321472114687, +STORE, 140321472114688, 140321480507391, +STORE, 140321463726080, 140321472114687, +SNULL, 140321472118783, 140321480507391, +STORE, 140321472114688, 140321472118783, +STORE, 140321472118784, 140321480507391, +SNULL, 140321757306880, 140321765695487, +STORE, 140321765695488, 140321774088191, +STORE, 140321757306880, 140321765695487, +SNULL, 140321765699583, 140321774088191, +STORE, 140321765695488, 140321765699583, +STORE, 140321765699584, 140321774088191, +STORE, 140321455329280, 140321463721983, +SNULL, 140321455333375, 140321463721983, +STORE, 140321455329280, 140321455333375, +STORE, 140321455333376, 140321463721983, +STORE, 140321446936576, 140321455329279, +STORE, 140321438543872, 140321455329279, +STORE, 140321430151168, 140321455329279, +SNULL, 140321430155263, 140321455329279, +STORE, 140321430151168, 140321430155263, +STORE, 140321430155264, 140321455329279, +SNULL, 140321430155264, 140321446936575, +STORE, 140321446936576, 140321455329279, +STORE, 140321430155264, 140321446936575, +SNULL, 140321446940671, 140321455329279, +STORE, 140321446936576, 140321446940671, +STORE, 140321446940672, 140321455329279, +SNULL, 140321430155264, 140321438543871, +STORE, 140321438543872, 140321446936575, +STORE, 140321430155264, 140321438543871, +SNULL, 140321438547967, 140321446936575, +STORE, 140321438543872, 140321438547967, +STORE, 140321438547968, 140321446936575, +STORE, 140321421758464, 140321430151167, +SNULL, 140321421762559, 140321430151167, +STORE, 140321421758464, 140321421762559, +STORE, 140321421762560, 140321430151167, +STORE, 140321413365760, 140321421758463, +SNULL, 140321413369855, 140321421758463, +STORE, 140321413365760, 140321413369855, +STORE, 140321413369856, 140321421758463, +STORE, 140321404973056, 140321413365759, +SNULL, 140321404977151, 140321413365759, +STORE, 140321404973056, 140321404977151, +STORE, 140321404977152, 140321413365759, +STORE, 140321396580352, 140321404973055, +STORE, 140321388187648, 140321404973055, +STORE, 140321253969920, 140321388187647, +SNULL, 140321253969920, 140321279180799, +STORE, 140321279180800, 140321388187647, +STORE, 140321253969920, 140321279180799, +ERASE, 140321253969920, 140321279180799, +SNULL, 140321346289663, 140321388187647, +STORE, 140321279180800, 140321346289663, +STORE, 140321346289664, 140321388187647, +ERASE, 140321346289664, 140321388187647, +STORE, 140321144963072, 140321346289663, +STORE, 140321379794944, 140321404973055, +STORE, 140321371402240, 140321404973055, +STORE, 140321010745344, 140321346289663, +STORE, 140321363009536, 140321404973055, +SNULL, 140321077854207, 140321346289663, +STORE, 140321010745344, 140321077854207, +STORE, 140321077854208, 140321346289663, +SNULL, 140321077854208, 140321144963071, +STORE, 140321144963072, 140321346289663, +STORE, 140321077854208, 140321144963071, +ERASE, 140321077854208, 140321144963071, +STORE, 140321354616832, 140321404973055, +STORE, 140321136570368, 140321144963071, +STORE, 140320943636480, 140321077854207, +STORE, 140320876527616, 140321077854207, +STORE, 140321128177664, 140321144963071, +SNULL, 140320876662783, 140321077854207, +STORE, 140320876527616, 140320876662783, +STORE, 140320876662784, 140321077854207, +STORE, 140321119784960, 140321144963071, +STORE, 140321111392256, 140321144963071, +STORE, 140320742309888, 140320876527615, +STORE, 140321102999552, 140321144963071, +STORE, 140320608092160, 140320876527615, +SNULL, 140320675201023, 140320876527615, +STORE, 140320608092160, 140320675201023, +STORE, 140320675201024, 140320876527615, +SNULL, 140320675201024, 140320742309887, +STORE, 140320742309888, 140320876527615, +STORE, 140320675201024, 140320742309887, +ERASE, 140320675201024, 140320742309887, +STORE, 140321094606848, 140321144963071, +STORE, 140321086214144, 140321144963071, +STORE, 140320608092160, 140320876527615, +SNULL, 140320608092160, 140320675201023, +STORE, 140320675201024, 140320876527615, +STORE, 140320608092160, 140320675201023, +SNULL, 140320675336191, 140320876527615, +STORE, 140320675201024, 140320675336191, +STORE, 140320675336192, 140320876527615, +STORE, 140320599699456, 140320608092159, +STORE, 140320591306752, 140320608092159, +STORE, 140320457089024, 140320591306751, +STORE, 140320448696320, 140320457089023, +STORE, 140320314478592, 140320448696319, +SNULL, 140321144963072, 140321279180799, +STORE, 140321279180800, 140321346289663, +STORE, 140321144963072, 140321279180799, +SNULL, 140321279315967, 140321346289663, +STORE, 140321279180800, 140321279315967, +STORE, 140321279315968, 140321346289663, +SNULL, 140321086214144, 140321136570367, +STORE, 140321136570368, 140321144963071, +STORE, 140321086214144, 140321136570367, +SNULL, 140321136574463, 140321144963071, +STORE, 140321136570368, 140321136574463, +STORE, 140321136574464, 140321144963071, +SNULL, 140321212071935, 140321279180799, +STORE, 140321144963072, 140321212071935, +STORE, 140321212071936, 140321279180799, +ERASE, 140321212071936, 140321279180799, +SNULL, 140321145098239, 140321212071935, +STORE, 140321144963072, 140321145098239, +STORE, 140321145098240, 140321212071935, +SNULL, 140320876662784, 140321010745343, +STORE, 140321010745344, 140321077854207, +STORE, 140320876662784, 140321010745343, +SNULL, 140321010880511, 140321077854207, +STORE, 140321010745344, 140321010880511, +STORE, 140321010880512, 140321077854207, +SNULL, 140321354616832, 140321379794943, +STORE, 140321379794944, 140321404973055, +STORE, 140321354616832, 140321379794943, +SNULL, 140321379799039, 140321404973055, +STORE, 140321379794944, 140321379799039, +STORE, 140321379799040, 140321404973055, +SNULL, 140320876662784, 140320943636479, +STORE, 140320943636480, 140321010745343, +STORE, 140320876662784, 140320943636479, +SNULL, 140320943771647, 140321010745343, +STORE, 140320943636480, 140320943771647, +STORE, 140320943771648, 140321010745343, +SNULL, 140320809418751, 140320876527615, +STORE, 140320675336192, 140320809418751, +STORE, 140320809418752, 140320876527615, +ERASE, 140320809418752, 140320876527615, +SNULL, 140320675336192, 140320742309887, +STORE, 140320742309888, 140320809418751, +STORE, 140320675336192, 140320742309887, +SNULL, 140320742445055, 140320809418751, +STORE, 140320742309888, 140320742445055, +STORE, 140320742445056, 140320809418751, +SNULL, 140320608227327, 140320675201023, +STORE, 140320608092160, 140320608227327, +STORE, 140320608227328, 140320675201023, +SNULL, 140320457089024, 140320473874431, +STORE, 140320473874432, 140320591306751, +STORE, 140320457089024, 140320473874431, +ERASE, 140320457089024, 140320473874431, +SNULL, 140320540983295, 140320591306751, +STORE, 140320473874432, 140320540983295, +STORE, 140320540983296, 140320591306751, +ERASE, 140320540983296, 140320591306751, +SNULL, 140320314478592, 140320339656703, +STORE, 140320339656704, 140320448696319, +STORE, 140320314478592, 140320339656703, +ERASE, 140320314478592, 140320339656703, +SNULL, 140321086214144, 140321128177663, +STORE, 140321128177664, 140321136570367, +STORE, 140321086214144, 140321128177663, +SNULL, 140321128181759, 140321136570367, +STORE, 140321128177664, 140321128181759, +STORE, 140321128181760, 140321136570367, +SNULL, 140321354616832, 140321371402239, +STORE, 140321371402240, 140321379794943, +STORE, 140321354616832, 140321371402239, +SNULL, 140321371406335, 140321379794943, +STORE, 140321371402240, 140321371406335, +STORE, 140321371406336, 140321379794943, +SNULL, 140320591310847, 140320608092159, +STORE, 140320591306752, 140320591310847, +STORE, 140320591310848, 140320608092159, +SNULL, 140321354616832, 140321363009535, +STORE, 140321363009536, 140321371402239, +STORE, 140321354616832, 140321363009535, +SNULL, 140321363013631, 140321371402239, +STORE, 140321363009536, 140321363013631, +STORE, 140321363013632, 140321371402239, +SNULL, 140321086214144, 140321119784959, +STORE, 140321119784960, 140321128177663, +STORE, 140321086214144, 140321119784959, +SNULL, 140321119789055, 140321128177663, +STORE, 140321119784960, 140321119789055, +STORE, 140321119789056, 140321128177663, +SNULL, 140321086218239, 140321119784959, +STORE, 140321086214144, 140321086218239, +STORE, 140321086218240, 140321119784959, +SNULL, 140321086218240, 140321094606847, +STORE, 140321094606848, 140321119784959, +STORE, 140321086218240, 140321094606847, +SNULL, 140321094610943, 140321119784959, +STORE, 140321094606848, 140321094610943, +STORE, 140321094610944, 140321119784959, +SNULL, 140320474009599, 140320540983295, +STORE, 140320473874432, 140320474009599, +STORE, 140320474009600, 140320540983295, +SNULL, 140320406765567, 140320448696319, +STORE, 140320339656704, 140320406765567, +STORE, 140320406765568, 140320448696319, +ERASE, 140320406765568, 140320448696319, +SNULL, 140320339791871, 140320406765567, +STORE, 140320339656704, 140320339791871, +STORE, 140320339791872, 140320406765567, +STORE, 140321270788096, 140321279180799, +STORE, 140321262395392, 140321279180799, +STORE, 140321254002688, 140321279180799, +SNULL, 140321254002688, 140321262395391, +STORE, 140321262395392, 140321279180799, +STORE, 140321254002688, 140321262395391, +SNULL, 140321262399487, 140321279180799, +STORE, 140321262395392, 140321262399487, +STORE, 140321262399488, 140321279180799, +STORE, 140321245609984, 140321262395391, +STORE, 140321237217280, 140321262395391, +SNULL, 140321237217280, 140321245609983, +STORE, 140321245609984, 140321262395391, +STORE, 140321237217280, 140321245609983, +SNULL, 140321245614079, 140321262395391, +STORE, 140321245609984, 140321245614079, +STORE, 140321245614080, 140321262395391, +SNULL, 140321379799040, 140321388187647, +STORE, 140321388187648, 140321404973055, +STORE, 140321379799040, 140321388187647, +SNULL, 140321388191743, 140321404973055, +STORE, 140321388187648, 140321388191743, +STORE, 140321388191744, 140321404973055, +SNULL, 140321354620927, 140321363009535, +STORE, 140321354616832, 140321354620927, +STORE, 140321354620928, 140321363009535, +SNULL, 140321388191744, 140321396580351, +STORE, 140321396580352, 140321404973055, +STORE, 140321388191744, 140321396580351, +SNULL, 140321396584447, 140321404973055, +STORE, 140321396580352, 140321396584447, +STORE, 140321396584448, 140321404973055, +SNULL, 140321094610944, 140321111392255, +STORE, 140321111392256, 140321119784959, +STORE, 140321094610944, 140321111392255, +SNULL, 140321111396351, 140321119784959, +STORE, 140321111392256, 140321111396351, +STORE, 140321111396352, 140321119784959, +STORE, 140321228824576, 140321245609983, +SNULL, 140321094610944, 140321102999551, +STORE, 140321102999552, 140321111392255, +STORE, 140321094610944, 140321102999551, +SNULL, 140321103003647, 140321111392255, +STORE, 140321102999552, 140321103003647, +STORE, 140321103003648, 140321111392255, +STORE, 140321220431872, 140321245609983, +SNULL, 140321220435967, 140321245609983, +STORE, 140321220431872, 140321220435967, +STORE, 140321220435968, 140321245609983, +STORE, 140320868134912, 140320876527615, +SNULL, 140320868139007, 140320876527615, +STORE, 140320868134912, 140320868139007, +STORE, 140320868139008, 140320876527615, +SNULL, 140320591310848, 140320599699455, +STORE, 140320599699456, 140320608092159, +STORE, 140320591310848, 140320599699455, +SNULL, 140320599703551, 140320608092159, +STORE, 140320599699456, 140320599703551, +STORE, 140320599703552, 140320608092159, +STORE, 140320859742208, 140320868134911, +SNULL, 140321262399488, 140321270788095, +STORE, 140321270788096, 140321279180799, +STORE, 140321262399488, 140321270788095, +SNULL, 140321270792191, 140321279180799, +STORE, 140321270788096, 140321270792191, +STORE, 140321270792192, 140321279180799, +STORE, 140320851349504, 140320868134911, +STORE, 140320842956800, 140320868134911, +STORE, 140320834564096, 140320868134911, +STORE, 140320826171392, 140320868134911, +SNULL, 140320826171392, 140320834564095, +STORE, 140320834564096, 140320868134911, +STORE, 140320826171392, 140320834564095, +SNULL, 140320834568191, 140320868134911, +STORE, 140320834564096, 140320834568191, +STORE, 140320834568192, 140320868134911, +SNULL, 140321220435968, 140321228824575, +STORE, 140321228824576, 140321245609983, +STORE, 140321220435968, 140321228824575, +SNULL, 140321228828671, 140321245609983, +STORE, 140321228824576, 140321228828671, +STORE, 140321228828672, 140321245609983, +STORE, 140320817778688, 140320834564095, +SNULL, 140320817782783, 140320834564095, +STORE, 140320817778688, 140320817782783, +STORE, 140320817782784, 140320834564095, +STORE, 140320582914048, 140320591306751, +SNULL, 140321228828672, 140321237217279, +STORE, 140321237217280, 140321245609983, +STORE, 140321228828672, 140321237217279, +SNULL, 140321237221375, 140321245609983, +STORE, 140321237217280, 140321237221375, +STORE, 140321237221376, 140321245609983, +SNULL, 140320448700415, 140320457089023, +STORE, 140320448696320, 140320448700415, +STORE, 140320448700416, 140320457089023, +SNULL, 140321245614080, 140321254002687, +STORE, 140321254002688, 140321262395391, +STORE, 140321245614080, 140321254002687, +SNULL, 140321254006783, 140321262395391, +STORE, 140321254002688, 140321254006783, +STORE, 140321254006784, 140321262395391, +STORE, 140320574521344, 140320591306751, +SNULL, 140320574525439, 140320591306751, +STORE, 140320574521344, 140320574525439, +STORE, 140320574525440, 140320591306751, +STORE, 140320566128640, 140320574521343, +SNULL, 140320566132735, 140320574521343, +STORE, 140320566128640, 140320566132735, +STORE, 140320566132736, 140320574521343, +SNULL, 140320574525440, 140320582914047, +STORE, 140320582914048, 140320591306751, +STORE, 140320574525440, 140320582914047, +SNULL, 140320582918143, 140320591306751, +STORE, 140320582914048, 140320582918143, +STORE, 140320582918144, 140320591306751, +STORE, 140320557735936, 140320566128639, +SNULL, 140320557740031, 140320566128639, +STORE, 140320557735936, 140320557740031, +STORE, 140320557740032, 140320566128639, +STORE, 140320549343232, 140320557735935, +STORE, 140320465481728, 140320473874431, +STORE, 140320448700416, 140320473874431, +SNULL, 140320834568192, 140320859742207, +STORE, 140320859742208, 140320868134911, +STORE, 140320834568192, 140320859742207, +SNULL, 140320859746303, 140320868134911, +STORE, 140320859742208, 140320859746303, +STORE, 140320859746304, 140320868134911, +STORE, 140320440303616, 140320448696319, +STORE, 140320431910912, 140320448696319, +SNULL, 140320834568192, 140320851349503, +STORE, 140320851349504, 140320859742207, +STORE, 140320834568192, 140320851349503, +SNULL, 140320851353599, 140320859742207, +STORE, 140320851349504, 140320851353599, +STORE, 140320851353600, 140320859742207, +SNULL, 140320817782784, 140320826171391, +STORE, 140320826171392, 140320834564095, +STORE, 140320817782784, 140320826171391, +SNULL, 140320826175487, 140320834564095, +STORE, 140320826171392, 140320826175487, +STORE, 140320826175488, 140320834564095, +SNULL, 140320834568192, 140320842956799, +STORE, 140320842956800, 140320851349503, +STORE, 140320834568192, 140320842956799, +SNULL, 140320842960895, 140320851349503, +STORE, 140320842956800, 140320842960895, +STORE, 140320842960896, 140320851349503, +STORE, 140320423518208, 140320448696319, +SNULL, 140320423522303, 140320448696319, +STORE, 140320423518208, 140320423522303, +STORE, 140320423522304, 140320448696319, +STORE, 140320415125504, 140320423518207, +STORE, 140320331264000, 140320339656703, +STORE, 140320322871296, 140320339656703, +STORE, 140320314478592, 140320339656703, +SNULL, 140320314482687, 140320339656703, +STORE, 140320314478592, 140320314482687, +STORE, 140320314482688, 140320339656703, +STORE, 140320306085888, 140320314478591, +SNULL, 140320306089983, 140320314478591, +STORE, 140320306085888, 140320306089983, +STORE, 140320306089984, 140320314478591, +STORE, 140320297693184, 140320306085887, +SNULL, 140320297697279, 140320306085887, +STORE, 140320297693184, 140320297697279, +STORE, 140320297697280, 140320306085887, +STORE, 140320289300480, 140320297693183, +STORE, 140320280907776, 140320297693183, +SNULL, 140320280911871, 140320297693183, +STORE, 140320280907776, 140320280911871, +STORE, 140320280911872, 140320297693183, +SNULL, 140320423522304, 140320431910911, +STORE, 140320431910912, 140320448696319, +STORE, 140320423522304, 140320431910911, +SNULL, 140320431915007, 140320448696319, +STORE, 140320431910912, 140320431915007, +STORE, 140320431915008, 140320448696319, +SNULL, 140320549347327, 140320557735935, +STORE, 140320549343232, 140320549347327, +STORE, 140320549347328, 140320557735935, +STORE, 140320272515072, 140320280907775, +SNULL, 140320448700416, 140320457089023, +STORE, 140320457089024, 140320473874431, +STORE, 140320448700416, 140320457089023, +SNULL, 140320457093119, 140320473874431, +STORE, 140320457089024, 140320457093119, +STORE, 140320457093120, 140320473874431, +STORE, 140320264122368, 140320280907775, +SNULL, 140320457093120, 140320465481727, +STORE, 140320465481728, 140320473874431, +STORE, 140320457093120, 140320465481727, +SNULL, 140320465485823, 140320473874431, +STORE, 140320465481728, 140320465485823, +STORE, 140320465485824, 140320473874431, +SNULL, 140320431915008, 140320440303615, +STORE, 140320440303616, 140320448696319, +STORE, 140320431915008, 140320440303615, +SNULL, 140320440307711, 140320448696319, +STORE, 140320440303616, 140320440307711, +STORE, 140320440307712, 140320448696319, +STORE, 140320255729664, 140320280907775, +STORE, 140320247336960, 140320280907775, +SNULL, 140320247341055, 140320280907775, +STORE, 140320247336960, 140320247341055, +STORE, 140320247341056, 140320280907775, +STORE, 140320238944256, 140320247336959, +STORE, 140320230551552, 140320247336959, +SNULL, 140320230551552, 140320238944255, +STORE, 140320238944256, 140320247336959, +STORE, 140320230551552, 140320238944255, +SNULL, 140320238948351, 140320247336959, +STORE, 140320238944256, 140320238948351, +STORE, 140320238948352, 140320247336959, +SNULL, 140320314482688, 140320331263999, +STORE, 140320331264000, 140320339656703, +STORE, 140320314482688, 140320331263999, +SNULL, 140320331268095, 140320339656703, +STORE, 140320331264000, 140320331268095, +STORE, 140320331268096, 140320339656703, +SNULL, 140320280911872, 140320289300479, +STORE, 140320289300480, 140320297693183, +STORE, 140320280911872, 140320289300479, +SNULL, 140320289304575, 140320297693183, +STORE, 140320289300480, 140320289304575, +STORE, 140320289304576, 140320297693183, +SNULL, 140320415129599, 140320423518207, +STORE, 140320415125504, 140320415129599, +STORE, 140320415129600, 140320423518207, +STORE, 140320222158848, 140320238944255, +STORE, 140320213766144, 140320238944255, +STORE, 140320205373440, 140320238944255, +SNULL, 140320205377535, 140320238944255, +STORE, 140320205373440, 140320205377535, +STORE, 140320205377536, 140320238944255, +SNULL, 140320314482688, 140320322871295, +STORE, 140320322871296, 140320331263999, +STORE, 140320314482688, 140320322871295, +SNULL, 140320322875391, 140320331263999, +STORE, 140320322871296, 140320322875391, +STORE, 140320322875392, 140320331263999, +SNULL, 140320247341056, 140320272515071, +STORE, 140320272515072, 140320280907775, +STORE, 140320247341056, 140320272515071, +SNULL, 140320272519167, 140320280907775, +STORE, 140320272515072, 140320272519167, +STORE, 140320272519168, 140320280907775, +SNULL, 140320247341056, 140320264122367, +STORE, 140320264122368, 140320272515071, +STORE, 140320247341056, 140320264122367, +SNULL, 140320264126463, 140320272515071, +STORE, 140320264122368, 140320264126463, +STORE, 140320264126464, 140320272515071, +SNULL, 140320205377536, 140320230551551, +STORE, 140320230551552, 140320238944255, +STORE, 140320205377536, 140320230551551, +SNULL, 140320230555647, 140320238944255, +STORE, 140320230551552, 140320230555647, +STORE, 140320230555648, 140320238944255, +STORE, 140320196980736, 140320205373439, +SNULL, 140320196984831, 140320205373439, +STORE, 140320196980736, 140320196984831, +STORE, 140320196984832, 140320205373439, +STORE, 140320188588032, 140320196980735, +SNULL, 140320247341056, 140320255729663, +STORE, 140320255729664, 140320264122367, +STORE, 140320247341056, 140320255729663, +SNULL, 140320255733759, 140320264122367, +STORE, 140320255729664, 140320255733759, +STORE, 140320255733760, 140320264122367, +STORE, 140320180195328, 140320196980735, +SNULL, 140320180199423, 140320196980735, +STORE, 140320180195328, 140320180199423, +STORE, 140320180199424, 140320196980735, +STORE, 140320171802624, 140320180195327, +STORE, 140320163409920, 140320180195327, +SNULL, 140320163414015, 140320180195327, +STORE, 140320163409920, 140320163414015, +STORE, 140320163414016, 140320180195327, +SNULL, 140320205377536, 140320222158847, +STORE, 140320222158848, 140320230551551, +STORE, 140320205377536, 140320222158847, +SNULL, 140320222162943, 140320230551551, +STORE, 140320222158848, 140320222162943, +STORE, 140320222162944, 140320230551551, +SNULL, 140320205377536, 140320213766143, +STORE, 140320213766144, 140320222158847, +STORE, 140320205377536, 140320213766143, +SNULL, 140320213770239, 140320222158847, +STORE, 140320213766144, 140320213770239, +STORE, 140320213770240, 140320222158847, +STORE, 140320155017216, 140320163409919, +SNULL, 140320180199424, 140320188588031, +STORE, 140320188588032, 140320196980735, +STORE, 140320180199424, 140320188588031, +SNULL, 140320188592127, 140320196980735, +STORE, 140320188588032, 140320188592127, +STORE, 140320188592128, 140320196980735, +SNULL, 140320155021311, 140320163409919, +STORE, 140320155017216, 140320155021311, +STORE, 140320155021312, 140320163409919, +SNULL, 140320163414016, 140320171802623, +STORE, 140320171802624, 140320180195327, +STORE, 140320163414016, 140320171802623, +SNULL, 140320171806719, 140320180195327, +STORE, 140320171802624, 140320171806719, +STORE, 140320171806720, 140320180195327, +STORE, 140320146624512, 140320155017215, +SNULL, 140320146628607, 140320155017215, +STORE, 140320146624512, 140320146628607, +STORE, 140320146628608, 140320155017215, +STORE, 140321937321984, 140321937350655, +STORE, 140321884942336, 140321887133695, +SNULL, 140321884942336, 140321885032447, +STORE, 140321885032448, 140321887133695, +STORE, 140321884942336, 140321885032447, +SNULL, 140321887125503, 140321887133695, +STORE, 140321885032448, 140321887125503, +STORE, 140321887125504, 140321887133695, +ERASE, 140321887125504, 140321887133695, +STORE, 140321887125504, 140321887133695, +SNULL, 140321887129599, 140321887133695, +STORE, 140321887125504, 140321887129599, +STORE, 140321887129600, 140321887133695, +ERASE, 140321937321984, 140321937350655, +ERASE, 140321086214144, 140321086218239, +ERASE, 140321086218240, 140321094606847, +ERASE, 140321119784960, 140321119789055, +ERASE, 140321119789056, 140321128177663, +ERASE, 140321245609984, 140321245614079, +ERASE, 140321245614080, 140321254002687, +ERASE, 140320574521344, 140320574525439, +ERASE, 140320574525440, 140320582914047, +ERASE, 140320297693184, 140320297697279, +ERASE, 140320297697280, 140320306085887, +ERASE, 140321354616832, 140321354620927, +ERASE, 140321354620928, 140321363009535, +ERASE, 140320834564096, 140320834568191, +ERASE, 140320834568192, 140320842956799, +ERASE, 140320591306752, 140320591310847, +ERASE, 140320591310848, 140320599699455, +ERASE, 140321136570368, 140321136574463, +ERASE, 140321136574464, 140321144963071, +ERASE, 140321237217280, 140321237221375, +ERASE, 140321237221376, 140321245609983, +ERASE, 140321363009536, 140321363013631, +ERASE, 140321363013632, 140321371402239, +ERASE, 140320599699456, 140320599703551, +ERASE, 140320599703552, 140320608092159, +ERASE, 140321396580352, 140321396584447, +ERASE, 140321396584448, 140321404973055, +ERASE, 140320566128640, 140320566132735, +ERASE, 140320566132736, 140320574521343, +ERASE, 140321094606848, 140321094610943, +ERASE, 140321094610944, 140321102999551, +ERASE, 140320582914048, 140320582918143, +ERASE, 140320582918144, 140320591306751, +ERASE, 140320289300480, 140320289304575, +ERASE, 140320289304576, 140320297693183, +ERASE, 140320163409920, 140320163414015, + }; + unsigned long set41[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140728157171712, 140737488351231, +SNULL, 140728157175807, 140737488351231, +STORE, 140728157171712, 140728157175807, +STORE, 140728157040640, 140728157175807, +STORE, 94376106364928, 94376108613631, +SNULL, 94376106487807, 94376108613631, +STORE, 94376106364928, 94376106487807, +STORE, 94376106487808, 94376108613631, +SNULL, 94376106487808, 94376108613631, +STORE, 94376108584960, 94376108593151, +STORE, 94376108593152, 94376108613631, +STORE, 140113496432640, 140113498685439, +SNULL, 140113496575999, 140113498685439, +STORE, 140113496432640, 140113496575999, +STORE, 140113496576000, 140113498685439, +SNULL, 140113496576000, 140113498685439, +STORE, 140113498673152, 140113498681343, +STORE, 140113498681344, 140113498685439, +STORE, 140728157609984, 140728157618175, +STORE, 140728157593600, 140728157609983, +STORE, 140113498636288, 140113498673151, +STORE, 140113498628096, 140113498636287, +STORE, 140113492635648, 140113496432639, +SNULL, 140113492635648, 140113494294527, +STORE, 140113494294528, 140113496432639, +STORE, 140113492635648, 140113494294527, +SNULL, 140113496391679, 140113496432639, +STORE, 140113494294528, 140113496391679, +STORE, 140113496391680, 140113496432639, +SNULL, 140113496391680, 140113496416255, +STORE, 140113496416256, 140113496432639, +STORE, 140113496391680, 140113496416255, +SNULL, 140113496391680, 140113496416255, +STORE, 140113496391680, 140113496416255, +SNULL, 140113496416256, 140113496432639, +STORE, 140113496416256, 140113496432639, +SNULL, 140113496408063, 140113496416255, +STORE, 140113496391680, 140113496408063, +STORE, 140113496408064, 140113496416255, +SNULL, 94376108589055, 94376108593151, +STORE, 94376108584960, 94376108589055, +STORE, 94376108589056, 94376108593151, +SNULL, 140113498677247, 140113498681343, +STORE, 140113498673152, 140113498677247, +STORE, 140113498677248, 140113498681343, +SNULL, 140113498636288, 140113498673151, +STORE, 94376135090176, 94376135094271, +STORE, 94376135090176, 94376135098367, +STORE, 94376139288576, 94376139292671, +STORE, 94376143482880, 94376143486975, +STORE, 94376147677184, 94376147681279, +STORE, 94376151871488, 94376151875583, +STORE, 94376156065792, 94376156069887, +STORE, 94376160260096, 94376160264191, +STORE, 94376164454400, 94376164458495, +STORE, 94376168648704, 94376168652799, +STORE, 94376172843008, 94376172847103, +STORE, 94376177037312, 94376177041407, +STORE, 94376181231616, 94376181235711, +STORE, 94376185425920, 94376185430015, +STORE, 94376189620224, 94376189624319, +STORE, 94376193814528, 94376193818623, +STORE, 94376198008832, 94376198012927, +STORE, 94376202203136, 94376202207231, +STORE, 94376206397440, 94376206401535, +STORE, 94376210591744, 94376210595839, +STORE, 94376214786048, 94376214790143, +STORE, 94376218980352, 94376218984447, +STORE, 94376223174656, 94376223178751, +STORE, 94376227368960, 94376227373055, +STORE, 94376231563264, 94376231567359, +STORE, 94376235757568, 94376235761663, +STORE, 94376239951872, 94376239955967, +STORE, 94376244146176, 94376244150271, +STORE, 94376248340480, 94376248344575, +STORE, 94376252534784, 94376252538879, +STORE, 94376256729088, 94376256733183, +STORE, 94376260923392, 94376260927487, +STORE, 94376265117696, 94376265121791, +STORE, 94376269312000, 94376269316095, +STORE, 94376273506304, 94376273510399, +STORE, 94376277700608, 94376277704703, +STORE, 94376281894912, 94376281899007, +STORE, 94376286089216, 94376286093311, +STORE, 94376290283520, 94376290287615, +STORE, 94376294477824, 94376294481919, +STORE, 94376298672128, 94376298676223, +STORE, 94376302866432, 94376302870527, +STORE, 94376307060736, 94376307064831, +STORE, 94376311255040, 94376311259135, +STORE, 94376315449344, 94376315453439, +STORE, 94376319643648, 94376319647743, +STORE, 94376323837952, 94376323842047, +STORE, 94376328032256, 94376328036351, +STORE, 94376332226560, 94376332230655, +STORE, 94376336420864, 94376336424959, +STORE, 94376340615168, 94376340619263, +STORE, 94376344809472, 94376344813567, +STORE, 94376349003776, 94376349007871, +STORE, 94376353198080, 94376353202175, +STORE, 94376357392384, 94376357396479, +STORE, 94376361586688, 94376361590783, +STORE, 94376365780992, 94376365785087, +STORE, 94376369975296, 94376369979391, +STORE, 94376374169600, 94376374173695, +STORE, 94376378363904, 94376378367999, +STORE, 94376382558208, 94376382562303, +STORE, 94376386752512, 94376386756607, +STORE, 94376390946816, 94376390950911, +STORE, 94376395141120, 94376395145215, +STORE, 94376399335424, 94376399339519, +STORE, 94376403529728, 94376403533823, +STORE, 94376407724032, 94376407728127, +STORE, 94376411918336, 94376411922431, +STORE, 94376416112640, 94376416116735, +STORE, 94376420306944, 94376420311039, +STORE, 94376424501248, 94376424505343, +STORE, 94376428695552, 94376428699647, +STORE, 94376432889856, 94376432893951, +STORE, 94376437084160, 94376437088255, +STORE, 94376441278464, 94376441282559, +STORE, 94376445472768, 94376445476863, +STORE, 94376449667072, 94376449671167, +STORE, 94376453861376, 94376453865471, +STORE, 94376458055680, 94376458059775, +STORE, 94376462249984, 94376462254079, +STORE, 94376466444288, 94376466448383, +STORE, 94376470638592, 94376470642687, +STORE, 94376474832896, 94376474836991, +STORE, 94376479027200, 94376479031295, +STORE, 94376483221504, 94376483225599, +STORE, 94376487415808, 94376487419903, +STORE, 94376491610112, 94376491614207, +STORE, 94376495804416, 94376495808511, +STORE, 94376499998720, 94376500002815, +STORE, 94376504193024, 94376504197119, +STORE, 94376508387328, 94376508391423, +STORE, 94376512581632, 94376512585727, +STORE, 94376516775936, 94376516780031, +STORE, 94376520970240, 94376520974335, +STORE, 94376525164544, 94376525168639, +STORE, 94376529358848, 94376529362943, +STORE, 94376533553152, 94376533557247, +STORE, 94376537747456, 94376537751551, +STORE, 94376541941760, 94376541945855, +STORE, 94376546136064, 94376546140159, +STORE, 94376550330368, 94376550334463, +STORE, 94376554524672, 94376554528767, +STORE, 94376558718976, 94376558723071, +STORE, 94376562913280, 94376562917375, +STORE, 94376567107584, 94376567111679, +STORE, 94376571301888, 94376571305983, +STORE, 94376575496192, 94376575500287, +STORE, 94376579690496, 94376579694591, +STORE, 94376583884800, 94376583888895, +STORE, 94376588079104, 94376588083199, +STORE, 94376592273408, 94376592277503, +STORE, 94376596467712, 94376596471807, +STORE, 94376600662016, 94376600666111, +STORE, 94376604856320, 94376604860415, +STORE, 94376609050624, 94376609054719, +STORE, 94376613244928, 94376613249023, +STORE, 94376617439232, 94376617443327, +STORE, 94376621633536, 94376621637631, +STORE, 94376625827840, 94376625831935, +STORE, 94376630022144, 94376630026239, +STORE, 94376634216448, 94376634220543, +STORE, 94376638410752, 94376638414847, +STORE, 94376642605056, 94376642609151, +STORE, 94376646799360, 94376646803455, +STORE, 94376650993664, 94376650997759, +STORE, 94376655187968, 94376655192063, +STORE, 94376659382272, 94376659386367, +STORE, 94376663576576, 94376663580671, +STORE, 94376667770880, 94376667774975, +STORE, 94376671965184, 94376671969279, +STORE, 94376676159488, 94376676163583, +STORE, 94376680353792, 94376680357887, +STORE, 94376684548096, 94376684552191, +STORE, 94376688742400, 94376688746495, +STORE, 94376692936704, 94376692940799, +STORE, 94376697131008, 94376697135103, +STORE, 94376701325312, 94376701329407, +STORE, 94376705519616, 94376705523711, +STORE, 94376709713920, 94376709718015, +STORE, 94376713908224, 94376713912319, +STORE, 94376718102528, 94376718106623, +STORE, 94376722296832, 94376722300927, +STORE, 94376726491136, 94376726495231, +STORE, 94376730685440, 94376730689535, +STORE, 94376734879744, 94376734883839, +STORE, 94376739074048, 94376739078143, +STORE, 94376743268352, 94376743272447, +STORE, 94376747462656, 94376747466751, +STORE, 94376751656960, 94376751661055, +STORE, 94376755851264, 94376755855359, +STORE, 94376760045568, 94376760049663, +STORE, 94376764239872, 94376764243967, +STORE, 94376768434176, 94376768438271, +STORE, 94376772628480, 94376772632575, +STORE, 94376776822784, 94376776826879, +STORE, 94376781017088, 94376781021183, +STORE, 94376785211392, 94376785215487, +STORE, 94376789405696, 94376789409791, +STORE, 94376793600000, 94376793604095, +STORE, 94376797794304, 94376797798399, +STORE, 94376801988608, 94376801992703, +STORE, 94376806182912, 94376806187007, +STORE, 94376810377216, 94376810381311, +STORE, 94376814571520, 94376814575615, +STORE, 94376818765824, 94376818769919, +STORE, 94376822960128, 94376822964223, +STORE, 94376827154432, 94376827158527, +STORE, 94376831348736, 94376831352831, +STORE, 94376835543040, 94376835547135, +STORE, 94376839737344, 94376839741439, +STORE, 94376843931648, 94376843935743, +STORE, 94376848125952, 94376848130047, +STORE, 94376852320256, 94376852324351, +STORE, 94376856514560, 94376856518655, +STORE, 94376860708864, 94376860712959, +STORE, 94376864903168, 94376864907263, +STORE, 94376869097472, 94376869101567, +STORE, 94376873291776, 94376873295871, +STORE, 94376877486080, 94376877490175, +STORE, 94376881680384, 94376881684479, +STORE, 94376885874688, 94376885878783, +STORE, 94376890068992, 94376890073087, +STORE, 94376894263296, 94376894267391, +STORE, 94376898457600, 94376898461695, +STORE, 94376902651904, 94376902655999, +STORE, 94376906846208, 94376906850303, +STORE, 94376911040512, 94376911044607, +STORE, 94376915234816, 94376915238911, +STORE, 94376919429120, 94376919433215, +STORE, 94376923623424, 94376923627519, +STORE, 94376927817728, 94376927821823, +STORE, 94376932012032, 94376932016127, +STORE, 94376936206336, 94376936210431, +STORE, 94376940400640, 94376940404735, +STORE, 94376944594944, 94376944599039, +STORE, 94376948789248, 94376948793343, +STORE, 94376952983552, 94376952987647, +STORE, 94376957177856, 94376957181951, +STORE, 94376961372160, 94376961376255, +STORE, 94376965566464, 94376965570559, +STORE, 94376969760768, 94376969764863, +STORE, 94376973955072, 94376973959167, +STORE, 94376978149376, 94376978153471, +STORE, 94376982343680, 94376982347775, +STORE, 94376986537984, 94376986542079, +STORE, 94376990732288, 94376990736383, +STORE, 94376994926592, 94376994930687, +STORE, 94376999120896, 94376999124991, +STORE, 94377003315200, 94377003319295, +STORE, 94377007509504, 94377007513599, +STORE, 94377011703808, 94377011707903, +STORE, 94377015898112, 94377015902207, +STORE, 94377020092416, 94377020096511, +STORE, 94377024286720, 94377024290815, +STORE, 94377028481024, 94377028485119, +STORE, 94377032675328, 94377032679423, +STORE, 94377036869632, 94377036873727, +STORE, 94377041063936, 94377041068031, +STORE, 94377045258240, 94377045262335, +STORE, 94377049452544, 94377049456639, +STORE, 94377053646848, 94377053650943, +STORE, 94377057841152, 94377057845247, +STORE, 94377062035456, 94377062039551, +STORE, 94377066229760, 94377066233855, +STORE, 94377070424064, 94377070428159, +STORE, 94377074618368, 94377074622463, +STORE, 94377078812672, 94377078816767, +STORE, 94377083006976, 94377083011071, +STORE, 94377087201280, 94377087205375, +STORE, 94377091395584, 94377091399679, +STORE, 94377095589888, 94377095593983, +STORE, 94377099784192, 94377099788287, +STORE, 94377103978496, 94377103982591, +STORE, 94377108172800, 94377108176895, +STORE, 94377112367104, 94377112371199, +STORE, 94377116561408, 94377116565503, +STORE, 94377120755712, 94377120759807, +STORE, 94377124950016, 94377124954111, +STORE, 94377129144320, 94377129148415, +STORE, 94377133338624, 94377133342719, +STORE, 94377137532928, 94377137537023, +STORE, 94377141727232, 94377141731327, +STORE, 94377145921536, 94377145925631, +STORE, 94377150115840, 94377150119935, +STORE, 94377154310144, 94377154314239, +STORE, 94377158504448, 94377158508543, +STORE, 94377162698752, 94377162702847, +STORE, 94377166893056, 94377166897151, +STORE, 94377171087360, 94377171091455, +STORE, 94377175281664, 94377175285759, +STORE, 94377179475968, 94377179480063, +STORE, 94377183670272, 94377183674367, +STORE, 94377187864576, 94377187868671, +STORE, 94377192058880, 94377192062975, +STORE, 94377196253184, 94377196257279, +STORE, 94377200447488, 94377200451583, +STORE, 94377204641792, 94377204645887, +SNULL, 94376135094271, 94376135098367, +STORE, 94376135090176, 94376135094271, +STORE, 94376135094272, 94376135098367, +SNULL, 94376135094272, 94377208836095, + }; + unsigned long set42[] = { +STORE, 314572800, 1388314623, +STORE, 1462157312, 1462169599, +STORE, 1462169600, 1462185983, +STORE, 1462185984, 1462190079, +STORE, 1462190080, 1462194175, +STORE, 1462194176, 1462198271, +STORE, 1879986176, 1881800703, +STORE, 1881800704, 1882034175, +STORE, 1882034176, 1882193919, +STORE, 1882193920, 1882406911, +STORE, 1882406912, 1882451967, +STORE, 1882451968, 1882996735, +STORE, 1882996736, 1885892607, +STORE, 1885892608, 1885896703, +STORE, 1885896704, 1885904895, +STORE, 1885904896, 1885908991, +STORE, 1885908992, 1885913087, +STORE, 1885913088, 1885966335, +STORE, 1885966336, 1886232575, +STORE, 1886232576, 1886236671, +STORE, 1886236672, 1886240767, +STORE, 1886240768, 1886244863, +STORE, 1886244864, 1886248959, +STORE, 1886248960, 1886294015, +STORE, 1886294016, 1886494719, +STORE, 1886494720, 1886498815, +STORE, 1886498816, 1886502911, +STORE, 1886502912, 1886507007, +STORE, 1886507008, 1886511103, +STORE, 1886511104, 1886556159, +STORE, 1886556160, 1886629887, +STORE, 1886629888, 1886633983, +STORE, 1886633984, 1886638079, +STORE, 1886638080, 1886642175, +STORE, 1886642176, 1886646271, +STORE, 1886646272, 1886666751, +STORE, 1886666752, 1886670847, +STORE, 1886670848, 1886674943, +STORE, 1886674944, 1886679039, +STORE, 1886679040, 1895419903, +STORE, 1895419904, 1895550975, +STORE, 1895550976, 1896148991, +STORE, 1896148992, 1897189375, +STORE, 1897189376, 1897701375, +STORE, 1897701376, 1897803775, +STORE, 1897803776, 1897816063, +STORE, 1897816064, 1899913215, +STORE, 1899913216, 1909379071, +STORE, 1909379072, 1909387263, +STORE, 1909387264, 1909391359, +STORE, 1909391360, 1909432319, +STORE, 1909432320, 1909436415, +STORE, 1909436416, 1909440511, +STORE, 1909440512, 1909460991, +STORE, 1909460992, 1909547007, +STORE, 1909547008, 1909551103, +STORE, 1909551104, 1909555199, +STORE, 1909555200, 1909559295, +STORE, 1909559296, 1909563391, +STORE, 1909563392, 1909739519, +STORE, 1909739520, 1910566911, +STORE, 1910566912, 1910571007, +STORE, 1910571008, 1910575103, +STORE, 1910575104, 1910579199, +STORE, 1910579200, 1910583295, +STORE, 1910583296, 1910587391, +STORE, 1910587392, 1910620159, +STORE, 1910620160, 1910624255, +STORE, 1910624256, 1910628351, +STORE, 1910628352, 1910632447, +STORE, 1910632448, 1910652927, +STORE, 1910652928, 1910657023, +STORE, 1910657024, 1910661119, +STORE, 1910661120, 1910665215, +STORE, 1910665216, 1910669311, +STORE, 1910669312, 1910677503, +STORE, 1910677504, 1910681599, +STORE, 1910681600, 1910685695, +STORE, 1910685696, 1910689791, +STORE, 1910689792, 1910697983, +STORE, 1910697984, 1910702079, +STORE, 1910702080, 1910706175, +STORE, 1910706176, 1910710271, +STORE, 1910710272, 1914093567, +STORE, 1914093568, 1914097663, +STORE, 1914097664, 1969434623, +STORE, 1969434624, 1977819135, +STORE, 3290435584, 3426750463, +STORE, 3426750464, 3426754559, +STORE, 3426754560, 3426762751, +STORE, 3426762752, 3426766847, +STORE, 3426766848, 3426770943, +STORE, 3427037184, 3427061759, +STORE, 3427061760, 3427135487, +STORE, 3427135488, 3427143679, +STORE, 3427143680, 3427147775, +STORE, 3427147776, 3427209215, +STORE, 3427319808, 3432116223, +STORE, 3432116224, 3450130431, +STORE, 3450130432, 3451027455, +STORE, 3451027456, 3451031551, +STORE, 3451031552, 3451461631, +STORE, 3451736064, 3456688127, +STORE, 3456688128, 3475222527, +STORE, 3475222528, 3476119551, +STORE, 3476119552, 3476127743, +STORE, 3476127744, 3476553727, +STORE, 3476631552, 3477315583, +STORE, 3477315584, 3479949311, +STORE, 3479949312, 3480002559, +STORE, 3480002560, 3480006655, +STORE, 3480006656, 3480432639, +STORE, 3480539136, 3480543231, +STORE, 3480543232, 3480547327, +STORE, 3480547328, 3480555519, +STORE, 3480854528, 3480903679, +STORE, 3480903680, 3480969215, +STORE, 3480969216, 3480977407, +STORE, 3480977408, 3480981503, +STORE, 3481030656, 3481092095, +STORE, 3481092096, 3481235455, +STORE, 3481235456, 3481243647, +STORE, 3481243648, 3481247743, +STORE, 3481436160, 3481444351, +STORE, 3481444352, 3481456639, +STORE, 3481456640, 3481460735, +STORE, 3481460736, 3481464831, +STORE, 3481587712, 3481645055, +STORE, 3481645056, 3481772031, +STORE, 3481772032, 3481776127, +STORE, 3481776128, 3481780223, +STORE, 3481874432, 3481935871, +STORE, 3481935872, 3482030079, +STORE, 3482030080, 3482038271, +STORE, 3482038272, 3482042367, +STORE, 3482198016, 3482230783, +STORE, 3482230784, 3482271743, +STORE, 3482271744, 3482279935, +STORE, 3482279936, 3482284031, +STORE, 3482562560, 3482566655, +STORE, 3482566656, 3482570751, +STORE, 3482570752, 3482574847, +STORE, 3482636288, 3482689535, +STORE, 3482689536, 3482746879, +STORE, 3482746880, 3482755071, +STORE, 3482755072, 3482759167, +STORE, 3482972160, 3483062271, +STORE, 3483062272, 3483242495, +STORE, 3483242496, 3483246591, +STORE, 3483246592, 3483250687, +STORE, 3483398144, 3483688959, +STORE, 3483688960, 3484114943, +STORE, 3484114944, 3484131327, +STORE, 3484131328, 3484135423, +STORE, 3484135424, 3484143615, +STORE, 3484184576, 3484475391, +STORE, 3484475392, 3485028351, +STORE, 3485028352, 3485057023, +STORE, 3485057024, 3485061119, +STORE, 3485360128, 3485364223, +STORE, 3485364224, 3485368319, +STORE, 3485368320, 3485372415, +STORE, 3485589504, 3485593599, +STORE, 3485593600, 3485597695, +STORE, 3485597696, 3485601791, +STORE, 3485913088, 3485937663, +STORE, 3485937664, 3485974527, +STORE, 3485974528, 3485982719, +STORE, 3485982720, 3485986815, +STORE, 3486052352, 3486056447, +STORE, 3486056448, 3486064639, +STORE, 3486064640, 3486068735, +STORE, 3486068736, 3486072831, +STORE, 3486294016, 3486302207, +STORE, 3486302208, 3486306303, +STORE, 3486306304, 3486310399, +STORE, 3486310400, 3486314495, +STORE, 3486670848, 3486679039, +STORE, 3486679040, 3486683135, +STORE, 3486683136, 3486687231, +STORE, 3486687232, 3486691327, +STORE, 3486863360, 3486871551, +STORE, 3486871552, 3486875647, +STORE, 3486875648, 3486879743, +STORE, 3486879744, 3486883839, +STORE, 3487584256, 3522543615, +STORE, 3522543616, 3523321855, +STORE, 3523321856, 3523342335, +STORE, 3523342336, 3523387391, +STORE, 3523387392, 3523391487, +STORE, 3523391488, 3523395583, +STORE, 3523477504, 3523686399, +STORE, 3523686400, 3523981311, +STORE, 3523981312, 3523997695, +STORE, 3523997696, 3524001791, +STORE, 3524177920, 3525013503, +STORE, 3525013504, 3526582271, +STORE, 3526582272, 3526606847, +STORE, 3526606848, 3526610943, +STORE, 3526610944, 3526615039, +STORE, 3526672384, 3526746111, +STORE, 3526746112, 3526860799, +STORE, 3526860800, 3526868991, +STORE, 3526868992, 3526873087, +STORE, 3527000064, 3527475199, +STORE, 3527475200, 3527479295, +STORE, 3527479296, 3527573503, +STORE, 3527573504, 3527581695, +STORE, 3527581696, 3527585791, +STORE, 3527585792, 3527606271, +STORE, 3527909376, 3527913471, +STORE, 3527913472, 3527917567, +STORE, 3527917568, 3527921663, +STORE, 3527950336, 3528011775, +STORE, 3528011776, 3528093695, +STORE, 3528093696, 3528101887, +STORE, 3528101888, 3528105983, +STORE, 3528228864, 3528241151, +STORE, 3528241152, 3528261631, +STORE, 3528261632, 3528265727, +STORE, 3528273920, 3528593407, +STORE, 3528593408, 3528609791, +STORE, 3528609792, 3528638463, +STORE, 3528638464, 3528642559, +STORE, 3528642560, 3528646655, +STORE, 3528880128, 3528912895, +STORE, 3528912896, 3528962047, +STORE, 3528962048, 3528966143, +STORE, 3528966144, 3528970239, +STORE, 3528982528, 3530293247, +STORE, 3530366976, 3530825727, +STORE, 3530825728, 3531317247, +STORE, 3531317248, 3541041151, +STORE, 3541041152, 3541303295, +STORE, 3541430272, 3566206975, +STORE, 3566206976, 3566993407, +STORE, 3567239168, 3587571711, +STORE, 3587571712, 3588284415, +STORE, 3588284416, 3588661247, +STORE, 3588661248, 3589066751, +STORE, 3589066752, 3589574655, +STORE, 3589574656, 3590078463, +STORE, 3590078464, 3590373375, +STORE, 3590373376, 3590668287, +STORE, 3590668288, 3590963199, +STORE, 3590963200, 3591294975, +STORE, 3591294976, 3591602175, +STORE, 3591602176, 3591933951, +STORE, 3591933952, 3592241151, +STORE, 3592241152, 3592572927, +STORE, 3592572928, 3592876031, +STORE, 3592876032, 3593211903, +STORE, 3593211904, 3593547775, +STORE, 3593547776, 3593650175, +STORE, 3593650176, 3593928703, +STORE, 3593928704, 3593936895, +STORE, 3593936896, 3593940991, +STORE, 3594006528, 3594301439, +STORE, 3594301440, 3594739711, +STORE, 3594739712, 3594756095, +STORE, 3594756096, 3594760191, +STORE, 3594760192, 3594768383, +STORE, 3594952704, 3595051007, +STORE, 3595051008, 3595223039, +STORE, 3595223040, 3595227135, +STORE, 3595227136, 3595235327, +STORE, 3595431936, 3595775999, +STORE, 3595776000, 3596701695, +STORE, 3596701696, 3596742655, +STORE, 3596742656, 3596746751, +STORE, 3596746752, 3596750847, +STORE, 3596767232, 3597070335, +STORE, 3597070336, 3597402111, +STORE, 3597402112, 3598188543, +STORE, 3598262272, 3623428095, +STORE, 3623428096, 3623432191, +STORE, 3623432192, 3623436287, +STORE, 3623436288, 3623440383, +STORE, 3623616512, 3623878655, +STORE, 3624169472, 3624300543, +STORE, 3627524096, 3628523519, +STORE, 3628523520, 3629522943, +STORE, 3696631808, 3730186239, +STORE, 3730186240, 3763740671, +STORE, 3763740672, 3764027391, +STORE, 3764027392, 3765133311, +STORE, 3765133312, 3765145599, +STORE, 3765145600, 3765149695, +STORE, 3765178368, 3766022143, +STORE, 3766022144, 3768791039, +STORE, 3768791040, 3768840191, +STORE, 3768840192, 3768844287, +STORE, 3768897536, 3768913919, +STORE, 3768913920, 3768934399, +STORE, 3768934400, 3768938495, +STORE, 3769016320, 3769147391, +STORE, 3769147392, 3769233407, +STORE, 3769233408, 3769356287, +STORE, 3769356288, 3769360383, +STORE, 3769360384, 3769368575, +STORE, 3769376768, 3794542591, +STORE, 3794542592, 3794599935, +STORE, 3794599936, 3794731007, +STORE, 3794731008, 3794735103, +STORE, 3794735104, 3794743295, +STORE, 3794849792, 3794980863, +STORE, 3794980864, 3794984959, +STORE, 3794984960, 3794989055, +STORE, 3794989056, 3794993151, +STORE, 3794993152, 3794997247, +STORE, 3795103744, 3795128319, +STORE, 3795128320, 3795165183, +STORE, 3795165184, 3795169279, +STORE, 3795169280, 3795173375, +STORE, 3795210240, 3795357695, +STORE, 3795357696, 3795365887, +STORE, 3795365888, 3795374079, +STORE, 3795374080, 3795378175, +STORE, 3795378176, 3795382271, +STORE, 3795406848, 3795738623, +STORE, 3795738624, 3795742719, +STORE, 3795742720, 3795755007, +STORE, 3795755008, 3795759103, +STORE, 3795763200, 3795894271, +STORE, 3795894272, 3796041727, +STORE, 3796041728, 3796054015, +STORE, 3796054016, 3796066303, +STORE, 3796066304, 3796070399, +STORE, 3796176896, 3796205567, +STORE, 3796205568, 3796250623, +STORE, 3796250624, 3796254719, +STORE, 3796254720, 3796258815, +STORE, 3796262912, 3796393983, +STORE, 3796393984, 3796516863, +STORE, 3796516864, 3796873215, +STORE, 3796873216, 3796885503, +STORE, 3796885504, 3796889599, +STORE, 3796963328, 3796967423, +STORE, 3796967424, 3796975615, +STORE, 3796975616, 3796979711, +STORE, 3797000192, 3797307391, +STORE, 3797307392, 3797311487, +STORE, 3797311488, 3797315583, +STORE, 3797315584, 3797323775, +STORE, 3797327872, 3797450751, +STORE, 3797450752, 3797458943, +STORE, 3797458944, 3797471231, +STORE, 3797471232, 3797475327, +STORE, 3797577728, 3797700607, +STORE, 3797700608, 3797721087, +STORE, 3797721088, 3797733375, +STORE, 3797733376, 3797741567, +STORE, 3797741568, 3797864447, +STORE, 3797864448, 3797995519, +STORE, 3797995520, 3798048767, +STORE, 3798048768, 3798179839, +STORE, 3798179840, 3798188031, +STORE, 3798188032, 3798192127, +STORE, 3798290432, 3798302719, +STORE, 3798302720, 3798323199, +STORE, 3798323200, 3798327295, +STORE, 3798327296, 3798331391, +STORE, 3798429696, 3798433791, +STORE, 3798433792, 3798552575, +STORE, 3798552576, 3798556671, +STORE, 3798556672, 3798568959, +STORE, 3798568960, 3798573055, +STORE, 3798573056, 3798581247, +STORE, 3798618112, 3798749183, +STORE, 3798749184, 3798855679, +STORE, 3798855680, 3798966271, +STORE, 3798966272, 3798982655, +STORE, 3798982656, 3798986751, +STORE, 3799101440, 3799171071, +STORE, 3799171072, 3799240703, +STORE, 3799240704, 3799248895, +STORE, 3799248896, 3799252991, +STORE, 3799326720, 3799650303, +STORE, 3799650304, 3800629247, +STORE, 3800629248, 3800641535, +STORE, 3800641536, 3800645631, +STORE, 3800645632, 3800649727, +STORE, 3800649728, 3800903679, +STORE, 3800903680, 3800936447, +STORE, 3800936448, 3800969215, +STORE, 3800969216, 3800981503, +STORE, 3800981504, 3800985599, +STORE, 3801001984, 3801133055, +STORE, 3801133056, 3801202687, +STORE, 3801202688, 3801591807, +STORE, 3801591808, 3801599999, +STORE, 3801600000, 3801604095, +STORE, 3801604096, 3801608191, +STORE, 3801608192, 3801739263, +STORE, 3801739264, 3801755647, +STORE, 3801755648, 3801796607, +STORE, 3801796608, 3801804799, +STORE, 3801804800, 3801808895, +STORE, 3801878528, 3801944063, +STORE, 3801944064, 3802116095, +STORE, 3802116096, 3802124287, +STORE, 3802124288, 3802128383, +STORE, 3802136576, 3803447295, +STORE, 3803492352, 3803553791, +STORE, 3803553792, 3804233727, +STORE, 3804233728, 3806068735, +STORE, 3806121984, 3806253055, +STORE, 3806253056, 3806674943, +STORE, 3806674944, 3807117311, +STORE, 3807117312, 3807379455, +STORE, 3807379456, 3807432703, +STORE, 3807432704, 3807563775, +STORE, 3807563776, 3809202175, +STORE, 3809202176, 3810250751, +STORE, 3810250752, 3827027967, +STORE, 3827027968, 3829125119, +STORE, 3829125120, 3837513727, +STORE, 3837513728, 3839610879, +STORE, 3839610880, 3847999487, +STORE, 3847999488, 3856392191, +STORE, 3856392192, 3864784895, +STORE, 3864784896, 3868983295, +STORE, 3868983296, 3885760511, +STORE, 3885760512, 3886809087, +STORE, 3886809088, 3887857663, +STORE, 3887857664, 3888119807, +STORE, 3888144384, 3888148479, +STORE, 3888148480, 3888218111, +STORE, 3888218112, 3888222207, +STORE, 3888222208, 3888353279, +STORE, 3888353280, 3889172479, +STORE, 3889172480, 3892314111, +STORE, 3892314112, 3892576255, +STORE, 3892588544, 3892637695, +STORE, 3892637696, 3892686847, +STORE, 3892686848, 3892744191, +STORE, 3892748288, 3892785151, +STORE, 3892785152, 3895459839, +STORE, 3895459840, 3895721983, +STORE, 3895738368, 3895885823, +STORE, 3895885824, 3897081855, +STORE, 3897081856, 3906482175, +STORE, 3906482176, 3916144639, +STORE, 3916144640, 3925766143, +STORE, 3925766144, 3926974463, +STORE, 3926974464, 3928367103, +STORE, 3928367104, 3928911871, +STORE, 3928911872, 3933995007, +STORE, 3933995008, 3935830015, +STORE, 3935830016, 3935846399, +STORE, 3935879168, 3936010239, +STORE, 3936010240, 3936026623, +STORE, 3936026624, 3936034815, +STORE, 3936034816, 3936051199, +STORE, 3936051200, 3936055295, +STORE, 3936071680, 3936137215, +STORE, 3936137216, 3936202751, +STORE, 3936202752, 3936219135, +STORE, 3936235520, 3936251903, +STORE, 3936268288, 3936276479, +STORE, 3936276480, 3936284671, +STORE, 3936284672, 3936288767, +STORE, 3936288768, 3936292863, +STORE, 3936296960, 3936354303, +STORE, 3936354304, 3936616447, +STORE, 3936628736, 3936669695, +STORE, 3936669696, 3936747519, +STORE, 3936747520, 3936870399, +STORE, 3936870400, 3936874495, +STORE, 3936874496, 3936878591, +STORE, 3936882688, 3936903167, +STORE, 3936911360, 3936948223, +STORE, 3936948224, 3936964607, +STORE, 3936964608, 3937103871, +STORE, 3937103872, 3937107967, +STORE, 3937132544, 3937161215, +STORE, 3937189888, 3937255423, +STORE, 3937255424, 3938512895, +STORE, 3938512896, 3945435135, +STORE, 3945435136, 3945476095, +STORE, 3945476096, 3945484287, +STORE, 3945484288, 3945496575, +STORE, 3945500672, 3945541631, +STORE, 3945558016, 3945566207, +STORE, 3945566208, 3945594879, +STORE, 3945594880, 3945598975, +STORE, 3945598976, 3945603071, +STORE, 3945611264, 3945742335, +STORE, 3945742336, 3945844735, +STORE, 3945844736, 3945848831, +STORE, 3945848832, 3945861119, +STORE, 3945861120, 3945865215, +STORE, 3945869312, 3945897983, +STORE, 3945897984, 3946303487, +STORE, 3946303488, 3946397695, +STORE, 3946397696, 3946569727, +STORE, 3946569728, 3946573823, +STORE, 3946573824, 3946594303, +STORE, 3946594304, 3946663935, +STORE, 3946663936, 3946708991, +STORE, 3946708992, 3946823679, +STORE, 3946823680, 3946827775, +STORE, 3946827776, 3946831871, +STORE, 3946831872, 3946860543, +STORE, 3946893312, 3946897407, +STORE, 3946897408, 3946905599, +STORE, 3946905600, 3946909695, +STORE, 3946909696, 3946913791, +STORE, 3946913792, 3946930175, +STORE, 3946930176, 3946967039, +STORE, 3946967040, 3947102207, +STORE, 3947102208, 3948412927, +STORE, 3948441600, 3948556287, +STORE, 3948556288, 3948576767, +STORE, 3948576768, 3948597247, +STORE, 3948597248, 3948605439, +STORE, 3948605440, 3948609535, +STORE, 3948609536, 3948654591, +STORE, 3948654592, 3948781567, +STORE, 3948781568, 3948822527, +STORE, 3948822528, 3948904447, +STORE, 3948904448, 3948908543, +STORE, 3948908544, 3948912639, +STORE, 3948945408, 3949043711, +STORE, 3949043712, 3949174783, +STORE, 3949174784, 3949191167, +STORE, 3949191168, 3949195263, +STORE, 3949207552, 3949252607, +STORE, 3949252608, 3949256703, +STORE, 3949256704, 3949363199, +STORE, 3949363200, 3949367295, +STORE, 3949367296, 3949379583, +STORE, 3949379584, 3949383679, +STORE, 3949383680, 3949400063, +STORE, 3949400064, 3949404159, +STORE, 3949416448, 3949481983, +STORE, 3949481984, 3949486079, +STORE, 3949486080, 3949592575, +STORE, 3949592576, 3949596671, +STORE, 3949596672, 3949621247, +STORE, 3949621248, 3949662207, +STORE, 3949662208, 3949666303, +STORE, 3949694976, 3949727743, +STORE, 3949727744, 3949731839, +STORE, 3949731840, 3949838335, +STORE, 3949838336, 3949842431, +STORE, 3949842432, 3949846527, +STORE, 3949846528, 3949854719, +STORE, 3949854720, 3949858815, +STORE, 3949858816, 3949862911, +STORE, 3949867008, 3949891583, +STORE, 3949891584, 3949928447, +STORE, 3949928448, 3949993983, +STORE, 3949993984, 3950043135, +STORE, 3950043136, 3950059519, +STORE, 3950059520, 3950096383, +STORE, 3950096384, 3950100479, +STORE, 3950100480, 3950104575, +STORE, 3950104576, 3950157823, +STORE, 3950157824, 3950292991, +STORE, 3950292992, 3950346239, +STORE, 3950346240, 3950477311, +STORE, 3950477312, 3950485503, +STORE, 3950485504, 3950489599, +STORE, 3950493696, 3950510079, +STORE, 3950510080, 3950661631, +STORE, 3950661632, 3951005695, +STORE, 3951005696, 3951026175, +STORE, 3951026176, 3951030271, +STORE, 3951030272, 3951054847, +STORE, 3951054848, 3951116287, +STORE, 3951116288, 3951144959, +STORE, 3951144960, 3951149055, +STORE, 3951149056, 3951194111, +STORE, 3951194112, 3951202303, +STORE, 3951202304, 3951206399, +STORE, 3951210496, 3951226879, +STORE, 3951226880, 3951329279, +STORE, 3951329280, 3951366143, +STORE, 3951366144, 3951411199, +STORE, 3951411200, 3951415295, +STORE, 3951415296, 3951419391, +STORE, 3951419392, 3951452159, +STORE, 3951452160, 3951566847, +STORE, 3951566848, 3951812607, +STORE, 3951812608, 3952173055, +STORE, 3952173056, 3952214015, +STORE, 3952214016, 3952218111, +STORE, 3952222208, 3952250879, +STORE, 3952250880, 3952369663, +STORE, 3952369664, 3952488447, +STORE, 3952488448, 3952627711, +STORE, 3952627712, 3952635903, +STORE, 3952635904, 3952639999, +STORE, 3952652288, 3952668671, +STORE, 3952668672, 3953000447, +STORE, 3953000448, 3953004543, +STORE, 3953004544, 3953008639, +STORE, 3953008640, 3953012735, +STORE, 3953012736, 3953037311, +STORE, 3953037312, 3953151999, +STORE, 3953152000, 3953291263, +STORE, 3953291264, 3953324031, +STORE, 3953324032, 3953364991, +STORE, 3953364992, 3953373183, +STORE, 3953373184, 3953377279, +STORE, 3953381376, 3953410047, +STORE, 3953410048, 3953491967, +STORE, 3953491968, 3953643519, +STORE, 3953643520, 3953651711, +STORE, 3953651712, 3953655807, +STORE, 3953659904, 3953766399, +STORE, 3953766400, 3953774591, +STORE, 3953774592, 3953786879, +STORE, 3953786880, 3953790975, +STORE, 3953790976, 3953823743, +STORE, 3953823744, 3953963007, +STORE, 3953963008, 3954024447, +STORE, 3954024448, 3954118655, +STORE, 3954118656, 3954122751, +STORE, 3954122752, 3954126847, +STORE, 3954130944, 3954184191, +STORE, 3954184192, 3954294783, +STORE, 3954294784, 3954323455, +STORE, 3954323456, 3954393087, +STORE, 3954393088, 3954397183, +STORE, 3954397184, 3954401279, +STORE, 3954401280, 3954405375, +STORE, 3954409472, 3954528255, +STORE, 3954528256, 3954737151, +STORE, 3954737152, 3955052543, +STORE, 3955052544, 3955060735, +STORE, 3955060736, 3955064831, +STORE, 3955068928, 3955105791, +STORE, 3955105792, 3955167231, +STORE, 3955167232, 3955277823, +STORE, 3955277824, 3955310591, +STORE, 3955310592, 3955351551, +STORE, 3955351552, 3955359743, +STORE, 3955359744, 3955363839, +STORE, 3955363840, 3955392511, +STORE, 3955392512, 3955453951, +STORE, 3955453952, 3955601407, +STORE, 3955601408, 3955777535, +STORE, 3955777536, 3955982335, +STORE, 3955982336, 3956011007, +STORE, 3956011008, 3956015103, +STORE, 3956023296, 3956039679, +STORE, 3956039680, 3956125695, +STORE, 3956125696, 3956129791, +STORE, 3956129792, 3956133887, +STORE, 3956133888, 3956137983, +STORE, 3956142080, 3956449279, +STORE, 3956449280, 3956543487, +STORE, 3956543488, 3956719615, +STORE, 3956719616, 3956731903, +STORE, 3956731904, 3956735999, +STORE, 3956744192, 3956793343, +STORE, 3956793344, 3956887551, +STORE, 3956887552, 3956953087, +STORE, 3956953088, 3957035007, +STORE, 3957035008, 3957039103, +STORE, 3957039104, 3957047295, +STORE, 3957047296, 3957071871, +STORE, 3957071872, 3957231615, +STORE, 3957231616, 3957563391, +STORE, 3957563392, 3957579775, +STORE, 3957579776, 3957583871, +STORE, 3957592064, 3957608447, +STORE, 3957608448, 3957878783, +STORE, 3957878784, 3958591487, +STORE, 3958591488, 3958599679, +STORE, 3958599680, 3958607871, +STORE, 3958607872, 3958620159, +STORE, 3958620160, 3958624255, +STORE, 3958624256, 3963199487, +STORE, 3963199488, 3963285503, +STORE, 3963285504, 3963371519, +STORE, 3963371520, 3963428863, +STORE, 3963428864, 3963555839, +STORE, 3963555840, 3963559935, +STORE, 3963559936, 3963564031, +STORE, 3963568128, 3963596799, +STORE, 3963596800, 3963682815, +STORE, 3963682816, 3963695103, +STORE, 3963695104, 3963711487, +STORE, 3963711488, 3963715583, +STORE, 3963719680, 3963752447, +STORE, 3963752448, 3963846655, +STORE, 3963846656, 3963932671, +STORE, 3963932672, 3964444671, +STORE, 3964444672, 3964448767, +STORE, 3964448768, 3965808639, +STORE, 3965808640, 3965845503, +STORE, 3965845504, 3965849599, +STORE, 3965853696, 3965935615, +STORE, 3965935616, 3966017535, +STORE, 3966017536, 3966103551, +STORE, 3966103552, 3966685183, +STORE, 3966685184, 3967705087, +STORE, 3967705088, 3967758335, +STORE, 3967758336, 3967762431, +STORE, 3967762432, 3967770623, +STORE, 3967770624, 3967799295, +STORE, 3967799296, 3967848447, +STORE, 3967848448, 3967868927, +STORE, 3967868928, 3967901695, +STORE, 3967901696, 3967905791, +STORE, 3967905792, 3967909887, +STORE, 3967909888, 3967995903, +STORE, 3967995904, 3968077823, +STORE, 3968077824, 3968159743, +STORE, 3968159744, 3968167935, +STORE, 3968167936, 3968172031, +STORE, 3968172032, 3968192511, +STORE, 3968192512, 3968196607, +STORE, 3968196608, 3968200703, +STORE, 3968208896, 3968516095, +STORE, 3968516096, 3968528383, +STORE, 3968528384, 3968552959, +STORE, 3968552960, 3968557055, +STORE, 3968561152, 3968593919, +STORE, 3968593920, 3968626687, +STORE, 3968626688, 3971153919, +STORE, 3971153920, 3973754879, +STORE, 3973754880, 3973804031, +STORE, 3973804032, 3973820415, +STORE, 3973820416, 3973832703, +STORE, 3973840896, 3973873663, +STORE, 3973873664, 3973967871, +STORE, 3973967872, 3973976063, +STORE, 3973976064, 3973984255, +STORE, 3973984256, 3973988351, +STORE, 3973988352, 3973992447, +STORE, 3973996544, 3974008831, +STORE, 3974008832, 3974045695, +STORE, 3974045696, 3974139903, +STORE, 3974139904, 3974254591, +STORE, 3974254592, 3974275071, +STORE, 3974275072, 3974291455, +STORE, 3974291456, 3974295551, +STORE, 3974295552, 3974373375, +STORE, 3974373376, 3974524927, +STORE, 3974524928, 3974529023, +STORE, 3974529024, 3974537215, +STORE, 3974537216, 3974541311, +STORE, 3974541312, 3974545407, +STORE, 3974545408, 3974627327, +STORE, 3974627328, 3974680575, +STORE, 3974680576, 3974811647, +STORE, 3974811648, 3974819839, +STORE, 3974819840, 3974823935, +STORE, 3974832128, 3974918143, +STORE, 3974918144, 3974963199, +STORE, 3974963200, 3975077887, +STORE, 3975077888, 3975090175, +STORE, 3975090176, 3975094271, +STORE, 3975094272, 3975102463, +STORE, 3975102464, 3975114751, +STORE, 3975114752, 3975266303, +STORE, 3975266304, 3975274495, +STORE, 3975274496, 3975286783, +STORE, 3975286784, 3975290879, +STORE, 3975290880, 3975299071, +STORE, 3975299072, 3975315455, +STORE, 3975315456, 3975430143, +STORE, 3975430144, 3975536639, +STORE, 3975536640, 3975651327, +STORE, 3975651328, 3975655423, +STORE, 3975655424, 3975659519, +STORE, 3975659520, 3975770111, +STORE, 3975770112, 3975778303, +STORE, 3975778304, 3975790591, +STORE, 3975790592, 3975794687, +STORE, 3975794688, 3975798783, +STORE, 3975798784, 3975831551, +STORE, 3975831552, 3975872511, +STORE, 3975872512, 3975987199, +STORE, 3975987200, 3976134655, +STORE, 3976134656, 3977175039, +STORE, 3977175040, 3977183231, +STORE, 3977183232, 3977191423, +STORE, 3977191424, 3977195519, +STORE, 3977199616, 3977248767, +STORE, 3977248768, 3977539583, +STORE, 3977539584, 3977965567, +STORE, 3977965568, 3977981951, +STORE, 3977981952, 3977986047, +STORE, 3977986048, 3977994239, +STORE, 3977994240, 3978002431, +STORE, 3978002432, 3978084351, +STORE, 3978084352, 3978125311, +STORE, 3978125312, 3978174463, +STORE, 3978174464, 3978178559, +STORE, 3978178560, 3978182655, +STORE, 3978182656, 3978207231, +STORE, 3978207232, 3978297343, +STORE, 3978297344, 3978301439, +STORE, 3978301440, 3978305535, +STORE, 3978305536, 3978309631, +STORE, 3978309632, 3978317823, +STORE, 3978317824, 3978625023, +STORE, 3978625024, 3978657791, +STORE, 3978657792, 3978727423, +STORE, 3978727424, 3978735615, +STORE, 3978735616, 3978739711, +STORE, 3978739712, 3978760191, +STORE, 3978760192, 3978842111, +STORE, 3978842112, 3978850303, +STORE, 3978850304, 3978858495, +STORE, 3978858496, 3978862591, +STORE, 3978862592, 3978895359, +STORE, 3978895360, 3979014143, +STORE, 3979014144, 3979132927, +STORE, 3979132928, 3979288575, +STORE, 3979288576, 3979481087, +STORE, 3979481088, 3979489279, +STORE, 3979489280, 3979493375, +STORE, 3979497472, 3979583487, +STORE, 3979583488, 3979673599, +STORE, 3979673600, 3979718655, +STORE, 3979718656, 3979829247, +STORE, 3979829248, 3979841535, +STORE, 3979841536, 3979882495, +STORE, 3979882496, 3979964415, +STORE, 3979964416, 3980013567, +STORE, 3980013568, 3980148735, +STORE, 3980148736, 3980152831, +STORE, 3980152832, 3980320767, +STORE, 3980320768, 3980337151, +STORE, 3980337152, 3980341247, +STORE, 3980345344, 3980365823, +STORE, 3980365824, 3980423167, +STORE, 3980423168, 3980460031, +STORE, 3980460032, 3980500991, +STORE, 3980500992, 3980509183, +STORE, 3980509184, 3980513279, +STORE, 3980513280, 3980546047, +STORE, 3980546048, 3980660735, +STORE, 3980660736, 3980951551, +STORE, 3980951552, 3981500415, +STORE, 3981500416, 3981529087, +STORE, 3981529088, 3981533183, +STORE, 3981537280, 3981549567, +STORE, 3981549568, 3981598719, +STORE, 3981598720, 3981717503, +STORE, 3981717504, 3982127103, +STORE, 3982127104, 3982675967, +STORE, 3982675968, 3982733311, +STORE, 3982733312, 3982737407, +STORE, 3982741504, 3982860287, +STORE, 3982860288, 3982905343, +STORE, 3982905344, 3982966783, +STORE, 3982966784, 3982974975, +STORE, 3982974976, 3982979071, +STORE, 3982979072, 3983032319, +STORE, 3983032320, 3983085567, +STORE, 3983085568, 3983208447, +STORE, 3983208448, 3983212543, +STORE, 3983212544, 3983220735, +STORE, 3983220736, 3983224831, +STORE, 3983224832, 3983237119, +STORE, 3983237120, 3983351807, +STORE, 3983351808, 3983376383, +STORE, 3983376384, 3983392767, +STORE, 3983392768, 3983396863, +STORE, 3983396864, 3983400959, +STORE, 3983400960, 3983417343, +STORE, 3983417344, 3983753215, +STORE, 3983753216, 3983757311, +STORE, 3983757312, 3983761407, +STORE, 3983761408, 3983765503, +STORE, 3983765504, 3983769599, +STORE, 3983769600, 3983880191, +STORE, 3983880192, 3983892479, +STORE, 3983892480, 3983900671, +STORE, 3983900672, 3983904767, +STORE, 3983904768, 3983908863, +STORE, 3983908864, 3983941631, +STORE, 3983941632, 3983990783, +STORE, 3983990784, 3984097279, +STORE, 3984097280, 3984105471, +STORE, 3984105472, 3984117759, +STORE, 3984117760, 3984121855, +STORE, 3984121856, 3984125951, +STORE, 3984125952, 3984134143, +STORE, 3984134144, 3984150527, +STORE, 3984150528, 3984416767, +STORE, 3984416768, 3984470015, +STORE, 3984470016, 3984564223, +STORE, 3984564224, 3984568319, +STORE, 3984572416, 3984629759, +STORE, 3984629760, 3984805887, +STORE, 3984805888, 3985096703, +STORE, 3985096704, 3985104895, +STORE, 3985104896, 3985108991, +STORE, 3985113088, 3986862079, +STORE, 3986862080, 3993640959, +STORE, 3993640960, 3993739263, +STORE, 3993739264, 3993743359, +STORE, 3993743360, 3993759743, +STORE, 3993759744, 3993780223, +STORE, 3993780224, 3993784319, +STORE, 3993784320, 3993792511, +STORE, 3993792512, 3993796607, +STORE, 3993796608, 3993800703, +STORE, 3993804800, 3994214399, +STORE, 3994214400, 3994218495, +STORE, 3994218496, 3994222591, +STORE, 3994222592, 3994226687, +STORE, 3994230784, 3994243071, +STORE, 3994243072, 3994255359, +STORE, 3994255360, 3994304511, +STORE, 3994304512, 3994386431, +STORE, 3994386432, 3994509311, +STORE, 3994509312, 3994521599, +STORE, 3994521600, 3994525695, +STORE, 3994529792, 3994542079, +STORE, 3994542080, 3994660863, +STORE, 3994660864, 3994705919, +STORE, 3994705920, 3994796031, +STORE, 3994796032, 3994800127, +STORE, 3994800128, 3994804223, +STORE, 3994804224, 3994812415, +STORE, 3994812416, 3994845183, +STORE, 3994845184, 3994898431, +STORE, 3994898432, 3994902527, +STORE, 3994902528, 3994906623, +STORE, 3994910720, 3994931199, +STORE, 3994931200, 3995181055, +STORE, 3995181056, 3995222015, +STORE, 3995222016, 3995275263, +STORE, 3995275264, 3995279359, +STORE, 3995279360, 3995283455, +STORE, 3995283456, 3995291647, +STORE, 3995291648, 3995324415, +STORE, 3995324416, 3995451391, +STORE, 3995451392, 3995697151, +STORE, 3995697152, 3996078079, +STORE, 3996078080, 3996086271, +STORE, 3996086272, 3996090367, +STORE, 3996094464, 3996119039, +STORE, 3996119040, 3996200959, +STORE, 3996200960, 3996229631, +STORE, 3996229632, 3996233727, +STORE, 3996233728, 3996282879, +STORE, 3996282880, 3996291071, +STORE, 3996291072, 3996295167, +STORE, 3996299264, 3996311551, +STORE, 3996311552, 3996430335, +STORE, 3996430336, 3996467199, +STORE, 3996467200, 3996504063, +STORE, 3996504064, 3996512255, +STORE, 3996512256, 3996516351, +STORE, 3996516352, 3996540927, +STORE, 3996540928, 3996671999, +STORE, 3996672000, 3996676095, +STORE, 3996676096, 3996684287, +STORE, 3996684288, 3996688383, +STORE, 3996688384, 3996692479, +STORE, 3996692480, 3996717055, +STORE, 3996717056, 3997048831, +STORE, 3997048832, 3997057023, +STORE, 3997057024, 3997073407, +STORE, 3997073408, 3997077503, +STORE, 3997077504, 3997081599, +STORE, 3997081600, 3997097983, +STORE, 3997097984, 3997179903, +STORE, 3997179904, 3997356031, +STORE, 3997356032, 3997650943, +STORE, 3997650944, 3997675519, +STORE, 3997675520, 3997679615, +STORE, 3997683712, 3997700095, +STORE, 3997700096, 3997745151, +STORE, 3997745152, 3997802495, +STORE, 3997802496, 3997810687, +STORE, 3997810688, 3997814783, +STORE, 3997814784, 3998064639, +STORE, 3998064640, 3998081023, +STORE, 3998081024, 3998085119, +STORE, 3998085120, 3998130175, +STORE, 3998130176, 3998134271, +STORE, 3998134272, 3998142463, +STORE, 3998142464, 3998179327, +STORE, 3998179328, 3998212095, +STORE, 3998212096, 3998326783, +STORE, 3998326784, 3998351359, +STORE, 3998351360, 3998392319, +STORE, 3998392320, 3998396415, +STORE, 3998396416, 3998400511, +STORE, 3998400512, 3998433279, +STORE, 3998433280, 3998466047, +STORE, 3998466048, 3998613503, +STORE, 3998613504, 3998666751, +STORE, 3998666752, 3998724095, +STORE, 3998724096, 3998732287, +STORE, 3998732288, 3998736383, +STORE, 3998736384, 3998760959, +STORE, 3998760960, 3998777343, +STORE, 3998777344, 3998822399, +STORE, 3998822400, 3998826495, +STORE, 3998826496, 3998830591, +STORE, 3998830592, 3998863359, +STORE, 3998863360, 3998900223, +STORE, 3998900224, 3999043583, +STORE, 3999043584, 3999121407, +STORE, 3999121408, 3999215615, +STORE, 3999215616, 3999223807, +STORE, 3999223808, 3999227903, +STORE, 3999227904, 3999236095, +STORE, 3999236096, 3999268863, +STORE, 3999268864, 3999301631, +STORE, 3999301632, 3999354879, +STORE, 3999354880, 3999428607, +STORE, 3999428608, 3999436799, +STORE, 3999436800, 3999440895, +STORE, 3999444992, 3999461375, +STORE, 3999461376, 3999584255, +STORE, 3999584256, 3999760383, +STORE, 3999760384, 4000219135, +STORE, 4000219136, 4000235519, +STORE, 4000235520, 4000251903, +STORE, 4000251904, 4000501759, +STORE, 4000501760, 4000505855, +STORE, 4000505856, 4000509951, +STORE, 4000509952, 4000518143, +STORE, 4000518144, 4000522239, +STORE, 4000522240, 4000587775, +STORE, 4000587776, 4000645119, +STORE, 4000645120, 4000813055, +STORE, 4000813056, 4000817151, +STORE, 4000821248, 4000837631, +STORE, 4000837632, 4000870399, +STORE, 4000870400, 4000874495, +STORE, 4000874496, 4000878591, +STORE, 4000878592, 4000882687, +STORE, 4000882688, 4000886783, +STORE, 4000886784, 4000890879, +STORE, 4000890880, 4000907263, +STORE, 4000907264, 4001214463, +STORE, 4001214464, 4001558527, +STORE, 4001558528, 4002484223, +STORE, 4002484224, 4002525183, +STORE, 4002525184, 4002529279, +STORE, 4002529280, 4002533375, +STORE, 4002533376, 4002537471, +STORE, 4002537472, 4002660351, +STORE, 4002660352, 4002779135, +STORE, 4002779136, 4002791423, +STORE, 4002791424, 4002799615, +STORE, 4002799616, 4002807807, +STORE, 4002807808, 4002811903, +STORE, 4002811904, 4002828287, +STORE, 4002828288, 4002910207, +STORE, 4002910208, 4003028991, +STORE, 4003028992, 4003037183, +STORE, 4003037184, 4003045375, +STORE, 4003045376, 4003049471, +STORE, 4003049472, 4003053567, +STORE, 4003053568, 4003057663, +STORE, 4003057664, 4003065855, +STORE, 4003065856, 4003135487, +STORE, 4003135488, 4003446783, +STORE, 4003446784, 4003450879, +STORE, 4003450880, 4003454975, +STORE, 4003454976, 4003459071, +STORE, 4003459072, 4003463167, +STORE, 4003463168, 4003495935, +STORE, 4003495936, 4003569663, +STORE, 4003569664, 4003573759, +STORE, 4003573760, 4003704831, +STORE, 4003704832, 4003708927, +STORE, 4003708928, 4003713023, +STORE, 4003713024, 4003737599, +STORE, 4003737600, 4003770367, +STORE, 4003770368, 4003876863, +STORE, 4003876864, 4003880959, +STORE, 4003880960, 4003885055, +STORE, 4003885056, 4003889151, +STORE, 4003889152, 4003893247, +STORE, 4003893248, 4003897343, +STORE, 4003897344, 4003962879, +STORE, 4003962880, 4004069375, +STORE, 4004069376, 4004093951, +STORE, 4004093952, 4004118527, +STORE, 4004118528, 4004122623, +STORE, 4004122624, 4004126719, +STORE, 4004126720, 4004155391, +STORE, 4004155392, 4004286463, +STORE, 4004286464, 4004384767, +STORE, 4004384768, 4004388863, +STORE, 4004388864, 4004646911, +STORE, 4004646912, 4004655103, +STORE, 4004655104, 4004659199, +STORE, 4004659200, 4004667391, +STORE, 4004667392, 4004683775, +STORE, 4004683776, 4004814847, +STORE, 4004814848, 4004818943, +STORE, 4004818944, 4004823039, +STORE, 4004823040, 4004827135, +STORE, 4004827136, 4004835327, +STORE, 4004835328, 4004954111, +STORE, 4004954112, 4005085183, +STORE, 4005085184, 4005306367, +STORE, 4005306368, 4005765119, +STORE, 4005765120, 4005789695, +STORE, 4005789696, 4005793791, +STORE, 4005793792, 4005801983, +STORE, 4005801984, 4005920767, +STORE, 4005920768, 4005945343, +STORE, 4005945344, 4005949439, +STORE, 4005949440, 4005986303, +STORE, 4005986304, 4005990399, +STORE, 4005990400, 4005994495, +STORE, 4005994496, 4006002687, +STORE, 4006002688, 4006109183, +STORE, 4006109184, 4006117375, +STORE, 4006117376, 4006121471, +STORE, 4006121472, 4006133759, +STORE, 4006133760, 4006137855, +STORE, 4006137856, 4006141951, +STORE, 4006141952, 4006150143, +STORE, 4006150144, 4006391807, +STORE, 4006391808, 4006445055, +STORE, 4006445056, 4006563839, +STORE, 4006563840, 4006572031, +STORE, 4006572032, 4006576127, +STORE, 4006576128, 4006584319, +STORE, 4006584320, 4006694911, +STORE, 4006694912, 4006739967, +STORE, 4006739968, 4006776831, +STORE, 4006776832, 4006785023, +STORE, 4006785024, 4006789119, +STORE, 4006789120, 4006797311, +STORE, 4006797312, 4006813695, +STORE, 4006813696, 4006846463, +STORE, 4006846464, 4006977535, +STORE, 4006977536, 4007006207, +STORE, 4007006208, 4007010303, +STORE, 4007010304, 4007067647, +STORE, 4007067648, 4007075839, +STORE, 4007075840, 4007084031, +STORE, 4007084032, 4007100415, +STORE, 4007100416, 4007116799, +STORE, 4007116800, 4007133183, +STORE, 4007133184, 4007153663, +STORE, 4007153664, 4007178239, +STORE, 4007178240, 4007202815, +STORE, 4007202816, 4007206911, +STORE, 4007206912, 4007272447, +STORE, 4007272448, 4007276543, +STORE, 4007276544, 4007280639, +STORE, 4007280640, 4007284735, +STORE, 4007284736, 4007292927, +STORE, 4007292928, 4007423999, +STORE, 4007424000, 4007448575, +STORE, 4007448576, 4007452671, +STORE, 4007452672, 4007505919, +STORE, 4007505920, 4007510015, +STORE, 4007510016, 4007514111, +STORE, 4007514112, 4007645183, +STORE, 4007645184, 4007776255, +STORE, 4007776256, 4007780351, +STORE, 4007780352, 4007784447, +STORE, 4007784448, 4007788543, +STORE, 4007788544, 4007809023, +STORE, 4007809024, 4007829503, +STORE, 4007829504, 4007960575, +STORE, 4007960576, 4008091647, +STORE, 4008091648, 4008296447, +STORE, 4008296448, 4008890367, +STORE, 4008890368, 4008898559, +STORE, 4008898560, 4008902655, +STORE, 4008902656, 4008996863, +STORE, 4008996864, 4009041919, +STORE, 4009041920, 4009082879, +STORE, 4009082880, 4009091071, +STORE, 4009091072, 4009107455, +STORE, 4009107456, 4009349119, +STORE, 4009349120, 4009373695, +STORE, 4009373696, 4009414655, +STORE, 4009414656, 4009422847, +STORE, 4009422848, 4009426943, +STORE, 4009426944, 4009447423, +STORE, 4009447424, 4009471999, +STORE, 4009472000, 4009512959, +STORE, 4009512960, 4009594879, +STORE, 4009594880, 4009598975, +STORE, 4009598976, 4009697279, +STORE, 4009697280, 4009713663, +STORE, 4009713664, 4009717759, +STORE, 4009717760, 4009721855, +STORE, 4009721856, 4009730047, +STORE, 4009730048, 4009861119, +STORE, 4009861120, 4009951231, +STORE, 4009951232, 4010131455, +STORE, 4010131456, 4010135551, +STORE, 4010135552, 4010139647, +STORE, 4010139648, 4010143743, +STORE, 4010143744, 4010164223, +STORE, 4010164224, 4010295295, +STORE, 4010295296, 4010299391, +STORE, 4010299392, 4010491903, +STORE, 4010491904, 4010495999, +STORE, 4010496000, 4010668031, +STORE, 4010668032, 4011028479, +STORE, 4011028480, 4011053055, +STORE, 4011053056, 4011057151, +STORE, 4011057152, 4011118591, +STORE, 4011118592, 4011126783, +STORE, 4011126784, 4011130879, +STORE, 4011130880, 4011143167, +STORE, 4011143168, 4011147263, +STORE, 4011147264, 4011167743, +STORE, 4011167744, 4011171839, +STORE, 4011171840, 4011360255, +STORE, 4011360256, 4011364351, +STORE, 4011364352, 4011626495, +STORE, 4011626496, 4012216319, +STORE, 4012216320, 4012228607, +STORE, 4012228608, 4012232703, +STORE, 4012232704, 4012236799, +STORE, 4012236800, 4012240895, +STORE, 4012240896, 4012261375, +STORE, 4012261376, 4012392447, +STORE, 4012392448, 4012466175, +STORE, 4012466176, 4012597247, +STORE, 4012597248, 4012601343, +STORE, 4012601344, 4012605439, +STORE, 4012605440, 4012609535, +STORE, 4012609536, 4012679167, +STORE, 4012679168, 4013563903, +STORE, 4013563904, 4015366143, +STORE, 4015366144, 4015411199, +STORE, 4015411200, 4015415295, +STORE, 4015415296, 4015419391, +STORE, 4015419392, 4015542271, +STORE, 4015542272, 4015550463, +STORE, 4015550464, 4015558655, +STORE, 4015558656, 4015562751, +STORE, 4015562752, 4015583231, +STORE, 4015583232, 4015587327, +STORE, 4015587328, 4015603711, +STORE, 4015665152, 4015669247, +STORE, 4015669248, 4015812607, +STORE, 4015812608, 4015816703, +STORE, 4015816704, 4016111615, +STORE, 4016111616, 4016467967, +STORE, 4016467968, 4016508927, +STORE, 4016508928, 4016517119, +STORE, 4016517120, 4016525311, +STORE, 4016525312, 4016586751, +STORE, 4016586752, 4016664575, +STORE, 4016664576, 4016697343, +STORE, 4016697344, 4016742399, +STORE, 4016742400, 4016746495, +STORE, 4016746496, 4016750591, +STORE, 4016750592, 4016758783, +STORE, 4016799744, 4016844799, +STORE, 4016844800, 4016902143, +STORE, 4016902144, 4016992255, +STORE, 4016992256, 4017000447, +STORE, 4017000448, 4017004543, +STORE, 4017004544, 4017008639, +STORE, 4017008640, 4017016831, +STORE, 4017016832, 4017020927, +STORE, 4017020928, 4017127423, +STORE, 4017127424, 4017131519, +STORE, 4017131520, 4017229823, +STORE, 4017229824, 4017422335, +STORE, 4017422336, 4017438719, +STORE, 4017438720, 4017442815, +STORE, 4017442816, 4017446911, +STORE, 4017446912, 4017455103, +STORE, 4017455104, 4017766399, +STORE, 4017766400, 4017909759, +STORE, 4017909760, 4018081791, +STORE, 4018081792, 4018089983, +STORE, 4018089984, 4018094079, +STORE, 4018094080, 4018098175, +STORE, 4018098176, 4018327551, +STORE, 4018327552, 4018331647, +STORE, 4018331648, 4018339839, +STORE, 4018339840, 4018348031, +STORE, 4018348032, 4018610175, +STORE, 4018610176, 4018626559, +STORE, 4018626560, 4018647039, +STORE, 4018647040, 4018651135, +STORE, 4018651136, 4018749439, +STORE, 4018749440, 4018761727, +STORE, 4018761728, 4018802687, +STORE, 4018802688, 4018806783, +STORE, 4018806784, 4018810879, +STORE, 4018810880, 4018814975, +STORE, 4018814976, 4018823167, +STORE, 4018823168, 4018954239, +STORE, 4018954240, 4019007487, +STORE, 4019007488, 4019068927, +STORE, 4019068928, 4019077119, +STORE, 4019077120, 4019081215, +STORE, 4019081216, 4019093503, +STORE, 4019093504, 4019208191, +STORE, 4019208192, 4019232767, +STORE, 4019232768, 4019265535, +STORE, 4019265536, 4019269631, +STORE, 4019269632, 4019277823, +STORE, 4019277824, 4019458047, +STORE, 4019458048, 4019519487, +STORE, 4019519488, 4019613695, +STORE, 4019613696, 4019621887, +STORE, 4019621888, 4019625983, +STORE, 4019625984, 4019630079, +STORE, 4019630080, 4019744767, +STORE, 4019744768, 4019822591, +STORE, 4019822592, 4019929087, +STORE, 4019929088, 4019941375, +STORE, 4019941376, 4019945471, +STORE, 4019945472, 4019961855, +STORE, 4019961856, 4019994623, +STORE, 4019994624, 4019998719, +STORE, 4019998720, 4020002815, +STORE, 4020002816, 4020006911, +STORE, 4020006912, 4020011007, +STORE, 4020011008, 4020256767, +STORE, 4020256768, 4020326399, +STORE, 4020326400, 4020457471, +STORE, 4020457472, 4020469759, +STORE, 4020469760, 4020473855, +STORE, 4020473856, 4020482047, +STORE, 4020482048, 4020711423, +STORE, 4020711424, 4020715519, +STORE, 4020715520, 4020719615, +STORE, 4020719616, 4020723711, +STORE, 4020723712, 4020805631, +STORE, 4020805632, 4021051391, +STORE, 4021051392, 4021460991, +STORE, 4021460992, 4021469183, +STORE, 4021469184, 4021473279, +STORE, 4021473280, 4021571583, +STORE, 4021571584, 4021633023, +STORE, 4021633024, 4021727231, +STORE, 4021727232, 4021735423, +STORE, 4021735424, 4021739519, +STORE, 4021739520, 4021747711, +STORE, 4021747712, 4021829631, +STORE, 4021829632, 4021866495, +STORE, 4021866496, 4021919743, +STORE, 4021919744, 4021927935, +STORE, 4021927936, 4021932031, +STORE, 4021932032, 4021944319, +STORE, 4021944320, 4022157311, +STORE, 4022157312, 4022161407, +STORE, 4022161408, 4022173695, +STORE, 4022173696, 4022177791, +STORE, 4022177792, 4022472703, +STORE, 4022472704, 4022509567, +STORE, 4022509568, 4022583295, +STORE, 4022583296, 4022587391, +STORE, 4022587392, 4022591487, +STORE, 4022591488, 4022607871, +STORE, 4022607872, 4022657023, +STORE, 4022657024, 4022722559, +STORE, 4022722560, 4022730751, +STORE, 4022730752, 4022734847, +STORE, 4022734848, 4022865919, +STORE, 4022865920, 4022943743, +STORE, 4022943744, 4023062527, +STORE, 4023062528, 4023074815, +STORE, 4023074816, 4023078911, +STORE, 4023078912, 4023128063, +STORE, 4023128064, 4023218175, +STORE, 4023218176, 4023361535, +STORE, 4023361536, 4023373823, +STORE, 4023373824, 4023377919, +STORE, 4023377920, 4023558143, +STORE, 4023558144, 4023631871, +STORE, 4023631872, 4023816191, +STORE, 4023816192, 4023820287, +STORE, 4023820288, 4023824383, +STORE, 4023824384, 4023832575, +STORE, 4023832576, 4024078335, +STORE, 4024078336, 4024197119, +STORE, 4024197120, 4024389631, +STORE, 4024389632, 4024406015, +STORE, 4024406016, 4024410111, +STORE, 4024410112, 4024422399, +STORE, 4024422400, 4024619007, +STORE, 4024619008, 4024639487, +STORE, 4024639488, 4024655871, +STORE, 4024655872, 4024664063, +STORE, 4024664064, 4024668159, +STORE, 4024668160, 4024676351, +STORE, 4024676352, 4024905727, +STORE, 4024905728, 4024909823, +STORE, 4024909824, 4024918015, +STORE, 4024918016, 4024922111, +STORE, 4024922112, 4024930303, +STORE, 4024930304, 4025110527, +STORE, 4025110528, 4025176063, +STORE, 4025176064, 4025208831, +STORE, 4025208832, 4025212927, +STORE, 4025212928, 4025217023, +STORE, 4025217024, 4025348095, +STORE, 4025348096, 4025372671, +STORE, 4025372672, 4025458687, +STORE, 4025458688, 4025466879, +STORE, 4025466880, 4025565183, +STORE, 4025565184, 4025757695, +STORE, 4025757696, 4026249215, +STORE, 4026249216, 4026261503, +STORE, 4026261504, 4026265599, +STORE, 4026265600, 4026269695, +STORE, 4026269696, 4026302463, +STORE, 4026302464, 4026306559, +STORE, 4026306560, 4026314751, +STORE, 4026314752, 4026318847, +STORE, 4026318848, 4026322943, +STORE, 4026322944, 4026327039, +STORE, 4026327040, 4026654719, +STORE, 4026654720, 4026671103, +STORE, 4026671104, 4026720255, +STORE, 4026720256, 4026724351, +STORE, 4026724352, 4026728447, +STORE, 4026728448, 4026732543, +STORE, 4026732544, 4026863615, +STORE, 4026863616, 4027027455, +STORE, 4027027456, 4027031551, +STORE, 4027031552, 4027514879, +STORE, 4027514880, 4027531263, +STORE, 4027531264, 4027535359, +STORE, 4027535360, 4027539455, +STORE, 4027539456, 4027785215, +STORE, 4027785216, 4027789311, +STORE, 4027789312, 4027793407, +STORE, 4027793408, 4027797503, +STORE, 4027797504, 4027863039, +STORE, 4027863040, 4027899903, +STORE, 4027899904, 4027949055, +STORE, 4027949056, 4027957247, +STORE, 4027957248, 4027961343, +STORE, 4027961344, 4027965439, +STORE, 4027965440, 4028194815, +STORE, 4028194816, 4028252159, +STORE, 4028252160, 4028338175, +STORE, 4028338176, 4028350463, +STORE, 4028350464, 4028354559, +STORE, 4028354560, 4028452863, +STORE, 4028452864, 4028489727, +STORE, 4028489728, 4028530687, +STORE, 4028530688, 4028538879, +STORE, 4028538880, 4028542975, +STORE, 4028542976, 4028551167, +STORE, 4028551168, 4028665855, +STORE, 4028665856, 4029349887, +STORE, 4029349888, 4030468095, +STORE, 4030468096, 4030513151, +STORE, 4030513152, 4030517247, +STORE, 4030517248, 4030525439, +STORE, 4030525440, 4030529535, +STORE, 4030529536, 4030758911, +STORE, 4030758912, 4030828543, +STORE, 4030828544, 4030943231, +STORE, 4030943232, 4030951423, +STORE, 4030951424, 4030955519, +STORE, 4030955520, 4030967807, +STORE, 4030967808, 4031131647, +STORE, 4031131648, 4031135743, +STORE, 4031135744, 4031139839, +STORE, 4031139840, 4031148031, +STORE, 4031148032, 4031152127, +STORE, 4031152128, 4031160319, +STORE, 4031160320, 4031504383, +STORE, 4031504384, 4031598591, +STORE, 4031598592, 4031754239, +STORE, 4031754240, 4031766527, +STORE, 4031766528, 4031770623, +STORE, 4031770624, 4031774719, +STORE, 4031774720, 4031782911, +STORE, 4031782912, 4031799295, +STORE, 4031799296, 4031856639, +STORE, 4031856640, 4031983615, +STORE, 4031983616, 4031987711, +STORE, 4031987712, 4031991807, +STORE, 4031991808, 4032270335, +STORE, 4032270336, 4032274431, +STORE, 4032274432, 4032282623, +STORE, 4032282624, 4032286719, +STORE, 4032286720, 4032290815, +STORE, 4032290816, 4032389119, +STORE, 4032389120, 4032397311, +STORE, 4032397312, 4032405503, +STORE, 4032405504, 4032413695, +STORE, 4032413696, 4032417791, +STORE, 4032417792, 4032565247, +STORE, 4032565248, 4032593919, +STORE, 4032593920, 4032737279, +STORE, 4032737280, 4032741375, +STORE, 4032741376, 4032745471, +STORE, 4032745472, 4032770047, +STORE, 4032770048, 4032933887, +STORE, 4032933888, 4032999423, +STORE, 4032999424, 4033032191, +STORE, 4033032192, 4033036287, +STORE, 4033036288, 4033040383, +STORE, 4033040384, 4033105919, +STORE, 4033105920, 4033396735, +STORE, 4033396736, 4033822719, +STORE, 4033822720, 4033839103, +STORE, 4033839104, 4033843199, +STORE, 4033843200, 4033851391, +STORE, 4033851392, 4033863679, +STORE, 4033863680, 4033880063, +STORE, 4033880064, 4033933311, +STORE, 4033933312, 4034023423, +STORE, 4034023424, 4034031615, +STORE, 4034031616, 4034035711, +STORE, 4034035712, 4034043903, +STORE, 4034043904, 4034142207, +STORE, 4034142208, 4034191359, +STORE, 4034191360, 4034260991, +STORE, 4034260992, 4034269183, +STORE, 4034269184, 4034273279, +STORE, 4034273280, 4034281471, +STORE, 4034281472, 4034412543, +STORE, 4034412544, 4034445311, +STORE, 4034445312, 4034490367, +STORE, 4034490368, 4034494463, +STORE, 4034494464, 4034498559, +STORE, 4034498560, 4034662399, +STORE, 4034662400, 4034666495, +STORE, 4034666496, 4034670591, +STORE, 4034670592, 4034674687, +STORE, 4034674688, 4034678783, +STORE, 4034678784, 4034682879, +STORE, 4034682880, 4034781183, +STORE, 4034781184, 4035043327, +STORE, 4035043328, 4035047423, +STORE, 4035047424, 4035055615, +STORE, 4035055616, 4035059711, +STORE, 4035059712, 4035063807, +STORE, 4035063808, 4035067903, +STORE, 4035067904, 4035100671, +STORE, 4035100672, 4035375103, +STORE, 4035375104, 4035383295, +STORE, 4035383296, 4035395583, +STORE, 4035395584, 4035399679, +STORE, 4035399680, 4035403775, +STORE, 4035403776, 4035407871, +STORE, 4035407872, 4035411967, +STORE, 4035411968, 4035477503, +STORE, 4035477504, 4035608575, +STORE, 4035608576, 4035641343, +STORE, 4035641344, 4035682303, +STORE, 4035682304, 4035686399, +STORE, 4035686400, 4035690495, +STORE, 4035690496, 4035694591, +STORE, 4035694592, 4035743743, +STORE, 4035743744, 4035784703, +STORE, 4035784704, 4035829759, +STORE, 4035829760, 4035837951, +STORE, 4035837952, 4035842047, +STORE, 4035842048, 4035846143, +STORE, 4035846144, 4035850239, +STORE, 4035850240, 4036001791, +STORE, 4036001792, 4036005887, +STORE, 4036005888, 4036214783, +STORE, 4036214784, 4036218879, +STORE, 4036218880, 4036603903, +STORE, 4036603904, 4036648959, +STORE, 4036648960, 4036653055, +STORE, 4036653056, 4036657151, +STORE, 4036657152, 4036665343, +STORE, 4036665344, 4036780031, +STORE, 4036780032, 4036829183, +STORE, 4036829184, 4036984831, +STORE, 4036984832, 4036993023, +STORE, 4036993024, 4036997119, +STORE, 4036997120, 4037001215, +STORE, 4037001216, 4037009407, +STORE, 4037009408, 4037025791, +STORE, 4037025792, 4037095423, +STORE, 4037095424, 4037181439, +STORE, 4037181440, 4037193727, +STORE, 4037193728, 4037197823, +STORE, 4037197824, 4037206015, +STORE, 4037206016, 4037320703, +STORE, 4037320704, 4037337087, +STORE, 4037337088, 4037349375, +STORE, 4037349376, 4037357567, +STORE, 4037357568, 4037361663, +STORE, 4037369856, 4037386239, +STORE, 4037386240, 4037672959, +STORE, 4037672960, 4037689343, +STORE, 4037689344, 4037730303, +STORE, 4037730304, 4037734399, +STORE, 4037734400, 4037738495, +STORE, 4037738496, 4037742591, +STORE, 4037742592, 4037758975, +STORE, 4037758976, 4037890047, +STORE, 4037890048, 4037931007, +STORE, 4037931008, 4037976063, +STORE, 4037976064, 4037984255, +STORE, 4037984256, 4037988351, +STORE, 4037988352, 4038053887, +STORE, 4038053888, 4038184959, +STORE, 4038184960, 4038189055, +STORE, 4038189056, 4038197247, +STORE, 4038197248, 4038201343, +STORE, 4038201344, 4038205439, +STORE, 4038205440, 4038209535, +STORE, 4038217728, 4038250495, +STORE, 4038250496, 4038512639, +STORE, 4038512640, 4038516735, +STORE, 4038516736, 4038520831, +STORE, 4038520832, 4038524927, +STORE, 4038524928, 4038529023, +STORE, 4038529024, 4038533119, +STORE, 4038541312, 4038623231, +STORE, 4038623232, 4038754303, +STORE, 4038754304, 4038885375, +STORE, 4038885376, 4038889471, +STORE, 4038897664, 4038963199, +STORE, 4038963200, 4038967295, +STORE, 4038967296, 4038983679, +STORE, 4038983680, 4039114751, +STORE, 4039114752, 4039245823, +STORE, 4039245824, 4039376895, +STORE, 4039376896, 4040687615, +STORE, 4040687616, 4040691711, +STORE, 4040691712, 4040806399, +STORE, 4040806400, 4040937471, +STORE, 4040937472, 4040941567, +STORE, 4040945664, 4040949759, +STORE, 4040949760, 4041080831, +STORE, 4041080832, 4041211903, +STORE, 4041211904, 4043046911, +STORE, 4043046912, 4043051007, +STORE, 4043051008, 4043055103, +STORE, 4043055104, 4043137023, +STORE, 4043137024, 4043141119, +STORE, 4043141120, 4043145215, +STORE, 4043145216, 4043153407, +STORE, 4043153408, 4043186175, +STORE, 4043186176, 4043317247, +STORE, 4043317248, 4043448319, +STORE, 4043448320, 4043579391, +STORE, 4043579392, 4043583487, +STORE, 4043583488, 4043599871, +STORE, 4043599872, 4043661311, +STORE, 4043661312, 4043792383, +STORE, 4043792384, 4043796479, +STORE, 4043796480, 4043800575, +STORE, 4043800576, 4043816959, +STORE, 4043816960, 4043821055, +STORE, 4043821056, 4043825151, +STORE, 4043825152, 4043829247, +STORE, 4043829248, 4043833343, +STORE, 4043833344, 4047241215, +STORE, 4047241216, 4047249407, +STORE, 4047249408, 4047253503, +STORE, 4047253504, 4047323135, +STORE, 4047323136, 4047327231, +STORE, 4047327232, 4047458303, +STORE, 4047458304, 4047589375, +STORE, 4047589376, 4047720447, +STORE, 4047720448, 4047773695, +STORE, 4047773696, 4047790079, +STORE, 4047790080, 4047921151, +STORE, 4047921152, 4048052223, +STORE, 4048052224, 4048183295, +STORE, 4048183296, 4049002495, +STORE, 4049002496, 4049133567, +STORE, 4049133568, 4049154047, +STORE, 4049154048, 4049158143, +STORE, 4049158144, 4049162239, +STORE, 4049162240, 4049166335, +STORE, 4049166336, 4049174527, +STORE, 4049174528, 4049182719, +STORE, 4049182720, 4049186815, +STORE, 4049186816, 4049190911, +STORE, 4049190912, 4049195007, +STORE, 4049195008, 4049203199, +STORE, 4049203200, 4049207295, +STORE, 4049207296, 4049211391, +STORE, 4049211392, 4049215487, +STORE, 4049215488, 4049219583, +STORE, 4049219584, 4049227775, +STORE, 4049227776, 4049231871, +STORE, 4049231872, 4049235967, +STORE, 4049235968, 4049244159, +STORE, 4049244160, 4049248255, +STORE, 4049248256, 4049252351, +STORE, 4049252352, 4049256447, +STORE, 4049256448, 4049268735, +STORE, 4049268736, 4049272831, +STORE, 4049272832, 4049313791, +STORE, 4049313792, 4049723391, +STORE, 4049723392, 4049727487, +STORE, 4049727488, 4049858559, +STORE, 4049858560, 4049989631, +STORE, 4049989632, 4049993727, +STORE, 4049993728, 4050026495, +STORE, 4050026496, 4050030591, +STORE, 4050030592, 4050161663, +STORE, 4050161664, 4050169855, +STORE, 4050169856, 4050223103, +STORE, 4050223104, 4050632703, +STORE, 4050632704, 4050636799, +STORE, 4050636800, 4050640895, +STORE, 4050640896, 4050644991, +STORE, 4050644992, 4050661375, +STORE, 4050661376, 4050665471, +STORE, 4050665472, 4050673663, +STORE, 4050673664, 4050677759, +STORE, 4050677760, 4050694143, +STORE, 4050694144, 4050702335, +STORE, 4050702336, 4050956287, +STORE, 4050956288, 4051963903, +STORE, 4051963904, 4051980287, +STORE, 4051980288, 4051988479, +STORE, 4051988480, 4052000767, +STORE, 4052000768, 4052004863, +STORE, 4052004864, 4052029439, +STORE, 4284014592, 4284018687, +STORE, 4284018688, 4292403199, +SNULL, 4041080832, 4041211903, +SNULL, 3795763200, 3795894271, +STORE, 3629522944, 3696631807, +SNULL, 3663077375, 3696631807, +STORE, 3629522944, 3663077375, +STORE, 3663077376, 3696631807, +SNULL, 3663077376, 3696631807, +STORE, 3663077376, 3696631807, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3626471424, 3627524095, +SNULL, 3626471424, 3626475519, +STORE, 3626475520, 3627524095, +STORE, 3626471424, 3626475519, +SNULL, 3627519999, 3627524095, +STORE, 3626475520, 3627519999, +STORE, 3627520000, 3627524095, +STORE, 3625418752, 3626475519, +SNULL, 3625418752, 3625422847, +STORE, 3625422848, 3626475519, +STORE, 3625418752, 3625422847, +SNULL, 3626467327, 3626475519, +STORE, 3625422848, 3626467327, +STORE, 3626467328, 3626475519, +STORE, 3624366080, 3625422847, +SNULL, 3624366080, 3624370175, +STORE, 3624370176, 3625422847, +STORE, 3624366080, 3624370175, +SNULL, 3625414655, 3625422847, +STORE, 3624370176, 3625414655, +STORE, 3625414656, 3625422847, +STORE, 4041191424, 4041211903, +SNULL, 4041195519, 4041211903, +STORE, 4041191424, 4041195519, +STORE, 4041195520, 4041211903, +STORE, 4041170944, 4041191423, +SNULL, 4041175039, 4041191423, +STORE, 4041170944, 4041175039, +STORE, 4041175040, 4041191423, +SNULL, 3625426943, 3626467327, +STORE, 3625422848, 3625426943, +STORE, 3625426944, 3626467327, +STORE, 4041162752, 4041170943, +SNULL, 3626479615, 3627519999, +STORE, 3626475520, 3626479615, +STORE, 3626479616, 3627519999, +STORE, 4041154560, 4041162751, +STORE, 4041154560, 4041170943, +STORE, 4041134080, 4041154559, +SNULL, 4041138175, 4041154559, +STORE, 4041134080, 4041138175, +STORE, 4041138176, 4041154559, +SNULL, 3624374271, 3625414655, +STORE, 3624370176, 3624374271, +STORE, 3624374272, 3625414655, +STORE, 4041125888, 4041134079, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3487174656, 3487584255, +STORE, 4041121792, 4041125887, +SNULL, 4041121792, 4041125887, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 3487174656, 3487584255, +STORE, 3222274048, 3223326719, +SNULL, 3222274048, 3222278143, +STORE, 3222278144, 3223326719, +STORE, 3222274048, 3222278143, +SNULL, 3223322623, 3223326719, +STORE, 3222278144, 3223322623, +STORE, 3223322624, 3223326719, +STORE, 3221221376, 3222278143, +SNULL, 3221221376, 3221225471, +STORE, 3221225472, 3222278143, +STORE, 3221221376, 3221225471, +SNULL, 3222269951, 3222278143, +STORE, 3221225472, 3222269951, +STORE, 3222269952, 3222278143, +STORE, 3220168704, 3221225471, +SNULL, 3220168704, 3220172799, +STORE, 3220172800, 3221225471, +STORE, 3220168704, 3220172799, +SNULL, 3221217279, 3221225471, +STORE, 3220172800, 3221217279, +STORE, 3221217280, 3221225471, +STORE, 4041117696, 4041125887, +STORE, 4041117696, 4041134079, +STORE, 3219083264, 3220172799, +SNULL, 3219083264, 3219087359, +STORE, 3219087360, 3220172799, +STORE, 3219083264, 3219087359, +SNULL, 3220164607, 3220172799, +STORE, 3219087360, 3220164607, +STORE, 3220164608, 3220172799, +STORE, 4041109504, 4041117695, +STORE, 4041109504, 4041134079, +STORE, 3217997824, 3219087359, +SNULL, 3217997824, 3218001919, +STORE, 3218001920, 3219087359, +STORE, 3217997824, 3218001919, +SNULL, 3219079167, 3219087359, +STORE, 3218001920, 3219079167, +STORE, 3219079168, 3219087359, +STORE, 4041101312, 4041109503, +STORE, 4041101312, 4041134079, +STORE, 3216912384, 3218001919, +SNULL, 3216912384, 3216916479, +STORE, 3216916480, 3218001919, +STORE, 3216912384, 3216916479, +SNULL, 3217993727, 3218001919, +STORE, 3216916480, 3217993727, +STORE, 3217993728, 3218001919, +STORE, 4041093120, 4041101311, +STORE, 4041093120, 4041134079, +STORE, 3215826944, 3216916479, +SNULL, 3215826944, 3215831039, +STORE, 3215831040, 3216916479, +STORE, 3215826944, 3215831039, +SNULL, 3216908287, 3216916479, +STORE, 3215831040, 3216908287, +STORE, 3216908288, 3216916479, +STORE, 4016779264, 4016799743, +SNULL, 4016783359, 4016799743, +STORE, 4016779264, 4016783359, +STORE, 4016783360, 4016799743, +STORE, 4016758784, 4016779263, +SNULL, 4016762879, 4016779263, +STORE, 4016758784, 4016762879, +STORE, 4016762880, 4016779263, +SNULL, 3222282239, 3223322623, +STORE, 3222278144, 3222282239, +STORE, 3222282240, 3223322623, +STORE, 4041084928, 4041093119, +STORE, 4041084928, 4041134079, +SNULL, 3221229567, 3222269951, +STORE, 3221225472, 3221229567, +STORE, 3221229568, 3222269951, +STORE, 4015644672, 4015665151, +STORE, 4038889472, 4038897663, +SNULL, 4015648767, 4015665151, +STORE, 4015644672, 4015648767, +STORE, 4015648768, 4015665151, +STORE, 4015624192, 4015644671, +SNULL, 4015628287, 4015644671, +STORE, 4015624192, 4015628287, +STORE, 4015628288, 4015644671, +SNULL, 3219091455, 3220164607, +STORE, 3219087360, 3219091455, +STORE, 3219091456, 3220164607, +STORE, 4015603712, 4015624191, +SNULL, 4015607807, 4015624191, +STORE, 4015603712, 4015607807, +STORE, 4015607808, 4015624191, +SNULL, 3218006015, 3219079167, +STORE, 3218001920, 3218006015, +STORE, 3218006016, 3219079167, +STORE, 3949674496, 3949694975, +SNULL, 3949678591, 3949694975, +STORE, 3949674496, 3949678591, +STORE, 3949678592, 3949694975, +SNULL, 3216920575, 3217993727, +STORE, 3216916480, 3216920575, +STORE, 3216920576, 3217993727, +STORE, 3948924928, 3948945407, +SNULL, 3948929023, 3948945407, +STORE, 3948924928, 3948929023, +STORE, 3948929024, 3948945407, +SNULL, 3215835135, 3216908287, +STORE, 3215831040, 3215835135, +STORE, 3215835136, 3216908287, +SNULL, 3220176895, 3221217279, +STORE, 3220172800, 3220176895, +STORE, 3220176896, 3221217279, +STORE, 3214786560, 3215826943, +STORE, 3213733888, 3214786559, +SNULL, 3213733888, 3213737983, +STORE, 3213737984, 3214786559, +STORE, 3213733888, 3213737983, +SNULL, 3214782463, 3214786559, +STORE, 3213737984, 3214782463, +STORE, 3214782464, 3214786559, +STORE, 4038533120, 4038541311, +STORE, 3948421120, 3948441599, +SNULL, 3948425215, 3948441599, +STORE, 3948421120, 3948425215, +STORE, 3948425216, 3948441599, +SNULL, 3213742079, 3214782463, +STORE, 3213737984, 3213742079, +STORE, 3213742080, 3214782463, +STORE, 4038209536, 4038217727, +STORE, 3212681216, 3213737983, +SNULL, 3212681216, 3212685311, +STORE, 3212685312, 3213737983, +STORE, 3212681216, 3212685311, +SNULL, 3213729791, 3213737983, +STORE, 3212685312, 3213729791, +STORE, 3213729792, 3213737983, +STORE, 3795763200, 3795894271, +STORE, 3946872832, 3946893311, +SNULL, 3946876927, 3946893311, +STORE, 3946872832, 3946876927, +STORE, 3946876928, 3946893311, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3487174656, 3487584255, +SNULL, 3212689407, 3213729791, +STORE, 3212685312, 3212689407, +STORE, 3212689408, 3213729791, +STORE, 4041080832, 4041084927, +STORE, 4040941568, 4040945663, +STORE, 4037361664, 4037369855, +STORE, 4000817152, 4000821247, +STORE, 3999440896, 3999444991, +STORE, 3212161024, 3212681215, +SNULL, 3212161024, 3212439551, +STORE, 3212439552, 3212681215, +STORE, 3212161024, 3212439551, +SNULL, 3212161024, 3212439551, +SNULL, 3212464127, 3212681215, +STORE, 3212439552, 3212464127, +STORE, 3212464128, 3212681215, +SNULL, 3212464128, 3212681215, +SNULL, 3212439552, 3212451839, +STORE, 3212451840, 3212464127, +STORE, 3212439552, 3212451839, +SNULL, 3212439552, 3212451839, +STORE, 3212439552, 3212451839, +SNULL, 3212451840, 3212455935, +STORE, 3212455936, 3212464127, +STORE, 3212451840, 3212455935, +SNULL, 3212451840, 3212455935, +STORE, 3212451840, 3212455935, +SNULL, 3212455936, 3212460031, +STORE, 3212460032, 3212464127, +STORE, 3212455936, 3212460031, +SNULL, 3212455936, 3212460031, +STORE, 3212455936, 3212460031, +SNULL, 3212460032, 3212464127, +STORE, 3212460032, 3212464127, +STORE, 3997679616, 3997683711, +SNULL, 4049235968, 4049240063, +STORE, 4049240064, 4049244159, +STORE, 4049235968, 4049240063, +SNULL, 4049240064, 4049244159, +STORE, 4049240064, 4049244159, +SNULL, 3997679616, 3997683711, +SNULL, 3999440896, 3999444991, +SNULL, 4000817152, 4000821247, +SNULL, 4040941568, 4040945663, +SNULL, 4041080832, 4041084927, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 3487174656, 3487584255, +SNULL, 3212451840, 3212455935, +STORE, 3212451840, 3212455935, +STORE, 4041080832, 4041084927, +STORE, 3623890944, 3624169471, +SNULL, 4041080832, 4041084927, +STORE, 4041080832, 4041084927, +SNULL, 4041080832, 4041084927, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 4041080832, 4041084927, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3211386880, 3212439551, +SNULL, 3211386880, 3211390975, +STORE, 3211390976, 3212439551, +STORE, 3211386880, 3211390975, +SNULL, 3212435455, 3212439551, +STORE, 3211390976, 3212435455, +STORE, 3212435456, 3212439551, +STORE, 4040941568, 4040945663, +STORE, 3937169408, 3937189887, +STORE, 3623485440, 3623616511, +SNULL, 717225983, 1388314623, +STORE, 314572800, 717225983, +STORE, 717225984, 1388314623, +SNULL, 717225984, 1388314623, +STORE, 3937112064, 3937132543, +SNULL, 3937116159, 3937132543, +STORE, 3937112064, 3937116159, +STORE, 3937116160, 3937132543, +SNULL, 3211395071, 3212435455, +STORE, 3211390976, 3211395071, +STORE, 3211395072, 3212435455, +STORE, 4000817152, 4000821247, +STORE, 3974823936, 3974832127, +STORE, 3595284480, 3595431935, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +STORE, 3487174656, 3487584255, +STORE, 3999440896, 3999444991, +STORE, 3997679616, 3997683711, +STORE, 3996295168, 3996299263, +STORE, 3996090368, 3996094463, +STORE, 3210866688, 3211386879, +SNULL, 3210866688, 3211001855, +STORE, 3211001856, 3211386879, +STORE, 3210866688, 3211001855, +SNULL, 3210866688, 3211001855, +SNULL, 3211038719, 3211386879, +STORE, 3211001856, 3211038719, +STORE, 3211038720, 3211386879, +SNULL, 3211038720, 3211386879, +SNULL, 3211001856, 3211022335, +STORE, 3211022336, 3211038719, +STORE, 3211001856, 3211022335, +SNULL, 3211001856, 3211022335, +STORE, 3211001856, 3211022335, +SNULL, 3211022336, 3211030527, +STORE, 3211030528, 3211038719, +STORE, 3211022336, 3211030527, +SNULL, 3211022336, 3211030527, +STORE, 3211022336, 3211030527, +SNULL, 3211030528, 3211034623, +STORE, 3211034624, 3211038719, +STORE, 3211030528, 3211034623, +SNULL, 3211030528, 3211034623, +STORE, 3211030528, 3211034623, +SNULL, 3211034624, 3211038719, +STORE, 3211034624, 3211038719, +STORE, 3994906624, 3994910719, +SNULL, 4049240064, 4049244159, +STORE, 4049240064, 4049244159, +SNULL, 3994906624, 3994910719, +SNULL, 3996090368, 3996094463, +SNULL, 3996295168, 3996299263, +SNULL, 3997679616, 3997683711, +SNULL, 3999440896, 3999444991, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 3487174656, 3487584255, +SNULL, 3211022336, 3211030527, +STORE, 3211022336, 3211030527, +STORE, 3999440896, 3999444991, +STORE, 3210199040, 3211001855, +SNULL, 3999440896, 3999444991, +STORE, 3999440896, 3999444991, +SNULL, 3999440896, 3999444991, +STORE, 3594821632, 3594952703, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 4048183296, 4048592895, +STORE, 4048592896, 4049002495, +STORE, 4048183296, 4048592895, +STORE, 4048183296, 4049002495, +SNULL, 1914101759, 1969434623, +STORE, 1914097664, 1914101759, +STORE, 1914101760, 1969434623, +STORE, 3567108096, 3567239167, +STORE, 3973832704, 3973840895, +STORE, 3209113600, 3210199039, +SNULL, 3209113600, 3209117695, +STORE, 3209117696, 3210199039, +STORE, 3209113600, 3209117695, +SNULL, 3210194943, 3210199039, +STORE, 3209117696, 3210194943, +STORE, 3210194944, 3210199039, +STORE, 3935858688, 3935879167, +SNULL, 3935862783, 3935879167, +STORE, 3935858688, 3935862783, +STORE, 3935862784, 3935879167, +SNULL, 3209121791, 3210194943, +STORE, 3209117696, 3209121791, +STORE, 3209121792, 3210194943, +STORE, 3528749056, 3528880127, +STORE, 3968200704, 3968208895, +STORE, 3208028160, 3209117695, +SNULL, 3208028160, 3208032255, +STORE, 3208032256, 3209117695, +STORE, 3208028160, 3208032255, +SNULL, 3209109503, 3209117695, +STORE, 3208032256, 3209109503, +STORE, 3209109504, 3209117695, +STORE, 3888123904, 3888144383, +SNULL, 3888127999, 3888144383, +STORE, 3888123904, 3888127999, +STORE, 3888128000, 3888144383, +SNULL, 3208036351, 3209109503, +STORE, 3208032256, 3208036351, +STORE, 3208036352, 3209109503, +SNULL, 3968200704, 3968208895, +SNULL, 3888123904, 3888144383, +SNULL, 3209109504, 3209113599, +STORE, 3209113600, 3209117695, +STORE, 3209109504, 3209113599, +SNULL, 3208028160, 3209113599, +STORE, 3208060928, 3209117695, +SNULL, 3208060928, 3208065023, +STORE, 3208065024, 3209117695, +STORE, 3208060928, 3208065023, +SNULL, 3209109503, 3209117695, +STORE, 3208065024, 3209109503, +STORE, 3209109504, 3209117695, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3888123904, 3888144383, +SNULL, 3888127999, 3888144383, +STORE, 3888123904, 3888127999, +STORE, 3888128000, 3888144383, +SNULL, 3208069119, 3209109503, +STORE, 3208065024, 3208069119, +STORE, 3208069120, 3209109503, +STORE, 3968200704, 3968208895, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3527778304, 3527909375, +STORE, 3999440896, 3999444991, +STORE, 3997679616, 3997683711, +STORE, 1914097664, 1914105855, +STORE, 1914105856, 1969434623, +STORE, 3957583872, 3957592063, +STORE, 3206975488, 3208065023, +SNULL, 3206975488, 3206979583, +STORE, 3206979584, 3208065023, +STORE, 3206975488, 3206979583, +SNULL, 3208056831, 3208065023, +STORE, 3206979584, 3208056831, +STORE, 3208056832, 3208065023, +STORE, 3956736000, 3956744191, +STORE, 3205890048, 3206979583, +SNULL, 3205890048, 3205894143, +STORE, 3205894144, 3206979583, +STORE, 3205890048, 3205894143, +SNULL, 3206971391, 3206979583, +STORE, 3205894144, 3206971391, +STORE, 3206971392, 3206979583, +STORE, 3806101504, 3806121983, +SNULL, 3806105599, 3806121983, +STORE, 3806101504, 3806105599, +STORE, 3806105600, 3806121983, +SNULL, 3206983679, 3208056831, +STORE, 3206979584, 3206983679, +STORE, 3206983680, 3208056831, +STORE, 3806081024, 3806101503, +SNULL, 3806085119, 3806101503, +STORE, 3806081024, 3806085119, +STORE, 3806085120, 3806101503, +SNULL, 3205898239, 3206971391, +STORE, 3205894144, 3205898239, +STORE, 3205898240, 3206971391, +STORE, 3956015104, 3956023295, +STORE, 3204804608, 3205894143, +SNULL, 3204804608, 3204808703, +STORE, 3204808704, 3205894143, +STORE, 3204804608, 3204808703, +SNULL, 3205885951, 3205894143, +STORE, 3204808704, 3205885951, +STORE, 3205885952, 3205894143, +STORE, 3803471872, 3803492351, +STORE, 3803451392, 3803471871, +STORE, 3803451392, 3803492351, +SNULL, 3957583872, 3957592063, +SNULL, 3806101504, 3806121983, +SNULL, 3206975487, 3206979583, +STORE, 3206971392, 3206975487, +STORE, 3206975488, 3206979583, +SNULL, 3208056832, 3208060927, +STORE, 3208060928, 3208065023, +STORE, 3208056832, 3208060927, +SNULL, 3206975488, 3208060927, +STORE, 3801845760, 3801878527, +STORE, 3806101504, 3806121983, +SNULL, 3806105599, 3806121983, +STORE, 3806101504, 3806105599, +STORE, 3806105600, 3806121983, +SNULL, 3204812799, 3205885951, +STORE, 3204808704, 3204812799, +STORE, 3204812800, 3205885951, +STORE, 1914097664, 1914109951, +STORE, 1914109952, 1969434623, +STORE, 3957583872, 3957592063, +STORE, 3206971392, 3208065023, +SNULL, 3206971392, 3206979583, +STORE, 3206979584, 3208065023, +STORE, 3206971392, 3206979583, +SNULL, 3208056831, 3208065023, +STORE, 3206979584, 3208056831, +STORE, 3208056832, 3208065023, +STORE, 3801825280, 3801845759, +SNULL, 3801829375, 3801845759, +STORE, 3801825280, 3801829375, +STORE, 3801829376, 3801845759, +SNULL, 3206983679, 3208056831, +STORE, 3206979584, 3206983679, +STORE, 3206983680, 3208056831, +STORE, 3202707456, 3204804607, +SNULL, 3202707456, 3204804607, +STORE, 3202707456, 3204804607, +STORE, 3200610304, 3202707455, +SNULL, 3202707456, 3204804607, +SNULL, 3200610304, 3202707455, +STORE, 3202707456, 3204804607, +SNULL, 3202707456, 3204804607, +STORE, 3202707456, 3204804607, +SNULL, 3202707456, 3204804607, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3527647232, 3527778303, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +STORE, 3487059968, 3487584255, +SNULL, 3487059968, 3487301631, +STORE, 3487301632, 3487584255, +STORE, 3487059968, 3487301631, +SNULL, 3487059968, 3487301631, +SNULL, 3487563775, 3487584255, +STORE, 3487301632, 3487563775, +STORE, 3487563776, 3487584255, +SNULL, 3487563776, 3487584255, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3524046848, 3524177919, +STORE, 3487170560, 3487301631, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3487039488, 3487170559, +STORE, 3487039488, 3487301631, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3204280320, 3204804607, +SNULL, 3204280320, 3204448255, +STORE, 3204448256, 3204804607, +STORE, 3204280320, 3204448255, +SNULL, 3204280320, 3204448255, +SNULL, 3204710399, 3204804607, +STORE, 3204448256, 3204710399, +STORE, 3204710400, 3204804607, +SNULL, 3204710400, 3204804607, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3996295168, 3996299263, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +SNULL, 3996295168, 3996299263, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3486908416, 3487039487, +STORE, 3486908416, 3487301631, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3223326720, 3290435583, +SNULL, 3223326720, 3256881151, +STORE, 3256881152, 3290435583, +STORE, 3223326720, 3256881151, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +STORE, 3201826816, 3202351103, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +STORE, 3202351104, 3204448255, +SNULL, 3202351104, 3204448255, +SNULL, 3803471871, 3803492351, +STORE, 3803451392, 3803471871, +STORE, 3803471872, 3803492351, +SNULL, 3803471872, 3803492351, +SNULL, 3803451392, 3803471871, +STORE, 3798999040, 3799101439, +SNULL, 3798999040, 3799101439, +STORE, 3952644096, 3952652287, +STORE, 3203362816, 3204448255, +SNULL, 3203362816, 3203366911, +STORE, 3203366912, 3204448255, +STORE, 3203362816, 3203366911, +SNULL, 3204444159, 3204448255, +STORE, 3203366912, 3204444159, +STORE, 3204444160, 3204448255, +STORE, 3803471872, 3803492351, +SNULL, 3803475967, 3803492351, +STORE, 3803471872, 3803475967, +STORE, 3803475968, 3803492351, +SNULL, 3203371007, 3204444159, +STORE, 3203366912, 3203371007, +STORE, 3203371008, 3204444159, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3199729664, 3201826815, +SNULL, 3199729664, 3201826815, +STORE, 3200774144, 3201826815, +SNULL, 3200774144, 3200778239, +STORE, 3200778240, 3201826815, +STORE, 3200774144, 3200778239, +SNULL, 3201822719, 3201826815, +STORE, 3200778240, 3201822719, +STORE, 3201822720, 3201826815, +STORE, 3803451392, 3803471871, +SNULL, 3803455487, 3803471871, +STORE, 3803451392, 3803455487, +STORE, 3803455488, 3803471871, +SNULL, 3200782335, 3201822719, +STORE, 3200778240, 3200782335, +STORE, 3200782336, 3201822719, +STORE, 3949666304, 3949674495, +STORE, 3949408256, 3949416447, +STORE, 3199688704, 3200778239, +SNULL, 3199688704, 3199692799, +STORE, 3199692800, 3200778239, +STORE, 3199688704, 3199692799, +SNULL, 3200770047, 3200778239, +STORE, 3199692800, 3200770047, +STORE, 3200770048, 3200778239, +STORE, 3799306240, 3799326719, +SNULL, 3799310335, 3799326719, +STORE, 3799306240, 3799310335, +STORE, 3799310336, 3799326719, +SNULL, 3199696895, 3200770047, +STORE, 3199692800, 3199696895, +STORE, 3199696896, 3200770047, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +STORE, 3799277568, 3799306239, +SNULL, 3799277568, 3799306239, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +SNULL, 4041162751, 4041170943, +STORE, 4041154560, 4041162751, +STORE, 4041162752, 4041170943, +SNULL, 4041162752, 4041170943, +SNULL, 4041154560, 4041162751, +SNULL, 4041191424, 4041211903, +SNULL, 4041170944, 4041191423, +SNULL, 3626471423, 3626475519, +STORE, 3626467328, 3626471423, +STORE, 3626471424, 3626475519, +SNULL, 3626471424, 3627524095, +SNULL, 3625418751, 3625422847, +STORE, 3625414656, 3625418751, +STORE, 3625418752, 3625422847, +SNULL, 3625418752, 3626471423, +STORE, 3627393024, 3627524095, +STORE, 3627261952, 3627393023, +STORE, 3627261952, 3627524095, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +STORE, 3195494400, 3197591551, +SNULL, 3197591552, 3199688703, +SNULL, 3195494400, 3197591551, +STORE, 3197591552, 3199688703, +SNULL, 3197591552, 3199688703, +STORE, 3197591552, 3199688703, +STORE, 3195494400, 3197591551, +SNULL, 3197591552, 3199688703, +SNULL, 3195494400, 3197591551, +STORE, 3798999040, 3799101439, +SNULL, 3798999040, 3799101439, +/* + * mmap: unmapped_area_topdown: ffff9a9f14ddaa80 + * Gap was found: mt 4041162752 gap_end 4041183232 + * mmap: window was 4052029440 - 4096 size 28672 + * mmap: mas.min 4041154560 max 4041191423 mas.last 4041191423 + * mmap: mas.index 4041162752 align mask 0 offset 0 + * mmap: rb_find_vma find on 4041162752 => ffff9a9f03d19678 (ffff9a9f03d19678) + */ + }; + + unsigned long set43[] = { +STORE, 140737488347136, 140737488351231, +STORE, 140734187720704, 140737488351231, +SNULL, 140734187724800, 140737488351231, +STORE, 140734187589632, 140734187724799, +STORE, 4194304, 6443007, +STORE, 4337664, 6443007, +STORE, 4194304, 4337663, +SNULL, 4337664, 6443007, +STORE, 6430720, 6443007, +STORE, 206158430208, 206160674815, +STORE, 206158569472, 206160674815, +STORE, 206158430208, 206158569471, +SNULL, 206158569472, 206160674815, +STORE, 206160662528, 206160670719, +STORE, 206160670720, 206160674815, +STORE, 140734188756992, 140734188765183, +STORE, 140734188740608, 140734188756991, +STORE, 140501948112896, 140501948116991, + }; + + int count = 0; + void *ptr = NULL; + + MA_STATE(mas, mt, 0, 0); + + mt_set_non_kernel(3); + check_erase2_testset(mt, set, ARRAY_SIZE(set)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set2, ARRAY_SIZE(set2)); + start = 140735933894656; + MT_BUG_ON(mt, !!mt_find(mt, &start, 140735933906943UL)); + mtree_destroy(mt); + + mt_set_non_kernel(2); + mt_init_flags(mt, 0); + check_erase2_testset(mt, set3, ARRAY_SIZE(set3)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, 0); + check_erase2_testset(mt, set4, ARRAY_SIZE(set4)); + rcu_read_lock(); + mas_for_each(&mas, entry, ULONG_MAX) { + if (xa_is_zero(entry)) + continue; + } + rcu_read_unlock(); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mt_set_non_kernel(100); + check_erase2_testset(mt, set5, ARRAY_SIZE(set5)); + rcu_barrier(); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set6, ARRAY_SIZE(set6)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set7, ARRAY_SIZE(set7)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set8, ARRAY_SIZE(set8)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set9, ARRAY_SIZE(set9)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set10, ARRAY_SIZE(set10)); + rcu_barrier(); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set11, ARRAY_SIZE(set11)); + rcu_barrier(); + mas_empty_area_rev(&mas, 12288, 140014592737280, 0x2000); + MT_BUG_ON(mt, mas.last != 140014592573439); + mtree_destroy(mt); + + mas_reset(&mas); + mas.tree = mt; + count = 0; + mas.index = 0; + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set12, ARRAY_SIZE(set12)); + rcu_barrier(); + mas_for_each(&mas, entry, ULONG_MAX) { + if (xa_is_zero(entry)) + continue; + BUG_ON(count > 12); + count++; + } + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set13, ARRAY_SIZE(set13)); + mtree_erase(mt, 140373516443648); + rcu_read_lock(); + mas_empty_area_rev(&mas, 0, 140373518663680, 4096); + rcu_read_unlock(); + mtree_destroy(mt); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set14, ARRAY_SIZE(set14)); + rcu_barrier(); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set15, ARRAY_SIZE(set15)); + rcu_barrier(); + mtree_destroy(mt); + + /* set16 was to find a bug on limit updating at slot 0. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set16, ARRAY_SIZE(set16)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 139921865637888, 0x6000); + MT_BUG_ON(mt, mas.last != 139921865547775); + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set17 found a bug in walking backwards and not counting nulls at + * the end. This could cause a gap to be missed if the null had any + * size. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set17, ARRAY_SIZE(set17)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 139953197334528, 0x1000); + MT_BUG_ON(mt, mas.last != 139953197322239); +/* MT_BUG_ON(mt, mas.index != 139953197318144); */ + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set18 found a bug in walking backwards and not setting the max from + * the node, but using the parent node. This was only an issue if the + * next slot in the parent had what we needed. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set18, ARRAY_SIZE(set18)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 140222972858368, 2215936); + MT_BUG_ON(mt, mas.last != 140222968475647); + /*MT_BUG_ON(mt, mas.index != 140222966259712); */ + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set19 found 2 bugs in prev. + * 1. If we hit root without finding anything, then there was an + * infinite loop. + * 2. The first ascending wasn't using the correct slot which may have + * caused missed entries. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set19, ARRAY_SIZE(set19)); + rcu_barrier(); + mas.index = 140656779083776; + entry = mas_find(&mas, ULONG_MAX); + MT_BUG_ON(mt, entry != xa_mk_value(140656779083776)); + entry = mas_prev(&mas, 0); + MT_BUG_ON(mt, entry != xa_mk_value(140656766251008)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + /* + * set20 found a bug in mas_may_move_gap due to the slot being + * overwritten during the __mas_add operation and setting it to zero. + */ + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set20, ARRAY_SIZE(set20)); + rcu_barrier(); + check_load(mt, 94849009414144, NULL); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set21, ARRAY_SIZE(set21)); + rcu_barrier(); + mt_validate(mt); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_set_non_kernel(999); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set22, ARRAY_SIZE(set22)); + rcu_barrier(); + mt_validate(mt); + ptr = mtree_load(mt, 140551363362816); + MT_BUG_ON(mt, ptr == mtree_load(mt, 140551363420159)); + mt_set_non_kernel(0); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set23, ARRAY_SIZE(set23)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set24, ARRAY_SIZE(set24)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set25, ARRAY_SIZE(set25)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* Split on NULL followed by delete - causes gap issues. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set26, ARRAY_SIZE(set26)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 140109042671616, 409600); + MT_BUG_ON(mt, mas.last != 140109040959487); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* Split on NULL followed by delete - causes gap issues. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set27, ARRAY_SIZE(set27)); + rcu_barrier(); + MT_BUG_ON(mt, 0 != mtree_load(mt, 140415537422336)); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set28, ARRAY_SIZE(set28)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 139918413357056, 2097152); + /* Search for the size of gap then align it (offset 0) */ + mas.index = (mas.last + 1 - 2097152 - 0) & (~2093056); + MT_BUG_ON(mt, mas.index != 139918401601536); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* This test found issues with retry moving rebalanced nodes so the + * incorrect parent pivot was updated. + */ + mt_set_non_kernel(999); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set29, ARRAY_SIZE(set29)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* This test found issues with deleting all entries in a node when + * surrounded by entries in the next nodes, then deleting the entries + * surrounding the node filled with deleted entries. + */ + mt_set_non_kernel(999); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set30, ARRAY_SIZE(set30)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* This test found an issue with deleting all entries in a node that was + * the end node and mas_gap incorrectly set next = curr, and curr = prev + * then moved next to the left, losing data. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set31, ARRAY_SIZE(set31)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set32, ARRAY_SIZE(set32)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + +/* + * mmap: empty_area_topdown: ffff88821c9cb600 Gap was found: + * mt 140582827569152 gap_end 140582869532672 + * mmap: window was 140583656296448 - 4096 size 134217728 + * mmap: mas.min 94133881868288 max 140582961786879 mas.last 140582961786879 + * mmap: mas.index 140582827569152 align mask 0 offset 0 + * mmap: rb_find_vma find on + * 140582827569152 => ffff88821c5bad00 (ffff88821c5bad00) + */ + + /* move gap failed due to an entirely empty node */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set33, ARRAY_SIZE(set33)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 140583656296448, 134217728); + MT_BUG_ON(mt, mas.last != 140583003750399); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* + * Incorrect gap in tree caused by mas_prev not setting the limits + * correctly while walking down. + */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set34, ARRAY_SIZE(set34)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* Empty leaf at the end of a parent caused incorrect gap. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set35, ARRAY_SIZE(set35)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mt_set_non_kernel(99); + /* Empty leaf at the end of a parent caused incorrect gap. */ + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set36, ARRAY_SIZE(set36)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set37, ARRAY_SIZE(set37)); + rcu_barrier(); + MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set38, ARRAY_SIZE(set38)); + rcu_barrier(); + MT_BUG_ON(mt, 0 != mtree_load(mt, 94637033459712)); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set39, ARRAY_SIZE(set39)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set40, ARRAY_SIZE(set40)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set41, ARRAY_SIZE(set41)); + rcu_barrier(); + mt_validate(mt); + mtree_destroy(mt); + + /* move gap failed due to an entirely empty node. */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set42, ARRAY_SIZE(set42)); + rcu_barrier(); + mas_empty_area_rev(&mas, 4096, 4052029440, 28672); + MT_BUG_ON(mt, mas.last != 4041211903); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); + + /* gap calc off by one */ + mt_set_non_kernel(99); + mas_reset(&mas); + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + check_erase2_testset(mt, set43, ARRAY_SIZE(set43)); + rcu_barrier(); + mt_set_non_kernel(0); + mt_validate(mt); + mtree_destroy(mt); +} +#endif + +/* End of VM testcases */ + +/* RCU stress testing */ + +/* RCU reader helper function */ +static void rcu_reader_register(struct rcu_test_struct2 *test) +{ + rcu_register_thread(); + uatomic_inc(&test->thread_count); + + while (!test->start) + usleep(test->pause * 100); +} + +static void rcu_reader_setup(struct rcu_reader_struct *reader, + unsigned int id, struct rcu_test_struct2 *test) +{ + reader->id = id; + reader->test = test; + reader->mod = reader->id % 10; + reader->del = (reader->mod + 1) % 10; + reader->flip = (reader->mod + 2) % 10; + reader->add = (reader->mod + 3) % 10; + reader->next = (reader->mod + 4) % 10; +} +/* RCU reader in increasing index */ +static void *rcu_reader_fwd(void *ptr) + +{ + struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; + struct rcu_test_struct2 *test = reader->test; + unsigned long index = reader->id; + bool toggled, modified, deleted, added; + int i; + void *entry, *prev = NULL; + MA_STATE(mas, test->mt, 0, 0); + + rcu_reader_register(test); + toggled = modified = deleted = added = false; + + while (!test->stop) { + i = 0; + /* mas_for_each ?*/ + rcu_read_lock(); + mas_set(&mas, test->index[index]); + mas_for_each(&mas, entry, test->last[index + 9]) { + unsigned long r_start, r_end, alt_start; + void *expected, *alt; + + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + + if (i == reader->del) { + if (!deleted) { + alt_start = test->index[index + reader->flip]; + /* delete occurred. */ + if (mas.index == alt_start) { + uatomic_inc(&test->seen_deleted); + deleted = true; + } + } + if (deleted) { + i = reader->flip; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + + if (!added && (i == reader->add)) { + alt_start = test->index[index + reader->next]; + if (mas.index == r_start) { + uatomic_inc(&test->seen_added); + added = true; + } else if (mas.index == alt_start) { + i = reader->next; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + + RCU_MT_BUG_ON(test, mas.index != r_start); + RCU_MT_BUG_ON(test, mas.last != r_end); + + if (i == reader->flip) { + alt = xa_mk_value(index + i + RCU_RANGE_COUNT); + if (prev) { + if (toggled && entry == expected) + uatomic_inc(&test->seen_toggle); + else if (!toggled && entry == alt) + uatomic_inc(&test->seen_toggle); + } + + if (entry == expected) + toggled = false; + else if (entry == alt) + toggled = true; + else { + printk("!!%lu-%lu -> %p not %p or %p\n", mas.index, mas.last, entry, expected, alt); + RCU_MT_BUG_ON(test, 1); + } + + prev = entry; + } else if (i == reader->mod) { + alt = xa_mk_value(index + i * 2 + 1 + + RCU_RANGE_COUNT); + if (entry != expected) { + if (!modified) + uatomic_inc(&test->seen_modified); + modified = true; + } else { + if (modified) + uatomic_inc(&test->seen_modified); + modified = false; + } + + if (modified) + RCU_MT_BUG_ON(test, entry != alt); + + } else { + if (entry != expected) + printk("!!%lu-%lu -> %p not %p\n", mas.index, mas.last, entry, expected); + RCU_MT_BUG_ON(test, entry != expected); + } + + i++; + } + rcu_read_unlock(); + usleep(test->pause); + } + + rcu_unregister_thread(); + return NULL; +} +/* RCU reader in decreasing index */ +static void *rcu_reader_rev(void *ptr) +{ + struct rcu_reader_struct *reader = (struct rcu_reader_struct *)ptr; + struct rcu_test_struct2 *test = reader->test; + unsigned long index = reader->id; + bool toggled, modified, deleted, added; + int i; + void *prev = NULL; + MA_STATE(mas, test->mt, 0, 0); + + rcu_reader_register(test); + toggled = modified = deleted = added = false; + + + while (!test->stop) { + void *entry; + + i = 9; + mas_set(&mas, test->index[index + i]); + + rcu_read_lock(); + while (i--) { + unsigned long r_start, r_end, alt_start; + void *expected, *alt; + int line = __LINE__; + + entry = mas_prev(&mas, test->index[index]); + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + + if (i == reader->del) { + alt_start = test->index[index + reader->mod]; + if (mas.index == alt_start) { + line = __LINE__; + if (!deleted) + uatomic_inc(&test->seen_deleted); + deleted = true; + } + if (deleted) { + line = __LINE__; + i = reader->mod; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + if (!added && (i == reader->add)) { + alt_start = test->index[index + reader->flip]; + if (mas.index == r_start) { + line = __LINE__; + uatomic_inc(&test->seen_added); + added = true; + } else if (mas.index == alt_start) { + line = __LINE__; + i = reader->flip; + r_start = test->index[index + i]; + r_end = test->last[index + i]; + expected = xa_mk_value(r_start); + } + } + + if (i == reader->mod) + line = __LINE__; + else if (i == reader->flip) + line = __LINE__; + + if (mas.index != r_start) { + alt = xa_mk_value(index + i * 2 + 1 + + RCU_RANGE_COUNT); + mt_dump(test->mt); + printk("Error: %lu-%lu %p != %lu-%lu %p %p line %d i %d\n", + mas.index, mas.last, entry, + r_start, r_end, expected, alt, + line, i); + } + RCU_MT_BUG_ON(test, mas.index != r_start); + RCU_MT_BUG_ON(test, mas.last != r_end); + + if (i == reader->mod) { + alt = xa_mk_value(index + i * 2 + 1 + + RCU_RANGE_COUNT); + + if (entry != expected) { + if (!modified) + uatomic_inc(&test->seen_modified); + modified = true; + } else { + if (modified) + uatomic_inc(&test->seen_modified); + modified = false; + } + if (modified) + RCU_MT_BUG_ON(test, entry != alt); + + + } else if (i == reader->flip) { + alt = xa_mk_value(index + i + + RCU_RANGE_COUNT); + if (prev) { + if (toggled && entry == expected) + uatomic_inc(&test->seen_toggle); + else if (!toggled && entry == alt) + uatomic_inc(&test->seen_toggle); + } + + if (entry == expected) + toggled = false; + else if (entry == alt) + toggled = true; + else { + printk("%lu-%lu %p != %p or %p\n", + mas.index, mas.last, entry, + expected, alt); + RCU_MT_BUG_ON(test, 1); + } + + prev = entry; + } else { + if (entry != expected) + printk("%lu-%lu %p != %p\n", mas.index, + mas.last, entry, expected); + RCU_MT_BUG_ON(test, entry != expected); + } + } + rcu_read_unlock(); + usleep(test->pause); + } + + rcu_unregister_thread(); + return NULL; +} + +static void rcu_stress_rev(struct maple_tree *mt, struct rcu_test_struct2 *test, + int count, struct rcu_reader_struct *test_reader) +{ + int i, j = 10000; + bool toggle = true; + + test->start = true; /* Release the hounds! */ + usleep(5); + + while (j--) { + toggle = !toggle; + i = count; + while (i--) { + unsigned long start, end; + struct rcu_reader_struct *this = &test_reader[i]; + + /* Mod offset */ + if (j == 600) { + start = test->index[this->id + this->mod]; + end = test->last[this->id + this->mod]; + mtree_store_range(mt, start, end, + xa_mk_value(this->id + this->mod * 2 + + 1 + RCU_RANGE_COUNT), + GFP_KERNEL); + } + + /* Toggle */ + if (!(j % 5)) { + start = test->index[this->id + this->flip]; + end = test->last[this->id + this->flip]; + mtree_store_range(mt, start, end, + xa_mk_value((toggle ? start : + this->id + this->flip + + RCU_RANGE_COUNT)), + GFP_KERNEL); + } + + /* delete */ + if (j == 400) { + start = test->index[this->id + this->del]; + end = test->last[this->id + this->del]; + mtree_store_range(mt, start, end, NULL, GFP_KERNEL); + } + + /* add */ + if (j == 500) { + start = test->index[this->id + this->add]; + end = test->last[this->id + this->add]; + mtree_store_range(mt, start, end, + xa_mk_value(start), GFP_KERNEL); + } + } + usleep(test->pause); + /* If a test fails, don't flood the console */ + if (test->stop) + break; + } +} + +static void rcu_stress_fwd(struct maple_tree *mt, struct rcu_test_struct2 *test, + int count, struct rcu_reader_struct *test_reader) +{ + int j, i; + bool toggle = true; + + test->start = true; /* Release the hounds! */ + usleep(5); + for (j = 0; j < 10000; j++) { + toggle = !toggle; + for (i = 0; i < count; i++) { + unsigned long start, end; + struct rcu_reader_struct *this = &test_reader[i]; + + /* Mod offset */ + if (j == 600) { + start = test->index[this->id + this->mod]; + end = test->last[this->id + this->mod]; + mtree_store_range(mt, start, end, + xa_mk_value(this->id + this->mod * 2 + + 1 + RCU_RANGE_COUNT), + GFP_KERNEL); + } + + /* Toggle */ + if (!(j % 5)) { + start = test->index[this->id + this->flip]; + end = test->last[this->id + this->flip]; + mtree_store_range(mt, start, end, + xa_mk_value((toggle ? start : + this->id + this->flip + + RCU_RANGE_COUNT)), + GFP_KERNEL); + } + + /* delete */ + if (j == 400) { + start = test->index[this->id + this->del]; + end = test->last[this->id + this->del]; + mtree_store_range(mt, start, end, NULL, GFP_KERNEL); + } + + /* add */ + if (j == 500) { + start = test->index[this->id + this->add]; + end = test->last[this->id + this->add]; + mtree_store_range(mt, start, end, + xa_mk_value(start), GFP_KERNEL); + } + } + usleep(test->pause); + /* If a test fails, don't flood the console */ + if (test->stop) + break; + } +} + +/* + * This is to check: + * 1. Range that is not ever present + * 2. Range that is always present + * 3. Things being added but not removed. + * 4. Things being removed but not added. + * 5. Things are being added and removed, searches my succeed or fail + * + * This sets up two readers for every 10 entries; one forward and one reverse + * reading. + */ +static void rcu_stress(struct maple_tree *mt, bool forward) +{ + unsigned int count, i; + unsigned long r, seed; + pthread_t readers[RCU_RANGE_COUNT / 5]; + struct rcu_test_struct2 test; + struct rcu_reader_struct test_reader[RCU_RANGE_COUNT / 5]; + void *(*function)(void *); + + /* Test setup */ + test.mt = mt; + test.pause = 5; + test.seen_toggle = 0; + test.seen_deleted = 0; + test.seen_added = 0; + test.seen_modified = 0; + test.thread_count = 0; + test.start = test.stop = false; + seed = time(NULL); + srand(seed); + for (i = 0; i < RCU_RANGE_COUNT; i++) { + r = seed + rand(); + mtree_store_range(mt, seed, r, + xa_mk_value(seed), GFP_KERNEL); + + /* Record start and end of entry */ + test.index[i] = seed; + test.last[i] = r; + seed = 1 + r + rand() % 10; + } + + i = count = ARRAY_SIZE(readers); + while (i--) { + unsigned long id; + + id = i / 2 * 10; + if (i % 2) + function = rcu_reader_fwd; + else + function = rcu_reader_rev; + + rcu_reader_setup(&test_reader[i], id, &test); + if (pthread_create(&readers[i], NULL, *function, + &test_reader[i])) { + perror("creating reader thread"); + exit(1); + } + } + + for (i = 0; i < ARRAY_SIZE(readers); i++) { + struct rcu_reader_struct *this = &test_reader[i]; + int add = this->id + this->add; + + /* Remove add entries from the tree for later addition */ + mtree_store_range(mt, test.index[add], test.last[add], + NULL, GFP_KERNEL); + } + + mt_set_in_rcu(mt); + do { + usleep(5); + } while (test.thread_count > ARRAY_SIZE(readers)); + + if (forward) + rcu_stress_fwd(mt, &test, count, test_reader); + else + rcu_stress_rev(mt, &test, count, test_reader); + + test.stop = true; + while (count--) + pthread_join(readers[count], NULL); + + mt_validate(mt); +} + + +struct rcu_test_struct { + struct maple_tree *mt; /* the maple tree */ + int count; /* Number of times to check value(s) */ + unsigned long index; /* The first index to check */ + void *entry1; /* The first entry value */ + void *entry2; /* The second entry value */ + void *entry3; /* The third entry value */ + + bool update_2; + bool update_3; + unsigned long range_start; + unsigned long range_end; + unsigned int loop_sleep; + unsigned int val_sleep; + + unsigned int failed; /* failed detection for other threads */ + unsigned int seen_entry2; /* Number of threads that have seen the new value */ + unsigned int seen_entry3; /* Number of threads that have seen the new value */ + unsigned int seen_both; /* Number of threads that have seen both new values */ + unsigned int seen_toggle; + unsigned int seen_added; + unsigned int seen_removed; + unsigned long last; /* The end of the range to write. */ + + unsigned long removed; /* The index of the removed entry */ + unsigned long added; /* The index of the removed entry */ + unsigned long toggle; /* The index of the removed entry */ +}; + +static inline +int eval_rcu_entry(struct rcu_test_struct *test, void *entry, bool *update_2, + bool *update_3) +{ + if (entry == test->entry1) + return 0; + + if (entry == test->entry2) { + if (!(*update_2)) { + uatomic_inc(&test->seen_entry2); + *update_2 = true; + if (update_3) + uatomic_inc(&test->seen_both); + } + return 0; + } + + if (entry == test->entry3) { + if (!(*update_3)) { + uatomic_inc(&test->seen_entry3); + *update_3 = true; + if (update_2) + uatomic_inc(&test->seen_both); + } + return 0; + } + + return 1; +} + +/* + * rcu_val() - Read a given value in the tree test->count times using the + * regular API + * + * @ptr: The pointer to the rcu_test_struct + */ +static void *rcu_val(void *ptr) +{ + struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; + unsigned long count = test->count; + bool update_2 = false; + bool update_3 = false; + void *entry; + + rcu_register_thread(); + while (count--) { + usleep(test->val_sleep); + /* + * No locking required, regular API locking is handled in the + * maple tree code + */ + entry = mtree_load(test->mt, test->index); + MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, &update_2, + &update_3)); + } + rcu_unregister_thread(); + return NULL; +} + +/* + * rcu_loop() - Loop over a section of the maple tree, checking for an expected + * value using the advanced API + * + * @ptr - The pointer to the rcu_test_struct + */ +static void *rcu_loop(void *ptr) +{ + struct rcu_test_struct *test = (struct rcu_test_struct *)ptr; + unsigned long count = test->count; + void *entry, *expected; + bool update_2 = false; + bool update_3 = false; + MA_STATE(mas, test->mt, test->range_start, test->range_start); + + rcu_register_thread(); + + /* + * Loop through the test->range_start - test->range_end test->count + * times + */ + while (count--) { + usleep(test->loop_sleep); + rcu_read_lock(); + mas_for_each(&mas, entry, test->range_end) { + /* The expected value is based on the start range. */ + expected = xa_mk_value(mas.index ? mas.index / 10 : 0); + + /* Out of the interesting range */ + if (mas.index < test->index || mas.index > test->last) { + if (entry != expected) { + printk("%lx - %lx = %p not %p\n", + mas.index, mas.last, entry, expected); + } + MT_BUG_ON(test->mt, entry != expected); + continue; + } + + if (entry == expected) + continue; /* Not seen. */ + + /* In the interesting range */ + MT_BUG_ON(test->mt, eval_rcu_entry(test, entry, + &update_2, + &update_3)); + } + rcu_read_unlock(); + mas_set(&mas, test->range_start); + } + + rcu_unregister_thread(); + return NULL; +} + +static noinline +void run_check_rcu(struct maple_tree *mt, struct rcu_test_struct *vals) +{ + + int i; + void *(*function)(void *); + pthread_t readers[20]; + + mt_set_in_rcu(mt); + MT_BUG_ON(mt, !mt_in_rcu(mt)); + + for (i = 0; i < ARRAY_SIZE(readers); i++) { + if (i % 2) + function = rcu_loop; + else + function = rcu_val; + + if (pthread_create(&readers[i], NULL, *function, vals)) { + perror("creating reader thread"); + exit(1); + } + } + + usleep(5); /* small yield to ensure all threads are at least started. */ + mtree_store_range(mt, vals->index, vals->last, vals->entry2, + GFP_KERNEL); + while (i--) + pthread_join(readers[i], NULL); + + /* Make sure the test caught at least one update. */ + MT_BUG_ON(mt, !vals->seen_entry2); +} + +static noinline +void run_check_rcu_slowread(struct maple_tree *mt, struct rcu_test_struct *vals) +{ + + int i; + void *(*function)(void *); + pthread_t readers[20]; + unsigned int index = vals->index; + + mt_set_in_rcu(mt); + MT_BUG_ON(mt, !mt_in_rcu(mt)); + + for (i = 0; i < ARRAY_SIZE(readers); i++) { + if (i % 2) + function = rcu_loop; + else + function = rcu_val; + + if (pthread_create(&readers[i], NULL, *function, vals)) { + perror("creating reader thread"); + exit(1); + } + } + + usleep(5); /* small yield to ensure all threads are at least started. */ + + while (index <= vals->last) { + mtree_store(mt, index, + (index % 2 ? vals->entry2 : vals->entry3), + GFP_KERNEL); + index++; + usleep(5); + } + + while (i--) + pthread_join(readers[i], NULL); + + /* Make sure the test caught at least one update. */ + MT_BUG_ON(mt, !vals->seen_entry2); + MT_BUG_ON(mt, !vals->seen_entry3); + MT_BUG_ON(mt, !vals->seen_both); +} +static noinline void check_rcu_simulated(struct maple_tree *mt) +{ + unsigned long i, nr_entries = 1000; + unsigned long target = 4320; + unsigned long val = 0xDEAD; + + MA_STATE(mas_writer, mt, 0, 0); + MA_STATE(mas_reader, mt, target, target); + + rcu_register_thread(); + + mt_set_in_rcu(mt); + mas_lock(&mas_writer); + for (i = 0; i <= nr_entries; i++) { + mas_writer.index = i * 10; + mas_writer.last = i * 10 + 5; + mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); + } + mas_unlock(&mas_writer); + + /* Overwrite one entry with a new value. */ + mas_set_range(&mas_writer, target, target + 5); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + + /* Overwrite 1/2 the entry */ + mas_set_range(&mas_writer, target, target + 2); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite last 1/2 the entry */ + mas_set_range(&mas_writer, target + 2, target + 5); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + rcu_read_unlock(); + + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite more than the entry */ + mas_set_range(&mas_writer, target - 5, target + 15); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite more than the node. */ + mas_set_range(&mas_writer, target - 400, target + 400); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + mas_reset(&mas_reader); + + /* Overwrite the tree */ + mas_set_range(&mas_writer, 0, ULONG_MAX); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Clear out tree & recreate it */ + mas_lock(&mas_writer); + mas_set_range(&mas_writer, 0, ULONG_MAX); + mas_store_gfp(&mas_writer, NULL, GFP_KERNEL); + mas_set_range(&mas_writer, 0, 0); + for (i = 0; i <= nr_entries; i++) { + mas_writer.index = i * 10; + mas_writer.last = i * 10 + 5; + mas_store_gfp(&mas_writer, xa_mk_value(i), GFP_KERNEL); + } + mas_unlock(&mas_writer); + + /* next check */ + /* Overwrite one entry with a new value. */ + mas_reset(&mas_reader); + mas_set_range(&mas_writer, target, target + 5); + mas_set_range(&mas_reader, target, target); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_prev(&mas_reader, 0); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_next(&mas_reader, ULONG_MAX) != xa_mk_value(val)); + rcu_read_unlock(); + + /* Restore value. */ + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(target/10), GFP_KERNEL); + mas_unlock(&mas_writer); + + /* prev check */ + /* Overwrite one entry with a new value. */ + mas_reset(&mas_reader); + mas_set_range(&mas_writer, target, target + 5); + mas_set_range(&mas_reader, target, target); + rcu_read_lock(); + MT_BUG_ON(mt, mas_walk(&mas_reader) != xa_mk_value(target/10)); + mas_next(&mas_reader, ULONG_MAX); + mas_lock(&mas_writer); + mas_store_gfp(&mas_writer, xa_mk_value(val), GFP_KERNEL); + mas_unlock(&mas_writer); + MT_BUG_ON(mt, mas_prev(&mas_reader, 0) != xa_mk_value(val)); + rcu_read_unlock(); + + rcu_unregister_thread(); +} + +static noinline void check_rcu_threaded(struct maple_tree *mt) +{ + unsigned long i, nr_entries = 1000; + struct rcu_test_struct vals; + + vals.val_sleep = 200; + vals.loop_sleep = 110; + + rcu_register_thread(); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + /* Store across several slots. */ + vals.count = 1000; + vals.mt = mt; + vals.index = 8650; + vals.last = 8666; + vals.entry1 = xa_mk_value(865); + vals.entry2 = xa_mk_value(8650); + vals.entry3 = xa_mk_value(8650); + vals.range_start = 0; + vals.range_end = ULONG_MAX; + vals.seen_entry2 = 0; + vals.seen_entry3 = 0; + + run_check_rcu(mt, &vals); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + /* 4390-4395: value 439 (0x1b7) [0x36f] */ + /* Store across several slots. */ + /* Spanning store. */ + vals.count = 10000; + vals.mt = mt; + vals.index = 4390; + vals.last = 4398; + vals.entry1 = xa_mk_value(4390); + vals.entry2 = xa_mk_value(439); + vals.entry3 = xa_mk_value(439); + vals.seen_entry2 = 0; + vals.range_start = 4316; + vals.range_end = 5035; + run_check_rcu(mt, &vals); + mtree_destroy(mt); + + + /* Forward writer for rcu stress */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + rcu_stress(mt, true); + mtree_destroy(mt); + + /* Reverse writer for rcu stress */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + rcu_stress(mt, false); + mtree_destroy(mt); + + /* Slow reader test with spanning store. */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= nr_entries; i++) + mtree_store_range(mt, i*10, i*10 + 5, + xa_mk_value(i), GFP_KERNEL); + + /* 4390-4395: value 439 (0x1b7) [0x36f] */ + /* Store across several slots. */ + /* Spanning store. */ + vals.count = 15000; + vals.mt = mt; + vals.index = 4390; + vals.last = 4398; + vals.entry1 = xa_mk_value(4390); + vals.entry2 = xa_mk_value(439); + vals.entry3 = xa_mk_value(4391); + vals.seen_toggle = 0; + vals.seen_added = 0; + vals.seen_removed = 0; + vals.range_start = 4316; + vals.range_end = 5035; + vals.removed = 4360; + vals.added = 4396; + vals.toggle = 4347; + vals.val_sleep = 400; + vals.loop_sleep = 200; + vals.seen_entry2 = 0; + vals.seen_entry3 = 0; + vals.seen_both = 0; + vals.entry3 = xa_mk_value(438); + + run_check_rcu_slowread(mt, &vals); + rcu_unregister_thread(); +} +/* End of RCU stress testing */ + +/* Check tree structure by depth first searching */ +static void mas_dfs_preorder(struct ma_state *mas) +{ + + struct maple_enode *prev; + unsigned char end, slot = 0; + + if (mas->node == MAS_START) { + mas_start(mas); + return; + } + + if (mte_is_leaf(mas->node) && mte_is_root(mas->node)) + goto done; + +walk_up: + end = mas_data_end(mas); + if (mte_is_leaf(mas->node) || + (slot > end)) { + if (mte_is_root(mas->node)) + goto done; + + slot = mte_parent_slot(mas->node) + 1; + mas_ascend(mas); + goto walk_up; + } + + prev = mas->node; + mas->node = mas_get_slot(mas, slot); + if (!mas->node || slot > end) { + if (mte_is_root(prev)) + goto done; + + mas->node = prev; + slot = mte_parent_slot(mas->node) + 1; + mas_ascend(mas); + goto walk_up; + } + + return; +done: + mas->node = MAS_NONE; +} + + +static void check_dfs_preorder(struct maple_tree *mt) +{ + unsigned long e, count = 0, max = 1000; + + MA_STATE(mas, mt, 0, 0); + + if (MAPLE_32BIT) + e = 37; + else + e = 74; + + check_seq(mt, max, false); + do { + count++; + mas_dfs_preorder(&mas); + } while (!mas_is_none(&mas)); + MT_BUG_ON(mt, count != e); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_reset(&mas); + count = 0; + if (!MAPLE_32BIT) + e = 77; + + check_seq(mt, max, false); + do { + count++; + mas_dfs_preorder(&mas); + } while (!mas_is_none(&mas)); + /*printk("count %lu\n", count); */ + MT_BUG_ON(mt, count != e); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_reset(&mas); + count = 0; + check_rev_seq(mt, max, false); + do { + count++; + mas_dfs_preorder(&mas); + } while (!mas_is_none(&mas)); + /*printk("count %lu\n", count); */ + MT_BUG_ON(mt, count != e); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + mas_reset(&mas); + mt_zero_nr_tallocated(); + mt_set_non_kernel(200); + mas_expected_entries(&mas, max); + for (count = 0; count <= max; count++) { + mas.index = mas.last = count; + mas_store(&mas, xa_mk_value(count)); + MT_BUG_ON(mt, mas_is_err(&mas)); + } + mas_destroy(&mas); + rcu_barrier(); + /* + * pr_info(" ->seq test of 0-%lu %luK in %d active (%d total)\n", + * max, mt_get_alloc_size()/1024, mt_nr_allocated(), + * mt_nr_tallocated()); + */ + +} +/* End of depth first search tests */ + +/* Preallocation testing */ +static noinline void check_prealloc(struct maple_tree *mt) +{ + unsigned long i, max = 100; + unsigned long allocated; + unsigned char height; + struct maple_node *mn; + void *ptr = check_prealloc; + MA_STATE(mas, mt, 10, 20); + + mt_set_non_kernel(1000); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); + ma_free_rcu(mn); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + ma_free_rcu(mn); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mn = mas_pop_node(&mas); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated - 1); + mas_push_node(&mas, mn); + MT_BUG_ON(mt, mas_allocated(&mas) != allocated); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + mas_destroy(&mas); + allocated = mas_allocated(&mas); + MT_BUG_ON(mt, allocated != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mt_set_non_kernel(1); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated != 0); + mas_destroy(&mas); + + + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL) != 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated == 0); + MT_BUG_ON(mt, allocated != 1 + height * 3); + mas_store_prealloc(&mas, ptr); + MT_BUG_ON(mt, mas_allocated(&mas) != 0); + mt_set_non_kernel(1); + MT_BUG_ON(mt, mas_preallocate(&mas, ptr, GFP_KERNEL & GFP_NOWAIT) == 0); + allocated = mas_allocated(&mas); + height = mas_mt_height(&mas); + MT_BUG_ON(mt, allocated != 0); +} +/* End of preallocation testing */ + +/* Spanning writes, writes that span nodes and layers of the tree */ +static noinline void check_spanning_write(struct maple_tree *mt) +{ + unsigned long i, max = 5000; + MA_STATE(mas, mt, 1200, 2380); + + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 1205); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + for (i = 1; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mtree_lock(mt); + mas_set_range(&mas, 9, 50006); /* Will expand to 0 - ULONG_MAX */ + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 1205); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mt_validate(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a right cousin rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 0, 12900); /* Spans more than 2 levels */ + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 1205); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test non-alloc tree spanning store */ + mt_init_flags(mt, 0); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 0, 300); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 15); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a right sibling rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 0, 12865); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 15); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a left sibling rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 90, 13665); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 95); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store that requires a left cousin rebalance */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 46805, 49995); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 46815); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* + * Test spanning store that requires a left cousin rebalance all the way + * to root + */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mas_set_range(&mas, 32395, 49995); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 46815); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* + * Test spanning store that requires a right cousin rebalance all the + * way to root + */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + mas_set_range(&mas, 38875, 43190); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 38900); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning store ending at full node (depth 2)*/ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + mtree_lock(mt); + mas_set(&mas, 47606); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + mas_set(&mas, 47607); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + mas_set(&mas, 47608); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + mas_set(&mas, 47609); + mas_store_gfp(&mas, check_spanning_write, GFP_KERNEL); + /* Ensure the parent node is full */ + mas_ascend(&mas); + MT_BUG_ON(mt, (mas_data_end(&mas)) != mt_slot_count(mas.node) - 1); + mas_set_range(&mas, 11516, 48940); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mtree_unlock(mt); + mtree_destroy(mt); + + /* Test spanning write with many levels of no siblings */ + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + mas_set_range(&mas, 43200, 49999); + mtree_lock(mt); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mas_set(&mas, 43200); + MT_BUG_ON(mt, mas_walk(&mas) != NULL); + mtree_unlock(mt); + mtree_destroy(mt); + + mt_init_flags(mt, MT_FLAGS_ALLOC_RANGE); + for (i = 0; i <= 100; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + + mtree_lock(mt); + mas_set_range(&mas, 76, 875); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + mtree_unlock(mt); +} +/* End of spanning write testing */ + +/* Writes to a NULL area that are adjacent to other NULLs */ +static noinline void check_null_expand(struct maple_tree *mt) +{ + unsigned long i, max = 100; + unsigned char data_end; + MA_STATE(mas, mt, 959, 959); + + for (i = 0; i <= max; i++) + mtree_test_store_range(mt, i * 10, i * 10 + 5, &i); + /* Test expanding null at start. */ + mas_lock(&mas); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 959, 963); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 963) != NULL); + MT_BUG_ON(mt, data_end != mas_data_end(&mas)); + + /* Test expanding null at end. */ + mas_set(&mas, 880); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 884, 887); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 884) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 889) != NULL); +#if CONFIG_64BIT + MT_BUG_ON(mt, data_end != mas_data_end(&mas)); +#endif + + /* Test expanding null at start and end. */ + mas_set(&mas, 890); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 900, 905); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 899) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 900) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 905) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 906) != NULL); +#if CONFIG_64BIT + MT_BUG_ON(mt, data_end - 2 != mas_data_end(&mas)); +#endif + + /* Test expanding null across multiple slots. */ + mas_set(&mas, 800); + mas_walk(&mas); + data_end = mas_data_end(&mas); + mas_set_range(&mas, 810, 825); + mas_store_gfp(&mas, NULL, GFP_KERNEL); + MT_BUG_ON(mt, mtree_load(mt, 809) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 810) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 825) != NULL); + MT_BUG_ON(mt, mtree_load(mt, 826) != NULL); +#if CONFIG_64BIT + MT_BUG_ON(mt, data_end - 4 != mas_data_end(&mas)); +#endif + mas_unlock(&mas); +} +/* End of NULL area expansions */ + +/* Checking for no memory is best done outside the kernel */ +static noinline void check_nomem(struct maple_tree *mt) +{ + MA_STATE(ms, mt, 1, 1); + + MT_BUG_ON(mt, !mtree_empty(mt)); + /* Ensure no bypassing of allocation failures */ + mt_set_non_kernel(0); + + /* Storing something at 1 requires memory allocation */ + MT_BUG_ON(mt, mtree_insert(mt, 1, &ms, GFP_ATOMIC) != -ENOMEM); + /* Storing something at 0 does not */ + MT_BUG_ON(mt, mtree_insert(mt, 0, &ms, GFP_ATOMIC) != 0); + + /* + * Simulate two threads racing; the first one fails to allocate + * memory to insert an entry at 1, then the second one succeeds + * in allocating memory to insert an entry at 2. The first one + * then needs to free the node it allocated. LeakSanitizer will + * notice this, as will the 'nr_allocated' debugging aid in the + * userspace test suite. + */ + mtree_lock(mt); + mas_store(&ms, &ms); /* insert 1 -> &ms, fails. */ + MT_BUG_ON(mt, ms.node != MA_ERROR(-ENOMEM)); + mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ + MT_BUG_ON(mt, ms.node != MAS_START); + mtree_unlock(mt); + MT_BUG_ON(mt, mtree_insert(mt, 2, mt, GFP_KERNEL) != 0); + mtree_lock(mt); + mas_store(&ms, &ms); /* insert 1 -> &ms */ + mas_nomem(&ms, GFP_KERNEL); /* Node allocated in here. */ + mtree_unlock(mt); + mtree_destroy(mt); +} + +static noinline void check_locky(struct maple_tree *mt) +{ + MA_STATE(ms, mt, 2, 2); + MA_STATE(reader, mt, 2, 2); + + mt_set_non_kernel(2); + mt_set_in_rcu(mt); + mas_lock(&ms); + mas_store(&ms, &ms); + mas_set_range(&ms, 1, 3); + mas_store(&ms, &reader); + mas_unlock(&ms); + mt_clear_in_rcu(mt); +} + +extern void test_kmem_cache_bulk(void); + void farmer_tests(void) { struct maple_node *node; @@ -39,6 +35756,59 @@ void farmer_tests(void) mt_dump(&tree); ma_free_rcu(node); + + /* Check things that will make lockdep angry */ + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_locky(&tree); + mtree_destroy(&tree); + test_kmem_cache_bulk(); + + mt_init_flags(&tree, 0); + check_dfs_preorder(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_prealloc(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_spanning_write(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_null_expand(&tree); + mtree_destroy(&tree); + + /* RCU testing */ + mt_init_flags(&tree, 0); + check_erase_testset(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, 0); + check_new_node(&tree); + mtree_destroy(&tree); + + if (!MAPLE_32BIT) { + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_rcu_simulated(&tree); + mtree_destroy(&tree); + + mt_init_flags(&tree, MT_FLAGS_ALLOC_RANGE); + check_rcu_threaded(&tree); + mtree_destroy(&tree); + } + + +#if defined(CONFIG_64BIT) + /* Captures from VMs that found previous errors */ + mt_init_flags(&tree, 0); + check_erase2_sets(&tree); + mtree_destroy(&tree); +#endif + + + /* No memory handling */ + check_nomem(&tree); } void maple_tree_tests(void) -- GitLab From 8625147cafaa9ba74713d682f5185eb62cb2aedb Mon Sep 17 00:00:00 2001 From: James Houghton Date: Tue, 18 Oct 2022 20:01:25 +0000 Subject: [PATCH 1249/1927] hugetlbfs: don't delete error page from pagecache This change is very similar to the change that was made for shmem [1], and it solves the same problem but for HugeTLBFS instead. Currently, when poison is found in a HugeTLB page, the page is removed from the page cache. That means that attempting to map or read that hugepage in the future will result in a new hugepage being allocated instead of notifying the user that the page was poisoned. As [1] states, this is effectively memory corruption. The fix is to leave the page in the page cache. If the user attempts to use a poisoned HugeTLB page with a syscall, the syscall will fail with EIO, the same error code that shmem uses. For attempts to map the page, the thread will get a BUS_MCEERR_AR SIGBUS. [1]: commit a76054266661 ("mm: shmem: don't truncate page if memory failure happens") Link: https://lkml.kernel.org/r/20221018200125.848471-1-jthoughton@google.com Signed-off-by: James Houghton Reviewed-by: Mike Kravetz Reviewed-by: Naoya Horiguchi Tested-by: Naoya Horiguchi Reviewed-by: Yang Shi Cc: Axel Rasmussen Cc: James Houghton Cc: Miaohe Lin Cc: Muchun Song Cc: Signed-off-by: Andrew Morton --- fs/hugetlbfs/inode.c | 13 ++++++------- mm/hugetlb.c | 4 ++++ mm/memory-failure.c | 5 ++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index dd54f67e47fdf..df7772335dc0e 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -328,6 +328,12 @@ static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to) } else { unlock_page(page); + if (PageHWPoison(page)) { + put_page(page); + retval = -EIO; + break; + } + /* * We have the page, copy it to user space buffer. */ @@ -1111,13 +1117,6 @@ static int hugetlbfs_migrate_folio(struct address_space *mapping, static int hugetlbfs_error_remove_page(struct address_space *mapping, struct page *page) { - struct inode *inode = mapping->host; - pgoff_t index = page->index; - - hugetlb_delete_from_page_cache(page); - if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1))) - hugetlb_fix_reserve_counts(inode); - return 0; } diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 546df97c31e4c..e48f8ef45b17f 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6111,6 +6111,10 @@ int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, ptl = huge_pte_lock(h, dst_mm, dst_pte); + ret = -EIO; + if (PageHWPoison(page)) + goto out_release_unlock; + /* * We allow to overwrite a pte marker: consider when both MISSING|WP * registered, we firstly wr-protect a none pte which has no page cache diff --git a/mm/memory-failure.c b/mm/memory-failure.c index 145bb561ddb3a..bead6bccc7f28 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -1080,6 +1080,7 @@ static int me_huge_page(struct page_state *ps, struct page *p) int res; struct page *hpage = compound_head(p); struct address_space *mapping; + bool extra_pins = false; if (!PageHuge(hpage)) return MF_DELAYED; @@ -1087,6 +1088,8 @@ static int me_huge_page(struct page_state *ps, struct page *p) mapping = page_mapping(hpage); if (mapping) { res = truncate_error_page(hpage, page_to_pfn(p), mapping); + /* The page is kept in page cache. */ + extra_pins = true; unlock_page(hpage); } else { unlock_page(hpage); @@ -1104,7 +1107,7 @@ static int me_huge_page(struct page_state *ps, struct page *p) } } - if (has_extra_refcount(ps, p, false)) + if (has_extra_refcount(ps, p, extra_pins)) res = MF_FAILED; return res; -- GitLab From cc674ab3c0188002917c8a2c28e4424131f1fd7e Mon Sep 17 00:00:00 2001 From: Li Zetao Date: Fri, 28 Oct 2022 15:37:17 +0800 Subject: [PATCH 1250/1927] mm/mmap: fix memory leak in mmap_region() There is a memory leak reported by kmemleak: unreferenced object 0xffff88817231ce40 (size 224): comm "mount.cifs", pid 19308, jiffies 4295917571 (age 405.880s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 60 c0 b2 00 81 88 ff ff 98 83 01 42 81 88 ff ff `..........B.... backtrace: [] __alloc_file+0x21/0x250 [] alloc_empty_file+0x41/0xf0 [] alloc_file+0x59/0x710 [] alloc_file_pseudo+0x154/0x210 [] __shmem_file_setup+0xff/0x2a0 [] shmem_zero_setup+0x8d/0x160 [] mmap_region+0x1075/0x19d0 [] do_mmap+0x727/0x1110 [] vm_mmap_pgoff+0x112/0x1e0 [] do_syscall_64+0x35/0x80 [] entry_SYSCALL_64_after_hwframe+0x46/0xb0 The root cause was traced to an error handing path in mmap_region() when arch_validate_flags() or mas_preallocate() fails. In the shared anonymous mapping sence, vma will be setuped and mapped with a new shared anonymous file via shmem_zero_setup(). So in this case, the file resource needs to be released. Fix it by calling fput(vma->vm_file) and unmap_region() when arch_validate_flags() or mas_preallocate() returns an error in the shared anonymous mapping sence. Link: https://lkml.kernel.org/r/20221028073717.1179380-1-lizetao1@huawei.com Fixes: d4af56c5c7c6 ("mm: start tracking VMAs with maple tree") Fixes: c462ac288f2c ("mm: Introduce arch_validate_flags()") Signed-off-by: Li Zetao Reviewed-by: Liam R. Howlett Signed-off-by: Andrew Morton --- mm/mmap.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mm/mmap.c b/mm/mmap.c index 2def55555e05f..c3c5c1d6103d1 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -2674,6 +2674,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr, error = -EINVAL; if (file) goto close_and_free_vma; + else if (vma->vm_file) + goto unmap_and_free_vma; else goto free_vma; } @@ -2682,6 +2684,8 @@ unsigned long mmap_region(struct file *file, unsigned long addr, error = -ENOMEM; if (file) goto close_and_free_vma; + else if (vma->vm_file) + goto unmap_and_free_vma; else goto free_vma; } @@ -2751,7 +2755,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, /* Undo any partial mapping done by a device driver. */ unmap_region(mm, mas.tree, vma, prev, next, vma->vm_start, vma->vm_end); - if (vm_flags & VM_SHARED) + if (file && (vm_flags & VM_SHARED)) mapping_unmap_writable(file->f_mapping); free_vma: vm_area_free(vma); -- GitLab From 8ac932a4921a96ca52f61935dbba64ea87bbd5dc Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Sat, 29 Oct 2022 13:49:12 +0900 Subject: [PATCH 1251/1927] nilfs2: fix deadlock in nilfs_count_free_blocks() A semaphore deadlock can occur if nilfs_get_block() detects metadata corruption while locating data blocks and a superblock writeback occurs at the same time: task 1 task 2 ------ ------ * A file operation * nilfs_truncate() nilfs_get_block() down_read(rwsem A) <-- nilfs_bmap_lookup_contig() ... generic_shutdown_super() nilfs_put_super() * Prepare to write superblock * down_write(rwsem B) <-- nilfs_cleanup_super() * Detect b-tree corruption * nilfs_set_log_cursor() nilfs_bmap_convert_error() nilfs_count_free_blocks() __nilfs_error() down_read(rwsem A) <-- nilfs_set_error() down_write(rwsem B) <-- *** DEADLOCK *** Here, nilfs_get_block() readlocks rwsem A (= NILFS_MDT(dat_inode)->mi_sem) and then calls nilfs_bmap_lookup_contig(), but if it fails due to metadata corruption, __nilfs_error() is called from nilfs_bmap_convert_error() inside the lock section. Since __nilfs_error() calls nilfs_set_error() unless the filesystem is read-only and nilfs_set_error() attempts to writelock rwsem B (= nilfs->ns_sem) to write back superblock exclusively, hierarchical lock acquisition occurs in the order rwsem A -> rwsem B. Now, if another task starts updating the superblock, it may writelock rwsem B during the lock sequence above, and can deadlock trying to readlock rwsem A in nilfs_count_free_blocks(). However, there is actually no need to take rwsem A in nilfs_count_free_blocks() because it, within the lock section, only reads a single integer data on a shared struct with nilfs_sufile_get_ncleansegs(). This has been the case after commit aa474a220180 ("nilfs2: add local variable to cache the number of clean segments"), that is, even before this bug was introduced. So, this resolves the deadlock problem by just not taking the semaphore in nilfs_count_free_blocks(). Link: https://lkml.kernel.org/r/20221029044912.9139-1-konishi.ryusuke@gmail.com Fixes: e828949e5b42 ("nilfs2: call nilfs_error inside bmap routines") Signed-off-by: Ryusuke Konishi Reported-by: syzbot+45d6ce7b7ad7ef455d03@syzkaller.appspotmail.com Tested-by: Ryusuke Konishi Cc: [2.6.38+ Signed-off-by: Andrew Morton --- fs/nilfs2/the_nilfs.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index 3b4a079c9617c..c8b89b4f94e0e 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -690,9 +690,7 @@ int nilfs_count_free_blocks(struct the_nilfs *nilfs, sector_t *nblocks) { unsigned long ncleansegs; - down_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); ncleansegs = nilfs_sufile_get_ncleansegs(nilfs->ns_sufile); - up_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); *nblocks = (sector_t)ncleansegs * nilfs->ns_blocks_per_segment; return 0; } -- GitLab From 624a2c94f5b7a08120aaf26b3275a17463789273 Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Tue, 8 Nov 2022 15:57:23 -0800 Subject: [PATCH 1252/1927] Partly revert "mm/thp: carry over dirty bit when thp splits on pmd" Anatoly Pugachev reported sparc64 breakage on the patch: https://lore.kernel.org/r/20221021160603.GA23307@u164.east.ru The sparc64 impl of pte_mkdirty() is definitely slightly special in that it leverages a code patching mechanism for sun4u/sun4v on relevant pgtable entry operations. Before having a clue of why the sparc64 is special and caused the patch to SIGSEGV the processes, revert the patch for now. The swap path of dirty bit inheritage is kept because that's using the swap shared code so we assume it'll not be affected. Link: https://lkml.kernel.org/r/Y1Wbi4yyVvDtg4zN@x1n Fixes: 0ccf7f168e17 ("mm/thp: carry over dirty bit when thp splits on pmd") Signed-off-by: Peter Xu Reported-by: Anatoly Pugachev Tested-by: Anatoly Pugachev Cc: Alistair Popple Cc: Andi Kleen Cc: Andrea Arcangeli Cc: David Hildenbrand Cc: David S. Miller Cc: "Huang, Ying" Cc: Hugh Dickins Cc: "Kirill A . Shutemov" Cc: Minchan Kim Cc: Nadav Amit Cc: Vlastimil Babka Signed-off-by: Andrew Morton --- mm/huge_memory.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 561a42567477d..811d19b5c4f60 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -2206,9 +2206,12 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, entry = pte_wrprotect(entry); if (!young) entry = pte_mkold(entry); - /* NOTE: this may set soft-dirty too on some archs */ - if (dirty) - entry = pte_mkdirty(entry); + /* + * NOTE: we don't do pte_mkdirty when dirty==true + * because it breaks sparc64 which can sigsegv + * random process. Need to revisit when we figure + * out what is special with sparc64. + */ if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) -- GitLab From 867400af90f1f953ff9e10b1b87ecaf9369a7eb8 Mon Sep 17 00:00:00 2001 From: Pankaj Gupta Date: Wed, 2 Nov 2022 11:07:28 -0500 Subject: [PATCH 1253/1927] mm/memremap.c: map FS_DAX device memory as decrypted virtio_pmem use devm_memremap_pages() to map the device memory. By default this memory is mapped as encrypted with SEV. Guest reboot changes the current encryption key and guest no longer properly decrypts the FSDAX device meta data. Mark the corresponding device memory region for FSDAX devices (mapped with memremap_pages) as decrypted to retain the persistent memory property. Link: https://lkml.kernel.org/r/20221102160728.3184016-1-pankaj.gupta@amd.com Fixes: b7b3c01b19159 ("mm/memremap_pages: support multiple ranges per invocation") Signed-off-by: Pankaj Gupta Cc: Dan Williams Cc: Tom Lendacky Cc: Signed-off-by: Andrew Morton --- mm/memremap.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/memremap.c b/mm/memremap.c index 421bec3a29ee7..08cbf54fe0370 100644 --- a/mm/memremap.c +++ b/mm/memremap.c @@ -335,6 +335,7 @@ void *memremap_pages(struct dev_pagemap *pgmap, int nid) WARN(1, "File system DAX not supported\n"); return ERR_PTR(-EINVAL); } + params.pgprot = pgprot_decrypted(params.pgprot); break; case MEMORY_DEVICE_GENERIC: break; -- GitLab From 93b0d9178743a68723babe8448981f658aebc58e Mon Sep 17 00:00:00 2001 From: Peter Xu Date: Wed, 2 Nov 2022 14:41:52 -0400 Subject: [PATCH 1254/1927] mm/shmem: use page_mapping() to detect page cache for uffd continue mfill_atomic_install_pte() checks page->mapping to detect whether one page is used in the page cache. However as pointed out by Matthew, the page can logically be a tail page rather than always the head in the case of uffd minor mode with UFFDIO_CONTINUE. It means we could wrongly install one pte with shmem thp tail page assuming it's an anonymous page. It's not that clear even for anonymous page, since normally anonymous pages also have page->mapping being setup with the anon vma. It's safe here only because the only such caller to mfill_atomic_install_pte() is always passing in a newly allocated page (mcopy_atomic_pte()), whose page->mapping is not yet setup. However that's not extremely obvious either. For either of above, use page_mapping() instead. Link: https://lkml.kernel.org/r/Y2K+y7wnhC4vbnP2@x1n Fixes: 153132571f02 ("userfaultfd/shmem: support UFFDIO_CONTINUE for shmem") Signed-off-by: Peter Xu Reported-by: Matthew Wilcox Cc: Andrea Arcangeli Cc: Hugh Dickins Cc: Axel Rasmussen Cc: Signed-off-by: Andrew Morton --- mm/userfaultfd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 3d0fef3980b3f..650ab6cfd5f49 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -64,7 +64,7 @@ int mfill_atomic_install_pte(struct mm_struct *dst_mm, pmd_t *dst_pmd, pte_t _dst_pte, *dst_pte; bool writable = dst_vma->vm_flags & VM_WRITE; bool vm_shared = dst_vma->vm_flags & VM_SHARED; - bool page_in_cache = page->mapping; + bool page_in_cache = page_mapping(page); spinlock_t *ptl; struct inode *inode; pgoff_t offset, max_off; -- GitLab From db5e8d84319bcdb51e1d3cfa42b410291d6d1cfa Mon Sep 17 00:00:00 2001 From: Vasily Gorbik Date: Wed, 2 Nov 2022 19:09:17 +0100 Subject: [PATCH 1255/1927] mm: hugetlb_vmemmap: include missing linux/moduleparam.h The kernel test robot reported build failures with a 'randconfig' on s390: >> mm/hugetlb_vmemmap.c:421:11: error: a function declaration without a prototype is deprecated in all versions of C [-Werror,-Wstrict-prototypes] core_param(hugetlb_free_vmemmap, vmemmap_optimize_enabled, bool, 0); ^ Link: https://lore.kernel.org/linux-mm/202210300751.rG3UDsuc-lkp@intel.com/ Link: https://lkml.kernel.org/r/patch.git-296b83ca939b.your-ad-here.call-01667411912-ext-5073@work.hours Fixes: 30152245c63b ("mm: hugetlb_vmemmap: replace early_param() with core_param()") Signed-off-by: Vasily Gorbik Reported-by: kernel test robot Reviewed-by: Muchun Song Cc: Gerald Schaefer Cc: Mike Kravetz Cc: Signed-off-by: Andrew Morton --- mm/hugetlb_vmemmap.c | 1 + 1 file changed, 1 insertion(+) diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index ba2a2596fb4e8..4962dd1ba4a68 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -11,6 +11,7 @@ #define pr_fmt(fmt) "HugeTLB: " fmt #include +#include #include #include #include -- GitLab From cbadaf71f7cf9e67c073eec673c6c050cecd0ec8 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 2 Nov 2022 12:06:07 +0100 Subject: [PATCH 1256/1927] kmsan: core: kmsan_in_runtime() should return true in NMI context Without that, every call to __msan_poison_alloca() in NMI may end up allocating memory, which is NMI-unsafe. Link: https://lkml.kernel.org/r/20221102110611.1085175-1-glider@google.com Link: https://lore.kernel.org/lkml/20221025221755.3810809-1-glider@google.com/ Signed-off-by: Alexander Potapenko Acked-by: Peter Zijlstra (Intel) Cc: Dmitry Vyukov Cc: Marco Elver Cc: Borislav Petkov Cc: Dave Hansen Cc: Ingo Molnar Cc: Kees Cook Cc: Masahiro Yamada Cc: Nick Desaulniers Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- mm/kmsan/kmsan.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mm/kmsan/kmsan.h b/mm/kmsan/kmsan.h index 7019c46d33a74..a147442054351 100644 --- a/mm/kmsan/kmsan.h +++ b/mm/kmsan/kmsan.h @@ -124,6 +124,8 @@ static __always_inline bool kmsan_in_runtime(void) { if ((hardirq_count() >> HARDIRQ_SHIFT) > 1) return true; + if (in_nmi()) + return true; return kmsan_get_context()->kmsan_in_runtime; } -- GitLab From 11385b2612004298ac2fbc9877e73f1410cfd3c0 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 2 Nov 2022 12:06:08 +0100 Subject: [PATCH 1257/1927] x86/uaccess: instrument copy_from_user_nmi() Make sure usercopy hooks from linux/instrumented.h are invoked for copy_from_user_nmi(). This fixes KMSAN false positives reported when dumping opcodes for a stack trace. Link: https://lkml.kernel.org/r/20221102110611.1085175-2-glider@google.com Signed-off-by: Alexander Potapenko Acked-by: Peter Zijlstra (Intel) Cc: Dave Hansen Cc: Kees Cook Cc: Borislav Petkov Cc: Dmitry Vyukov Cc: Ingo Molnar Cc: Marco Elver Cc: Masahiro Yamada Cc: Nick Desaulniers Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- arch/x86/lib/usercopy.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c index f1bb186171562..24b48af274173 100644 --- a/arch/x86/lib/usercopy.c +++ b/arch/x86/lib/usercopy.c @@ -6,6 +6,7 @@ #include #include +#include #include @@ -44,7 +45,9 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n) * called from other contexts. */ pagefault_disable(); + instrument_copy_from_user_before(to, from, n); ret = raw_copy_from_user(to, from, n); + instrument_copy_from_user_after(to, from, n, ret); pagefault_enable(); return ret; -- GitLab From ac66998df30a3c2888c077879786a3589ed5170b Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 2 Nov 2022 12:06:09 +0100 Subject: [PATCH 1258/1927] Kconfig.debug: ensure early check for KMSAN in CONFIG_KMSAN_WARN As pointed out by Masahiro Yamada, Kconfig picks up the first default entry which has true 'if' condition. Hence, the previously added check for KMSAN was never used, because it followed the checks for 64BIT and !64BIT. Put KMSAN check before others to ensure it is always applied. Link: https://lkml.kernel.org/r/20221102110611.1085175-3-glider@google.com Link: https://github.com/google/kmsan/issues/89 Link: https://lore.kernel.org/linux-mm/20221024212144.2852069-3-glider@google.com/ Fixes: 921757bc9b61 ("Kconfig.debug: disable CONFIG_FRAME_WARN for KMSAN by default") Signed-off-by: Alexander Potapenko Cc: Kees Cook Cc: Masahiro Yamada Cc: Nick Desaulniers Cc: Borislav Petkov Cc: Dave Hansen Cc: Dmitry Vyukov Cc: Ingo Molnar Cc: Marco Elver Cc: Peter Zijlstra (Intel) Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- lib/Kconfig.debug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index be69844e40e64..c3c0b077ade33 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -395,12 +395,12 @@ endif # DEBUG_INFO config FRAME_WARN int "Warn for stack frames larger than" range 0 8192 + default 0 if KMSAN default 2048 if GCC_PLUGIN_LATENT_ENTROPY default 2048 if PARISC default 1536 if (!64BIT && XTENSA) default 1024 if !64BIT default 2048 if 64BIT - default 0 if KMSAN help Tell the compiler to warn at build time for stack frames larger than this. Setting this too low will cause a lot of warnings. -- GitLab From 83d0edfa04eeca46b3eff554fb42b2fefe97bdf1 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 2 Nov 2022 12:06:10 +0100 Subject: [PATCH 1259/1927] kmsan: make sure PREEMPT_RT is off As pointed out by Peter Zijlstra, __msan_poison_alloca() does not play well with IRQ code when PREEMPT_RT is on, because in that mode even GFP_ATOMIC allocations cannot be performed. Fixing this would require making stackdepot completely lockless, which is quite challenging and may be excessive for the time being. Instead, make sure KMSAN is incompatible with PREEMPT_RT, like other debug configs are. Link: https://lkml.kernel.org/r/20221102110611.1085175-4-glider@google.com Link: https://lore.kernel.org/lkml/20221025221755.3810809-1-glider@google.com/ Signed-off-by: Alexander Potapenko Acked-by: Peter Zijlstra (Intel) Cc: Dmitry Vyukov Cc: Marco Elver Cc: Borislav Petkov Cc: Dave Hansen Cc: Ingo Molnar Cc: Kees Cook Cc: Masahiro Yamada Cc: Nick Desaulniers Cc: Thomas Gleixner Signed-off-by: Andrew Morton --- lib/Kconfig.kmsan | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Kconfig.kmsan b/lib/Kconfig.kmsan index b2489dd6503fa..ef2c8f256c57d 100644 --- a/lib/Kconfig.kmsan +++ b/lib/Kconfig.kmsan @@ -12,6 +12,7 @@ config KMSAN bool "KMSAN: detector of uninitialized values use" depends on HAVE_ARCH_KMSAN && HAVE_KMSAN_COMPILER depends on SLUB && DEBUG_KERNEL && !KASAN && !KCSAN + depends on !PREEMPT_RT select STACKDEPOT select STACKDEPOT_ALWAYS_INIT help -- GitLab From ba54d194f8daad8943802d6dfe06e205f882c391 Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Wed, 2 Nov 2022 12:06:11 +0100 Subject: [PATCH 1260/1927] x86/traps: avoid KMSAN bugs originating from handle_bug() There is a case in exc_invalid_op handler that is executed outside the irqentry_enter()/irqentry_exit() region when an UD2 instruction is used to encode a call to __warn(). In that case the `struct pt_regs` passed to the interrupt handler is never unpoisoned by KMSAN (this is normally done in irqentry_enter()), which leads to false positives inside handle_bug(). Use kmsan_unpoison_entry_regs() to explicitly unpoison those registers before using them. Link: https://lkml.kernel.org/r/20221102110611.1085175-5-glider@google.com Signed-off-by: Alexander Potapenko Cc: Borislav Petkov Cc: Dave Hansen Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Dmitry Vyukov Cc: Kees Cook Cc: Marco Elver Cc: Masahiro Yamada Cc: Nick Desaulniers Cc: Peter Zijlstra (Intel) Signed-off-by: Andrew Morton --- arch/x86/kernel/traps.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 178015a820f08..d3fdec706f1d2 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -301,6 +302,12 @@ static noinstr bool handle_bug(struct pt_regs *regs) { bool handled = false; + /* + * Normally @regs are unpoisoned by irqentry_enter(), but handle_bug() + * is a rare case that uses @regs without passing them to + * irqentry_enter(). + */ + kmsan_unpoison_entry_regs(regs); if (!is_valid_bugaddr(regs->ip)) return handled; -- GitLab From 8cccf05fe857a18ee26e20d11a8455a73ffd4efd Mon Sep 17 00:00:00 2001 From: Ryusuke Konishi Date: Fri, 4 Nov 2022 23:29:59 +0900 Subject: [PATCH 1261/1927] nilfs2: fix use-after-free bug of ns_writer on remount If a nilfs2 filesystem is downgraded to read-only due to metadata corruption on disk and is remounted read/write, or if emergency read-only remount is performed, detaching a log writer and synchronizing the filesystem can be done at the same time. In these cases, use-after-free of the log writer (hereinafter nilfs->ns_writer) can happen as shown in the scenario below: Task1 Task2 -------------------------------- ------------------------------ nilfs_construct_segment nilfs_segctor_sync init_wait init_waitqueue_entry add_wait_queue schedule nilfs_remount (R/W remount case) nilfs_attach_log_writer nilfs_detach_log_writer nilfs_segctor_destroy kfree finish_wait _raw_spin_lock_irqsave __raw_spin_lock_irqsave do_raw_spin_lock debug_spin_lock_before <-- use-after-free While Task1 is sleeping, nilfs->ns_writer is freed by Task2. After Task1 waked up, Task1 accesses nilfs->ns_writer which is already freed. This scenario diagram is based on the Shigeru Yoshida's post [1]. This patch fixes the issue by not detaching nilfs->ns_writer on remount so that this UAF race doesn't happen. Along with this change, this patch also inserts a few necessary read-only checks with superblock instance where only the ns_writer pointer was used to check if the filesystem is read-only. Link: https://syzkaller.appspot.com/bug?id=79a4c002e960419ca173d55e863bd09e8112df8b Link: https://lkml.kernel.org/r/20221103141759.1836312-1-syoshida@redhat.com [1] Link: https://lkml.kernel.org/r/20221104142959.28296-1-konishi.ryusuke@gmail.com Signed-off-by: Ryusuke Konishi Reported-by: syzbot+f816fa82f8783f7a02bb@syzkaller.appspotmail.com Reported-by: Shigeru Yoshida Tested-by: Ryusuke Konishi Cc: Signed-off-by: Andrew Morton --- fs/nilfs2/segment.c | 15 ++++++++------- fs/nilfs2/super.c | 2 -- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index b4cebad21b484..3335ef3529155 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -317,7 +317,7 @@ void nilfs_relax_pressure_in_lock(struct super_block *sb) struct the_nilfs *nilfs = sb->s_fs_info; struct nilfs_sc_info *sci = nilfs->ns_writer; - if (!sci || !sci->sc_flush_request) + if (sb_rdonly(sb) || unlikely(!sci) || !sci->sc_flush_request) return; set_bit(NILFS_SC_PRIOR_FLUSH, &sci->sc_flags); @@ -2242,7 +2242,7 @@ int nilfs_construct_segment(struct super_block *sb) struct nilfs_sc_info *sci = nilfs->ns_writer; struct nilfs_transaction_info *ti; - if (!sci) + if (sb_rdonly(sb) || unlikely(!sci)) return -EROFS; /* A call inside transactions causes a deadlock. */ @@ -2280,7 +2280,7 @@ int nilfs_construct_dsync_segment(struct super_block *sb, struct inode *inode, struct nilfs_transaction_info ti; int err = 0; - if (!sci) + if (sb_rdonly(sb) || unlikely(!sci)) return -EROFS; nilfs_transaction_lock(sb, &ti, 0); @@ -2776,11 +2776,12 @@ int nilfs_attach_log_writer(struct super_block *sb, struct nilfs_root *root) if (nilfs->ns_writer) { /* - * This happens if the filesystem was remounted - * read/write after nilfs_error degenerated it into a - * read-only mount. + * This happens if the filesystem is made read-only by + * __nilfs_error or nilfs_remount and then remounted + * read/write. In these cases, reuse the existing + * writer. */ - nilfs_detach_log_writer(sb); + return 0; } nilfs->ns_writer = nilfs_segctor_new(sb, root); diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c index ba108f915391e..6edb6e0dd61f7 100644 --- a/fs/nilfs2/super.c +++ b/fs/nilfs2/super.c @@ -1133,8 +1133,6 @@ static int nilfs_remount(struct super_block *sb, int *flags, char *data) if ((bool)(*flags & SB_RDONLY) == sb_rdonly(sb)) goto out; if (*flags & SB_RDONLY) { - /* Shutting down log writer */ - nilfs_detach_log_writer(sb); sb->s_flags |= SB_RDONLY; /* -- GitLab From 82e60d00b753bb5cfecce22b8e952436b14d02a3 Mon Sep 17 00:00:00 2001 From: Johannes Weiner Date: Thu, 3 Nov 2022 17:34:31 -0400 Subject: [PATCH 1262/1927] fs: fix leaked psi pressure state When psi annotations were added to to btrfs compression reads, the psi state tracking over add_ra_bio_pages and btrfs_submit_compressed_read was faulty. A pressure state, once entered, is never left. This results in incorrectly elevated pressure, which triggers OOM kills. pflags record the *previous* memstall state when we enter a new one. The code tried to initialize pflags to 1, and then optimize the leave call when we either didn't enter a memstall, or were already inside a nested stall. However, there can be multiple PageWorkingset pages in the bio, at which point it's that path itself that enters repeatedly and overwrites pflags. This causes us to miss the exit. Enter the stall only once if needed, then unwind correctly. erofs has the same problem, fix that up too. And move the memstall exit past submit_bio() to restore submit accounting originally added by b8e24a9300b0 ("block: annotate refault stalls from IO submission"). Link: https://lkml.kernel.org/r/Y2UHRqthNUwuIQGS@cmpxchg.org Fixes: 4088a47e78f9 ("btrfs: add manual PSI accounting for compressed reads") Fixes: 99486c511f68 ("erofs: add manual PSI accounting for the compressed address space") Fixes: 118f3663fbc6 ("block: remove PSI accounting from the bio layer") Link: https://lore.kernel.org/r/d20a0a85-e415-cf78-27f9-77dd7a94bc8d@leemhuis.info/ Signed-off-by: Johannes Weiner Reported-by: Thorsten Leemhuis Tested-by: Thorsten Leemhuis Cc: Chao Yu Cc: Chris Mason Cc: Christoph Hellwig Cc: David Sterba Cc: Gao Xiang Cc: Jens Axboe Cc: Josef Bacik Cc: Suren Baghdasaryan Signed-off-by: Andrew Morton --- fs/btrfs/compression.c | 14 ++++++++------ fs/erofs/zdata.c | 18 +++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index f1f051ad31474..e6635fe700678 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -512,7 +512,7 @@ static u64 bio_end_offset(struct bio *bio) static noinline int add_ra_bio_pages(struct inode *inode, u64 compressed_end, struct compressed_bio *cb, - unsigned long *pflags) + int *memstall, unsigned long *pflags) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); unsigned long end_index; @@ -581,8 +581,10 @@ static noinline int add_ra_bio_pages(struct inode *inode, continue; } - if (PageWorkingset(page)) + if (!*memstall && PageWorkingset(page)) { psi_memstall_enter(pflags); + *memstall = 1; + } ret = set_page_extent_mapped(page); if (ret < 0) { @@ -670,8 +672,8 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, u64 em_len; u64 em_start; struct extent_map *em; - /* Initialize to 1 to make skip psi_memstall_leave unless needed */ - unsigned long pflags = 1; + unsigned long pflags; + int memstall = 0; blk_status_t ret; int ret2; int i; @@ -727,7 +729,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, goto fail; } - add_ra_bio_pages(inode, em_start + em_len, cb, &pflags); + add_ra_bio_pages(inode, em_start + em_len, cb, &memstall, &pflags); /* include any pages we added in add_ra-bio_pages */ cb->len = bio->bi_iter.bi_size; @@ -807,7 +809,7 @@ void btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, } } - if (!pflags) + if (memstall) psi_memstall_leave(&pflags); if (refcount_dec_and_test(&cb->pending_ios)) diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index c7f24fc7efd59..064a166324a77 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -1412,8 +1412,8 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, struct block_device *last_bdev; unsigned int nr_bios = 0; struct bio *bio = NULL; - /* initialize to 1 to make skip psi_memstall_leave unless needed */ - unsigned long pflags = 1; + unsigned long pflags; + int memstall = 0; bi_private = jobqueueset_init(sb, q, fgq, force_fg); qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; @@ -1463,14 +1463,18 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, if (bio && (cur != last_index + 1 || last_bdev != mdev.m_bdev)) { submit_bio_retry: - if (!pflags) - psi_memstall_leave(&pflags); submit_bio(bio); + if (memstall) { + psi_memstall_leave(&pflags); + memstall = 0; + } bio = NULL; } - if (unlikely(PageWorkingset(page))) + if (unlikely(PageWorkingset(page)) && !memstall) { psi_memstall_enter(&pflags); + memstall = 1; + } if (!bio) { bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, @@ -1500,9 +1504,9 @@ static void z_erofs_submit_queue(struct z_erofs_decompress_frontend *f, } while (owned_head != Z_EROFS_PCLUSTER_TAIL); if (bio) { - if (!pflags) - psi_memstall_leave(&pflags); submit_bio(bio); + if (memstall) + psi_memstall_leave(&pflags); } /* -- GitLab From 1fdbed657a4726639c4f17841fd2a0fb646c746e Mon Sep 17 00:00:00 2001 From: Naoya Horiguchi Date: Mon, 7 Nov 2022 11:10:10 +0900 Subject: [PATCH 1263/1927] arch/x86/mm/hugetlbpage.c: pud_huge() returns 0 when using 2-level paging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following bug is reported to be triggered when starting X on x86-32 system with i915: [ 225.777375] kernel BUG at mm/memory.c:2664! [ 225.777391] invalid opcode: 0000 [#1] PREEMPT SMP [ 225.777405] CPU: 0 PID: 2402 Comm: Xorg Not tainted 6.1.0-rc3-bdg+ #86 [ 225.777415] Hardware name: /8I865G775-G, BIOS F1 08/29/2006 [ 225.777421] EIP: __apply_to_page_range+0x24d/0x31c [ 225.777437] Code: ff ff 8b 55 e8 8b 45 cc e8 0a 11 ec ff 89 d8 83 c4 28 5b 5e 5f 5d c3 81 7d e0 a0 ef 96 c1 74 ad 8b 45 d0 e8 2d 83 49 00 eb a3 <0f> 0b 25 00 f0 ff ff 81 eb 00 00 00 40 01 c3 8b 45 ec 8b 00 e8 76 [ 225.777446] EAX: 00000001 EBX: c53a3b58 ECX: b5c00000 EDX: c258aa00 [ 225.777454] ESI: b5c00000 EDI: b5900000 EBP: c4b0fdb4 ESP: c4b0fd80 [ 225.777462] DS: 007b ES: 007b FS: 00d8 GS: 0033 SS: 0068 EFLAGS: 00010202 [ 225.777470] CR0: 80050033 CR2: b5900000 CR3: 053a3000 CR4: 000006d0 [ 225.777479] Call Trace: [ 225.777486] ? i915_memcpy_init_early+0x63/0x63 [i915] [ 225.777684] apply_to_page_range+0x21/0x27 [ 225.777694] ? i915_memcpy_init_early+0x63/0x63 [i915] [ 225.777870] remap_io_mapping+0x49/0x75 [i915] [ 225.778046] ? i915_memcpy_init_early+0x63/0x63 [i915] [ 225.778220] ? mutex_unlock+0xb/0xd [ 225.778231] ? i915_vma_pin_fence+0x6d/0xf7 [i915] [ 225.778420] vm_fault_gtt+0x2a9/0x8f1 [i915] [ 225.778644] ? lock_is_held_type+0x56/0xe7 [ 225.778655] ? lock_is_held_type+0x7a/0xe7 [ 225.778663] ? 0xc1000000 [ 225.778670] __do_fault+0x21/0x6a [ 225.778679] handle_mm_fault+0x708/0xb21 [ 225.778686] ? mt_find+0x21e/0x5ae [ 225.778696] exc_page_fault+0x185/0x705 [ 225.778704] ? doublefault_shim+0x127/0x127 [ 225.778715] handle_exception+0x130/0x130 [ 225.778723] EIP: 0xb700468a Recently pud_huge() got aware of non-present entry by commit 3a194f3f8ad0 ("mm/hugetlb: make pud_huge() and follow_huge_pud() aware of non-present pud entry") to handle some special states of gigantic page. However, it's overlooked that pud_none() always returns false when running with 2-level paging, and as a result pud_huge() can return true pointlessly. Introduce "#if CONFIG_PGTABLE_LEVELS > 2" to pud_huge() to deal with this. Link: https://lkml.kernel.org/r/20221107021010.2449306-1-naoya.horiguchi@linux.dev Fixes: 3a194f3f8ad0 ("mm/hugetlb: make pud_huge() and follow_huge_pud() aware of non-present pud entry") Signed-off-by: Naoya Horiguchi Reported-by: Ville Syrjälä Tested-by: Ville Syrjälä Reviewed-by: Miaohe Lin Cc: David Hildenbrand Cc: Liu Shixin Cc: Mike Kravetz Cc: Muchun Song Cc: Oscar Salvador Cc: Yang Shi Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: Signed-off-by: Andrew Morton --- arch/x86/mm/hugetlbpage.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c index 6b3033845c6d3..5804bbae4f012 100644 --- a/arch/x86/mm/hugetlbpage.c +++ b/arch/x86/mm/hugetlbpage.c @@ -37,8 +37,12 @@ int pmd_huge(pmd_t pmd) */ int pud_huge(pud_t pud) { +#if CONFIG_PGTABLE_LEVELS > 2 return !pud_none(pud) && (pud_val(pud) & (_PAGE_PRESENT|_PAGE_PSE)) != _PAGE_PRESENT; +#else + return 0; +#endif } #ifdef CONFIG_HUGETLB_PAGE -- GitLab From 9bbba5633488ee3e2903647c3484c4390ff39ea7 Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Mon, 7 Nov 2022 16:38:35 +0000 Subject: [PATCH 1264/1927] maple_tree: fix depth tracking in maple_state It is possible to confuse the depth tracking in the maple state by searching the same node for values. Fix the depth tracking by moving where the depth is incremented closer to where the node changes level. Also change the initial depth setting when using the root node. Link: https://lkml.kernel.org/r/20221107163814.866612-1-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Signed-off-by: Andrew Morton --- lib/maple_tree.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index f23f11da4113b..59d0cebc774bc 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -1351,6 +1351,7 @@ static inline struct maple_enode *mas_start(struct ma_state *mas) root = mas_root(mas); /* Tree with nodes */ if (likely(xa_is_node(root))) { + mas->depth = 1; mas->node = mte_safe_root(root); return NULL; } @@ -3727,7 +3728,6 @@ static bool mas_is_span_wr(struct ma_wr_state *wr_mas) static inline void mas_wr_walk_descend(struct ma_wr_state *wr_mas) { - wr_mas->mas->depth++; wr_mas->type = mte_node_type(wr_mas->mas->node); mas_wr_node_walk(wr_mas); wr_mas->slots = ma_slots(wr_mas->node, wr_mas->type); @@ -3739,6 +3739,7 @@ static inline void mas_wr_walk_traverse(struct ma_wr_state *wr_mas) wr_mas->mas->min = wr_mas->r_min; wr_mas->mas->node = wr_mas->content; wr_mas->mas->offset = 0; + wr_mas->mas->depth++; } /* * mas_wr_walk() - Walk the tree for a write. -- GitLab From 7dc5ba6254bb242a9f45e43549171a2d84d25e6a Mon Sep 17 00:00:00 2001 From: Liam Howlett Date: Mon, 7 Nov 2022 16:39:02 +0000 Subject: [PATCH 1265/1927] maple_tree: don't set a new maximum on the node when not reusing nodes In RCU mode, the node limits were being updated to the last pivot which may not be correct and would cause the metadata to be set when it shouldn't. Fix this by not setting a new limit in this case. Link: https://lkml.kernel.org/r/20221107163857.867377-1-Liam.Howlett@oracle.com Signed-off-by: Liam R. Howlett Signed-off-by: Andrew Morton --- lib/maple_tree.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 59d0cebc774bc..df352f6ccc240 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -3603,8 +3603,7 @@ static inline int mas_commit_b_node(struct ma_wr_state *wr_mas, node = mas_pop_node(wr_mas->mas); node->parent = mas_mn(wr_mas->mas)->parent; wr_mas->mas->node = mt_mk_node(node, b_type); - mab_mas_cp(b_node, 0, b_end, wr_mas->mas, true); - + mab_mas_cp(b_node, 0, b_end, wr_mas->mas, false); mas_replace(wr_mas->mas, false); reuse_node: mas_update_gap(wr_mas->mas); -- GitLab From 1de09a7281edecfdba19b3a07417f6d65243ab5f Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Mon, 7 Nov 2022 16:50:00 +0000 Subject: [PATCH 1266/1927] mm/damon/dbgfs: check if rm_contexts input is for a real context A user could write a name of a file under 'damon/' debugfs directory, which is not a user-created context, to 'rm_contexts' file. In the case, 'dbgfs_rm_context()' just assumes it's the valid DAMON context directory only if a file of the name exist. As a result, invalid memory access could happen as below. Fix the bug by checking if the given input is for a directory. This check can filter out non-context inputs because directories under 'damon/' debugfs directory can be created via only 'mk_contexts' file. This bug has found by syzbot[1]. [1] https://lore.kernel.org/damon/000000000000ede3ac05ec4abf8e@google.com/ Link: https://lkml.kernel.org/r/20221107165001.5717-2-sj@kernel.org Fixes: 75c1c2b53c78 ("mm/damon/dbgfs: support multiple contexts") Signed-off-by: SeongJae Park Reported-by: syzbot+6087eafb76a94c4ac9eb@syzkaller.appspotmail.com Cc: [5.15.x] Signed-off-by: Andrew Morton --- mm/damon/dbgfs.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c index 6f0ae7d3ae39b..b3f454a5c6828 100644 --- a/mm/damon/dbgfs.c +++ b/mm/damon/dbgfs.c @@ -890,6 +890,7 @@ static ssize_t dbgfs_mk_context_write(struct file *file, static int dbgfs_rm_context(char *name) { struct dentry *root, *dir, **new_dirs; + struct inode *inode; struct damon_ctx **new_ctxs; int i, j; int ret = 0; @@ -905,6 +906,12 @@ static int dbgfs_rm_context(char *name) if (!dir) return -ENOENT; + inode = d_inode(dir); + if (!S_ISDIR(inode->i_mode)) { + ret = -EINVAL; + goto out_dput; + } + new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs), GFP_KERNEL); if (!new_dirs) { -- GitLab From 436fa4a699bcd1c368bea9199e22b0a1b9cf9e0f Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Mon, 7 Nov 2022 15:22:55 +0100 Subject: [PATCH 1267/1927] docs: kmsan: fix formatting of "Example report" Add a blank line to make the sentence before the list render as a separate paragraph, not a definition. Link: https://lkml.kernel.org/r/20221107142255.4038811-1-glider@google.com Fixes: 93858ae70cf4 ("kmsan: add ReST documentation") Signed-off-by: Alexander Potapenko Suggested-by: Bagas Sanjaya Cc: Jonathan Corbet Signed-off-by: Andrew Morton --- Documentation/dev-tools/kmsan.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/dev-tools/kmsan.rst b/Documentation/dev-tools/kmsan.rst index 2a53a801198cb..55fa82212eb25 100644 --- a/Documentation/dev-tools/kmsan.rst +++ b/Documentation/dev-tools/kmsan.rst @@ -67,6 +67,7 @@ uninitialized in the local variable, as well as the stack where the value was copied to another memory location before use. A use of uninitialized value ``v`` is reported by KMSAN in the following cases: + - in a condition, e.g. ``if (v) { ... }``; - in an indexing or pointer dereferencing, e.g. ``array[v]`` or ``*v``; - when it is copied to userspace or hardware, e.g. ``copy_to_user(..., &v, ...)``; -- GitLab From d62f8e982cb857f451a9055d8cc446b1212a6500 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:34 +1000 Subject: [PATCH 1268/1927] drm/nouveau/kms: switch hpd_lock from mutex to spinlock There's no good reason for this to be a mutex, and once the layers of workqueues have been untangled, nouveau_connector_hpd() can be called from IRQ context and won't be able to take a mutex. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nouveau_connector.c | 5 +++-- drivers/gpu/drm/nouveau/nouveau_display.c | 13 ++++++------- drivers/gpu/drm/nouveau/nouveau_drv.h | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index 1991bbb1d05c3..c2ff8e91d90d0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -1166,13 +1166,14 @@ nouveau_connector_hpd(struct drm_connector *connector) { struct nouveau_drm *drm = nouveau_drm(connector->dev); u32 mask = drm_connector_mask(connector); + unsigned long flags; - mutex_lock(&drm->hpd_lock); + spin_lock_irqsave(&drm->hpd_lock, flags); if (!(drm->hpd_pending & mask)) { drm->hpd_pending |= mask; schedule_work(&drm->hpd_work); } - mutex_unlock(&drm->hpd_lock); + spin_unlock_irqrestore(&drm->hpd_lock, flags); } static int diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 2e97186090c86..fb22ebed5424d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -456,9 +456,9 @@ nouveau_display_hpd_resume(struct drm_device *dev) { struct nouveau_drm *drm = nouveau_drm(dev); - mutex_lock(&drm->hpd_lock); + spin_lock_irq(&drm->hpd_lock); drm->hpd_pending = ~0; - mutex_unlock(&drm->hpd_lock); + spin_unlock_irq(&drm->hpd_lock); schedule_work(&drm->hpd_work); } @@ -475,10 +475,10 @@ nouveau_display_hpd_work(struct work_struct *work) pm_runtime_get_sync(dev->dev); - mutex_lock(&drm->hpd_lock); + spin_lock_irq(&drm->hpd_lock); pending = drm->hpd_pending; drm->hpd_pending = 0; - mutex_unlock(&drm->hpd_lock); + spin_unlock_irq(&drm->hpd_lock); /* Nothing to do, exit early without updating the last busy counter */ if (!pending) @@ -732,7 +732,7 @@ nouveau_display_create(struct drm_device *dev) } INIT_WORK(&drm->hpd_work, nouveau_display_hpd_work); - mutex_init(&drm->hpd_lock); + spin_lock_init(&drm->hpd_lock); #ifdef CONFIG_ACPI drm->acpi_nb.notifier_call = nouveau_display_acpi_ntfy; register_acpi_notifier(&drm->acpi_nb); @@ -766,8 +766,7 @@ nouveau_display_destroy(struct drm_device *dev) nvif_disp_dtor(&disp->disp); - nouveau_drm(dev)->display = NULL; - mutex_destroy(&drm->hpd_lock); + drm->display = NULL; kfree(disp); } diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 84df5ddae4d06..1a92c980cb735 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -201,7 +201,7 @@ struct nouveau_drm { struct nvbios vbios; struct nouveau_display *display; struct work_struct hpd_work; - struct mutex hpd_lock; + spinlock_t hpd_lock; u32 hpd_pending; struct work_struct fbcon_work; int fbcon_new_state; -- GitLab From 016dacb60e6d4b301c5941a0dedb49d337926832 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:34 +1000 Subject: [PATCH 1269/1927] drm/nouveau/kms: pass event mask to hpd handler Will be moving the DP link status check / re-train here so it's safe from racing with modeset routing changes. MST message handling etc. will remain where it is. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nouveau_connector.c | 11 +++++------ drivers/gpu/drm/nouveau/nouveau_connector.h | 3 ++- drivers/gpu/drm/nouveau/nouveau_display.c | 20 +++++++++++++++++--- drivers/gpu/drm/nouveau/nouveau_dp.c | 12 +++++------- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index c2ff8e91d90d0..bbd17ee608538 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -1162,14 +1162,15 @@ nouveau_connector_funcs_lvds = { }; void -nouveau_connector_hpd(struct drm_connector *connector) +nouveau_connector_hpd(struct nouveau_connector *nv_connector, u64 bits) { - struct nouveau_drm *drm = nouveau_drm(connector->dev); - u32 mask = drm_connector_mask(connector); + struct nouveau_drm *drm = nouveau_drm(nv_connector->base.dev); + u32 mask = drm_connector_mask(&nv_connector->base); unsigned long flags; spin_lock_irqsave(&drm->hpd_lock, flags); if (!(drm->hpd_pending & mask)) { + nv_connector->hpd_pending |= bits; drm->hpd_pending |= mask; schedule_work(&drm->hpd_work); } @@ -1185,15 +1186,13 @@ nouveau_connector_hotplug(struct nvif_notify *notify) struct drm_device *dev = connector->dev; struct nouveau_drm *drm = nouveau_drm(dev); const struct nvif_notify_conn_rep_v0 *rep = notify->data; - bool plugged = (rep->mask != NVIF_NOTIFY_CONN_V0_UNPLUG); if (rep->mask & NVIF_NOTIFY_CONN_V0_IRQ) { nouveau_dp_irq(drm, nv_connector); return NVIF_NOTIFY_KEEP; } - NV_DEBUG(drm, "%splugged %s\n", plugged ? "" : "un", connector->name); - nouveau_connector_hpd(connector); + nouveau_connector_hpd(nv_connector, rep->mask); return NVIF_NOTIFY_KEEP; } diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h index f4e17ff68bf93..1bbf8bf6ba445 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.h +++ b/drivers/gpu/drm/nouveau/nouveau_connector.h @@ -124,6 +124,7 @@ struct nouveau_connector { u8 *dcb; struct nvif_conn conn; + u64 hpd_pending; struct nvif_notify hpd; struct drm_dp_aux aux; @@ -198,7 +199,7 @@ nouveau_crtc_connector_get(struct nouveau_crtc *nv_crtc) struct drm_connector * nouveau_connector_create(struct drm_device *, const struct dcb_output *); -void nouveau_connector_hpd(struct drm_connector *connector); +void nouveau_connector_hpd(struct nouveau_connector *, u64 bits); extern int nouveau_tv_disable; extern int nouveau_ignorelid; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index fb22ebed5424d..a21dc2a0f92b3 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -488,14 +488,28 @@ nouveau_display_hpd_work(struct work_struct *work) drm_connector_list_iter_begin(dev, &conn_iter); nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) { + struct nouveau_connector *nv_connector = nouveau_connector(connector); enum drm_connector_status old_status = connector->status; - u64 old_epoch_counter = connector->epoch_counter; + u64 bits, old_epoch_counter = connector->epoch_counter; if (!(pending & drm_connector_mask(connector))) continue; - connector->status = drm_helper_probe_detect(connector, NULL, - false); + spin_lock_irq(&drm->hpd_lock); + bits = nv_connector->hpd_pending; + nv_connector->hpd_pending = 0; + spin_unlock_irq(&drm->hpd_lock); + + drm_dbg_kms(dev, "[CONNECTOR:%d:%s] plug:%d unplug:%d irq:%d\n", + connector->base.id, connector->name, + !!(bits & NVIF_NOTIFY_CONN_V0_PLUG), + !!(bits & NVIF_NOTIFY_CONN_V0_UNPLUG), + !!(bits & NVIF_NOTIFY_CONN_V0_IRQ)); + + if (bits & NVIF_NOTIFY_CONN_V0_IRQ) + continue; + + connector->status = drm_helper_probe_detect(connector, NULL, false); if (old_epoch_counter == connector->epoch_counter) continue; diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c index b7104e676eb29..d349dc24a0031 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dp.c +++ b/drivers/gpu/drm/nouveau/nouveau_dp.c @@ -29,8 +29,7 @@ #include "nouveau_encoder.h" #include "nouveau_crtc.h" -#include -#include +#include MODULE_PARM_DESC(mst, "Enable DisplayPort multi-stream (default: enabled)"); static int nouveau_mst = 1; @@ -218,8 +217,8 @@ void nouveau_dp_irq(struct nouveau_drm *drm, struct drm_connector *connector = &nv_connector->base; struct nouveau_encoder *outp = find_encoder(connector, DCB_OUTPUT_DP); struct nv50_mstm *mstm; + u64 hpd = 0; int ret; - bool send_hpd = false; if (!outp) return; @@ -231,14 +230,14 @@ void nouveau_dp_irq(struct nouveau_drm *drm, if (mstm && mstm->is_mst) { if (!nv50_mstm_service(drm, nv_connector, mstm)) - send_hpd = true; + hpd |= NVIF_NOTIFY_CONN_V0_UNPLUG; } else { drm_dp_cec_irq(&nv_connector->aux); if (nouveau_dp_has_sink_count(connector, outp)) { ret = drm_dp_read_sink_count(&nv_connector->aux); if (ret != outp->dp.sink_count) - send_hpd = true; + hpd |= NVIF_NOTIFY_CONN_V0_PLUG; if (ret >= 0) outp->dp.sink_count = ret; } @@ -246,8 +245,7 @@ void nouveau_dp_irq(struct nouveau_drm *drm, mutex_unlock(&outp->dp.hpd_irq_lock); - if (send_hpd) - nouveau_connector_hpd(connector); + nouveau_connector_hpd(nv_connector, NVIF_NOTIFY_CONN_V0_IRQ | hpd); } /* TODO: -- GitLab From 8bb30c882334c0fdbec9b604f09d2e9120b509ab Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:35 +1000 Subject: [PATCH 1270/1927] drm/nouveau/disp: add method to trigger DP link retrain This moves control of link retraining in response to HPD IRQ to the KMS driver's HPD IRQ handler. NVKM still handles checking link status for the moment, this can be moved to the KMS driver when it takes explicit control of link rate selection. v2: - skip source config on retrain (fixes some retrain failures) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 6 + drivers/gpu/drm/nouveau/include/nvif/outp.h | 1 + drivers/gpu/drm/nouveau/nouveau_display.c | 6 +- drivers/gpu/drm/nouveau/nouveau_dp.c | 11 ++ drivers/gpu/drm/nouveau/nouveau_encoder.h | 1 + drivers/gpu/drm/nouveau/nvif/outp.c | 8 + drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c | 138 ++++++++++-------- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 15 ++ 8 files changed, 124 insertions(+), 62 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index d1583271f5d4d..0d2e9a949d98f 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -16,6 +16,7 @@ union nvif_outp_args { #define NVIF_OUTP_V0_INFOFRAME 0x03 #define NVIF_OUTP_V0_HDA_ELD 0x04 #define NVIF_OUTP_V0_DP_AUX_PWR 0x05 +#define NVIF_OUTP_V0_DP_RETRAIN 0x06 union nvif_outp_load_detect_args { struct nvif_outp_load_detect_v0 { @@ -100,4 +101,9 @@ union nvif_outp_dp_aux_pwr_args { __u8 pad02[6]; } v0; }; + +union nvif_outp_dp_retrain_args { + struct nvif_outp_dp_retrain_vn { + } vn; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index 1c960f6fdd7a1..ea3e0a936c8a5 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -27,4 +27,5 @@ void nvif_outp_release(struct nvif_outp *); int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_v0 *, u32 size); int nvif_outp_hda_eld(struct nvif_outp *, int head, void *data, u32 size); int nvif_outp_dp_aux_pwr(struct nvif_outp *, bool enable); +int nvif_outp_dp_retrain(struct nvif_outp *); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index a21dc2a0f92b3..965e1c62d4974 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -506,8 +506,10 @@ nouveau_display_hpd_work(struct work_struct *work) !!(bits & NVIF_NOTIFY_CONN_V0_UNPLUG), !!(bits & NVIF_NOTIFY_CONN_V0_IRQ)); - if (bits & NVIF_NOTIFY_CONN_V0_IRQ) - continue; + if (bits & NVIF_NOTIFY_CONN_V0_IRQ) { + if (nouveau_dp_link_check(nv_connector)) + continue; + } connector->status = drm_helper_probe_detect(connector, NULL, false); if (old_epoch_counter == connector->epoch_counter) diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c index d349dc24a0031..bde58d00fbf53 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dp.c +++ b/drivers/gpu/drm/nouveau/nouveau_dp.c @@ -211,6 +211,17 @@ nouveau_dp_detect(struct nouveau_connector *nv_connector, return ret; } +bool +nouveau_dp_link_check(struct nouveau_connector *nv_connector) +{ + struct nouveau_encoder *nv_encoder = find_encoder(&nv_connector->base, DCB_OUTPUT_DP); + + if (!nv_encoder || nv_encoder->outp.or.id < 0) + return true; + + return nvif_outp_dp_retrain(&nv_encoder->outp) == 0; +} + void nouveau_dp_irq(struct nouveau_drm *drm, struct nouveau_connector *nv_connector) { diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h index b6c51fc60d137..8524d211f4e10 100644 --- a/drivers/gpu/drm/nouveau/nouveau_encoder.h +++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h @@ -141,6 +141,7 @@ enum nouveau_dp_status { }; int nouveau_dp_detect(struct nouveau_connector *, struct nouveau_encoder *); +bool nouveau_dp_link_check(struct nouveau_connector *); void nouveau_dp_irq(struct nouveau_drm *drm, struct nouveau_connector *nv_connector); enum drm_mode_status nv50_dp_mode_valid(struct drm_connector *, diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 9bd9cc21a0396..56b7c703b1373 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -25,6 +25,14 @@ #include +int +nvif_outp_dp_retrain(struct nvif_outp *outp) +{ + int ret = nvif_object_mthd(&outp->object, NVIF_OUTP_V0_DP_RETRAIN, NULL, 0); + NVIF_ERRON(ret, &outp->object, "[DP_RETRAIN]"); + return ret; +} + int nvif_outp_dp_aux_pwr(struct nvif_outp *outp, bool enable) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c index 92c9faecffae8..730c3a6f33626 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c @@ -274,71 +274,18 @@ nvkm_dp_train_cr(struct lt_state *lt) } static int -nvkm_dp_train_links(struct nvkm_outp *outp, int rate) +nvkm_dp_train_link(struct nvkm_outp *outp, int rate) { struct nvkm_ior *ior = outp->ior; - struct nvkm_disp *disp = outp->disp; - struct nvkm_subdev *subdev = &disp->engine.subdev; - struct nvkm_bios *bios = subdev->device->bios; struct lt_state lt = { .outp = outp, + .pc2 = outp->dp.dpcd[DPCD_RC02] & DPCD_RC02_TPS3_SUPPORTED, }; - u32 lnkcmp; u8 sink[2], data; int ret; OUTP_DBG(outp, "training %dx%02x", ior->dp.nr, ior->dp.bw); - /* Intersect misc. capabilities of the OR and sink. */ - if (disp->engine.subdev.device->chipset < 0x110) - outp->dp.dpcd[DPCD_RC03] &= ~DPCD_RC03_TPS4_SUPPORTED; - if (disp->engine.subdev.device->chipset < 0xd0) - outp->dp.dpcd[DPCD_RC02] &= ~DPCD_RC02_TPS3_SUPPORTED; - lt.pc2 = outp->dp.dpcd[DPCD_RC02] & DPCD_RC02_TPS3_SUPPORTED; - - if (AMPERE_IED_HACK(disp) && (lnkcmp = lt.outp->dp.info.script[0])) { - /* Execute BeforeLinkTraining script from DP Info table. */ - while (ior->dp.bw < nvbios_rd08(bios, lnkcmp)) - lnkcmp += 3; - lnkcmp = nvbios_rd16(bios, lnkcmp + 1); - - nvbios_init(&outp->disp->engine.subdev, lnkcmp, - init.outp = &outp->info; - init.or = ior->id; - init.link = ior->asy.link; - ); - } - - /* Set desired link configuration on the source. */ - if ((lnkcmp = lt.outp->dp.info.lnkcmp)) { - if (outp->dp.version < 0x30) { - while ((ior->dp.bw * 2700) < nvbios_rd16(bios, lnkcmp)) - lnkcmp += 4; - lnkcmp = nvbios_rd16(bios, lnkcmp + 2); - } else { - while (ior->dp.bw < nvbios_rd08(bios, lnkcmp)) - lnkcmp += 3; - lnkcmp = nvbios_rd16(bios, lnkcmp + 1); - } - - nvbios_init(subdev, lnkcmp, - init.outp = &outp->info; - init.or = ior->id; - init.link = ior->asy.link; - ); - } - - ret = ior->func->dp->links(ior, outp->dp.aux); - if (ret) { - if (ret < 0) { - OUTP_ERR(outp, "train failed with %d", ret); - return ret; - } - return 0; - } - - ior->func->dp->power(ior, ior->dp.nr); - /* Select LTTPR non-transparent mode if we have a valid configuration, * use transparent mode otherwise. */ @@ -393,6 +340,71 @@ nvkm_dp_train_links(struct nvkm_outp *outp, int rate) return ret; } +static int +nvkm_dp_train_links(struct nvkm_outp *outp, int rate) +{ + struct nvkm_ior *ior = outp->ior; + struct nvkm_disp *disp = outp->disp; + struct nvkm_subdev *subdev = &disp->engine.subdev; + struct nvkm_bios *bios = subdev->device->bios; + u32 lnkcmp; + int ret; + + OUTP_DBG(outp, "programming link for %dx%02x", ior->dp.nr, ior->dp.bw); + + /* Intersect misc. capabilities of the OR and sink. */ + if (disp->engine.subdev.device->chipset < 0x110) + outp->dp.dpcd[DPCD_RC03] &= ~DPCD_RC03_TPS4_SUPPORTED; + if (disp->engine.subdev.device->chipset < 0xd0) + outp->dp.dpcd[DPCD_RC02] &= ~DPCD_RC02_TPS3_SUPPORTED; + + if (AMPERE_IED_HACK(disp) && (lnkcmp = outp->dp.info.script[0])) { + /* Execute BeforeLinkTraining script from DP Info table. */ + while (ior->dp.bw < nvbios_rd08(bios, lnkcmp)) + lnkcmp += 3; + lnkcmp = nvbios_rd16(bios, lnkcmp + 1); + + nvbios_init(&outp->disp->engine.subdev, lnkcmp, + init.outp = &outp->info; + init.or = ior->id; + init.link = ior->asy.link; + ); + } + + /* Set desired link configuration on the source. */ + if ((lnkcmp = outp->dp.info.lnkcmp)) { + if (outp->dp.version < 0x30) { + while ((ior->dp.bw * 2700) < nvbios_rd16(bios, lnkcmp)) + lnkcmp += 4; + lnkcmp = nvbios_rd16(bios, lnkcmp + 2); + } else { + while (ior->dp.bw < nvbios_rd08(bios, lnkcmp)) + lnkcmp += 3; + lnkcmp = nvbios_rd16(bios, lnkcmp + 1); + } + + nvbios_init(subdev, lnkcmp, + init.outp = &outp->info; + init.or = ior->id; + init.link = ior->asy.link; + ); + } + + ret = ior->func->dp->links(ior, outp->dp.aux); + if (ret) { + if (ret < 0) { + OUTP_ERR(outp, "train failed with %d", ret); + return ret; + } + return 0; + } + + ior->func->dp->power(ior, ior->dp.nr); + + /* Attempt to train the link in this configuration. */ + return nvkm_dp_train_link(outp, rate); +} + static void nvkm_dp_train_fini(struct nvkm_outp *outp) { @@ -439,6 +451,16 @@ nvkm_dp_train(struct nvkm_outp *outp, u32 dataKBps) int ret = -EINVAL, nr, rate; u8 pwr; + /* Retraining link? Skip source configuration, it can mess up the active modeset. */ + if (atomic_read(&outp->dp.lt.done)) { + for (rate = 0; rate < outp->dp.rates; rate++) { + if (outp->dp.rate[rate].rate == ior->dp.bw * 27000) + return nvkm_dp_train_link(outp, ret); + } + WARN_ON(1); + return -EINVAL; + } + /* Ensure sink is not in a low-power state. */ if (!nvkm_rdaux(outp->dp.aux, DPCD_SC00, &pwr, 1)) { if ((pwr & DPCD_SC00_SET_POWER) != DPCD_SC00_SET_POWER_D0) { @@ -726,12 +748,8 @@ nvkm_dp_hpd(struct nvkm_notify *notify) struct nvif_notify_conn_rep_v0 rep = {}; OUTP_DBG(outp, "HPD: %d", line->mask); - if (line->mask & NVKM_I2C_IRQ) { - if (atomic_read(&outp->dp.lt.done)) - outp->func->acquire(outp); + if (line->mask & NVKM_I2C_IRQ) rep.mask |= NVIF_NOTIFY_CONN_V0_IRQ; - } - if (line->mask & NVKM_I2C_UNPLUG) rep.mask |= NVIF_NOTIFY_CONN_V0_UNPLUG; if (line->mask & NVKM_I2C_PLUG) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index 3bfc3e39778e2..e12a31d428d63 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -27,6 +27,20 @@ #include +static int +nvkm_uoutp_mthd_dp_retrain(struct nvkm_outp *outp, void *argv, u32 argc) +{ + union nvif_outp_dp_retrain_args *args = argv; + + if (argc != sizeof(args->vn)) + return -ENOSYS; + + if (!atomic_read(&outp->dp.lt.done)) + return 0; + + return outp->func->acquire(outp); +} + static int nvkm_uoutp_mthd_dp_aux_pwr(struct nvkm_outp *outp, void *argv, u32 argc) { @@ -251,6 +265,7 @@ nvkm_uoutp_mthd_acquired(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc) case NVIF_OUTP_V0_RELEASE : return nvkm_uoutp_mthd_release (outp, argv, argc); case NVIF_OUTP_V0_INFOFRAME : return nvkm_uoutp_mthd_infoframe (outp, argv, argc); case NVIF_OUTP_V0_HDA_ELD : return nvkm_uoutp_mthd_hda_eld (outp, argv, argc); + case NVIF_OUTP_V0_DP_RETRAIN : return nvkm_uoutp_mthd_dp_retrain (outp, argv, argc); default: break; } -- GitLab From 8c7d980da9ba3eb67a1b40fd4b33bcf49397084b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:35 +1000 Subject: [PATCH 1271/1927] drm/nouveau/disp: move DP MST payload config method Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 26 +++--------- drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 18 --------- drivers/gpu/drm/nouveau/include/nvif/if0012.h | 12 ++++++ drivers/gpu/drm/nouveau/include/nvif/outp.h | 2 + drivers/gpu/drm/nouveau/nvif/outp.c | 21 ++++++++++ .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 40 ------------------- .../gpu/drm/nouveau/nvkm/engine/disp/uoutp.c | 17 ++++++++ 7 files changed, 57 insertions(+), 79 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index ca58c6b65339e..67087940e3042 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -884,16 +884,6 @@ nv50_msto_prepare(struct drm_atomic_state *state, struct nv50_mstc *mstc = msto->mstc; struct nv50_mstm *mstm = mstc->mstm; struct drm_dp_mst_atomic_payload *payload; - struct { - struct nv50_disp_mthd_v1 base; - struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi; - } args = { - .base.version = 1, - .base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI, - .base.hasht = mstm->outp->dcb->hasht, - .base.hashm = (0xf0ff & mstm->outp->dcb->hashm) | - (0x0100 << msto->head->base.index), - }; NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name); @@ -902,22 +892,16 @@ nv50_msto_prepare(struct drm_atomic_state *state, // TODO: Figure out if we want to do a better job of handling VCPI allocation failures here? if (msto->disabled) { drm_dp_remove_payload(mgr, mst_state, payload); + + nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index, 0, 0, 0, 0); } else { if (msto->enabled) drm_dp_add_payload_part1(mgr, mst_state, payload); - args.vcpi.start_slot = payload->vc_start_slot; - args.vcpi.num_slots = payload->time_slots; - args.vcpi.pbn = payload->pbn; - args.vcpi.aligned_pbn = payload->time_slots * mst_state->pbn_div; + nvif_outp_dp_mst_vcpi(&mstm->outp->outp, msto->head->base.index, + payload->vc_start_slot, payload->time_slots, + payload->pbn, payload->time_slots * mst_state->pbn_div); } - - NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n", - msto->encoder.name, msto->head->base.base.name, - args.vcpi.start_slot, args.vcpi.num_slots, - args.vcpi.pbn, args.vcpi.aligned_pbn); - - nvif_mthd(&drm->display->disp.object, 0, &args, sizeof(args)); } static int diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h index 15e30fbdc2152..75264c89af826 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h @@ -25,22 +25,4 @@ struct nv50_disp_scanoutpos_v0 { __u16 htotal; __u16 hline; }; - -struct nv50_disp_mthd_v1 { - __u8 version; -#define NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI 0x26 - __u8 method; - __u16 hasht; - __u16 hashm; - __u8 pad06[2]; -}; - -struct nv50_disp_sor_dp_mst_vcpi_v0 { - __u8 version; - __u8 pad01[1]; - __u8 start_slot; - __u8 num_slots; - __u16 pbn; - __u16 aligned_pbn; -}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0012.h b/drivers/gpu/drm/nouveau/include/nvif/if0012.h index 0d2e9a949d98f..eb99d84eb8443 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0012.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0012.h @@ -17,6 +17,7 @@ union nvif_outp_args { #define NVIF_OUTP_V0_HDA_ELD 0x04 #define NVIF_OUTP_V0_DP_AUX_PWR 0x05 #define NVIF_OUTP_V0_DP_RETRAIN 0x06 +#define NVIF_OUTP_V0_DP_MST_VCPI 0x07 union nvif_outp_load_detect_args { struct nvif_outp_load_detect_v0 { @@ -106,4 +107,15 @@ union nvif_outp_dp_retrain_args { struct nvif_outp_dp_retrain_vn { } vn; }; + +union nvif_outp_dp_mst_vcpi_args { + struct nvif_outp_dp_mst_vcpi_v0 { + __u8 version; + __u8 head; + __u8 start_slot; + __u8 num_slots; + __u16 pbn; + __u16 aligned_pbn; + } v0; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/outp.h b/drivers/gpu/drm/nouveau/include/nvif/outp.h index ea3e0a936c8a5..45daadec3c0c7 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/outp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/outp.h @@ -28,4 +28,6 @@ int nvif_outp_infoframe(struct nvif_outp *, u8 type, struct nvif_outp_infoframe_ int nvif_outp_hda_eld(struct nvif_outp *, int head, void *data, u32 size); int nvif_outp_dp_aux_pwr(struct nvif_outp *, bool enable); int nvif_outp_dp_retrain(struct nvif_outp *); +int nvif_outp_dp_mst_vcpi(struct nvif_outp *, int head, + u8 start_slot, u8 num_slots, u16 pbn, u16 aligned_pbn); #endif diff --git a/drivers/gpu/drm/nouveau/nvif/outp.c b/drivers/gpu/drm/nouveau/nvif/outp.c index 56b7c703b1373..7da39f1eae9fb 100644 --- a/drivers/gpu/drm/nouveau/nvif/outp.c +++ b/drivers/gpu/drm/nouveau/nvif/outp.c @@ -25,6 +25,27 @@ #include +int +nvif_outp_dp_mst_vcpi(struct nvif_outp *outp, int head, + u8 start_slot, u8 num_slots, u16 pbn, u16 aligned_pbn) +{ + struct nvif_outp_dp_mst_vcpi_v0 args; + int ret; + + args.version = 0; + args.head = head; + args.start_slot = start_slot; + args.num_slots = num_slots; + args.pbn = pbn; + args.aligned_pbn = aligned_pbn; + + ret = nvif_object_mthd(&outp->object, NVIF_OUTP_V0_DP_MST_VCPI, &args, sizeof(args)); + NVIF_ERRON(ret, &outp->object, + "[DP_MST_VCPI head:%d start_slot:%02x num_slots:%02x pbn:%04x aligned_pbn:%04x]", + args.head, args.start_slot, args.num_slots, args.pbn, args.aligned_pbn); + return ret; +} + int nvif_outp_dp_retrain(struct nvif_outp *outp) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c index 2f42940367a0f..3226faff6f382 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c @@ -37,7 +37,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) { union { struct nv50_disp_mthd_v0 v0; - struct nv50_disp_mthd_v1 v1; } *args = data; struct nvkm_disp *disp = nvkm_udisp(object); struct nvkm_outp *temp, *outp = NULL; @@ -54,16 +53,6 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) args->v0.version, args->v0.method, args->v0.head); mthd = args->v0.method; hidx = args->v0.head; - } else - if (!(ret = nvif_unpack(ret, &data, &size, args->v1, 1, 1, true))) { - nvif_ioctl(object, "disp mthd vers %d mthd %02x " - "type %04x mask %04x\n", - args->v1.version, args->v1.method, - args->v1.hasht, args->v1.hashm); - mthd = args->v1.method; - type = args->v1.hasht; - mask = args->v1.hashm; - hidx = ffs((mask >> 8) & 0x0f) - 1; } else return ret; @@ -90,34 +79,5 @@ nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) break; } - switch (mthd * !!outp) { - case NV50_DISP_MTHD_V1_SOR_DP_MST_VCPI: { - union { - struct nv50_disp_sor_dp_mst_vcpi_v0 v0; - } *args = data; - int ret = -ENOSYS; - nvif_ioctl(object, "disp sor dp mst vcpi size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "disp sor dp mst vcpi vers %d " - "slot %02x/%02x pbn %04x/%04x\n", - args->v0.version, args->v0.start_slot, - args->v0.num_slots, args->v0.pbn, - args->v0.aligned_pbn); - if (!outp->ior->func->dp->vcpi) - return -ENODEV; - outp->ior->func->dp->vcpi(outp->ior, hidx, - args->v0.start_slot, - args->v0.num_slots, - args->v0.pbn, - args->v0.aligned_pbn); - return 0; - } else - return ret; - } - break; - default: - break; - } - return -EINVAL; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c index e12a31d428d63..d10ce1e04d325 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uoutp.c @@ -27,6 +27,22 @@ #include +static int +nvkm_uoutp_mthd_dp_mst_vcpi(struct nvkm_outp *outp, void *argv, u32 argc) +{ + struct nvkm_ior *ior = outp->ior; + union nvif_outp_dp_mst_vcpi_args *args = argv; + + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + if (!ior->func->dp || !ior->func->dp->vcpi || !nvkm_head_find(outp->disp, args->v0.head)) + return -EINVAL; + + ior->func->dp->vcpi(ior, args->v0.head, args->v0.start_slot, args->v0.num_slots, + args->v0.pbn, args->v0.aligned_pbn); + return 0; +} + static int nvkm_uoutp_mthd_dp_retrain(struct nvkm_outp *outp, void *argv, u32 argc) { @@ -266,6 +282,7 @@ nvkm_uoutp_mthd_acquired(struct nvkm_outp *outp, u32 mthd, void *argv, u32 argc) case NVIF_OUTP_V0_INFOFRAME : return nvkm_uoutp_mthd_infoframe (outp, argv, argc); case NVIF_OUTP_V0_HDA_ELD : return nvkm_uoutp_mthd_hda_eld (outp, argv, argc); case NVIF_OUTP_V0_DP_RETRAIN : return nvkm_uoutp_mthd_dp_retrain (outp, argv, argc); + case NVIF_OUTP_V0_DP_MST_VCPI: return nvkm_uoutp_mthd_dp_mst_vcpi(outp, argv, argc); default: break; } -- GitLab From a2b7eadfef5963138a5aeaba90c4f513414823c2 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:36 +1000 Subject: [PATCH 1272/1927] drm/nouveau/disp: add head class v2: remove extra whitespace Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv04/crtc.c | 5 ++ drivers/gpu/drm/nouveau/dispnv50/head.c | 5 ++ drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + drivers/gpu/drm/nouveau/include/nvif/disp.h | 1 + drivers/gpu/drm/nouveau/include/nvif/head.h | 13 ++++ drivers/gpu/drm/nouveau/include/nvif/if0010.h | 1 + drivers/gpu/drm/nouveau/include/nvif/if0013.h | 12 ++++ drivers/gpu/drm/nouveau/nouveau_crtc.h | 3 +- drivers/gpu/drm/nouveau/nvif/Kbuild | 1 + drivers/gpu/drm/nouveau/nvif/disp.c | 5 +- drivers/gpu/drm/nouveau/nvif/head.c | 48 ++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/disp/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/engine/disp/head.h | 3 + .../gpu/drm/nouveau/nvkm/engine/disp/priv.h | 1 + .../gpu/drm/nouveau/nvkm/engine/disp/udisp.c | 12 ++++ .../gpu/drm/nouveau/nvkm/engine/disp/uhead.c | 66 +++++++++++++++++++ 16 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/head.h create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if0013.h create mode 100644 drivers/gpu/drm/nouveau/nvif/head.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c index ee92d576d2777..f35c25607508c 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c +++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c @@ -762,6 +762,7 @@ static void nv_crtc_destroy(struct drm_crtc *crtc) nouveau_bo_unpin(nv_crtc->cursor.nvbo); nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo); nvif_notify_dtor(&nv_crtc->vblank); + nvif_head_dtor(&nv_crtc->head); kfree(nv_crtc); } @@ -1341,6 +1342,10 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num) nv04_cursor_init(nv_crtc); + ret = nvif_head_ctor(&disp->disp, nv_crtc->base.name, nv_crtc->index, &nv_crtc->head); + if (ret) + return ret; + ret = nvif_notify_ctor(&disp->disp.object, "kmsVbl", nv04_crtc_vblank_handler, false, NV04_DISP_NTFY_VBLANK, &(struct nvif_notify_head_req_v0) { diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.c b/drivers/gpu/drm/nouveau/dispnv50/head.c index c3c57be54e1ce..8ac3474111c54 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/head.c +++ b/drivers/gpu/drm/nouveau/dispnv50/head.c @@ -518,6 +518,7 @@ nv50_head_destroy(struct drm_crtc *crtc) struct nv50_head *head = nv50_head(crtc); nvif_notify_dtor(&head->base.vblank); + nvif_head_dtor(&head->base.head); nv50_lut_fini(&head->olut); drm_crtc_cleanup(crtc); kfree(head); @@ -624,6 +625,10 @@ nv50_head_create(struct drm_device *dev, int index) } } + ret = nvif_head_ctor(disp->disp, head->base.base.name, head->base.index, &head->base.head); + if (ret) + return ERR_PTR(ret); + ret = nvif_notify_ctor(&disp->disp->object, "kmsVbl", nv50_head_vblank_handler, false, NV04_DISP_NTFY_VBLANK, &(struct nvif_notify_head_req_v0) { diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 8641db649f484..03f0f7c299f23 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -35,6 +35,7 @@ #define NVIF_CLASS_DISP /* if0010.h */ 0x80000010 #define NVIF_CLASS_CONN /* if0011.h */ 0x80000011 #define NVIF_CLASS_OUTP /* if0012.h */ 0x80000012 +#define NVIF_CLASS_HEAD /* if0013.h */ 0x80000013 #define NVIF_CLASS_DISP_CHAN /* if0014.h */ 0x80000014 /* the below match nvidia-assigned (either in hw, or sw) class numbers */ diff --git a/drivers/gpu/drm/nouveau/include/nvif/disp.h b/drivers/gpu/drm/nouveau/include/nvif/disp.h index 742632ad3bea9..56eb7293e01cd 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/disp.h +++ b/drivers/gpu/drm/nouveau/include/nvif/disp.h @@ -7,6 +7,7 @@ struct nvif_disp { struct nvif_object object; unsigned long conn_mask; unsigned long outp_mask; + unsigned long head_mask; }; int nvif_disp_ctor(struct nvif_device *, const char *name, s32 oclass, diff --git a/drivers/gpu/drm/nouveau/include/nvif/head.h b/drivers/gpu/drm/nouveau/include/nvif/head.h new file mode 100644 index 0000000000000..09170c7bc0afd --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/head.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVIF_HEAD_H__ +#define __NVIF_HEAD_H__ +#include +struct nvif_disp; + +struct nvif_head { + struct nvif_object object; +}; + +int nvif_head_ctor(struct nvif_disp *, const char *name, int id, struct nvif_head *); +void nvif_head_dtor(struct nvif_head *); +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0010.h b/drivers/gpu/drm/nouveau/include/nvif/if0010.h index fc236ef289654..4c835bbe6fe3f 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0010.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0010.h @@ -8,6 +8,7 @@ union nvif_disp_args { __u8 pad01[3]; __u32 conn_mask; __u32 outp_mask; + __u32 head_mask; } v0; }; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0013.h b/drivers/gpu/drm/nouveau/include/nvif/if0013.h new file mode 100644 index 0000000000000..218ba6eeba7b0 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if0013.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVIF_IF0013_H__ +#define __NVIF_IF0013_H__ + +union nvif_head_args { + struct nvif_head_v0 { + __u8 version; + __u8 id; + __u8 pad02[6]; + } v0; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h index 7f63be2ec35db..e85e74f9a28dc 100644 --- a/drivers/gpu/drm/nouveau/nouveau_crtc.h +++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h @@ -26,14 +26,15 @@ #ifndef __NOUVEAU_CRTC_H__ #define __NOUVEAU_CRTC_H__ - #include +#include #include struct nouveau_crtc { struct drm_crtc base; + struct nvif_head head; int index; struct nvif_notify vblank; diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 6abc4bc42e355..6281291c70dc5 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -6,6 +6,7 @@ nvif-y += nvif/device.o nvif-y += nvif/disp.o nvif-y += nvif/driver.o nvif-y += nvif/fifo.o +nvif-y += nvif/head.o nvif-y += nvif/mem.o nvif-y += nvif/mmu.o nvif-y += nvif/notify.o diff --git a/drivers/gpu/drm/nouveau/nvif/disp.c b/drivers/gpu/drm/nouveau/nvif/disp.c index 926b0c04b1e87..09915f2715afd 100644 --- a/drivers/gpu/drm/nouveau/nvif/disp.c +++ b/drivers/gpu/drm/nouveau/nvif/disp.c @@ -72,9 +72,10 @@ nvif_disp_ctor(struct nvif_device *device, const char *name, s32 oclass, struct if (ret) return ret; - NVIF_DEBUG(&disp->object, "[NEW] conn_mask:%08x outp_mask:%08x", - args.conn_mask, args.outp_mask); + NVIF_DEBUG(&disp->object, "[NEW] conn_mask:%08x outp_mask:%08x head_mask:%08x", + args.conn_mask, args.outp_mask, args.head_mask); disp->conn_mask = args.conn_mask; disp->outp_mask = args.outp_mask; + disp->head_mask = args.head_mask; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvif/head.c b/drivers/gpu/drm/nouveau/nvif/head.c new file mode 100644 index 0000000000000..01deba4626000 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvif/head.c @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include +#include + +#include +#include + +void +nvif_head_dtor(struct nvif_head *head) +{ + nvif_object_dtor(&head->object); +} + +int +nvif_head_ctor(struct nvif_disp *disp, const char *name, int id, struct nvif_head *head) +{ + struct nvif_head_v0 args; + int ret; + + args.version = 0; + args.id = id; + + ret = nvif_object_ctor(&disp->object, name ? name : "nvifHead", id, NVIF_CLASS_HEAD, + &args, sizeof(args), &head->object); + NVIF_ERRON(ret, &disp->object, "[NEW head id:%d]", args.id); + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index 600072a904bed..e403040658b6c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -34,3 +34,4 @@ nvkm-y += nvkm/engine/disp/rootnv50.o nvkm-y += nvkm/engine/disp/udisp.o nvkm-y += nvkm/engine/disp/uconn.o nvkm-y += nvkm/engine/disp/uoutp.o +nvkm-y += nvkm/engine/disp/uhead.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h index 84a2989193cf4..d95d4b591b2e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef __NVKM_DISP_HEAD_H__ #define __NVKM_DISP_HEAD_H__ +#include #include "priv.h" struct nvkm_head { @@ -26,6 +27,8 @@ struct nvkm_head { u8 depth; } or; } arm, asy; + + struct nvkm_object object; }; int nvkm_head_new_(const struct nvkm_head_func *, struct nvkm_disp *, int id); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h index cb25dfe849f0f..9e89615c4f67e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h @@ -86,4 +86,5 @@ extern const struct nvkm_event_func gv100_disp_chan_uevent; int nvkm_udisp_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); int nvkm_uconn_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); int nvkm_uoutp_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); +int nvkm_uhead_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c index 0841e7ce03436..93a0b816b33b0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c @@ -21,6 +21,7 @@ */ #include "priv.h" #include "conn.h" +#include "head.h" #include "outp.h" #include @@ -43,6 +44,12 @@ nvkm_udisp_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *scl return 0; } + if (index-- == 0) { + sclass->base = (struct nvkm_sclass) { 0, 0, NVIF_CLASS_HEAD }; + sclass->ctor = nvkm_uhead_new; + return 0; + } + if (disp->func->user[index].ctor) { sclass->base = disp->func->user[index].base; sclass->ctor = disp->func->user[index].ctor; @@ -89,6 +96,7 @@ nvkm_udisp_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nv struct nvkm_disp *disp = nvkm_disp(oclass->engine); struct nvkm_conn *conn; struct nvkm_outp *outp; + struct nvkm_head *head; union nvif_disp_args *args = argv; if (argc != sizeof(args->v0) || args->v0.version != 0) @@ -111,5 +119,9 @@ nvkm_udisp_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nv list_for_each_entry(outp, &disp->outps, head) args->v0.outp_mask |= BIT(outp->index); + args->v0.head_mask = 0; + list_for_each_entry(head, &disp->heads, head) + args->v0.head_mask |= BIT(head->id); + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c new file mode 100644 index 0000000000000..a58f4b5fcd014 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c @@ -0,0 +1,66 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define nvkm_uhead(p) container_of((p), struct nvkm_head, object) +#include "head.h" + +#include + +static void * +nvkm_uhead_dtor(struct nvkm_object *object) +{ + struct nvkm_head *head = nvkm_uhead(object); + struct nvkm_disp *disp = head->disp; + + spin_lock(&disp->client.lock); + head->object.func = NULL; + spin_unlock(&disp->client.lock); + return NULL; +} + +static const struct nvkm_object_func +nvkm_uhead = { + .dtor = nvkm_uhead_dtor, +}; + +int +nvkm_uhead_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) +{ + struct nvkm_disp *disp = nvkm_udisp(oclass->parent); + struct nvkm_head *head; + union nvif_head_args *args = argv; + int ret; + + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + if (!(head = nvkm_head_find(disp, args->v0.id))) + return -EINVAL; + + ret = -EBUSY; + spin_lock(&disp->client.lock); + if (!head->object.func) { + nvkm_object_ctor(&nvkm_uhead, oclass, &head->object); + *pobject = &head->object; + ret = 0; + } + spin_unlock(&disp->client.lock); + return ret; +} -- GitLab From 361863ceab1eaa171a304bda84636f2ff0a1d820 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:36 +1000 Subject: [PATCH 1273/1927] drm/nouveau/disp: move head scanoutpos method Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 1 - drivers/gpu/drm/nouveau/include/nvif/cl0046.h | 22 ----- drivers/gpu/drm/nouveau/include/nvif/cl5070.h | 28 ------- drivers/gpu/drm/nouveau/include/nvif/if0013.h | 18 ++++ drivers/gpu/drm/nouveau/nouveau_display.c | 27 +++--- .../gpu/drm/nouveau/nvkm/engine/disp/Kbuild | 3 - .../gpu/drm/nouveau/nvkm/engine/disp/head.c | 38 --------- .../gpu/drm/nouveau/nvkm/engine/disp/head.h | 2 - .../gpu/drm/nouveau/nvkm/engine/disp/priv.h | 2 - .../drm/nouveau/nvkm/engine/disp/rootnv04.c | 62 -------------- .../drm/nouveau/nvkm/engine/disp/rootnv50.c | 83 ------------------- .../gpu/drm/nouveau/nvkm/engine/disp/udisp.c | 12 --- .../gpu/drm/nouveau/nvkm/engine/disp/uhead.c | 42 ++++++++++ 13 files changed, 71 insertions(+), 269 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl5070.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 67087940e3042..e720c01841b26 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -46,7 +46,6 @@ #include #include -#include #include #include #include diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0046.h b/drivers/gpu/drm/nouveau/include/nvif/cl0046.h index d490d401870ad..b3ac930b01fa5 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl0046.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl0046.h @@ -4,26 +4,4 @@ #define NV04_DISP_NTFY_VBLANK 0x00 #define NV04_DISP_NTFY_CONN 0x01 - -struct nv04_disp_mthd_v0 { - __u8 version; -#define NV04_DISP_SCANOUTPOS 0x00 - __u8 method; - __u8 head; - __u8 pad03[5]; -}; - -struct nv04_disp_scanoutpos_v0 { - __u8 version; - __u8 pad01[7]; - __s64 time[2]; - __u16 vblanks; - __u16 vblanke; - __u16 vtotal; - __u16 vline; - __u16 hblanks; - __u16 hblanke; - __u16 htotal; - __u16 hline; -}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h b/drivers/gpu/drm/nouveau/include/nvif/cl5070.h deleted file mode 100644 index 75264c89af826..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl5070.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL5070_H__ -#define __NVIF_CL5070_H__ - -#define NV50_DISP_MTHD 0x00 - -struct nv50_disp_mthd_v0 { - __u8 version; -#define NV50_DISP_SCANOUTPOS 0x00 - __u8 method; - __u8 head; - __u8 pad03[5]; -}; - -struct nv50_disp_scanoutpos_v0 { - __u8 version; - __u8 pad01[7]; - __s64 time[2]; - __u16 vblanks; - __u16 vblanke; - __u16 vtotal; - __u16 vline; - __u16 hblanks; - __u16 hblanke; - __u16 htotal; - __u16 hline; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0013.h b/drivers/gpu/drm/nouveau/include/nvif/if0013.h index 218ba6eeba7b0..4bd9507a200bd 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0013.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0013.h @@ -9,4 +9,22 @@ union nvif_head_args { __u8 pad02[6]; } v0; }; + +#define NVIF_HEAD_V0_SCANOUTPOS 0x00 + +union nvif_head_scanoutpos_args { + struct nvif_head_scanoutpos_v0 { + __u8 version; + __u8 pad01[7]; + __s64 time[2]; + __u16 vblanks; + __u16 vblanke; + __u16 vtotal; + __u16 vline; + __u16 hblanks; + __u16 hblanke; + __u16 htotal; + __u16 hline; + } v0; +}; #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 965e1c62d4974..2034346f3934d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -42,7 +42,7 @@ #include "nv50_display.h" #include -#include +#include #include #include @@ -84,24 +84,20 @@ static bool nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos, ktime_t *stime, ktime_t *etime) { - struct { - struct nv04_disp_mthd_v0 base; - struct nv04_disp_scanoutpos_v0 scan; - } args = { - .base.method = NV04_DISP_SCANOUTPOS, - .base.head = nouveau_crtc(crtc)->index, - }; - struct nouveau_display *disp = nouveau_display(crtc->dev); struct drm_vblank_crtc *vblank = &crtc->dev->vblank[drm_crtc_index(crtc)]; + struct nvif_head *head = &nouveau_crtc(crtc)->head; + struct nvif_head_scanoutpos_v0 args; int retry = 20; bool ret = false; + args.version = 0; + do { - ret = nvif_mthd(&disp->disp.object, 0, &args, sizeof(args)); + ret = nvif_mthd(&head->object, NVIF_HEAD_V0_SCANOUTPOS, &args, sizeof(args)); if (ret != 0) return false; - if (args.scan.vline) { + if (args.vline) { ret = true; break; } @@ -109,11 +105,10 @@ nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos, if (retry) ndelay(vblank->linedur_ns); } while (retry--); - *hpos = args.scan.hline; - *vpos = calc(args.scan.vblanks, args.scan.vblanke, - args.scan.vtotal, args.scan.vline); - if (stime) *stime = ns_to_ktime(args.scan.time[0]); - if (etime) *etime = ns_to_ktime(args.scan.time[1]); + *hpos = args.hline; + *vpos = calc(args.vblanks, args.vblanke, args.vtotal, args.vline); + if (stime) *stime = ns_to_ktime(args.time[0]); + if (etime) *etime = ns_to_ktime(args.time[1]); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index e403040658b6c..e1aecd3fe96c1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -28,9 +28,6 @@ nvkm-y += nvkm/engine/disp/gv100.o nvkm-y += nvkm/engine/disp/tu102.o nvkm-y += nvkm/engine/disp/ga102.o -nvkm-y += nvkm/engine/disp/rootnv04.o -nvkm-y += nvkm/engine/disp/rootnv50.o - nvkm-y += nvkm/engine/disp/udisp.o nvkm-y += nvkm/engine/disp/uconn.o nvkm-y += nvkm/engine/disp/uoutp.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.c index 83152c26fe3e0..7f5d13d13c946 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.c @@ -39,44 +39,6 @@ nvkm_head_find(struct nvkm_disp *disp, int id) return NULL; } -int -nvkm_head_mthd_scanoutpos(struct nvkm_object *object, - struct nvkm_head *head, void *data, u32 size) -{ - union { - struct nv04_disp_scanoutpos_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(object, "head scanoutpos size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "head scanoutpos vers %d\n", - args->v0.version); - - head->func->state(head, &head->arm); - args->v0.vtotal = head->arm.vtotal; - args->v0.vblanks = head->arm.vblanks; - args->v0.vblanke = head->arm.vblanke; - args->v0.htotal = head->arm.htotal; - args->v0.hblanks = head->arm.hblanks; - args->v0.hblanke = head->arm.hblanke; - - /* We don't support reading htotal/vtotal on pre-NV50 VGA, - * so we have to give up and trigger the timestamping - * fallback in the drm core. - */ - if (!args->v0.vtotal || !args->v0.htotal) - return -ENOTSUPP; - - args->v0.time[0] = ktime_to_ns(ktime_get()); - head->func->rgpos(head, &args->v0.hline, &args->v0.vline); - args->v0.time[1] = ktime_to_ns(ktime_get()); - } else - return ret; - - return 0; -} - void nvkm_head_del(struct nvkm_head **phead) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h index d95d4b591b2e0..856252bf559a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h @@ -33,8 +33,6 @@ struct nvkm_head { int nvkm_head_new_(const struct nvkm_head_func *, struct nvkm_disp *, int id); void nvkm_head_del(struct nvkm_head **); -int nvkm_head_mthd_scanoutpos(struct nvkm_object *, - struct nvkm_head *, void *, u32); struct nvkm_head *nvkm_head_find(struct nvkm_disp *, int id); struct nvkm_head_func { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h index 9e89615c4f67e..a98b57f82fe77 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h @@ -43,8 +43,6 @@ struct nvkm_disp_func { }; int nvkm_disp_ntfy(struct nvkm_object *, u32, struct nvkm_event **); -int nv04_disp_mthd(struct nvkm_object *, u32, void *, u32); -int nv50_disp_root_mthd_(struct nvkm_object *, u32, void *, u32); int nv50_disp_oneinit(struct nvkm_disp *); int nv50_disp_init(struct nvkm_disp *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c deleted file mode 100644 index 9acaec5c271e6..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv04.c +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "priv.h" -#include "head.h" - -#include - -#include -#include - -int -nv04_disp_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size) -{ - struct nvkm_disp *disp = nvkm_disp(object->engine); - union { - struct nv04_disp_mthd_v0 v0; - } *args = data; - struct nvkm_head *head; - int id, ret = -ENOSYS; - - nvif_ioctl(object, "disp mthd size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { - nvif_ioctl(object, "disp mthd vers %d mthd %02x head %d\n", - args->v0.version, args->v0.method, args->v0.head); - mthd = args->v0.method; - id = args->v0.head; - } else - return ret; - - if (!(head = nvkm_head_find(disp, id))) - return -ENXIO; - - switch (mthd) { - case NV04_DISP_SCANOUTPOS: - return nvkm_head_mthd_scanoutpos(object, head, data, size); - default: - break; - } - - return -EINVAL; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c deleted file mode 100644 index 3226faff6f382..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.c +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "chan.h" -#include "head.h" -#include "ior.h" -#include "outp.h" - -#include - -#include -#include -#include - -int -nv50_disp_root_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) -{ - union { - struct nv50_disp_mthd_v0 v0; - } *args = data; - struct nvkm_disp *disp = nvkm_udisp(object); - struct nvkm_outp *temp, *outp = NULL; - struct nvkm_head *head; - u16 type, mask = 0; - int hidx, ret = -ENOSYS; - - if (mthd != NV50_DISP_MTHD) - return -EINVAL; - - nvif_ioctl(object, "disp mthd size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { - nvif_ioctl(object, "disp mthd vers %d mthd %02x head %d\n", - args->v0.version, args->v0.method, args->v0.head); - mthd = args->v0.method; - hidx = args->v0.head; - } else - return ret; - - if (!(head = nvkm_head_find(disp, hidx))) - return -ENXIO; - - if (mask) { - list_for_each_entry(temp, &disp->outps, head) { - if ((temp->info.hasht == type) && - (temp->info.hashm & mask) == mask) { - outp = temp; - break; - } - } - if (outp == NULL) - return -ENXIO; - } - - switch (mthd) { - case NV50_DISP_SCANOUTPOS: { - return nvkm_head_mthd_scanoutpos(object, head, data, size); - } - default: - break; - } - - return -EINVAL; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c index 93a0b816b33b0..c9bd32220fe17 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c @@ -59,17 +59,6 @@ nvkm_udisp_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *scl return -EINVAL; } -static int -nvkm_udisp_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) -{ - struct nvkm_disp *disp = nvkm_udisp(object); - - if (disp->engine.subdev.device->card_type >= NV_50) - return nv50_disp_root_mthd_(object, mthd, argv, argc); - - return nv04_disp_mthd(object, mthd, argv, argc); -} - static void * nvkm_udisp_dtor(struct nvkm_object *object) { @@ -85,7 +74,6 @@ nvkm_udisp_dtor(struct nvkm_object *object) static const struct nvkm_object_func nvkm_udisp = { .dtor = nvkm_udisp_dtor, - .mthd = nvkm_udisp_mthd, .ntfy = nvkm_disp_ntfy, .sclass = nvkm_udisp_sclass, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c index a58f4b5fcd014..5f3c4b7dbb75b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c @@ -24,6 +24,47 @@ #include +static int +nvkm_uhead_mthd_scanoutpos(struct nvkm_head *head, void *argv, u32 argc) +{ + union nvif_head_scanoutpos_args *args = argv; + + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + head->func->state(head, &head->arm); + args->v0.vtotal = head->arm.vtotal; + args->v0.vblanks = head->arm.vblanks; + args->v0.vblanke = head->arm.vblanke; + args->v0.htotal = head->arm.htotal; + args->v0.hblanks = head->arm.hblanks; + args->v0.hblanke = head->arm.hblanke; + + /* We don't support reading htotal/vtotal on pre-NV50 VGA, + * so we have to give up and trigger the timestamping + * fallback in the drm core. + */ + if (!args->v0.vtotal || !args->v0.htotal) + return -ENOTSUPP; + + args->v0.time[0] = ktime_to_ns(ktime_get()); + head->func->rgpos(head, &args->v0.hline, &args->v0.vline); + args->v0.time[1] = ktime_to_ns(ktime_get()); + return 0; +} + +static int +nvkm_uhead_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) +{ + struct nvkm_head *head = nvkm_uhead(object); + + switch (mthd) { + case NVIF_HEAD_V0_SCANOUTPOS: return nvkm_uhead_mthd_scanoutpos(head, argv, argc); + default: + return -EINVAL; + } +} + static void * nvkm_uhead_dtor(struct nvkm_object *object) { @@ -39,6 +80,7 @@ nvkm_uhead_dtor(struct nvkm_object *object) static const struct nvkm_object_func nvkm_uhead = { .dtor = nvkm_uhead_dtor, + .mthd = nvkm_uhead_mthd, }; int -- GitLab From f43e47c090dc7fe32d5410d8740c3a004eb2676f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:36 +1000 Subject: [PATCH 1274/1927] drm/nouveau/nvkm: add a replacement for nvkm_notify This replaces the twisty, confusing, relationship between nvkm_event and nvkm_notify with something much simpler, and less racey. It also places events in the object tree hierarchy, which will allow a heap of the code tracking events across allocation/teardown/suspend to be removed. This commit just adds the new interfaces, and passes the owning subdev to the event constructor to enable debug-tracing in the new code. v2: - use ?: (lyude) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/class.h | 2 + drivers/gpu/drm/nouveau/include/nvif/event.h | 32 ++++ drivers/gpu/drm/nouveau/include/nvif/if000e.h | 26 +++ .../drm/nouveau/include/nvkm/core/client.h | 2 + .../gpu/drm/nouveau/include/nvkm/core/event.h | 38 ++++- .../drm/nouveau/include/nvkm/core/object.h | 2 + drivers/gpu/drm/nouveau/nouveau_nvif.c | 15 +- drivers/gpu/drm/nouveau/nvif/Kbuild | 1 + drivers/gpu/drm/nouveau/nvif/event.c | 81 +++++++++ drivers/gpu/drm/nouveau/nvkm/core/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/core/client.c | 5 +- drivers/gpu/drm/nouveau/nvkm/core/event.c | 149 ++++++++++++++++- drivers/gpu/drm/nouveau/nvkm/core/ioctl.c | 27 ++- drivers/gpu/drm/nouveau/nvkm/core/oproxy.c | 13 ++ drivers/gpu/drm/nouveau/nvkm/core/uevent.c | 157 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/disp/base.c | 11 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/fault/base.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/gpio/base.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/i2c/base.c | 2 +- 21 files changed, 547 insertions(+), 29 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if000e.h create mode 100644 drivers/gpu/drm/nouveau/nvif/event.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/core/uevent.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 03f0f7c299f23..29448f77069b7 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -32,6 +32,8 @@ #define NVIF_CLASS_VMM_GM200 /* ifb00d.h */ 0x8000b00d #define NVIF_CLASS_VMM_GP100 /* ifc00d.h */ 0x8000c00d +#define NVIF_CLASS_EVENT /* if000e.h */ 0x8000000e + #define NVIF_CLASS_DISP /* if0010.h */ 0x80000010 #define NVIF_CLASS_CONN /* if0011.h */ 0x80000011 #define NVIF_CLASS_OUTP /* if0012.h */ 0x80000012 diff --git a/drivers/gpu/drm/nouveau/include/nvif/event.h b/drivers/gpu/drm/nouveau/include/nvif/event.h index a6b1ee4f10caf..a510ba4ad4879 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/event.h +++ b/drivers/gpu/drm/nouveau/include/nvif/event.h @@ -1,6 +1,38 @@ /* SPDX-License-Identifier: MIT */ #ifndef __NVIF_EVENT_H__ #define __NVIF_EVENT_H__ +#include +#include +struct nvif_event; + +#define NVIF_EVENT_KEEP 0 +#define NVIF_EVENT_DROP 1 +typedef int (*nvif_event_func)(struct nvif_event *, void *repv, u32 repc); + +struct nvif_event { + struct nvif_object object; + nvif_event_func func; +}; + +static inline bool +nvif_event_constructed(struct nvif_event *event) +{ + return nvif_object_constructed(&event->object); +} + +int nvif_event_ctor_(struct nvif_object *, const char *, u32, nvif_event_func, bool, + struct nvif_event_v0 *, u32, bool, struct nvif_event *); + +static inline int +nvif_event_ctor(struct nvif_object *parent, const char *name, u32 handle, nvif_event_func func, + bool wait, struct nvif_event_v0 *args, u32 argc, struct nvif_event *event) +{ + return nvif_event_ctor_(parent, name, handle, func, wait, args, argc, true, event); +} + +void nvif_event_dtor(struct nvif_event *); +int nvif_event_allow(struct nvif_event *); +int nvif_event_block(struct nvif_event *); struct nvif_notify_req_v0 { __u8 version; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if000e.h b/drivers/gpu/drm/nouveau/include/nvif/if000e.h new file mode 100644 index 0000000000000..90a936cb17668 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if000e.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVIF_IF000E_H__ +#define __NVIF_IF000E_H__ + +union nvif_event_args { + struct nvif_event_v0 { + __u8 version; + __u8 wait; + __u8 pad02[6]; + __u8 data[]; + } v0; +}; + +#define NVIF_EVENT_V0_ALLOW 0x00 +#define NVIF_EVENT_V0_BLOCK 0x01 + +union nvif_event_allow_args { + struct nvif_event_allow_vn { + } vn; +}; + +union nvif_event_block_args { + struct nvif_event_block_vn { + } vn; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h index 2f86606e708c9..2eb22aaed700b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h @@ -15,6 +15,7 @@ struct nvkm_client { void *data; int (*ntfy)(const void *, u32, const void *, u32); + int (*event)(u64 token, void *argv, u32 argc); struct list_head umem; spinlock_t lock; @@ -23,6 +24,7 @@ struct nvkm_client { int nvkm_client_new(const char *name, u64 device, const char *cfg, const char *dbg, int (*)(const void *, u32, const void *, u32), + int (*)(u64, void *, u32), struct nvkm_client **); struct nvkm_client *nvkm_client_search(struct nvkm_client *, u64 handle); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/event.h b/drivers/gpu/drm/nouveau/include/nvkm/core/event.h index a7a413f07a789..d6755a89f5877 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/event.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/event.h @@ -4,9 +4,12 @@ #include struct nvkm_notify; struct nvkm_object; +struct nvkm_oclass; +struct nvkm_uevent; struct nvkm_event { const struct nvkm_event_func *func; + struct nvkm_subdev *subdev; int types_nr; int index_nr; @@ -15,6 +18,8 @@ struct nvkm_event { spinlock_t list_lock; struct list_head list; int *refs; + + struct list_head ntfy; }; struct nvkm_event_func { @@ -25,11 +30,42 @@ struct nvkm_event_func { void (*fini)(struct nvkm_event *, int type, int index); }; -int nvkm_event_init(const struct nvkm_event_func *func, int types_nr, +int nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *, int types_nr, int index_nr, struct nvkm_event *); void nvkm_event_fini(struct nvkm_event *); void nvkm_event_get(struct nvkm_event *, u32 types, int index); void nvkm_event_put(struct nvkm_event *, u32 types, int index); void nvkm_event_send(struct nvkm_event *, u32 types, int index, void *data, u32 size); + +#define NVKM_EVENT_KEEP 0 +#define NVKM_EVENT_DROP 1 +struct nvkm_event_ntfy; +typedef int (*nvkm_event_func)(struct nvkm_event_ntfy *, u32 bits); + +struct nvkm_event_ntfy { + struct nvkm_event *event; + int id; + u32 bits; + bool wait; + nvkm_event_func func; + + atomic_t allowed; + bool running; + + struct list_head head; +}; + +void nvkm_event_ntfy(struct nvkm_event *, int id, u32 bits); +bool nvkm_event_ntfy_valid(struct nvkm_event *, int id, u32 bits); +void nvkm_event_ntfy_add(struct nvkm_event *, int id, u32 bits, bool wait, nvkm_event_func, + struct nvkm_event_ntfy *); +void nvkm_event_ntfy_del(struct nvkm_event_ntfy *); +void nvkm_event_ntfy_allow(struct nvkm_event_ntfy *); +void nvkm_event_ntfy_block(struct nvkm_event_ntfy *); + +typedef int (*nvkm_uevent_func)(struct nvkm_object *, u64 token, u32 bits); + +int nvkm_uevent_new(const struct nvkm_oclass *, void *argv, u32 argc, struct nvkm_object **); +int nvkm_uevent_add(struct nvkm_uevent *, struct nvkm_event *, int id, u32 bits, nvkm_uevent_func); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h index 7efcd5d2f2fff..ed1f66360782a 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/object.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/object.h @@ -4,6 +4,7 @@ #include struct nvkm_event; struct nvkm_gpuobj; +struct nvkm_uevent; struct nvkm_object { const struct nvkm_object_func *func; @@ -43,6 +44,7 @@ struct nvkm_object_func { int (*bind)(struct nvkm_object *, struct nvkm_gpuobj *, int align, struct nvkm_gpuobj **); int (*sclass)(struct nvkm_object *, int index, struct nvkm_oclass *); + int (*uevent)(struct nvkm_object *, void *argv, u32 argc, struct nvkm_uevent *); }; void nvkm_object_ctor(const struct nvkm_object_func *, diff --git a/drivers/gpu/drm/nouveau/nouveau_nvif.c b/drivers/gpu/drm/nouveau/nouveau_nvif.c index df0fe58ca3ab2..be6d404a34596 100644 --- a/drivers/gpu/drm/nouveau/nouveau_nvif.c +++ b/drivers/gpu/drm/nouveau/nouveau_nvif.c @@ -71,11 +71,24 @@ nvkm_client_suspend(void *priv) return nvkm_object_fini(&client->object, true); } +static int +nvkm_client_event(u64 token, void *repv, u32 repc) +{ + struct nvif_object *object = (void *)(unsigned long)token; + struct nvif_event *event = container_of(object, typeof(*event), object); + + if (event->func(event, repv, repc) == NVIF_EVENT_KEEP) + return NVKM_EVENT_KEEP; + + return NVKM_EVENT_DROP; +} + static int nvkm_client_driver_init(const char *name, u64 device, const char *cfg, const char *dbg, void **ppriv) { - return nvkm_client_new(name, device, cfg, dbg, nvif_notify, (struct nvkm_client **)ppriv); + return nvkm_client_new(name, device, cfg, dbg, nvif_notify, nvkm_client_event, + (struct nvkm_client **)ppriv); } const struct nvif_driver diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 6281291c70dc5..7ce040ca23940 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -5,6 +5,7 @@ nvif-y += nvif/conn.o nvif-y += nvif/device.o nvif-y += nvif/disp.o nvif-y += nvif/driver.o +nvif-y += nvif/event.o nvif-y += nvif/fifo.o nvif-y += nvif/head.o nvif-y += nvif/mem.o diff --git a/drivers/gpu/drm/nouveau/nvif/event.c b/drivers/gpu/drm/nouveau/nvif/event.c new file mode 100644 index 0000000000000..61ff4d6eba9f8 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvif/event.c @@ -0,0 +1,81 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include +#include + +#include +#include + +int +nvif_event_block(struct nvif_event *event) +{ + if (nvif_event_constructed(event)) { + int ret = nvif_mthd(&event->object, NVIF_EVENT_V0_BLOCK, NULL, 0); + NVIF_ERRON(ret, &event->object, "[BLOCK]"); + return ret; + } + return 0; +} + +int +nvif_event_allow(struct nvif_event *event) +{ + if (nvif_event_constructed(event)) { + int ret = nvif_mthd(&event->object, NVIF_EVENT_V0_ALLOW, NULL, 0); + NVIF_ERRON(ret, &event->object, "[ALLOW]"); + return ret; + } + return 0; +} + +void +nvif_event_dtor(struct nvif_event *event) +{ + nvif_object_dtor(&event->object); +} + +int +nvif_event_ctor_(struct nvif_object *parent, const char *name, u32 handle, nvif_event_func func, + bool wait, struct nvif_event_v0 *args, u32 argc, bool warn, + struct nvif_event *event) +{ + struct nvif_event_v0 _args; + int ret; + + if (!args) { + args = &_args; + argc = sizeof(_args); + } + + args->version = 0; + args->wait = wait; + + ret = nvif_object_ctor(parent, name ?: "nvifEvent", handle, + NVIF_CLASS_EVENT, args, argc, &event->object); + NVIF_ERRON(ret && warn, parent, "[NEW EVENT wait:%d size:%zd]", + args->wait, argc - sizeof(*args)); + if (ret) + return ret; + + event->func = func; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/core/Kbuild b/drivers/gpu/drm/nouveau/nvkm/core/Kbuild index 2b471ab585b48..216a3db821ee9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/core/Kbuild @@ -14,3 +14,4 @@ nvkm-y += nvkm/core/oproxy.o nvkm-y += nvkm/core/option.o nvkm-y += nvkm/core/ramht.o nvkm-y += nvkm/core/subdev.o +nvkm-y += nvkm/core/uevent.o diff --git a/drivers/gpu/drm/nouveau/nvkm/core/client.c b/drivers/gpu/drm/nouveau/nvkm/core/client.c index 0c8c55c73b122..e1d978fbfe724 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/client.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/client.c @@ -44,7 +44,7 @@ nvkm_uclient_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))){ args->v0.name[sizeof(args->v0.name) - 1] = 0; ret = nvkm_client_new(args->v0.name, args->v0.device, NULL, - NULL, oclass->client->ntfy, &client); + NULL, oclass->client->ntfy, oclass->client->event, &client); if (ret) return ret; } else @@ -286,7 +286,7 @@ int nvkm_client_new(const char *name, u64 device, const char *cfg, const char *dbg, int (*ntfy)(const void *, u32, const void *, u32), - struct nvkm_client **pclient) + int (*event)(u64, void *, u32), struct nvkm_client **pclient) { struct nvkm_oclass oclass = { .base = nvkm_uclient_sclass }; struct nvkm_client *client; @@ -301,6 +301,7 @@ nvkm_client_new(const char *name, u64 device, const char *cfg, client->debug = nvkm_dbgopt(dbg, "CLIENT"); client->objroot = RB_ROOT; client->ntfy = ntfy; + client->event = event; INIT_LIST_HEAD(&client->umem); spin_lock_init(&client->lock); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/core/event.c b/drivers/gpu/drm/nouveau/nvkm/core/event.c index 006618d77aa46..ec508d1bbe1da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/event.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/event.c @@ -21,14 +21,19 @@ */ #include #include +#include void nvkm_event_put(struct nvkm_event *event, u32 types, int index) { assert_spin_locked(&event->refs_lock); + + nvkm_trace(event->subdev, "event: decr %08x on %d\n", types, index); + while (types) { int type = __ffs(types); types &= ~(1 << type); if (--event->refs[index * event->types_nr + type] == 0) { + nvkm_trace(event->subdev, "event: blocking %d on %d\n", type, index); if (event->func->fini) event->func->fini(event, 1 << type, index); } @@ -39,18 +44,146 @@ void nvkm_event_get(struct nvkm_event *event, u32 types, int index) { assert_spin_locked(&event->refs_lock); + + nvkm_trace(event->subdev, "event: incr %08x on %d\n", types, index); + while (types) { int type = __ffs(types); types &= ~(1 << type); if (++event->refs[index * event->types_nr + type] == 1) { + nvkm_trace(event->subdev, "event: allowing %d on %d\n", type, index); if (event->func->init) event->func->init(event, 1 << type, index); } } } +static void +nvkm_event_ntfy_state(struct nvkm_event_ntfy *ntfy) +{ + struct nvkm_event *event = ntfy->event; + unsigned long flags; + + nvkm_trace(event->subdev, "event: ntfy state changed\n"); + spin_lock_irqsave(&event->refs_lock, flags); + + if (atomic_read(&ntfy->allowed) != ntfy->running) { + if (ntfy->running) { + nvkm_event_put(ntfy->event, ntfy->bits, ntfy->id); + ntfy->running = false; + } else { + nvkm_event_get(ntfy->event, ntfy->bits, ntfy->id); + ntfy->running = true; + } + } + + spin_unlock_irqrestore(&event->refs_lock, flags); +} + +static void +nvkm_event_ntfy_remove(struct nvkm_event_ntfy *ntfy) +{ + spin_lock_irq(&ntfy->event->list_lock); + list_del_init(&ntfy->head); + spin_unlock_irq(&ntfy->event->list_lock); +} + +static void +nvkm_event_ntfy_insert(struct nvkm_event_ntfy *ntfy) +{ + spin_lock_irq(&ntfy->event->list_lock); + list_add_tail(&ntfy->head, &ntfy->event->ntfy); + spin_unlock_irq(&ntfy->event->list_lock); +} + +static void +nvkm_event_ntfy_block_(struct nvkm_event_ntfy *ntfy, bool wait) +{ + struct nvkm_subdev *subdev = ntfy->event->subdev; + + nvkm_trace(subdev, "event: ntfy block %08x on %d wait:%d\n", ntfy->bits, ntfy->id, wait); + + if (atomic_xchg(&ntfy->allowed, 0) == 1) { + nvkm_event_ntfy_state(ntfy); + if (wait) + nvkm_event_ntfy_remove(ntfy); + } +} + +void +nvkm_event_ntfy_block(struct nvkm_event_ntfy *ntfy) +{ + if (ntfy->event) + nvkm_event_ntfy_block_(ntfy, ntfy->wait); +} + +void +nvkm_event_ntfy_allow(struct nvkm_event_ntfy *ntfy) +{ + nvkm_trace(ntfy->event->subdev, "event: ntfy allow %08x on %d\n", ntfy->bits, ntfy->id); + + if (atomic_xchg(&ntfy->allowed, 1) == 0) { + nvkm_event_ntfy_state(ntfy); + if (ntfy->wait) + nvkm_event_ntfy_insert(ntfy); + } +} + +void +nvkm_event_ntfy_del(struct nvkm_event_ntfy *ntfy) +{ + struct nvkm_event *event = ntfy->event; + + if (!event) + return; + + nvkm_trace(event->subdev, "event: ntfy del %08x on %d\n", ntfy->bits, ntfy->id); + + nvkm_event_ntfy_block_(ntfy, false); + nvkm_event_ntfy_remove(ntfy); + ntfy->event = NULL; +} + +void +nvkm_event_ntfy_add(struct nvkm_event *event, int id, u32 bits, bool wait, nvkm_event_func func, + struct nvkm_event_ntfy *ntfy) +{ + nvkm_trace(event->subdev, "event: ntfy add %08x on %d wait:%d\n", id, bits, wait); + + ntfy->event = event; + ntfy->id = id; + ntfy->bits = bits; + ntfy->wait = wait; + ntfy->func = func; + atomic_set(&ntfy->allowed, 0); + ntfy->running = false; + INIT_LIST_HEAD(&ntfy->head); + if (!ntfy->wait) + nvkm_event_ntfy_insert(ntfy); +} + +bool +nvkm_event_ntfy_valid(struct nvkm_event *event, int id, u32 bits) +{ + return true; +} + +void +nvkm_event_ntfy(struct nvkm_event *event, int id, u32 bits) +{ + struct nvkm_event_ntfy *ntfy, *ntmp; + + nvkm_trace(event->subdev, "event: ntfy %08x on %d\n", bits, id); + + list_for_each_entry_safe(ntfy, ntmp, &event->ntfy, head) { + if (ntfy->id == id && ntfy->bits & bits) { + if (atomic_read(&ntfy->allowed)) + ntfy->func(ntfy, ntfy->bits & bits); + } + } +} + void -nvkm_event_send(struct nvkm_event *event, u32 types, int index, - void *data, u32 size) +nvkm_event_send(struct nvkm_event *event, u32 types, int index, void *data, u32 size) { struct nvkm_notify *notify; unsigned long flags; @@ -59,6 +192,8 @@ nvkm_event_send(struct nvkm_event *event, u32 types, int index, return; spin_lock_irqsave(&event->list_lock, flags); + nvkm_event_ntfy(event, index, types); + list_for_each_entry(notify, &event->list, head) { if (notify->index == index && (notify->types & types)) { if (event->func->send) { @@ -81,20 +216,20 @@ nvkm_event_fini(struct nvkm_event *event) } int -nvkm_event_init(const struct nvkm_event_func *func, int types_nr, int index_nr, - struct nvkm_event *event) +nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, + int types_nr, int index_nr, struct nvkm_event *event) { - event->refs = kzalloc(array3_size(index_nr, types_nr, - sizeof(*event->refs)), - GFP_KERNEL); + event->refs = kzalloc(array3_size(index_nr, types_nr, sizeof(*event->refs)), GFP_KERNEL); if (!event->refs) return -ENOMEM; event->func = func; + event->subdev = subdev; event->types_nr = types_nr; event->index_nr = index_nr; spin_lock_init(&event->refs_lock); spin_lock_init(&event->list_lock); INIT_LIST_HEAD(&event->list); + INIT_LIST_HEAD(&event->ntfy); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index 45f920da89af5..b42d0bf2c492b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -47,6 +47,26 @@ nvkm_ioctl_nop(struct nvkm_client *client, return ret; } +#include + +static int +nvkm_ioctl_sclass_(struct nvkm_object *object, int index, struct nvkm_oclass *oclass) +{ + if ( object->func->uevent && + !object->func->uevent(object, NULL, 0, NULL) && index-- == 0) { + oclass->ctor = nvkm_uevent_new; + oclass->base.minver = 0; + oclass->base.maxver = 0; + oclass->base.oclass = NVIF_CLASS_EVENT; + return 0; + } + + if (object->func->sclass) + return object->func->sclass(object, index, oclass); + + return -ENOSYS; +} + static int nvkm_ioctl_sclass(struct nvkm_client *client, struct nvkm_object *object, void *data, u32 size) @@ -64,8 +84,7 @@ nvkm_ioctl_sclass(struct nvkm_client *client, if (size != args->v0.count * sizeof(args->v0.oclass[0])) return -EINVAL; - while (object->func->sclass && - object->func->sclass(object, i, &oclass) >= 0) { + while (nvkm_ioctl_sclass_(object, i, &oclass) >= 0) { if (i < args->v0.count) { args->v0.oclass[i].oclass = oclass.base.oclass; args->v0.oclass[i].minver = oclass.base.minver; @@ -100,7 +119,7 @@ nvkm_ioctl_new(struct nvkm_client *client, } else return ret; - if (!parent->func->sclass) { + if (!parent->func->sclass && !parent->func->uevent) { nvif_ioctl(parent, "cannot have children\n"); return -EINVAL; } @@ -113,7 +132,7 @@ nvkm_ioctl_new(struct nvkm_client *client, oclass.object = args->v0.object; oclass.client = client; oclass.parent = parent; - ret = parent->func->sclass(parent, i++, &oclass); + ret = nvkm_ioctl_sclass_(parent, i++, &oclass); if (ret) return ret; } while (oclass.base.oclass != args->v0.oclass); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c index 16299837a296c..3ffd4845d9e5b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c @@ -105,6 +105,18 @@ nvkm_oproxy_sclass(struct nvkm_object *object, int index, return oproxy->object->func->sclass(oproxy->object, index, oclass); } +static int +nvkm_oproxy_uevent(struct nvkm_object *object, void *argv, u32 argc, + struct nvkm_uevent *uevent) +{ + struct nvkm_oproxy *oproxy = nvkm_oproxy(object); + + if (!oproxy->object->func->uevent) + return -ENOSYS; + + return oproxy->object->func->uevent(oproxy->object, argv, argc, uevent); +} + static int nvkm_oproxy_fini(struct nvkm_object *object, bool suspend) { @@ -188,6 +200,7 @@ nvkm_oproxy_func = { .wr32 = nvkm_oproxy_wr32, .bind = nvkm_oproxy_bind, .sclass = nvkm_oproxy_sclass, + .uevent = nvkm_oproxy_uevent, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/core/uevent.c b/drivers/gpu/drm/nouveau/nvkm/core/uevent.c new file mode 100644 index 0000000000000..ba9d9edaec755 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/core/uevent.c @@ -0,0 +1,157 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define nvkm_uevent(p) container_of((p), struct nvkm_uevent, object) +#include +#include + +#include + +struct nvkm_uevent { + struct nvkm_object object; + struct nvkm_object *parent; + nvkm_uevent_func func; + bool wait; + + struct nvkm_event_ntfy ntfy; + atomic_t allowed; +}; + +static int +nvkm_uevent_mthd_block(struct nvkm_uevent *uevent, union nvif_event_block_args *args, u32 argc) +{ + if (argc != sizeof(args->vn)) + return -ENOSYS; + + nvkm_event_ntfy_block(&uevent->ntfy); + atomic_set(&uevent->allowed, 0); + return 0; +} + +static int +nvkm_uevent_mthd_allow(struct nvkm_uevent *uevent, union nvif_event_allow_args *args, u32 argc) +{ + if (argc != sizeof(args->vn)) + return -ENOSYS; + + nvkm_event_ntfy_allow(&uevent->ntfy); + atomic_set(&uevent->allowed, 1); + return 0; +} + +static int +nvkm_uevent_mthd(struct nvkm_object *object, u32 mthd, void *argv, u32 argc) +{ + struct nvkm_uevent *uevent = nvkm_uevent(object); + + switch (mthd) { + case NVIF_EVENT_V0_ALLOW: return nvkm_uevent_mthd_allow(uevent, argv, argc); + case NVIF_EVENT_V0_BLOCK: return nvkm_uevent_mthd_block(uevent, argv, argc); + default: + break; + } + + return -EINVAL; +} + +static int +nvkm_uevent_fini(struct nvkm_object *object, bool suspend) +{ + struct nvkm_uevent *uevent = nvkm_uevent(object); + + nvkm_event_ntfy_block(&uevent->ntfy); + return 0; +} + +static int +nvkm_uevent_init(struct nvkm_object *object) +{ + struct nvkm_uevent *uevent = nvkm_uevent(object); + + if (atomic_read(&uevent->allowed)) + nvkm_event_ntfy_allow(&uevent->ntfy); + + return 0; +} + +static void * +nvkm_uevent_dtor(struct nvkm_object *object) +{ + struct nvkm_uevent *uevent = nvkm_uevent(object); + + nvkm_event_ntfy_del(&uevent->ntfy); + return uevent; +} + +static const struct nvkm_object_func +nvkm_uevent = { + .dtor = nvkm_uevent_dtor, + .init = nvkm_uevent_init, + .fini = nvkm_uevent_fini, + .mthd = nvkm_uevent_mthd, +}; + +static int +nvkm_uevent_ntfy(struct nvkm_event_ntfy *ntfy, u32 bits) +{ + struct nvkm_uevent *uevent = container_of(ntfy, typeof(*uevent), ntfy); + struct nvkm_client *client = uevent->object.client; + + if (uevent->func) + return uevent->func(uevent->parent, uevent->object.token, bits); + + return client->event(uevent->object.token, NULL, 0); +} + +int +nvkm_uevent_add(struct nvkm_uevent *uevent, struct nvkm_event *event, int id, u32 bits, + nvkm_uevent_func func) +{ + if (WARN_ON(uevent->func)) + return -EBUSY; + + nvkm_event_ntfy_add(event, id, bits, uevent->wait, nvkm_uevent_ntfy, &uevent->ntfy); + uevent->func = func; + return 0; +} + +int +nvkm_uevent_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + struct nvkm_object *parent = oclass->parent; + struct nvkm_uevent *uevent; + union nvif_event_args *args = argv; + + if (argc < sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + if (!(uevent = kzalloc(sizeof(*uevent), GFP_KERNEL))) + return -ENOMEM; + *pobject = &uevent->object; + + nvkm_object_ctor(&nvkm_uevent, oclass, &uevent->object); + uevent->parent = parent; + uevent->func = NULL; + uevent->wait = args->v0.wait; + uevent->ntfy.event = NULL; + return parent->func->uevent(parent, &args->v0.data, argc - sizeof(args->v0), uevent); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c index 65c99d948b686..55c97dc314e16 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c @@ -343,9 +343,7 @@ nvkm_disp_oneinit(struct nvkm_engine *engine) /* Apparently we need to create a new one! */ ret = nvkm_conn_new(disp, i, &connE, &outp->conn); if (ret) { - nvkm_error(&disp->engine.subdev, - "failed to create outp %d conn: %d\n", - outp->index, ret); + nvkm_error(subdev, "failed to create outp %d conn: %d\n", outp->index, ret); nvkm_conn_del(&outp->conn); list_del(&outp->head); nvkm_outp_del(&outp); @@ -355,7 +353,7 @@ nvkm_disp_oneinit(struct nvkm_engine *engine) list_add_tail(&outp->conn->head, &disp->conns); } - ret = nvkm_event_init(&nvkm_disp_hpd_func, 3, hpd, &disp->hpd); + ret = nvkm_event_init(&nvkm_disp_hpd_func, subdev, 3, hpd, &disp->hpd); if (ret) return ret; @@ -382,7 +380,7 @@ nvkm_disp_oneinit(struct nvkm_engine *engine) list_for_each_entry(head, &disp->heads, head) i = max(i, head->id + 1); - return nvkm_event_init(&nvkm_disp_vblank_func, 1, i, &disp->vblank); + return nvkm_event_init(&nvkm_disp_vblank_func, subdev, 1, i, &disp->vblank); } static void * @@ -473,5 +471,6 @@ nvkm_disp_new_(const struct nvkm_disp_func *func, struct nvkm_device *device, mutex_init(&disp->super.mutex); } - return nvkm_event_init(func->uevent, 1, ARRAY_SIZE(disp->chan), &disp->uevent); + return nvkm_event_init(func->uevent, &disp->engine.subdev, 1, ARRAY_SIZE(disp->chan), + &disp->uevent); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 58b8df75fc402..c94b2b9b93294 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -347,11 +347,11 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, return ret; if (func->uevent_init) { - ret = nvkm_event_init(&nvkm_fifo_uevent_func, 1, 1, + ret = nvkm_event_init(&nvkm_fifo_uevent_func, &fifo->engine.subdev, 1, 1, &fifo->uevent); if (ret) return ret; } - return nvkm_event_init(&nvkm_fifo_kevent_func, 1, nr, &fifo->kevent); + return nvkm_event_init(&nvkm_fifo_kevent_func, &fifo->engine.subdev, 1, nr, &fifo->kevent); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c index f28967065639e..d156f86768965 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c @@ -107,5 +107,5 @@ nvkm_sw_chan_ctor(const struct nvkm_sw_chan_func *func, struct nvkm_sw *sw, list_add(&chan->head, &sw->chan); spin_unlock_irqrestore(&sw->engine.lock, flags); - return nvkm_event_init(&nvkm_sw_chan_event, 1, 1, &chan->event); + return nvkm_event_init(&nvkm_sw_chan_event, &sw->engine.subdev, 1, 1, &chan->event); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index fd54fa504efa0..cc6fa1bb50343 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -130,8 +130,7 @@ nvkm_fault_oneinit(struct nvkm_subdev *subdev) } } - ret = nvkm_event_init(&nvkm_fault_ntfy, 1, fault->buffer_nr, - &fault->event); + ret = nvkm_event_init(&nvkm_fault_ntfy, subdev, 1, fault->buffer_nr, &fault->event); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c index 048bcc70c3f48..f2ccbcf219ca8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c @@ -251,6 +251,5 @@ nvkm_gpio_new_(const struct nvkm_gpio_func *func, struct nvkm_device *device, nvkm_subdev_ctor(&nvkm_gpio, device, type, inst, &gpio->subdev); gpio->func = func; - return nvkm_event_init(&nvkm_gpio_intr_func, 2, func->lines, - &gpio->event); + return nvkm_event_init(&nvkm_gpio_intr_func, &gpio->subdev, 2, func->lines, &gpio->event); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c index cb5cb533d91c8..49a84ef146e9f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c @@ -427,5 +427,5 @@ nvkm_i2c_new_(const struct nvkm_i2c_func *func, struct nvkm_device *device, } } - return nvkm_event_init(&nvkm_i2c_intr_func, 4, i, &i2c->event); + return nvkm_event_init(&nvkm_i2c_intr_func, &i2c->subdev, 4, i, &i2c->event); } -- GitLab From 55520832d6e40c1e2099ce2c6c1e5ab9ecf57ff7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:37 +1000 Subject: [PATCH 1275/1927] drm/nouveau/fault: switch non-replayable faults to nvkm_event_ntfy v2: fix flush_work() being called uninitialised during init Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/subdev/fault.h | 5 ++- .../gpu/drm/nouveau/nvkm/subdev/fault/base.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/fault/gp100.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/fault/gv100.c | 41 ++++++++++++------- .../gpu/drm/nouveau/nvkm/subdev/fault/priv.h | 1 + .../gpu/drm/nouveau/nvkm/subdev/fault/tu102.c | 17 +++++--- 6 files changed, 44 insertions(+), 24 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h index 9c78f072d62b4..e781c5e4dcaeb 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h @@ -2,7 +2,6 @@ #define __NVKM_FAULT_H__ #include #include -#include struct nvkm_fault { const struct nvkm_fault_func *func; @@ -11,9 +10,11 @@ struct nvkm_fault { struct nvkm_fault_buffer *buffer[2]; int buffer_nr; +#define NVKM_FAULT_BUFFER_EVENT_PENDING BIT(0) struct nvkm_event event; - struct nvkm_notify nrpfb; + struct nvkm_event_ntfy nrpfb; + struct work_struct nrpfb_work; struct nvkm_device_oclass user; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index cc6fa1bb50343..7dd722c9b6601 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -145,7 +145,7 @@ nvkm_fault_dtor(struct nvkm_subdev *subdev) struct nvkm_fault *fault = nvkm_fault(subdev); int i; - nvkm_notify_fini(&fault->nrpfb); + nvkm_event_ntfy_del(&fault->nrpfb); nvkm_event_fini(&fault->event); for (i = 0; i < fault->buffer_nr; i++) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c index 6af7959e02ea6..a96a1d633cb31 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c @@ -65,7 +65,7 @@ gp100_fault_buffer_info(struct nvkm_fault_buffer *buffer) void gp100_fault_intr(struct nvkm_fault *fault) { - nvkm_event_send(&fault->event, 1, 0, NULL, 0); + nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 0, NULL, 0); } static const struct nvkm_fault_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c index cd9d2ade5ac7a..21756d65663b7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c @@ -27,10 +27,12 @@ #include -static void -gv100_fault_buffer_process(struct nvkm_fault_buffer *buffer) +void +gv100_fault_buffer_process(struct work_struct *work) { - struct nvkm_device *device = buffer->fault->subdev.device; + struct nvkm_fault *fault = container_of(work, typeof(*fault), nrpfb_work); + struct nvkm_fault_buffer *buffer = fault->buffer[0]; + struct nvkm_device *device = fault->subdev.device; struct nvkm_memory *mem = buffer->mem; u32 get = nvkm_rd32(device, buffer->get); u32 put = nvkm_rd32(device, buffer->put); @@ -115,11 +117,12 @@ gv100_fault_buffer_info(struct nvkm_fault_buffer *buffer) } static int -gv100_fault_ntfy_nrpfb(struct nvkm_notify *notify) +gv100_fault_ntfy_nrpfb(struct nvkm_event_ntfy *ntfy, u32 bits) { - struct nvkm_fault *fault = container_of(notify, typeof(*fault), nrpfb); - gv100_fault_buffer_process(fault->buffer[0]); - return NVKM_NOTIFY_KEEP; + struct nvkm_fault *fault = container_of(ntfy, typeof(*fault), nrpfb); + + schedule_work(&fault->nrpfb_work); + return NVKM_EVENT_KEEP; } static void @@ -163,14 +166,14 @@ gv100_fault_intr(struct nvkm_fault *fault) if (stat & 0x20000000) { if (fault->buffer[0]) { - nvkm_event_send(&fault->event, 1, 0, NULL, 0); + nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 0, NULL, 0); stat &= ~0x20000000; } } if (stat & 0x08000000) { if (fault->buffer[1]) { - nvkm_event_send(&fault->event, 1, 1, NULL, 0); + nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 1, NULL, 0); stat &= ~0x08000000; } } @@ -183,9 +186,12 @@ gv100_fault_intr(struct nvkm_fault *fault) static void gv100_fault_fini(struct nvkm_fault *fault) { - nvkm_notify_put(&fault->nrpfb); + nvkm_event_ntfy_block(&fault->nrpfb); + flush_work(&fault->nrpfb_work); + if (fault->buffer[0]) fault->func->buffer.fini(fault->buffer[0]); + nvkm_mask(fault->subdev.device, 0x100a34, 0x80000000, 0x80000000); } @@ -194,15 +200,15 @@ gv100_fault_init(struct nvkm_fault *fault) { nvkm_mask(fault->subdev.device, 0x100a2c, 0x80000000, 0x80000000); fault->func->buffer.init(fault->buffer[0]); - nvkm_notify_get(&fault->nrpfb); + nvkm_event_ntfy_allow(&fault->nrpfb); } int gv100_fault_oneinit(struct nvkm_fault *fault) { - return nvkm_notify_init(&fault->buffer[0]->object, &fault->event, - gv100_fault_ntfy_nrpfb, true, NULL, 0, 0, - &fault->nrpfb); + nvkm_event_ntfy_add(&fault->event, 0, NVKM_FAULT_BUFFER_EVENT_PENDING, true, + gv100_fault_ntfy_nrpfb, &fault->nrpfb); + return 0; } static const struct nvkm_fault_func @@ -231,5 +237,10 @@ int gv100_fault_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fault **pfault) { - return nvkm_fault_new_(&gv100_fault, device, type, inst, pfault); + int ret = nvkm_fault_new_(&gv100_fault, device, type, inst, pfault); + if (ret) + return ret; + + INIT_WORK(&(*pfault)->nrpfb_work, gv100_fault_buffer_process); + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h index 36681c347fb59..30cc0d0888972 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h @@ -46,6 +46,7 @@ void gp100_fault_buffer_fini(struct nvkm_fault_buffer *); void gp100_fault_buffer_init(struct nvkm_fault_buffer *); u64 gp100_fault_buffer_pin(struct nvkm_fault_buffer *); void gp100_fault_buffer_info(struct nvkm_fault_buffer *); +void gv100_fault_buffer_process(struct work_struct *); void gp100_fault_intr(struct nvkm_fault *); u64 gp10b_fault_buffer_pin(struct nvkm_fault_buffer *); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c index 91eb6729c84d6..2d3660d21b9a9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c @@ -126,7 +126,7 @@ tu102_fault_intr(struct nvkm_fault *fault) nvkm_wr32(device, 0xb81010, 0x10); if (fault->buffer[0]) { - nvkm_event_send(&fault->event, 1, 0, NULL, 0); + nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 0, NULL, 0); stat &= ~0x00000200; } } @@ -137,7 +137,7 @@ tu102_fault_intr(struct nvkm_fault *fault) nvkm_wr32(device, 0xb81008, 0x1); if (fault->buffer[1]) { - nvkm_event_send(&fault->event, 1, 1, NULL, 0); + nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 1, NULL, 0); stat &= ~0x00000100; } } @@ -150,7 +150,9 @@ tu102_fault_intr(struct nvkm_fault *fault) static void tu102_fault_fini(struct nvkm_fault *fault) { - nvkm_notify_put(&fault->nrpfb); + nvkm_event_ntfy_block(&fault->nrpfb); + flush_work(&fault->nrpfb_work); + if (fault->buffer[0]) fault->func->buffer.fini(fault->buffer[0]); /*XXX: disable priv faults */ @@ -161,7 +163,7 @@ tu102_fault_init(struct nvkm_fault *fault) { /*XXX: enable priv faults */ fault->func->buffer.init(fault->buffer[0]); - nvkm_notify_get(&fault->nrpfb); + nvkm_event_ntfy_allow(&fault->nrpfb); } static const struct nvkm_fault_func @@ -184,5 +186,10 @@ int tu102_fault_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fault **pfault) { - return nvkm_fault_new_(&tu102_fault, device, type, inst, pfault); + int ret = nvkm_fault_new_(&tu102_fault, device, type, inst, pfault); + if (ret) + return ret; + + INIT_WORK(&(*pfault)->nrpfb_work, gv100_fault_buffer_process); + return 0; } -- GitLab From b418ff8863eec01b39f32eee0417a216f4cdb24c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:37 +1000 Subject: [PATCH 1276/1927] drm/nouveau/fault: expose replayable fault buffer event class Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/clb069.h | 5 +- drivers/gpu/drm/nouveau/nouveau_svm.c | 50 ++++++++++++------- .../gpu/drm/nouveau/nvkm/subdev/fault/base.c | 16 ------ .../gpu/drm/nouveau/nvkm/subdev/fault/user.c | 30 ++++++----- 4 files changed, 53 insertions(+), 48 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/clb069.h b/drivers/gpu/drm/nouveau/include/nvif/clb069.h index eef5d0227babe..d7689de35ab2b 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/clb069.h +++ b/drivers/gpu/drm/nouveau/include/nvif/clb069.h @@ -8,5 +8,8 @@ struct nvif_clb069_v0 { __u32 put; }; -#define NVB069_V0_NTFY_FAULT 0x00 +union nvif_clb069_event_args { + struct nvif_clb069_event_vn { + } vn; +}; #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_svm.c b/drivers/gpu/drm/nouveau/nouveau_svm.c index 31a5b81ee9fc4..a74ba8d84ba7f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_svm.c +++ b/drivers/gpu/drm/nouveau/nouveau_svm.c @@ -24,7 +24,7 @@ #include "nouveau_chan.h" #include "nouveau_dmem.h" -#include +#include #include #include @@ -51,7 +51,8 @@ struct nouveau_svm { u32 putaddr; u32 get; u32 put; - struct nvif_notify notify; + struct nvif_event notify; + struct work_struct work; struct nouveau_svm_fault { u64 inst; @@ -711,13 +712,11 @@ static int nouveau_range_fault(struct nouveau_svmm *svmm, return ret; } -static int -nouveau_svm_fault(struct nvif_notify *notify) +static void +nouveau_svm_fault(struct work_struct *work) { - struct nouveau_svm_fault_buffer *buffer = - container_of(notify, typeof(*buffer), notify); - struct nouveau_svm *svm = - container_of(buffer, typeof(*svm), buffer[buffer->id]); + struct nouveau_svm_fault_buffer *buffer = container_of(work, typeof(*buffer), work); + struct nouveau_svm *svm = container_of(buffer, typeof(*svm), buffer[buffer->id]); struct nvif_object *device = &svm->drm->client.device.object; struct nouveau_svmm *svmm; struct { @@ -737,7 +736,7 @@ nouveau_svm_fault(struct nvif_notify *notify) buffer->put = nvif_rd32(device, buffer->putaddr); buffer->get = nvif_rd32(device, buffer->getaddr); if (buffer->get == buffer->put) - return NVIF_NOTIFY_KEEP; + return; } buffer->fault_nr = 0; @@ -881,7 +880,15 @@ nouveau_svm_fault(struct nvif_notify *notify) /* Issue fault replay to the GPU. */ if (replay) nouveau_svm_fault_replay(svm); - return NVIF_NOTIFY_KEEP; +} + +static int +nouveau_svm_event(struct nvif_event *event, void *argv, u32 argc) +{ + struct nouveau_svm_fault_buffer *buffer = container_of(event, typeof(*buffer), notify); + + schedule_work(&buffer->work); + return NVIF_EVENT_KEEP; } static struct nouveau_pfnmap_args * @@ -936,7 +943,9 @@ static void nouveau_svm_fault_buffer_fini(struct nouveau_svm *svm, int id) { struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; - nvif_notify_put(&buffer->notify); + + nvif_event_block(&buffer->notify); + flush_work(&buffer->work); } static int @@ -944,10 +953,12 @@ nouveau_svm_fault_buffer_init(struct nouveau_svm *svm, int id) { struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; struct nvif_object *device = &svm->drm->client.device.object; + buffer->get = nvif_rd32(device, buffer->getaddr); buffer->put = nvif_rd32(device, buffer->putaddr); SVM_DBG(svm, "get %08x put %08x (init)", buffer->get, buffer->put); - return nvif_notify_get(&buffer->notify); + + return nvif_event_allow(&buffer->notify); } static void @@ -956,15 +967,18 @@ nouveau_svm_fault_buffer_dtor(struct nouveau_svm *svm, int id) struct nouveau_svm_fault_buffer *buffer = &svm->buffer[id]; int i; + if (!nvif_object_constructed(&buffer->object)) + return; + + nouveau_svm_fault_buffer_fini(svm, id); + if (buffer->fault) { for (i = 0; buffer->fault[i] && i < buffer->entries; i++) kfree(buffer->fault[i]); kvfree(buffer->fault); } - nouveau_svm_fault_buffer_fini(svm, id); - - nvif_notify_dtor(&buffer->notify); + nvif_event_dtor(&buffer->notify); nvif_object_dtor(&buffer->object); } @@ -990,10 +1004,10 @@ nouveau_svm_fault_buffer_ctor(struct nouveau_svm *svm, s32 oclass, int id) buffer->entries = args.entries; buffer->getaddr = args.get; buffer->putaddr = args.put; + INIT_WORK(&buffer->work, nouveau_svm_fault); - ret = nvif_notify_ctor(&buffer->object, "svmFault", nouveau_svm_fault, - true, NVB069_V0_NTFY_FAULT, NULL, 0, 0, - &buffer->notify); + ret = nvif_event_ctor(&buffer->object, "svmFault", id, nouveau_svm_event, true, NULL, 0, + &buffer->notify); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c index 7dd722c9b6601..b53ac9a2552fa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/base.c @@ -22,7 +22,6 @@ #include "priv.h" #include -#include static void nvkm_fault_ntfy_fini(struct nvkm_event *event, int type, int index) @@ -38,23 +37,8 @@ nvkm_fault_ntfy_init(struct nvkm_event *event, int type, int index) fault->func->buffer.intr(fault->buffer[index], true); } -static int -nvkm_fault_ntfy_ctor(struct nvkm_object *object, void *argv, u32 argc, - struct nvkm_notify *notify) -{ - struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); - if (argc == 0) { - notify->size = 0; - notify->types = 1; - notify->index = buffer->id; - return 0; - } - return -ENOSYS; -} - static const struct nvkm_event_func nvkm_fault_ntfy = { - .ctor = nvkm_fault_ntfy_ctor, .init = nvkm_fault_ntfy_init, .fini = nvkm_fault_ntfy_fini, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c index ac835c9582fd2..c123e5893d76e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/user.c @@ -22,11 +22,27 @@ #include "priv.h" #include +#include #include #include #include +static int +nvkm_ufault_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) +{ + struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); + union nvif_clb069_event_args *args = argv; + + if (!uevent) + return 0; + if (argc != sizeof(args->vn)) + return -ENOSYS; + + return nvkm_uevent_add(uevent, &buffer->fault->event, buffer->id, + NVKM_FAULT_BUFFER_EVENT_PENDING, NULL); +} + static int nvkm_ufault_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) @@ -39,18 +55,6 @@ nvkm_ufault_map(struct nvkm_object *object, void *argv, u32 argc, return 0; } -static int -nvkm_ufault_ntfy(struct nvkm_object *object, u32 type, - struct nvkm_event **pevent) -{ - struct nvkm_fault_buffer *buffer = nvkm_fault_buffer(object); - if (type == NVB069_V0_NTFY_FAULT) { - *pevent = &buffer->fault->event; - return 0; - } - return -EINVAL; -} - static int nvkm_ufault_fini(struct nvkm_object *object, bool suspend) { @@ -78,8 +82,8 @@ nvkm_ufault = { .dtor = nvkm_ufault_dtor, .init = nvkm_ufault_init, .fini = nvkm_ufault_fini, - .ntfy = nvkm_ufault_ntfy, .map = nvkm_ufault_map, + .uevent = nvkm_ufault_uevent, }; int -- GitLab From 1d4dce284164de21cfbab05d0b763711c428df45 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:37 +1000 Subject: [PATCH 1277/1927] drm/nouveau/disp: switch vblank semaphore release to nvkm_event_ntfy Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/event.h | 2 -- .../drm/nouveau/include/nvkm/engine/disp.h | 1 + .../gpu/drm/nouveau/nvkm/engine/disp/base.c | 3 +-- .../gpu/drm/nouveau/nvkm/engine/sw/gf100.c | 20 +++++----------- drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c | 24 +++++++------------ drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.h | 4 ++-- 6 files changed, 19 insertions(+), 35 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/event.h b/drivers/gpu/drm/nouveau/include/nvif/event.h index a510ba4ad4879..75211961c27b0 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/event.h +++ b/drivers/gpu/drm/nouveau/include/nvif/event.h @@ -61,8 +61,6 @@ struct nvif_notify_head_req_v0 { struct nvif_notify_head_rep_v0 { /* nvif_notify_rep ... */ - __u8 version; - __u8 pad01[7]; }; struct nvif_notify_conn_req_v0 { diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h index eaf10f5d505e1..ad9aef2df48f7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h @@ -16,6 +16,7 @@ struct nvkm_disp { struct list_head conns; struct nvkm_event hpd; +#define NVKM_DISP_HEAD_EVENT_VBLANK BIT(0) struct nvkm_event vblank; struct { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c index 55c97dc314e16..7d286138e5cd8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c @@ -90,8 +90,7 @@ nvkm_disp_vblank_func = { void nvkm_disp_vblank(struct nvkm_disp *disp, int head) { - struct nvif_notify_head_rep_v0 rep = {}; - nvkm_event_send(&disp->vblank, 1, head, &rep, sizeof(rep)); + nvkm_event_send(&disp->vblank, NVKM_DISP_HEAD_EVENT_VBLANK, head, NULL, 0); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c index 55abf839f29d5..c3cf6f2ff86c3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/gf100.c @@ -36,10 +36,10 @@ ******************************************************************************/ static int -gf100_sw_chan_vblsem_release(struct nvkm_notify *notify) +gf100_sw_chan_vblsem_release(struct nvkm_event_ntfy *notify, u32 bits) { struct nv50_sw_chan *chan = - container_of(notify, typeof(*chan), vblank.notify[notify->index]); + container_of(notify, typeof(*chan), vblank.notify[notify->id]); struct nvkm_sw *sw = chan->base.sw; struct nvkm_device *device = sw->engine.subdev.device; u32 inst = chan->base.fifo->inst->addr >> 12; @@ -50,7 +50,7 @@ gf100_sw_chan_vblsem_release(struct nvkm_notify *notify) nvkm_wr32(device, 0x060010, lower_32_bits(chan->vblank.offset)); nvkm_wr32(device, 0x060014, chan->vblank.value); - return NVKM_NOTIFY_DROP; + return NVKM_EVENT_DROP; } static bool @@ -73,7 +73,7 @@ gf100_sw_chan_mthd(struct nvkm_sw_chan *base, int subc, u32 mthd, u32 data) return true; case 0x040c: if (data < device->disp->vblank.index_nr) { - nvkm_notify_get(&chan->vblank.notify[data]); + nvkm_event_ntfy_allow(&chan->vblank.notify[data]); return true; } break; @@ -120,16 +120,8 @@ gf100_sw_chan_new(struct nvkm_sw *sw, struct nvkm_fifo_chan *fifoch, return ret; for (i = 0; disp && i < disp->vblank.index_nr; i++) { - ret = nvkm_notify_init(NULL, &disp->vblank, - gf100_sw_chan_vblsem_release, false, - &(struct nvif_notify_head_req_v0) { - .head = i, - }, - sizeof(struct nvif_notify_head_req_v0), - sizeof(struct nvif_notify_head_rep_v0), - &chan->vblank.notify[i]); - if (ret) - return ret; + nvkm_event_ntfy_add(&disp->vblank, i, NVKM_DISP_HEAD_EVENT_VBLANK, true, + gf100_sw_chan_vblsem_release, &chan->vblank.notify[i]); } return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c index 1fdd094c8b7e5..9d7a9b7d5be3a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.c @@ -36,10 +36,10 @@ ******************************************************************************/ static int -nv50_sw_chan_vblsem_release(struct nvkm_notify *notify) +nv50_sw_chan_vblsem_release(struct nvkm_event_ntfy *notify, u32 bits) { struct nv50_sw_chan *chan = - container_of(notify, typeof(*chan), vblank.notify[notify->index]); + container_of(notify, typeof(*chan), vblank.notify[notify->id]); struct nvkm_sw *sw = chan->base.sw; struct nvkm_device *device = sw->engine.subdev.device; @@ -55,7 +55,7 @@ nv50_sw_chan_vblsem_release(struct nvkm_notify *notify) nvkm_wr32(device, 0x060014, chan->vblank.value); } - return NVKM_NOTIFY_DROP; + return NVKM_EVENT_DROP; } static bool @@ -70,7 +70,7 @@ nv50_sw_chan_mthd(struct nvkm_sw_chan *base, int subc, u32 mthd, u32 data) case 0x0404: chan->vblank.value = data; return true; case 0x0408: if (data < device->disp->vblank.index_nr) { - nvkm_notify_get(&chan->vblank.notify[data]); + nvkm_event_ntfy_allow(&chan->vblank.notify[data]); return true; } break; @@ -85,8 +85,10 @@ nv50_sw_chan_dtor(struct nvkm_sw_chan *base) { struct nv50_sw_chan *chan = nv50_sw_chan(base); int i; + for (i = 0; i < ARRAY_SIZE(chan->vblank.notify); i++) - nvkm_notify_fini(&chan->vblank.notify[i]); + nvkm_event_ntfy_del(&chan->vblank.notify[i]); + return chan; } @@ -113,16 +115,8 @@ nv50_sw_chan_new(struct nvkm_sw *sw, struct nvkm_fifo_chan *fifoch, return ret; for (i = 0; disp && i < disp->vblank.index_nr; i++) { - ret = nvkm_notify_init(NULL, &disp->vblank, - nv50_sw_chan_vblsem_release, false, - &(struct nvif_notify_head_req_v0) { - .head = i, - }, - sizeof(struct nvif_notify_head_req_v0), - sizeof(struct nvif_notify_head_rep_v0), - &chan->vblank.notify[i]); - if (ret) - return ret; + nvkm_event_ntfy_add(&disp->vblank, i, NVKM_DISP_HEAD_EVENT_VBLANK, true, + nv50_sw_chan_vblsem_release, &chan->vblank.notify[i]); } return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.h index 6d364d7b406a4..b42289ce8826a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nv50.h @@ -5,12 +5,12 @@ #include "priv.h" #include "chan.h" #include "nvsw.h" -#include +#include struct nv50_sw_chan { struct nvkm_sw_chan base; struct { - struct nvkm_notify notify[4]; + struct nvkm_event_ntfy notify[4]; u32 ctxdma; u64 offset; u32 value; -- GitLab From ffd2664114c8fb9f12c4d4fd09c6d57cc3f4d951 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:38 +1000 Subject: [PATCH 1278/1927] drm/nouveau/disp: expose head event class Also fixes vblank interrupts being left enabled when they're not meant to be as a result of races/bugs in previous event handling code. v2: - use ?: (lyude) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv04/crtc.c | 22 +++++---------- drivers/gpu/drm/nouveau/dispnv50/head.c | 20 +++++--------- drivers/gpu/drm/nouveau/include/nvif/cl0046.h | 1 - drivers/gpu/drm/nouveau/include/nvif/event.h | 11 -------- drivers/gpu/drm/nouveau/include/nvif/head.h | 10 +++++++ drivers/gpu/drm/nouveau/include/nvif/if0013.h | 5 ++++ drivers/gpu/drm/nouveau/nouveau_crtc.h | 4 +-- drivers/gpu/drm/nouveau/nouveau_display.c | 4 +-- drivers/gpu/drm/nouveau/nvif/head.c | 10 +++++++ .../gpu/drm/nouveau/nvkm/engine/disp/base.c | 27 ------------------- .../gpu/drm/nouveau/nvkm/engine/disp/uhead.c | 19 +++++++++++++ 11 files changed, 62 insertions(+), 71 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c index f35c25607508c..80b0b5cf4079e 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c +++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c @@ -761,7 +761,7 @@ static void nv_crtc_destroy(struct drm_crtc *crtc) nouveau_bo_unmap(nv_crtc->cursor.nvbo); nouveau_bo_unpin(nv_crtc->cursor.nvbo); nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo); - nvif_notify_dtor(&nv_crtc->vblank); + nvif_event_dtor(&nv_crtc->vblank); nvif_head_dtor(&nv_crtc->head); kfree(nv_crtc); } @@ -1280,13 +1280,13 @@ static const struct drm_plane_funcs nv04_primary_plane_funcs = { DRM_PLANE_NON_ATOMIC_FUNCS, }; -static int nv04_crtc_vblank_handler(struct nvif_notify *notify) +static int +nv04_crtc_vblank_handler(struct nvif_event *event, void *repv, u32 repc) { - struct nouveau_crtc *nv_crtc = - container_of(notify, struct nouveau_crtc, vblank); + struct nouveau_crtc *nv_crtc = container_of(event, struct nouveau_crtc, vblank); drm_crtc_handle_vblank(&nv_crtc->base); - return NVIF_NOTIFY_KEEP; + return NVIF_EVENT_KEEP; } int @@ -1346,14 +1346,6 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num) if (ret) return ret; - ret = nvif_notify_ctor(&disp->disp.object, "kmsVbl", nv04_crtc_vblank_handler, - false, NV04_DISP_NTFY_VBLANK, - &(struct nvif_notify_head_req_v0) { - .head = nv_crtc->index, - }, - sizeof(struct nvif_notify_head_req_v0), - sizeof(struct nvif_notify_head_rep_v0), - &nv_crtc->vblank); - - return ret; + return nvif_head_vblank_event_ctor(&nv_crtc->head, "kmsVbl", nv04_crtc_vblank_handler, + false, &nv_crtc->vblank); } diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.c b/drivers/gpu/drm/nouveau/dispnv50/head.c index 8ac3474111c54..f006e56e1e08b 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/head.c +++ b/drivers/gpu/drm/nouveau/dispnv50/head.c @@ -517,7 +517,7 @@ nv50_head_destroy(struct drm_crtc *crtc) { struct nv50_head *head = nv50_head(crtc); - nvif_notify_dtor(&head->base.vblank); + nvif_event_dtor(&head->base.vblank); nvif_head_dtor(&head->base.head); nv50_lut_fini(&head->olut); drm_crtc_cleanup(crtc); @@ -555,15 +555,15 @@ nvd9_head_func = { .late_register = nv50_head_late_register, }; -static int nv50_head_vblank_handler(struct nvif_notify *notify) +static int +nv50_head_vblank_handler(struct nvif_event *event, void *repv, u32 repc) { - struct nouveau_crtc *nv_crtc = - container_of(notify, struct nouveau_crtc, vblank); + struct nouveau_crtc *nv_crtc = container_of(event, struct nouveau_crtc, vblank); if (drm_crtc_handle_vblank(&nv_crtc->base)) nv50_crc_handle_vblank(nv50_head(&nv_crtc->base)); - return NVIF_NOTIFY_KEEP; + return NVIF_EVENT_KEEP; } struct nv50_head * @@ -629,14 +629,8 @@ nv50_head_create(struct drm_device *dev, int index) if (ret) return ERR_PTR(ret); - ret = nvif_notify_ctor(&disp->disp->object, "kmsVbl", nv50_head_vblank_handler, - false, NV04_DISP_NTFY_VBLANK, - &(struct nvif_notify_head_req_v0) { - .head = nv_crtc->index, - }, - sizeof(struct nvif_notify_head_req_v0), - sizeof(struct nvif_notify_head_rep_v0), - &nv_crtc->vblank); + ret = nvif_head_vblank_event_ctor(&head->base.head, "kmsVbl", nv50_head_vblank_handler, + false, &nv_crtc->vblank); if (ret) return ERR_PTR(ret); diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0046.h b/drivers/gpu/drm/nouveau/include/nvif/cl0046.h index b3ac930b01fa5..eca7c3950654a 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl0046.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl0046.h @@ -2,6 +2,5 @@ #ifndef __NVIF_CL0046_H__ #define __NVIF_CL0046_H__ -#define NV04_DISP_NTFY_VBLANK 0x00 #define NV04_DISP_NTFY_CONN 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/event.h b/drivers/gpu/drm/nouveau/include/nvif/event.h index 75211961c27b0..679950400f536 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/event.h +++ b/drivers/gpu/drm/nouveau/include/nvif/event.h @@ -52,17 +52,6 @@ struct nvif_notify_rep_v0 { __u8 data[]; /* reply data (below) */ }; -struct nvif_notify_head_req_v0 { - /* nvif_notify_req ... */ - __u8 version; - __u8 head; - __u8 pad02[6]; -}; - -struct nvif_notify_head_rep_v0 { - /* nvif_notify_rep ... */ -}; - struct nvif_notify_conn_req_v0 { /* nvif_notify_req ... */ __u8 version; diff --git a/drivers/gpu/drm/nouveau/include/nvif/head.h b/drivers/gpu/drm/nouveau/include/nvif/head.h index 09170c7bc0afd..3ec36999e9560 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/head.h +++ b/drivers/gpu/drm/nouveau/include/nvif/head.h @@ -2,6 +2,7 @@ #ifndef __NVIF_HEAD_H__ #define __NVIF_HEAD_H__ #include +#include struct nvif_disp; struct nvif_head { @@ -10,4 +11,13 @@ struct nvif_head { int nvif_head_ctor(struct nvif_disp *, const char *name, int id, struct nvif_head *); void nvif_head_dtor(struct nvif_head *); + +static inline int +nvif_head_id(struct nvif_head *head) +{ + return head->object.handle; +} + +int nvif_head_vblank_event_ctor(struct nvif_head *, const char *name, nvif_event_func, bool wait, + struct nvif_event *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0013.h b/drivers/gpu/drm/nouveau/include/nvif/if0013.h index 4bd9507a200bd..6756c7467ae4e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0013.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0013.h @@ -10,6 +10,11 @@ union nvif_head_args { } v0; }; +union nvif_head_event_args { + struct nvif_head_event_vn { + } vn; +}; + #define NVIF_HEAD_V0_SCANOUTPOS 0x00 union nvif_head_scanoutpos_args { diff --git a/drivers/gpu/drm/nouveau/nouveau_crtc.h b/drivers/gpu/drm/nouveau/nouveau_crtc.h index e85e74f9a28dc..c717f664a7b8d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_crtc.h +++ b/drivers/gpu/drm/nouveau/nouveau_crtc.h @@ -29,14 +29,14 @@ #include #include -#include +#include struct nouveau_crtc { struct drm_crtc base; struct nvif_head head; int index; - struct nvif_notify vblank; + struct nvif_event vblank; uint32_t dpms_saved_fp_control; uint32_t fp_users; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 2034346f3934d..e88845ae75204 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -52,7 +52,7 @@ nouveau_display_vblank_enable(struct drm_crtc *crtc) struct nouveau_crtc *nv_crtc; nv_crtc = nouveau_crtc(crtc); - nvif_notify_get(&nv_crtc->vblank); + nvif_event_allow(&nv_crtc->vblank); return 0; } @@ -63,7 +63,7 @@ nouveau_display_vblank_disable(struct drm_crtc *crtc) struct nouveau_crtc *nv_crtc; nv_crtc = nouveau_crtc(crtc); - nvif_notify_put(&nv_crtc->vblank); + nvif_event_block(&nv_crtc->vblank); } static inline int diff --git a/drivers/gpu/drm/nouveau/nvif/head.c b/drivers/gpu/drm/nouveau/nvif/head.c index 01deba4626000..f00e01d232db4 100644 --- a/drivers/gpu/drm/nouveau/nvif/head.c +++ b/drivers/gpu/drm/nouveau/nvif/head.c @@ -26,6 +26,16 @@ #include #include +int +nvif_head_vblank_event_ctor(struct nvif_head *head, const char *name, nvif_event_func func, + bool wait, struct nvif_event *event) +{ + int ret = nvif_event_ctor(&head->object, name ?: "nvifHeadVBlank", nvif_head_id(head), + func, wait, NULL, 0, event); + NVIF_ERRON(ret, &head->object, "[NEW EVENT:VBLANK]"); + return ret; +} + void nvif_head_dtor(struct nvif_head *head) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c index 7d286138e5cd8..399195946823e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c @@ -57,32 +57,8 @@ nvkm_disp_vblank_init(struct nvkm_event *event, int type, int id) head->func->vblank_get(head); } -static int -nvkm_disp_vblank_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - struct nvkm_disp *disp = - container_of(notify->event, typeof(*disp), vblank); - union { - struct nvif_notify_head_req_v0 v0; - } *req = data; - int ret = -ENOSYS; - - if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, false))) { - notify->size = sizeof(struct nvif_notify_head_rep_v0); - if (ret = -ENXIO, req->v0.head <= disp->vblank.index_nr) { - notify->types = 1; - notify->index = req->v0.head; - return 0; - } - } - - return ret; -} - static const struct nvkm_event_func nvkm_disp_vblank_func = { - .ctor = nvkm_disp_vblank_ctor, .init = nvkm_disp_vblank_init, .fini = nvkm_disp_vblank_fini, }; @@ -132,9 +108,6 @@ nvkm_disp_ntfy(struct nvkm_object *object, u32 type, struct nvkm_event **event) { struct nvkm_disp *disp = nvkm_disp(object->engine); switch (type) { - case NV04_DISP_NTFY_VBLANK: - *event = &disp->vblank; - return 0; case NV04_DISP_NTFY_CONN: *event = &disp->hpd; return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c index 5f3c4b7dbb75b..f072cec160402 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uhead.c @@ -21,9 +21,27 @@ */ #define nvkm_uhead(p) container_of((p), struct nvkm_head, object) #include "head.h" +#include #include +#include + +static int +nvkm_uhead_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) +{ + struct nvkm_head *head = nvkm_uhead(object); + union nvif_head_event_args *args = argv; + + if (!uevent) + return 0; + if (argc != sizeof(args->vn)) + return -ENOSYS; + + return nvkm_uevent_add(uevent, &head->disp->vblank, head->id, + NVKM_DISP_HEAD_EVENT_VBLANK, NULL); +} + static int nvkm_uhead_mthd_scanoutpos(struct nvkm_head *head, void *argv, u32 argc) { @@ -81,6 +99,7 @@ static const struct nvkm_object_func nvkm_uhead = { .dtor = nvkm_uhead_dtor, .mthd = nvkm_uhead_mthd, + .uevent = nvkm_uhead_uevent, }; int -- GitLab From 773eb04d14a11552b2c3953097ed09cde2ab4831 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:38 +1000 Subject: [PATCH 1279/1927] drm/nouveau/disp: expose conn event class This removes some now-unnecessary nesting of workqueues. v2: - use ?: (lyude) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/conn.h | 10 +++ drivers/gpu/drm/nouveau/include/nvif/event.h | 19 ----- drivers/gpu/drm/nouveau/include/nvif/if0011.h | 11 +++ .../drm/nouveau/include/nvkm/subdev/gpio.h | 6 +- .../gpu/drm/nouveau/include/nvkm/subdev/i2c.h | 19 ++--- drivers/gpu/drm/nouveau/nouveau_connector.c | 67 +++++++++-------- drivers/gpu/drm/nouveau/nouveau_connector.h | 6 +- drivers/gpu/drm/nouveau/nouveau_display.c | 16 ++-- drivers/gpu/drm/nouveau/nouveau_dp.c | 15 ++-- drivers/gpu/drm/nouveau/nouveau_encoder.h | 3 +- drivers/gpu/drm/nouveau/nvif/conn.c | 19 +++++ .../gpu/drm/nouveau/nvkm/engine/disp/base.c | 54 ------------- .../gpu/drm/nouveau/nvkm/engine/disp/conn.c | 39 ---------- .../gpu/drm/nouveau/nvkm/engine/disp/conn.h | 3 - drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c | 40 ---------- .../gpu/drm/nouveau/nvkm/engine/disp/outp.h | 2 - .../gpu/drm/nouveau/nvkm/engine/disp/priv.h | 2 - .../gpu/drm/nouveau/nvkm/engine/disp/uconn.c | 75 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/disp/udisp.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/gpio/base.c | 24 +----- .../gpu/drm/nouveau/nvkm/subdev/i2c/base.c | 25 +------ 21 files changed, 187 insertions(+), 269 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/conn.h b/drivers/gpu/drm/nouveau/include/nvif/conn.h index f72a8f138f47f..dc355e1dfafa0 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/conn.h +++ b/drivers/gpu/drm/nouveau/include/nvif/conn.h @@ -2,6 +2,7 @@ #ifndef __NVIF_CONN_H__ #define __NVIF_CONN_H__ #include +#include struct nvif_disp; struct nvif_conn { @@ -11,8 +12,17 @@ struct nvif_conn { int nvif_conn_ctor(struct nvif_disp *, const char *name, int id, struct nvif_conn *); void nvif_conn_dtor(struct nvif_conn *); +static inline int +nvif_conn_id(struct nvif_conn *conn) +{ + return conn->object.handle; +} + #define NVIF_CONN_HPD_STATUS_UNSUPPORTED 0 /* negative if query fails */ #define NVIF_CONN_HPD_STATUS_NOT_PRESENT 1 #define NVIF_CONN_HPD_STATUS_PRESENT 2 int nvif_conn_hpd_status(struct nvif_conn *); + +int nvif_conn_event_ctor(struct nvif_conn *, const char *name, nvif_event_func, u8 types, + struct nvif_event *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/event.h b/drivers/gpu/drm/nouveau/include/nvif/event.h index 679950400f536..3afcda2034d65 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/event.h +++ b/drivers/gpu/drm/nouveau/include/nvif/event.h @@ -52,25 +52,6 @@ struct nvif_notify_rep_v0 { __u8 data[]; /* reply data (below) */ }; -struct nvif_notify_conn_req_v0 { - /* nvif_notify_req ... */ - __u8 version; -#define NVIF_NOTIFY_CONN_V0_PLUG 0x01 -#define NVIF_NOTIFY_CONN_V0_UNPLUG 0x02 -#define NVIF_NOTIFY_CONN_V0_IRQ 0x04 -#define NVIF_NOTIFY_CONN_V0_ANY 0x07 - __u8 mask; - __u8 conn; - __u8 pad03[5]; -}; - -struct nvif_notify_conn_rep_v0 { - /* nvif_notify_rep ... */ - __u8 version; - __u8 mask; - __u8 pad02[6]; -}; - struct nvif_notify_uevent_req { /* nvif_notify_req ... */ }; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0011.h b/drivers/gpu/drm/nouveau/include/nvif/if0011.h index 04ba6581f8403..69b0b779f9424 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0011.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0011.h @@ -10,6 +10,17 @@ union nvif_conn_args { } v0; }; +union nvif_conn_event_args { + struct nvif_conn_event_v0 { + __u8 version; +#define NVIF_CONN_EVENT_V0_PLUG 0x01 +#define NVIF_CONN_EVENT_V0_UNPLUG 0x02 +#define NVIF_CONN_EVENT_V0_IRQ 0x04 + __u8 types; + __u8 pad02[6]; + } v0; +}; + #define NVIF_CONN_V0_HPD_STATUS 0x00000000 union nvif_conn_hpd_status_args { diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gpio.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gpio.h index 0e46ea1fe9729..537c4fc58b4fa 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gpio.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gpio.h @@ -8,9 +8,6 @@ #include struct nvkm_gpio_ntfy_req { -#define NVKM_GPIO_HI 0x01 -#define NVKM_GPIO_LO 0x02 -#define NVKM_GPIO_TOGGLED 0x03 u8 mask; u8 line; }; @@ -23,6 +20,9 @@ struct nvkm_gpio { const struct nvkm_gpio_func *func; struct nvkm_subdev subdev; +#define NVKM_GPIO_HI BIT(0) +#define NVKM_GPIO_LO BIT(1) +#define NVKM_GPIO_TOGGLED (NVKM_GPIO_HI | NVKM_GPIO_LO) struct nvkm_event event; }; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h index 146e13292203d..40a1065ae626e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/i2c.h @@ -7,20 +7,6 @@ #include #include -struct nvkm_i2c_ntfy_req { -#define NVKM_I2C_PLUG 0x01 -#define NVKM_I2C_UNPLUG 0x02 -#define NVKM_I2C_IRQ 0x04 -#define NVKM_I2C_DONE 0x08 -#define NVKM_I2C_ANY 0x0f - u8 mask; - u8 port; -}; - -struct nvkm_i2c_ntfy_rep { - u8 mask; -}; - struct nvkm_i2c_bus_probe { struct i2c_board_info dev; u8 udelay; /* set to 0 to use the standard delay */ @@ -79,6 +65,11 @@ struct nvkm_i2c { struct list_head bus; struct list_head aux; +#define NVKM_I2C_PLUG BIT(0) +#define NVKM_I2C_UNPLUG BIT(1) +#define NVKM_I2C_IRQ BIT(2) +#define NVKM_I2C_DONE BIT(3) +#define NVKM_I2C_ANY (NVKM_I2C_PLUG | NVKM_I2C_UNPLUG | NVKM_I2C_IRQ | NVKM_I2C_DONE) struct nvkm_event event; }; diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index bbd17ee608538..086b66b60d918 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -47,8 +47,7 @@ #include "nouveau_crtc.h" #include -#include -#include +#include struct drm_display_mode * nouveau_conn_native_mode(struct drm_connector *connector) @@ -396,7 +395,8 @@ static void nouveau_connector_destroy(struct drm_connector *connector) { struct nouveau_connector *nv_connector = nouveau_connector(connector); - nvif_notify_dtor(&nv_connector->hpd); + nvif_event_dtor(&nv_connector->irq); + nvif_event_dtor(&nv_connector->hpd); kfree(nv_connector->edid); drm_connector_unregister(connector); drm_connector_cleanup(connector); @@ -1178,23 +1178,22 @@ nouveau_connector_hpd(struct nouveau_connector *nv_connector, u64 bits) } static int -nouveau_connector_hotplug(struct nvif_notify *notify) +nouveau_connector_irq(struct nvif_event *event, void *repv, u32 repc) { - struct nouveau_connector *nv_connector = - container_of(notify, typeof(*nv_connector), hpd); - struct drm_connector *connector = &nv_connector->base; - struct drm_device *dev = connector->dev; - struct nouveau_drm *drm = nouveau_drm(dev); - const struct nvif_notify_conn_rep_v0 *rep = notify->data; + struct nouveau_connector *nv_connector = container_of(event, typeof(*nv_connector), irq); - if (rep->mask & NVIF_NOTIFY_CONN_V0_IRQ) { - nouveau_dp_irq(drm, nv_connector); - return NVIF_NOTIFY_KEEP; - } + schedule_work(&nv_connector->irq_work); + return NVIF_EVENT_KEEP; +} - nouveau_connector_hpd(nv_connector, rep->mask); +static int +nouveau_connector_hotplug(struct nvif_event *event, void *repv, u32 repc) +{ + struct nouveau_connector *nv_connector = container_of(event, typeof(*nv_connector), hpd); + struct nvif_conn_event_v0 *rep = repv; - return NVIF_NOTIFY_KEEP; + nouveau_connector_hpd(nv_connector, rep->types); + return NVIF_EVENT_KEEP; } static ssize_t @@ -1290,6 +1289,7 @@ nouveau_connector_create(struct drm_device *dev, connector = &nv_connector->base; nv_connector->index = index; + INIT_WORK(&nv_connector->irq_work, nouveau_dp_irq); /* attempt to parse vbios connector type and hotplug gpio */ nv_connector->dcb = olddcb_conn(dev, index); @@ -1401,6 +1401,7 @@ nouveau_connector_create(struct drm_device *dev, drm_connector_init(dev, connector, funcs, type); drm_connector_helper_add(connector, &nouveau_connector_helper_funcs); + connector->polled = DRM_CONNECTOR_POLL_CONNECT; if (nv_connector->dcb && (disp->disp.conn_mask & BIT(nv_connector->index))) { ret = nvif_conn_ctor(&disp->disp, nv_connector->base.name, nv_connector->index, @@ -1409,6 +1410,25 @@ nouveau_connector_create(struct drm_device *dev, kfree(nv_connector); return ERR_PTR(ret); } + + ret = nvif_conn_event_ctor(&nv_connector->conn, "kmsHotplug", + nouveau_connector_hotplug, + NVIF_CONN_EVENT_V0_PLUG | NVIF_CONN_EVENT_V0_UNPLUG, + &nv_connector->hpd); + if (ret == 0) + connector->polled = DRM_CONNECTOR_POLL_HPD; + + if (nv_connector->aux.transfer) { + ret = nvif_conn_event_ctor(&nv_connector->conn, "kmsDpIrq", + nouveau_connector_irq, NVIF_CONN_EVENT_V0_IRQ, + &nv_connector->irq); + if (ret) { + nvif_event_dtor(&nv_connector->hpd); + nvif_conn_dtor(&nv_connector->conn); + kfree(nv_connector); + return ERR_PTR(ret); + } + } } connector->funcs->reset(connector); @@ -1452,21 +1472,6 @@ nouveau_connector_create(struct drm_device *dev, break; } - ret = nvif_notify_ctor(&disp->disp.object, "kmsHotplug", - nouveau_connector_hotplug, - true, NV04_DISP_NTFY_CONN, - &(struct nvif_notify_conn_req_v0) { - .mask = NVIF_NOTIFY_CONN_V0_ANY, - .conn = index, - }, - sizeof(struct nvif_notify_conn_req_v0), - sizeof(struct nvif_notify_conn_rep_v0), - &nv_connector->hpd); - if (ret) - connector->polled = DRM_CONNECTOR_POLL_CONNECT; - else - connector->polled = DRM_CONNECTOR_POLL_HPD; - drm_connector_register(connector); return connector; } diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h index 1bbf8bf6ba445..35bcb541722b8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.h +++ b/drivers/gpu/drm/nouveau/nouveau_connector.h @@ -27,7 +27,7 @@ #ifndef __NOUVEAU_CONNECTOR_H__ #define __NOUVEAU_CONNECTOR_H__ #include -#include +#include #include #include @@ -125,7 +125,9 @@ struct nouveau_connector { struct nvif_conn conn; u64 hpd_pending; - struct nvif_notify hpd; + struct nvif_event hpd; + struct nvif_event irq; + struct work_struct irq_work; struct drm_dp_aux aux; diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index e88845ae75204..55099d9ce1c8f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -42,8 +42,8 @@ #include "nv50_display.h" #include +#include #include -#include #include int @@ -497,11 +497,11 @@ nouveau_display_hpd_work(struct work_struct *work) drm_dbg_kms(dev, "[CONNECTOR:%d:%s] plug:%d unplug:%d irq:%d\n", connector->base.id, connector->name, - !!(bits & NVIF_NOTIFY_CONN_V0_PLUG), - !!(bits & NVIF_NOTIFY_CONN_V0_UNPLUG), - !!(bits & NVIF_NOTIFY_CONN_V0_IRQ)); + !!(bits & NVIF_CONN_EVENT_V0_PLUG), + !!(bits & NVIF_CONN_EVENT_V0_UNPLUG), + !!(bits & NVIF_CONN_EVENT_V0_IRQ)); - if (bits & NVIF_NOTIFY_CONN_V0_IRQ) { + if (bits & NVIF_CONN_EVENT_V0_IRQ) { if (nouveau_dp_link_check(nv_connector)) continue; } @@ -584,7 +584,8 @@ nouveau_display_init(struct drm_device *dev, bool resume, bool runtime) drm_connector_list_iter_begin(dev, &conn_iter); nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) { struct nouveau_connector *conn = nouveau_connector(connector); - nvif_notify_get(&conn->hpd); + nvif_event_allow(&conn->hpd); + nvif_event_allow(&conn->irq); } drm_connector_list_iter_end(&conn_iter); @@ -619,7 +620,8 @@ nouveau_display_fini(struct drm_device *dev, bool suspend, bool runtime) drm_connector_list_iter_begin(dev, &conn_iter); nouveau_for_each_non_mst_connector_iter(connector, &conn_iter) { struct nouveau_connector *conn = nouveau_connector(connector); - nvif_notify_put(&conn->hpd); + nvif_event_block(&conn->irq); + nvif_event_block(&conn->hpd); } drm_connector_list_iter_end(&conn_iter); diff --git a/drivers/gpu/drm/nouveau/nouveau_dp.c b/drivers/gpu/drm/nouveau/nouveau_dp.c index bde58d00fbf53..e00876f92aeea 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dp.c +++ b/drivers/gpu/drm/nouveau/nouveau_dp.c @@ -29,7 +29,7 @@ #include "nouveau_encoder.h" #include "nouveau_crtc.h" -#include +#include MODULE_PARM_DESC(mst, "Enable DisplayPort multi-stream (default: enabled)"); static int nouveau_mst = 1; @@ -222,11 +222,14 @@ nouveau_dp_link_check(struct nouveau_connector *nv_connector) return nvif_outp_dp_retrain(&nv_encoder->outp) == 0; } -void nouveau_dp_irq(struct nouveau_drm *drm, - struct nouveau_connector *nv_connector) +void +nouveau_dp_irq(struct work_struct *work) { + struct nouveau_connector *nv_connector = + container_of(work, typeof(*nv_connector), irq_work); struct drm_connector *connector = &nv_connector->base; struct nouveau_encoder *outp = find_encoder(connector, DCB_OUTPUT_DP); + struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev); struct nv50_mstm *mstm; u64 hpd = 0; int ret; @@ -241,14 +244,14 @@ void nouveau_dp_irq(struct nouveau_drm *drm, if (mstm && mstm->is_mst) { if (!nv50_mstm_service(drm, nv_connector, mstm)) - hpd |= NVIF_NOTIFY_CONN_V0_UNPLUG; + hpd |= NVIF_CONN_EVENT_V0_UNPLUG; } else { drm_dp_cec_irq(&nv_connector->aux); if (nouveau_dp_has_sink_count(connector, outp)) { ret = drm_dp_read_sink_count(&nv_connector->aux); if (ret != outp->dp.sink_count) - hpd |= NVIF_NOTIFY_CONN_V0_PLUG; + hpd |= NVIF_CONN_EVENT_V0_PLUG; if (ret >= 0) outp->dp.sink_count = ret; } @@ -256,7 +259,7 @@ void nouveau_dp_irq(struct nouveau_drm *drm, mutex_unlock(&outp->dp.hpd_irq_lock); - nouveau_connector_hpd(nv_connector, NVIF_NOTIFY_CONN_V0_IRQ | hpd); + nouveau_connector_hpd(nv_connector, NVIF_CONN_EVENT_V0_IRQ | hpd); } /* TODO: diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h index 8524d211f4e10..70c1ad6c4d9db 100644 --- a/drivers/gpu/drm/nouveau/nouveau_encoder.h +++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h @@ -142,8 +142,7 @@ enum nouveau_dp_status { int nouveau_dp_detect(struct nouveau_connector *, struct nouveau_encoder *); bool nouveau_dp_link_check(struct nouveau_connector *); -void nouveau_dp_irq(struct nouveau_drm *drm, - struct nouveau_connector *nv_connector); +void nouveau_dp_irq(struct work_struct *); enum drm_mode_status nv50_dp_mode_valid(struct drm_connector *, struct nouveau_encoder *, const struct drm_display_mode *, diff --git a/drivers/gpu/drm/nouveau/nvif/conn.c b/drivers/gpu/drm/nouveau/nvif/conn.c index 4ce935d58c906..a3cf91aeae2de 100644 --- a/drivers/gpu/drm/nouveau/nvif/conn.c +++ b/drivers/gpu/drm/nouveau/nvif/conn.c @@ -26,6 +26,25 @@ #include #include +int +nvif_conn_event_ctor(struct nvif_conn *conn, const char *name, nvif_event_func func, u8 types, + struct nvif_event *event) +{ + struct { + struct nvif_event_v0 base; + struct nvif_conn_event_v0 conn; + } args; + int ret; + + args.conn.version = 0; + args.conn.types = types; + + ret = nvif_event_ctor_(&conn->object, name ?: "nvifConnHpd", nvif_conn_id(conn), + func, true, &args.base, sizeof(args), false, event); + NVIF_DEBUG(&conn->object, "[NEW EVENT:HPD types:%02x]", types); + return ret; +} + int nvif_conn_hpd_status(struct nvif_conn *conn) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c index 399195946823e..c1d81cf74c57c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c @@ -29,7 +29,6 @@ #include "outp.h" #include -#include #include #include #include @@ -69,54 +68,6 @@ nvkm_disp_vblank(struct nvkm_disp *disp, int head) nvkm_event_send(&disp->vblank, NVKM_DISP_HEAD_EVENT_VBLANK, head, NULL, 0); } -static int -nvkm_disp_hpd_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - struct nvkm_disp *disp = - container_of(notify->event, typeof(*disp), hpd); - union { - struct nvif_notify_conn_req_v0 v0; - } *req = data; - struct nvkm_outp *outp; - int ret = -ENOSYS; - - if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, false))) { - notify->size = sizeof(struct nvif_notify_conn_rep_v0); - list_for_each_entry(outp, &disp->outps, head) { - if (ret = -ENXIO, outp->conn->index == req->v0.conn) { - if (ret = -ENODEV, outp->conn->hpd.event) { - notify->types = req->v0.mask; - notify->index = req->v0.conn; - ret = 0; - } - break; - } - } - } - - return ret; -} - -static const struct nvkm_event_func -nvkm_disp_hpd_func = { - .ctor = nvkm_disp_hpd_ctor -}; - -int -nvkm_disp_ntfy(struct nvkm_object *object, u32 type, struct nvkm_event **event) -{ - struct nvkm_disp *disp = nvkm_disp(object->engine); - switch (type) { - case NV04_DISP_NTFY_CONN: - *event = &disp->hpd; - return 0; - default: - break; - } - return -EINVAL; -} - static int nvkm_disp_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, void *data, u32 size, @@ -325,10 +276,6 @@ nvkm_disp_oneinit(struct nvkm_engine *engine) list_add_tail(&outp->conn->head, &disp->conns); } - ret = nvkm_event_init(&nvkm_disp_hpd_func, subdev, 3, hpd, &disp->hpd); - if (ret) - return ret; - if (disp->func->oneinit) { ret = disp->func->oneinit(disp); if (ret) @@ -376,7 +323,6 @@ nvkm_disp_dtor(struct nvkm_engine *engine) } nvkm_event_fini(&disp->vblank); - nvkm_event_fini(&disp->hpd); while (!list_empty(&disp->conns)) { conn = list_first_entry(&disp->conns, typeof(*conn), head); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.c index 7ed11801a3ae6..fbdae11378646 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.c @@ -29,38 +29,14 @@ #include -static int -nvkm_conn_hpd(struct nvkm_notify *notify) -{ - struct nvkm_conn *conn = container_of(notify, typeof(*conn), hpd); - struct nvkm_disp *disp = conn->disp; - struct nvkm_gpio *gpio = disp->engine.subdev.device->gpio; - const struct nvkm_gpio_ntfy_rep *line = notify->data; - struct nvif_notify_conn_rep_v0 rep; - int index = conn->index; - - CONN_DBG(conn, "HPD: %d", line->mask); - - if (!nvkm_gpio_get(gpio, 0, DCB_GPIO_UNUSED, conn->hpd.index)) - rep.mask = NVIF_NOTIFY_CONN_V0_UNPLUG; - else - rep.mask = NVIF_NOTIFY_CONN_V0_PLUG; - rep.version = 0; - - nvkm_event_send(&disp->hpd, rep.mask, index, &rep, sizeof(rep)); - return NVKM_NOTIFY_KEEP; -} - void nvkm_conn_fini(struct nvkm_conn *conn) { - nvkm_notify_put(&conn->hpd); } void nvkm_conn_init(struct nvkm_conn *conn) { - nvkm_notify_get(&conn->hpd); } void @@ -68,7 +44,6 @@ nvkm_conn_del(struct nvkm_conn **pconn) { struct nvkm_conn *conn = *pconn; if (conn) { - nvkm_notify_fini(&conn->hpd); kfree(*pconn); *pconn = NULL; } @@ -106,20 +81,6 @@ nvkm_conn_ctor(struct nvkm_disp *disp, int index, struct nvbios_connE *info, } conn->info.hpd = func.line; - - ret = nvkm_notify_init(NULL, &gpio->event, nvkm_conn_hpd, - true, &(struct nvkm_gpio_ntfy_req) { - .mask = NVKM_GPIO_TOGGLED, - .line = func.line, - }, - sizeof(struct nvkm_gpio_ntfy_req), - sizeof(struct nvkm_gpio_ntfy_rep), - &conn->hpd); - if (ret) { - CONN_ERR(conn, "func %02x failed, %d", info->hpd, ret); - } else { - CONN_DBG(conn, "func %02x (HPD)", info->hpd); - } } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.h index f109634ce5ca8..a0600e72b0ecd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/conn.h @@ -3,7 +3,6 @@ #define __NVKM_DISP_CONN_H__ #include "priv.h" -#include #include #include @@ -12,8 +11,6 @@ struct nvkm_conn { int index; struct nvbios_connE info; - struct nvkm_notify hpd; - struct list_head head; struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c index 730c3a6f33626..40c8ea43c42f2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dp.c @@ -738,31 +738,9 @@ nvkm_dp_enable(struct nvkm_outp *outp, bool auxpwr) } } -static int -nvkm_dp_hpd(struct nvkm_notify *notify) -{ - const struct nvkm_i2c_ntfy_rep *line = notify->data; - struct nvkm_outp *outp = container_of(notify, typeof(*outp), dp.hpd); - struct nvkm_conn *conn = outp->conn; - struct nvkm_disp *disp = outp->disp; - struct nvif_notify_conn_rep_v0 rep = {}; - - OUTP_DBG(outp, "HPD: %d", line->mask); - if (line->mask & NVKM_I2C_IRQ) - rep.mask |= NVIF_NOTIFY_CONN_V0_IRQ; - if (line->mask & NVKM_I2C_UNPLUG) - rep.mask |= NVIF_NOTIFY_CONN_V0_UNPLUG; - if (line->mask & NVKM_I2C_PLUG) - rep.mask |= NVIF_NOTIFY_CONN_V0_PLUG; - - nvkm_event_send(&disp->hpd, rep.mask, conn->index, &rep, sizeof(rep)); - return NVKM_NOTIFY_KEEP; -} - static void nvkm_dp_fini(struct nvkm_outp *outp) { - nvkm_notify_put(&outp->dp.hpd); nvkm_dp_enable(outp, false); } @@ -770,14 +748,11 @@ static void nvkm_dp_init(struct nvkm_outp *outp) { nvkm_dp_enable(outp, outp->dp.enabled); - nvkm_notify_put(&outp->conn->hpd); - nvkm_notify_get(&outp->dp.hpd); } static void * nvkm_dp_dtor(struct nvkm_outp *outp) { - nvkm_notify_fini(&outp->dp.hpd); return outp; } @@ -826,21 +801,6 @@ nvkm_dp_new(struct nvkm_disp *disp, int index, struct dcb_output *dcbE, struct n OUTP_DBG(outp, "bios dp %02x %02x %02x %02x", outp->dp.version, hdr, cnt, len); - /* hotplug detect, replaces gpio-based mechanism with aux events */ - ret = nvkm_notify_init(NULL, &i2c->event, nvkm_dp_hpd, true, - &(struct nvkm_i2c_ntfy_req) { - .mask = NVKM_I2C_PLUG | NVKM_I2C_UNPLUG | - NVKM_I2C_IRQ, - .port = outp->dp.aux->id, - }, - sizeof(struct nvkm_i2c_ntfy_req), - sizeof(struct nvkm_i2c_ntfy_rep), - &outp->dp.hpd); - if (ret) { - OUTP_ERR(outp, "error monitoring aux hpd: %d", ret); - return ret; - } - mutex_init(&outp->dp.mutex); atomic_set(&outp->dp.lt.done, 0); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h index 66def8ae31655..b7631c1ab2420 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/outp.h @@ -2,7 +2,6 @@ #ifndef __NVKM_DISP_OUTP_H__ #define __NVKM_DISP_OUTP_H__ #include "priv.h" -#include #include #include @@ -38,7 +37,6 @@ struct nvkm_outp { struct nvkm_i2c_aux *aux; - struct nvkm_notify hpd; bool enabled; bool aux_pwr; bool aux_pwr_pu; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h index a98b57f82fe77..ec5292a8f3c85 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h @@ -42,8 +42,6 @@ struct nvkm_disp_func { } user[]; }; -int nvkm_disp_ntfy(struct nvkm_object *, u32, struct nvkm_event **); - int nv50_disp_oneinit(struct nvkm_disp *); int nv50_disp_init(struct nvkm_disp *); void nv50_disp_fini(struct nvkm_disp *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c index fd9f18144c265..dad942be6679c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/uconn.c @@ -21,11 +21,85 @@ */ #define nvkm_uconn(p) container_of((p), struct nvkm_conn, object) #include "conn.h" +#include "outp.h" +#include +#include #include +#include #include +static int +nvkm_uconn_uevent_aux(struct nvkm_object *object, u64 token, u32 bits) +{ + union nvif_conn_event_args args; + + args.v0.version = 0; + args.v0.types = 0; + if (bits & NVKM_I2C_PLUG) + args.v0.types |= NVIF_CONN_EVENT_V0_PLUG; + if (bits & NVKM_I2C_UNPLUG) + args.v0.types |= NVIF_CONN_EVENT_V0_UNPLUG; + if (bits & NVKM_I2C_IRQ) + args.v0.types |= NVIF_CONN_EVENT_V0_IRQ; + + return object->client->event(token, &args, sizeof(args.v0)); +} + +static int +nvkm_uconn_uevent_gpio(struct nvkm_object *object, u64 token, u32 bits) +{ + union nvif_conn_event_args args; + + args.v0.version = 0; + args.v0.types = 0; + if (bits & NVKM_GPIO_HI) + args.v0.types |= NVIF_CONN_EVENT_V0_PLUG; + if (bits & NVKM_GPIO_LO) + args.v0.types |= NVIF_CONN_EVENT_V0_UNPLUG; + + return object->client->event(token, &args, sizeof(args.v0)); +} + +static int +nvkm_uconn_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) +{ + struct nvkm_conn *conn = nvkm_uconn(object); + struct nvkm_device *device = conn->disp->engine.subdev.device; + struct nvkm_outp *outp; + union nvif_conn_event_args *args = argv; + u64 bits = 0; + + if (!uevent) { + if (conn->info.hpd == DCB_GPIO_UNUSED) + return -ENOSYS; + return 0; + } + + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + list_for_each_entry(outp, &conn->disp->outps, head) { + if (outp->info.connector == conn->index && outp->dp.aux) { + if (args->v0.types & NVIF_CONN_EVENT_V0_PLUG ) bits |= NVKM_I2C_PLUG; + if (args->v0.types & NVIF_CONN_EVENT_V0_UNPLUG) bits |= NVKM_I2C_UNPLUG; + if (args->v0.types & NVIF_CONN_EVENT_V0_IRQ ) bits |= NVKM_I2C_IRQ; + + return nvkm_uevent_add(uevent, &device->i2c->event, outp->dp.aux->id, bits, + nvkm_uconn_uevent_aux); + } + } + + if (args->v0.types & NVIF_CONN_EVENT_V0_PLUG ) bits |= NVKM_GPIO_HI; + if (args->v0.types & NVIF_CONN_EVENT_V0_UNPLUG) bits |= NVKM_GPIO_LO; + if (args->v0.types & NVIF_CONN_EVENT_V0_IRQ) + return -EINVAL; + + return nvkm_uevent_add(uevent, &device->gpio->event, conn->info.hpd, bits, + nvkm_uconn_uevent_gpio); +} + static int nvkm_uconn_mthd_hpd_status(struct nvkm_conn *conn, void *argv, u32 argc) { @@ -82,6 +156,7 @@ static const struct nvkm_object_func nvkm_uconn = { .dtor = nvkm_uconn_dtor, .mthd = nvkm_uconn_mthd, + .uevent = nvkm_uconn_uevent, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c index c9bd32220fe17..0268d1d75805b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/udisp.c @@ -74,7 +74,6 @@ nvkm_udisp_dtor(struct nvkm_object *object) static const struct nvkm_object_func nvkm_udisp = { .dtor = nvkm_udisp_dtor, - .ntfy = nvkm_disp_ntfy, .sclass = nvkm_udisp_sclass, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c index f2ccbcf219ca8..2c2c23b8663b6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c @@ -24,7 +24,6 @@ #include "priv.h" #include -#include static int nvkm_gpio_drive(struct nvkm_gpio *gpio, int idx, int line, int dir, int out) @@ -123,23 +122,8 @@ nvkm_gpio_intr_init(struct nvkm_event *event, int type, int index) gpio->func->intr_mask(gpio, type, 1 << index, 1 << index); } -static int -nvkm_gpio_intr_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - struct nvkm_gpio_ntfy_req *req = data; - if (!WARN_ON(size != sizeof(*req))) { - notify->size = sizeof(struct nvkm_gpio_ntfy_rep); - notify->types = req->mask; - notify->index = req->line; - return 0; - } - return -EINVAL; -} - static const struct nvkm_event_func nvkm_gpio_intr_func = { - .ctor = nvkm_gpio_intr_ctor, .init = nvkm_gpio_intr_init, .fini = nvkm_gpio_intr_fini, }; @@ -153,11 +137,9 @@ nvkm_gpio_intr(struct nvkm_subdev *subdev) gpio->func->intr_stat(gpio, &hi, &lo); for (i = 0; (hi | lo) && i < gpio->func->lines; i++) { - struct nvkm_gpio_ntfy_rep rep = { - .mask = (NVKM_GPIO_HI * !!(hi & (1 << i))) | - (NVKM_GPIO_LO * !!(lo & (1 << i))), - }; - nvkm_event_send(&gpio->event, rep.mask, i, &rep, sizeof(rep)); + u32 mask = (NVKM_GPIO_HI * !!(hi & (1 << i))) | + (NVKM_GPIO_LO * !!(lo & (1 << i))); + nvkm_event_send(&gpio->event, mask, i, NULL, 0); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c index 49a84ef146e9f..feb7de985d2d7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c @@ -26,7 +26,6 @@ #include "bus.h" #include "pad.h" -#include #include #include #include @@ -104,23 +103,8 @@ nvkm_i2c_intr_init(struct nvkm_event *event, int type, int id) i2c->func->aux_mask(i2c, type, aux->intr, aux->intr); } -static int -nvkm_i2c_intr_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - struct nvkm_i2c_ntfy_req *req = data; - if (!WARN_ON(size != sizeof(*req))) { - notify->size = sizeof(struct nvkm_i2c_ntfy_rep); - notify->types = req->mask; - notify->index = req->port; - return 0; - } - return -EINVAL; -} - static const struct nvkm_event_func nvkm_i2c_intr_func = { - .ctor = nvkm_i2c_intr_ctor, .init = nvkm_i2c_intr_init, .fini = nvkm_i2c_intr_fini, }; @@ -145,13 +129,8 @@ nvkm_i2c_intr(struct nvkm_subdev *subdev) if (lo & aux->intr) mask |= NVKM_I2C_UNPLUG; if (rq & aux->intr) mask |= NVKM_I2C_IRQ; if (tx & aux->intr) mask |= NVKM_I2C_DONE; - if (mask) { - struct nvkm_i2c_ntfy_rep rep = { - .mask = mask, - }; - nvkm_event_send(&i2c->event, rep.mask, aux->id, - &rep, sizeof(rep)); - } + if (mask) + nvkm_event_send(&i2c->event, mask, aux->id, NULL, 0); } } -- GitLab From 801bc8584e58f718491ca25827647129d9b6ef1f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:39 +1000 Subject: [PATCH 1280/1927] drm/nouveau/disp: expose page flip event class Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/dispnv04/crtc.c | 8 ++--- drivers/gpu/drm/nouveau/dispnv04/disp.c | 15 ++++---- drivers/gpu/drm/nouveau/dispnv04/disp.h | 7 ++-- drivers/gpu/drm/nouveau/include/nvif/if0004.h | 5 ++- drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c | 22 +----------- drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c | 35 ++++++++++--------- 7 files changed, 42 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c index 80b0b5cf4079e..4e8b84f592800 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c +++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c @@ -1081,10 +1081,10 @@ nv04_finish_page_flip(struct nouveau_channel *chan, } int -nv04_flip_complete(struct nvif_notify *notify) +nv04_flip_complete(struct nvif_event *event, void *argv, u32 argc) { - struct nouveau_cli *cli = (void *)notify->object->client; - struct nouveau_drm *drm = cli->drm; + struct nv04_display *disp = container_of(event, typeof(*disp), flip); + struct nouveau_drm *drm = disp->drm; struct nouveau_channel *chan = drm->channel; struct nv04_page_flip_state state; @@ -1095,7 +1095,7 @@ nv04_flip_complete(struct nvif_notify *notify) state.bpp / 8); } - return NVIF_NOTIFY_KEEP; + return NVIF_EVENT_KEEP; } static int diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c index 99fee4d8cd318..e9ac3fb27ff78 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c @@ -61,7 +61,7 @@ nv04_display_fini(struct drm_device *dev, bool runtime, bool suspend) struct drm_crtc *crtc; /* Disable flip completion events. */ - nvif_notify_put(&disp->flip); + nvif_event_block(&disp->flip); /* Disable vblank interrupts. */ NVWriteCRTC(dev, 0, NV_PCRTC_INTR_EN_0, 0); @@ -121,7 +121,7 @@ nv04_display_init(struct drm_device *dev, bool resume, bool runtime) encoder->enc_save(&encoder->base.base); /* Enable flip completion events. */ - nvif_notify_get(&disp->flip); + nvif_event_allow(&disp->flip); if (!resume) return 0; @@ -202,7 +202,7 @@ nv04_display_destroy(struct drm_device *dev) nouveau_hw_save_vga_fonts(dev, 0); - nvif_notify_dtor(&disp->flip); + nvif_event_dtor(&disp->flip); nouveau_display(dev)->priv = NULL; vfree(disp); @@ -227,6 +227,8 @@ nv04_display_create(struct drm_device *dev) if (!disp) return -ENOMEM; + disp->drm = drm; + nvif_object_map(&drm->client.device.object, NULL, 0); nouveau_display(dev)->priv = disp; @@ -239,9 +241,10 @@ nv04_display_create(struct drm_device *dev) /* Request page flip completion event. */ if (drm->channel) { - nvif_notify_ctor(&drm->channel->nvsw, "kmsFlip", nv04_flip_complete, - false, NV04_NVSW_NTFY_UEVENT, - NULL, 0, 0, &disp->flip); + ret = nvif_event_ctor(&drm->channel->nvsw, "kmsFlip", 0, nv04_flip_complete, + true, NULL, 0, &disp->flip); + if (ret) + return ret; } nouveau_hw_save_vga_fonts(dev, 1); diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.h b/drivers/gpu/drm/nouveau/dispnv04/disp.h index f0a24126641ab..11a6663758ec9 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/disp.h +++ b/drivers/gpu/drm/nouveau/dispnv04/disp.h @@ -6,6 +6,8 @@ #include "nouveau_display.h" +#include + struct nouveau_encoder; enum nv04_fp_display_regs { @@ -84,7 +86,8 @@ struct nv04_display { uint32_t saved_vga_font[4][16384]; uint32_t dac_users[4]; struct nouveau_bo *image[2]; - struct nvif_notify flip; + struct nvif_event flip; + struct nouveau_drm *drm; }; static inline struct nv04_display * @@ -179,5 +182,5 @@ nouveau_bios_run_init_table(struct drm_device *dev, u16 table, ); } -int nv04_flip_complete(struct nvif_notify *); +int nv04_flip_complete(struct nvif_event *, void *, u32); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0004.h b/drivers/gpu/drm/nouveau/include/nvif/if0004.h index d324c73c27fb5..1d916a137941e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0004.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0004.h @@ -2,7 +2,10 @@ #ifndef __NVIF_IF0004_H__ #define __NVIF_IF0004_H__ -#define NV04_NVSW_NTFY_UEVENT 0x00 +union nv04_nvsw_event_args { + struct nv04_nvsw_event_vn { + } vn; +}; #define NV04_NVSW_GET_REF 0x00 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c index d156f86768965..5e65e13b00533 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c @@ -23,7 +23,6 @@ */ #include "chan.h" -#include #include #include @@ -36,7 +35,7 @@ nvkm_sw_chan_mthd(struct nvkm_sw_chan *chan, int subc, u32 mthd, u32 data) case 0x0000: return true; case 0x0500: - nvkm_event_send(&chan->event, 1, 0, NULL, 0); + nvkm_event_send(&chan->event, NVKM_SW_CHAN_EVENT_PAGE_FLIP, 0, NULL, 0); return true; default: if (chan->func->mthd) @@ -46,27 +45,8 @@ nvkm_sw_chan_mthd(struct nvkm_sw_chan *chan, int subc, u32 mthd, u32 data) return false; } -static int -nvkm_sw_chan_event_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - union { - struct nvif_notify_uevent_req none; - } *req = data; - int ret = -ENOSYS; - - if (!(ret = nvif_unvers(ret, &data, &size, req->none))) { - notify->size = sizeof(struct nvif_notify_uevent_rep); - notify->types = 1; - notify->index = 0; - } - - return ret; -} - static const struct nvkm_event_func nvkm_sw_chan_event = { - .ctor = nvkm_sw_chan_event_ctor, }; static void * diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h index 32de53427aa44..67b2e5ea93d92 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.h @@ -14,6 +14,7 @@ struct nvkm_sw_chan { struct nvkm_fifo_chan *fifo; struct list_head head; +#define NVKM_SW_CHAN_EVENT_PAGE_FLIP BIT(0) struct nvkm_event event; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c index 33dd03fff3c40..f5affa1c8f343 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/nvsw.c @@ -27,33 +27,34 @@ #include static int -nvkm_nvsw_mthd_(struct nvkm_object *object, u32 mthd, void *data, u32 size) +nvkm_nvsw_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) { - struct nvkm_nvsw *nvsw = nvkm_nvsw(object); - if (nvsw->func->mthd) - return nvsw->func->mthd(nvsw, mthd, data, size); - return -ENODEV; + union nv04_nvsw_event_args *args = argv; + + if (!uevent) + return 0; + if (argc != sizeof(args->vn)) + return -ENOSYS; + + return nvkm_uevent_add(uevent, &nvkm_nvsw(object)->chan->event, 0, + NVKM_SW_CHAN_EVENT_PAGE_FLIP, NULL); } static int -nvkm_nvsw_ntfy_(struct nvkm_object *object, u32 mthd, - struct nvkm_event **pevent) +nvkm_nvsw_mthd(struct nvkm_object *object, u32 mthd, void *data, u32 size) { struct nvkm_nvsw *nvsw = nvkm_nvsw(object); - switch (mthd) { - case NV04_NVSW_NTFY_UEVENT: - *pevent = &nvsw->chan->event; - return 0; - default: - break; - } - return -EINVAL; + + if (nvsw->func->mthd) + return nvsw->func->mthd(nvsw, mthd, data, size); + + return -ENODEV; } static const struct nvkm_object_func nvkm_nvsw_ = { - .mthd = nvkm_nvsw_mthd_, - .ntfy = nvkm_nvsw_ntfy_, + .mthd = nvkm_nvsw_mthd, + .uevent = nvkm_nvsw_uevent, }; int -- GitLab From 8c880fd4c155d9b1b3e294169d7dc3f3a1e04bd3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:39 +1000 Subject: [PATCH 1281/1927] drm/nouveau/fifo: expose non-stall intr in host channel event class Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/cl826e.h | 2 - drivers/gpu/drm/nouveau/include/nvif/cl826f.h | 2 - drivers/gpu/drm/nouveau/include/nvif/cl906f.h | 1 - drivers/gpu/drm/nouveau/include/nvif/cla06f.h | 1 - drivers/gpu/drm/nouveau/include/nvif/class.h | 2 + drivers/gpu/drm/nouveau/include/nvif/clc36f.h | 1 - drivers/gpu/drm/nouveau/include/nvif/if0020.h | 12 ++++++ .../drm/nouveau/include/nvkm/engine/fifo.h | 1 + drivers/gpu/drm/nouveau/nouveau_fence.c | 42 +++++++++---------- drivers/gpu/drm/nouveau/nouveau_fence.h | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 23 +--------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 25 +++++++++++ .../drm/nouveau/nvkm/engine/fifo/chang84.c | 15 ------- .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 3 -- 14 files changed, 64 insertions(+), 70 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if0020.h diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h index 1b6496d315803..ef3033b836f0e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h @@ -10,6 +10,4 @@ struct g82_channel_dma_v0 { __u64 pushbuf; __u64 offset; }; - -#define NV826E_V0_NTFY_NON_STALL_INTERRUPT 0x00 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h index 148602264a764..c5d5e56b04cc0 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h @@ -11,6 +11,4 @@ struct g82_channel_gpfifo_v0 { __u64 pushbuf; __u64 vmm; }; - -#define NV826F_V0_NTFY_NON_STALL_INTERRUPT 0x00 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h index 3823d6891b556..0e17b21e8c6ee 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h @@ -11,6 +11,5 @@ struct fermi_channel_gpfifo_v0 { __u64 vmm; }; -#define NV906F_V0_NTFY_NON_STALL_INTERRUPT 0x00 #define NV906F_V0_NTFY_KILLED 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h index cfa18f1fbf838..00057ddfa41f2 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h @@ -13,6 +13,5 @@ struct kepler_channel_gpfifo_a_v0 { __u64 inst; }; -#define NVA06F_V0_NTFY_NON_STALL_INTERRUPT 0x00 #define NVA06F_V0_NTFY_KILLED 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 29448f77069b7..3e9d181b5dda1 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -40,6 +40,8 @@ #define NVIF_CLASS_HEAD /* if0013.h */ 0x80000013 #define NVIF_CLASS_DISP_CHAN /* if0014.h */ 0x80000014 +#define NVIF_CLASS_CHAN /* if0020.h */ 0x80000020 + /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 diff --git a/drivers/gpu/drm/nouveau/include/nvif/clc36f.h b/drivers/gpu/drm/nouveau/include/nvif/clc36f.h index f66885891238e..b36624760dea3 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/clc36f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/clc36f.h @@ -14,6 +14,5 @@ struct volta_channel_gpfifo_a_v0 { __u32 token; }; -#define NVC36F_V0_NTFY_NON_STALL_INTERRUPT 0x00 #define NVC36F_V0_NTFY_KILLED 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0020.h b/drivers/gpu/drm/nouveau/include/nvif/if0020.h new file mode 100644 index 0000000000000..29a9fe2a8d871 --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if0020.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVIF_IF0020_H__ +#define __NVIF_IF0020_H__ + +union nvif_chan_event_args { + struct nvif_chan_event_v0 { + __u8 version; +#define NVIF_CHAN_EVENT_V0_NON_STALL_INTR 0x00 + __u8 type; + } v0; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 15099913504d5..ebfca3ffe9ae4 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -42,6 +42,7 @@ struct nvkm_fifo { spinlock_t lock; struct mutex mutex; +#define NVKM_FIFO_EVENT_NON_STALL_INTR BIT(0) struct nvkm_event uevent; /* async user trigger */ struct nvkm_event kevent; /* channel killed */ }; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index abcac7db43473..62560e30febb4 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -29,9 +29,7 @@ #include #include -#include -#include -#include +#include #include "nouveau_drv.h" #include "nouveau_dma.h" @@ -99,7 +97,7 @@ nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error) dma_fence_set_error(&fence->base, error); if (nouveau_fence_signal(fence)) - nvif_notify_put(&fctx->notify); + nvif_event_block(&fctx->event); } spin_unlock_irq(&fctx->lock); } @@ -108,7 +106,7 @@ void nouveau_fence_context_del(struct nouveau_fence_chan *fctx) { nouveau_fence_context_kill(fctx, 0); - nvif_notify_dtor(&fctx->notify); + nvif_event_dtor(&fctx->event); fctx->dead = 1; /* @@ -150,12 +148,11 @@ nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fc } static int -nouveau_fence_wait_uevent_handler(struct nvif_notify *notify) +nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc) { - struct nouveau_fence_chan *fctx = - container_of(notify, typeof(*fctx), notify); + struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event); unsigned long flags; - int ret = NVIF_NOTIFY_KEEP; + int ret = NVIF_EVENT_KEEP; spin_lock_irqsave(&fctx->lock, flags); if (!list_empty(&fctx->pending)) { @@ -165,7 +162,7 @@ nouveau_fence_wait_uevent_handler(struct nvif_notify *notify) fence = list_entry(fctx->pending.next, typeof(*fence), head); chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); if (nouveau_fence_update(chan, fctx)) - ret = NVIF_NOTIFY_DROP; + ret = NVIF_EVENT_DROP; } spin_unlock_irqrestore(&fctx->lock, flags); @@ -177,6 +174,10 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha { struct nouveau_fence_priv *priv = (void*)chan->drm->fence; struct nouveau_cli *cli = (void *)chan->user.client; + struct { + struct nvif_event_v0 base; + struct nvif_chan_event_v0 host; + } args; int ret; INIT_LIST_HEAD(&fctx->flip); @@ -195,13 +196,12 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha if (!priv->uevent) return; - ret = nvif_notify_ctor(&chan->user, "fenceNonStallIntr", - nouveau_fence_wait_uevent_handler, - false, NV826E_V0_NTFY_NON_STALL_INTERRUPT, - &(struct nvif_notify_uevent_req) { }, - sizeof(struct nvif_notify_uevent_req), - sizeof(struct nvif_notify_uevent_rep), - &fctx->notify); + args.host.version = 0; + args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR; + + ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", chan->chid, + nouveau_fence_wait_uevent_handler, false, + &args.base, sizeof(args), &fctx->event); WARN_ON(ret); } @@ -230,7 +230,7 @@ nouveau_fence_emit(struct nouveau_fence *fence, struct nouveau_channel *chan) spin_lock_irq(&fctx->lock); if (nouveau_fence_update(chan, fctx)) - nvif_notify_put(&fctx->notify); + nvif_event_block(&fctx->event); list_add_tail(&fence->head, &fctx->pending); spin_unlock_irq(&fctx->lock); @@ -254,7 +254,7 @@ nouveau_fence_done(struct nouveau_fence *fence) spin_lock_irqsave(&fctx->lock, flags); chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock)); if (chan && nouveau_fence_update(chan, fctx)) - nvif_notify_put(&fctx->notify); + nvif_event_block(&fctx->event); spin_unlock_irqrestore(&fctx->lock, flags); } return dma_fence_is_signaled(&fence->base); @@ -505,13 +505,13 @@ static bool nouveau_fence_enable_signaling(struct dma_fence *f) bool ret; if (!fctx->notify_ref++) - nvif_notify_get(&fctx->notify); + nvif_event_allow(&fctx->event); ret = nouveau_fence_no_signaling(f); if (ret) set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags); else if (!--fctx->notify_ref) - nvif_notify_put(&fctx->notify); + nvif_event_block(&fctx->event); return ret; } diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h index 4887caa69c657..0ca2bc85adf61 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.h +++ b/drivers/gpu/drm/nouveau/nouveau_fence.h @@ -3,7 +3,7 @@ #define __NOUVEAU_FENCE_H__ #include -#include +#include struct nouveau_drm; struct nouveau_bo; @@ -44,7 +44,7 @@ struct nouveau_fence_chan { u32 context; char name[32]; - struct nvif_notify notify; + struct nvif_event event; int notify_ref, dead; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index c94b2b9b93294..ec790af9b24f0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -158,27 +158,8 @@ nvkm_fifo_uevent_init(struct nvkm_event *event, int type, int index) fifo->func->uevent_init(fifo); } -static int -nvkm_fifo_uevent_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - union { - struct nvif_notify_uevent_req none; - } *req = data; - int ret = -ENOSYS; - - if (!(ret = nvif_unvers(ret, &data, &size, req->none))) { - notify->size = sizeof(struct nvif_notify_uevent_rep); - notify->types = 1; - notify->index = 0; - } - - return ret; -} - static const struct nvkm_event_func nvkm_fifo_uevent_func = { - .ctor = nvkm_fifo_uevent_ctor, .init = nvkm_fifo_uevent_init, .fini = nvkm_fifo_uevent_fini, }; @@ -186,9 +167,7 @@ nvkm_fifo_uevent_func = { void nvkm_fifo_uevent(struct nvkm_fifo *fifo) { - struct nvif_notify_uevent_rep rep = { - }; - nvkm_event_send(&fifo->uevent, 1, 0, &rep, sizeof(rep)); + nvkm_event_send(&fifo->uevent, NVKM_FIFO_EVENT_NON_STALL_INTR, 0, NULL, 0); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 2e7f32cebf2a1..08f09f40defb3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -29,6 +29,8 @@ #include #include +#include + struct nvkm_fifo_chan_object { struct nvkm_oproxy oproxy; struct nvkm_fifo_chan *chan; @@ -250,6 +252,28 @@ nvkm_fifo_chan_child_get(struct nvkm_object *object, int index, return -EINVAL; } +static int +nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) +{ + struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); + union nvif_chan_event_args *args = argv; + + if (!uevent) + return 0; + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + switch (args->v0.type) { + case NVIF_CHAN_EVENT_V0_NON_STALL_INTR: + return nvkm_uevent_add(uevent, &chan->fifo->uevent, 0, + NVKM_FIFO_EVENT_NON_STALL_INTR, NULL); + default: + break; + } + + return -ENOSYS; +} + static int nvkm_fifo_chan_ntfy(struct nvkm_object *object, u32 type, struct nvkm_event **pevent) @@ -320,6 +344,7 @@ nvkm_fifo_chan_func = { .ntfy = nvkm_fifo_chan_ntfy, .map = nvkm_fifo_chan_map, .sclass = nvkm_fifo_chan_child_get, + .uevent = nvkm_fifo_chan_uevent, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 3492c561f2cfc..903b54c07750e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -30,20 +30,6 @@ #include -static int -g84_fifo_chan_ntfy(struct nvkm_fifo_chan *chan, u32 type, - struct nvkm_event **pevent) -{ - switch (type) { - case NV826E_V0_NTFY_NON_STALL_INTERRUPT: - *pevent = &chan->fifo->uevent; - return 0; - default: - break; - } - return -EINVAL; -} - static int g84_fifo_chan_engine_addr(struct nvkm_engine *engine) { @@ -198,7 +184,6 @@ g84_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, .init = g84_fifo_chan_init, .fini = nv50_fifo_chan_fini, - .ntfy = g84_fifo_chan_ntfy, .engine_ctor = g84_fifo_chan_engine_ctor, .engine_dtor = nv50_fifo_chan_engine_dtor, .engine_init = g84_fifo_chan_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 4e78bbe3b94bb..569b5eedb84d6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -37,9 +37,6 @@ gf100_fifo_chan_ntfy(struct nvkm_fifo_chan *chan, u32 type, struct nvkm_event **pevent) { switch (type) { - case NV906F_V0_NTFY_NON_STALL_INTERRUPT: - *pevent = &chan->fifo->uevent; - return 0; case NV906F_V0_NTFY_KILLED: *pevent = &chan->fifo->kevent; return 0; -- GitLab From 6c9705f68465661eede90ce435eda7fd49f70d79 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:39 +1000 Subject: [PATCH 1282/1927] drm/nouveau/fifo: expose channel killed in host channel event class Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/cl906f.h | 2 -- drivers/gpu/drm/nouveau/include/nvif/cla06f.h | 2 -- drivers/gpu/drm/nouveau/include/nvif/clc36f.h | 2 -- drivers/gpu/drm/nouveau/include/nvif/if0020.h | 1 + .../drm/nouveau/include/nvkm/engine/fifo.h | 1 + drivers/gpu/drm/nouveau/nouveau_chan.c | 28 +++++++++++++------ drivers/gpu/drm/nouveau/nouveau_chan.h | 4 +-- drivers/gpu/drm/nouveau/nouveau_fence.c | 5 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 18 +----------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 14 ++-------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 1 - .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 15 ---------- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 1 - .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 1 - .../nouveau/nvkm/engine/fifo/gpfifotu102.c | 1 - 15 files changed, 30 insertions(+), 66 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h index 0e17b21e8c6ee..5ccc8fd8c458e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h @@ -10,6 +10,4 @@ struct fermi_channel_gpfifo_v0 { __u64 ioffset; __u64 vmm; }; - -#define NV906F_V0_NTFY_KILLED 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h index 00057ddfa41f2..10449accd3e84 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h @@ -12,6 +12,4 @@ struct kepler_channel_gpfifo_a_v0 { __u64 vmm; __u64 inst; }; - -#define NVA06F_V0_NTFY_KILLED 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/clc36f.h b/drivers/gpu/drm/nouveau/include/nvif/clc36f.h index b36624760dea3..cdf6708e1d9a6 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/clc36f.h +++ b/drivers/gpu/drm/nouveau/include/nvif/clc36f.h @@ -13,6 +13,4 @@ struct volta_channel_gpfifo_a_v0 { __u64 inst; __u32 token; }; - -#define NVC36F_V0_NTFY_KILLED 0x01 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0020.h b/drivers/gpu/drm/nouveau/include/nvif/if0020.h index 29a9fe2a8d871..1893b8aa0abb4 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0020.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0020.h @@ -6,6 +6,7 @@ union nvif_chan_event_args { struct nvif_chan_event_v0 { __u8 version; #define NVIF_CHAN_EVENT_V0_NON_STALL_INTR 0x00 +#define NVIF_CHAN_EVENT_V0_KILLED 0x01 __u8 type; } v0; }; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index ebfca3ffe9ae4..5a19c71f7ccbd 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -44,6 +44,7 @@ struct nvkm_fifo { #define NVKM_FIFO_EVENT_NON_STALL_INTR BIT(0) struct nvkm_event uevent; /* async user trigger */ +#define NVKM_FIFO_EVENT_KILLED BIT(0) struct nvkm_event kevent; /* channel killed */ }; diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 48dea5d0c580f..497d889c20b6b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "nouveau_drv.h" @@ -46,15 +47,17 @@ int nouveau_vram_pushbuf; module_param_named(vram_pushbuf, nouveau_vram_pushbuf, int, 0400); static int -nouveau_channel_killed(struct nvif_notify *ntfy) +nouveau_channel_killed(struct nvif_event *event, void *repv, u32 repc) { - struct nouveau_channel *chan = container_of(ntfy, typeof(*chan), kill); + struct nouveau_channel *chan = container_of(event, typeof(*chan), kill); struct nouveau_cli *cli = (void *)chan->user.client; + NV_PRINTK(warn, cli, "channel %d killed!\n", chan->chid); atomic_set(&chan->killed, 1); if (chan->fence) nouveau_fence_context_kill(chan->fence, -ENODEV); - return NVIF_NOTIFY_DROP; + + return NVIF_EVENT_DROP; } int @@ -96,7 +99,7 @@ nouveau_channel_del(struct nouveau_channel **pchan) nvif_object_dtor(&chan->nvsw); nvif_object_dtor(&chan->gart); nvif_object_dtor(&chan->vram); - nvif_notify_dtor(&chan->kill); + nvif_event_dtor(&chan->kill); nvif_object_dtor(&chan->user); nvif_object_dtor(&chan->push.ctxdma); nouveau_vma_del(&chan->push.vma); @@ -391,12 +394,19 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO && chan->user.oclass < AMPERE_CHANNEL_GPFIFO_B) { - ret = nvif_notify_ctor(&chan->user, "abi16ChanKilled", - nouveau_channel_killed, - true, NV906F_V0_NTFY_KILLED, - NULL, 0, 0, &chan->kill); + struct { + struct nvif_event_v0 base; + struct nvif_chan_event_v0 host; + } args; + + args.host.version = 0; + args.host.type = NVIF_CHAN_EVENT_V0_KILLED; + + ret = nvif_event_ctor(&chan->user, "abi16ChanKilled", chan->chid, + nouveau_channel_killed, false, + &args.base, sizeof(args), &chan->kill); if (ret == 0) - ret = nvif_notify_get(&chan->kill); + ret = nvif_event_allow(&chan->kill); if (ret) { NV_ERROR(drm, "Failed to request channel kill " "notification: %d\n", ret); diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h index 98ba9d27e6b46..195b38c847717 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.h +++ b/drivers/gpu/drm/nouveau/nouveau_chan.h @@ -2,7 +2,7 @@ #ifndef __NOUVEAU_CHAN_H__ #define __NOUVEAU_CHAN_H__ #include -#include +#include #include struct nvif_device; @@ -50,7 +50,7 @@ struct nouveau_channel { struct nvif_object user; - struct nvif_notify kill; + struct nvif_event kill; atomic_t killed; }; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index 62560e30febb4..c4c8af16af026 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -88,8 +88,9 @@ void nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error) { struct nouveau_fence *fence; + unsigned long flags; - spin_lock_irq(&fctx->lock); + spin_lock_irqsave(&fctx->lock, flags); while (!list_empty(&fctx->pending)) { fence = list_entry(fctx->pending.next, typeof(*fence), head); @@ -99,7 +100,7 @@ nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error) if (nouveau_fence_signal(fence)) nvif_event_block(&fctx->event); } - spin_unlock_irq(&fctx->lock); + spin_unlock_irqrestore(&fctx->lock, flags); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index ec790af9b24f0..a5d7c7d0ff7f3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -122,26 +121,11 @@ nvkm_fifo_chan_chid(struct nvkm_fifo *fifo, int chid, unsigned long *rflags) void nvkm_fifo_kevent(struct nvkm_fifo *fifo, int chid) { - nvkm_event_send(&fifo->kevent, 1, chid, NULL, 0); -} - -static int -nvkm_fifo_kevent_ctor(struct nvkm_object *object, void *data, u32 size, - struct nvkm_notify *notify) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - if (size == 0) { - notify->size = 0; - notify->types = 1; - notify->index = chan->chid; - return 0; - } - return -ENOSYS; + nvkm_event_send(&fifo->kevent, NVKM_FIFO_EVENT_KILLED, chid, NULL, 0); } static const struct nvkm_event_func nvkm_fifo_kevent_func = { - .ctor = nvkm_fifo_kevent_ctor, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 08f09f40defb3..39d6b9217f8d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -267,6 +267,9 @@ nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct n case NVIF_CHAN_EVENT_V0_NON_STALL_INTR: return nvkm_uevent_add(uevent, &chan->fifo->uevent, 0, NVKM_FIFO_EVENT_NON_STALL_INTR, NULL); + case NVIF_CHAN_EVENT_V0_KILLED: + return nvkm_uevent_add(uevent, &chan->fifo->kevent, chan->chid, + NVKM_FIFO_EVENT_KILLED, NULL); default: break; } @@ -274,16 +277,6 @@ nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct n return -ENOSYS; } -static int -nvkm_fifo_chan_ntfy(struct nvkm_object *object, u32 type, - struct nvkm_event **pevent) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - if (chan->func->ntfy) - return chan->func->ntfy(chan, type, pevent); - return -ENODEV; -} - static int nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) @@ -341,7 +334,6 @@ nvkm_fifo_chan_func = { .dtor = nvkm_fifo_chan_dtor, .init = nvkm_fifo_chan_init, .fini = nvkm_fifo_chan_fini, - .ntfy = nvkm_fifo_chan_ntfy, .map = nvkm_fifo_chan_map, .sclass = nvkm_fifo_chan_child_get, .uevent = nvkm_fifo_chan_uevent, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index e53504354841b..8ef972134f9be 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -8,7 +8,6 @@ struct nvkm_fifo_chan_func { void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); void (*fini)(struct nvkm_fifo_chan *); - int (*ntfy)(struct nvkm_fifo_chan *, u32 type, struct nvkm_event **); int (*engine_ctor)(struct nvkm_fifo_chan *, struct nvkm_engine *, struct nvkm_object *); void (*engine_dtor)(struct nvkm_fifo_chan *, struct nvkm_engine *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 569b5eedb84d6..4b1f6702680a6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -32,20 +32,6 @@ #include #include -int -gf100_fifo_chan_ntfy(struct nvkm_fifo_chan *chan, u32 type, - struct nvkm_event **pevent) -{ - switch (type) { - case NV906F_V0_NTFY_KILLED: - *pevent = &chan->fifo->kevent; - return 0; - default: - break; - } - return -EINVAL; -} - static u32 gf100_fifo_gpfifo_engine_addr(struct nvkm_engine *engine) { @@ -207,7 +193,6 @@ gf100_fifo_gpfifo_func = { .dtor = gf100_fifo_gpfifo_dtor, .init = gf100_fifo_gpfifo_init, .fini = gf100_fifo_gpfifo_fini, - .ntfy = gf100_fifo_chan_ntfy, .engine_ctor = gf100_fifo_gpfifo_engine_ctor, .engine_dtor = gf100_fifo_gpfifo_engine_dtor, .engine_init = gf100_fifo_gpfifo_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 80456ec70e8a0..ea9852a5fdd83 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -246,7 +246,6 @@ gk104_fifo_gpfifo_func = { .dtor = gk104_fifo_gpfifo_dtor, .init = gk104_fifo_gpfifo_init, .fini = gk104_fifo_gpfifo_fini, - .ntfy = gf100_fifo_chan_ntfy, .engine_ctor = gk104_fifo_gpfifo_engine_ctor, .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gk104_fifo_gpfifo_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 428f9b41165c9..e8546fed40241 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -125,7 +125,6 @@ gv100_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, .init = gk104_fifo_gpfifo_init, .fini = gk104_fifo_gpfifo_fini, - .ntfy = gf100_fifo_chan_ntfy, .engine_ctor = gk104_fifo_gpfifo_engine_ctor, .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index 99aafa103a317..4d35bee0ab54b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -40,7 +40,6 @@ tu102_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, .init = gk104_fifo_gpfifo_init, .fini = gk104_fifo_gpfifo_fini, - .ntfy = gf100_fifo_chan_ntfy, .engine_ctor = gk104_fifo_gpfifo_engine_ctor, .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, -- GitLab From 99d0701afdaf09ab5eb42b6578f2cc9913e7d85c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:40 +1000 Subject: [PATCH 1283/1927] drm/nouveau/nvkm: rip out old notify Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/event.h | 27 --- drivers/gpu/drm/nouveau/include/nvif/ioctl.h | 51 +---- drivers/gpu/drm/nouveau/include/nvif/notify.h | 35 --- .../drm/nouveau/include/nvkm/core/client.h | 15 +- .../gpu/drm/nouveau/include/nvkm/core/event.h | 9 - .../drm/nouveau/include/nvkm/core/notify.h | 39 ---- drivers/gpu/drm/nouveau/nouveau_drv.h | 5 - drivers/gpu/drm/nouveau/nouveau_nvif.c | 4 +- drivers/gpu/drm/nouveau/nouveau_usif.c | 6 - drivers/gpu/drm/nouveau/nvif/Kbuild | 1 - drivers/gpu/drm/nouveau/nvif/notify.c | 210 ------------------ drivers/gpu/drm/nouveau/nvkm/core/Kbuild | 1 - drivers/gpu/drm/nouveau/nvkm/core/client.c | 127 +---------- drivers/gpu/drm/nouveau/nvkm/core/event.c | 33 +-- drivers/gpu/drm/nouveau/nvkm/core/ioctl.c | 88 -------- drivers/gpu/drm/nouveau/nvkm/core/notify.c | 163 -------------- .../gpu/drm/nouveau/nvkm/engine/disp/base.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/disp/nv50.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/fault/gp100.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/fault/gv100.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/fault/tu102.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/gpio/base.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/i2c/base.c | 2 +- 25 files changed, 33 insertions(+), 805 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/notify.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvkm/core/notify.h delete mode 100644 drivers/gpu/drm/nouveau/nvif/notify.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/core/notify.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/event.h b/drivers/gpu/drm/nouveau/include/nvif/event.h index 3afcda2034d65..68bf6635841f7 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/event.h +++ b/drivers/gpu/drm/nouveau/include/nvif/event.h @@ -33,31 +33,4 @@ nvif_event_ctor(struct nvif_object *parent, const char *name, u32 handle, nvif_e void nvif_event_dtor(struct nvif_event *); int nvif_event_allow(struct nvif_event *); int nvif_event_block(struct nvif_event *); - -struct nvif_notify_req_v0 { - __u8 version; - __u8 reply; - __u8 pad02[5]; -#define NVIF_NOTIFY_V0_ROUTE_NVIF 0x00 - __u8 route; - __u64 token; /* must be unique */ - __u8 data[]; /* request data (below) */ -}; - -struct nvif_notify_rep_v0 { - __u8 version; - __u8 pad01[6]; - __u8 route; - __u64 token; - __u8 data[]; /* reply data (below) */ -}; - -struct nvif_notify_uevent_req { - /* nvif_notify_req ... */ -}; - -struct nvif_notify_uevent_rep { - /* nvif_notify_rep ... */ -}; - #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/ioctl.h b/drivers/gpu/drm/nouveau/include/nvif/ioctl.h index 886c63fe753fb..4e047bb1fc072 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/ioctl.h +++ b/drivers/gpu/drm/nouveau/include/nvif/ioctl.h @@ -15,10 +15,6 @@ struct nvif_ioctl_v0 { #define NVIF_IOCTL_V0_WR 0x06 #define NVIF_IOCTL_V0_MAP 0x07 #define NVIF_IOCTL_V0_UNMAP 0x08 -#define NVIF_IOCTL_V0_NTFY_NEW 0x09 -#define NVIF_IOCTL_V0_NTFY_DEL 0x0a -#define NVIF_IOCTL_V0_NTFY_GET 0x0b -#define NVIF_IOCTL_V0_NTFY_PUT 0x0c __u8 type; __u8 pad02[4]; #define NVIF_IOCTL_V0_OWNER_NVIF 0x00 @@ -63,6 +59,14 @@ struct nvif_ioctl_new_v0 { struct nvif_ioctl_del { }; +struct nvif_ioctl_mthd_v0 { + /* nvif_ioctl ... */ + __u8 version; + __u8 method; + __u8 pad02[6]; + __u8 data[]; /* method data (class.h) */ +}; + struct nvif_ioctl_rd_v0 { /* nvif_ioctl ... */ __u8 version; @@ -95,43 +99,4 @@ struct nvif_ioctl_map_v0 { struct nvif_ioctl_unmap { }; - -struct nvif_ioctl_ntfy_new_v0 { - /* nvif_ioctl ... */ - __u8 version; - __u8 event; - __u8 index; - __u8 pad03[5]; - __u8 data[]; /* event request data (event.h) */ -}; - -struct nvif_ioctl_ntfy_del_v0 { - /* nvif_ioctl ... */ - __u8 version; - __u8 index; - __u8 pad02[6]; -}; - -struct nvif_ioctl_ntfy_get_v0 { - /* nvif_ioctl ... */ - __u8 version; - __u8 index; - __u8 pad02[6]; -}; - -struct nvif_ioctl_ntfy_put_v0 { - /* nvif_ioctl ... */ - __u8 version; - __u8 index; - __u8 pad02[6]; -}; - -struct nvif_ioctl_mthd_v0 { - /* nvif_ioctl ... */ - __u8 version; - __u8 method; - __u8 pad02[6]; - __u8 data[]; /* method data (class.h) */ -}; - #endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/notify.h b/drivers/gpu/drm/nouveau/include/nvif/notify.h deleted file mode 100644 index 39f6b7ee1719e..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/notify.h +++ /dev/null @@ -1,35 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_NOTIFY_H__ -#define __NVIF_NOTIFY_H__ - -struct nvif_notify { - struct nvif_object *object; - const char *name; - int index; - -#define NVIF_NOTIFY_USER 0 -#define NVIF_NOTIFY_WORK 1 - unsigned long flags; - atomic_t putcnt; - void (*dtor)(struct nvif_notify *); -#define NVIF_NOTIFY_DROP 0 -#define NVIF_NOTIFY_KEEP 1 - int (*func)(struct nvif_notify *); - - /* this is const for a *very* good reason - the data might be on the - * stack from an irq handler. if you're not nvif/notify.c then you - * should probably think twice before casting it away... - */ - const void *data; - u32 size; - struct work_struct work; -}; - -int nvif_notify_ctor(struct nvif_object *, const char *name, - int (*func)(struct nvif_notify *), bool work, u8 type, - void *data, u32 size, u32 reply, struct nvif_notify *); -int nvif_notify_dtor(struct nvif_notify *); -int nvif_notify_get(struct nvif_notify *); -int nvif_notify_put(struct nvif_notify *); -int nvif_notify(const void *, u32, const void *, u32); -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h index 2eb22aaed700b..0d9fc741a7193 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/client.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/client.h @@ -10,30 +10,19 @@ struct nvkm_client { u64 device; u32 debug; - struct nvkm_client_notify *notify[32]; struct rb_root objroot; void *data; - int (*ntfy)(const void *, u32, const void *, u32); int (*event)(u64 token, void *argv, u32 argc); struct list_head umem; spinlock_t lock; }; -int nvkm_client_new(const char *name, u64 device, const char *cfg, - const char *dbg, - int (*)(const void *, u32, const void *, u32), - int (*)(u64, void *, u32), - struct nvkm_client **); +int nvkm_client_new(const char *name, u64 device, const char *cfg, const char *dbg, + int (*)(u64, void *, u32), struct nvkm_client **); struct nvkm_client *nvkm_client_search(struct nvkm_client *, u64 handle); -int nvkm_client_notify_new(struct nvkm_object *, struct nvkm_event *, - void *data, u32 size); -int nvkm_client_notify_del(struct nvkm_client *, int index); -int nvkm_client_notify_get(struct nvkm_client *, int index); -int nvkm_client_notify_put(struct nvkm_client *, int index); - /* logging for client-facing objects */ #define nvif_printk(o,l,p,f,a...) do { \ const struct nvkm_object *_object = (o); \ diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/event.h b/drivers/gpu/drm/nouveau/include/nvkm/core/event.h index d6755a89f5877..8e8fb4b6bca4f 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/event.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/event.h @@ -2,7 +2,6 @@ #ifndef __NVKM_EVENT_H__ #define __NVKM_EVENT_H__ #include -struct nvkm_notify; struct nvkm_object; struct nvkm_oclass; struct nvkm_uevent; @@ -16,16 +15,12 @@ struct nvkm_event { spinlock_t refs_lock; spinlock_t list_lock; - struct list_head list; int *refs; struct list_head ntfy; }; struct nvkm_event_func { - int (*ctor)(struct nvkm_object *, void *data, u32 size, - struct nvkm_notify *); - void (*send)(void *data, u32 size, struct nvkm_notify *); void (*init)(struct nvkm_event *, int type, int index); void (*fini)(struct nvkm_event *, int type, int index); }; @@ -33,10 +28,6 @@ struct nvkm_event_func { int nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *, int types_nr, int index_nr, struct nvkm_event *); void nvkm_event_fini(struct nvkm_event *); -void nvkm_event_get(struct nvkm_event *, u32 types, int index); -void nvkm_event_put(struct nvkm_event *, u32 types, int index); -void nvkm_event_send(struct nvkm_event *, u32 types, int index, - void *data, u32 size); #define NVKM_EVENT_KEEP 0 #define NVKM_EVENT_DROP 1 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/notify.h b/drivers/gpu/drm/nouveau/include/nvkm/core/notify.h deleted file mode 100644 index 3d358a66db3a3..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/notify.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVKM_NOTIFY_H__ -#define __NVKM_NOTIFY_H__ -#include -struct nvkm_object; - -struct nvkm_notify { - struct nvkm_event *event; - struct list_head head; -#define NVKM_NOTIFY_USER 0 -#define NVKM_NOTIFY_WORK 1 - unsigned long flags; - int block; -#define NVKM_NOTIFY_DROP 0 -#define NVKM_NOTIFY_KEEP 1 - int (*func)(struct nvkm_notify *); - - /* set by nvkm_event ctor */ - u32 types; - int index; - u32 size; - - struct work_struct work; - /* this is const for a *very* good reason - the data might be on the - * stack from an irq handler. if you're not core/notify.c then you - * should probably think twice before casting it away... - */ - const void *data; -}; - -int nvkm_notify_init(struct nvkm_object *, struct nvkm_event *, - int (*func)(struct nvkm_notify *), bool work, - void *data, u32 size, u32 reply, - struct nvkm_notify *); -void nvkm_notify_fini(struct nvkm_notify *); -void nvkm_notify_get(struct nvkm_notify *); -void nvkm_notify_put(struct nvkm_notify *); -void nvkm_notify_send(struct nvkm_notify *, void *data, u32 size); -#endif diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 1a92c980cb735..2b9045b12ac70 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -78,11 +78,6 @@ enum nouveau_drm_object_route { NVDRM_OBJECT_ANY = NVIF_IOCTL_V0_OWNER_ANY, }; -enum nouveau_drm_notify_route { - NVDRM_NOTIFY_NVIF = 0, - NVDRM_NOTIFY_USIF -}; - enum nouveau_drm_handle { NVDRM_CHAN = 0xcccc0000, /* |= client chid */ NVDRM_NVSW = 0x55550000, diff --git a/drivers/gpu/drm/nouveau/nouveau_nvif.c b/drivers/gpu/drm/nouveau/nouveau_nvif.c index be6d404a34596..1d49ebdfd5dc1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_nvif.c +++ b/drivers/gpu/drm/nouveau/nouveau_nvif.c @@ -27,12 +27,10 @@ ******************************************************************************/ #include -#include #include #include #include -#include #include #include @@ -87,7 +85,7 @@ static int nvkm_client_driver_init(const char *name, u64 device, const char *cfg, const char *dbg, void **ppriv) { - return nvkm_client_new(name, device, cfg, dbg, nvif_notify, nvkm_client_event, + return nvkm_client_new(name, device, cfg, dbg, nvkm_client_event, (struct nvkm_client **)ppriv); } diff --git a/drivers/gpu/drm/nouveau/nouveau_usif.c b/drivers/gpu/drm/nouveau/nouveau_usif.c index 36df6840c0994..002d1479ba89f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_usif.c +++ b/drivers/gpu/drm/nouveau/nouveau_usif.c @@ -151,12 +151,6 @@ usif_ioctl(struct drm_file *filp, void __user *user, u32 argc) case NVIF_IOCTL_V0_NEW: ret = usif_object_new(filp, data, size, argv, argc, abi16); break; - case NVIF_IOCTL_V0_NTFY_NEW: - case NVIF_IOCTL_V0_NTFY_DEL: - case NVIF_IOCTL_V0_NTFY_GET: - case NVIF_IOCTL_V0_NTFY_PUT: - ret = -ENOSYS; - break; default: ret = nvif_client_ioctl(client, argv, argc); break; diff --git a/drivers/gpu/drm/nouveau/nvif/Kbuild b/drivers/gpu/drm/nouveau/nvif/Kbuild index 7ce040ca23940..b7963a39dd91b 100644 --- a/drivers/gpu/drm/nouveau/nvif/Kbuild +++ b/drivers/gpu/drm/nouveau/nvif/Kbuild @@ -10,7 +10,6 @@ nvif-y += nvif/fifo.o nvif-y += nvif/head.o nvif-y += nvif/mem.o nvif-y += nvif/mmu.o -nvif-y += nvif/notify.o nvif-y += nvif/outp.o nvif-y += nvif/timer.o nvif-y += nvif/vmm.o diff --git a/drivers/gpu/drm/nouveau/nvif/notify.c b/drivers/gpu/drm/nouveau/nvif/notify.c deleted file mode 100644 index 143c8dc6889e9..0000000000000 --- a/drivers/gpu/drm/nouveau/nvif/notify.c +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright 2014 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ - -#include -#include -#include -#include -#include -#include - -static inline int -nvif_notify_put_(struct nvif_notify *notify) -{ - struct nvif_object *object = notify->object; - struct { - struct nvif_ioctl_v0 ioctl; - struct nvif_ioctl_ntfy_put_v0 ntfy; - } args = { - .ioctl.type = NVIF_IOCTL_V0_NTFY_PUT, - .ntfy.index = notify->index, - }; - - if (atomic_inc_return(¬ify->putcnt) != 1) - return 0; - - return nvif_object_ioctl(object, &args, sizeof(args), NULL); -} - -int -nvif_notify_put(struct nvif_notify *notify) -{ - if (likely(notify->object) && - test_and_clear_bit(NVIF_NOTIFY_USER, ¬ify->flags)) { - int ret = nvif_notify_put_(notify); - if (test_bit(NVIF_NOTIFY_WORK, ¬ify->flags)) - flush_work(¬ify->work); - return ret; - } - return 0; -} - -static inline int -nvif_notify_get_(struct nvif_notify *notify) -{ - struct nvif_object *object = notify->object; - struct { - struct nvif_ioctl_v0 ioctl; - struct nvif_ioctl_ntfy_get_v0 ntfy; - } args = { - .ioctl.type = NVIF_IOCTL_V0_NTFY_GET, - .ntfy.index = notify->index, - }; - - if (atomic_dec_return(¬ify->putcnt) != 0) - return 0; - - return nvif_object_ioctl(object, &args, sizeof(args), NULL); -} - -int -nvif_notify_get(struct nvif_notify *notify) -{ - if (likely(notify->object) && - !test_and_set_bit(NVIF_NOTIFY_USER, ¬ify->flags)) - return nvif_notify_get_(notify); - return 0; -} - -static inline int -nvif_notify_func(struct nvif_notify *notify, bool keep) -{ - int ret = notify->func(notify); - if (ret == NVIF_NOTIFY_KEEP || - !test_and_clear_bit(NVIF_NOTIFY_USER, ¬ify->flags)) { - if (!keep) - atomic_dec(¬ify->putcnt); - else - nvif_notify_get_(notify); - } - return ret; -} - -static void -nvif_notify_work(struct work_struct *work) -{ - struct nvif_notify *notify = container_of(work, typeof(*notify), work); - nvif_notify_func(notify, true); -} - -int -nvif_notify(const void *header, u32 length, const void *data, u32 size) -{ - struct nvif_notify *notify = NULL; - const union { - struct nvif_notify_rep_v0 v0; - } *args = header; - int ret = NVIF_NOTIFY_DROP; - - if (length == sizeof(args->v0) && args->v0.version == 0) { - if (WARN_ON(args->v0.route)) - return NVIF_NOTIFY_DROP; - notify = (void *)(unsigned long)args->v0.token; - } - - if (!WARN_ON(notify == NULL)) { - struct nvif_client *client = notify->object->client; - if (!WARN_ON(notify->size != size)) { - atomic_inc(¬ify->putcnt); - if (test_bit(NVIF_NOTIFY_WORK, ¬ify->flags)) { - memcpy((void *)notify->data, data, size); - schedule_work(¬ify->work); - return NVIF_NOTIFY_DROP; - } - notify->data = data; - ret = nvif_notify_func(notify, client->driver->keep); - notify->data = NULL; - } - } - - return ret; -} - -int -nvif_notify_dtor(struct nvif_notify *notify) -{ - struct nvif_object *object = notify->object; - struct { - struct nvif_ioctl_v0 ioctl; - struct nvif_ioctl_ntfy_del_v0 ntfy; - } args = { - .ioctl.type = NVIF_IOCTL_V0_NTFY_DEL, - .ntfy.index = notify->index, - }; - int ret = nvif_notify_put(notify); - if (ret >= 0 && object) { - ret = nvif_object_ioctl(object, &args, sizeof(args), NULL); - notify->object = NULL; - kfree((void *)notify->data); - } - return ret; -} - -int -nvif_notify_ctor(struct nvif_object *object, const char *name, - int (*func)(struct nvif_notify *), bool work, u8 event, - void *data, u32 size, u32 reply, struct nvif_notify *notify) -{ - struct { - struct nvif_ioctl_v0 ioctl; - struct nvif_ioctl_ntfy_new_v0 ntfy; - struct nvif_notify_req_v0 req; - } *args; - int ret = -ENOMEM; - - notify->object = object; - notify->name = name ? name : "nvifNotify"; - notify->flags = 0; - atomic_set(¬ify->putcnt, 1); - notify->func = func; - notify->data = NULL; - notify->size = reply; - if (work) { - INIT_WORK(¬ify->work, nvif_notify_work); - set_bit(NVIF_NOTIFY_WORK, ¬ify->flags); - notify->data = kmalloc(notify->size, GFP_KERNEL); - if (!notify->data) - goto done; - } - - if (!(args = kmalloc(sizeof(*args) + size, GFP_KERNEL))) - goto done; - args->ioctl.version = 0; - args->ioctl.type = NVIF_IOCTL_V0_NTFY_NEW; - args->ntfy.version = 0; - args->ntfy.event = event; - args->req.version = 0; - args->req.reply = notify->size; - args->req.route = 0; - args->req.token = (unsigned long)(void *)notify; - - memcpy(args->req.data, data, size); - ret = nvif_object_ioctl(object, args, sizeof(*args) + size, NULL); - notify->index = args->ntfy.index; - kfree(args); -done: - if (ret) - nvif_notify_dtor(notify); - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/core/Kbuild b/drivers/gpu/drm/nouveau/nvkm/core/Kbuild index 216a3db821ee9..65dad05f047c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/core/Kbuild @@ -8,7 +8,6 @@ nvkm-y += nvkm/core/gpuobj.o nvkm-y += nvkm/core/ioctl.o nvkm-y += nvkm/core/memory.o nvkm-y += nvkm/core/mm.o -nvkm-y += nvkm/core/notify.o nvkm-y += nvkm/core/object.o nvkm-y += nvkm/core/oproxy.o nvkm-y += nvkm/core/option.o diff --git a/drivers/gpu/drm/nouveau/nvkm/core/client.c b/drivers/gpu/drm/nouveau/nvkm/core/client.c index e1d978fbfe724..ebdeb8eb9e774 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/client.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/client.c @@ -23,7 +23,6 @@ */ #include #include -#include #include #include @@ -44,7 +43,7 @@ nvkm_uclient_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))){ args->v0.name[sizeof(args->v0.name) - 1] = 0; ret = nvkm_client_new(args->v0.name, args->v0.device, NULL, - NULL, oclass->client->ntfy, oclass->client->event, &client); + NULL, oclass->client->event, &client); if (ret) return ret; } else @@ -68,113 +67,6 @@ nvkm_uclient_sclass = { .ctor = nvkm_uclient_new, }; -struct nvkm_client_notify { - struct nvkm_client *client; - struct nvkm_notify n; - u8 version; - u8 size; - union { - struct nvif_notify_rep_v0 v0; - } rep; -}; - -static int -nvkm_client_notify(struct nvkm_notify *n) -{ - struct nvkm_client_notify *notify = container_of(n, typeof(*notify), n); - struct nvkm_client *client = notify->client; - return client->ntfy(¬ify->rep, notify->size, n->data, n->size); -} - -int -nvkm_client_notify_put(struct nvkm_client *client, int index) -{ - if (index < ARRAY_SIZE(client->notify)) { - if (client->notify[index]) { - nvkm_notify_put(&client->notify[index]->n); - return 0; - } - } - return -ENOENT; -} - -int -nvkm_client_notify_get(struct nvkm_client *client, int index) -{ - if (index < ARRAY_SIZE(client->notify)) { - if (client->notify[index]) { - nvkm_notify_get(&client->notify[index]->n); - return 0; - } - } - return -ENOENT; -} - -int -nvkm_client_notify_del(struct nvkm_client *client, int index) -{ - if (index < ARRAY_SIZE(client->notify)) { - if (client->notify[index]) { - nvkm_notify_fini(&client->notify[index]->n); - kfree(client->notify[index]); - client->notify[index] = NULL; - return 0; - } - } - return -ENOENT; -} - -int -nvkm_client_notify_new(struct nvkm_object *object, - struct nvkm_event *event, void *data, u32 size) -{ - struct nvkm_client *client = object->client; - struct nvkm_client_notify *notify; - union { - struct nvif_notify_req_v0 v0; - } *req = data; - u8 index, reply; - int ret = -ENOSYS; - - for (index = 0; index < ARRAY_SIZE(client->notify); index++) { - if (!client->notify[index]) - break; - } - - if (index == ARRAY_SIZE(client->notify)) - return -ENOSPC; - - notify = kzalloc(sizeof(*notify), GFP_KERNEL); - if (!notify) - return -ENOMEM; - - nvif_ioctl(object, "notify new size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, req->v0, 0, 0, true))) { - nvif_ioctl(object, "notify new vers %d reply %d route %02x " - "token %llx\n", req->v0.version, - req->v0.reply, req->v0.route, req->v0.token); - notify->version = req->v0.version; - notify->size = sizeof(notify->rep.v0); - notify->rep.v0.version = req->v0.version; - notify->rep.v0.route = req->v0.route; - notify->rep.v0.token = req->v0.token; - reply = req->v0.reply; - } - - if (ret == 0) { - ret = nvkm_notify_init(object, event, nvkm_client_notify, - false, data, size, reply, ¬ify->n); - if (ret == 0) { - client->notify[index] = notify; - notify->client = client; - return index; - } - } - - kfree(notify); - return ret; -} - static const struct nvkm_object_func nvkm_client; struct nvkm_client * nvkm_client_search(struct nvkm_client *client, u64 handle) @@ -255,23 +147,13 @@ nvkm_client_child_get(struct nvkm_object *object, int index, static int nvkm_client_fini(struct nvkm_object *object, bool suspend) { - struct nvkm_client *client = nvkm_client(object); - const char *name[2] = { "fini", "suspend" }; - int i; - nvif_debug(object, "%s notify\n", name[suspend]); - for (i = 0; i < ARRAY_SIZE(client->notify); i++) - nvkm_client_notify_put(client, i); return 0; } static void * nvkm_client_dtor(struct nvkm_object *object) { - struct nvkm_client *client = nvkm_client(object); - int i; - for (i = 0; i < ARRAY_SIZE(client->notify); i++) - nvkm_client_notify_del(client, i); - return client; + return nvkm_client(object); } static const struct nvkm_object_func @@ -283,9 +165,7 @@ nvkm_client = { }; int -nvkm_client_new(const char *name, u64 device, const char *cfg, - const char *dbg, - int (*ntfy)(const void *, u32, const void *, u32), +nvkm_client_new(const char *name, u64 device, const char *cfg, const char *dbg, int (*event)(u64, void *, u32), struct nvkm_client **pclient) { struct nvkm_oclass oclass = { .base = nvkm_uclient_sclass }; @@ -300,7 +180,6 @@ nvkm_client_new(const char *name, u64 device, const char *cfg, client->device = device; client->debug = nvkm_dbgopt(dbg, "CLIENT"); client->objroot = RB_ROOT; - client->ntfy = ntfy; client->event = event; INIT_LIST_HEAD(&client->umem); spin_lock_init(&client->lock); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/event.c b/drivers/gpu/drm/nouveau/nvkm/core/event.c index ec508d1bbe1da..622df36391be9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/event.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/event.c @@ -20,10 +20,9 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include -#include #include -void +static void nvkm_event_put(struct nvkm_event *event, u32 types, int index) { assert_spin_locked(&event->refs_lock); @@ -40,7 +39,7 @@ nvkm_event_put(struct nvkm_event *event, u32 types, int index) } } -void +static void nvkm_event_get(struct nvkm_event *event, u32 types, int index) { assert_spin_locked(&event->refs_lock); @@ -171,8 +170,13 @@ void nvkm_event_ntfy(struct nvkm_event *event, int id, u32 bits) { struct nvkm_event_ntfy *ntfy, *ntmp; + unsigned long flags; + + if (!event->refs || WARN_ON(id >= event->index_nr)) + return; nvkm_trace(event->subdev, "event: ntfy %08x on %d\n", bits, id); + spin_lock_irqsave(&event->list_lock, flags); list_for_each_entry_safe(ntfy, ntmp, &event->ntfy, head) { if (ntfy->id == id && ntfy->bits & bits) { @@ -180,29 +184,7 @@ nvkm_event_ntfy(struct nvkm_event *event, int id, u32 bits) ntfy->func(ntfy, ntfy->bits & bits); } } -} -void -nvkm_event_send(struct nvkm_event *event, u32 types, int index, void *data, u32 size) -{ - struct nvkm_notify *notify; - unsigned long flags; - - if (!event->refs || WARN_ON(index >= event->index_nr)) - return; - - spin_lock_irqsave(&event->list_lock, flags); - nvkm_event_ntfy(event, index, types); - - list_for_each_entry(notify, &event->list, head) { - if (notify->index == index && (notify->types & types)) { - if (event->func->send) { - event->func->send(data, size, notify); - continue; - } - nvkm_notify_send(notify, data, size); - } - } spin_unlock_irqrestore(&event->list_lock, flags); } @@ -229,7 +211,6 @@ nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, event->index_nr = index_nr; spin_lock_init(&event->refs_lock); spin_lock_init(&event->list_lock); - INIT_LIST_HEAD(&event->list); INIT_LIST_HEAD(&event->ntfy); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c index b42d0bf2c492b..0b33287e43a70 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/ioctl.c @@ -313,90 +313,6 @@ nvkm_ioctl_unmap(struct nvkm_client *client, return ret; } -static int -nvkm_ioctl_ntfy_new(struct nvkm_client *client, - struct nvkm_object *object, void *data, u32 size) -{ - union { - struct nvif_ioctl_ntfy_new_v0 v0; - } *args = data; - struct nvkm_event *event; - int ret = -ENOSYS; - - nvif_ioctl(object, "ntfy new size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, true))) { - nvif_ioctl(object, "ntfy new vers %d event %02x\n", - args->v0.version, args->v0.event); - ret = nvkm_object_ntfy(object, args->v0.event, &event); - if (ret == 0) { - ret = nvkm_client_notify_new(object, event, data, size); - if (ret >= 0) { - args->v0.index = ret; - ret = 0; - } - } - } - - return ret; -} - -static int -nvkm_ioctl_ntfy_del(struct nvkm_client *client, - struct nvkm_object *object, void *data, u32 size) -{ - union { - struct nvif_ioctl_ntfy_del_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(object, "ntfy del size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "ntfy del vers %d index %d\n", - args->v0.version, args->v0.index); - ret = nvkm_client_notify_del(client, args->v0.index); - } - - return ret; -} - -static int -nvkm_ioctl_ntfy_get(struct nvkm_client *client, - struct nvkm_object *object, void *data, u32 size) -{ - union { - struct nvif_ioctl_ntfy_get_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(object, "ntfy get size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "ntfy get vers %d index %d\n", - args->v0.version, args->v0.index); - ret = nvkm_client_notify_get(client, args->v0.index); - } - - return ret; -} - -static int -nvkm_ioctl_ntfy_put(struct nvkm_client *client, - struct nvkm_object *object, void *data, u32 size) -{ - union { - struct nvif_ioctl_ntfy_put_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(object, "ntfy put size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(object, "ntfy put vers %d index %d\n", - args->v0.version, args->v0.index); - ret = nvkm_client_notify_put(client, args->v0.index); - } - - return ret; -} - static struct { int version; int (*func)(struct nvkm_client *, struct nvkm_object *, void *, u32); @@ -411,10 +327,6 @@ nvkm_ioctl_v0[] = { { 0x00, nvkm_ioctl_wr }, { 0x00, nvkm_ioctl_map }, { 0x00, nvkm_ioctl_unmap }, - { 0x00, nvkm_ioctl_ntfy_new }, - { 0x00, nvkm_ioctl_ntfy_del }, - { 0x00, nvkm_ioctl_ntfy_get }, - { 0x00, nvkm_ioctl_ntfy_put }, }; static int diff --git a/drivers/gpu/drm/nouveau/nvkm/core/notify.c b/drivers/gpu/drm/nouveau/nvkm/core/notify.c deleted file mode 100644 index 023610d01458d..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/core/notify.c +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Copyright 2014 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include -#include - -static inline void -nvkm_notify_put_locked(struct nvkm_notify *notify) -{ - if (notify->block++ == 0) - nvkm_event_put(notify->event, notify->types, notify->index); -} - -void -nvkm_notify_put(struct nvkm_notify *notify) -{ - struct nvkm_event *event = notify->event; - unsigned long flags; - if (likely(event) && - test_and_clear_bit(NVKM_NOTIFY_USER, ¬ify->flags)) { - spin_lock_irqsave(&event->refs_lock, flags); - nvkm_notify_put_locked(notify); - spin_unlock_irqrestore(&event->refs_lock, flags); - if (test_bit(NVKM_NOTIFY_WORK, ¬ify->flags)) - flush_work(¬ify->work); - } -} - -static inline void -nvkm_notify_get_locked(struct nvkm_notify *notify) -{ - if (--notify->block == 0) - nvkm_event_get(notify->event, notify->types, notify->index); -} - -void -nvkm_notify_get(struct nvkm_notify *notify) -{ - struct nvkm_event *event = notify->event; - unsigned long flags; - if (likely(event) && - !test_and_set_bit(NVKM_NOTIFY_USER, ¬ify->flags)) { - spin_lock_irqsave(&event->refs_lock, flags); - nvkm_notify_get_locked(notify); - spin_unlock_irqrestore(&event->refs_lock, flags); - } -} - -static inline void -nvkm_notify_func(struct nvkm_notify *notify) -{ - struct nvkm_event *event = notify->event; - int ret = notify->func(notify); - unsigned long flags; - if ((ret == NVKM_NOTIFY_KEEP) || - !test_and_clear_bit(NVKM_NOTIFY_USER, ¬ify->flags)) { - spin_lock_irqsave(&event->refs_lock, flags); - nvkm_notify_get_locked(notify); - spin_unlock_irqrestore(&event->refs_lock, flags); - } -} - -static void -nvkm_notify_work(struct work_struct *work) -{ - struct nvkm_notify *notify = container_of(work, typeof(*notify), work); - nvkm_notify_func(notify); -} - -void -nvkm_notify_send(struct nvkm_notify *notify, void *data, u32 size) -{ - struct nvkm_event *event = notify->event; - unsigned long flags; - - assert_spin_locked(&event->list_lock); - BUG_ON(size != notify->size); - - spin_lock_irqsave(&event->refs_lock, flags); - if (notify->block) { - spin_unlock_irqrestore(&event->refs_lock, flags); - return; - } - nvkm_notify_put_locked(notify); - spin_unlock_irqrestore(&event->refs_lock, flags); - - if (test_bit(NVKM_NOTIFY_WORK, ¬ify->flags)) { - memcpy((void *)notify->data, data, size); - schedule_work(¬ify->work); - } else { - notify->data = data; - nvkm_notify_func(notify); - notify->data = NULL; - } -} - -void -nvkm_notify_fini(struct nvkm_notify *notify) -{ - unsigned long flags; - if (notify->event) { - nvkm_notify_put(notify); - spin_lock_irqsave(¬ify->event->list_lock, flags); - list_del(¬ify->head); - spin_unlock_irqrestore(¬ify->event->list_lock, flags); - kfree((void *)notify->data); - notify->event = NULL; - } -} - -int -nvkm_notify_init(struct nvkm_object *object, struct nvkm_event *event, - int (*func)(struct nvkm_notify *), bool work, - void *data, u32 size, u32 reply, - struct nvkm_notify *notify) -{ - unsigned long flags; - int ret = -ENODEV; - if ((notify->event = event), event->refs) { - ret = event->func->ctor(object, data, size, notify); - if (ret == 0 && (ret = -EINVAL, notify->size == reply)) { - notify->flags = 0; - notify->block = 1; - notify->func = func; - notify->data = NULL; - if (ret = 0, work) { - INIT_WORK(¬ify->work, nvkm_notify_work); - set_bit(NVKM_NOTIFY_WORK, ¬ify->flags); - notify->data = kmalloc(reply, GFP_KERNEL); - if (!notify->data) - ret = -ENOMEM; - } - } - if (ret == 0) { - spin_lock_irqsave(&event->list_lock, flags); - list_add_tail(¬ify->head, &event->list); - spin_unlock_irqrestore(&event->list_lock, flags); - } - } - if (ret) - notify->event = NULL; - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c index c1d81cf74c57c..73104b59f97fe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/base.c @@ -65,7 +65,7 @@ nvkm_disp_vblank_func = { void nvkm_disp_vblank(struct nvkm_disp *disp, int head) { - nvkm_event_send(&disp->vblank, NVKM_DISP_HEAD_EVENT_VBLANK, head, NULL, 0); + nvkm_event_ntfy(&disp->vblank, head, NVKM_DISP_HEAD_EVENT_VBLANK); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c index 55ac6d70e9500..be81168029604 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c @@ -503,7 +503,7 @@ nv50_disp_chan_uevent_init(struct nvkm_event *event, int types, int index) void nv50_disp_chan_uevent_send(struct nvkm_disp *disp, int chid) { - nvkm_event_send(&disp->uevent, NVKM_DISP_EVENT_CHAN_AWAKEN, chid, NULL, 0); + nvkm_event_ntfy(&disp->uevent, chid, NVKM_DISP_EVENT_CHAN_AWAKEN); } const struct nvkm_event_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index a5d7c7d0ff7f3..518f3c492f017 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -121,7 +121,7 @@ nvkm_fifo_chan_chid(struct nvkm_fifo *fifo, int chid, unsigned long *rflags) void nvkm_fifo_kevent(struct nvkm_fifo *fifo, int chid) { - nvkm_event_send(&fifo->kevent, NVKM_FIFO_EVENT_KILLED, chid, NULL, 0); + nvkm_event_ntfy(&fifo->kevent, chid, NVKM_FIFO_EVENT_KILLED); } static const struct nvkm_event_func @@ -151,7 +151,7 @@ nvkm_fifo_uevent_func = { void nvkm_fifo_uevent(struct nvkm_fifo *fifo) { - nvkm_event_send(&fifo->uevent, NVKM_FIFO_EVENT_NON_STALL_INTR, 0, NULL, 0); + nvkm_event_ntfy(&fifo->uevent, 0, NVKM_FIFO_EVENT_NON_STALL_INTR); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c index 5e65e13b00533..834b8cbed51d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/chan.c @@ -35,7 +35,7 @@ nvkm_sw_chan_mthd(struct nvkm_sw_chan *chan, int subc, u32 mthd, u32 data) case 0x0000: return true; case 0x0500: - nvkm_event_send(&chan->event, NVKM_SW_CHAN_EVENT_PAGE_FLIP, 0, NULL, 0); + nvkm_event_ntfy(&chan->event, 0, NVKM_SW_CHAN_EVENT_PAGE_FLIP); return true; default: if (chan->func->mthd) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c index a96a1d633cb31..04c7526888bc4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gp100.c @@ -65,7 +65,7 @@ gp100_fault_buffer_info(struct nvkm_fault_buffer *buffer) void gp100_fault_intr(struct nvkm_fault *fault) { - nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 0, NULL, 0); + nvkm_event_ntfy(&fault->event, 0, NVKM_FAULT_BUFFER_EVENT_PENDING); } static const struct nvkm_fault_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c index 21756d65663b7..8e34d40e7649b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/gv100.c @@ -166,14 +166,14 @@ gv100_fault_intr(struct nvkm_fault *fault) if (stat & 0x20000000) { if (fault->buffer[0]) { - nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 0, NULL, 0); + nvkm_event_ntfy(&fault->event, 0, NVKM_FAULT_BUFFER_EVENT_PENDING); stat &= ~0x20000000; } } if (stat & 0x08000000) { if (fault->buffer[1]) { - nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 1, NULL, 0); + nvkm_event_ntfy(&fault->event, 1, NVKM_FAULT_BUFFER_EVENT_PENDING); stat &= ~0x08000000; } } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c index 2d3660d21b9a9..917a7343db315 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c @@ -126,7 +126,7 @@ tu102_fault_intr(struct nvkm_fault *fault) nvkm_wr32(device, 0xb81010, 0x10); if (fault->buffer[0]) { - nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 0, NULL, 0); + nvkm_event_ntfy(&fault->event, 0, NVKM_FAULT_BUFFER_EVENT_PENDING); stat &= ~0x00000200; } } @@ -137,7 +137,7 @@ tu102_fault_intr(struct nvkm_fault *fault) nvkm_wr32(device, 0xb81008, 0x1); if (fault->buffer[1]) { - nvkm_event_send(&fault->event, NVKM_FAULT_BUFFER_EVENT_PENDING, 1, NULL, 0); + nvkm_event_ntfy(&fault->event, 1, NVKM_FAULT_BUFFER_EVENT_PENDING); stat &= ~0x00000100; } } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c index 2c2c23b8663b6..b196baa376dcd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gpio/base.c @@ -139,7 +139,7 @@ nvkm_gpio_intr(struct nvkm_subdev *subdev) for (i = 0; (hi | lo) && i < gpio->func->lines; i++) { u32 mask = (NVKM_GPIO_HI * !!(hi & (1 << i))) | (NVKM_GPIO_LO * !!(lo & (1 << i))); - nvkm_event_send(&gpio->event, mask, i, NULL, 0); + nvkm_event_ntfy(&gpio->event, i, mask); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c index feb7de985d2d7..976539de4220c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/i2c/base.c @@ -130,7 +130,7 @@ nvkm_i2c_intr(struct nvkm_subdev *subdev) if (rq & aux->intr) mask |= NVKM_I2C_IRQ; if (tx & aux->intr) mask |= NVKM_I2C_DONE; if (mask) - nvkm_event_send(&i2c->event, mask, aux->id, NULL, 0); + nvkm_event_ntfy(&i2c->event, aux->id, mask); } } -- GitLab From 4a16dd9d18a04742292cc7bfdbb79aedfe24e2c8 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:40 +1000 Subject: [PATCH 1284/1927] drm/nouveau/kms: switch to drm fbdev helpers This removes support for accelerated fbcon rendering, and fixes a number of races/crashes/issues around suspend/resume/module unload etc. Losing HW accelerated rendering isn't ideal, but it's been significantly reduced in performance since the removal of accelerated scrolling in the kernel anyway - not to mention, can be racey (skips cpu<->gpu sync) from certain contexts. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/Kbuild | 4 - drivers/gpu/drm/nouveau/dispnv04/crtc.c | 10 +- drivers/gpu/drm/nouveau/dispnv50/disp.c | 3 +- drivers/gpu/drm/nouveau/nouveau_display.c | 10 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 14 +- drivers/gpu/drm/nouveau/nouveau_drv.h | 3 - drivers/gpu/drm/nouveau/nouveau_fbcon.c | 614 ---------------------- drivers/gpu/drm/nouveau/nouveau_fbcon.h | 82 --- drivers/gpu/drm/nouveau/nouveau_vga.c | 1 - drivers/gpu/drm/nouveau/nv04_fbcon.c | 259 --------- drivers/gpu/drm/nouveau/nv50_fbcon.c | 299 ----------- drivers/gpu/drm/nouveau/nvc0_fbcon.c | 297 ----------- 12 files changed, 15 insertions(+), 1581 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/nouveau_fbcon.c delete mode 100644 drivers/gpu/drm/nouveau/nouveau_fbcon.h delete mode 100644 drivers/gpu/drm/nouveau/nv04_fbcon.c delete mode 100644 drivers/gpu/drm/nouveau/nv50_fbcon.c delete mode 100644 drivers/gpu/drm/nouveau/nvc0_fbcon.c diff --git a/drivers/gpu/drm/nouveau/Kbuild b/drivers/gpu/drm/nouveau/Kbuild index 60586fb8275e4..5e5617006da50 100644 --- a/drivers/gpu/drm/nouveau/Kbuild +++ b/drivers/gpu/drm/nouveau/Kbuild @@ -54,10 +54,6 @@ nouveau-y += nouveau_bios.o nouveau-y += nouveau_connector.o nouveau-y += nouveau_display.o nouveau-y += nouveau_dp.o -nouveau-y += nouveau_fbcon.o -nouveau-y += nv04_fbcon.o -nouveau-y += nv50_fbcon.o -nouveau-y += nvc0_fbcon.o include $(src)/dispnv04/Kbuild include $(src)/dispnv50/Kbuild diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c index 4e8b84f592800..0e0f117bc70b1 100644 --- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c +++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c @@ -23,6 +23,7 @@ * DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include @@ -37,7 +38,6 @@ #include "nouveau_crtc.h" #include "hw.h" #include "nvreg.h" -#include "nouveau_fbcon.h" #include "disp.h" #include "nouveau_dma.h" @@ -915,14 +915,6 @@ nv04_crtc_mode_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, int x, int y, enum mode_set_atomic state) { - struct nouveau_drm *drm = nouveau_drm(crtc->dev); - struct drm_device *dev = drm->dev; - - if (state == ENTER_ATOMIC_MODE_SET) - nouveau_fbcon_accel_save_disable(dev); - else - nouveau_fbcon_accel_restore(dev); - return nv04_crtc_do_mode_set_base(crtc, fb, x, y, true); } diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index e720c01841b26..b7084c17f9c1f 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -64,7 +64,6 @@ #include "nouveau_connector.h" #include "nouveau_encoder.h" #include "nouveau_fence.h" -#include "nouveau_fbcon.h" #include @@ -2446,7 +2445,7 @@ nv50_disp_atomic_state_alloc(struct drm_device *dev) static const struct drm_mode_config_funcs nv50_disp_func = { .fb_create = nouveau_user_framebuffer_create, - .output_poll_changed = nouveau_fbcon_output_poll_changed, + .output_poll_changed = drm_fb_helper_output_poll_changed, .atomic_check = nv50_disp_atomic_check, .atomic_commit = nv50_disp_atomic_commit, .atomic_state_alloc = nv50_disp_atomic_state_alloc, diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c index 55099d9ce1c8f..ec3ffff487fcd 100644 --- a/drivers/gpu/drm/nouveau/nouveau_display.c +++ b/drivers/gpu/drm/nouveau/nouveau_display.c @@ -35,7 +35,6 @@ #include #include -#include "nouveau_fbcon.h" #include "nouveau_crtc.h" #include "nouveau_gem.h" #include "nouveau_connector.h" @@ -392,7 +391,7 @@ nouveau_user_framebuffer_create(struct drm_device *dev, static const struct drm_mode_config_funcs nouveau_mode_config_funcs = { .fb_create = nouveau_user_framebuffer_create, - .output_poll_changed = nouveau_fbcon_output_poll_changed, + .output_poll_changed = drm_fb_helper_output_poll_changed, }; @@ -788,6 +787,9 @@ nouveau_display_suspend(struct drm_device *dev, bool runtime) { struct nouveau_display *disp = nouveau_display(dev); + /* Disable console. */ + drm_fb_helper_set_suspend_unlocked(dev->fb_helper, true); + if (drm_drv_uses_atomic_modeset(dev)) { if (!runtime) { disp->suspend = drm_atomic_helper_suspend(dev); @@ -815,8 +817,10 @@ nouveau_display_resume(struct drm_device *dev, bool runtime) drm_atomic_helper_resume(dev, disp->suspend); disp->suspend = NULL; } - return; } + + /* Enable console. */ + drm_fb_helper_set_suspend_unlocked(dev->fb_helper, false); } int diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index fd99ec0f4257a..9df276bff3c7e 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -62,7 +63,6 @@ #include "nouveau_bios.h" #include "nouveau_ioctl.h" #include "nouveau_abi16.h" -#include "nouveau_fbcon.h" #include "nouveau_fence.h" #include "nouveau_debugfs.h" #include "nouveau_usif.h" @@ -611,7 +611,6 @@ nouveau_drm_device_init(struct drm_device *dev) nouveau_hwmon_init(dev); nouveau_svm_init(drm); nouveau_dmem_init(drm); - nouveau_fbcon_init(dev); nouveau_led_init(dev); if (nouveau_pmops_runtime()) { @@ -655,7 +654,6 @@ nouveau_drm_device_fini(struct drm_device *dev) } nouveau_led_fini(dev); - nouveau_fbcon_fini(dev); nouveau_dmem_fini(drm); nouveau_svm_fini(drm); nouveau_hwmon_fini(dev); @@ -809,6 +807,11 @@ static int nouveau_drm_probe(struct pci_dev *pdev, if (ret) goto fail_drm_dev_init; + if (nouveau_drm(drm_dev)->client.device.info.ram_size <= 32 * 1024 * 1024) + drm_fbdev_generic_setup(drm_dev, 8); + else + drm_fbdev_generic_setup(drm_dev, 32); + quirk_broken_nv_runpm(pdev); return 0; @@ -865,8 +868,6 @@ nouveau_do_suspend(struct drm_device *dev, bool runtime) nouveau_led_suspend(dev); if (dev->mode_config.num_crtc) { - NV_DEBUG(drm, "suspending console...\n"); - nouveau_fbcon_set_suspend(dev, 1); NV_DEBUG(drm, "suspending display...\n"); ret = nouveau_display_suspend(dev, runtime); if (ret) @@ -940,8 +941,6 @@ nouveau_do_resume(struct drm_device *dev, bool runtime) if (dev->mode_config.num_crtc) { NV_DEBUG(drm, "resuming display...\n"); nouveau_display_resume(dev, runtime); - NV_DEBUG(drm, "resuming console...\n"); - nouveau_fbcon_set_suspend(dev, 0); } nouveau_led_resume(dev); @@ -1296,7 +1295,6 @@ static void nouveau_display_options(void) DRM_DEBUG_DRIVER("... tv_disable : %d\n", nouveau_tv_disable); DRM_DEBUG_DRIVER("... ignorelid : %d\n", nouveau_ignorelid); DRM_DEBUG_DRIVER("... duallink : %d\n", nouveau_duallink); - DRM_DEBUG_DRIVER("... nofbaccel : %d\n", nouveau_nofbaccel); DRM_DEBUG_DRIVER("... config : %s\n", nouveau_config); DRM_DEBUG_DRIVER("... debug : %s\n", nouveau_debug); DRM_DEBUG_DRIVER("... noaccel : %d\n", nouveau_noaccel); diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 2b9045b12ac70..6fb636020f7fc 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -183,7 +183,6 @@ struct nouveau_drm { struct nouveau_channel *cechan; struct nouveau_channel *channel; struct nvkm_gpuobj *notify; - struct nouveau_fbdev *fbcon; struct nvif_object ntfy; /* nv10-nv40 tiling regions */ @@ -198,8 +197,6 @@ struct nouveau_drm { struct work_struct hpd_work; spinlock_t hpd_lock; u32 hpd_pending; - struct work_struct fbcon_work; - int fbcon_new_state; #ifdef CONFIG_ACPI struct notifier_block acpi_nb; #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c deleted file mode 100644 index 3c7e0c9d6baf1..0000000000000 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c +++ /dev/null @@ -1,614 +0,0 @@ -/* - * Copyright © 2007 David Airlie - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - * - * Authors: - * David Airlie - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "nouveau_drv.h" -#include "nouveau_gem.h" -#include "nouveau_bo.h" -#include "nouveau_fbcon.h" -#include "nouveau_chan.h" -#include "nouveau_vmm.h" - -#include "nouveau_crtc.h" - -MODULE_PARM_DESC(nofbaccel, "Disable fbcon acceleration"); -int nouveau_nofbaccel = 0; -module_param_named(nofbaccel, nouveau_nofbaccel, int, 0400); - -MODULE_PARM_DESC(fbcon_bpp, "fbcon bits-per-pixel (default: auto)"); -static int nouveau_fbcon_bpp; -module_param_named(fbcon_bpp, nouveau_fbcon_bpp, int, 0400); - -static void -nouveau_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - struct nvif_device *device = &drm->client.device; - int ret; - - if (info->state != FBINFO_STATE_RUNNING) - return; - - ret = -ENODEV; - if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) && - mutex_trylock(&drm->client.mutex)) { - if (device->info.family < NV_DEVICE_INFO_V0_TESLA) - ret = nv04_fbcon_fillrect(info, rect); - else - if (device->info.family < NV_DEVICE_INFO_V0_FERMI) - ret = nv50_fbcon_fillrect(info, rect); - else - ret = nvc0_fbcon_fillrect(info, rect); - mutex_unlock(&drm->client.mutex); - } - - if (ret == 0) - return; - - if (ret != -ENODEV) - nouveau_fbcon_gpu_lockup(info); - drm_fb_helper_cfb_fillrect(info, rect); -} - -static void -nouveau_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *image) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - struct nvif_device *device = &drm->client.device; - int ret; - - if (info->state != FBINFO_STATE_RUNNING) - return; - - ret = -ENODEV; - if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) && - mutex_trylock(&drm->client.mutex)) { - if (device->info.family < NV_DEVICE_INFO_V0_TESLA) - ret = nv04_fbcon_copyarea(info, image); - else - if (device->info.family < NV_DEVICE_INFO_V0_FERMI) - ret = nv50_fbcon_copyarea(info, image); - else - ret = nvc0_fbcon_copyarea(info, image); - mutex_unlock(&drm->client.mutex); - } - - if (ret == 0) - return; - - if (ret != -ENODEV) - nouveau_fbcon_gpu_lockup(info); - drm_fb_helper_cfb_copyarea(info, image); -} - -static void -nouveau_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - struct nvif_device *device = &drm->client.device; - int ret; - - if (info->state != FBINFO_STATE_RUNNING) - return; - - ret = -ENODEV; - if (!in_interrupt() && !(info->flags & FBINFO_HWACCEL_DISABLED) && - mutex_trylock(&drm->client.mutex)) { - if (device->info.family < NV_DEVICE_INFO_V0_TESLA) - ret = nv04_fbcon_imageblit(info, image); - else - if (device->info.family < NV_DEVICE_INFO_V0_FERMI) - ret = nv50_fbcon_imageblit(info, image); - else - ret = nvc0_fbcon_imageblit(info, image); - mutex_unlock(&drm->client.mutex); - } - - if (ret == 0) - return; - - if (ret != -ENODEV) - nouveau_fbcon_gpu_lockup(info); - drm_fb_helper_cfb_imageblit(info, image); -} - -static int -nouveau_fbcon_sync(struct fb_info *info) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - struct nouveau_channel *chan = drm->channel; - int ret; - - if (!chan || !chan->accel_done || in_interrupt() || - info->state != FBINFO_STATE_RUNNING || - info->flags & FBINFO_HWACCEL_DISABLED) - return 0; - - if (!mutex_trylock(&drm->client.mutex)) - return 0; - - ret = nouveau_channel_idle(chan); - mutex_unlock(&drm->client.mutex); - if (ret) { - nouveau_fbcon_gpu_lockup(info); - return 0; - } - - chan->accel_done = false; - return 0; -} - -static int -nouveau_fbcon_open(struct fb_info *info, int user) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - int ret = pm_runtime_get_sync(drm->dev->dev); - if (ret < 0 && ret != -EACCES) { - pm_runtime_put(drm->dev->dev); - return ret; - } - return 0; -} - -static int -nouveau_fbcon_release(struct fb_info *info, int user) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - pm_runtime_put(drm->dev->dev); - return 0; -} - -static const struct fb_ops nouveau_fbcon_ops = { - .owner = THIS_MODULE, - DRM_FB_HELPER_DEFAULT_OPS, - .fb_open = nouveau_fbcon_open, - .fb_release = nouveau_fbcon_release, - .fb_fillrect = nouveau_fbcon_fillrect, - .fb_copyarea = nouveau_fbcon_copyarea, - .fb_imageblit = nouveau_fbcon_imageblit, - .fb_sync = nouveau_fbcon_sync, -}; - -static const struct fb_ops nouveau_fbcon_sw_ops = { - .owner = THIS_MODULE, - DRM_FB_HELPER_DEFAULT_OPS, - .fb_open = nouveau_fbcon_open, - .fb_release = nouveau_fbcon_release, - .fb_fillrect = drm_fb_helper_cfb_fillrect, - .fb_copyarea = drm_fb_helper_cfb_copyarea, - .fb_imageblit = drm_fb_helper_cfb_imageblit, -}; - -void -nouveau_fbcon_accel_save_disable(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - if (drm->fbcon && drm->fbcon->helper.fbdev) { - drm->fbcon->saved_flags = drm->fbcon->helper.fbdev->flags; - drm->fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED; - } -} - -void -nouveau_fbcon_accel_restore(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - if (drm->fbcon && drm->fbcon->helper.fbdev) { - drm->fbcon->helper.fbdev->flags = drm->fbcon->saved_flags; - } -} - -static void -nouveau_fbcon_accel_fini(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_fbdev *fbcon = drm->fbcon; - if (fbcon && drm->channel) { - console_lock(); - if (fbcon->helper.fbdev) - fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED; - console_unlock(); - nouveau_channel_idle(drm->channel); - nvif_object_dtor(&fbcon->twod); - nvif_object_dtor(&fbcon->blit); - nvif_object_dtor(&fbcon->gdi); - nvif_object_dtor(&fbcon->patt); - nvif_object_dtor(&fbcon->rop); - nvif_object_dtor(&fbcon->clip); - nvif_object_dtor(&fbcon->surf2d); - } -} - -static void -nouveau_fbcon_accel_init(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_fbdev *fbcon = drm->fbcon; - struct fb_info *info = fbcon->helper.fbdev; - int ret; - - if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) - ret = nv04_fbcon_accel_init(info); - else - if (drm->client.device.info.family < NV_DEVICE_INFO_V0_FERMI) - ret = nv50_fbcon_accel_init(info); - else - ret = nvc0_fbcon_accel_init(info); - - if (ret == 0) - info->fbops = &nouveau_fbcon_ops; -} - -static void -nouveau_fbcon_zfill(struct drm_device *dev, struct nouveau_fbdev *fbcon) -{ - struct fb_info *info = fbcon->helper.fbdev; - struct fb_fillrect rect; - - /* Clear the entire fbcon. The drm will program every connector - * with it's preferred mode. If the sizes differ, one display will - * quite likely have garbage around the console. - */ - rect.dx = rect.dy = 0; - rect.width = info->var.xres_virtual; - rect.height = info->var.yres_virtual; - rect.color = 0; - rect.rop = ROP_COPY; - info->fbops->fb_fillrect(info, &rect); -} - -static int -nouveau_fbcon_create(struct drm_fb_helper *helper, - struct drm_fb_helper_surface_size *sizes) -{ - struct nouveau_fbdev *fbcon = - container_of(helper, struct nouveau_fbdev, helper); - struct drm_device *dev = fbcon->helper.dev; - struct nouveau_drm *drm = nouveau_drm(dev); - struct nvif_device *device = &drm->client.device; - struct fb_info *info; - struct drm_framebuffer *fb; - struct nouveau_channel *chan; - struct nouveau_bo *nvbo; - struct drm_mode_fb_cmd2 mode_cmd = {}; - int ret; - - mode_cmd.width = sizes->surface_width; - mode_cmd.height = sizes->surface_height; - - mode_cmd.pitches[0] = mode_cmd.width * (sizes->surface_bpp >> 3); - mode_cmd.pitches[0] = roundup(mode_cmd.pitches[0], 256); - - mode_cmd.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp, - sizes->surface_depth); - - ret = nouveau_gem_new(&drm->client, mode_cmd.pitches[0] * - mode_cmd.height, 0, NOUVEAU_GEM_DOMAIN_VRAM, - 0, 0x0000, &nvbo); - if (ret) { - NV_ERROR(drm, "failed to allocate framebuffer\n"); - goto out; - } - - ret = nouveau_framebuffer_new(dev, &mode_cmd, &nvbo->bo.base, &fb); - if (ret) - goto out_unref; - - ret = nouveau_bo_pin(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, false); - if (ret) { - NV_ERROR(drm, "failed to pin fb: %d\n", ret); - goto out_unref; - } - - ret = nouveau_bo_map(nvbo); - if (ret) { - NV_ERROR(drm, "failed to map fb: %d\n", ret); - goto out_unpin; - } - - chan = nouveau_nofbaccel ? NULL : drm->channel; - if (chan && device->info.family >= NV_DEVICE_INFO_V0_TESLA) { - ret = nouveau_vma_new(nvbo, chan->vmm, &fbcon->vma); - if (ret) { - NV_ERROR(drm, "failed to map fb into chan: %d\n", ret); - chan = NULL; - } - } - - info = drm_fb_helper_alloc_fbi(helper); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto out_unlock; - } - - /* setup helper */ - fbcon->helper.fb = fb; - - if (!chan) - info->flags = FBINFO_HWACCEL_DISABLED; - else - info->flags = FBINFO_HWACCEL_COPYAREA | - FBINFO_HWACCEL_FILLRECT | - FBINFO_HWACCEL_IMAGEBLIT; - info->fbops = &nouveau_fbcon_sw_ops; - info->fix.smem_start = nvbo->bo.resource->bus.offset; - info->fix.smem_len = nvbo->bo.base.size; - - info->screen_base = nvbo_kmap_obj_iovirtual(nvbo); - info->screen_size = nvbo->bo.base.size; - - drm_fb_helper_fill_info(info, &fbcon->helper, sizes); - - /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */ - - if (chan) - nouveau_fbcon_accel_init(dev); - nouveau_fbcon_zfill(dev, fbcon); - - /* To allow resizeing without swapping buffers */ - NV_INFO(drm, "allocated %dx%d fb: 0x%llx, bo %p\n", - fb->width, fb->height, nvbo->offset, nvbo); - - if (dev_is_pci(dev->dev)) - vga_switcheroo_client_fb_set(to_pci_dev(dev->dev), info); - - return 0; - -out_unlock: - if (chan) - nouveau_vma_del(&fbcon->vma); - nouveau_bo_unmap(nvbo); -out_unpin: - nouveau_bo_unpin(nvbo); -out_unref: - nouveau_bo_ref(NULL, &nvbo); -out: - return ret; -} - -static int -nouveau_fbcon_destroy(struct drm_device *dev, struct nouveau_fbdev *fbcon) -{ - struct drm_framebuffer *fb = fbcon->helper.fb; - struct nouveau_bo *nvbo; - - drm_fb_helper_unregister_fbi(&fbcon->helper); - drm_fb_helper_fini(&fbcon->helper); - - if (fb && fb->obj[0]) { - nvbo = nouveau_gem_object(fb->obj[0]); - nouveau_vma_del(&fbcon->vma); - nouveau_bo_unmap(nvbo); - nouveau_bo_unpin(nvbo); - drm_framebuffer_put(fb); - } - - return 0; -} - -void nouveau_fbcon_gpu_lockup(struct fb_info *info) -{ - struct nouveau_fbdev *fbcon = info->par; - struct nouveau_drm *drm = nouveau_drm(fbcon->helper.dev); - - NV_ERROR(drm, "GPU lockup - switching to software fbcon\n"); - info->flags |= FBINFO_HWACCEL_DISABLED; -} - -static const struct drm_fb_helper_funcs nouveau_fbcon_helper_funcs = { - .fb_probe = nouveau_fbcon_create, -}; - -static void -nouveau_fbcon_set_suspend_work(struct work_struct *work) -{ - struct nouveau_drm *drm = container_of(work, typeof(*drm), fbcon_work); - int state = READ_ONCE(drm->fbcon_new_state); - - if (state == FBINFO_STATE_RUNNING) - pm_runtime_get_sync(drm->dev->dev); - - console_lock(); - if (state == FBINFO_STATE_RUNNING) - nouveau_fbcon_accel_restore(drm->dev); - drm_fb_helper_set_suspend(&drm->fbcon->helper, state); - if (state != FBINFO_STATE_RUNNING) - nouveau_fbcon_accel_save_disable(drm->dev); - console_unlock(); - - if (state == FBINFO_STATE_RUNNING) { - nouveau_fbcon_hotplug_resume(drm->fbcon); - pm_runtime_mark_last_busy(drm->dev->dev); - pm_runtime_put_autosuspend(drm->dev->dev); - } -} - -void -nouveau_fbcon_set_suspend(struct drm_device *dev, int state) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - - if (!drm->fbcon) - return; - - drm->fbcon_new_state = state; - /* Since runtime resume can happen as a result of a sysfs operation, - * it's possible we already have the console locked. So handle fbcon - * init/deinit from a seperate work thread - */ - schedule_work(&drm->fbcon_work); -} - -void -nouveau_fbcon_output_poll_changed(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_fbdev *fbcon = drm->fbcon; - int ret; - - if (!fbcon) - return; - - mutex_lock(&fbcon->hotplug_lock); - - ret = pm_runtime_get(dev->dev); - if (ret == 1 || ret == -EACCES) { - drm_fb_helper_hotplug_event(&fbcon->helper); - - pm_runtime_mark_last_busy(dev->dev); - pm_runtime_put_autosuspend(dev->dev); - } else if (ret == 0) { - /* If the GPU was already in the process of suspending before - * this event happened, then we can't block here as we'll - * deadlock the runtime pmops since they wait for us to - * finish. So, just defer this event for when we runtime - * resume again. It will be handled by fbcon_work. - */ - NV_DEBUG(drm, "fbcon HPD event deferred until runtime resume\n"); - fbcon->hotplug_waiting = true; - pm_runtime_put_noidle(drm->dev->dev); - } else { - DRM_WARN("fbcon HPD event lost due to RPM failure: %d\n", - ret); - } - - mutex_unlock(&fbcon->hotplug_lock); -} - -void -nouveau_fbcon_hotplug_resume(struct nouveau_fbdev *fbcon) -{ - struct nouveau_drm *drm; - - if (!fbcon) - return; - drm = nouveau_drm(fbcon->helper.dev); - - mutex_lock(&fbcon->hotplug_lock); - if (fbcon->hotplug_waiting) { - fbcon->hotplug_waiting = false; - - NV_DEBUG(drm, "Handling deferred fbcon HPD events\n"); - drm_fb_helper_hotplug_event(&fbcon->helper); - } - mutex_unlock(&fbcon->hotplug_lock); -} - -int -nouveau_fbcon_init(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_fbdev *fbcon; - int preferred_bpp = nouveau_fbcon_bpp; - int ret; - - if (!dev->mode_config.num_crtc || - (to_pci_dev(dev->dev)->class >> 8) != PCI_CLASS_DISPLAY_VGA) - return 0; - - fbcon = kzalloc(sizeof(struct nouveau_fbdev), GFP_KERNEL); - if (!fbcon) - return -ENOMEM; - - drm->fbcon = fbcon; - INIT_WORK(&drm->fbcon_work, nouveau_fbcon_set_suspend_work); - mutex_init(&fbcon->hotplug_lock); - - drm_fb_helper_prepare(dev, &fbcon->helper, &nouveau_fbcon_helper_funcs); - - ret = drm_fb_helper_init(dev, &fbcon->helper); - if (ret) - goto free; - - if (preferred_bpp != 8 && preferred_bpp != 16 && preferred_bpp != 32) { - if (drm->client.device.info.ram_size <= 32 * 1024 * 1024) - preferred_bpp = 8; - else - if (drm->client.device.info.ram_size <= 64 * 1024 * 1024) - preferred_bpp = 16; - else - preferred_bpp = 32; - } - - /* disable all the possible outputs/crtcs before entering KMS mode */ - if (!drm_drv_uses_atomic_modeset(dev)) - drm_helper_disable_unused_functions(dev); - - ret = drm_fb_helper_initial_config(&fbcon->helper, preferred_bpp); - if (ret) - goto fini; - - if (fbcon->helper.fbdev) - fbcon->helper.fbdev->pixmap.buf_align = 4; - return 0; - -fini: - drm_fb_helper_fini(&fbcon->helper); -free: - kfree(fbcon); - drm->fbcon = NULL; - return ret; -} - -void -nouveau_fbcon_fini(struct drm_device *dev) -{ - struct nouveau_drm *drm = nouveau_drm(dev); - - if (!drm->fbcon) - return; - - drm_kms_helper_poll_fini(dev); - nouveau_fbcon_accel_fini(dev); - nouveau_fbcon_destroy(dev, drm->fbcon); - kfree(drm->fbcon); - drm->fbcon = NULL; -} diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.h b/drivers/gpu/drm/nouveau/nouveau_fbcon.h deleted file mode 100644 index 1796d8824580b..0000000000000 --- a/drivers/gpu/drm/nouveau/nouveau_fbcon.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2008 Maarten Maathuis. - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining - * a copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial - * portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE - * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#ifndef __NOUVEAU_FBCON_H__ -#define __NOUVEAU_FBCON_H__ - -#include - -#include "nouveau_display.h" - -struct nouveau_vma; - -struct nouveau_fbdev { - struct drm_fb_helper helper; /* must be first */ - unsigned int saved_flags; - struct nvif_object surf2d; - struct nvif_object clip; - struct nvif_object rop; - struct nvif_object patt; - struct nvif_object gdi; - struct nvif_object blit; - struct nvif_object twod; - struct nouveau_vma *vma; - - struct mutex hotplug_lock; - bool hotplug_waiting; -}; - -void nouveau_fbcon_restore(void); - -int nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region); -int nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect); -int nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image); -int nv04_fbcon_accel_init(struct fb_info *info); - -int nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect); -int nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region); -int nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image); -int nv50_fbcon_accel_init(struct fb_info *info); - -int nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect); -int nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region); -int nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image); -int nvc0_fbcon_accel_init(struct fb_info *info); - -void nouveau_fbcon_gpu_lockup(struct fb_info *info); - -int nouveau_fbcon_init(struct drm_device *dev); -void nouveau_fbcon_fini(struct drm_device *dev); -void nouveau_fbcon_set_suspend(struct drm_device *dev, int state); -void nouveau_fbcon_accel_save_disable(struct drm_device *dev); -void nouveau_fbcon_accel_restore(struct drm_device *dev); - -void nouveau_fbcon_output_poll_changed(struct drm_device *dev); -void nouveau_fbcon_hotplug_resume(struct nouveau_fbdev *fbcon); -extern int nouveau_nofbaccel; - -#endif /* __NV50_FBCON_H__ */ - diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c index 60cd8c0463dff..789393b942911 100644 --- a/drivers/gpu/drm/nouveau/nouveau_vga.c +++ b/drivers/gpu/drm/nouveau/nouveau_vga.c @@ -7,7 +7,6 @@ #include "nouveau_drv.h" #include "nouveau_acpi.h" -#include "nouveau_fbcon.h" #include "nouveau_vga.h" static unsigned int diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c deleted file mode 100644 index c30b8dacd86b3..0000000000000 --- a/drivers/gpu/drm/nouveau/nv04_fbcon.c +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright 2009 Ben Skeggs - * Copyright 2008 Stuart Bennett - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice (including the next - * paragraph) shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#define NVIF_DEBUG_PRINT_DISABLE -#include "nouveau_drv.h" -#include "nouveau_dma.h" -#include "nouveau_fbcon.h" - -#include - -int -nv04_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - int ret; - - ret = PUSH_WAIT(push, 4); - if (ret) - return ret; - - PUSH_NVSQ(push, NV05F, 0x0300, (region->sy << 16) | region->sx, - 0x0304, (region->dy << 16) | region->dx, - 0x0308, (region->height << 16) | region->width); - PUSH_KICK(push); - return 0; -} - -int -nv04_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - int ret; - - ret = PUSH_WAIT(push, 7); - if (ret) - return ret; - - PUSH_NVSQ(push, NV04A, 0x02fc, (rect->rop != ROP_COPY) ? 1 : 3); - if (info->fix.visual == FB_VISUAL_TRUECOLOR || - info->fix.visual == FB_VISUAL_DIRECTCOLOR) - PUSH_NVSQ(push, NV04A, 0x03fc, ((uint32_t *)info->pseudo_palette)[rect->color]); - else - PUSH_NVSQ(push, NV04A, 0x03fc, rect->color); - PUSH_NVSQ(push, NV04A, 0x0400, (rect->dx << 16) | rect->dy, - 0x0404, (rect->width << 16) | rect->height); - PUSH_KICK(push); - return 0; -} - -int -nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - uint32_t fg; - uint32_t bg; - uint32_t dsize; - uint32_t *data = (uint32_t *)image->data; - int ret; - - if (image->depth != 1) - return -ENODEV; - - ret = PUSH_WAIT(push, 8); - if (ret) - return ret; - - if (info->fix.visual == FB_VISUAL_TRUECOLOR || - info->fix.visual == FB_VISUAL_DIRECTCOLOR) { - fg = ((uint32_t *) info->pseudo_palette)[image->fg_color]; - bg = ((uint32_t *) info->pseudo_palette)[image->bg_color]; - } else { - fg = image->fg_color; - bg = image->bg_color; - } - - PUSH_NVSQ(push, NV04A, 0x0be4, (image->dy << 16) | (image->dx & 0xffff), - 0x0be8, ((image->dy + image->height) << 16) | - ((image->dx + image->width) & 0xffff), - 0x0bec, bg, - 0x0bf0, fg, - 0x0bf4, (image->height << 16) | ALIGN(image->width, 8), - 0x0bf8, (image->height << 16) | image->width, - 0x0bfc, (image->dy << 16) | (image->dx & 0xffff)); - - dsize = ALIGN(ALIGN(image->width, 8) * image->height, 32) >> 5; - while (dsize) { - int iter_len = dsize > 128 ? 128 : dsize; - - ret = PUSH_WAIT(push, iter_len + 1); - if (ret) - return ret; - - PUSH_NVSQ(push, NV04A, 0x0c00, data, iter_len); - data += iter_len; - dsize -= iter_len; - } - - PUSH_KICK(push); - return 0; -} - -int -nv04_fbcon_accel_init(struct fb_info *info) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct drm_device *dev = nfbdev->helper.dev; - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_device *device = &drm->client.device; - struct nvif_push *push = chan->chan.push; - struct nvkm_device *nvkm_device = nvxx_device(&drm->client.device); - resource_size_t fb_base = nvkm_device->func->resource_addr(nvkm_device, 1); - int surface_fmt, pattern_fmt, rect_fmt; - int ret; - - switch (info->var.bits_per_pixel) { - case 8: - surface_fmt = 1; - pattern_fmt = 3; - rect_fmt = 3; - break; - case 16: - surface_fmt = 4; - pattern_fmt = 1; - rect_fmt = 1; - break; - case 32: - switch (info->var.transp.length) { - case 0: /* depth 24 */ - case 8: /* depth 32 */ - break; - default: - return -EINVAL; - } - - surface_fmt = 6; - pattern_fmt = 3; - rect_fmt = 3; - break; - default: - return -EINVAL; - } - - ret = nvif_object_ctor(&chan->user, "fbconCtxSurf2d", 0x0062, - device->info.family >= NV_DEVICE_INFO_V0_CELSIUS ? - 0x0062 : 0x0042, NULL, 0, &nfbdev->surf2d); - if (ret) - return ret; - - ret = nvif_object_ctor(&chan->user, "fbconCtxClip", 0x0019, 0x0019, - NULL, 0, &nfbdev->clip); - if (ret) - return ret; - - ret = nvif_object_ctor(&chan->user, "fbconCtxRop", 0x0043, 0x0043, - NULL, 0, &nfbdev->rop); - if (ret) - return ret; - - ret = nvif_object_ctor(&chan->user, "fbconCtxPatt", 0x0044, 0x0044, - NULL, 0, &nfbdev->patt); - if (ret) - return ret; - - ret = nvif_object_ctor(&chan->user, "fbconGdiRectText", 0x004a, 0x004a, - NULL, 0, &nfbdev->gdi); - if (ret) - return ret; - - ret = nvif_object_ctor(&chan->user, "fbconImageBlit", 0x005f, - device->info.chipset >= 0x11 ? 0x009f : 0x005f, - NULL, 0, &nfbdev->blit); - if (ret) - return ret; - - if (PUSH_WAIT(push, 49 + (device->info.chipset >= 0x11 ? 4 : 0))) { - nouveau_fbcon_gpu_lockup(info); - return 0; - } - - PUSH_NVSQ(push, NV042, 0x0000, nfbdev->surf2d.handle); - PUSH_NVSQ(push, NV042, 0x0184, chan->vram.handle, - 0x0188, chan->vram.handle); - PUSH_NVSQ(push, NV042, 0x0300, surface_fmt, - 0x0304, info->fix.line_length | (info->fix.line_length << 16), - 0x0308, info->fix.smem_start - fb_base, - 0x030c, info->fix.smem_start - fb_base); - - PUSH_NVSQ(push, NV043, 0x0000, nfbdev->rop.handle); - PUSH_NVSQ(push, NV043, 0x0300, 0x55); - - PUSH_NVSQ(push, NV044, 0x0000, nfbdev->patt.handle); - PUSH_NVSQ(push, NV044, 0x0300, pattern_fmt, -#ifdef __BIG_ENDIAN - 0x0304, 2, -#else - 0x0304, 1, -#endif - 0x0308, 0, - 0x030c, 1, - 0x0310, ~0, - 0x0314, ~0, - 0x0318, ~0, - 0x031c, ~0); - - PUSH_NVSQ(push, NV019, 0x0000, nfbdev->clip.handle); - PUSH_NVSQ(push, NV019, 0x0300, 0, - 0x0304, (info->var.yres_virtual << 16) | info->var.xres_virtual); - - PUSH_NVSQ(push, NV05F, 0x0000, nfbdev->blit.handle); - PUSH_NVSQ(push, NV05F, 0x019c, nfbdev->surf2d.handle); - PUSH_NVSQ(push, NV05F, 0x02fc, 3); - if (nfbdev->blit.oclass == 0x009f) { - PUSH_NVSQ(push, NV09F, 0x0120, 0, - 0x0124, 1, - 0x0128, 2); - } - - PUSH_NVSQ(push, NV04A, 0x0000, nfbdev->gdi.handle); - PUSH_NVSQ(push, NV04A, 0x0198, nfbdev->surf2d.handle); - PUSH_NVSQ(push, NV04A, 0x0188, nfbdev->patt.handle, - 0x018c, nfbdev->rop.handle); - PUSH_NVSQ(push, NV04A, 0x0304, 1); - PUSH_NVSQ(push, NV04A, 0x0300, rect_fmt); - PUSH_NVSQ(push, NV04A, 0x02fc, 3); - - PUSH_KICK(push); - return 0; -} - diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c deleted file mode 100644 index 71f92e4750f91..0000000000000 --- a/drivers/gpu/drm/nouveau/nv50_fbcon.c +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright 2010 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#define NVIF_DEBUG_PRINT_DISABLE -#include "nouveau_drv.h" -#include "nouveau_dma.h" -#include "nouveau_fbcon.h" -#include "nouveau_vmm.h" - -#include - -#include - -int -nv50_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - u32 colour; - int ret; - - if (info->fix.visual == FB_VISUAL_TRUECOLOR || - info->fix.visual == FB_VISUAL_DIRECTCOLOR) - colour = ((uint32_t *)info->pseudo_palette)[rect->color]; - else - colour = rect->color; - - ret = PUSH_WAIT(push, rect->rop == ROP_COPY ? 7 : 11); - if (ret) - return ret; - - if (rect->rop != ROP_COPY) { - PUSH_MTHD(push, NV502D, SET_OPERATION, - NVDEF(NV502D, SET_OPERATION, V, ROP_AND)); - } - - PUSH_MTHD(push, NV502D, SET_RENDER_SOLID_PRIM_COLOR, colour); - - PUSH_MTHD(push, NV502D, RENDER_SOLID_PRIM_POINT_SET_X(0), rect->dx, - RENDER_SOLID_PRIM_POINT_Y(0), rect->dy, - RENDER_SOLID_PRIM_POINT_SET_X(1), rect->dx + rect->width, - RENDER_SOLID_PRIM_POINT_Y(1), rect->dy + rect->height); - - if (rect->rop != ROP_COPY) { - PUSH_MTHD(push, NV502D, SET_OPERATION, - NVDEF(NV502D, SET_OPERATION, V, SRCCOPY)); - } - - PUSH_KICK(push); - return 0; -} - -int -nv50_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - int ret; - - ret = PUSH_WAIT(push, 12); - if (ret) - return ret; - - PUSH_MTHD(push, NV502D, WAIT_FOR_IDLE, 0); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_MEMORY_DST_X0, region->dx, - SET_PIXELS_FROM_MEMORY_DST_Y0, region->dy, - SET_PIXELS_FROM_MEMORY_DST_WIDTH, region->width, - SET_PIXELS_FROM_MEMORY_DST_HEIGHT, region->height); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_MEMORY_SRC_X0_FRAC, 0, - SET_PIXELS_FROM_MEMORY_SRC_X0_INT, region->sx, - SET_PIXELS_FROM_MEMORY_SRC_Y0_FRAC, 0, - PIXELS_FROM_MEMORY_SRC_Y0_INT, region->sy); - PUSH_KICK(push); - return 0; -} - -int -nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - uint32_t dwords, *data = (uint32_t *)image->data; - uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel)); - uint32_t *palette = info->pseudo_palette, bg, fg; - int ret; - - if (image->depth != 1) - return -ENODEV; - - if (info->fix.visual == FB_VISUAL_TRUECOLOR || - info->fix.visual == FB_VISUAL_DIRECTCOLOR) { - bg = palette[image->bg_color] | mask; - fg = palette[image->fg_color] | mask; - } else { - bg = image->bg_color; - fg = image->fg_color; - } - - ret = PUSH_WAIT(push, 11); - if (ret) - return ret; - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_CPU_COLOR0, bg, - SET_PIXELS_FROM_CPU_COLOR1, fg); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_CPU_SRC_WIDTH, image->width, - SET_PIXELS_FROM_CPU_SRC_HEIGHT, image->height); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_CPU_DST_X0_FRAC, 0, - SET_PIXELS_FROM_CPU_DST_X0_INT, image->dx, - SET_PIXELS_FROM_CPU_DST_Y0_FRAC, 0, - SET_PIXELS_FROM_CPU_DST_Y0_INT, image->dy); - - dwords = ALIGN(ALIGN(image->width, 8) * image->height, 32) >> 5; - while (dwords) { - int count = dwords > 2047 ? 2047 : dwords; - - ret = PUSH_WAIT(push, count + 1); - if (ret) - return ret; - - dwords -= count; - - PUSH_NINC(push, NV502D, PIXELS_FROM_CPU_DATA, data, count); - data += count; - } - - PUSH_KICK(push); - return 0; -} - -int -nv50_fbcon_accel_init(struct fb_info *info) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct drm_device *dev = nfbdev->helper.dev; - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - int ret, format; - - switch (info->var.bits_per_pixel) { - case 8: - format = NV502D_SET_DST_FORMAT_V_Y8; - break; - case 15: - format = NV502D_SET_DST_FORMAT_V_X1R5G5B5; - break; - case 16: - format = NV502D_SET_DST_FORMAT_V_R5G6B5; - break; - case 32: - switch (info->var.transp.length) { - case 0: /* depth 24 */ - case 8: /* depth 32, just use 24.. */ - format = NV502D_SET_DST_FORMAT_V_X8R8G8B8; - break; - case 2: /* depth 30 */ - format = NV502D_SET_DST_FORMAT_V_A2B10G10R10; - break; - default: - return -EINVAL; - } - break; - default: - return -EINVAL; - } - - ret = nvif_object_ctor(&chan->user, "fbconTwoD", 0x502d, 0x502d, - NULL, 0, &nfbdev->twod); - if (ret) - return ret; - - ret = PUSH_WAIT(push, 56); - if (ret) { - nouveau_fbcon_gpu_lockup(info); - return ret; - } - - PUSH_MTHD(push, NV502D, SET_OBJECT, nfbdev->twod.handle); - PUSH_MTHD(push, NV502D, SET_DST_CONTEXT_DMA, chan->vram.handle, - SET_SRC_CONTEXT_DMA, chan->vram.handle, - SET_SEMAPHORE_CONTEXT_DMA, chan->vram.handle); - - PUSH_MTHD(push, NV502D, SET_DST_FORMAT, - NVVAL(NV502D, SET_DST_FORMAT, V, format), - - SET_DST_MEMORY_LAYOUT, - NVDEF(NV502D, SET_DST_MEMORY_LAYOUT, V, PITCH)); - - PUSH_MTHD(push, NV502D, SET_DST_PITCH, info->fix.line_length, - SET_DST_WIDTH, info->var.xres_virtual, - SET_DST_HEIGHT, info->var.yres_virtual, - - SET_DST_OFFSET_UPPER, - NVVAL(NV502D, SET_DST_OFFSET_UPPER, V, upper_32_bits(nfbdev->vma->addr)), - - SET_DST_OFFSET_LOWER, - NVVAL(NV502D, SET_DST_OFFSET_LOWER, V, lower_32_bits(nfbdev->vma->addr))); - - PUSH_MTHD(push, NV502D, SET_SRC_FORMAT, - NVVAL(NV502D, SET_SRC_FORMAT, V, format), - - SET_SRC_MEMORY_LAYOUT, - NVDEF(NV502D, SET_SRC_MEMORY_LAYOUT, V, PITCH)); - - PUSH_MTHD(push, NV502D, SET_SRC_PITCH, info->fix.line_length, - SET_SRC_WIDTH, info->var.xres_virtual, - SET_SRC_HEIGHT, info->var.yres_virtual, - - SET_SRC_OFFSET_UPPER, - NVVAL(NV502D, SET_SRC_OFFSET_UPPER, V, upper_32_bits(nfbdev->vma->addr)), - - SET_SRC_OFFSET_LOWER, - NVVAL(NV502D, SET_SRC_OFFSET_LOWER, V, lower_32_bits(nfbdev->vma->addr))); - - PUSH_MTHD(push, NV502D, SET_CLIP_ENABLE, - NVDEF(NV502D, SET_CLIP_ENABLE, V, FALSE)); - - PUSH_MTHD(push, NV502D, SET_ROP, - NVVAL(NV502D, SET_ROP, V, 0x55)); - - PUSH_MTHD(push, NV502D, SET_OPERATION, - NVDEF(NV502D, SET_OPERATION, V, SRCCOPY)); - - PUSH_MTHD(push, NV502D, SET_MONOCHROME_PATTERN_COLOR_FORMAT, - NVDEF(NV502D, SET_MONOCHROME_PATTERN_COLOR_FORMAT, V, A8R8G8B8), - - SET_MONOCHROME_PATTERN_FORMAT, - NVDEF(NV502D, SET_MONOCHROME_PATTERN_FORMAT, V, LE_M1)); - - PUSH_MTHD(push, NV502D, RENDER_SOLID_PRIM_MODE, - NVDEF(NV502D, RENDER_SOLID_PRIM_MODE, V, RECTS), - - SET_RENDER_SOLID_PRIM_COLOR_FORMAT, - NVVAL(NV502D, SET_RENDER_SOLID_PRIM_COLOR_FORMAT, V, format)); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_CPU_DATA_TYPE, - NVDEF(NV502D, SET_PIXELS_FROM_CPU_DATA_TYPE, V, INDEX), - - SET_PIXELS_FROM_CPU_COLOR_FORMAT, - NVVAL(NV502D, SET_PIXELS_FROM_CPU_COLOR_FORMAT, V, format), - - SET_PIXELS_FROM_CPU_INDEX_FORMAT, - NVDEF(NV502D, SET_PIXELS_FROM_CPU_INDEX_FORMAT, V, I1), - - SET_PIXELS_FROM_CPU_MONO_FORMAT, - NVDEF(NV502D, SET_PIXELS_FROM_CPU_MONO_FORMAT, V, CGA6_M1), - - SET_PIXELS_FROM_CPU_WRAP, - NVDEF(NV502D, SET_PIXELS_FROM_CPU_WRAP, V, WRAP_BYTE)); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_CPU_MONO_OPACITY, - NVDEF(NV502D, SET_PIXELS_FROM_CPU_MONO_OPACITY, V, OPAQUE)); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_CPU_DX_DU_FRAC, 0, - SET_PIXELS_FROM_CPU_DX_DU_INT, 1, - SET_PIXELS_FROM_CPU_DY_DV_FRAC, 0, - SET_PIXELS_FROM_CPU_DY_DV_INT, 1); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_MEMORY_SAFE_OVERLAP, - NVDEF(NV502D, SET_PIXELS_FROM_MEMORY_SAFE_OVERLAP, V, TRUE)); - - PUSH_MTHD(push, NV502D, SET_PIXELS_FROM_MEMORY_DU_DX_FRAC, 0, - SET_PIXELS_FROM_MEMORY_DU_DX_INT, 1, - SET_PIXELS_FROM_MEMORY_DV_DY_FRAC, 0, - SET_PIXELS_FROM_MEMORY_DV_DY_INT, 1); - PUSH_KICK(push); - return 0; -} - diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c deleted file mode 100644 index 7908a1a3e00ff..0000000000000 --- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Copyright 2010 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#define NVIF_DEBUG_PRINT_DISABLE -#include "nouveau_drv.h" -#include "nouveau_dma.h" -#include "nouveau_fbcon.h" -#include "nouveau_vmm.h" - -#include - -#include - -int -nvc0_fbcon_fillrect(struct fb_info *info, const struct fb_fillrect *rect) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - u32 colour; - int ret; - - if (info->fix.visual == FB_VISUAL_TRUECOLOR || - info->fix.visual == FB_VISUAL_DIRECTCOLOR) - colour = ((uint32_t *)info->pseudo_palette)[rect->color]; - else - colour = rect->color; - - ret = PUSH_WAIT(push, rect->rop == ROP_COPY ? 7 : 9); - if (ret) - return ret; - - if (rect->rop != ROP_COPY) { - PUSH_IMMD(push, NV902D, SET_OPERATION, - NVDEF(NV902D, SET_OPERATION, V, ROP_AND)); - } - - PUSH_MTHD(push, NV902D, SET_RENDER_SOLID_PRIM_COLOR, colour); - - PUSH_MTHD(push, NV902D, RENDER_SOLID_PRIM_POINT_SET_X(0), rect->dx, - RENDER_SOLID_PRIM_POINT_Y(0), rect->dy, - RENDER_SOLID_PRIM_POINT_SET_X(1), rect->dx + rect->width, - RENDER_SOLID_PRIM_POINT_Y(1), rect->dy + rect->height); - - if (rect->rop != ROP_COPY) { - PUSH_IMMD(push, NV902D, SET_OPERATION, - NVDEF(NV902D, SET_OPERATION, V, SRCCOPY)); - } - - PUSH_KICK(push); - return 0; -} - -int -nvc0_fbcon_copyarea(struct fb_info *info, const struct fb_copyarea *region) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - int ret; - - ret = PUSH_WAIT(push, 11); - if (ret) - return ret; - - PUSH_IMMD(push, NV902D, WAIT_FOR_IDLE, 0); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_MEMORY_DST_X0, region->dx, - SET_PIXELS_FROM_MEMORY_DST_Y0, region->dy, - SET_PIXELS_FROM_MEMORY_DST_WIDTH, region->width, - SET_PIXELS_FROM_MEMORY_DST_HEIGHT, region->height); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_MEMORY_SRC_X0_FRAC, 0, - SET_PIXELS_FROM_MEMORY_SRC_X0_INT, region->sx, - SET_PIXELS_FROM_MEMORY_SRC_Y0_FRAC, 0, - PIXELS_FROM_MEMORY_SRC_Y0_INT, region->sy); - PUSH_KICK(push); - return 0; -} - -int -nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct nouveau_drm *drm = nouveau_drm(nfbdev->helper.dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - uint32_t dwords, *data = (uint32_t *)image->data; - uint32_t mask = ~(~0 >> (32 - info->var.bits_per_pixel)); - uint32_t *palette = info->pseudo_palette, bg, fg; - int ret; - - if (image->depth != 1) - return -ENODEV; - - if (info->fix.visual == FB_VISUAL_TRUECOLOR || - info->fix.visual == FB_VISUAL_DIRECTCOLOR) { - bg = palette[image->bg_color] | mask; - fg = palette[image->fg_color] | mask; - } else { - bg = image->bg_color; - fg = image->fg_color; - } - - ret = PUSH_WAIT(push, 11); - if (ret) - return ret; - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_CPU_COLOR0, bg, - SET_PIXELS_FROM_CPU_COLOR1, fg); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_CPU_SRC_WIDTH, image->width, - SET_PIXELS_FROM_CPU_SRC_HEIGHT, image->height); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_CPU_DST_X0_FRAC, 0, - SET_PIXELS_FROM_CPU_DST_X0_INT, image->dx, - SET_PIXELS_FROM_CPU_DST_Y0_FRAC, 0, - SET_PIXELS_FROM_CPU_DST_Y0_INT, image->dy); - - dwords = ALIGN(ALIGN(image->width, 8) * image->height, 32) >> 5; - while (dwords) { - int count = dwords > 2047 ? 2047 : dwords; - - ret = PUSH_WAIT(push, count + 1); - if (ret) - return ret; - - dwords -= count; - - PUSH_NINC(push, NV902D, PIXELS_FROM_CPU_DATA, data, count); - data += count; - } - - PUSH_KICK(push); - return 0; -} - -int -nvc0_fbcon_accel_init(struct fb_info *info) -{ - struct nouveau_fbdev *nfbdev = info->par; - struct drm_device *dev = nfbdev->helper.dev; - struct nouveau_drm *drm = nouveau_drm(dev); - struct nouveau_channel *chan = drm->channel; - struct nvif_push *push = chan->chan.push; - int ret, format; - - ret = nvif_object_ctor(&chan->user, "fbconTwoD", 0x902d, 0x902d, - NULL, 0, &nfbdev->twod); - if (ret) - return ret; - - switch (info->var.bits_per_pixel) { - case 8: - format = NV902D_SET_DST_FORMAT_V_Y8; - break; - case 15: - format = NV902D_SET_DST_FORMAT_V_X1R5G5B5; - break; - case 16: - format = NV902D_SET_DST_FORMAT_V_R5G6B5; - break; - case 32: - switch (info->var.transp.length) { - case 0: /* depth 24 */ - case 8: /* depth 32, just use 24.. */ - format = NV902D_SET_DST_FORMAT_V_X8R8G8B8; - break; - case 2: /* depth 30 */ - format = NV902D_SET_DST_FORMAT_V_A2B10G10R10; - break; - default: - return -EINVAL; - } - break; - default: - return -EINVAL; - } - - ret = PUSH_WAIT(push, 52); - if (ret) { - WARN_ON(1); - nouveau_fbcon_gpu_lockup(info); - return ret; - } - - PUSH_MTHD(push, NV902D, SET_OBJECT, nfbdev->twod.handle); - - PUSH_MTHD(push, NV902D, SET_DST_FORMAT, - NVVAL(NV902D, SET_DST_FORMAT, V, format), - - SET_DST_MEMORY_LAYOUT, - NVDEF(NV902D, SET_DST_MEMORY_LAYOUT, V, PITCH)); - - PUSH_MTHD(push, NV902D, SET_DST_PITCH, info->fix.line_length, - SET_DST_WIDTH, info->var.xres_virtual, - SET_DST_HEIGHT, info->var.yres_virtual, - - SET_DST_OFFSET_UPPER, - NVVAL(NV902D, SET_DST_OFFSET_UPPER, V, upper_32_bits(nfbdev->vma->addr)), - - SET_DST_OFFSET_LOWER, - NVVAL(NV902D, SET_DST_OFFSET_LOWER, V, lower_32_bits(nfbdev->vma->addr))); - - PUSH_MTHD(push, NV902D, SET_SRC_FORMAT, - NVVAL(NV902D, SET_SRC_FORMAT, V, format), - - SET_SRC_MEMORY_LAYOUT, - NVDEF(NV902D, SET_SRC_MEMORY_LAYOUT, V, PITCH)); - - PUSH_MTHD(push, NV902D, SET_SRC_PITCH, info->fix.line_length, - SET_SRC_WIDTH, info->var.xres_virtual, - SET_SRC_HEIGHT, info->var.yres_virtual, - - SET_SRC_OFFSET_UPPER, - NVVAL(NV902D, SET_SRC_OFFSET_UPPER, V, upper_32_bits(nfbdev->vma->addr)), - - SET_SRC_OFFSET_LOWER, - NVVAL(NV902D, SET_SRC_OFFSET_LOWER, V, lower_32_bits(nfbdev->vma->addr))); - - PUSH_IMMD(push, NV902D, SET_CLIP_ENABLE, - NVDEF(NV902D, SET_CLIP_ENABLE, V, FALSE)); - - PUSH_IMMD(push, NV902D, SET_ROP, - NVVAL(NV902D, SET_ROP, V, 0x55)); - - PUSH_IMMD(push, NV902D, SET_OPERATION, - NVDEF(NV902D, SET_OPERATION, V, SRCCOPY)); - - PUSH_MTHD(push, NV902D, SET_MONOCHROME_PATTERN_COLOR_FORMAT, - NVDEF(NV902D, SET_MONOCHROME_PATTERN_COLOR_FORMAT, V, A8R8G8B8), - - SET_MONOCHROME_PATTERN_FORMAT, - NVDEF(NV902D, SET_MONOCHROME_PATTERN_FORMAT, V, LE_M1)); - - PUSH_MTHD(push, NV902D, RENDER_SOLID_PRIM_MODE, - NVDEF(NV902D, RENDER_SOLID_PRIM_MODE, V, RECTS), - - SET_RENDER_SOLID_PRIM_COLOR_FORMAT, - NVVAL(NV902D, SET_RENDER_SOLID_PRIM_COLOR_FORMAT, V, format)); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_CPU_DATA_TYPE, - NVDEF(NV902D, SET_PIXELS_FROM_CPU_DATA_TYPE, V, INDEX), - - SET_PIXELS_FROM_CPU_COLOR_FORMAT, - NVVAL(NV902D, SET_PIXELS_FROM_CPU_COLOR_FORMAT, V, format), - - SET_PIXELS_FROM_CPU_INDEX_FORMAT, - NVDEF(NV902D, SET_PIXELS_FROM_CPU_INDEX_FORMAT, V, I1), - - SET_PIXELS_FROM_CPU_MONO_FORMAT, - NVDEF(NV902D, SET_PIXELS_FROM_CPU_MONO_FORMAT, V, CGA6_M1), - - SET_PIXELS_FROM_CPU_WRAP, - NVDEF(NV902D, SET_PIXELS_FROM_CPU_WRAP, V, WRAP_BYTE)); - - PUSH_IMMD(push, NV902D, SET_PIXELS_FROM_CPU_MONO_OPACITY, - NVDEF(NV902D, SET_PIXELS_FROM_CPU_MONO_OPACITY, V, OPAQUE)); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_CPU_DX_DU_FRAC, 0, - SET_PIXELS_FROM_CPU_DX_DU_INT, 1, - SET_PIXELS_FROM_CPU_DY_DV_FRAC, 0, - SET_PIXELS_FROM_CPU_DY_DV_INT, 1); - - PUSH_IMMD(push, NV902D, SET_PIXELS_FROM_MEMORY_SAFE_OVERLAP, - NVDEF(NV902D, SET_PIXELS_FROM_MEMORY_SAFE_OVERLAP, V, TRUE)); - - PUSH_MTHD(push, NV902D, SET_PIXELS_FROM_MEMORY_DU_DX_FRAC, 0, - SET_PIXELS_FROM_MEMORY_DU_DX_INT, 1, - SET_PIXELS_FROM_MEMORY_DV_DY_FRAC, 0, - SET_PIXELS_FROM_MEMORY_DV_DY_INT, 1); - PUSH_KICK(push); - return 0; -} - -- GitLab From ab4f75eb1cc261263ac51814fed14fde08b6716f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:51 +1000 Subject: [PATCH 1285/1927] drm/nouveau/nvkm: give each nvkm_event its own lockdep class The vblank and nonstall events have some annoying interactions with DRM locking, and aren't able to do certain things as a result. However, other uses of event notifications don't have such requirements, and upcoming patches take advantage of this for various improvements. Having separate classes for each nvkm_event's spinlocks allows lockdep to distinguish between them and avoid false-positives. v2: __always_inline + comment Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/core/event.h | 19 +++++++++++++++++-- drivers/gpu/drm/nouveau/nvkm/core/event.c | 6 ++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/event.h b/drivers/gpu/drm/nouveau/include/nvkm/core/event.h index 8e8fb4b6bca4f..82b267c111470 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/event.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/event.h @@ -25,8 +25,23 @@ struct nvkm_event_func { void (*fini)(struct nvkm_event *, int type, int index); }; -int nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *, int types_nr, - int index_nr, struct nvkm_event *); +int __nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *, int types_nr, + int index_nr, struct nvkm_event *); + +/* Each nvkm_event needs its own lockdep class due to inter-dependencies, to + * prevent lockdep false-positives. + * + * Inlining the spinlock initialisation ensures each is unique. + */ +static __always_inline int +nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, + int types_nr, int index_nr, struct nvkm_event *event) +{ + spin_lock_init(&event->refs_lock); + spin_lock_init(&event->list_lock); + return __nvkm_event_init(func, subdev, types_nr, index_nr, event); +} + void nvkm_event_fini(struct nvkm_event *); #define NVKM_EVENT_KEEP 0 diff --git a/drivers/gpu/drm/nouveau/nvkm/core/event.c b/drivers/gpu/drm/nouveau/nvkm/core/event.c index 622df36391be9..a6c877135598f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/event.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/event.c @@ -198,8 +198,8 @@ nvkm_event_fini(struct nvkm_event *event) } int -nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, - int types_nr, int index_nr, struct nvkm_event *event) +__nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, + int types_nr, int index_nr, struct nvkm_event *event) { event->refs = kzalloc(array3_size(index_nr, types_nr, sizeof(*event->refs)), GFP_KERNEL); if (!event->refs) @@ -209,8 +209,6 @@ nvkm_event_init(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, event->subdev = subdev; event->types_nr = types_nr; event->index_nr = index_nr; - spin_lock_init(&event->refs_lock); - spin_lock_init(&event->list_lock); INIT_LIST_HEAD(&event->ntfy); return 0; } -- GitLab From eec3f6dfedc0b8c5aef7619667dee61a77a37e35 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:51 +1000 Subject: [PATCH 1286/1927] drm/nouveau/top: parse device topology right after devinit We're going to want this information available earlier than it is now. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/device/base.c | 4 ++++ drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c | 13 ++++++++----- drivers/gpu/drm/nouveau/nvkm/subdev/top/ga100.c | 4 ++-- drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c | 4 ++-- drivers/gpu/drm/nouveau/nvkm/subdev/top/priv.h | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h index ee75c5524c436..73e717b980b85 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h @@ -21,6 +21,7 @@ struct nvkm_top_device { struct list_head head; }; +int nvkm_top_parse(struct nvkm_device *); u32 nvkm_top_addr(struct nvkm_device *, enum nvkm_subdev_type, int); u32 nvkm_top_reset(struct nvkm_device *, enum nvkm_subdev_type, int); u32 nvkm_top_intr_mask(struct nvkm_device *, enum nvkm_subdev_type, int); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index d8cf71fb05128..c45ff9df07273 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2775,6 +2775,10 @@ nvkm_device_preinit(struct nvkm_device *device) if (ret) goto fail; + ret = nvkm_top_parse(device); + if (ret) + goto fail; + time = ktime_to_us(ktime_get()) - time; nvdev_trace(device, "preinit completed in %lldus\n", time); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c index 28d0789f50fec..eb348dfc1d7a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c @@ -117,11 +117,15 @@ nvkm_top_fault(struct nvkm_device *device, int fault) return NULL; } -static int -nvkm_top_oneinit(struct nvkm_subdev *subdev) +int +nvkm_top_parse(struct nvkm_device *device) { - struct nvkm_top *top = nvkm_top(subdev); - return top->func->oneinit(top); + struct nvkm_top *top = device->top; + + if (!top || !list_empty(&top->device)) + return 0; + + return top->func->parse(top); } static void * @@ -141,7 +145,6 @@ nvkm_top_dtor(struct nvkm_subdev *subdev) static const struct nvkm_subdev_func nvkm_top = { .dtor = nvkm_top_dtor, - .oneinit = nvkm_top_oneinit, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/ga100.c index c982d834c8d98..84790cf52b902 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/ga100.c @@ -22,7 +22,7 @@ #include "priv.h" static int -ga100_top_oneinit(struct nvkm_top *top) +ga100_top_parse(struct nvkm_top *top) { struct nvkm_subdev *subdev = &top->subdev; struct nvkm_device *device = subdev->device; @@ -97,7 +97,7 @@ ga100_top_oneinit(struct nvkm_top *top) static const struct nvkm_top_func ga100_top = { - .oneinit = ga100_top_oneinit, + .parse = ga100_top_parse, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c index 4dcad97bd505b..2bbba8244cbf1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c @@ -24,7 +24,7 @@ #include "priv.h" static int -gk104_top_oneinit(struct nvkm_top *top) +gk104_top_parse(struct nvkm_top *top) { struct nvkm_subdev *subdev = &top->subdev; struct nvkm_device *device = subdev->device; @@ -108,7 +108,7 @@ gk104_top_oneinit(struct nvkm_top *top) static const struct nvkm_top_func gk104_top = { - .oneinit = gk104_top_oneinit, + .parse = gk104_top_parse, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/top/priv.h index 8e103a8367059..532be91d8fd90 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/priv.h @@ -5,7 +5,7 @@ #include struct nvkm_top_func { - int (*oneinit)(struct nvkm_top *); + int (*parse)(struct nvkm_top *); }; int nvkm_top_new_(const struct nvkm_top_func *, struct nvkm_device *, enum nvkm_subdev_type, int, -- GitLab From 727fd72f2402afe7cc320844b0aef165f7eb544e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:52 +1000 Subject: [PATCH 1287/1927] drm/nouveau/intr: add shared interrupt plumbing between pci/tegra Unifies the handling between PCI-based and Tegra GPUs, and makes more explicit/obvious where device interrupts can be expected. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/device.h | 9 ++ .../gpu/drm/nouveau/include/nvkm/core/intr.h | 12 ++ .../gpu/drm/nouveau/include/nvkm/core/tegra.h | 1 - .../gpu/drm/nouveau/include/nvkm/subdev/pci.h | 2 +- drivers/gpu/drm/nouveau/nvkm/core/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/core/intr.c | 109 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/device/base.c | 11 +- .../gpu/drm/nouveau/nvkm/engine/device/pci.c | 7 ++ .../drm/nouveau/nvkm/engine/device/tegra.c | 40 +------ .../gpu/drm/nouveau/nvkm/subdev/pci/base.c | 44 ++----- 10 files changed, 161 insertions(+), 75 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/include/nvkm/core/intr.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/core/intr.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h index efede1f11e1d9..6752a4c317e2a 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h @@ -2,6 +2,7 @@ #ifndef __NVKM_DEVICE_H__ #define __NVKM_DEVICE_H__ #include +#include enum nvkm_subdev_type; enum nvkm_device_type { @@ -60,6 +61,13 @@ struct nvkm_device { #undef NVKM_LAYOUT_INST #undef NVKM_LAYOUT_ONCE struct list_head subdev; + + struct { + spinlock_t lock; + int irq; + bool alloc; + bool armed; + } intr; }; struct nvkm_subdev *nvkm_device_subdev(struct nvkm_device *, int type, int inst); @@ -72,6 +80,7 @@ struct nvkm_device_func { int (*preinit)(struct nvkm_device *); int (*init)(struct nvkm_device *); void (*fini)(struct nvkm_device *, bool suspend); + int (*irq)(struct nvkm_device *); resource_size_t (*resource_addr)(struct nvkm_device *, unsigned bar); resource_size_t (*resource_size)(struct nvkm_device *, unsigned bar); bool cpu_coherent; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h b/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h new file mode 100644 index 0000000000000..b4d49ff40c6ad --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_INTR_H__ +#define __NVKM_INTR_H__ +#include +struct nvkm_device; + +void nvkm_intr_ctor(struct nvkm_device *); +void nvkm_intr_dtor(struct nvkm_device *); +int nvkm_intr_install(struct nvkm_device *); +void nvkm_intr_unarm(struct nvkm_device *); +void nvkm_intr_rearm(struct nvkm_device *); +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h b/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h index 924009dd2bb07..ccee53d4e4ec6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h @@ -8,7 +8,6 @@ struct nvkm_device_tegra { const struct nvkm_device_tegra_func *func; struct nvkm_device device; struct platform_device *pdev; - int irq; struct reset_control *rst; struct clk *clk; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h index 74c19bdfb757c..3c103101d5fcc 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h @@ -13,7 +13,6 @@ struct nvkm_pci { const struct nvkm_pci_func *func; struct nvkm_subdev subdev; struct pci_dev *pdev; - int irq; struct { struct agp_bridge_data *bridge; @@ -38,6 +37,7 @@ void nvkm_pci_wr08(struct nvkm_pci *, u16 addr, u8 data); void nvkm_pci_wr32(struct nvkm_pci *, u16 addr, u32 data); u32 nvkm_pci_mask(struct nvkm_pci *, u16 addr, u32 mask, u32 value); void nvkm_pci_rom_shadow(struct nvkm_pci *, bool shadow); +void nvkm_pci_msi_rearm(struct nvkm_device *); int nv04_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **); int nv40_pci_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_pci **); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/Kbuild b/drivers/gpu/drm/nouveau/nvkm/core/Kbuild index 65dad05f047c7..e40712023c73a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/core/Kbuild @@ -5,6 +5,7 @@ nvkm-y += nvkm/core/enum.o nvkm-y += nvkm/core/event.o nvkm-y += nvkm/core/firmware.o nvkm-y += nvkm/core/gpuobj.o +nvkm-y += nvkm/core/intr.o nvkm-y += nvkm/core/ioctl.o nvkm-y += nvkm/core/memory.o nvkm-y += nvkm/core/mm.o diff --git a/drivers/gpu/drm/nouveau/nvkm/core/intr.c b/drivers/gpu/drm/nouveau/nvkm/core/intr.c new file mode 100644 index 0000000000000..89f6015677497 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/core/intr.c @@ -0,0 +1,109 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include + +#include +#include + +static void +nvkm_intr_rearm_locked(struct nvkm_device *device) +{ + nvkm_mc_intr_rearm(device); +} + +static void +nvkm_intr_unarm_locked(struct nvkm_device *device) +{ + nvkm_mc_intr_unarm(device); +} + +static irqreturn_t +nvkm_intr(int irq, void *arg) +{ + struct nvkm_device *device = arg; + irqreturn_t ret = IRQ_NONE; + bool handled; + + spin_lock(&device->intr.lock); + if (!device->intr.armed) + goto done_unlock; + + nvkm_intr_unarm_locked(device); + nvkm_pci_msi_rearm(device); + + nvkm_mc_intr(device, &handled); + if (handled) + ret = IRQ_HANDLED; + + nvkm_intr_rearm_locked(device); +done_unlock: + spin_unlock(&device->intr.lock); + return ret; +} + +void +nvkm_intr_rearm(struct nvkm_device *device) +{ + spin_lock_irq(&device->intr.lock); + nvkm_intr_rearm_locked(device); + device->intr.armed = true; + spin_unlock_irq(&device->intr.lock); +} + +void +nvkm_intr_unarm(struct nvkm_device *device) +{ + spin_lock_irq(&device->intr.lock); + nvkm_intr_unarm_locked(device); + device->intr.armed = false; + spin_unlock_irq(&device->intr.lock); +} + +int +nvkm_intr_install(struct nvkm_device *device) +{ + int ret; + + device->intr.irq = device->func->irq(device); + if (device->intr.irq < 0) + return device->intr.irq; + + ret = request_irq(device->intr.irq, nvkm_intr, IRQF_SHARED, "nvkm", device); + if (ret) + return ret; + + device->intr.alloc = true; + return 0; +} + +void +nvkm_intr_dtor(struct nvkm_device *device) +{ + if (device->intr.alloc) + free_irq(device->intr.irq, device); +} + +void +nvkm_intr_ctor(struct nvkm_device *device) +{ + spin_lock_init(&device->intr.lock); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index c45ff9df07273..44fdb6f5f225f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2734,6 +2734,8 @@ nvkm_device_fini(struct nvkm_device *device, bool suspend) if (device->func->fini) device->func->fini(device, suspend); + nvkm_intr_unarm(device); + time = ktime_to_us(ktime_get()) - time; nvdev_trace(device, "%s completed in %lldus...\n", action, time); return 0; @@ -2759,6 +2761,8 @@ nvkm_device_preinit(struct nvkm_device *device) nvdev_trace(device, "preinit running...\n"); time = ktime_to_us(ktime_get()); + nvkm_intr_unarm(device); + if (device->func->preinit) { ret = device->func->preinit(device); if (ret) @@ -2804,6 +2808,8 @@ nvkm_device_init(struct nvkm_device *device) nvdev_trace(device, "init running...\n"); time = ktime_to_us(ktime_get()); + nvkm_intr_rearm(device); + if (device->func->init) { ret = device->func->init(device); if (ret) @@ -2841,6 +2847,8 @@ nvkm_device_del(struct nvkm_device **pdevice) if (device) { mutex_lock(&nv_devices_mutex); + nvkm_intr_dtor(device); + list_for_each_entry_safe_reverse(subdev, subtmp, &device->subdev, head) nvkm_subdev_del(&subdev); @@ -3148,6 +3156,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, device->name = device->chip->name; mutex_init(&device->mutex); + nvkm_intr_ctor(device); #define NVKM_LAYOUT_ONCE(type,data,ptr) \ if (device->chip->ptr.inst && (subdev_mask & (BIT_ULL(type)))) { \ @@ -3189,7 +3198,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func, #undef NVKM_LAYOUT_INST #undef NVKM_LAYOUT_ONCE - ret = 0; + ret = nvkm_intr_install(device); done: if (device->pri && (!mmio || ret)) { iounmap(device->pri); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c index f302d2b5782a1..abccb2bb68a6c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c @@ -1574,6 +1574,12 @@ nvkm_device_pci_resource_size(struct nvkm_device *device, unsigned bar) return pci_resource_len(pdev->pdev, bar); } +static int +nvkm_device_pci_irq(struct nvkm_device *device) +{ + return nvkm_device_pci(device)->pdev->irq; +} + static void nvkm_device_pci_fini(struct nvkm_device *device, bool suspend) { @@ -1612,6 +1618,7 @@ nvkm_device_pci_func = { .dtor = nvkm_device_pci_dtor, .preinit = nvkm_device_pci_preinit, .fini = nvkm_device_pci_fini, + .irq = nvkm_device_pci_irq, .resource_addr = nvkm_device_pci_resource_addr, .resource_size = nvkm_device_pci_resource_size, .cpu_coherent = !IS_ENABLED(CONFIG_ARM), diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c index ac9e122586bc9..87caa4a72921d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c @@ -206,45 +206,12 @@ nvkm_device_tegra_resource_size(struct nvkm_device *device, unsigned bar) return res ? resource_size(res) : 0; } -static irqreturn_t -nvkm_device_tegra_intr(int irq, void *arg) -{ - struct nvkm_device_tegra *tdev = arg; - struct nvkm_device *device = &tdev->device; - bool handled = false; - nvkm_mc_intr_unarm(device); - nvkm_mc_intr(device, &handled); - nvkm_mc_intr_rearm(device); - return handled ? IRQ_HANDLED : IRQ_NONE; -} - -static void -nvkm_device_tegra_fini(struct nvkm_device *device, bool suspend) -{ - struct nvkm_device_tegra *tdev = nvkm_device_tegra(device); - if (tdev->irq) { - free_irq(tdev->irq, tdev); - tdev->irq = 0; - } -} - static int -nvkm_device_tegra_init(struct nvkm_device *device) +nvkm_device_tegra_irq(struct nvkm_device *device) { struct nvkm_device_tegra *tdev = nvkm_device_tegra(device); - int irq, ret; - - irq = platform_get_irq_byname(tdev->pdev, "stall"); - if (irq < 0) - return irq; - ret = request_irq(irq, nvkm_device_tegra_intr, - IRQF_SHARED, "nvkm", tdev); - if (ret) - return ret; - - tdev->irq = irq; - return 0; + return platform_get_irq_byname(tdev->pdev, "stall"); } static void * @@ -260,8 +227,7 @@ static const struct nvkm_device_func nvkm_device_tegra_func = { .tegra = nvkm_device_tegra, .dtor = nvkm_device_tegra_dtor, - .init = nvkm_device_tegra_init, - .fini = nvkm_device_tegra_fini, + .irq = nvkm_device_tegra_irq, .resource_addr = nvkm_device_tegra_resource_addr, .resource_size = nvkm_device_tegra_resource_size, .cpu_coherent = false, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c index a7d42ea8ba286..5a0de45d36ce0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c @@ -26,7 +26,15 @@ #include #include -#include + +void +nvkm_pci_msi_rearm(struct nvkm_device *device) +{ + struct nvkm_pci *pci = device->pci; + + if (pci && pci->msi) + pci->func->msi_rearm(pci); +} u32 nvkm_pci_rd32(struct nvkm_pci *pci, u16 addr) @@ -65,24 +73,6 @@ nvkm_pci_rom_shadow(struct nvkm_pci *pci, bool shadow) nvkm_pci_wr32(pci, 0x0050, data); } -static irqreturn_t -nvkm_pci_intr(int irq, void *arg) -{ - struct nvkm_pci *pci = arg; - struct nvkm_device *device = pci->subdev.device; - bool handled = false; - - if (pci->irq < 0) - return IRQ_HANDLED; - - nvkm_mc_intr_unarm(device); - if (pci->msi) - pci->func->msi_rearm(pci); - nvkm_mc_intr(device, &handled); - nvkm_mc_intr_rearm(device); - return handled ? IRQ_HANDLED : IRQ_NONE; -} - static int nvkm_pci_fini(struct nvkm_subdev *subdev, bool suspend) { @@ -107,7 +97,6 @@ static int nvkm_pci_oneinit(struct nvkm_subdev *subdev) { struct nvkm_pci *pci = nvkm_pci(subdev); - struct pci_dev *pdev = pci->pdev; int ret; if (pci_is_pcie(pci->pdev)) { @@ -116,11 +105,6 @@ nvkm_pci_oneinit(struct nvkm_subdev *subdev) return ret; } - ret = request_irq(pdev->irq, nvkm_pci_intr, IRQF_SHARED, "nvkm", pci); - if (ret) - return ret; - - pci->irq = pdev->irq; return 0; } @@ -157,15 +141,6 @@ nvkm_pci_dtor(struct nvkm_subdev *subdev) nvkm_agp_dtor(pci); - if (pci->irq >= 0) { - /* freq_irq() will call the handler, we use pci->irq == -1 - * to signal that it's been torn down and should be a noop. - */ - int irq = pci->irq; - pci->irq = -1; - free_irq(irq, pci); - } - if (pci->msi) pci_disable_msi(pci->pdev); @@ -192,7 +167,6 @@ nvkm_pci_new_(const struct nvkm_pci_func *func, struct nvkm_device *device, nvkm_subdev_ctor(&nvkm_pci_func, device, type, inst, &pci->subdev); pci->func = func; pci->pdev = device->func->pci(device)->pdev; - pci->irq = -1; pci->pcie.speed = -1; pci->pcie.width = -1; -- GitLab From 3ebd64aa3c4fe7fa2e73f6fa5f81490721a9c4e1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:52 +1000 Subject: [PATCH 1288/1927] drm/nouveau/intr: support multiple trees, and explicit interfaces Turing adds a second top-level interrupt tree in HW, in addition to the trees available via NV_PMC. Most of the interrupts we care about are exposed in both trees, but not all of them, and we have some rather nasty hacks to route the fault buffer interrupts. Ampere removes the NV_PMC trees entirely. Here we add some infrastructure to be able to handle all of this more cleanly, as well as providing more explicit control over handlers. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/device.h | 2 + .../gpu/drm/nouveau/include/nvkm/core/intr.h | 60 ++++ drivers/gpu/drm/nouveau/nvkm/core/intr.c | 284 +++++++++++++++++- 3 files changed, 344 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h index 6752a4c317e2a..555e3fe063414 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h @@ -63,6 +63,8 @@ struct nvkm_device { struct list_head subdev; struct { + struct list_head intr; + struct list_head prio[NVKM_INTR_PRIO_NR]; spinlock_t lock; int irq; bool alloc; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h b/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h index b4d49ff40c6ad..ec7db1312277b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h @@ -3,10 +3,70 @@ #define __NVKM_INTR_H__ #include struct nvkm_device; +struct nvkm_subdev; + +enum nvkm_intr_prio { + NVKM_INTR_PRIO_VBLANK = 0, + NVKM_INTR_PRIO_NORMAL, + NVKM_INTR_PRIO_NR +}; + +enum nvkm_intr_type { + NVKM_INTR_SUBDEV = -1, /* lookup vector by requesting subdev, in mapping table. */ + NVKM_INTR_VECTOR_0 = 0, +}; + +struct nvkm_intr { + const struct nvkm_intr_func { + bool (*pending)(struct nvkm_intr *); + void (*unarm)(struct nvkm_intr *); + void (*rearm)(struct nvkm_intr *); + void (*block)(struct nvkm_intr *, int leaf, u32 mask); + void (*allow)(struct nvkm_intr *, int leaf, u32 mask); + void (*reset)(struct nvkm_intr *, int leaf, u32 mask); + } *func; + const struct nvkm_intr_data { + int type; /* enum nvkm_subdev_type (+ve), enum nvkm_intr_type (-ve) */ + int inst; + int leaf; + u32 mask; /* 0-terminated. */ + } *data; + + struct nvkm_subdev *subdev; + int leaves; + u32 *stat; + u32 *mask; + + struct list_head head; +}; void nvkm_intr_ctor(struct nvkm_device *); void nvkm_intr_dtor(struct nvkm_device *); int nvkm_intr_install(struct nvkm_device *); void nvkm_intr_unarm(struct nvkm_device *); void nvkm_intr_rearm(struct nvkm_device *); + +int nvkm_intr_add(const struct nvkm_intr_func *, const struct nvkm_intr_data *, + struct nvkm_subdev *, int leaves, struct nvkm_intr *); +void nvkm_intr_block(struct nvkm_subdev *, enum nvkm_intr_type); +void nvkm_intr_allow(struct nvkm_subdev *, enum nvkm_intr_type); + +struct nvkm_inth; +typedef irqreturn_t (*nvkm_inth_func)(struct nvkm_inth *); + +struct nvkm_inth { + struct nvkm_intr *intr; + int leaf; + u32 mask; + nvkm_inth_func func; + + atomic_t allowed; + + struct list_head head; +}; + +int nvkm_inth_add(struct nvkm_intr *, enum nvkm_intr_type, enum nvkm_intr_prio, + struct nvkm_subdev *, nvkm_inth_func, struct nvkm_inth *); +void nvkm_inth_allow(struct nvkm_inth *); +void nvkm_inth_block(struct nvkm_inth *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/core/intr.c b/drivers/gpu/drm/nouveau/nvkm/core/intr.c index 89f6015677497..ff95e018fa78a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/intr.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/intr.c @@ -20,19 +20,147 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include - +#include +#include #include +#include + #include +static int +nvkm_intr_xlat(struct nvkm_subdev *subdev, struct nvkm_intr *intr, + enum nvkm_intr_type type, int *leaf, u32 *mask) +{ + struct nvkm_device *device = subdev->device; + + if (type < NVKM_INTR_VECTOR_0) { + if (type == NVKM_INTR_SUBDEV) { + const struct nvkm_intr_data *data = intr->data; + struct nvkm_top_device *tdev; + + while (data && data->mask) { + if (data->type == NVKM_SUBDEV_TOP) { + list_for_each_entry(tdev, &device->top->device, head) { + if (tdev->intr >= 0 && + tdev->type == subdev->type && + tdev->inst == subdev->inst) { + if (data->mask & BIT(tdev->intr)) { + *leaf = data->leaf; + *mask = BIT(tdev->intr); + return 0; + } + } + } + } else + if (data->type == subdev->type && data->inst == subdev->inst) { + *leaf = data->leaf; + *mask = data->mask; + return 0; + } + + data++; + } + } else { + return -ENOSYS; + } + } else { + if (type < intr->leaves * sizeof(*intr->stat) * 8) { + *leaf = type / 32; + *mask = BIT(type % 32); + return 0; + } + } + + return -EINVAL; +} + +static struct nvkm_intr * +nvkm_intr_find(struct nvkm_subdev *subdev, enum nvkm_intr_type type, int *leaf, u32 *mask) +{ + struct nvkm_intr *intr; + int ret; + + list_for_each_entry(intr, &subdev->device->intr.intr, head) { + ret = nvkm_intr_xlat(subdev, intr, type, leaf, mask); + if (ret == 0) + return intr; + } + + return NULL; +} + +static void +nvkm_intr_allow_locked(struct nvkm_intr *intr, int leaf, u32 mask) +{ + intr->mask[leaf] |= mask; + if (intr->func->allow) { + if (intr->func->reset) + intr->func->reset(intr, leaf, mask); + intr->func->allow(intr, leaf, mask); + } +} + +void +nvkm_intr_allow(struct nvkm_subdev *subdev, enum nvkm_intr_type type) +{ + struct nvkm_device *device = subdev->device; + struct nvkm_intr *intr; + unsigned long flags; + int leaf; + u32 mask; + + intr = nvkm_intr_find(subdev, type, &leaf, &mask); + if (intr) { + nvkm_debug(intr->subdev, "intr %d/%08x allowed by %s\n", leaf, mask, subdev->name); + spin_lock_irqsave(&device->intr.lock, flags); + nvkm_intr_allow_locked(intr, leaf, mask); + spin_unlock_irqrestore(&device->intr.lock, flags); + } +} + +static void +nvkm_intr_block_locked(struct nvkm_intr *intr, int leaf, u32 mask) +{ + intr->mask[leaf] &= ~mask; + if (intr->func->block) + intr->func->block(intr, leaf, mask); +} + +void +nvkm_intr_block(struct nvkm_subdev *subdev, enum nvkm_intr_type type) +{ + struct nvkm_device *device = subdev->device; + struct nvkm_intr *intr; + unsigned long flags; + int leaf; + u32 mask; + + intr = nvkm_intr_find(subdev, type, &leaf, &mask); + if (intr) { + nvkm_debug(intr->subdev, "intr %d/%08x blocked by %s\n", leaf, mask, subdev->name); + spin_lock_irqsave(&device->intr.lock, flags); + nvkm_intr_block_locked(intr, leaf, mask); + spin_unlock_irqrestore(&device->intr.lock, flags); + } +} + static void nvkm_intr_rearm_locked(struct nvkm_device *device) { + struct nvkm_intr *intr; + + list_for_each_entry(intr, &device->intr.intr, head) + intr->func->rearm(intr); nvkm_mc_intr_rearm(device); } static void nvkm_intr_unarm_locked(struct nvkm_device *device) { + struct nvkm_intr *intr; + + list_for_each_entry(intr, &device->intr.intr, head) + intr->func->unarm(intr); nvkm_mc_intr_unarm(device); } @@ -40,9 +168,13 @@ static irqreturn_t nvkm_intr(int irq, void *arg) { struct nvkm_device *device = arg; + struct nvkm_intr *intr; + struct nvkm_inth *inth; irqreturn_t ret = IRQ_NONE; - bool handled; + bool pending = false, handled; + int prio, leaf; + /* Disable all top-level interrupt sources, and re-arm MSI interrupts. */ spin_lock(&device->intr.lock); if (!device->intr.armed) goto done_unlock; @@ -50,20 +182,103 @@ nvkm_intr(int irq, void *arg) nvkm_intr_unarm_locked(device); nvkm_pci_msi_rearm(device); + /* Fetch pending interrupt masks. */ + list_for_each_entry(intr, &device->intr.intr, head) { + if (intr->func->pending(intr)) + pending = true; + } + nvkm_mc_intr(device, &handled); if (handled) ret = IRQ_HANDLED; + if (!pending) + goto done; + + /* Check that GPU is still on the bus by reading NV_PMC_BOOT_0. */ + if (WARN_ON(nvkm_rd32(device, 0x000000) == 0xffffffff)) + goto done; + + /* Execute handlers. */ + for (prio = 0; prio < ARRAY_SIZE(device->intr.prio); prio++) { + list_for_each_entry(inth, &device->intr.prio[prio], head) { + struct nvkm_intr *intr = inth->intr; + + if (intr->stat[inth->leaf] & inth->mask) { + if (atomic_read(&inth->allowed)) { + if (intr->func->reset) + intr->func->reset(intr, inth->leaf, inth->mask); + if (inth->func(inth) == IRQ_HANDLED) + ret = IRQ_HANDLED; + } + } + } + } + + /* Nothing handled? Some debugging/protection from IRQ storms is in order... */ + if (ret == IRQ_NONE) { + list_for_each_entry(intr, &device->intr.intr, head) { + for (leaf = 0; leaf < intr->leaves; leaf++) { + if (intr->stat[leaf]) { + nvkm_warn(intr->subdev, "intr%d: %08x\n", + leaf, intr->stat[leaf]); + nvkm_intr_block_locked(intr, leaf, intr->stat[leaf]); + } + } + } + } + +done: + /* Re-enable all top-level interrupt sources. */ nvkm_intr_rearm_locked(device); done_unlock: spin_unlock(&device->intr.lock); return ret; } +int +nvkm_intr_add(const struct nvkm_intr_func *func, const struct nvkm_intr_data *data, + struct nvkm_subdev *subdev, int leaves, struct nvkm_intr *intr) +{ + struct nvkm_device *device = subdev->device; + int i; + + intr->func = func; + intr->data = data; + intr->subdev = subdev; + intr->leaves = leaves; + intr->stat = kcalloc(leaves, sizeof(*intr->stat), GFP_KERNEL); + intr->mask = kcalloc(leaves, sizeof(*intr->mask), GFP_KERNEL); + if (!intr->stat || !intr->mask) { + kfree(intr->stat); + return -ENOMEM; + } + + if (intr->subdev->debug >= NV_DBG_DEBUG) { + for (i = 0; i < intr->leaves; i++) + intr->mask[i] = ~0; + } + + spin_lock_irq(&device->intr.lock); + list_add_tail(&intr->head, &device->intr.intr); + spin_unlock_irq(&device->intr.lock); + return 0; +} + void nvkm_intr_rearm(struct nvkm_device *device) { + struct nvkm_intr *intr; + int i; + spin_lock_irq(&device->intr.lock); + list_for_each_entry(intr, &device->intr.intr, head) { + for (i = 0; intr->func->block && i < intr->leaves; i++) { + intr->func->block(intr, i, ~0); + intr->func->allow(intr, i, intr->mask[i]); + } + } + nvkm_intr_rearm_locked(device); device->intr.armed = true; spin_unlock_irq(&device->intr.lock); @@ -98,6 +313,14 @@ nvkm_intr_install(struct nvkm_device *device) void nvkm_intr_dtor(struct nvkm_device *device) { + struct nvkm_intr *intr, *intt; + + list_for_each_entry_safe(intr, intt, &device->intr.intr, head) { + list_del(&intr->head); + kfree(intr->mask); + kfree(intr->stat); + } + if (device->intr.alloc) free_irq(device->intr.irq, device); } @@ -105,5 +328,62 @@ nvkm_intr_dtor(struct nvkm_device *device) void nvkm_intr_ctor(struct nvkm_device *device) { + int i; + + INIT_LIST_HEAD(&device->intr.intr); + for (i = 0; i < ARRAY_SIZE(device->intr.prio); i++) + INIT_LIST_HEAD(&device->intr.prio[i]); + spin_lock_init(&device->intr.lock); + device->intr.armed = false; +} + +void +nvkm_inth_block(struct nvkm_inth *inth) +{ + if (unlikely(!inth->intr)) + return; + + atomic_set(&inth->allowed, 0); +} + +void +nvkm_inth_allow(struct nvkm_inth *inth) +{ + struct nvkm_intr *intr = inth->intr; + unsigned long flags; + + if (unlikely(!inth->intr)) + return; + + spin_lock_irqsave(&intr->subdev->device->intr.lock, flags); + if (!atomic_xchg(&inth->allowed, 1)) { + if ((intr->mask[inth->leaf] & inth->mask) != inth->mask) + nvkm_intr_allow_locked(intr, inth->leaf, inth->mask); + } + spin_unlock_irqrestore(&intr->subdev->device->intr.lock, flags); +} + +int +nvkm_inth_add(struct nvkm_intr *intr, enum nvkm_intr_type type, enum nvkm_intr_prio prio, + struct nvkm_subdev *subdev, nvkm_inth_func func, struct nvkm_inth *inth) +{ + struct nvkm_device *device = subdev->device; + int ret; + + if (WARN_ON(inth->mask)) + return -EBUSY; + + ret = nvkm_intr_xlat(subdev, intr, type, &inth->leaf, &inth->mask); + if (ret) + return ret; + + nvkm_debug(intr->subdev, "intr %d/%08x requested by %s\n", + inth->leaf, inth->mask, subdev->name); + + inth->intr = intr; + inth->func = func; + atomic_set(&inth->allowed, 0); + list_add_tail(&inth->head, &device->intr.prio[prio]); + return 0; } -- GitLab From a7ab200aebc23095474e6dc8e335e60473086236 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:53 +1000 Subject: [PATCH 1289/1927] drm/nouveau/intr: add nvkm_subdev_intr() compatibility It's quite a lot of tedious and error-prone work to switch over all the subdevs at once, so allow an nvkm_intr to request new-style handlers to be created that wrap the existing interfaces. This will allow a more gradual transition. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/device.h | 1 + .../gpu/drm/nouveau/include/nvkm/core/intr.h | 1 + .../drm/nouveau/include/nvkm/core/subdev.h | 2 + drivers/gpu/drm/nouveau/nvkm/core/intr.c | 61 +++++++++++++++++++ 4 files changed, 65 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h index 555e3fe063414..f65b5009acf75 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h @@ -69,6 +69,7 @@ struct nvkm_device { int irq; bool alloc; bool armed; + bool legacy_done; } intr; }; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h b/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h index ec7db1312277b..a003d6a544b0e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/intr.h @@ -30,6 +30,7 @@ struct nvkm_intr { int inst; int leaf; u32 mask; /* 0-terminated. */ + bool legacy; /* auto-create "legacy" nvkm_subdev_intr() handler */ } *data; struct nvkm_subdev *subdev; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h index 96113c8bee8c5..20e1fc90c536e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h @@ -21,6 +21,8 @@ struct nvkm_subdev { u32 debug; struct list_head head; + struct nvkm_inth inth; + void **pself; bool oneinit; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/core/intr.c b/drivers/gpu/drm/nouveau/nvkm/core/intr.c index ff95e018fa78a..3f3fe3d416e26 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/intr.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/intr.c @@ -265,12 +265,73 @@ nvkm_intr_add(const struct nvkm_intr_func *func, const struct nvkm_intr_data *da return 0; } +static irqreturn_t +nvkm_intr_subdev(struct nvkm_inth *inth) +{ + struct nvkm_subdev *subdev = container_of(inth, typeof(*subdev), inth); + + nvkm_subdev_intr(subdev); + return IRQ_HANDLED; +} + +static void +nvkm_intr_subdev_add_dev(struct nvkm_intr *intr, enum nvkm_subdev_type type, int inst) +{ + struct nvkm_subdev *subdev; + enum nvkm_intr_prio prio; + int ret; + + subdev = nvkm_device_subdev(intr->subdev->device, type, inst); + if (!subdev || !subdev->func->intr) + return; + + if (type == NVKM_ENGINE_DISP) + prio = NVKM_INTR_PRIO_VBLANK; + else + prio = NVKM_INTR_PRIO_NORMAL; + + ret = nvkm_inth_add(intr, NVKM_INTR_SUBDEV, prio, subdev, nvkm_intr_subdev, &subdev->inth); + if (WARN_ON(ret)) + return; + + nvkm_inth_allow(&subdev->inth); +} + +static void +nvkm_intr_subdev_add(struct nvkm_intr *intr) +{ + const struct nvkm_intr_data *data; + struct nvkm_device *device = intr->subdev->device; + struct nvkm_top_device *tdev; + + for (data = intr->data; data && data->mask; data++) { + if (data->legacy) { + if (data->type == NVKM_SUBDEV_TOP) { + list_for_each_entry(tdev, &device->top->device, head) { + if (tdev->intr < 0 || !(data->mask & BIT(tdev->intr))) + continue; + + nvkm_intr_subdev_add_dev(intr, tdev->type, tdev->inst); + } + } else { + nvkm_intr_subdev_add_dev(intr, data->type, data->inst); + } + } + } +} + void nvkm_intr_rearm(struct nvkm_device *device) { struct nvkm_intr *intr; int i; + if (unlikely(!device->intr.legacy_done)) { + list_for_each_entry(intr, &device->intr.intr, head) + nvkm_intr_subdev_add(intr); + device->intr.legacy_done = true; + } + spin_lock_irq(&device->intr.lock); list_for_each_entry(intr, &device->intr.intr, head) { for (i = 0; intr->func->block && i < intr->leaves; i++) { -- GitLab From f83d1c3114182a0d826c929912e5ae94b7aeebc6 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:53 +1000 Subject: [PATCH 1290/1927] drm/nouveau/vfn: add stub subdev for dev_func Initially for NV_USERMODE class, and Turing/Ampere's new interrupt tree. v2. fixup for ga103 early merge Signed-off-by: Ben Skeggs --- .../drm/nouveau/include/nvkm/core/layout.h | 1 + .../gpu/drm/nouveau/include/nvkm/subdev/vfn.h | 18 +++++++ .../gpu/drm/nouveau/nvkm/engine/device/base.c | 12 +++++ .../gpu/drm/nouveau/nvkm/engine/device/priv.h | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild | 5 ++ .../gpu/drm/nouveau/nvkm/subdev/vfn/base.c | 48 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c | 33 +++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c | 33 +++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/vfn/priv.h | 12 +++++ .../gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c | 33 +++++++++++++ 11 files changed, 197 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h b/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h index 7afe1579b20f4..a344ddb77c808 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h @@ -1,4 +1,5 @@ /* SPDX-License-Identifier: MIT */ +NVKM_LAYOUT_ONCE(NVKM_SUBDEV_VFN , struct nvkm_vfn , vfn) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_PCI , struct nvkm_pci , pci) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_VBIOS , struct nvkm_bios , bios) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_DEVINIT , struct nvkm_devinit , devinit) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h new file mode 100644 index 0000000000000..85c1bd36ee50f --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_VFN_H__ +#define __NVKM_VFN_H__ +#include + +struct nvkm_vfn { + const struct nvkm_vfn_func *func; + struct nvkm_subdev subdev; + + struct { + u32 priv; + } addr; +}; + +int gv100_vfn_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_vfn **); +int tu102_vfn_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_vfn **); +int ga100_vfn_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_vfn **); +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 44fdb6f5f225f..88b92306dd496 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2385,6 +2385,7 @@ nv140_chipset = { .therm = { 0x00000001, gp100_therm_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, gk104_top_new }, + .vfn = { 0x00000001, gv100_vfn_new }, .ce = { 0x000001ff, gv100_ce_new }, .disp = { 0x00000001, gv100_disp_new }, .dma = { 0x00000001, gv100_dma_new }, @@ -2419,6 +2420,7 @@ nv162_chipset = { .therm = { 0x00000001, gp100_therm_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, gk104_top_new }, + .vfn = { 0x00000001, tu102_vfn_new }, .ce = { 0x0000001f, tu102_ce_new }, .disp = { 0x00000001, tu102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, @@ -2453,6 +2455,7 @@ nv164_chipset = { .therm = { 0x00000001, gp100_therm_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, gk104_top_new }, + .vfn = { 0x00000001, tu102_vfn_new }, .ce = { 0x0000001f, tu102_ce_new }, .disp = { 0x00000001, tu102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, @@ -2487,6 +2490,7 @@ nv166_chipset = { .therm = { 0x00000001, gp100_therm_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, gk104_top_new }, + .vfn = { 0x00000001, tu102_vfn_new }, .ce = { 0x0000001f, tu102_ce_new }, .disp = { 0x00000001, tu102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, @@ -2521,6 +2525,7 @@ nv167_chipset = { .therm = { 0x00000001, gp100_therm_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, gk104_top_new }, + .vfn = { 0x00000001, tu102_vfn_new }, .ce = { 0x0000001f, tu102_ce_new }, .disp = { 0x00000001, tu102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, @@ -2555,6 +2560,7 @@ nv168_chipset = { .therm = { 0x00000001, gp100_therm_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, gk104_top_new }, + .vfn = { 0x00000001, tu102_vfn_new }, .ce = { 0x0000001f, tu102_ce_new }, .disp = { 0x00000001, tu102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, @@ -2581,6 +2587,7 @@ nv170_chipset = { .privring = { 0x00000001, gm200_privring_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, + .vfn = { 0x00000001, ga100_vfn_new }, }; static const struct nvkm_device_chip @@ -2599,6 +2606,7 @@ nv172_chipset = { .privring = { 0x00000001, gm200_privring_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, + .vfn = { 0x00000001, ga100_vfn_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2620,6 +2628,7 @@ nv173_chipset = { .privring = { 0x00000001, gm200_privring_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, + .vfn = { 0x00000001, ga100_vfn_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2641,6 +2650,7 @@ nv174_chipset = { .privring = { 0x00000001, gm200_privring_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, + .vfn = { 0x00000001, ga100_vfn_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2662,6 +2672,7 @@ nv176_chipset = { .privring = { 0x00000001, gm200_privring_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, + .vfn = { 0x00000001, ga100_vfn_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2683,6 +2694,7 @@ nv177_chipset = { .privring = { 0x00000001, gm200_privring_new }, .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, + .vfn = { 0x00000001, ga100_vfn_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h index 93949b3c72143..24faaac158917 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/priv.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild index 2cb24fff7e321..4c2f6fc4ef58f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/Kbuild @@ -23,4 +23,5 @@ include $(src)/nvkm/subdev/privring/Kbuild include $(src)/nvkm/subdev/therm/Kbuild include $(src)/nvkm/subdev/timer/Kbuild include $(src)/nvkm/subdev/top/Kbuild +include $(src)/nvkm/subdev/vfn/Kbuild include $(src)/nvkm/subdev/volt/Kbuild diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild new file mode 100644 index 0000000000000..21c975a6b2060 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: MIT +nvkm-y += nvkm/subdev/vfn/base.o +nvkm-y += nvkm/subdev/vfn/gv100.o +nvkm-y += nvkm/subdev/vfn/tu102.o +nvkm-y += nvkm/subdev/vfn/ga100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c new file mode 100644 index 0000000000000..682ca3ed725b3 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c @@ -0,0 +1,48 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static void * +nvkm_vfn_dtor(struct nvkm_subdev *subdev) +{ + return nvkm_vfn(subdev); +} + +static const struct nvkm_subdev_func +nvkm_vfn = { + .dtor = nvkm_vfn_dtor, +}; + +int +nvkm_vfn_new_(const struct nvkm_vfn_func *func, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, u32 addr, struct nvkm_vfn **pvfn) +{ + struct nvkm_vfn *vfn; + + if (!(vfn = *pvfn = kzalloc(sizeof(*vfn), GFP_KERNEL))) + return -ENOMEM; + + nvkm_subdev_ctor(&nvkm_vfn, device, type, inst, &vfn->subdev); + vfn->func = func; + vfn->addr.priv = addr; + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c new file mode 100644 index 0000000000000..979402dff6595 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_vfn_func +ga100_vfn = { +}; + +int +ga100_vfn_new(struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_vfn **pvfn) +{ + return nvkm_vfn_new_(&ga100_vfn, device, type, inst, 0xb80000, pvfn); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c new file mode 100644 index 0000000000000..83dd79bddd867 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_vfn_func +gv100_vfn = { +}; + +int +gv100_vfn_new(struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_vfn **pvfn) +{ + return nvkm_vfn_new_(&gv100_vfn, device, type, inst, 0, pvfn); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h new file mode 100644 index 0000000000000..174817c76bf95 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_VFN_PRIV_H__ +#define __NVKM_VFN_PRIV_H__ +#define nvkm_vfn(p) container_of((p), struct nvkm_vfn, subdev) +#include + +struct nvkm_vfn_func { +}; + +int nvkm_vfn_new_(const struct nvkm_vfn_func *, struct nvkm_device *, enum nvkm_subdev_type, int, + u32 addr, struct nvkm_vfn **); +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c new file mode 100644 index 0000000000000..999e50f5c36b2 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_vfn_func +tu102_vfn = { +}; + +int +tu102_vfn_new(struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_vfn **pvfn) +{ + return nvkm_vfn_new_(&tu102_vfn, device, type, inst, 0xb80000, pvfn); +} -- GitLab From 58c3d3c837297051479d552753f04e4561453403 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:53 +1000 Subject: [PATCH 1291/1927] drm/nouveau/vfn: move NV_USERMODE class from host - uses proper class IDs for Turing/Ampere Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/class.h | 2 + .../gpu/drm/nouveau/include/nvkm/subdev/vfn.h | 3 ++ drivers/gpu/drm/nouveau/nvif/user.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/device/user.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 3 -- .../gpu/drm/nouveau/nvkm/engine/fifo/ga102.c | 20 +------- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/user.h | 8 ---- .../drm/nouveau/nvkm/engine/fifo/usertu102.c | 45 ------------------ .../gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/vfn/base.c | 4 ++ .../gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c | 3 ++ .../gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c | 3 ++ .../gpu/drm/nouveau/nvkm/subdev/vfn/priv.h | 8 ++++ .../gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c | 3 ++ .../fifo/usergv100.c => subdev/vfn/uvfn.c} | 46 ++++++++++++++----- 17 files changed, 67 insertions(+), 92 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c rename drivers/gpu/drm/nouveau/nvkm/{engine/fifo/usergv100.c => subdev/vfn/uvfn.c} (55%) diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 3e9d181b5dda1..a623fb67280b0 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -63,6 +63,8 @@ #define NV04_DISP /* cl0046.h */ 0x00000046 #define VOLTA_USERMODE_A 0x0000c361 +#define TURING_USERMODE_A 0x0000c461 +#define AMPERE_USERMODE_A 0x0000c561 #define MAXWELL_FAULT_BUFFER_A /* clb069.h */ 0x0000b069 #define VOLTA_FAULT_BUFFER_A /* clb069.h */ 0x0000c369 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h index 85c1bd36ee50f..fb04d5cd34bf4 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h @@ -9,7 +9,10 @@ struct nvkm_vfn { struct { u32 priv; + u32 user; } addr; + + struct nvkm_device_oclass user; }; int gv100_vfn_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_vfn **); diff --git a/drivers/gpu/drm/nouveau/nvif/user.c b/drivers/gpu/drm/nouveau/nvif/user.c index d89f5b67b3046..b648a5e036afb 100644 --- a/drivers/gpu/drm/nouveau/nvif/user.c +++ b/drivers/gpu/drm/nouveau/nvif/user.c @@ -41,7 +41,9 @@ nvif_user_ctor(struct nvif_device *device, const char *name) int version; const struct nvif_user_func *func; } users[] = { - { VOLTA_USERMODE_A, -1, &nvif_userc361 }, + { AMPERE_USERMODE_A, -1, &nvif_userc361 }, + { TURING_USERMODE_A, -1, &nvif_userc361 }, + { VOLTA_USERMODE_A, -1, &nvif_userc361 }, {} }; int cid, ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c index 45f509c11c361..9b39ec3416150 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c @@ -342,6 +342,8 @@ nvkm_udevice_child_get(struct nvkm_object *object, int index, sclass = &device->mmu->user; else if (device->fault && index-- == 0) sclass = &device->fault->user; + else if (device->vfn && index-- == 0) + sclass = &device->vfn->user; else return -EINVAL; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 5e831d347a957..0fa4df2990095 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -35,6 +35,3 @@ nvkm-y += nvkm/engine/fifo/gpfifogf100.o nvkm-y += nvkm/engine/fifo/gpfifogk104.o nvkm-y += nvkm/engine/fifo/gpfifogv100.o nvkm-y += nvkm/engine/fifo/gpfifotu102.o - -nvkm-y += nvkm/engine/fifo/usergv100.o -nvkm-y += nvkm/engine/fifo/usertu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c index c630dbd2911ae..e228ca3a52032 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c @@ -22,7 +22,6 @@ #define ga102_fifo(p) container_of((p), struct ga102_fifo, base.engine) #define ga102_chan(p) container_of((p), struct ga102_chan, object) #include -#include "user.h" #include #include @@ -242,33 +241,16 @@ ga102_chan_oclass = { .ctor = ga102_chan_new, }; -static int -ga102_user_new(struct nvkm_device *device, - const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) -{ - return tu102_fifo_user_new(oclass, argv, argc, pobject); -} - -static const struct nvkm_device_oclass -ga102_user_oclass = { - .ctor = ga102_user_new, -}; - static int ga102_fifo_sclass(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) { if (index == 0) { - oclass->base = (struct nvkm_sclass) { -1, -1, VOLTA_USERMODE_A }; - *class = &ga102_user_oclass; - return 0; - } else - if (index == 1) { oclass->base = (struct nvkm_sclass) { 0, 0, AMPERE_CHANNEL_GPFIFO_B }; *class = &ga102_chan_oclass; return 0; } - return 2; + return 1; } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index faf0fe9f704cf..f56369d592193 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -22,7 +22,6 @@ #include "gk104.h" #include "cgrp.h" #include "changk104.h" -#include "user.h" #include @@ -295,7 +294,6 @@ gv100_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .runlist = &gv100_fifo_runlist, - .user = {{-1,-1,VOLTA_USERMODE_A }, gv100_fifo_user_new }, .chan = {{ 0, 0,VOLTA_CHANNEL_GPFIFO_A}, gv100_fifo_gpfifo_new }, .cgrp_force = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 260b197f81bc3..b800cfc5bfd91 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -22,7 +22,6 @@ #include "gk104.h" #include "cgrp.h" #include "changk104.h" -#include "user.h" #include #include @@ -110,7 +109,6 @@ tu102_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .runlist = &tu102_fifo_runlist, - .user = {{-1,-1,VOLTA_USERMODE_A }, tu102_fifo_user_new }, .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, .cgrp_force = true, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h deleted file mode 100644 index 54a3a3092cc09..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/user.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __NVKM_FIFO_USER_H__ -#define __NVKM_FIFO_USER_H__ -#include "priv.h" -int gv100_fifo_user_new(const struct nvkm_oclass *, void *, u32, - struct nvkm_object **); -int tu102_fifo_user_new(const struct nvkm_oclass *, void *, u32, - struct nvkm_object **); -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c deleted file mode 100644 index 217268f8ccadf..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usertu102.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2018 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#include "user.h" - -static int -tu102_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc, - enum nvkm_object_map *type, u64 *addr, u64 *size) -{ - struct nvkm_device *device = object->engine->subdev.device; - *addr = 0xbb0000 + device->func->resource_addr(device, 0); - *size = 0x010000; - *type = NVKM_OBJECT_MAP_IO; - return 0; -} - -static const struct nvkm_object_func -tu102_fifo_user = { - .map = tu102_fifo_user_map, -}; - -int -tu102_fifo_user_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, - struct nvkm_object **pobject) -{ - return nvkm_object_new_(&tu102_fifo_user, oclass, argv, argc, pobject); -} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild index 21c975a6b2060..23cd21b40a25e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/Kbuild @@ -1,5 +1,6 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/subdev/vfn/base.o +nvkm-y += nvkm/subdev/vfn/uvfn.o nvkm-y += nvkm/subdev/vfn/gv100.o nvkm-y += nvkm/subdev/vfn/tu102.o nvkm-y += nvkm/subdev/vfn/ga100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c index 682ca3ed725b3..3aa042eec1042 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c @@ -44,5 +44,9 @@ nvkm_vfn_new_(const struct nvkm_vfn_func *func, struct nvkm_device *device, nvkm_subdev_ctor(&nvkm_vfn, device, type, inst, &vfn->subdev); vfn->func = func; vfn->addr.priv = addr; + vfn->addr.user = vfn->addr.priv + func->user.addr; + + vfn->user.ctor = nvkm_uvfn_new; + vfn->user.base = func->user.base; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c index 979402dff6595..626b64230fe59 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c @@ -21,8 +21,11 @@ */ #include "priv.h" +#include + static const struct nvkm_vfn_func ga100_vfn = { + .user = { 0x030000, 0x010000, { -1, -1, AMPERE_USERMODE_A } }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c index 83dd79bddd867..ddd39d714c4a9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/gv100.c @@ -21,8 +21,11 @@ */ #include "priv.h" +#include + static const struct nvkm_vfn_func gv100_vfn = { + .user = { 0x810000, 0x010000, { -1, -1, VOLTA_USERMODE_A } }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h index 174817c76bf95..e6f4365323926 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h @@ -5,8 +5,16 @@ #include struct nvkm_vfn_func { + struct { + u32 addr; + u32 size; + const struct nvkm_sclass base; + } user; }; int nvkm_vfn_new_(const struct nvkm_vfn_func *, struct nvkm_device *, enum nvkm_subdev_type, int, u32 addr, struct nvkm_vfn **); + +int nvkm_uvfn_new(struct nvkm_device *, const struct nvkm_oclass *, void *, u32, + struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c index 999e50f5c36b2..9e1745d31fbb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c @@ -21,8 +21,11 @@ */ #include "priv.h" +#include + static const struct nvkm_vfn_func tu102_vfn = { + .user = { 0x030000, 0x010000, { -1, -1, TURING_USERMODE_A } }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/uvfn.c similarity index 55% rename from drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c rename to drivers/gpu/drm/nouveau/nvkm/subdev/vfn/uvfn.c index 3dc3b8b312de3..c5460a14c5416 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/usergv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/uvfn.c @@ -1,5 +1,5 @@ /* - * Copyright 2018 Red Hat Inc. + * Copyright 2021 Red Hat Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -19,27 +19,49 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "user.h" +#define nvkm_uvfn(p) container_of((p), struct nvkm_uvfn, object) +#include "priv.h" + +#include + +struct nvkm_uvfn { + struct nvkm_object object; + struct nvkm_vfn *vfn; +}; static int -gv100_fifo_user_map(struct nvkm_object *object, void *argv, u32 argc, - enum nvkm_object_map *type, u64 *addr, u64 *size) +nvkm_uvfn_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) { - struct nvkm_device *device = object->engine->subdev.device; - *addr = 0x810000 + device->func->resource_addr(device, 0); - *size = 0x010000; + struct nvkm_vfn *vfn = nvkm_uvfn(object)->vfn; + struct nvkm_device *device = vfn->subdev.device; + + *addr = device->func->resource_addr(device, 0) + vfn->addr.user; + *size = vfn->func->user.size; *type = NVKM_OBJECT_MAP_IO; return 0; } static const struct nvkm_object_func -gv100_fifo_user = { - .map = gv100_fifo_user_map, +nvkm_uvfn = { + .map = nvkm_uvfn_map, }; int -gv100_fifo_user_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, - struct nvkm_object **pobject) +nvkm_uvfn_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) { - return nvkm_object_new_(&gv100_fifo_user, oclass, argv, argc, pobject); + struct nvkm_uvfn *uvfn; + + if (argc != 0) + return -ENOSYS; + + if (!(uvfn = kzalloc(sizeof(*uvfn), GFP_KERNEL))) + return -ENOMEM; + + nvkm_object_ctor(&nvkm_uvfn, oclass, &uvfn->object); + uvfn->vfn = device->vfn; + + *pobject = &uvfn->object; + return 0; } -- GitLab From e650738055cebe1dec82765232697052bf6c22d5 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:54 +1000 Subject: [PATCH 1292/1927] drm/nouveau/vfn/tu102-: support new-style interrupt tree - switches ampere over now, and removes its hack mc implementation Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/subdev/vfn.h | 2 + .../gpu/drm/nouveau/nvkm/subdev/mc/ga100.c | 16 +---- .../gpu/drm/nouveau/nvkm/subdev/vfn/base.c | 8 +++ .../gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c | 11 +++ .../gpu/drm/nouveau/nvkm/subdev/vfn/priv.h | 5 ++ .../gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c | 72 +++++++++++++++++++ 6 files changed, 99 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h index fb04d5cd34bf4..cc6d0796c265e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/vfn.h @@ -12,6 +12,8 @@ struct nvkm_vfn { u32 user; } addr; + struct nvkm_intr intr; + struct nvkm_device_oclass user; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c index 4105175dfccd6..ec3eedfa541aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c @@ -24,36 +24,23 @@ static void ga100_mc_intr_unarm(struct nvkm_mc *mc) { - nvkm_wr32(mc->subdev.device, 0xb81610, 0x00000004); } static void ga100_mc_intr_rearm(struct nvkm_mc *mc) { - nvkm_wr32(mc->subdev.device, 0xb81608, 0x00000004); -} - -static void -ga100_mc_intr_mask(struct nvkm_mc *mc, u32 mask, u32 intr) -{ - nvkm_wr32(mc->subdev.device, 0xb81210, mask & intr ); - nvkm_wr32(mc->subdev.device, 0xb81410, mask & ~(mask & intr)); } static u32 ga100_mc_intr_stat(struct nvkm_mc *mc) { - u32 intr_top = nvkm_rd32(mc->subdev.device, 0xb81600), intr = 0x00000000; - if (intr_top & 0x00000004) - intr = nvkm_mask(mc->subdev.device, 0xb81010, 0x00000000, 0x00000000); - return intr; + return 0; } static void ga100_mc_init(struct nvkm_mc *mc) { nv50_mc_init(mc); - nvkm_wr32(mc->subdev.device, 0xb81210, 0xffffffff); } static const struct nvkm_mc_func @@ -62,7 +49,6 @@ ga100_mc = { .intr = gp100_mc_intr, .intr_unarm = ga100_mc_intr_unarm, .intr_rearm = ga100_mc_intr_rearm, - .intr_mask = ga100_mc_intr_mask, .intr_stat = ga100_mc_intr_stat, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c index 3aa042eec1042..62e81d551f441 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/base.c @@ -37,6 +37,7 @@ nvkm_vfn_new_(const struct nvkm_vfn_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, u32 addr, struct nvkm_vfn **pvfn) { struct nvkm_vfn *vfn; + int ret; if (!(vfn = *pvfn = kzalloc(sizeof(*vfn), GFP_KERNEL))) return -ENOMEM; @@ -46,6 +47,13 @@ nvkm_vfn_new_(const struct nvkm_vfn_func *func, struct nvkm_device *device, vfn->addr.priv = addr; vfn->addr.user = vfn->addr.priv + func->user.addr; + if (vfn->func->intr) { + ret = nvkm_intr_add(vfn->func->intr, vfn->func->intrs, + &vfn->subdev, 8, &vfn->intr); + if (ret) + return ret; + } + vfn->user.ctor = nvkm_uvfn_new; vfn->user.base = func->user.base; return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c index 626b64230fe59..fd5c6931322dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/ga100.c @@ -23,8 +23,19 @@ #include +static const struct nvkm_intr_data +ga100_vfn_intrs[] = { + { NVKM_ENGINE_DISP , 0, 4, 0x04000000, true }, + { NVKM_SUBDEV_GPIO , 0, 4, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 4, 0x00200000, true }, + { NVKM_SUBDEV_PRIVRING, 0, 4, 0x40000000, true }, + {} +}; + static const struct nvkm_vfn_func ga100_vfn = { + .intr = &tu102_vfn_intr, + .intrs = ga100_vfn_intrs, .user = { 0x030000, 0x010000, { -1, -1, AMPERE_USERMODE_A } }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h index e6f4365323926..96d53c02041b5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/priv.h @@ -5,6 +5,9 @@ #include struct nvkm_vfn_func { + const struct nvkm_intr_func *intr; + const struct nvkm_intr_data *intrs; + struct { u32 addr; u32 size; @@ -15,6 +18,8 @@ struct nvkm_vfn_func { int nvkm_vfn_new_(const struct nvkm_vfn_func *, struct nvkm_device *, enum nvkm_subdev_type, int, u32 addr, struct nvkm_vfn **); +extern const struct nvkm_intr_func tu102_vfn_intr; + int nvkm_uvfn_new(struct nvkm_device *, const struct nvkm_oclass *, void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c index 9e1745d31fbb1..3d063fb5e1366 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/vfn/tu102.c @@ -23,8 +23,80 @@ #include +static void +tu102_vfn_intr_reset(struct nvkm_intr *intr, int leaf, u32 mask) +{ + struct nvkm_vfn *vfn = container_of(intr, typeof(*vfn), intr); + + nvkm_wr32(vfn->subdev.device, vfn->addr.priv + 0x1000 + (leaf * 4), mask); +} + +static void +tu102_vfn_intr_allow(struct nvkm_intr *intr, int leaf, u32 mask) +{ + struct nvkm_vfn *vfn = container_of(intr, typeof(*vfn), intr); + + nvkm_wr32(vfn->subdev.device, vfn->addr.priv + 0x1200 + (leaf * 4), mask); +} + +static void +tu102_vfn_intr_block(struct nvkm_intr *intr, int leaf, u32 mask) +{ + struct nvkm_vfn *vfn = container_of(intr, typeof(*vfn), intr); + + nvkm_wr32(vfn->subdev.device, vfn->addr.priv + 0x1400 + (leaf * 4), mask); +} + +static void +tu102_vfn_intr_rearm(struct nvkm_intr *intr) +{ + struct nvkm_vfn *vfn = container_of(intr, typeof(*vfn), intr); + + nvkm_wr32(vfn->subdev.device, vfn->addr.priv + 0x1608, 0x0000000f); +} + +static void +tu102_vfn_intr_unarm(struct nvkm_intr *intr) +{ + struct nvkm_vfn *vfn = container_of(intr, typeof(*vfn), intr); + + nvkm_wr32(vfn->subdev.device, vfn->addr.priv + 0x1610, 0x0000000f); +} + +static bool +tu102_vfn_intr_pending(struct nvkm_intr *intr) +{ + struct nvkm_vfn *vfn = container_of(intr, typeof(*vfn), intr); + struct nvkm_device *device = vfn->subdev.device; + u32 intr_top = nvkm_rd32(device, vfn->addr.priv + 0x1600); + int pending = 0, leaf; + + for (leaf = 0; leaf < 8; leaf++) { + if (intr_top & BIT(leaf / 2)) { + intr->stat[leaf] = nvkm_rd32(device, vfn->addr.priv + 0x1000 + (leaf * 4)); + if (intr->stat[leaf]) + pending++; + } else { + intr->stat[leaf] = 0; + } + } + + return pending != 0; +} + +const struct nvkm_intr_func +tu102_vfn_intr = { + .pending = tu102_vfn_intr_pending, + .unarm = tu102_vfn_intr_unarm, + .rearm = tu102_vfn_intr_rearm, + .block = tu102_vfn_intr_block, + .allow = tu102_vfn_intr_allow, + .reset = tu102_vfn_intr_reset, +}; + static const struct nvkm_vfn_func tu102_vfn = { + .intr = &tu102_vfn_intr, .user = { 0x030000, 0x010000, { -1, -1, TURING_USERMODE_A } }, }; -- GitLab From b59d810a47423e61ae4e0ded502585e6ee9b1d76 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:54 +1000 Subject: [PATCH 1293/1927] drm/nouveau/fault/tu102: switch to explicit intr handlers - reads vectors from HW, rather than being hardcoded - removes hacks to support routing via old interfaces Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/subdev/fault.h | 2 + .../gpu/drm/nouveau/nvkm/subdev/fault/priv.h | 2 + .../gpu/drm/nouveau/nvkm/subdev/fault/tu102.c | 107 ++++++++---------- .../gpu/drm/nouveau/nvkm/subdev/mc/tu102.c | 13 --- 4 files changed, 52 insertions(+), 72 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h index e781c5e4dcaeb..e40bbf378a8dd 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fault.h @@ -7,6 +7,8 @@ struct nvkm_fault { const struct nvkm_fault_func *func; struct nvkm_subdev subdev; + struct nvkm_inth info_fault; + struct nvkm_fault_buffer *buffer[2]; int buffer_nr; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h index 30cc0d0888972..a5510332c402f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/priv.h @@ -16,6 +16,8 @@ struct nvkm_fault_buffer { u32 put; struct nvkm_memory *mem; u64 addr; + + struct nvkm_inth inth; }; int nvkm_fault_new_(const struct nvkm_fault_func *, struct nvkm_device *, enum nvkm_subdev_type, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c index 917a7343db315..967efaddae281 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fault/tu102.c @@ -24,20 +24,27 @@ #include #include #include +#include #include #include +static irqreturn_t +tu102_fault_buffer_notify(struct nvkm_inth *inth) +{ + struct nvkm_fault_buffer *buffer = container_of(inth, typeof(*buffer), inth); + + nvkm_event_ntfy(&buffer->fault->event, buffer->id, NVKM_FAULT_BUFFER_EVENT_PENDING); + return IRQ_HANDLED; +} + static void tu102_fault_buffer_intr(struct nvkm_fault_buffer *buffer, bool enable) { - /*XXX: Earlier versions of RM touched the old regs on Turing, - * which don't appear to actually work anymore, but newer - * versions of RM don't appear to touch anything at all.. - */ - struct nvkm_device *device = buffer->fault->subdev.device; - - nvkm_mc_intr_mask(device, NVKM_SUBDEV_FAULT, 0, enable); + if (enable) + nvkm_inth_allow(&buffer->inth); + else + nvkm_inth_block(&buffer->inth); } static void @@ -46,10 +53,6 @@ tu102_fault_buffer_fini(struct nvkm_fault_buffer *buffer) struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x20; - /* Disable the fault interrupts */ - nvkm_wr32(device, 0xb81408, 0x1); - nvkm_wr32(device, 0xb81410, 0x10); - nvkm_mask(device, 0xb83010 + foff, 0x80000000, 0x00000000); } @@ -59,10 +62,6 @@ tu102_fault_buffer_init(struct nvkm_fault_buffer *buffer) struct nvkm_device *device = buffer->fault->subdev.device; const u32 foff = buffer->id * 0x20; - /* Enable the fault interrupts */ - nvkm_wr32(device, 0xb81208, 0x1); - nvkm_wr32(device, 0xb81210, 0x10); - nvkm_mask(device, 0xb83010 + foff, 0xc0000000, 0x40000000); nvkm_wr32(device, 0xb83004 + foff, upper_32_bits(buffer->addr)); nvkm_wr32(device, 0xb83000 + foff, lower_32_bits(buffer->addr)); @@ -82,9 +81,10 @@ tu102_fault_buffer_info(struct nvkm_fault_buffer *buffer) buffer->put = 0xb8300c + foff; } -static void -tu102_fault_intr_fault(struct nvkm_fault *fault) +static irqreturn_t +tu102_fault_info_fault(struct nvkm_inth *inth) { + struct nvkm_fault *fault = container_of(inth, typeof(*fault), info_fault); struct nvkm_subdev *subdev = &fault->subdev; struct nvkm_device *device = subdev->device; struct nvkm_fault_data info; @@ -106,45 +106,9 @@ tu102_fault_intr_fault(struct nvkm_fault *fault) info.reason = (info1 & 0x0000001f); nvkm_fifo_fault(device->fifo, &info); -} - -static void -tu102_fault_intr(struct nvkm_fault *fault) -{ - struct nvkm_subdev *subdev = &fault->subdev; - struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0xb83094); - - if (stat & 0x80000000) { - tu102_fault_intr_fault(fault); - nvkm_wr32(device, 0xb83094, 0x80000000); - stat &= ~0x80000000; - } - - if (stat & 0x00000200) { - /* Clear the associated interrupt flag */ - nvkm_wr32(device, 0xb81010, 0x10); - if (fault->buffer[0]) { - nvkm_event_ntfy(&fault->event, 0, NVKM_FAULT_BUFFER_EVENT_PENDING); - stat &= ~0x00000200; - } - } - - /* Replayable MMU fault */ - if (stat & 0x00000100) { - /* Clear the associated interrupt flag */ - nvkm_wr32(device, 0xb81008, 0x1); - - if (fault->buffer[1]) { - nvkm_event_ntfy(&fault->event, 1, NVKM_FAULT_BUFFER_EVENT_PENDING); - stat &= ~0x00000100; - } - } - - if (stat) { - nvkm_debug(subdev, "intr %08x\n", stat); - } + nvkm_wr32(device, 0xb83094, 0x80000000); + return IRQ_HANDLED; } static void @@ -155,23 +119,48 @@ tu102_fault_fini(struct nvkm_fault *fault) if (fault->buffer[0]) fault->func->buffer.fini(fault->buffer[0]); - /*XXX: disable priv faults */ + + nvkm_inth_block(&fault->info_fault); } static void tu102_fault_init(struct nvkm_fault *fault) { - /*XXX: enable priv faults */ + nvkm_inth_allow(&fault->info_fault); + fault->func->buffer.init(fault->buffer[0]); nvkm_event_ntfy_allow(&fault->nrpfb); } +static int +tu102_fault_oneinit(struct nvkm_fault *fault) +{ + struct nvkm_device *device = fault->subdev.device; + struct nvkm_intr *intr = &device->vfn->intr; + int ret, i; + + ret = nvkm_inth_add(intr, nvkm_rd32(device, 0x100ee0) & 0x0000ffff, + NVKM_INTR_PRIO_NORMAL, &fault->subdev, tu102_fault_info_fault, + &fault->info_fault); + if (ret) + return ret; + + for (i = 0; i < fault->buffer_nr; i++) { + ret = nvkm_inth_add(intr, nvkm_rd32(device, 0x100ee4 + (i * 4)) >> 16, + NVKM_INTR_PRIO_NORMAL, &fault->subdev, + tu102_fault_buffer_notify, &fault->buffer[i]->inth); + if (ret) + return ret; + } + + return gv100_fault_oneinit(fault); +} + static const struct nvkm_fault_func tu102_fault = { - .oneinit = gv100_fault_oneinit, + .oneinit = tu102_fault_oneinit, .init = tu102_fault_init, .fini = tu102_fault_fini, - .intr = tu102_fault_intr, .buffer.nr = 2, .buffer.entry_size = 32, .buffer.info = tu102_fault_buffer_info, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c index a96084b34a788..630a9c9e7ac97 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c @@ -39,11 +39,6 @@ tu102_mc_intr_update(struct tu102_mc *mc) nvkm_wr32(device, 0x000180 + (i * 0x04), ~mask); nvkm_wr32(device, 0x000160 + (i * 0x04), mask); } - - if (mask & 0x00000200) - nvkm_wr32(device, 0xb81608, 0x6); - else - nvkm_wr32(device, 0xb81610, 0x6); } static void @@ -88,14 +83,6 @@ tu102_mc_intr_stat(struct nvkm_mc *mc) struct nvkm_device *device = mc->subdev.device; u32 intr0 = nvkm_rd32(device, 0x000100); u32 intr1 = nvkm_rd32(device, 0x000104); - u32 intr_top = nvkm_rd32(device, 0xb81600); - - /* Turing and above route the MMU fault interrupts via a different - * interrupt tree with different control registers. For the moment remap - * them back to the old PMC vector. - */ - if (intr_top & 0x00000006) - intr0 |= 0x00000200; return intr0 | intr1; } -- GitLab From e5f92c8735b6ff415aafdd0af85232f2592174c3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:54 +1000 Subject: [PATCH 1294/1927] drm/nouveau/fault/ga100: initial support TU102 implementation should be OK for Ampere now. v2. fixup for ga103 early merge Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/device/base.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 88b92306dd496..a36c46aff242d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2577,6 +2577,7 @@ nv170_chipset = { .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, + .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga100_fb_new }, .gpio = { 0x00000001, gk104_gpio_new }, .i2c = { 0x00000001, gm200_i2c_new }, @@ -2596,6 +2597,7 @@ nv172_chipset = { .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, + .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, .i2c = { 0x00000001, gm200_i2c_new }, @@ -2618,6 +2620,7 @@ nv173_chipset = { .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, + .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, .i2c = { 0x00000001, gm200_i2c_new }, @@ -2640,6 +2643,7 @@ nv174_chipset = { .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, + .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, .i2c = { 0x00000001, gm200_i2c_new }, @@ -2662,6 +2666,7 @@ nv176_chipset = { .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, + .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, .i2c = { 0x00000001, gm200_i2c_new }, @@ -2684,6 +2689,7 @@ nv177_chipset = { .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, + .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, .i2c = { 0x00000001, gm200_i2c_new }, -- GitLab From fe76fe497c30080ce8962ea62c5c13c6814e1a83 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:55 +1000 Subject: [PATCH 1295/1927] drm/nouveau/mc: implement intr handling on top of nvkm_intr - new-style handlers can now be used here too - decent clean-up Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/subdev/mc.h | 6 +- drivers/gpu/drm/nouveau/nvkm/core/intr.c | 10 +- .../gpu/drm/nouveau/nvkm/engine/device/base.c | 10 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild | 1 - drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c | 113 +++------------- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c | 34 +++-- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c | 34 +++-- .../gpu/drm/nouveau/nvkm/subdev/mc/ga100.c | 28 +--- .../gpu/drm/nouveau/nvkm/subdev/mc/gf100.c | 81 ++++-------- .../gpu/drm/nouveau/nvkm/subdev/mc/gk104.c | 36 +++-- .../gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c | 8 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gp100.c | 124 +++++++----------- .../gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c | 10 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gt215.c | 62 +++++---- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c | 64 ++++++--- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c | 20 ++- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c | 22 ++-- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c | 6 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c | 28 ++-- drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h | 42 +++--- .../gpu/drm/nouveau/nvkm/subdev/mc/tu102.c | 123 ----------------- 21 files changed, 285 insertions(+), 577 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h index cb86a56e68d4f..127ac545e4b21 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h @@ -6,15 +6,14 @@ struct nvkm_mc { const struct nvkm_mc_func *func; struct nvkm_subdev subdev; + + struct nvkm_intr intr; }; void nvkm_mc_enable(struct nvkm_device *, enum nvkm_subdev_type, int); void nvkm_mc_disable(struct nvkm_device *, enum nvkm_subdev_type, int); bool nvkm_mc_enabled(struct nvkm_device *, enum nvkm_subdev_type, int); void nvkm_mc_reset(struct nvkm_device *, enum nvkm_subdev_type, int); -void nvkm_mc_intr(struct nvkm_device *, bool *handled); -void nvkm_mc_intr_unarm(struct nvkm_device *); -void nvkm_mc_intr_rearm(struct nvkm_device *); void nvkm_mc_intr_mask(struct nvkm_device *, enum nvkm_subdev_type, int, bool enable); void nvkm_mc_unk260(struct nvkm_device *, u32 data); @@ -31,6 +30,5 @@ int gk104_mc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct n int gk20a_mc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mc **); int gp100_mc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mc **); int gp10b_mc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mc **); -int tu102_mc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mc **); int ga100_mc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_mc **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/core/intr.c b/drivers/gpu/drm/nouveau/nvkm/core/intr.c index 3f3fe3d416e26..e20b7ca218c3d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/intr.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/intr.c @@ -25,8 +25,6 @@ #include #include -#include - static int nvkm_intr_xlat(struct nvkm_subdev *subdev, struct nvkm_intr *intr, enum nvkm_intr_type type, int *leaf, u32 *mask) @@ -151,7 +149,6 @@ nvkm_intr_rearm_locked(struct nvkm_device *device) list_for_each_entry(intr, &device->intr.intr, head) intr->func->rearm(intr); - nvkm_mc_intr_rearm(device); } static void @@ -161,7 +158,6 @@ nvkm_intr_unarm_locked(struct nvkm_device *device) list_for_each_entry(intr, &device->intr.intr, head) intr->func->unarm(intr); - nvkm_mc_intr_unarm(device); } static irqreturn_t @@ -171,7 +167,7 @@ nvkm_intr(int irq, void *arg) struct nvkm_intr *intr; struct nvkm_inth *inth; irqreturn_t ret = IRQ_NONE; - bool pending = false, handled; + bool pending = false; int prio, leaf; /* Disable all top-level interrupt sources, and re-arm MSI interrupts. */ @@ -188,10 +184,6 @@ nvkm_intr(int irq, void *arg) pending = true; } - nvkm_mc_intr(device, &handled); - if (handled) - ret = IRQ_HANDLED; - if (!pending) goto done; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index a36c46aff242d..587aebcda78c9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2412,7 +2412,7 @@ nv162_chipset = { .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .ltc = { 0x00000001, gp102_ltc_new }, - .mc = { 0x00000001, tu102_mc_new }, + .mc = { 0x00000001, gp100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, .pmu = { 0x00000001, gp102_pmu_new }, @@ -2447,7 +2447,7 @@ nv164_chipset = { .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .ltc = { 0x00000001, gp102_ltc_new }, - .mc = { 0x00000001, tu102_mc_new }, + .mc = { 0x00000001, gp100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, .pmu = { 0x00000001, gp102_pmu_new }, @@ -2482,7 +2482,7 @@ nv166_chipset = { .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .ltc = { 0x00000001, gp102_ltc_new }, - .mc = { 0x00000001, tu102_mc_new }, + .mc = { 0x00000001, gp100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, .pmu = { 0x00000001, gp102_pmu_new }, @@ -2517,7 +2517,7 @@ nv167_chipset = { .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .ltc = { 0x00000001, gp102_ltc_new }, - .mc = { 0x00000001, tu102_mc_new }, + .mc = { 0x00000001, gp100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, .pmu = { 0x00000001, gp102_pmu_new }, @@ -2552,7 +2552,7 @@ nv168_chipset = { .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .ltc = { 0x00000001, gp102_ltc_new }, - .mc = { 0x00000001, tu102_mc_new }, + .mc = { 0x00000001, gp100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, .pmu = { 0x00000001, gp102_pmu_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild index ac2b34e9ac6ad..2a3255ced8b7a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild @@ -13,5 +13,4 @@ nvkm-y += nvkm/subdev/mc/gk104.o nvkm-y += nvkm/subdev/mc/gk20a.o nvkm-y += nvkm/subdev/mc/gp100.o nvkm-y += nvkm/subdev/mc/gp10b.o -nvkm-y += nvkm/subdev/mc/tu102.o nvkm-y += nvkm/subdev/mc/ga100.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c index 21c4af3f81d5f..322237e2ff80e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c @@ -37,84 +37,14 @@ nvkm_mc_unk260(struct nvkm_device *device, u32 data) void nvkm_mc_intr_mask(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, bool en) { - struct nvkm_mc *mc = device->mc; - const struct nvkm_mc_map *map; - if (likely(mc) && mc->func->intr_mask) { - u32 mask = nvkm_top_intr_mask(device, type, inst); - for (map = mc->func->intr; !mask && map->stat; map++) { - if (map->type == type && map->inst == inst) - mask = map->stat; - } - mc->func->intr_mask(mc, mask, en ? mask : 0); - } -} - -void -nvkm_mc_intr_unarm(struct nvkm_device *device) -{ - struct nvkm_mc *mc = device->mc; - if (likely(mc)) - mc->func->intr_unarm(mc); -} - -void -nvkm_mc_intr_rearm(struct nvkm_device *device) -{ - struct nvkm_mc *mc = device->mc; - if (likely(mc)) - mc->func->intr_rearm(mc); -} - -static u32 -nvkm_mc_intr_stat(struct nvkm_mc *mc) -{ - u32 intr = mc->func->intr_stat(mc); - if (WARN_ON_ONCE(intr == 0xffffffff)) - intr = 0; /* likely fallen off the bus */ - return intr; -} - -void -nvkm_mc_intr(struct nvkm_device *device, bool *handled) -{ - struct nvkm_mc *mc = device->mc; - struct nvkm_top *top = device->top; - struct nvkm_top_device *tdev; - struct nvkm_subdev *subdev; - const struct nvkm_mc_map *map; - u32 stat, intr; - - if (unlikely(!mc)) - return; - - stat = intr = nvkm_mc_intr_stat(mc); + struct nvkm_subdev *subdev = nvkm_device_subdev(device, type, inst); - if (top) { - list_for_each_entry(tdev, &top->device, head) { - if (tdev->intr >= 0 && (stat & BIT(tdev->intr))) { - subdev = nvkm_device_subdev(device, tdev->type, tdev->inst); - if (subdev) { - nvkm_subdev_intr(subdev); - stat &= ~BIT(tdev->intr); - if (!stat) - break; - } - } - } - } - - for (map = mc->func->intr; map->stat; map++) { - if (intr & map->stat) { - subdev = nvkm_device_subdev(device, map->type, map->inst); - if (subdev) - nvkm_subdev_intr(subdev); - stat &= ~map->stat; - } + if (subdev) { + if (en) + nvkm_intr_allow(subdev, NVKM_INTR_SUBDEV); + else + nvkm_intr_block(subdev, NVKM_INTR_SUBDEV); } - - if (stat) - nvkm_error(&mc->subdev, "intr %08x\n", stat); - *handled = intr != 0; } static u32 @@ -177,20 +107,12 @@ nvkm_mc_enabled(struct nvkm_device *device, enum nvkm_subdev_type type, int inst } -static int -nvkm_mc_fini(struct nvkm_subdev *subdev, bool suspend) -{ - nvkm_mc_intr_unarm(subdev->device); - return 0; -} - static int nvkm_mc_init(struct nvkm_subdev *subdev) { struct nvkm_mc *mc = nvkm_mc(subdev); if (mc->func->init) mc->func->init(mc); - nvkm_mc_intr_rearm(subdev->device); return 0; } @@ -204,24 +126,27 @@ static const struct nvkm_subdev_func nvkm_mc = { .dtor = nvkm_mc_dtor, .init = nvkm_mc_init, - .fini = nvkm_mc_fini, }; -void -nvkm_mc_ctor(const struct nvkm_mc_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_mc *mc) -{ - nvkm_subdev_ctor(&nvkm_mc, device, type, inst, &mc->subdev); - mc->func = func; -} - int nvkm_mc_new_(const struct nvkm_mc_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_mc **pmc) { struct nvkm_mc *mc; + int ret; + if (!(mc = *pmc = kzalloc(sizeof(*mc), GFP_KERNEL))) return -ENOMEM; - nvkm_mc_ctor(func, device, type, inst, *pmc); + + nvkm_subdev_ctor(&nvkm_mc, device, type, inst, &mc->subdev); + mc->func = func; + + if (mc->func->intr) { + ret = nvkm_intr_add(mc->func->intr, mc->func->intrs, &mc->subdev, + mc->func->intr_nonstall ? 2 : 1, &mc->intr); + if (ret) + return ret; + } + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c index 4cfc1c9840068..8a8267ecfe1d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c @@ -34,30 +34,28 @@ g84_mc_reset[] = { {} }; -static const struct nvkm_mc_map -g84_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00020000, NVKM_ENGINE_VP }, - { 0x00008000, NVKM_ENGINE_BSP }, - { 0x00004000, NVKM_ENGINE_CIPHER }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000001, NVKM_ENGINE_MPEG }, - { 0x0002d101, NVKM_SUBDEV_FB }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, +static const struct nvkm_intr_data +g84_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_VP , 0, 0, 0x00020000, true }, + { NVKM_ENGINE_BSP , 0, 0, 0x00008000, true }, + { NVKM_ENGINE_CIPHER, 0, 0, 0x00004000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_MPEG , 0, 0, 0x00000001, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x0002d101, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, {}, }; static const struct nvkm_mc_func g84_mc = { .init = nv50_mc_init, - .intr = g84_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = g84_mc_intrs, .reset = g84_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c index b7e58d75d894b..e099d419f1ce5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c @@ -34,30 +34,28 @@ g98_mc_reset[] = { {} }; -static const struct nvkm_mc_map -g98_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00020000, NVKM_ENGINE_MSPDEC }, - { 0x00008000, NVKM_ENGINE_MSVLD }, - { 0x00004000, NVKM_ENGINE_SEC }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000001, NVKM_ENGINE_MSPPP }, - { 0x0002d101, NVKM_SUBDEV_FB }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, +static const struct nvkm_intr_data +g98_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_MSPDEC, 0, 0, 0x00020000, true }, + { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, + { NVKM_ENGINE_SEC , 0, 0, 0x00004000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_MSPPP , 0, 0, 0x00000001, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x0002d101, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, {}, }; static const struct nvkm_mc_func g98_mc = { .init = nv50_mc_init, - .intr = g98_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = g98_mc_intrs, .reset = g98_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c index ec3eedfa541aa..98ccce9585ebc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c @@ -21,35 +21,9 @@ */ #include "priv.h" -static void -ga100_mc_intr_unarm(struct nvkm_mc *mc) -{ -} - -static void -ga100_mc_intr_rearm(struct nvkm_mc *mc) -{ -} - -static u32 -ga100_mc_intr_stat(struct nvkm_mc *mc) -{ - return 0; -} - -static void -ga100_mc_init(struct nvkm_mc *mc) -{ - nv50_mc_init(mc); -} - static const struct nvkm_mc_func ga100_mc = { - .init = ga100_mc_init, - .intr = gp100_mc_intr, - .intr_unarm = ga100_mc_intr_unarm, - .intr_rearm = ga100_mc_intr_rearm, - .intr_stat = ga100_mc_intr_stat, + .init = nv50_mc_init, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c index 3a589c6f7fad6..b2846eaea34c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c @@ -36,63 +36,28 @@ gf100_mc_reset[] = { {} }; -static const struct nvkm_mc_map -gf100_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00020000, NVKM_ENGINE_MSPDEC }, - { 0x00008000, NVKM_ENGINE_MSVLD }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000040, NVKM_ENGINE_CE, 1 }, - { 0x00000020, NVKM_ENGINE_CE, 0 }, - { 0x00000001, NVKM_ENGINE_MSPPP }, - { 0x40000000, NVKM_SUBDEV_PRIVRING }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x08000000, NVKM_SUBDEV_FB }, - { 0x02000000, NVKM_SUBDEV_LTC }, - { 0x01000000, NVKM_SUBDEV_PMU }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, - { 0x00040000, NVKM_SUBDEV_THERM }, - { 0x00002000, NVKM_SUBDEV_FB }, +static const struct nvkm_intr_data +gf100_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_MSPDEC , 0, 0, 0x00020000, true }, + { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_CE , 1, 0, 0x00000040, true }, + { NVKM_ENGINE_CE , 0, 0, 0x00000020, true }, + { NVKM_ENGINE_MSPPP , 0, 0, 0x00000001, true }, + { NVKM_SUBDEV_PRIVRING, 0, 0, 0x40000000, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x08002000, true }, + { NVKM_SUBDEV_LTC , 0, 0, 0x02000000, true }, + { NVKM_SUBDEV_PMU , 0, 0, 0x01000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, + { NVKM_SUBDEV_THERM , 0, 0, 0x00040000, true }, {}, }; -void -gf100_mc_intr_unarm(struct nvkm_mc *mc) -{ - struct nvkm_device *device = mc->subdev.device; - nvkm_wr32(device, 0x000140, 0x00000000); - nvkm_wr32(device, 0x000144, 0x00000000); - nvkm_rd32(device, 0x000140); -} - -void -gf100_mc_intr_rearm(struct nvkm_mc *mc) -{ - struct nvkm_device *device = mc->subdev.device; - nvkm_wr32(device, 0x000140, 0x00000001); - nvkm_wr32(device, 0x000144, 0x00000001); -} - -u32 -gf100_mc_intr_stat(struct nvkm_mc *mc) -{ - struct nvkm_device *device = mc->subdev.device; - u32 intr0 = nvkm_rd32(device, 0x000100); - u32 intr1 = nvkm_rd32(device, 0x000104); - return intr0 | intr1; -} - -void -gf100_mc_intr_mask(struct nvkm_mc *mc, u32 mask, u32 stat) -{ - struct nvkm_device *device = mc->subdev.device; - nvkm_mask(device, 0x000640, mask, stat); - nvkm_mask(device, 0x000644, mask, stat); -} - void gf100_mc_unk260(struct nvkm_mc *mc, u32 data) { @@ -102,11 +67,9 @@ gf100_mc_unk260(struct nvkm_mc *mc, u32 data) static const struct nvkm_mc_func gf100_mc = { .init = nv50_mc_init, - .intr = gf100_mc_intr, - .intr_unarm = gf100_mc_intr_unarm, - .intr_rearm = gf100_mc_intr_rearm, - .intr_mask = gf100_mc_intr_mask, - .intr_stat = gf100_mc_intr_stat, + .intr = >215_mc_intr, + .intrs = gf100_mc_intrs, + .intr_nonstall = true, .reset = gf100_mc_reset, .unk260 = gf100_mc_unk260, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c index d9b9067fa93f5..edf82e4568bf9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c @@ -30,31 +30,29 @@ gk104_mc_reset[] = { {} }; -const struct nvkm_mc_map -gk104_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x40000000, NVKM_SUBDEV_PRIVRING }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x08000000, NVKM_SUBDEV_FB }, - { 0x02000000, NVKM_SUBDEV_LTC }, - { 0x01000000, NVKM_SUBDEV_PMU }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, - { 0x00040000, NVKM_SUBDEV_THERM }, - { 0x00002000, NVKM_SUBDEV_FB }, +const struct nvkm_intr_data +gk104_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_SUBDEV_PRIVRING, 0, 0, 0x40000000, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x08002000, true }, + { NVKM_SUBDEV_LTC , 0, 0, 0x02000000, true }, + { NVKM_SUBDEV_PMU , 0, 0, 0x01000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, + { NVKM_SUBDEV_THERM , 0, 0, 0x00040000, true }, + { NVKM_SUBDEV_TOP , 0, 0, 0xffffffff, true }, {}, }; static const struct nvkm_mc_func gk104_mc = { .init = nv50_mc_init, - .intr = gk104_mc_intr, - .intr_unarm = gf100_mc_intr_unarm, - .intr_rearm = gf100_mc_intr_rearm, - .intr_mask = gf100_mc_intr_mask, - .intr_stat = gf100_mc_intr_stat, + .intr = >215_mc_intr, + .intrs = gk104_mc_intrs, + .intr_nonstall = true, .reset = gk104_mc_reset, .unk260 = gf100_mc_unk260, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c index 03590292749ac..931985343ab52 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c @@ -26,11 +26,9 @@ static const struct nvkm_mc_func gk20a_mc = { .init = nv50_mc_init, - .intr = gk104_mc_intr, - .intr_unarm = gf100_mc_intr_unarm, - .intr_rearm = gf100_mc_intr_rearm, - .intr_mask = gf100_mc_intr_mask, - .intr_stat = gf100_mc_intr_stat, + .intr = >215_mc_intr, + .intrs = gk104_mc_intrs, + .intr_nonstall = true, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c index 5fd1a0595c334..5dfdf75af255c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c @@ -21,108 +21,80 @@ * * Authors: Ben Skeggs */ -#define gp100_mc(p) container_of((p), struct gp100_mc, base) #include "priv.h" -struct gp100_mc { - struct nvkm_mc base; - spinlock_t lock; - bool intr; - u32 mask; +const struct nvkm_intr_data +gp100_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_SUBDEV_FAULT , 0, 0, 0x00000200, true }, + { NVKM_SUBDEV_PRIVRING, 0, 0, 0x40000000, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x08002000, true }, + { NVKM_SUBDEV_LTC , 0, 0, 0x02000000, true }, + { NVKM_SUBDEV_PMU , 0, 0, 0x01000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, + { NVKM_SUBDEV_THERM , 0, 0, 0x00040000, true }, + { NVKM_SUBDEV_TOP , 0, 0, 0xffffffff, true }, + {}, }; static void -gp100_mc_intr_update(struct gp100_mc *mc) +gp100_mc_intr_allow(struct nvkm_intr *intr, int leaf, u32 mask) { - struct nvkm_device *device = mc->base.subdev.device; - u32 mask = mc->intr ? mc->mask : 0, i; - for (i = 0; i < 2; i++) { - nvkm_wr32(device, 0x000180 + (i * 0x04), ~mask); - nvkm_wr32(device, 0x000160 + (i * 0x04), mask); - } + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + + nvkm_wr32(mc->subdev.device, 0x000160 + (leaf * 4), mask); } -void -gp100_mc_intr_unarm(struct nvkm_mc *base) +static void +gp100_mc_intr_block(struct nvkm_intr *intr, int leaf, u32 mask) { - struct gp100_mc *mc = gp100_mc(base); - unsigned long flags; - spin_lock_irqsave(&mc->lock, flags); - mc->intr = false; - gp100_mc_intr_update(mc); - spin_unlock_irqrestore(&mc->lock, flags); + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + + nvkm_wr32(mc->subdev.device, 0x000180 + (leaf * 4), mask); } -void -gp100_mc_intr_rearm(struct nvkm_mc *base) +static void +gp100_mc_intr_rearm(struct nvkm_intr *intr) { - struct gp100_mc *mc = gp100_mc(base); - unsigned long flags; - spin_lock_irqsave(&mc->lock, flags); - mc->intr = true; - gp100_mc_intr_update(mc); - spin_unlock_irqrestore(&mc->lock, flags); + int i; + + for (i = 0; i < intr->leaves; i++) + intr->func->allow(intr, i, intr->mask[i]); } -void -gp100_mc_intr_mask(struct nvkm_mc *base, u32 mask, u32 intr) +static void +gp100_mc_intr_unarm(struct nvkm_intr *intr) { - struct gp100_mc *mc = gp100_mc(base); - unsigned long flags; - spin_lock_irqsave(&mc->lock, flags); - mc->mask = (mc->mask & ~mask) | intr; - gp100_mc_intr_update(mc); - spin_unlock_irqrestore(&mc->lock, flags); + int i; + + for (i = 0; i < intr->leaves; i++) + intr->func->block(intr, i, 0xffffffff); } -const struct nvkm_mc_map -gp100_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000200, NVKM_SUBDEV_FAULT }, - { 0x40000000, NVKM_SUBDEV_PRIVRING }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x08000000, NVKM_SUBDEV_FB }, - { 0x02000000, NVKM_SUBDEV_LTC }, - { 0x01000000, NVKM_SUBDEV_PMU }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, - { 0x00040000, NVKM_SUBDEV_THERM }, - { 0x00002000, NVKM_SUBDEV_FB }, - {}, +const struct nvkm_intr_func +gp100_mc_intr = { + .pending = nv04_mc_intr_pending, + .unarm = gp100_mc_intr_unarm, + .rearm = gp100_mc_intr_rearm, + .block = gp100_mc_intr_block, + .allow = gp100_mc_intr_allow, }; static const struct nvkm_mc_func gp100_mc = { .init = nv50_mc_init, - .intr = gp100_mc_intr, - .intr_unarm = gp100_mc_intr_unarm, - .intr_rearm = gp100_mc_intr_rearm, - .intr_mask = gp100_mc_intr_mask, - .intr_stat = gf100_mc_intr_stat, + .intr = &gp100_mc_intr, + .intrs = gp100_mc_intrs, + .intr_nonstall = true, .reset = gk104_mc_reset, }; -int -gp100_mc_new_(const struct nvkm_mc_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_mc **pmc) -{ - struct gp100_mc *mc; - - if (!(mc = kzalloc(sizeof(*mc), GFP_KERNEL))) - return -ENOMEM; - nvkm_mc_ctor(func, device, type, inst, &mc->base); - *pmc = &mc->base; - - spin_lock_init(&mc->lock); - mc->intr = false; - mc->mask = 0x7fffffff; - return 0; -} - int gp100_mc_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_mc **pmc) { - return gp100_mc_new_(&gp100_mc, device, type, inst, pmc); + return nvkm_mc_new_(&gp100_mc, device, type, inst, pmc); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c index dd581d030cede..6b83c6bf308c0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c @@ -34,16 +34,14 @@ gp10b_mc_init(struct nvkm_mc *mc) static const struct nvkm_mc_func gp10b_mc = { .init = gp10b_mc_init, - .intr = gp100_mc_intr, - .intr_unarm = gp100_mc_intr_unarm, - .intr_rearm = gp100_mc_intr_rearm, - .intr_mask = gp100_mc_intr_mask, - .intr_stat = gf100_mc_intr_stat, + .intr = &gp100_mc_intr, + .intrs = gp100_mc_intrs, + .intr_nonstall = true, .reset = gk104_mc_reset, }; int gp10b_mc_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_mc **pmc) { - return gp100_mc_new_(&gp10b_mc, device, type, inst, pmc); + return nvkm_mc_new_(&gp10b_mc, device, type, inst, pmc); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c index 1b4d43531dbac..330ef92f5e504 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c @@ -34,39 +34,55 @@ gt215_mc_reset[] = { {} }; -static const struct nvkm_mc_map -gt215_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00400000, NVKM_ENGINE_CE, 0 }, - { 0x00020000, NVKM_ENGINE_MSPDEC }, - { 0x00008000, NVKM_ENGINE_MSVLD }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000001, NVKM_ENGINE_MSPPP }, - { 0x00429101, NVKM_SUBDEV_FB }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, - { 0x00080000, NVKM_SUBDEV_THERM }, - { 0x00040000, NVKM_SUBDEV_PMU }, +static const struct nvkm_intr_data +gt215_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_CE , 0, 0, 0x00400000, true }, + { NVKM_ENGINE_MSPDEC, 0, 0, 0x00020000, true }, + { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_MSPPP , 0, 0, 0x00000001, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x00429101, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, + { NVKM_SUBDEV_THERM , 0, 0, 0x00080000, true }, + { NVKM_SUBDEV_PMU , 0, 0, 0x00040000, true }, {}, }; static void -gt215_mc_intr_mask(struct nvkm_mc *mc, u32 mask, u32 stat) +gt215_mc_intr_allow(struct nvkm_intr *intr, int leaf, u32 mask) +{ + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + + nvkm_mask(mc->subdev.device, 0x000640 + (leaf * 4), mask, mask); +} + +static void +gt215_mc_intr_block(struct nvkm_intr *intr, int leaf, u32 mask) { - nvkm_mask(mc->subdev.device, 0x000640, mask, stat); + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + + nvkm_mask(mc->subdev.device, 0x000640 + (leaf * 4), mask, 0); } +const struct nvkm_intr_func +gt215_mc_intr = { + .pending = nv04_mc_intr_pending, + .unarm = nv04_mc_intr_unarm, + .rearm = nv04_mc_intr_rearm, + .block = gt215_mc_intr_block, + .allow = gt215_mc_intr_allow, +}; + static const struct nvkm_mc_func gt215_mc = { .init = nv50_mc_init, - .intr = gt215_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_mask = gt215_mc_intr_mask, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = gt215_mc_intrs, .reset = gt215_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c index bc0d09bafa99c..00adf6833d265 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c @@ -30,37 +30,61 @@ nv04_mc_reset[] = { {} }; -static const struct nvkm_mc_map -nv04_mc_intr[] = { - { 0x01010000, NVKM_ENGINE_DISP }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00100000, NVKM_SUBDEV_TIMER }, +static const struct nvkm_intr_data +nv04_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x01010000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, {} }; void -nv04_mc_intr_unarm(struct nvkm_mc *mc) +nv04_mc_intr_rearm(struct nvkm_intr *intr) { - struct nvkm_device *device = mc->subdev.device; - nvkm_wr32(device, 0x000140, 0x00000000); - nvkm_rd32(device, 0x000140); + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + int leaf; + + for (leaf = 0; leaf < intr->leaves; leaf++) + nvkm_wr32(mc->subdev.device, 0x000140 + (leaf * 4), 0x00000001); } void -nv04_mc_intr_rearm(struct nvkm_mc *mc) +nv04_mc_intr_unarm(struct nvkm_intr *intr) { - struct nvkm_device *device = mc->subdev.device; - nvkm_wr32(device, 0x000140, 0x00000001); + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + int leaf; + + for (leaf = 0; leaf < intr->leaves; leaf++) + nvkm_wr32(mc->subdev.device, 0x000140 + (leaf * 4), 0x00000000); + + nvkm_rd32(mc->subdev.device, 0x000140); } -u32 -nv04_mc_intr_stat(struct nvkm_mc *mc) +bool +nv04_mc_intr_pending(struct nvkm_intr *intr) { - return nvkm_rd32(mc->subdev.device, 0x000100); + struct nvkm_mc *mc = container_of(intr, typeof(*mc), intr); + bool pending = false; + int leaf; + + for (leaf = 0; leaf < intr->leaves; leaf++) { + intr->stat[leaf] = nvkm_rd32(mc->subdev.device, 0x000100 + (leaf * 4)); + if (intr->stat[leaf]) + pending = true; + } + + return pending; } +const struct nvkm_intr_func +nv04_mc_intr = { + .pending = nv04_mc_intr_pending, + .unarm = nv04_mc_intr_unarm, + .rearm = nv04_mc_intr_rearm, +}; + void nv04_mc_init(struct nvkm_mc *mc) { @@ -72,10 +96,8 @@ nv04_mc_init(struct nvkm_mc *mc) static const struct nvkm_mc_func nv04_mc = { .init = nv04_mc_init, - .intr = nv04_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = nv04_mc_intrs, .reset = nv04_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c index ab59ca1ee0682..1cef78827f570 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c @@ -23,23 +23,21 @@ */ #include "priv.h" -static const struct nvkm_mc_map -nv11_mc_intr[] = { - { 0x03010000, NVKM_ENGINE_DISP }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00100000, NVKM_SUBDEV_TIMER }, +static const struct nvkm_intr_data +nv11_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x03010000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, {} }; static const struct nvkm_mc_func nv11_mc = { .init = nv04_mc_init, - .intr = nv11_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = nv11_mc_intrs, .reset = nv04_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c index 03d756e26e578..ac70c359bce0a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c @@ -31,24 +31,22 @@ nv17_mc_reset[] = { {} }; -const struct nvkm_mc_map -nv17_mc_intr[] = { - { 0x03010000, NVKM_ENGINE_DISP }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000001, NVKM_ENGINE_MPEG }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00100000, NVKM_SUBDEV_TIMER }, +const struct nvkm_intr_data +nv17_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x03010000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_MPEG , 0, 0, 0x00000001, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, {} }; static const struct nvkm_mc_func nv17_mc = { .init = nv04_mc_init, - .intr = nv17_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = nv17_mc_intrs, .reset = nv17_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c index 95f65766e8b00..0a054459a949a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c @@ -40,10 +40,8 @@ nv44_mc_init(struct nvkm_mc *mc) static const struct nvkm_mc_func nv44_mc = { .init = nv44_mc_init, - .intr = nv17_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = nv17_mc_intrs, .reset = nv17_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c index fce3613cdfa52..6a02ed754c3fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c @@ -23,17 +23,17 @@ */ #include "priv.h" -static const struct nvkm_mc_map -nv50_mc_intr[] = { - { 0x04000000, NVKM_ENGINE_DISP }, - { 0x00001000, NVKM_ENGINE_GR }, - { 0x00000100, NVKM_ENGINE_FIFO }, - { 0x00000001, NVKM_ENGINE_MPEG }, - { 0x00001101, NVKM_SUBDEV_FB }, - { 0x10000000, NVKM_SUBDEV_BUS }, - { 0x00200000, NVKM_SUBDEV_GPIO }, - { 0x00200000, NVKM_SUBDEV_I2C }, - { 0x00100000, NVKM_SUBDEV_TIMER }, +static const struct nvkm_intr_data +nv50_mc_intrs[] = { + { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_MPEG , 0, 0, 0x00000001, true }, + { NVKM_SUBDEV_FB , 0, 0, 0x00001101, true }, + { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, + { NVKM_SUBDEV_GPIO , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, + { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, {}, }; @@ -47,10 +47,8 @@ nv50_mc_init(struct nvkm_mc *mc) static const struct nvkm_mc_func nv50_mc = { .init = nv50_mc_init, - .intr = nv50_mc_intr, - .intr_unarm = nv04_mc_intr_unarm, - .intr_rearm = nv04_mc_intr_rearm, - .intr_stat = nv04_mc_intr_stat, + .intr = &nv04_mc_intr, + .intrs = nv50_mc_intrs, .reset = nv17_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h index c8bcabb98f999..3ecfe9b4fb05f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h @@ -4,8 +4,6 @@ #define nvkm_mc(p) container_of((p), struct nvkm_mc, subdev) #include -void nvkm_mc_ctor(const struct nvkm_mc_func *, struct nvkm_device *, enum nvkm_subdev_type, int, - struct nvkm_mc *); int nvkm_mc_new_(const struct nvkm_mc_func *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_mc **); @@ -18,46 +16,36 @@ struct nvkm_mc_map { struct nvkm_mc_func { void (*init)(struct nvkm_mc *); - const struct nvkm_mc_map *intr; - /* disable reporting of interrupts to host */ - void (*intr_unarm)(struct nvkm_mc *); - /* enable reporting of interrupts to host */ - void (*intr_rearm)(struct nvkm_mc *); - /* (un)mask delivery of specific interrupts */ - void (*intr_mask)(struct nvkm_mc *, u32 mask, u32 stat); - /* retrieve pending interrupt mask (NV_PMC_INTR) */ - u32 (*intr_stat)(struct nvkm_mc *); + + const struct nvkm_intr_func *intr; + const struct nvkm_intr_data *intrs; + bool intr_nonstall; + const struct nvkm_mc_map *reset; void (*unk260)(struct nvkm_mc *, u32); }; void nv04_mc_init(struct nvkm_mc *); -void nv04_mc_intr_unarm(struct nvkm_mc *); -void nv04_mc_intr_rearm(struct nvkm_mc *); -u32 nv04_mc_intr_stat(struct nvkm_mc *); +extern const struct nvkm_intr_func nv04_mc_intr; +bool nv04_mc_intr_pending(struct nvkm_intr *); +void nv04_mc_intr_unarm(struct nvkm_intr *); +void nv04_mc_intr_rearm(struct nvkm_intr *); extern const struct nvkm_mc_map nv04_mc_reset[]; -extern const struct nvkm_mc_map nv17_mc_intr[]; +extern const struct nvkm_intr_data nv17_mc_intrs[]; extern const struct nvkm_mc_map nv17_mc_reset[]; void nv44_mc_init(struct nvkm_mc *); void nv50_mc_init(struct nvkm_mc *); -void gk104_mc_init(struct nvkm_mc *); -void gf100_mc_intr_unarm(struct nvkm_mc *); -void gf100_mc_intr_rearm(struct nvkm_mc *); -void gf100_mc_intr_mask(struct nvkm_mc *, u32, u32); -u32 gf100_mc_intr_stat(struct nvkm_mc *); +extern const struct nvkm_intr_func gt215_mc_intr; void gf100_mc_unk260(struct nvkm_mc *, u32); -void gp100_mc_intr_unarm(struct nvkm_mc *); -void gp100_mc_intr_rearm(struct nvkm_mc *); -void gp100_mc_intr_mask(struct nvkm_mc *, u32, u32); -int gp100_mc_new_(const struct nvkm_mc_func *, struct nvkm_device *, enum nvkm_subdev_type, int, - struct nvkm_mc **); -extern const struct nvkm_mc_map gk104_mc_intr[]; +void gk104_mc_init(struct nvkm_mc *); +extern const struct nvkm_intr_data gk104_mc_intrs[]; extern const struct nvkm_mc_map gk104_mc_reset[]; -extern const struct nvkm_mc_map gp100_mc_intr[]; +extern const struct nvkm_intr_func gp100_mc_intr; +extern const struct nvkm_intr_data gp100_mc_intrs[]; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c deleted file mode 100644 index 630a9c9e7ac97..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/tu102.c +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright 2018 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#define tu102_mc(p) container_of((p), struct tu102_mc, base) -#include "priv.h" - -struct tu102_mc { - struct nvkm_mc base; - spinlock_t lock; - bool intr; - u32 mask; -}; - -static void -tu102_mc_intr_update(struct tu102_mc *mc) -{ - struct nvkm_device *device = mc->base.subdev.device; - u32 mask = mc->intr ? mc->mask : 0, i; - - for (i = 0; i < 2; i++) { - nvkm_wr32(device, 0x000180 + (i * 0x04), ~mask); - nvkm_wr32(device, 0x000160 + (i * 0x04), mask); - } -} - -static void -tu102_mc_intr_unarm(struct nvkm_mc *base) -{ - struct tu102_mc *mc = tu102_mc(base); - unsigned long flags; - - spin_lock_irqsave(&mc->lock, flags); - mc->intr = false; - tu102_mc_intr_update(mc); - spin_unlock_irqrestore(&mc->lock, flags); -} - -static void -tu102_mc_intr_rearm(struct nvkm_mc *base) -{ - struct tu102_mc *mc = tu102_mc(base); - unsigned long flags; - - spin_lock_irqsave(&mc->lock, flags); - mc->intr = true; - tu102_mc_intr_update(mc); - spin_unlock_irqrestore(&mc->lock, flags); -} - -static void -tu102_mc_intr_mask(struct nvkm_mc *base, u32 mask, u32 intr) -{ - struct tu102_mc *mc = tu102_mc(base); - unsigned long flags; - - spin_lock_irqsave(&mc->lock, flags); - mc->mask = (mc->mask & ~mask) | intr; - tu102_mc_intr_update(mc); - spin_unlock_irqrestore(&mc->lock, flags); -} - -static u32 -tu102_mc_intr_stat(struct nvkm_mc *mc) -{ - struct nvkm_device *device = mc->subdev.device; - u32 intr0 = nvkm_rd32(device, 0x000100); - u32 intr1 = nvkm_rd32(device, 0x000104); - - return intr0 | intr1; -} - - -static const struct nvkm_mc_func -tu102_mc = { - .init = nv50_mc_init, - .intr = gp100_mc_intr, - .intr_unarm = tu102_mc_intr_unarm, - .intr_rearm = tu102_mc_intr_rearm, - .intr_mask = tu102_mc_intr_mask, - .intr_stat = tu102_mc_intr_stat, - .reset = gk104_mc_reset, -}; - -static int -tu102_mc_new_(const struct nvkm_mc_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_mc **pmc) -{ - struct tu102_mc *mc; - - if (!(mc = kzalloc(sizeof(*mc), GFP_KERNEL))) - return -ENOMEM; - nvkm_mc_ctor(func, device, type, inst, &mc->base); - *pmc = &mc->base; - - spin_lock_init(&mc->lock); - mc->intr = false; - mc->mask = 0x7fffffff; - return 0; -} - -int -tu102_mc_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_mc **pmc) -{ - return tu102_mc_new_(&tu102_mc, device, type, inst, pmc); -} -- GitLab From ebb195dbb3390324100cb85c22129f6334064b5f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:55 +1000 Subject: [PATCH 1296/1927] drm/nouveau/mc: move NV_PMC_ENABLE bashing to chipset-specific code Ampere needs different handling here, most of what we touch has moved. We probably want to refactor these interfaces in general, but I'm not yet sure how they should look, this will get the job done for now. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c | 17 ++++------- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/ga100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/gf100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/gk104.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/gp100.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/mc/gt215.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c | 29 +++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c | 1 + drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h | 8 +++++ 16 files changed, 56 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c index 322237e2ff80e..c85600ba69f9d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c @@ -73,9 +73,8 @@ nvkm_mc_reset(struct nvkm_device *device, enum nvkm_subdev_type type, int inst) { u64 pmc_enable = nvkm_mc_reset_mask(device, true, type, inst); if (pmc_enable) { - nvkm_mask(device, 0x000200, pmc_enable, 0x00000000); - nvkm_mask(device, 0x000200, pmc_enable, pmc_enable); - nvkm_rd32(device, 0x000200); + device->mc->func->device->disable(device->mc, pmc_enable); + device->mc->func->device->enable(device->mc, pmc_enable); } } @@ -84,17 +83,15 @@ nvkm_mc_disable(struct nvkm_device *device, enum nvkm_subdev_type type, int inst { u64 pmc_enable = nvkm_mc_reset_mask(device, false, type, inst); if (pmc_enable) - nvkm_mask(device, 0x000200, pmc_enable, 0x00000000); + device->mc->func->device->disable(device->mc, pmc_enable); } void nvkm_mc_enable(struct nvkm_device *device, enum nvkm_subdev_type type, int inst) { u64 pmc_enable = nvkm_mc_reset_mask(device, false, type, inst); - if (pmc_enable) { - nvkm_mask(device, 0x000200, pmc_enable, pmc_enable); - nvkm_rd32(device, 0x000200); - } + if (pmc_enable) + device->mc->func->device->enable(device->mc, pmc_enable); } bool @@ -102,11 +99,9 @@ nvkm_mc_enabled(struct nvkm_device *device, enum nvkm_subdev_type type, int inst { u64 pmc_enable = nvkm_mc_reset_mask(device, false, type, inst); - return (pmc_enable != 0) && - ((nvkm_rd32(device, 0x000200) & pmc_enable) == pmc_enable); + return (pmc_enable != 0) && device->mc->func->device->enabled(device->mc, pmc_enable); } - static int nvkm_mc_init(struct nvkm_subdev *subdev) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c index 8a8267ecfe1d8..9feb7d167d236 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c @@ -56,6 +56,7 @@ g84_mc = { .init = nv50_mc_init, .intr = &nv04_mc_intr, .intrs = g84_mc_intrs, + .device = &nv04_mc_device, .reset = g84_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c index e099d419f1ce5..b0b328044e55a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c @@ -56,6 +56,7 @@ g98_mc = { .init = nv50_mc_init, .intr = &nv04_mc_intr, .intrs = g98_mc_intrs, + .device = &nv04_mc_device, .reset = g98_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c index 98ccce9585ebc..842326a243935 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c @@ -24,6 +24,7 @@ static const struct nvkm_mc_func ga100_mc = { .init = nv50_mc_init, + .device = &nv04_mc_device, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c index b2846eaea34c4..d5dba44235cf3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c @@ -71,6 +71,7 @@ gf100_mc = { .intrs = gf100_mc_intrs, .intr_nonstall = true, .reset = gf100_mc_reset, + .device = &nv04_mc_device, .unk260 = gf100_mc_unk260, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c index edf82e4568bf9..023dfda735cda 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c @@ -54,6 +54,7 @@ gk104_mc = { .intrs = gk104_mc_intrs, .intr_nonstall = true, .reset = gk104_mc_reset, + .device = &nv04_mc_device, .unk260 = gf100_mc_unk260, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c index 931985343ab52..d98a6563a4117 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c @@ -29,6 +29,7 @@ gk20a_mc = { .intr = >215_mc_intr, .intrs = gk104_mc_intrs, .intr_nonstall = true, + .device = &nv04_mc_device, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c index 5dfdf75af255c..bffde40bb237d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c @@ -90,6 +90,7 @@ gp100_mc = { .intr = &gp100_mc_intr, .intrs = gp100_mc_intrs, .intr_nonstall = true, + .device = &nv04_mc_device, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c index 6b83c6bf308c0..9bed9c5ea5d39 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp10b.c @@ -37,6 +37,7 @@ gp10b_mc = { .intr = &gp100_mc_intr, .intrs = gp100_mc_intrs, .intr_nonstall = true, + .device = &nv04_mc_device, .reset = gk104_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c index 330ef92f5e504..41e17a4703885 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c @@ -83,6 +83,7 @@ gt215_mc = { .init = nv50_mc_init, .intr = &nv04_mc_intr, .intrs = gt215_mc_intrs, + .device = &nv04_mc_device, .reset = gt215_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c index 00adf6833d265..04809caf36d6a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c @@ -30,6 +30,34 @@ nv04_mc_reset[] = { {} }; +static void +nv04_mc_device_disable(struct nvkm_mc *mc, u32 mask) +{ + nvkm_mask(mc->subdev.device, 0x000200, mask, 0x00000000); +} + +static void +nv04_mc_device_enable(struct nvkm_mc *mc, u32 mask) +{ + struct nvkm_device *device = mc->subdev.device; + + nvkm_mask(device, 0x000200, mask, mask); + nvkm_rd32(device, 0x000200); +} + +static bool +nv04_mc_device_enabled(struct nvkm_mc *mc, u32 mask) +{ + return (nvkm_rd32(mc->subdev.device, 0x000200) & mask) == mask; +} + +const struct nvkm_mc_device_func +nv04_mc_device = { + .enabled = nv04_mc_device_enabled, + .enable = nv04_mc_device_enable, + .disable = nv04_mc_device_disable, +}; + static const struct nvkm_intr_data nv04_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x01010000, true }, @@ -98,6 +126,7 @@ nv04_mc = { .init = nv04_mc_init, .intr = &nv04_mc_intr, .intrs = nv04_mc_intrs, + .device = &nv04_mc_device, .reset = nv04_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c index 1cef78827f570..6a3e2f4753ef0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c @@ -38,6 +38,7 @@ nv11_mc = { .init = nv04_mc_init, .intr = &nv04_mc_intr, .intrs = nv11_mc_intrs, + .device = &nv04_mc_device, .reset = nv04_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c index ac70c359bce0a..2edc1cb0e379c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c @@ -47,6 +47,7 @@ nv17_mc = { .init = nv04_mc_init, .intr = &nv04_mc_intr, .intrs = nv17_mc_intrs, + .device = &nv04_mc_device, .reset = nv17_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c index 0a054459a949a..649a9fcc0a2fe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c @@ -42,6 +42,7 @@ nv44_mc = { .init = nv44_mc_init, .intr = &nv04_mc_intr, .intrs = nv17_mc_intrs, + .device = &nv04_mc_device, .reset = nv17_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c index 6a02ed754c3fb..a42d20206e37e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c @@ -49,6 +49,7 @@ nv50_mc = { .init = nv50_mc_init, .intr = &nv04_mc_intr, .intrs = nv50_mc_intrs, + .device = &nv04_mc_device, .reset = nv17_mc_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h index 3ecfe9b4fb05f..7f38d54b4bc2a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h @@ -21,7 +21,14 @@ struct nvkm_mc_func { const struct nvkm_intr_data *intrs; bool intr_nonstall; + const struct nvkm_mc_device_func { + bool (*enabled)(struct nvkm_mc *, u32 mask); + void (*enable)(struct nvkm_mc *, u32 mask); + void (*disable)(struct nvkm_mc *, u32 mask); + } *device; + const struct nvkm_mc_map *reset; + void (*unk260)(struct nvkm_mc *, u32); }; @@ -30,6 +37,7 @@ extern const struct nvkm_intr_func nv04_mc_intr; bool nv04_mc_intr_pending(struct nvkm_intr *); void nv04_mc_intr_unarm(struct nvkm_intr *); void nv04_mc_intr_rearm(struct nvkm_intr *); +extern const struct nvkm_mc_device_func nv04_mc_device; extern const struct nvkm_mc_map nv04_mc_reset[]; extern const struct nvkm_intr_data nv17_mc_intrs[]; -- GitLab From 565bfaf1f26af0e9fc9aafbb7053da1187afe9f4 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:46:55 +1000 Subject: [PATCH 1297/1927] drm/nouveau/mc/ga100: switch to using NV_PMC_DEVICE_ENABLE - NV_PMC_ENABLE still exists, but we don't touch anything in it yet Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/subdev/mc/ga100.c | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c index 842326a243935..1e2eabec1a76c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/ga100.c @@ -21,11 +21,52 @@ */ #include "priv.h" +static void +ga100_mc_device_disable(struct nvkm_mc *mc, u32 mask) +{ + struct nvkm_device *device = mc->subdev.device; + + nvkm_mask(device, 0x000600, mask, 0x00000000); + nvkm_rd32(device, 0x000600); + nvkm_rd32(device, 0x000600); +} + +static void +ga100_mc_device_enable(struct nvkm_mc *mc, u32 mask) +{ + struct nvkm_device *device = mc->subdev.device; + + nvkm_mask(device, 0x000600, mask, mask); + nvkm_rd32(device, 0x000600); + nvkm_rd32(device, 0x000600); +} + +static bool +ga100_mc_device_enabled(struct nvkm_mc *mc, u32 mask) +{ + return (nvkm_rd32(mc->subdev.device, 0x000600) & mask) == mask; +} + +const struct nvkm_mc_device_func +ga100_mc_device = { + .enabled = ga100_mc_device_enabled, + .enable = ga100_mc_device_enable, + .disable = ga100_mc_device_disable, +}; + +static void +ga100_mc_init(struct nvkm_mc *mc) +{ + struct nvkm_device *device = mc->subdev.device; + + nvkm_wr32(device, 0x000200, 0xffffffff); + nvkm_wr32(device, 0x000600, 0xffffffff); +} + static const struct nvkm_mc_func ga100_mc = { - .init = nv50_mc_init, - .device = &nv04_mc_device, - .reset = gk104_mc_reset, + .init = ga100_mc_init, + .device = &ga100_mc_device, }; int -- GitLab From 8478cd5a740a092163c8ad5b6da1a1b488eb42bd Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:19 +1000 Subject: [PATCH 1298/1927] drm/nouveau/nvkm: add locking to subdev/engine init paths This wasn't really needed before; the main place this could race is with channel recovery, but (through potentially fragile means) shouldn't have been possible. However, a number of upcoming patches benefit from having better control over subdev init, necessitating some improvements here. - allows subdev/engine oneinit() without init() (host/fifo patches) - merges engine use locking/tracking into subdev, and extends it to fix some issues that will arise with future usage patterns (acr patches) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/engine.h | 6 - .../drm/nouveau/include/nvkm/core/subdev.h | 25 +++- drivers/gpu/drm/nouveau/nvkm/core/engine.c | 63 ++++------ drivers/gpu/drm/nouveau/nvkm/core/subdev.c | 117 +++++++++++++++--- 4 files changed, 142 insertions(+), 69 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h index e58923b67d743..6d15c13509bfb 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h @@ -12,12 +12,6 @@ struct nvkm_engine { const struct nvkm_engine_func *func; struct nvkm_subdev subdev; spinlock_t lock; - - struct { - refcount_t refcount; - struct mutex mutex; - bool enabled; - } use; }; struct nvkm_engine_func { diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h index 20e1fc90c536e..f920a2de17353 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h @@ -17,12 +17,19 @@ struct nvkm_subdev { struct nvkm_device *device; enum nvkm_subdev_type type; int inst; + char name[16]; u32 debug; - struct list_head head; + + struct { + refcount_t refcount; + struct mutex mutex; + bool enabled; + } use; struct nvkm_inth inth; + struct list_head head; void **pself; bool oneinit; }; @@ -40,11 +47,23 @@ struct nvkm_subdev_func { extern const char *nvkm_subdev_type[NVKM_SUBDEV_NR]; int nvkm_subdev_new_(const struct nvkm_subdev_func *, struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_subdev **); -void nvkm_subdev_ctor(const struct nvkm_subdev_func *, struct nvkm_device *, - enum nvkm_subdev_type, int inst, struct nvkm_subdev *); +void __nvkm_subdev_ctor(const struct nvkm_subdev_func *, struct nvkm_device *, + enum nvkm_subdev_type, int inst, struct nvkm_subdev *); + +static inline void +nvkm_subdev_ctor(const struct nvkm_subdev_func *func, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_subdev *subdev) +{ + __nvkm_subdev_ctor(func, device, type, inst, subdev); + mutex_init(&subdev->use.mutex); +} + void nvkm_subdev_disable(struct nvkm_device *, enum nvkm_subdev_type, int inst); void nvkm_subdev_del(struct nvkm_subdev **); +int nvkm_subdev_ref(struct nvkm_subdev *); +void nvkm_subdev_unref(struct nvkm_subdev *); int nvkm_subdev_preinit(struct nvkm_subdev *); +int nvkm_subdev_oneinit(struct nvkm_subdev *); int nvkm_subdev_init(struct nvkm_subdev *); int nvkm_subdev_fini(struct nvkm_subdev *, bool suspend); int nvkm_subdev_info(struct nvkm_subdev *, u64, u64 *); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c index e41a39ae1597a..558bd10e55188 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c @@ -39,12 +39,9 @@ void nvkm_engine_unref(struct nvkm_engine **pengine) { struct nvkm_engine *engine = *pengine; + if (engine) { - if (refcount_dec_and_mutex_lock(&engine->use.refcount, &engine->use.mutex)) { - nvkm_subdev_fini(&engine->subdev, false); - engine->use.enabled = false; - mutex_unlock(&engine->use.mutex); - } + nvkm_subdev_unref(&engine->subdev); *pengine = NULL; } } @@ -53,21 +50,13 @@ struct nvkm_engine * nvkm_engine_ref(struct nvkm_engine *engine) { int ret; + if (engine) { - if (!refcount_inc_not_zero(&engine->use.refcount)) { - mutex_lock(&engine->use.mutex); - if (!refcount_inc_not_zero(&engine->use.refcount)) { - engine->use.enabled = true; - if ((ret = nvkm_subdev_init(&engine->subdev))) { - engine->use.enabled = false; - mutex_unlock(&engine->use.mutex); - return ERR_PTR(ret); - } - refcount_set(&engine->use.refcount, 1); - } - mutex_unlock(&engine->use.mutex); - } + ret = nvkm_subdev_ref(&engine->subdev); + if (ret) + return ERR_PTR(ret); } + return engine; } @@ -117,26 +106,6 @@ nvkm_engine_init(struct nvkm_subdev *subdev) struct nvkm_engine *engine = nvkm_engine(subdev); struct nvkm_fb *fb = subdev->device->fb; int ret = 0, i; - s64 time; - - if (!engine->use.enabled) { - nvkm_trace(subdev, "init skipped, engine has no users\n"); - return ret; - } - - if (engine->func->oneinit && !engine->subdev.oneinit) { - nvkm_trace(subdev, "one-time init running...\n"); - time = ktime_to_us(ktime_get()); - ret = engine->func->oneinit(engine); - if (ret) { - nvkm_trace(subdev, "one-time init failed, %d\n", ret); - return ret; - } - - engine->subdev.oneinit = true; - time = ktime_to_us(ktime_get()) - time; - nvkm_trace(subdev, "one-time init completed in %lldus\n", time); - } if (engine->func->init) ret = engine->func->init(engine); @@ -146,6 +115,17 @@ nvkm_engine_init(struct nvkm_subdev *subdev) return ret; } +static int +nvkm_engine_oneinit(struct nvkm_subdev *subdev) +{ + struct nvkm_engine *engine = nvkm_engine(subdev); + + if (engine->func->oneinit) + return engine->func->oneinit(engine); + + return 0; +} + static int nvkm_engine_preinit(struct nvkm_subdev *subdev) { @@ -161,7 +141,6 @@ nvkm_engine_dtor(struct nvkm_subdev *subdev) struct nvkm_engine *engine = nvkm_engine(subdev); if (engine->func->dtor) return engine->func->dtor(engine); - mutex_destroy(&engine->use.mutex); return engine; } @@ -169,6 +148,7 @@ const struct nvkm_subdev_func nvkm_engine = { .dtor = nvkm_engine_dtor, .preinit = nvkm_engine_preinit, + .oneinit = nvkm_engine_oneinit, .init = nvkm_engine_init, .fini = nvkm_engine_fini, .info = nvkm_engine_info, @@ -179,10 +159,9 @@ int nvkm_engine_ctor(const struct nvkm_engine_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, bool enable, struct nvkm_engine *engine) { - nvkm_subdev_ctor(&nvkm_engine, device, type, inst, &engine->subdev); engine->func = func; - refcount_set(&engine->use.refcount, 0); - mutex_init(&engine->use.mutex); + nvkm_subdev_ctor(&nvkm_engine, device, type, inst, &engine->subdev); + refcount_set(&engine->subdev.use.refcount, 0); if (!nvkm_boolopt(device->cfgopt, engine->subdev.name, enable)) { nvkm_debug(&engine->subdev, "disabled\n"); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c index a74b7acb6832e..6c20e827a069a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c @@ -54,7 +54,7 @@ int nvkm_subdev_fini(struct nvkm_subdev *subdev, bool suspend) { struct nvkm_device *device = subdev->device; - const char *action = suspend ? "suspend" : "fini"; + const char *action = suspend ? "suspend" : subdev->use.enabled ? "fini" : "reset"; s64 time; nvkm_trace(subdev, "%s running...\n", action); @@ -68,6 +68,7 @@ nvkm_subdev_fini(struct nvkm_subdev *subdev, bool suspend) return ret; } } + subdev->use.enabled = false; nvkm_mc_reset(device, subdev->type, subdev->inst); @@ -97,30 +98,49 @@ nvkm_subdev_preinit(struct nvkm_subdev *subdev) return 0; } -int -nvkm_subdev_init(struct nvkm_subdev *subdev) +static int +nvkm_subdev_oneinit_(struct nvkm_subdev *subdev) { s64 time; int ret; - nvkm_trace(subdev, "init running...\n"); + if (!subdev->func->oneinit || subdev->oneinit) + return 0; + + nvkm_trace(subdev, "one-time init running...\n"); time = ktime_to_us(ktime_get()); + ret = subdev->func->oneinit(subdev); + if (ret) { + nvkm_error(subdev, "one-time init failed, %d\n", ret); + return ret; + } - if (subdev->func->oneinit && !subdev->oneinit) { - s64 time; - nvkm_trace(subdev, "one-time init running...\n"); - time = ktime_to_us(ktime_get()); - ret = subdev->func->oneinit(subdev); - if (ret) { - nvkm_error(subdev, "one-time init failed, %d\n", ret); - return ret; - } + subdev->oneinit = true; + time = ktime_to_us(ktime_get()) - time; + nvkm_trace(subdev, "one-time init completed in %lldus\n", time); + return 0; +} - subdev->oneinit = true; - time = ktime_to_us(ktime_get()) - time; - nvkm_trace(subdev, "one-time init completed in %lldus\n", time); +static int +nvkm_subdev_init_(struct nvkm_subdev *subdev) +{ + s64 time; + int ret; + + if (subdev->use.enabled) { + nvkm_trace(subdev, "init skipped, already running\n"); + return 0; } + nvkm_trace(subdev, "init running...\n"); + time = ktime_to_us(ktime_get()); + + ret = nvkm_subdev_oneinit_(subdev); + if (ret) + return ret; + + subdev->use.enabled = true; + if (subdev->func->init) { ret = subdev->func->init(subdev); if (ret) { @@ -134,6 +154,64 @@ nvkm_subdev_init(struct nvkm_subdev *subdev) return 0; } +int +nvkm_subdev_init(struct nvkm_subdev *subdev) +{ + int ret; + + mutex_lock(&subdev->use.mutex); + if (refcount_read(&subdev->use.refcount) == 0) { + nvkm_trace(subdev, "init skipped, no users\n"); + mutex_unlock(&subdev->use.mutex); + return 0; + } + + ret = nvkm_subdev_init_(subdev); + mutex_unlock(&subdev->use.mutex); + return ret; +} + +int +nvkm_subdev_oneinit(struct nvkm_subdev *subdev) +{ + int ret; + + mutex_lock(&subdev->use.mutex); + ret = nvkm_subdev_oneinit_(subdev); + mutex_unlock(&subdev->use.mutex); + return ret; +} + +void +nvkm_subdev_unref(struct nvkm_subdev *subdev) +{ + if (refcount_dec_and_mutex_lock(&subdev->use.refcount, &subdev->use.mutex)) { + nvkm_subdev_fini(subdev, false); + mutex_unlock(&subdev->use.mutex); + } +} + +int +nvkm_subdev_ref(struct nvkm_subdev *subdev) +{ + int ret; + + if (subdev && !refcount_inc_not_zero(&subdev->use.refcount)) { + mutex_lock(&subdev->use.mutex); + if (!refcount_inc_not_zero(&subdev->use.refcount)) { + if ((ret = nvkm_subdev_init_(subdev))) { + mutex_unlock(&subdev->use.mutex); + return ret; + } + + refcount_set(&subdev->use.refcount, 1); + } + mutex_unlock(&subdev->use.mutex); + } + + return 0; +} + void nvkm_subdev_del(struct nvkm_subdev **psubdev) { @@ -146,6 +224,7 @@ nvkm_subdev_del(struct nvkm_subdev **psubdev) list_del(&subdev->head); if (subdev->func->dtor) *psubdev = subdev->func->dtor(subdev); + mutex_destroy(&subdev->use.mutex); time = ktime_to_us(ktime_get()) - time; nvkm_trace(subdev, "destroy completed in %lldus\n", time); kfree(*psubdev); @@ -167,8 +246,8 @@ nvkm_subdev_disable(struct nvkm_device *device, enum nvkm_subdev_type type, int } void -nvkm_subdev_ctor(const struct nvkm_subdev_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_subdev *subdev) +__nvkm_subdev_ctor(const struct nvkm_subdev_func *func, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_subdev *subdev) { subdev->func = func; subdev->device = device; @@ -180,6 +259,8 @@ nvkm_subdev_ctor(const struct nvkm_subdev_func *func, struct nvkm_device *device else strscpy(subdev->name, nvkm_subdev_type[type], sizeof(subdev->name)); subdev->debug = nvkm_dbgopt(device->dbgopt, subdev->name); + + refcount_set(&subdev->use.refcount, 1); list_add_tail(&subdev->head, &device->subdev); } -- GitLab From e442f1e453143c801b9fb213f514ceabb6c3b746 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:20 +1000 Subject: [PATCH 1299/1927] drm/nouveau/flcn: show falcon user in debug output Displays both owner/user of the falcon (when they differ), and takes both subdevs' debug levels into account when deciding whether to log the message. - runlist debugging will use one of the alternate macros added here Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/core/falcon.h | 16 ++++++++-------- .../gpu/drm/nouveau/include/nvkm/core/subdev.h | 15 +++++++++++---- drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h | 7 +++---- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index fd9a3f9a518e8..d647d8acb15b6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -21,14 +21,14 @@ void nvkm_falcon_v1_disable(struct nvkm_falcon *); void gp102_sec2_flcn_bind_context(struct nvkm_falcon *, struct nvkm_memory *); int gp102_sec2_flcn_enable(struct nvkm_falcon *); -#define FLCN_PRINTK(t,f,fmt,a...) do { \ - if ((f)->owner->name != (f)->name) \ - nvkm_##t((f)->owner, "%s: "fmt"\n", (f)->name, ##a); \ - else \ - nvkm_##t((f)->owner, fmt"\n", ##a); \ -} while(0) -#define FLCN_DBG(f,fmt,a...) FLCN_PRINTK(debug, (f), fmt, ##a) -#define FLCN_ERR(f,fmt,a...) FLCN_PRINTK(error, (f), fmt, ##a) +#define FLCN_PRINTK(f,l,p,fmt,a...) ({ \ + if ((f)->owner->name != (f)->name) \ + nvkm_printk___((f)->owner, (f)->user, NV_DBG_##l, p, "%s:"fmt, (f)->name, ##a); \ + else \ + nvkm_printk___((f)->owner, (f)->user, NV_DBG_##l, p, fmt, ##a); \ +}) +#define FLCN_DBG(f,fmt,a...) FLCN_PRINTK((f), DEBUG, info, " "fmt"\n", ##a) +#define FLCN_ERR(f,fmt,a...) FLCN_PRINTK((f), ERROR, err, " "fmt"\n", ##a) /** * struct nvfw_falcon_msg - header for all messages diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h index f920a2de17353..bce6e1ba09ea0 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/subdev.h @@ -70,11 +70,18 @@ int nvkm_subdev_info(struct nvkm_subdev *, u64, u64 *); void nvkm_subdev_intr(struct nvkm_subdev *); /* subdev logging */ -#define nvkm_printk_(s,l,p,f,a...) do { \ - const struct nvkm_subdev *_subdev = (s); \ - if (CONFIG_NOUVEAU_DEBUG >= (l) && _subdev->debug >= (l)) \ - dev_##p(_subdev->device->dev, "%s: "f, _subdev->name, ##a); \ +#define nvkm_printk_ok(s,u,l) \ + ((CONFIG_NOUVEAU_DEBUG >= (l)) && ((s)->debug >= (l) || ((u) && (u)->debug >= (l)))) +#define nvkm_printk___(s,u,l,p,f,a...) do { \ + if (nvkm_printk_ok((s), (u), (l))) { \ + if ((u) && (u) != (s)) \ + dev_##p((s)->device->dev, "%s(%s):"f, (s)->name, (u)->name, ##a); \ + else \ + dev_##p((s)->device->dev, "%s:"f, (s)->name, ##a); \ + } \ } while(0) +#define nvkm_printk__(s,l,p,f,a...) nvkm_printk___((s), (s), (l), p, f, ##a) +#define nvkm_printk_(s,l,p,f,a...) nvkm_printk__((s), (l), p, " "f, ##a) #define nvkm_printk(s,l,p,f,a...) nvkm_printk_((s), NV_DBG_##l, p, f, ##a) #define nvkm_fatal(s,f,a...) nvkm_printk((s), FATAL, crit, f, ##a) #define nvkm_error(s,f,a...) nvkm_printk((s), ERROR, err, f, ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h b/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h index 976cb7b7aa999..33564ac906a8b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h @@ -82,8 +82,7 @@ struct nvkm_falcon_msgq { u32 position; }; -#define FLCNQ_PRINTK(t,q,f,a...) \ - FLCN_PRINTK(t, (q)->qmgr->falcon, "%s: "f, (q)->name, ##a) -#define FLCNQ_DBG(q,f,a...) FLCNQ_PRINTK(debug, (q), f, ##a) -#define FLCNQ_ERR(q,f,a...) FLCNQ_PRINTK(error, (q), f, ##a) +#define FLCNQ_PRINTK(q,l,p,f,a...) FLCN_PRINTK((q)->qmgr->falcon, l, p, "%s: "f, (q)->name, ##a) +#define FLCNQ_DBG(q,f,a...) FLCNQ_PRINTK((q), DEBUG, info, f, ##a) +#define FLCNQ_ERR(q,f,a...) FLCNQ_PRINTK((q), ERROR, err, f, ##a) #endif -- GitLab From 973b32443b090870903ad8346adfc911e7c0f188 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:21 +1000 Subject: [PATCH 1300/1927] drm/nouveau/imem: allow bar2 mapping of user allocations Will be used to init client-allocated USERD to default values. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/memory.h | 2 ++ .../drm/nouveau/include/nvkm/subdev/instmem.h | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c | 8 ++++++ .../drm/nouveau/nvkm/subdev/instmem/base.c | 12 +++++++++ .../drm/nouveau/nvkm/subdev/instmem/nv50.c | 27 +++++++++++++++---- .../drm/nouveau/nvkm/subdev/instmem/priv.h | 1 + 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h index 74d3f1a809d7c..d3b6a68ddda36 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/memory.h @@ -37,6 +37,7 @@ struct nvkm_memory_func { void (*release)(struct nvkm_memory *); int (*map)(struct nvkm_memory *, u64 offset, struct nvkm_vmm *, struct nvkm_vma *, void *argv, u32 argc); + int (*kmap)(struct nvkm_memory *, struct nvkm_memory **); }; struct nvkm_memory_ptrs { @@ -63,6 +64,7 @@ void nvkm_memory_tags_put(struct nvkm_memory *, struct nvkm_device *, #define nvkm_memory_boot(p,v) (p)->func->boot((p),(v)) #define nvkm_memory_map(p,o,vm,va,av,ac) \ (p)->func->map((p),(o),(vm),(va),(av),(ac)) +#define nvkm_memory_kmap(p,i) ((p)->func->kmap ? (p)->func->kmap((p), (i)) : -ENOSYS) /* accessor macros - kmap()/done() must bracket use of the other accessor * macros to guarantee correct behaviour across all chipsets diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h index f967b97d163c0..fcdaefc99fe85 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/instmem.h @@ -28,7 +28,7 @@ u32 nvkm_instmem_rd32(struct nvkm_instmem *, u32 addr); void nvkm_instmem_wr32(struct nvkm_instmem *, u32 addr, u32 data); int nvkm_instobj_new(struct nvkm_instmem *, u32 size, u32 align, bool zero, struct nvkm_memory **); - +int nvkm_instobj_wrap(struct nvkm_device *, struct nvkm_memory *, struct nvkm_memory **); int nv04_instmem_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_instmem **); int nv40_instmem_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_instmem **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c index 03b1bdb27770a..5c34416cb6379 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.c @@ -25,6 +25,7 @@ #include "ram.h" #include +#include #include struct nvkm_vram { @@ -34,6 +35,12 @@ struct nvkm_vram { struct nvkm_mm_node *mn; }; +static int +nvkm_vram_kmap(struct nvkm_memory *memory, struct nvkm_memory **pmemory) +{ + return nvkm_instobj_wrap(nvkm_vram(memory)->ram->fb->subdev.device, memory, pmemory); +} + static int nvkm_vram_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, struct nvkm_vma *vma, void *argv, u32 argc) @@ -98,6 +105,7 @@ nvkm_vram = { .addr = nvkm_vram_addr, .size = nvkm_vram_size, .map = nvkm_vram_map, + .kmap = nvkm_vram_kmap, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c index cd8163a52bb65..e0e4f97be0294 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/base.c @@ -89,6 +89,18 @@ nvkm_instobj_ctor(const struct nvkm_memory_func *func, spin_unlock(&imem->lock); } +int +nvkm_instobj_wrap(struct nvkm_device *device, + struct nvkm_memory *memory, struct nvkm_memory **pmemory) +{ + struct nvkm_instmem *imem = device->imem; + + if (!imem->func->memory_wrap) + return -ENOSYS; + + return imem->func->memory_wrap(imem, memory, pmemory); +} + int nvkm_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, struct nvkm_memory **pmemory) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c index c51bac76174c1..4b2d7465d22f7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/nv50.c @@ -348,13 +348,11 @@ nv50_instobj_func = { }; static int -nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, - struct nvkm_memory **pmemory) +nv50_instobj_wrap(struct nvkm_instmem *base, + struct nvkm_memory *memory, struct nvkm_memory **pmemory) { struct nv50_instmem *imem = nv50_instmem(base); struct nv50_instobj *iobj; - struct nvkm_device *device = imem->base.subdev.device; - u8 page = max(order_base_2(align), 12); if (!(iobj = kzalloc(sizeof(*iobj), GFP_KERNEL))) return -ENOMEM; @@ -365,7 +363,25 @@ nv50_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, refcount_set(&iobj->maps, 0); INIT_LIST_HEAD(&iobj->lru); - return nvkm_ram_get(device, 0, 1, page, size, true, true, &iobj->ram); + iobj->ram = nvkm_memory_ref(memory); + return 0; +} + +static int +nv50_instobj_new(struct nvkm_instmem *imem, u32 size, u32 align, bool zero, + struct nvkm_memory **pmemory) +{ + u8 page = max(order_base_2(align), 12); + struct nvkm_memory *ram; + int ret; + + ret = nvkm_ram_get(imem->subdev.device, 0, 1, page, size, true, true, &ram); + if (ret) + return ret; + + ret = nv50_instobj_wrap(imem, ram, pmemory); + nvkm_memory_unref(&ram); + return ret; } /****************************************************************************** @@ -382,6 +398,7 @@ static const struct nvkm_instmem_func nv50_instmem = { .fini = nv50_instmem_fini, .memory_new = nv50_instobj_new, + .memory_wrap = nv50_instobj_wrap, .zero = false, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h index 56c15e30a5dd7..fe92986a38858 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/priv.h @@ -12,6 +12,7 @@ struct nvkm_instmem_func { void (*wr32)(struct nvkm_instmem *, u32 addr, u32 data); int (*memory_new)(struct nvkm_instmem *, u32 size, u32 align, bool zero, struct nvkm_memory **); + int (*memory_wrap)(struct nvkm_instmem *, struct nvkm_memory *, struct nvkm_memory **); bool zero; }; -- GitLab From 8c18138c7633f7f9e609be6b11e48bb33a8dfb75 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:21 +1000 Subject: [PATCH 1301/1927] drm/nouveau/fifo: add chid_nr() - reads channel count from GPU from gm200 onwards - removes gm20b/gp10b (they become identical to gm200/gp100) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/core/os.h | 1 + .../drm/nouveau/include/nvkm/engine/fifo.h | 2 - .../gpu/drm/nouveau/nvkm/engine/device/base.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 20 ++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 11 ++++- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 9 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 9 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 9 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c | 45 ------------------ .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c | 46 ------------------- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 9 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 7 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 9 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 15 +++++- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 33 +++++++------ 25 files changed, 114 insertions(+), 137 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h index d7ba3205207fe..5a516b903fe9d 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h @@ -34,4 +34,5 @@ nvkm_blob_dtor(struct nvkm_blob *blob) blob->data = NULL; blob->size = 0; } + #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 5a19c71f7ccbd..3cc7cfebfa5e6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -72,9 +72,7 @@ int gk208_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gk20a_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gm107_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gm200_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); -int gm20b_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gp100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); -int gp10b_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gv100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int tu102_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int ga102_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 587aebcda78c9..532231f1a0f53 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2130,7 +2130,7 @@ nv12b_chipset = { .volt = { 0x00000001, gm20b_volt_new }, .ce = { 0x00000004, gm200_ce_new }, .dma = { 0x00000001, gf119_dma_new }, - .fifo = { 0x00000001, gm20b_fifo_new }, + .fifo = { 0x00000001, gm200_fifo_new }, .gr = { 0x00000001, gm20b_gr_new }, .sw = { 0x00000001, gf100_sw_new }, }; @@ -2356,7 +2356,7 @@ nv13b_chipset = { .top = { 0x00000001, gk104_top_new }, .ce = { 0x00000001, gp100_ce_new }, .dma = { 0x00000001, gf119_dma_new }, - .fifo = { 0x00000001, gp10b_fifo_new }, + .fifo = { 0x00000001, gp100_fifo_new }, .gr = { 0x00000001, gp10b_gr_new }, .sw = { 0x00000001, gf100_sw_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 0fa4df2990095..8599063e86223 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -13,9 +13,7 @@ nvkm-y += nvkm/engine/fifo/gk208.o nvkm-y += nvkm/engine/fifo/gk20a.o nvkm-y += nvkm/engine/fifo/gm107.o nvkm-y += nvkm/engine/fifo/gm200.o -nvkm-y += nvkm/engine/fifo/gm20b.o nvkm-y += nvkm/engine/fifo/gp100.o -nvkm-y += nvkm/engine/fifo/gp10b.o nvkm-y += nvkm/engine/fifo/gv100.o nvkm-y += nvkm/engine/fifo/tu102.o nvkm-y += nvkm/engine/fifo/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 518f3c492f017..b11a4cb18fc78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -24,14 +24,14 @@ #include "priv.h" #include "chan.h" -#include #include #include -#include #include #include +#include "gk104.h" + void nvkm_fifo_recover_chan(struct nvkm_fifo *fifo, int chid) { @@ -290,25 +290,27 @@ nvkm_fifo = { int nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, int nr, struct nvkm_fifo *fifo) + enum nvkm_subdev_type type, int inst, struct nvkm_fifo *fifo) { - int ret; + int ret, nr; fifo->func = func; - INIT_LIST_HEAD(&fifo->chan); spin_lock_init(&fifo->lock); mutex_init(&fifo->mutex); + ret = nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine); + if (ret) + return ret; + + INIT_LIST_HEAD(&fifo->chan); + + nr = func->chid_nr ? func->chid_nr(fifo) : gk104_fifo(fifo)->func->chid_nr(fifo); if (WARN_ON(fifo->nr > NVKM_FIFO_CHID_NR)) fifo->nr = NVKM_FIFO_CHID_NR; else fifo->nr = nr; bitmap_clear(fifo->mask, 0, fifo->nr); - ret = nvkm_engine_ctor(&nvkm_fifo, device, type, inst, true, &fifo->engine); - if (ret) - return ret; - if (func->uevent_init) { ret = nvkm_event_init(&nvkm_fifo_uevent_func, &fifo->engine.subdev, 1, 1, &fifo->uevent); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 3885c3830b949..c452388fd2a59 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -110,6 +110,7 @@ static const struct nvkm_fifo_func g84_fifo = { .dtor = nv50_fifo_dtor, .oneinit = nv50_fifo_oneinit, + .chid_nr = nv50_fifo_chid_nr, .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = g84_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 8b4f36b3e34b5..e10c094c9a7de 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -669,6 +669,7 @@ static const struct nvkm_fifo_func gf100_fifo = { .dtor = gf100_fifo_dtor, .oneinit = gf100_fifo_oneinit, + .chid_nr = nv50_fifo_chid_nr, .init = gf100_fifo_init, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, @@ -695,5 +696,5 @@ gf100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, INIT_WORK(&fifo->recover.work, gf100_fifo_recover_work); *pfifo = &fifo->base; - return nvkm_fifo_ctor(&gf100_fifo, device, type, inst, 128, &fifo->base); + return nvkm_fifo_ctor(&gf100_fifo, device, type, inst, &fifo->base); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index e771bd519ee24..e11b5eb8e196e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -1055,6 +1055,12 @@ gk104_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002140, 0x7fffffff); } +int +gk104_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 4096; +} + void * gk104_fifo_dtor(struct nvkm_fifo *base) { @@ -1103,7 +1109,7 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, INIT_WORK(&fifo->recover.work, gk104_fifo_recover_work); *pfifo = &fifo->base; - return nvkm_fifo_ctor(&gk104_fifo_, device, type, inst, nr, &fifo->base); + return nvkm_fifo_ctor(&gk104_fifo_, device, type, inst, &fifo->base); } const struct nvkm_enum @@ -1230,6 +1236,7 @@ gk104_fifo_fault_gpcclient[] = { static const struct gk104_fifo_func gk104_fifo = { + .chid_nr = gk104_fifo_chid_nr, .intr.fault = gf100_fifo_intr_fault, .pbdma = &gk104_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -1245,5 +1252,5 @@ int gk104_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk104_fifo, device, type, inst, 4096, pfifo); + return gk104_fifo_new_(&gk104_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index f2d12ae739449..5eca8c7579662 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -46,6 +46,8 @@ struct gk104_fifo { }; struct gk104_fifo_func { + int (*chid_nr)(struct nvkm_fifo *); + struct { void (*fault)(struct nvkm_fifo *, int unit); } intr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 915278c7e0128..047e128febd1d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -48,6 +48,7 @@ gk110_fifo_runlist = { static const struct gk104_fifo_func gk110_fifo = { + .chid_nr = gk104_fifo_chid_nr, .intr.fault = gf100_fifo_intr_fault, .pbdma = &gk104_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -63,5 +64,5 @@ int gk110_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk110_fifo, device, type, inst, 4096, pfifo); + return gk104_fifo_new_(&gk110_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index cb703693de52c..89b9c4fa35cc3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -43,8 +43,15 @@ gk208_fifo_pbdma = { .init_timeout = gk208_fifo_pbdma_init_timeout, }; +static int +gk208_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 1024; +} + static const struct gk104_fifo_func gk208_fifo = { + .chid_nr = gk208_fifo_chid_nr, .intr.fault = gf100_fifo_intr_fault, .pbdma = &gk208_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -60,5 +67,5 @@ int gk208_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk208_fifo, device, type, inst, 1024, pfifo); + return gk104_fifo_new_(&gk208_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 6e35cf44c640f..48cdcf9a4a727 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -26,6 +26,7 @@ static const struct gk104_fifo_func gk20a_fifo = { + .chid_nr = nv50_fifo_chid_nr, .intr.fault = gf100_fifo_intr_fault, .pbdma = &gk208_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -41,5 +42,5 @@ int gk20a_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk20a_fifo, device, type, inst, 128, pfifo); + return gk104_fifo_new_(&gk20a_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 7af6e687d4749..7b4244f184d24 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -92,8 +92,15 @@ gm107_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) nvkm_fifo_fault(fifo, &info); } +static int +gm107_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 2048; +} + static const struct gk104_fifo_func gm107_fifo = { + .chid_nr = gm107_fifo_chid_nr, .intr.fault = gm107_fifo_intr_fault, .pbdma = &gk208_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -109,5 +116,5 @@ int gm107_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gm107_fifo, device, type, inst, 2048, pfifo); + return gk104_fifo_new_(&gm107_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 573658cb6c73f..17836a383c760 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -40,8 +40,15 @@ gm200_fifo_pbdma = { .init_timeout = gk208_fifo_pbdma_init_timeout, }; +int +gm200_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return nvkm_rd32(fifo->engine.subdev.device, 0x002008); +} + static const struct gk104_fifo_func gm200_fifo = { + .chid_nr = gm200_fifo_chid_nr, .intr.fault = gm107_fifo_intr_fault, .pbdma = &gm200_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -57,5 +64,5 @@ int gm200_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gm200_fifo, device, type, inst, 4096, pfifo); + return gk104_fifo_new_(&gm200_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c deleted file mode 100644 index 556c97e54f143..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm20b.c +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#include "gk104.h" -#include "changk104.h" - -#include - -static const struct gk104_fifo_func -gm20b_fifo = { - .intr.fault = gm107_fifo_intr_fault, - .pbdma = &gm200_fifo_pbdma, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gm107_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, - .runlist = &gm107_fifo_runlist, - .chan = {{0,0,MAXWELL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, -}; - -int -gm20b_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, - struct nvkm_fifo **pfifo) -{ - return gk104_fifo_new_(&gm20b_fifo, device, type, inst, 512, pfifo); -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 6b46b6b65b877..7a1ac3d863ecd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -78,6 +78,7 @@ gp100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) static const struct gk104_fifo_func gp100_fifo = { + .chid_nr = gm200_fifo_chid_nr, .intr.fault = gp100_fifo_intr_fault, .pbdma = &gm200_fifo_pbdma, .fault.access = gk104_fifo_fault_access, @@ -94,5 +95,5 @@ int gp100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gp100_fifo, device, type, inst, 4096, pfifo); + return gk104_fifo_new_(&gp100_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c deleted file mode 100644 index 7a5929cb4d29b..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2017, NVIDIA CORPORATION. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#include "gk104.h" -#include "changk104.h" - -#include - -static const struct gk104_fifo_func -gp10b_fifo = { - .intr.fault = gp100_fifo_intr_fault, - .pbdma = &gm200_fifo_pbdma, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gp100_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, - .runlist = &gm107_fifo_runlist, - .chan = {{0,0,PASCAL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, - .cgrp_force = true, -}; - -int -gp10b_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, - struct nvkm_fifo **pfifo) -{ - return gk104_fifo_new_(&gp10b_fifo, device, type, inst, 512, pfifo); -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index f56369d592193..0be50120476f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -287,6 +287,7 @@ gv100_fifo_fault_access[] = { static const struct gk104_fifo_func gv100_fifo = { + .chid_nr = gm200_fifo_chid_nr, .pbdma = &gm200_fifo_pbdma, .fault.access = gv100_fifo_fault_access, .fault.engine = gv100_fifo_fault_engine, @@ -302,5 +303,5 @@ int gv100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gv100_fifo, device, type, inst, 4096, pfifo); + return gk104_fifo_new_(&gv100_fifo, device, type, inst, 0, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index c6730c124769c..e4b4133be3a4a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -356,6 +356,12 @@ nv04_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, NV03_PFIFO_CACHES, 1); } +static int +nv04_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 16; +} + int nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, @@ -369,7 +375,7 @@ nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, fifo->ramfc = ramfc; *pfifo = &fifo->base; - ret = nvkm_fifo_ctor(func, device, type, inst, nr, &fifo->base); + ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); if (ret) return ret; @@ -379,6 +385,7 @@ nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, static const struct nvkm_fifo_func nv04_fifo = { + .chid_nr = nv04_fifo_chid_nr, .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index f8887f0f2f82a..94ad6ee192977 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -39,8 +39,15 @@ nv10_fifo_ramfc[] = { {} }; +int +nv10_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 32; +} + static const struct nvkm_fifo_func nv10_fifo = { + .chid_nr = nv10_fifo_chid_nr, .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 3f94c7b5b054b..23d320362c861 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -79,6 +79,7 @@ nv17_fifo_init(struct nvkm_fifo *base) static const struct nvkm_fifo_func nv17_fifo = { + .chid_nr = nv10_fifo_chid_nr, .init = nv17_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index f9ea46809bc05..01e75c80b4f3a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -110,6 +110,7 @@ nv40_fifo_init(struct nvkm_fifo *base) static const struct nvkm_fifo_func nv40_fifo = { + .chid_nr = nv10_fifo_chid_nr, .init = nv40_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index a08742cf425aa..9d0cfb8564013 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -96,6 +96,12 @@ nv50_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002500, 0x00000001); } +int +nv50_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 128; +} + void * nv50_fifo_dtor(struct nvkm_fifo *base) { @@ -116,7 +122,7 @@ nv50_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, return -ENOMEM; *pfifo = &fifo->base; - ret = nvkm_fifo_ctor(func, device, type, inst, 128, &fifo->base); + ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); if (ret) return ret; @@ -129,6 +135,7 @@ static const struct nvkm_fifo_func nv50_fifo = { .dtor = nv50_fifo_dtor, .oneinit = nv50_fifo_oneinit, + .chid_nr = nv50_fifo_chid_nr, .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 79cec57647f02..66e93609c3c7f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -4,8 +4,6 @@ #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) #include -int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, - int nr, struct nvkm_fifo *); void nvkm_fifo_uevent(struct nvkm_fifo *); void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid); @@ -16,7 +14,10 @@ nvkm_fifo_chan_inst_locked(struct nvkm_fifo *, u64 inst); struct nvkm_fifo_chan_oclass; struct nvkm_fifo_func { void *(*dtor)(struct nvkm_fifo *); + int (*oneinit)(struct nvkm_fifo *); + int (*chid_nr)(struct nvkm_fifo *); + int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data); void (*init)(struct nvkm_fifo *); void (*fini)(struct nvkm_fifo *); @@ -35,14 +36,24 @@ struct nvkm_fifo_func { const struct nvkm_fifo_chan_oclass *chan[]; }; +int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, + struct nvkm_fifo *); + void nv04_fifo_intr(struct nvkm_fifo *); int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *nv04_fifo_id_engine(struct nvkm_fifo *, int); void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); +int nv10_fifo_chid_nr(struct nvkm_fifo *); + +int nv50_fifo_chid_nr(struct nvkm_fifo *); + void gf100_fifo_intr_fault(struct nvkm_fifo *, int); +int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); + +int gm200_fifo_chid_nr(struct nvkm_fifo *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index b800cfc5bfd91..63a3a2c0a824b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -24,12 +24,10 @@ #include "changk104.h" #include -#include +#include #include #include #include -#include -#include #include @@ -100,19 +98,6 @@ tu102_fifo_pbdma = { .init_timeout = gk208_fifo_pbdma_init_timeout, }; -static const struct gk104_fifo_func -tu102_fifo = { - .pbdma = &tu102_fifo_pbdma, - .fault.access = gv100_fifo_fault_access, - .fault.engine = tu102_fifo_fault_engine, - .fault.reason = gv100_fifo_fault_reason, - .fault.hubclient = gv100_fifo_fault_hubclient, - .fault.gpcclient = gv100_fifo_fault_gpcclient, - .runlist = &tu102_fifo_runlist, - .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, - .cgrp_force = true, -}; - static void tu102_fifo_recover_work(struct work_struct *w) { @@ -459,6 +444,20 @@ tu102_fifo_ = { .class_new = gk104_fifo_class_new, }; +static const struct gk104_fifo_func +tu102_fifo = { + .chid_nr = gm200_fifo_chid_nr, + .pbdma = &tu102_fifo_pbdma, + .fault.access = gv100_fifo_fault_access, + .fault.engine = tu102_fifo_fault_engine, + .fault.reason = gv100_fifo_fault_reason, + .fault.hubclient = gv100_fifo_fault_hubclient, + .fault.gpcclient = gv100_fifo_fault_gpcclient, + .runlist = &tu102_fifo_runlist, + .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, + .cgrp_force = true, +}; + int tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) @@ -471,5 +470,5 @@ tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, INIT_WORK(&fifo->recover.work, tu102_fifo_recover_work); *pfifo = &fifo->base; - return nvkm_fifo_ctor(&tu102_fifo_, device, type, inst, 4096, &fifo->base); + return nvkm_fifo_ctor(&tu102_fifo_, device, type, inst, &fifo->base); } -- GitLab From f5e4568991f60125712b03079bc037c6feb953f1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:21 +1000 Subject: [PATCH 1302/1927] drm/nouveau/fifo: unify handling of channel classes Adds the basic skeleton for common channel (group) interfaces. - common behaviour between =gk104 impl's - separates priv/user channel objects - passthrough to existing object for now, kludges removed later Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/class.h | 3 + .../gpu/drm/nouveau/include/nvkm/core/os.h | 4 + .../drm/nouveau/include/nvkm/engine/fifo.h | 5 +- .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 43 ++--- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 11 +- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 43 ++++- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 19 +- .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 4 - .../drm/nouveau/nvkm/engine/fifo/dmanv10.c | 4 - .../drm/nouveau/nvkm/engine/fifo/dmanv17.c | 4 - .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 4 - .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 14 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 12 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 50 +---- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 18 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 15 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 4 +- .../drm/nouveau/nvkm/engine/fifo/gpfifog84.c | 4 - .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 4 - .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 3 +- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 2 +- .../drm/nouveau/nvkm/engine/fifo/gpfifonv50.c | 4 - .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 12 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 19 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 14 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 14 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 14 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 14 +- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 34 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 12 +- .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 174 ++++++++++++++++++ 36 files changed, 410 insertions(+), 191 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index a623fb67280b0..e9d9f314a744e 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -41,6 +41,7 @@ #define NVIF_CLASS_DISP_CHAN /* if0014.h */ 0x80000014 #define NVIF_CLASS_CHAN /* if0020.h */ 0x80000020 +#define NVIF_CLASS_CGRP /* if0021.h */ 0x80000021 /* the below match nvidia-assigned (either in hw, or sw) class numbers */ #define NV_NULL_CLASS 0x00000030 @@ -74,6 +75,8 @@ #define NV17_CHANNEL_DMA /* cl506b.h */ 0x0000176e #define NV40_CHANNEL_DMA /* cl506b.h */ 0x0000406e +#define KEPLER_CHANNEL_GROUP_A /* if0021.h */ 0x0000a06c + #define NV50_CHANNEL_GPFIFO /* cl506f.h */ 0x0000506f #define G82_CHANNEL_GPFIFO /* cl826f.h */ 0x0000826f #define FERMI_CHANNEL_GPFIFO /* cl906f.h */ 0x0000906f diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h index 5a516b903fe9d..e8705216ae4e0 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h @@ -35,4 +35,8 @@ nvkm_blob_dtor(struct nvkm_blob *blob) blob->size = 0; } +/*FIXME: remove after */ +#define nvkm_fifo_chan nvkm_chan +#define nvkm_fifo_chan_func nvkm_chan_func +#define nvkm_fifo_cgrp nvkm_cgrp #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 3cc7cfebfa5e6..2f2059c5d4c9e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -15,8 +15,9 @@ struct nvkm_fifo_engn { int usecount; }; -struct nvkm_fifo_chan { - const struct nvkm_fifo_chan_func *func; +struct nvkm_chan { + const struct nvkm_chan_func *func; + struct nvkm_fifo *fifo; u32 engm; struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 8599063e86223..76d2bf1f6d8b4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -33,3 +33,5 @@ nvkm-y += nvkm/engine/fifo/gpfifogf100.o nvkm-y += nvkm/engine/fifo/gpfifogk104.o nvkm-y += nvkm/engine/fifo/gpfifogv100.o nvkm-y += nvkm/engine/fifo/gpfifotu102.o + +nvkm-y += nvkm/engine/fifo/uchan.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index b11a4cb18fc78..06e47728e3a4a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -155,27 +155,16 @@ nvkm_fifo_uevent(struct nvkm_fifo *fifo) } static int -nvkm_fifo_class_new_(struct nvkm_device *device, - const struct nvkm_oclass *oclass, void *data, u32 size, - struct nvkm_object **pobject) +nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) { struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); - return fifo->func->class_new(fifo, oclass, data, size, pobject); -} -static const struct nvkm_device_oclass -nvkm_fifo_class_ = { - .ctor = nvkm_fifo_class_new_, -}; + if (oclass->engn == &fifo->func->chan.user) + return nvkm_uchan_new(fifo, NULL, oclass, argv, argc, pobject); -static int -nvkm_fifo_class_new(struct nvkm_device *device, - const struct nvkm_oclass *oclass, void *data, u32 size, - struct nvkm_object **pobject) -{ - const struct nvkm_fifo_chan_oclass *sclass = oclass->engn; - struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); - return sclass->ctor(fifo, oclass, data, size, pobject); + WARN_ON(1); + return -ENOSYS; } static const struct nvkm_device_oclass @@ -184,24 +173,20 @@ nvkm_fifo_class = { }; static int -nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, - const struct nvkm_device_oclass **class) +nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) { struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); - const struct nvkm_fifo_chan_oclass *sclass; + const struct nvkm_fifo_func_chan *chan = &fifo->func->chan; int c = 0; - if (fifo->func->class_get) { - int ret = fifo->func->class_get(fifo, index, oclass); - if (ret == 0) - *class = &nvkm_fifo_class_; - return ret; - } + if (fifo->func->engine_id == gk104_fifo_engine_id) + chan = &gk104_fifo(fifo)->func->chan; - while ((sclass = fifo->func->chan[c])) { + /* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */ + if (chan->user.oclass) { if (c++ == index) { - oclass->base = sclass->base; - oclass->engn = sclass; + oclass->base = chan->user; + oclass->engn = &fifo->func->chan.user; *class = &nvkm_fifo_class; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index d0ac60b067206..9a677a1acf9ba 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -1,8 +1,11 @@ -#ifndef __NVKM_FIFO_CGRP_H__ -#define __NVKM_FIFO_CGRP_H__ -#include "priv.h" +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_CGRP_H__ +#define __NVKM_CGRP_H__ +#include -struct nvkm_fifo_cgrp { +struct nvkm_cgrp { + const struct nvkm_cgrp_func { + } *func; int id; struct list_head head; struct list_head chan; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 39d6b9217f8d1..4d4905be4b92b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "priv.h" #include #include @@ -140,7 +141,7 @@ nvkm_fifo_chan_child_func = { .fini[0] = nvkm_fifo_chan_child_fini, }; -static int +int nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, struct nvkm_object **pobject) { @@ -258,11 +259,6 @@ nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct n struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); union nvif_chan_event_args *args = argv; - if (!uevent) - return 0; - if (argc != sizeof(args->v0) || args->v0.version != 0) - return -ENOSYS; - switch (args->v0.type) { case NVIF_CHAN_EVENT_V0_NON_STALL_INTR: return nvkm_uevent_add(uevent, &chan->fifo->uevent, 0, @@ -304,6 +300,18 @@ nvkm_fifo_chan_init(struct nvkm_object *object) return 0; } +void +nvkm_chan_del(struct nvkm_chan **pchan) +{ + struct nvkm_chan *chan = *pchan; + + if (!chan) + return; + + chan = nvkm_object_dtor(&chan->object); + kfree(chan); +} + static void * nvkm_fifo_chan_dtor(struct nvkm_object *object) { @@ -326,6 +334,7 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object) nvkm_gpuobj_del(&chan->push); nvkm_gpuobj_del(&chan->inst); + kfree(chan->func); return data; } @@ -340,20 +349,38 @@ nvkm_fifo_chan_func = { }; int -nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *func, +nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, struct nvkm_fifo *fifo, u32 size, u32 align, bool zero, u64 hvmm, u64 push, u32 engm, int bar, u32 base, u32 user, const struct nvkm_oclass *oclass, struct nvkm_fifo_chan *chan) { + struct nvkm_chan_func *func; struct nvkm_client *client = oclass->client; struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_dmaobj *dmaobj; unsigned long flags; int ret; - nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); + /*FIXME: temp kludge to ease transition, remove later */ + if (!(func = kmalloc(sizeof(*func), GFP_KERNEL))) + return -ENOMEM; + + *func = *fifo->func->chan.func; + func->dtor = fn->dtor; + func->init = fn->init; + func->fini = fn->fini; + func->engine_ctor = fn->engine_ctor; + func->engine_dtor = fn->engine_dtor; + func->engine_init = fn->engine_init; + func->engine_fini = fn->engine_fini; + func->object_ctor = fn->object_ctor; + func->object_dtor = fn->object_dtor; + func->submit_token = fn->submit_token; + chan->func = func; + + nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); chan->fifo = fifo; chan->engm = engm; INIT_LIST_HEAD(&chan->head); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 8ef972134f9be..24fbe6cf70e1b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -1,10 +1,10 @@ /* SPDX-License-Identifier: MIT */ -#ifndef __NVKM_FIFO_CHAN_H__ -#define __NVKM_FIFO_CHAN_H__ -#define nvkm_fifo_chan(p) container_of((p), struct nvkm_fifo_chan, object) -#include "priv.h" +#ifndef __NVKM_CHAN_H__ +#define __NVKM_CHAN_H__ +#define nvkm_chan(p) container_of((p), struct nvkm_chan, object) /*FIXME: remove later */ +#include -struct nvkm_fifo_chan_func { +struct nvkm_chan_func { void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); void (*fini)(struct nvkm_fifo_chan *); @@ -23,12 +23,7 @@ int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, u32 size, u32 align, bool zero, u64 vm, u64 push, u32 engm, int bar, u32 base, u32 user, const struct nvkm_oclass *, struct nvkm_fifo_chan *); +void nvkm_chan_del(struct nvkm_chan **); -struct nvkm_fifo_chan_oclass { - int (*ctor)(struct nvkm_fifo *, const struct nvkm_oclass *, - void *data, u32 size, struct nvkm_object **); - struct nvkm_sclass base; -}; - -int gf100_fifo_chan_ntfy(struct nvkm_fifo_chan *, u32, struct nvkm_event **); +int nvkm_fifo_chan_child_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index dbcdc5fab990b..d17d8c65d1690 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -28,7 +28,6 @@ #include #include -#include #include #include @@ -219,8 +218,5 @@ nv04_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass nv04_fifo_dma_oclass = { - .base.oclass = NV03_CHANNEL_DMA, - .base.minver = 0, - .base.maxver = 0, .ctor = nv04_fifo_dma_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c index 07d80d54a07cf..f45655072f9d0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c @@ -28,7 +28,6 @@ #include #include -#include #include #include @@ -90,8 +89,5 @@ nv10_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass nv10_fifo_dma_oclass = { - .base.oclass = NV10_CHANNEL_DMA, - .base.minver = 0, - .base.maxver = 0, .ctor = nv10_fifo_dma_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c index edd70a1142185..cd3f9f84ab9c2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c @@ -28,7 +28,6 @@ #include #include -#include #include #include @@ -91,8 +90,5 @@ nv17_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass nv17_fifo_dma_oclass = { - .base.oclass = NV17_CHANNEL_DMA, - .base.minver = 0, - .base.maxver = 0, .ctor = nv17_fifo_dma_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c index 0411fb9084577..c42c2e01d8011 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c @@ -28,7 +28,6 @@ #include #include -#include #include #include @@ -247,8 +246,5 @@ nv40_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass nv40_fifo_dma_oclass = { - .base.oclass = NV40_CHANNEL_DMA, - .base.minver = 0, - .base.maxver = 0, .ctor = nv40_fifo_dma_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index c452388fd2a59..24732d355c831 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -21,9 +21,17 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "nv50.h" #include "channv50.h" +#include + +const struct nvkm_chan_func +g84_chan = { +}; + static void g84_fifo_uevent_fini(struct nvkm_fifo *fifo) { @@ -119,10 +127,8 @@ g84_fifo = { .start = nv04_fifo_start, .uevent_init = g84_fifo_uevent_init, .uevent_fini = g84_fifo_uevent_fini, - .chan = { - &g84_fifo_gpfifo_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index e10c094c9a7de..25eda4fd28675 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -21,6 +21,8 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "gf100.h" #include "changf100.h" @@ -47,6 +49,10 @@ gf100_fifo_uevent_fini(struct nvkm_fifo *fifo) nvkm_mask(device, 0x002140, 0x80000000, 0x00000000); } +static const struct nvkm_chan_func +gf100_chan = { +}; + void gf100_fifo_runlist_commit(struct gf100_fifo *fifo) { @@ -678,10 +684,8 @@ gf100_fifo = { .id_engine = gf100_fifo_id_engine, .uevent_init = gf100_fifo_uevent_init, .uevent_fini = gf100_fifo_uevent_fini, - .chan = { - &gf100_fifo_gpfifo_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index e11b5eb8e196e..1e15b88d59aea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -21,6 +21,8 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "gk104.h" #include "cgrp.h" #include "changk104.h" @@ -36,6 +38,10 @@ #include #include +static const struct nvkm_chan_func +gk104_chan = { +}; + void gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, struct gk104_fifo_engine_status *status) @@ -83,45 +89,6 @@ gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, status->chan == &status->next ? "*" : " "); } -int -gk104_fifo_class_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *argv, u32 argc, struct nvkm_object **pobject) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - if (oclass->engn == &fifo->func->chan) { - const struct gk104_fifo_chan_user *user = oclass->engn; - return user->ctor(fifo, oclass, argv, argc, pobject); - } else - if (oclass->engn == &fifo->func->user) { - const struct gk104_fifo_user_user *user = oclass->engn; - return user->ctor(oclass, argv, argc, pobject); - } - WARN_ON(1); - return -EINVAL; -} - -int -gk104_fifo_class_get(struct nvkm_fifo *base, int index, - struct nvkm_oclass *oclass) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - int c = 0; - - if (fifo->func->user.ctor && c++ == index) { - oclass->base = fifo->func->user.user; - oclass->engn = &fifo->func->user; - return 0; - } - - if (fifo->func->chan.ctor && c++ == index) { - oclass->base = fifo->func->chan.user; - oclass->engn = &fifo->func->chan; - return 0; - } - - return c; -} - void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo) { @@ -1093,8 +1060,6 @@ gk104_fifo_ = { .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, - .class_get = gk104_fifo_class_get, - .class_new = gk104_fifo_class_new, }; int @@ -1245,7 +1210,8 @@ gk104_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gk104_fifo_runlist, - .chan = {{0,0,KEPLER_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 5eca8c7579662..be426734da02e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -76,18 +76,8 @@ struct gk104_fifo_func { struct nvkm_memory *, int entries); } *runlist; - struct gk104_fifo_user_user { - struct nvkm_sclass user; - int (*ctor)(const struct nvkm_oclass *, void *, u32, - struct nvkm_object **); - } user; - - struct gk104_fifo_chan_user { - struct nvkm_sclass user; - int (*ctor)(struct gk104_fifo *, const struct nvkm_oclass *, - void *, u32, struct nvkm_object **); - } chan; - bool cgrp_force; + struct nvkm_fifo_func_cgrp cgrp; + struct nvkm_fifo_func_chan chan; }; struct gk104_fifo_engine_status { @@ -121,10 +111,6 @@ int gk104_fifo_oneinit(struct nvkm_fifo *base); int gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data); void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); -int gk104_fifo_class_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *argv, u32 argc, struct nvkm_object **pobject); -int gk104_fifo_class_get(struct nvkm_fifo *base, int index, - struct nvkm_oclass *oclass); void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo); void gk104_fifo_uevent_init(struct nvkm_fifo *fifo); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 047e128febd1d..df9b83cf382fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -21,14 +21,24 @@ * * Authors: Ben Skeggs */ -#include "gk104.h" #include "cgrp.h" +#include "chan.h" + +#include "gk104.h" #include "changk104.h" #include #include +const struct nvkm_chan_func +gk110_chan = { +}; + +const struct nvkm_cgrp_func +gk110_cgrp = { +}; + void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp, struct nvkm_memory *memory, u32 offset) @@ -57,7 +67,8 @@ gk110_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gk110_fifo_runlist, - .chan = {{0,0,KEPLER_CHANNEL_GPFIFO_B}, gk104_fifo_gpfifo_new }, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 89b9c4fa35cc3..003f244f89db8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -60,7 +60,8 @@ gk208_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gk110_fifo_runlist, - .chan = {{0,0,KEPLER_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 48cdcf9a4a727..661c9f0cf7b9c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -35,7 +35,8 @@ gk20a_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gk110_fifo_runlist, - .chan = {{0,0,KEPLER_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, + .cgrp = {{ }, &gk110_cgrp }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 7b4244f184d24..910d97911a98d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -21,6 +21,8 @@ * * Authors: Ben Skeggs */ +#include "priv.h" +#include "chan.h" #include "gk104.h" #include "changk104.h" @@ -29,6 +31,10 @@ #include +const struct nvkm_chan_func +gm107_chan = { +}; + static void gm107_fifo_runlist_chan(struct gk104_fifo_chan *chan, struct nvkm_memory *memory, u32 offset) @@ -109,7 +115,8 @@ gm107_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gm107_fifo_runlist, - .chan = {{0,0,KEPLER_CHANNEL_GPFIFO_B}, gk104_fifo_gpfifo_new }, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 17836a383c760..101027536a93e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -57,7 +57,8 @@ gm200_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gm107_fifo_runlist, - .chan = {{0,0,MAXWELL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, + .chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 7a1ac3d863ecd..c52a4a0b361b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -87,8 +87,8 @@ gp100_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .runlist = &gm107_fifo_runlist, - .chan = {{0,0,PASCAL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new }, - .cgrp_force = true, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, + .chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c index 2121f517b1dda..3157a76332311 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -88,8 +87,5 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass g84_fifo_gpfifo_oclass = { - .base.oclass = G82_CHANNEL_GPFIFO, - .base.minver = 0, - .base.maxver = 0, .ctor = g84_fifo_gpfifo_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 4b1f6702680a6..5d6e3a7b8f1e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -28,7 +28,6 @@ #include #include -#include #include #include @@ -283,8 +282,5 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass gf100_fifo_gpfifo_oclass = { - .base.oclass = FERMI_CHANNEL_GPFIFO, - .base.minver = 0, - .base.maxver = 0, .ctor = gf100_fifo_gpfifo_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index ea9852a5fdd83..6415e80a1fbf1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -30,7 +30,6 @@ #include #include -#include #include #include @@ -287,7 +286,7 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, /* Hack to support GPUs where even individual channels should be * part of a channel group. */ - if (fifo->func->cgrp_force) { + if (fifo->func->cgrp.force) { if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL))) return -ENOMEM; chan->cgrp->id = chan->base.chid; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index e8546fed40241..34a8e792c0e89 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -168,7 +168,7 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, /* Hack to support GPUs where even individual channels should be * part of a channel group. */ - if (fifo->func->cgrp_force) { + if (fifo->func->cgrp.force) { if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL))) return -ENOMEM; chan->cgrp->id = chan->base.chid; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c index d8f28ec1e4a81..410fc9e6c1f99 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c @@ -26,7 +26,6 @@ #include #include -#include #include #include @@ -86,8 +85,5 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, const struct nvkm_fifo_chan_oclass nv50_fifo_gpfifo_oclass = { - .base.oclass = NV50_CHANNEL_GPFIFO, - .base.minver = 0, - .base.maxver = 0, .ctor = nv50_fifo_gpfifo_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 0be50120476f1..ddca298528656 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -19,14 +19,20 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include "gk104.h" +#include "chan.h" #include "cgrp.h" + +#include "gk104.h" #include "changk104.h" #include #include +static const struct nvkm_chan_func +gv100_chan = { +}; + void gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan, struct nvkm_memory *memory, u32 offset) @@ -295,8 +301,8 @@ gv100_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .runlist = &gv100_fifo_runlist, - .chan = {{ 0, 0,VOLTA_CHANNEL_GPFIFO_A}, gv100_fifo_gpfifo_new }, - .cgrp_force = true, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, + .chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index e4b4133be3a4a..95a30fbf65a10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -21,6 +21,9 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" +#include "chan.h" + #include "nv04.h" #include "channv04.h" #include "regsnv04.h" @@ -31,6 +34,8 @@ #include #include +#include + static const struct nv04_fifo_ramfc nv04_fifo_ramfc[] = { { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, @@ -44,6 +49,14 @@ nv04_fifo_ramfc[] = { {} }; +static const struct nvkm_chan_func +nv04_chan = { +}; + +const struct nvkm_cgrp_func +nv04_cgrp = { +}; + void nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags) __acquires(fifo->base.lock) @@ -392,10 +405,8 @@ nv04_fifo = { .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .chan = { - &nv04_fifo_dma_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index 94ad6ee192977..f07304e4b8c50 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -21,10 +21,14 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "nv04.h" #include "channv04.h" #include "regsnv04.h" +#include + static const struct nv04_fifo_ramfc nv10_fifo_ramfc[] = { { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, @@ -39,6 +43,10 @@ nv10_fifo_ramfc[] = { {} }; +static const struct nvkm_chan_func +nv10_chan = { +}; + int nv10_fifo_chid_nr(struct nvkm_fifo *fifo) { @@ -54,10 +62,8 @@ nv10_fifo = { .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .chan = { - &nv10_fifo_dma_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, NV10_CHANNEL_DMA }, &nv10_chan, .oclass = &nv10_fifo_dma_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 23d320362c861..858c8ba158c94 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -21,6 +21,8 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "nv04.h" #include "channv04.h" #include "regsnv04.h" @@ -28,6 +30,8 @@ #include #include +#include + static const struct nv04_fifo_ramfc nv17_fifo_ramfc[] = { { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, @@ -47,6 +51,10 @@ nv17_fifo_ramfc[] = { {} }; +static const struct nvkm_chan_func +nv17_chan = { +}; + static void nv17_fifo_init(struct nvkm_fifo *base) { @@ -86,10 +94,8 @@ nv17_fifo = { .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .chan = { - &nv17_fifo_dma_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, NV17_CHANNEL_DMA }, &nv17_chan, .oclass = &nv17_fifo_dma_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index 01e75c80b4f3a..30f924e3da3c8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -21,6 +21,8 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "nv04.h" #include "channv04.h" #include "regsnv04.h" @@ -29,6 +31,8 @@ #include #include +#include + static const struct nv04_fifo_ramfc nv40_fifo_ramfc[] = { { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, @@ -56,6 +60,10 @@ nv40_fifo_ramfc[] = { {} }; +static const struct nvkm_chan_func +nv40_chan = { +}; + static void nv40_fifo_init(struct nvkm_fifo *base) { @@ -117,10 +125,8 @@ nv40_fifo = { .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .chan = { - &nv40_fifo_dma_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, NV40_CHANNEL_DMA }, &nv40_chan, .oclass = &nv40_fifo_dma_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 9d0cfb8564013..f3f11a324591d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -21,11 +21,19 @@ * * Authors: Ben Skeggs */ +#include "chan.h" + #include "nv50.h" #include "channv50.h" #include +#include + +static const struct nvkm_chan_func +nv50_chan = { +}; + static void nv50_fifo_runlist_update_locked(struct nv50_fifo *fifo) { @@ -142,10 +150,8 @@ nv50_fifo = { .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .chan = { - &nv50_fifo_gpfifo_oclass, - NULL - }, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan, .oclass = &nv50_fifo_gpfifo_oclass }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 66e93609c3c7f..b8031c212c8bd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -3,6 +3,8 @@ #define __NVKM_FIFO_PRIV_H__ #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) #include +struct nvkm_cgrp; +struct gk104_fifo; void nvkm_fifo_uevent(struct nvkm_fifo *); void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); @@ -30,10 +32,23 @@ struct nvkm_fifo_func { void (*uevent_init)(struct nvkm_fifo *); void (*uevent_fini)(struct nvkm_fifo *); void (*recover_chan)(struct nvkm_fifo *, int chid); - int (*class_get)(struct nvkm_fifo *, int index, struct nvkm_oclass *); - int (*class_new)(struct nvkm_fifo *, const struct nvkm_oclass *, - void *, u32, struct nvkm_object **); - const struct nvkm_fifo_chan_oclass *chan[]; + + struct nvkm_fifo_func_cgrp { + struct nvkm_sclass user; + const struct nvkm_cgrp_func *func; + bool force; + } cgrp; + + struct nvkm_fifo_func_chan { + struct nvkm_sclass user; + const struct nvkm_chan_func *func; + const struct nvkm_fifo_chan_oclass { + int (*ctor)(struct nvkm_fifo *, const struct nvkm_oclass *, + void *data, u32 size, struct nvkm_object **); + } *oclass; + int (*ctor)(struct gk104_fifo *, const struct nvkm_oclass *, void *, u32, + struct nvkm_object **); + } chan; }; int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, @@ -44,16 +59,27 @@ int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *nv04_fifo_id_engine(struct nvkm_fifo *, int); void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); +extern const struct nvkm_cgrp_func nv04_cgrp; int nv10_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_nr(struct nvkm_fifo *); +extern const struct nvkm_chan_func g84_chan; + void gf100_fifo_intr_fault(struct nvkm_fifo *, int); int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); +extern const struct nvkm_cgrp_func gk110_cgrp; +extern const struct nvkm_chan_func gk110_chan; + +extern const struct nvkm_chan_func gm107_chan; + int gm200_fifo_chid_nr(struct nvkm_fifo *); + +int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, + void *argv, u32 argc, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 63a3a2c0a824b..db9b802bd67c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -19,6 +19,8 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "chan.h" + #include "gk104.h" #include "cgrp.h" #include "changk104.h" @@ -31,6 +33,10 @@ #include +static const struct nvkm_chan_func +tu102_chan = { +}; + static void tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, struct nvkm_memory *mem, int nr) @@ -440,8 +446,6 @@ tu102_fifo_ = { .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = tu102_fifo_recover_chan, - .class_get = gk104_fifo_class_get, - .class_new = gk104_fifo_class_new, }; static const struct gk104_fifo_func @@ -454,8 +458,8 @@ tu102_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .runlist = &tu102_fifo_runlist, - .chan = {{ 0, 0,TURING_CHANNEL_GPFIFO_A}, tu102_fifo_gpfifo_new }, - .cgrp_force = true, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, + .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new }, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c new file mode 100644 index 0000000000000..a1d93f28b634b --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -0,0 +1,174 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define nvkm_uchan(p) container_of((p), struct nvkm_uchan, object) +#include "cgrp.h" +#include "chan.h" + +#include + +#include + +#include "gk104.h" + +struct nvkm_uchan { + struct nvkm_object object; + struct nvkm_chan *chan; +}; + +static int +nvkm_uchan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) +{ + struct nvkm_chan *chan = nvkm_uchan(object)->chan; + union nvif_chan_event_args *args = argv; + + if (!uevent) + return 0; + if (argc != sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + + switch (args->v0.type) { + case NVIF_CHAN_EVENT_V0_NON_STALL_INTR: + case NVIF_CHAN_EVENT_V0_KILLED: + return chan->object.func->uevent(&chan->object, argv, argc, uevent); + default: + break; + } + + return -ENOSYS; +} + +struct nvkm_uobj { + struct nvkm_oproxy oproxy; + struct nvkm_chan *chan; +}; + +static const struct nvkm_oproxy_func +nvkm_uchan_object = { +}; + +static int +nvkm_uchan_object_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + struct nvkm_chan *chan = nvkm_uchan(oclass->parent)->chan; + struct nvkm_uobj *uobj; + struct nvkm_oclass _oclass; + + if (!(uobj = kzalloc(sizeof(*uobj), GFP_KERNEL))) + return -ENOMEM; + + nvkm_oproxy_ctor(&nvkm_uchan_object, oclass, &uobj->oproxy); + uobj->chan = chan; + *pobject = &uobj->oproxy.base; + + _oclass = *oclass; + _oclass.parent = &chan->object; + return nvkm_fifo_chan_child_new(&_oclass, argv, argc, &uobj->oproxy.object); +} + +static int +nvkm_uchan_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *oclass) +{ + struct nvkm_chan *chan = nvkm_uchan(object)->chan; + int ret; + + ret = chan->object.func->sclass(&chan->object, index, oclass); + if (ret) + return ret; + + oclass->ctor = nvkm_uchan_object_new; + return 0; +} + +static int +nvkm_uchan_map(struct nvkm_object *object, void *argv, u32 argc, + enum nvkm_object_map *type, u64 *addr, u64 *size) +{ + struct nvkm_chan *chan = nvkm_uchan(object)->chan; + + return chan->object.func->map(&chan->object, argv, argc, type, addr, size); +} + +static int +nvkm_uchan_fini(struct nvkm_object *object, bool suspend) +{ + struct nvkm_chan *chan = nvkm_uchan(object)->chan; + int ret; + + ret = chan->object.func->fini(&chan->object, suspend); + if (ret && suspend) + return ret; + + return 0; +} + +static int +nvkm_uchan_init(struct nvkm_object *object) +{ + struct nvkm_chan *chan = nvkm_uchan(object)->chan; + + return chan->object.func->init(&chan->object); +} + +static void * +nvkm_uchan_dtor(struct nvkm_object *object) +{ + struct nvkm_uchan *uchan = nvkm_uchan(object); + + nvkm_chan_del(&uchan->chan); + return uchan; +} + +static const struct nvkm_object_func +nvkm_uchan = { + .dtor = nvkm_uchan_dtor, + .init = nvkm_uchan_init, + .fini = nvkm_uchan_fini, + .map = nvkm_uchan_map, + .sclass = nvkm_uchan_sclass, + .uevent = nvkm_uchan_uevent, +}; + +int +nvkm_uchan_new(struct nvkm_fifo *fifo, struct nvkm_cgrp *cgrp, const struct nvkm_oclass *oclass, + void *argv, u32 argc, struct nvkm_object **pobject) +{ + struct nvkm_object *object = NULL; + struct nvkm_uchan *uchan; + int ret; + + if (!(uchan = kzalloc(sizeof(*uchan), GFP_KERNEL))) + return -ENOMEM; + + nvkm_object_ctor(&nvkm_uchan, oclass, &uchan->object); + *pobject = &uchan->object; + + if (!fifo->func->chan.func) + ret = gk104_fifo(fifo)->func->chan.ctor(gk104_fifo(fifo), oclass, argv, argc, &object); + else + ret = fifo->func->chan.oclass->ctor(fifo, oclass, argv, argc, &object); + if (!object) + return ret; + + uchan->chan = container_of(object, typeof(*uchan->chan), object); + return ret; +} -- GitLab From fd67738a28bf51a8113c0d1cc8f31d4f2f3776e0 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:22 +1000 Subject: [PATCH 1303/1927] drm/nouveau/fifo: pre-move some blocks of code around - will make subsequent patches more obvious - no code changes Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 16 +- .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 66 --- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 172 +++---- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 448 +++++++++--------- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 66 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 32 +- 6 files changed, 400 insertions(+), 400 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 06e47728e3a4a..b06db3b3d4714 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -211,6 +211,14 @@ nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend) return 0; } +static int +nvkm_fifo_init(struct nvkm_engine *engine) +{ + struct nvkm_fifo *fifo = nvkm_fifo(engine); + fifo->func->init(fifo); + return 0; +} + static int nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) { @@ -240,14 +248,6 @@ nvkm_fifo_preinit(struct nvkm_engine *engine) nvkm_mc_reset(engine->subdev.device, NVKM_ENGINE_FIFO, 0); } -static int -nvkm_fifo_init(struct nvkm_engine *engine) -{ - struct nvkm_fifo *fifo = nvkm_fifo(engine); - fifo->func->init(fifo); - return 0; -} - static void * nvkm_fifo_dtor(struct nvkm_engine *engine) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index d17d8c65d1690..ed2ae9e699874 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -69,72 +69,6 @@ nv04_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, return hash; } -void -nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_memory *fctx = device->imem->ramfc; - const struct nv04_fifo_ramfc *c; - unsigned long flags; - u32 mask = fifo->base.nr - 1; - u32 data = chan->ramfc; - u32 chid; - - /* prevent fifo context switches */ - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_wr32(device, NV03_PFIFO_CACHES, 0); - - /* if this channel is active, replace it with a null context */ - chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & mask; - if (chid == chan->base.chid) { - nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); - nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); - - c = fifo->ramfc; - nvkm_kmap(fctx); - do { - u32 rm = ((1ULL << c->bits) - 1) << c->regs; - u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; - u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; - u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); - nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); - } while ((++c)->bits); - nvkm_done(fctx); - - c = fifo->ramfc; - do { - nvkm_wr32(device, c->regp, 0x00000000); - } while ((++c)->bits); - - nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, mask); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); - nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); - } - - /* restore normal operation, after disabling dma mode */ - nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); - nvkm_wr32(device, NV03_PFIFO_CACHES, 1); - spin_unlock_irqrestore(&fifo->base.lock, flags); -} - -void -nv04_fifo_dma_init(struct nvkm_fifo_chan *base) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 mask = 1 << chan->base.chid; - unsigned long flags; - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); - spin_unlock_irqrestore(&fifo->base.lock, flags); -} - void * nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 25eda4fd28675..bd0b0c66e28f6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -53,6 +53,52 @@ static const struct nvkm_chan_func gf100_chan = { }; +static const struct nvkm_bitfield +gf100_fifo_pbdma_intr[] = { +/* { 0x00008000, "" } seen with null ib push */ + { 0x00200000, "ILLEGAL_MTHD" }, + { 0x00800000, "EMPTY_SUBC" }, + {} +}; + +static void +gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit) +{ + struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_device *device = subdev->device; + u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000)); + u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000)); + u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000)); + u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0x7f; + u32 subc = (addr & 0x00070000) >> 16; + u32 mthd = (addr & 0x00003ffc); + struct nvkm_fifo_chan *chan; + unsigned long flags; + u32 show = stat; + char msg[128]; + + if (stat & 0x00800000) { + if (device->sw) { + if (nvkm_sw_mthd(device->sw, chid, subc, mthd, data)) + show &= ~0x00800000; + } + } + + if (show) { + nvkm_snprintbf(msg, sizeof(msg), gf100_fifo_pbdma_intr, show); + chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); + nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " + "subc %d mthd %04x data %08x\n", + unit, show, msg, chid, chan ? chan->inst->addr : 0, + chan ? chan->object.client->name : "unknown", + subc, mthd, data); + nvkm_fifo_chan_put(&fifo->base, flags, &chan); + } + + nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008); + nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); +} + void gf100_fifo_runlist_commit(struct gf100_fifo *fifo) { @@ -399,52 +445,6 @@ gf100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) nvkm_fifo_fault(fifo, &info); } -static const struct nvkm_bitfield -gf100_fifo_pbdma_intr[] = { -/* { 0x00008000, "" } seen with null ib push */ - { 0x00200000, "ILLEGAL_MTHD" }, - { 0x00800000, "EMPTY_SUBC" }, - {} -}; - -static void -gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000)); - u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000)); - u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000)); - u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0x7f; - u32 subc = (addr & 0x00070000) >> 16; - u32 mthd = (addr & 0x00003ffc); - struct nvkm_fifo_chan *chan; - unsigned long flags; - u32 show= stat; - char msg[128]; - - if (stat & 0x00800000) { - if (device->sw) { - if (nvkm_sw_mthd(device->sw, chid, subc, mthd, data)) - show &= ~0x00800000; - } - } - - if (show) { - nvkm_snprintbf(msg, sizeof(msg), gf100_fifo_pbdma_intr, show); - chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); - nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " - "subc %d mthd %04x data %08x\n", - unit, show, msg, chid, chan ? chan->inst->addr : 0, - chan ? chan->object.client->name : "unknown", - subc, mthd, data); - nvkm_fifo_chan_put(&fifo->base, flags, &chan); - } - - nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008); - nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); -} - static void gf100_fifo_intr_runlist(struct gf100_fifo *fifo) { @@ -576,46 +576,6 @@ gf100_fifo_intr(struct nvkm_fifo *base) } } -static int -gf100_fifo_oneinit(struct nvkm_fifo *base) -{ - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); - int ret; - - /* Determine number of PBDMAs by checking valid enable bits. */ - nvkm_wr32(device, 0x002204, 0xffffffff); - fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204)); - nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr); - - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, - false, &fifo->runlist.mem[0]); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, - false, &fifo->runlist.mem[1]); - if (ret) - return ret; - - init_waitqueue_head(&fifo->runlist.wait); - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000, - 0x1000, false, &fifo->user.mem); - if (ret) - return ret; - - ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem), - &fifo->user.bar); - if (ret) - return ret; - - return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); -} - static void gf100_fifo_fini(struct nvkm_fifo *base) { @@ -659,6 +619,46 @@ gf100_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ } +static int +gf100_fifo_oneinit(struct nvkm_fifo *base) +{ + struct gf100_fifo *fifo = gf100_fifo(base); + struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_device *device = subdev->device; + struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); + int ret; + + /* Determine number of PBDMAs by checking valid enable bits. */ + nvkm_wr32(device, 0x002204, 0xffffffff); + fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204)); + nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr); + + + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, + false, &fifo->runlist.mem[0]); + if (ret) + return ret; + + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, + false, &fifo->runlist.mem[1]); + if (ret) + return ret; + + init_waitqueue_head(&fifo->runlist.wait); + + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000, + 0x1000, false, &fifo->user.mem); + if (ret) + return ret; + + ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem), + &fifo->user.bar); + if (ret) + return ret; + + return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); +} + static void * gf100_fifo_dtor(struct nvkm_fifo *base) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 1e15b88d59aea..da26c5219e53e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -103,6 +103,70 @@ gk104_fifo_uevent_init(struct nvkm_fifo *fifo) nvkm_mask(device, 0x002140, 0x80000000, 0x80000000); } +static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = { + { 0x00000001, "HCE_RE_ILLEGAL_OP" }, + { 0x00000002, "HCE_RE_ALIGNB" }, + { 0x00000004, "HCE_PRIV" }, + { 0x00000008, "HCE_ILLEGAL_MTHD" }, + { 0x00000010, "HCE_ILLEGAL_CLASS" }, + {} +}; + +void +gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit) +{ + struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_device *device = subdev->device; + u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000)); + u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask; + u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff; + char msg[128]; + + if (stat) { + nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat); + nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n", + unit, stat, msg, chid, + nvkm_rd32(device, 0x040150 + (unit * 0x2000)), + nvkm_rd32(device, 0x040154 + (unit * 0x2000))); + } + + nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat); +} + +static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { + { 0x00000001, "MEMREQ" }, + { 0x00000002, "MEMACK_TIMEOUT" }, + { 0x00000004, "MEMACK_EXTRA" }, + { 0x00000008, "MEMDAT_TIMEOUT" }, + { 0x00000010, "MEMDAT_EXTRA" }, + { 0x00000020, "MEMFLUSH" }, + { 0x00000040, "MEMOP" }, + { 0x00000080, "LBCONNECT" }, + { 0x00000100, "LBREQ" }, + { 0x00000200, "LBACK_TIMEOUT" }, + { 0x00000400, "LBACK_EXTRA" }, + { 0x00000800, "LBDAT_TIMEOUT" }, + { 0x00001000, "LBDAT_EXTRA" }, + { 0x00002000, "GPFIFO" }, + { 0x00004000, "GPPTR" }, + { 0x00008000, "GPENTRY" }, + { 0x00010000, "GPCRC" }, + { 0x00020000, "PBPTR" }, + { 0x00040000, "PBENTRY" }, + { 0x00080000, "PBCRC" }, + { 0x00100000, "XBARCONNECT" }, + { 0x00200000, "METHOD" }, + { 0x00400000, "METHODCRC" }, + { 0x00800000, "DEVICE" }, + { 0x02000000, "SEMAPHORE" }, + { 0x04000000, "ACQUIRE" }, + { 0x08000000, "PRI" }, + { 0x20000000, "NO_CTXSW_SEG" }, + { 0x40000000, "PBSEG" }, + { 0x80000000, "SIGNATURE" }, + {} +}; + void gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, struct nvkm_memory *mem, int nr) @@ -443,6 +507,128 @@ gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn) schedule_work(&fifo->recover.work); } +const struct nvkm_enum +gk104_fifo_fault_access[] = { + { 0x0, "READ" }, + { 0x1, "WRITE" }, + {} +}; + +const struct nvkm_enum +gk104_fifo_fault_engine[] = { + { 0x00, "GR", NULL, NVKM_ENGINE_GR }, + { 0x01, "DISPLAY" }, + { 0x02, "CAPTURE" }, + { 0x03, "IFB", NULL, NVKM_ENGINE_IFB }, + { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, + { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, + { 0x06, "SCHED" }, + { 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO }, + { 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO }, + { 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO }, + { 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO }, + { 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO }, + { 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO }, + { 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO }, + { 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO }, + { 0x0f, "HOSTSR" }, + { 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD }, + { 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP }, + { 0x13, "PERF" }, + { 0x14, "MSPDEC", NULL, NVKM_ENGINE_MSPDEC }, + { 0x15, "CE0", NULL, NVKM_ENGINE_CE, 0 }, + { 0x16, "CE1", NULL, NVKM_ENGINE_CE, 1 }, + { 0x17, "PMU" }, + { 0x18, "PTP" }, + { 0x19, "MSENC", NULL, NVKM_ENGINE_MSENC }, + { 0x1b, "CE2", NULL, NVKM_ENGINE_CE, 2 }, + {} +}; + +const struct nvkm_enum +gk104_fifo_fault_reason[] = { + { 0x00, "PDE" }, + { 0x01, "PDE_SIZE" }, + { 0x02, "PTE" }, + { 0x03, "VA_LIMIT_VIOLATION" }, + { 0x04, "UNBOUND_INST_BLOCK" }, + { 0x05, "PRIV_VIOLATION" }, + { 0x06, "RO_VIOLATION" }, + { 0x07, "WO_VIOLATION" }, + { 0x08, "PITCH_MASK_VIOLATION" }, + { 0x09, "WORK_CREATION" }, + { 0x0a, "UNSUPPORTED_APERTURE" }, + { 0x0b, "COMPRESSION_FAILURE" }, + { 0x0c, "UNSUPPORTED_KIND" }, + { 0x0d, "REGION_VIOLATION" }, + { 0x0e, "BOTH_PTES_VALID" }, + { 0x0f, "INFO_TYPE_POISONED" }, + {} +}; + +const struct nvkm_enum +gk104_fifo_fault_hubclient[] = { + { 0x00, "VIP" }, + { 0x01, "CE0" }, + { 0x02, "CE1" }, + { 0x03, "DNISO" }, + { 0x04, "FE" }, + { 0x05, "FECS" }, + { 0x06, "HOST" }, + { 0x07, "HOST_CPU" }, + { 0x08, "HOST_CPU_NB" }, + { 0x09, "ISO" }, + { 0x0a, "MMU" }, + { 0x0b, "MSPDEC" }, + { 0x0c, "MSPPP" }, + { 0x0d, "MSVLD" }, + { 0x0e, "NISO" }, + { 0x0f, "P2P" }, + { 0x10, "PD" }, + { 0x11, "PERF" }, + { 0x12, "PMU" }, + { 0x13, "RASTERTWOD" }, + { 0x14, "SCC" }, + { 0x15, "SCC_NB" }, + { 0x16, "SEC" }, + { 0x17, "SSYNC" }, + { 0x18, "GR_CE" }, + { 0x19, "CE2" }, + { 0x1a, "XV" }, + { 0x1b, "MMU_NB" }, + { 0x1c, "MSENC" }, + { 0x1d, "DFALCON" }, + { 0x1e, "SKED" }, + { 0x1f, "AFALCON" }, + {} +}; + +const struct nvkm_enum +gk104_fifo_fault_gpcclient[] = { + { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, + { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, + { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, + { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, + { 0x0c, "RAST" }, + { 0x0d, "GCC" }, + { 0x0e, "GPCCS" }, + { 0x0f, "PROP_0" }, + { 0x10, "PROP_1" }, + { 0x11, "PROP_2" }, + { 0x12, "PROP_3" }, + { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, + { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, + { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, + { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, + { 0x1f, "GPM" }, + { 0x20, "LTP_UTLB_0" }, + { 0x21, "LTP_UTLB_1" }, + { 0x22, "LTP_UTLB_2" }, + { 0x23, "LTP_UTLB_3" }, + { 0x24, "GPC_RGG_UTLB" }, + {} +}; + static void gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) { @@ -624,40 +810,6 @@ gk104_fifo_intr_dropped_fault(struct gk104_fifo *fifo) nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat); } -static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { - { 0x00000001, "MEMREQ" }, - { 0x00000002, "MEMACK_TIMEOUT" }, - { 0x00000004, "MEMACK_EXTRA" }, - { 0x00000008, "MEMDAT_TIMEOUT" }, - { 0x00000010, "MEMDAT_EXTRA" }, - { 0x00000020, "MEMFLUSH" }, - { 0x00000040, "MEMOP" }, - { 0x00000080, "LBCONNECT" }, - { 0x00000100, "LBREQ" }, - { 0x00000200, "LBACK_TIMEOUT" }, - { 0x00000400, "LBACK_EXTRA" }, - { 0x00000800, "LBDAT_TIMEOUT" }, - { 0x00001000, "LBDAT_EXTRA" }, - { 0x00002000, "GPFIFO" }, - { 0x00004000, "GPPTR" }, - { 0x00008000, "GPENTRY" }, - { 0x00010000, "GPCRC" }, - { 0x00020000, "PBPTR" }, - { 0x00040000, "PBENTRY" }, - { 0x00080000, "PBCRC" }, - { 0x00100000, "XBARCONNECT" }, - { 0x00200000, "METHOD" }, - { 0x00400000, "METHODCRC" }, - { 0x00800000, "DEVICE" }, - { 0x02000000, "SEMAPHORE" }, - { 0x04000000, "ACQUIRE" }, - { 0x08000000, "PRI" }, - { 0x20000000, "NO_CTXSW_SEG" }, - { 0x40000000, "PBSEG" }, - { 0x80000000, "SIGNATURE" }, - {} -}; - void gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit) { @@ -698,36 +850,6 @@ gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit) nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); } -static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = { - { 0x00000001, "HCE_RE_ILLEGAL_OP" }, - { 0x00000002, "HCE_RE_ALIGNB" }, - { 0x00000004, "HCE_PRIV" }, - { 0x00000008, "HCE_ILLEGAL_MTHD" }, - { 0x00000010, "HCE_ILLEGAL_CLASS" }, - {} -}; - -void -gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000)); - u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask; - u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff; - char msg[128]; - - if (stat) { - nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat); - nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n", - unit, stat, msg, chid, - nvkm_rd32(device, 0x040150 + (unit * 0x2000)), - nvkm_rd32(device, 0x040154 + (unit * 0x2000))); - } - - nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat); -} - void gk104_fifo_intr_runlist(struct gk104_fifo *fifo) { @@ -899,6 +1021,44 @@ gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data) } } +void +gk104_fifo_init(struct nvkm_fifo *base) +{ + struct gk104_fifo *fifo = gk104_fifo(base); + struct nvkm_device *device = fifo->base.engine.subdev.device; + int i; + + /* Enable PBDMAs. */ + fifo->func->pbdma->init(fifo); + + /* PBDMA[n] */ + for (i = 0; i < fifo->pbdma_nr; i++) { + nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); + nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ + nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ + } + + /* PBDMA[n].HCE */ + for (i = 0; i < fifo->pbdma_nr; i++) { + nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */ + nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ + } + + nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); + + if (fifo->func->pbdma->init_timeout) + fifo->func->pbdma->init_timeout(fifo); + + nvkm_wr32(device, 0x002100, 0xffffffff); + nvkm_wr32(device, 0x002140, 0x7fffffff); +} + +int +gk104_fifo_chid_nr(struct nvkm_fifo *fifo) +{ + return 4096; +} + int gk104_fifo_oneinit(struct nvkm_fifo *base) { @@ -990,44 +1150,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); } -void -gk104_fifo_init(struct nvkm_fifo *base) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; - int i; - - /* Enable PBDMAs. */ - fifo->func->pbdma->init(fifo); - - /* PBDMA[n] */ - for (i = 0; i < fifo->pbdma_nr; i++) { - nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); - nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ - nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ - } - - /* PBDMA[n].HCE */ - for (i = 0; i < fifo->pbdma_nr; i++) { - nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */ - nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ - } - - nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); - - if (fifo->func->pbdma->init_timeout) - fifo->func->pbdma->init_timeout(fifo); - - nvkm_wr32(device, 0x002100, 0xffffffff); - nvkm_wr32(device, 0x002140, 0x7fffffff); -} - -int -gk104_fifo_chid_nr(struct nvkm_fifo *fifo) -{ - return 4096; -} - void * gk104_fifo_dtor(struct nvkm_fifo *base) { @@ -1077,128 +1199,6 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, return nvkm_fifo_ctor(&gk104_fifo_, device, type, inst, &fifo->base); } -const struct nvkm_enum -gk104_fifo_fault_access[] = { - { 0x0, "READ" }, - { 0x1, "WRITE" }, - {} -}; - -const struct nvkm_enum -gk104_fifo_fault_engine[] = { - { 0x00, "GR", NULL, NVKM_ENGINE_GR }, - { 0x01, "DISPLAY" }, - { 0x02, "CAPTURE" }, - { 0x03, "IFB", NULL, NVKM_ENGINE_IFB }, - { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, - { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, - { 0x06, "SCHED" }, - { 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO }, - { 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO }, - { 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO }, - { 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO }, - { 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO }, - { 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO }, - { 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO }, - { 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO }, - { 0x0f, "HOSTSR" }, - { 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD }, - { 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP }, - { 0x13, "PERF" }, - { 0x14, "MSPDEC", NULL, NVKM_ENGINE_MSPDEC }, - { 0x15, "CE0", NULL, NVKM_ENGINE_CE, 0 }, - { 0x16, "CE1", NULL, NVKM_ENGINE_CE, 1 }, - { 0x17, "PMU" }, - { 0x18, "PTP" }, - { 0x19, "MSENC", NULL, NVKM_ENGINE_MSENC }, - { 0x1b, "CE2", NULL, NVKM_ENGINE_CE, 2 }, - {} -}; - -const struct nvkm_enum -gk104_fifo_fault_reason[] = { - { 0x00, "PDE" }, - { 0x01, "PDE_SIZE" }, - { 0x02, "PTE" }, - { 0x03, "VA_LIMIT_VIOLATION" }, - { 0x04, "UNBOUND_INST_BLOCK" }, - { 0x05, "PRIV_VIOLATION" }, - { 0x06, "RO_VIOLATION" }, - { 0x07, "WO_VIOLATION" }, - { 0x08, "PITCH_MASK_VIOLATION" }, - { 0x09, "WORK_CREATION" }, - { 0x0a, "UNSUPPORTED_APERTURE" }, - { 0x0b, "COMPRESSION_FAILURE" }, - { 0x0c, "UNSUPPORTED_KIND" }, - { 0x0d, "REGION_VIOLATION" }, - { 0x0e, "BOTH_PTES_VALID" }, - { 0x0f, "INFO_TYPE_POISONED" }, - {} -}; - -const struct nvkm_enum -gk104_fifo_fault_hubclient[] = { - { 0x00, "VIP" }, - { 0x01, "CE0" }, - { 0x02, "CE1" }, - { 0x03, "DNISO" }, - { 0x04, "FE" }, - { 0x05, "FECS" }, - { 0x06, "HOST" }, - { 0x07, "HOST_CPU" }, - { 0x08, "HOST_CPU_NB" }, - { 0x09, "ISO" }, - { 0x0a, "MMU" }, - { 0x0b, "MSPDEC" }, - { 0x0c, "MSPPP" }, - { 0x0d, "MSVLD" }, - { 0x0e, "NISO" }, - { 0x0f, "P2P" }, - { 0x10, "PD" }, - { 0x11, "PERF" }, - { 0x12, "PMU" }, - { 0x13, "RASTERTWOD" }, - { 0x14, "SCC" }, - { 0x15, "SCC_NB" }, - { 0x16, "SEC" }, - { 0x17, "SSYNC" }, - { 0x18, "GR_CE" }, - { 0x19, "CE2" }, - { 0x1a, "XV" }, - { 0x1b, "MMU_NB" }, - { 0x1c, "MSENC" }, - { 0x1d, "DFALCON" }, - { 0x1e, "SKED" }, - { 0x1f, "AFALCON" }, - {} -}; - -const struct nvkm_enum -gk104_fifo_fault_gpcclient[] = { - { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, - { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, - { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, - { 0x09, "L1_3" }, { 0x0a, "T1_3" }, { 0x0b, "PE_3" }, - { 0x0c, "RAST" }, - { 0x0d, "GCC" }, - { 0x0e, "GPCCS" }, - { 0x0f, "PROP_0" }, - { 0x10, "PROP_1" }, - { 0x11, "PROP_2" }, - { 0x12, "PROP_3" }, - { 0x13, "L1_4" }, { 0x14, "T1_4" }, { 0x15, "PE_4" }, - { 0x16, "L1_5" }, { 0x17, "T1_5" }, { 0x18, "PE_5" }, - { 0x19, "L1_6" }, { 0x1a, "T1_6" }, { 0x1b, "PE_6" }, - { 0x1c, "L1_7" }, { 0x1d, "T1_7" }, { 0x1e, "PE_7" }, - { 0x1f, "GPM" }, - { 0x20, "LTP_UTLB_0" }, - { 0x21, "LTP_UTLB_1" }, - { 0x22, "LTP_UTLB_2" }, - { 0x23, "LTP_UTLB_3" }, - { 0x24, "GPC_RGG_UTLB" }, - {} -}; - static const struct gk104_fifo_func gk104_fifo = { .chid_nr = gk104_fifo_chid_nr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 95a30fbf65a10..f9f8371d37565 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -49,6 +49,72 @@ nv04_fifo_ramfc[] = { {} }; +void +nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) +{ + struct nv04_fifo_chan *chan = nv04_fifo_chan(base); + struct nv04_fifo *fifo = chan->fifo; + struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_memory *fctx = device->imem->ramfc; + const struct nv04_fifo_ramfc *c; + unsigned long flags; + u32 mask = fifo->base.nr - 1; + u32 data = chan->ramfc; + u32 chid; + + /* prevent fifo context switches */ + spin_lock_irqsave(&fifo->base.lock, flags); + nvkm_wr32(device, NV03_PFIFO_CACHES, 0); + + /* if this channel is active, replace it with a null context */ + chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & mask; + if (chid == chan->base.chid) { + nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); + nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); + + c = fifo->ramfc; + nvkm_kmap(fctx); + do { + u32 rm = ((1ULL << c->bits) - 1) << c->regs; + u32 cm = ((1ULL << c->bits) - 1) << c->ctxs; + u32 rv = (nvkm_rd32(device, c->regp) & rm) >> c->regs; + u32 cv = (nvkm_ro32(fctx, c->ctxp + data) & ~cm); + nvkm_wo32(fctx, c->ctxp + data, cv | (rv << c->ctxs)); + } while ((++c)->bits); + nvkm_done(fctx); + + c = fifo->ramfc; + do { + nvkm_wr32(device, c->regp, 0x00000000); + } while ((++c)->bits); + + nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, mask); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); + nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); + } + + /* restore normal operation, after disabling dma mode */ + nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); + nvkm_wr32(device, NV03_PFIFO_CACHES, 1); + spin_unlock_irqrestore(&fifo->base.lock, flags); +} + +void +nv04_fifo_dma_init(struct nvkm_fifo_chan *base) +{ + struct nv04_fifo_chan *chan = nv04_fifo_chan(base); + struct nv04_fifo *fifo = chan->fifo; + struct nvkm_device *device = fifo->base.engine.subdev.device; + u32 mask = 1 << chan->base.chid; + unsigned long flags; + spin_lock_irqsave(&fifo->base.lock, flags); + nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); + spin_unlock_irqrestore(&fifo->base.lock, flags); +} + static const struct nvkm_chan_func nv04_chan = { }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index f3f11a324591d..101015a1a7e94 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -64,22 +64,6 @@ nv50_fifo_runlist_update(struct nv50_fifo *fifo) mutex_unlock(&fifo->base.mutex); } -int -nv50_fifo_oneinit(struct nvkm_fifo *base) -{ - struct nv50_fifo *fifo = nv50_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; - int ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000, - false, &fifo->runlist[0]); - if (ret) - return ret; - - return nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000, - false, &fifo->runlist[1]); -} - void nv50_fifo_init(struct nvkm_fifo *base) { @@ -110,6 +94,22 @@ nv50_fifo_chid_nr(struct nvkm_fifo *fifo) return 128; } +int +nv50_fifo_oneinit(struct nvkm_fifo *base) +{ + struct nv50_fifo *fifo = nv50_fifo(base); + struct nvkm_device *device = fifo->base.engine.subdev.device; + int ret; + + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000, + false, &fifo->runlist[0]); + if (ret) + return ret; + + return nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000, + false, &fifo->runlist[1]); +} + void * nv50_fifo_dtor(struct nvkm_fifo *base) { -- GitLab From 9be9c606c4be74b5e8cbebe5b1ea96821e27b04f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:22 +1000 Subject: [PATCH 1304/1927] drm/nouveau/fifo: merge gk104_fifo_func into nvkm_host_func This makes it easier to transition everything. - a couple of function renames for collisions Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 9 +--- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 11 +++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 49 ++++++++++--------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 39 +-------------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 18 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 18 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 18 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 25 ++++++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 18 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 25 ++++++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 21 +++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 46 ++++++++++++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 29 +++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 4 +- 14 files changed, 219 insertions(+), 111 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index b06db3b3d4714..4e099bf0a3fa9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -30,8 +30,6 @@ #include #include -#include "gk104.h" - void nvkm_fifo_recover_chan(struct nvkm_fifo *fifo, int chid) { @@ -58,7 +56,7 @@ nvkm_fifo_start(struct nvkm_fifo *fifo, unsigned long *flags) void nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) { - return fifo->func->fault(fifo, info); + return fifo->func->mmu_fault->recover(fifo, info); } void @@ -179,9 +177,6 @@ nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_dev const struct nvkm_fifo_func_chan *chan = &fifo->func->chan; int c = 0; - if (fifo->func->engine_id == gk104_fifo_engine_id) - chan = &gk104_fifo(fifo)->func->chan; - /* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */ if (chan->user.oclass) { if (c++ == index) { @@ -289,7 +284,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, INIT_LIST_HEAD(&fifo->chan); - nr = func->chid_nr ? func->chid_nr(fifo) : gk104_fifo(fifo)->func->chid_nr(fifo); + nr = func->chid_nr(fifo); if (WARN_ON(fifo->nr > NVKM_FIFO_CHID_NR)) fifo->nr = NVKM_FIFO_CHID_NR; else diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index bd0b0c66e28f6..1a641b1fdfbf1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -359,6 +359,11 @@ gf100_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) nvkm_fifo_chan_put(&fifo->base, flags, &chan); } +static const struct nvkm_fifo_func_mmu_fault +gf100_fifo_mmu_fault = { + .recover = gf100_fifo_fault, +}; + static const struct nvkm_enum gf100_fifo_sched_reason[] = { { 0x0a, "CTXSW_TIMEOUT" }, @@ -422,7 +427,7 @@ gf100_fifo_intr_sched(struct gf100_fifo *fifo) } void -gf100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) +gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) { struct nvkm_device *device = fifo->engine.subdev.device; u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10)); @@ -541,7 +546,7 @@ gf100_fifo_intr(struct nvkm_fifo *base) u32 mask = nvkm_rd32(device, 0x00259c); while (mask) { u32 unit = __ffs(mask); - gf100_fifo_intr_fault(&fifo->base, unit); + gf100_fifo_intr_mmu_fault_unit(&fifo->base, unit); nvkm_wr32(device, 0x00259c, (1 << unit)); mask &= ~(1 << unit); } @@ -679,7 +684,7 @@ gf100_fifo = { .init = gf100_fifo_init, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, - .fault = gf100_fifo_fault, + .mmu_fault = &gf100_fifo_mmu_fault, .engine_id = gf100_fifo_engine_id, .id_engine = gf100_fifo_id_engine, .uevent_init = gf100_fifo_uevent_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index da26c5219e53e..880ddfec22f7a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -397,7 +397,7 @@ gk104_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid) return NULL; } -static void +void gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid) { struct gk104_fifo *fifo = gk104_fifo(base); @@ -629,7 +629,7 @@ gk104_fifo_fault_gpcclient[] = { {} }; -static void +void gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) { struct gk104_fifo *fifo = gk104_fifo(base); @@ -709,6 +709,11 @@ gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) spin_unlock_irqrestore(&fifo->base.lock, flags); } +const struct nvkm_fifo_func_mmu_fault +gk104_fifo_mmu_fault = { + .recover = gk104_fifo_fault, +}; + static const struct nvkm_enum gk104_fifo_bind_reason[] = { { 0x01, "BIND_NOT_UNBOUND" }, @@ -869,7 +874,7 @@ gk104_fifo_intr_engine(struct gk104_fifo *fifo) nvkm_fifo_uevent(&fifo->base); } -static void +void gk104_fifo_intr(struct nvkm_fifo *base) { struct gk104_fifo *fifo = gk104_fifo(base); @@ -924,7 +929,7 @@ gk104_fifo_intr(struct nvkm_fifo *base) u32 mask = nvkm_rd32(device, 0x00259c); while (mask) { u32 unit = __ffs(mask); - fifo->func->intr.fault(&fifo->base, unit); + fifo->func->intr_mmu_fault_unit(&fifo->base, unit); nvkm_wr32(device, 0x00259c, (1 << unit)); mask &= ~(1 << unit); } @@ -1168,22 +1173,6 @@ gk104_fifo_dtor(struct nvkm_fifo *base) return fifo; } -static const struct nvkm_fifo_func -gk104_fifo_ = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, - .info = gk104_fifo_info, - .init = gk104_fifo_init, - .fini = gk104_fifo_fini, - .intr = gk104_fifo_intr, - .fault = gk104_fifo_fault, - .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, - .recover_chan = gk104_fifo_recover_chan, -}; - int gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, int nr, struct nvkm_fifo **pfifo) @@ -1196,20 +1185,32 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, INIT_WORK(&fifo->recover.work, gk104_fifo_recover_work); *pfifo = &fifo->base; - return nvkm_fifo_ctor(&gk104_fifo_, device, type, inst, &fifo->base); + return nvkm_fifo_ctor(func, device, type, inst, &fifo->base); } -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gk104_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, - .intr.fault = gf100_fifo_intr_fault, - .pbdma = &gk104_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .mmu_fault = &gk104_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, + .pbdma = &gk104_fifo_pbdma, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index be426734da02e..9621877400106 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -8,6 +8,8 @@ struct nvkm_fifo_cgrp; #include #include +#define gk104_fifo_func nvkm_fifo_func + struct gk104_fifo_chan; struct gk104_fifo { const struct gk104_fifo_func *func; @@ -45,41 +47,6 @@ struct gk104_fifo { } user; }; -struct gk104_fifo_func { - int (*chid_nr)(struct nvkm_fifo *); - - struct { - void (*fault)(struct nvkm_fifo *, int unit); - } intr; - - const struct gk104_fifo_pbdma_func { - int (*nr)(struct gk104_fifo *); - void (*init)(struct gk104_fifo *); - void (*init_timeout)(struct gk104_fifo *); - } *pbdma; - - struct { - const struct nvkm_enum *access; - const struct nvkm_enum *engine; - const struct nvkm_enum *reason; - const struct nvkm_enum *hubclient; - const struct nvkm_enum *gpcclient; - } fault; - - const struct gk104_fifo_runlist_func { - u8 size; - void (*cgrp)(struct nvkm_fifo_cgrp *, - struct nvkm_memory *, u32 offset); - void (*chan)(struct gk104_fifo_chan *, - struct nvkm_memory *, u32 offset); - void (*commit)(struct gk104_fifo *, int runl, - struct nvkm_memory *, int entries); - } *runlist; - - struct nvkm_fifo_func_cgrp cgrp; - struct nvkm_fifo_func_chan chan; -}; - struct gk104_fifo_engine_status { bool busy; bool faulted; @@ -135,14 +102,12 @@ void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, extern const struct gk104_fifo_pbdma_func gk208_fifo_pbdma; void gk208_fifo_pbdma_init_timeout(struct gk104_fifo *); -void gm107_fifo_intr_fault(struct nvkm_fifo *, int); extern const struct nvkm_enum gm107_fifo_fault_engine[]; extern const struct gk104_fifo_runlist_func gm107_fifo_runlist; extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma; int gm200_fifo_pbdma_nr(struct gk104_fifo *); -void gp100_fifo_intr_fault(struct nvkm_fifo *, int); extern const struct nvkm_enum gp100_fifo_fault_engine[]; extern const struct nvkm_enum gv100_fifo_fault_access[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index df9b83cf382fb..9dcc54f1d016b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -56,17 +56,29 @@ gk110_fifo_runlist = { .commit = gk104_fifo_runlist_commit, }; -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gk110_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, - .intr.fault = gf100_fifo_intr_fault, - .pbdma = &gk104_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .mmu_fault = &gk104_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, + .pbdma = &gk104_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 003f244f89db8..28791baf77f8a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -49,17 +49,29 @@ gk208_fifo_chid_nr(struct nvkm_fifo *fifo) return 1024; } -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gk208_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gk208_fifo_chid_nr, - .intr.fault = gf100_fifo_intr_fault, - .pbdma = &gk208_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .mmu_fault = &gk104_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, + .pbdma = &gk208_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 661c9f0cf7b9c..b62e3bb0294ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -24,17 +24,29 @@ #include -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gk20a_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, - .intr.fault = gf100_fifo_intr_fault, - .pbdma = &gk208_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .mmu_fault = &gk104_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gk104_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, + .pbdma = &gk208_fifo_pbdma, .cgrp = {{ }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 910d97911a98d..f996f1c1141a0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -74,8 +74,13 @@ gm107_fifo_fault_engine[] = { {} }; +const struct nvkm_fifo_func_mmu_fault +gm107_fifo_mmu_fault = { + .recover = gk104_fifo_fault, +}; + void -gm107_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) +gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) { struct nvkm_device *device = fifo->engine.subdev.device; u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10)); @@ -104,17 +109,29 @@ gm107_fifo_chid_nr(struct nvkm_fifo *fifo) return 2048; } -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gm107_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gm107_fifo_chid_nr, - .intr.fault = gm107_fifo_intr_fault, - .pbdma = &gk208_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, + .mmu_fault = &gm107_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gm107_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, + .pbdma = &gk208_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 101027536a93e..8f3da21ce1e76 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -46,17 +46,29 @@ gm200_fifo_chid_nr(struct nvkm_fifo *fifo) return nvkm_rd32(fifo->engine.subdev.device, 0x002008); } -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gm200_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, - .intr.fault = gm107_fifo_intr_fault, - .pbdma = &gm200_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, + .mmu_fault = &gm107_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gm107_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, + .pbdma = &gm200_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index c52a4a0b361b3..dc590848c25c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -52,8 +52,13 @@ gp100_fifo_fault_engine[] = { {} }; +static const struct nvkm_fifo_func_mmu_fault +gp100_fifo_mmu_fault = { + .recover = gk104_fifo_fault, +}; + void -gp100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) +gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) { struct nvkm_device *device = fifo->engine.subdev.device; u32 inst = nvkm_rd32(device, 0x002800 + (unit * 0x10)); @@ -76,17 +81,29 @@ gp100_fifo_intr_fault(struct nvkm_fifo *fifo, int unit) nvkm_fifo_fault(fifo, &info); } -static const struct gk104_fifo_func +static const struct nvkm_fifo_func gp100_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, - .intr.fault = gp100_fifo_intr_fault, - .pbdma = &gm200_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .intr_mmu_fault_unit = gp100_fifo_intr_mmu_fault_unit, + .mmu_fault = &gp100_fifo_mmu_fault, .fault.access = gk104_fifo_fault_access, .fault.engine = gp100_fifo_fault_engine, .fault.reason = gk104_fifo_fault_reason, .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, + .pbdma = &gm200_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index ddca298528656..36c02b5aac0f9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -291,16 +291,33 @@ gv100_fifo_fault_access[] = { {} }; -static const struct gk104_fifo_func +static const struct nvkm_fifo_func_mmu_fault +gv100_fifo_mmu_fault = { + .recover = gk104_fifo_fault, +}; + +static const struct nvkm_fifo_func gv100_fifo = { + .dtor = gk104_fifo_dtor, + .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, - .pbdma = &gm200_fifo_pbdma, + .info = gk104_fifo_info, + .init = gk104_fifo_init, + .fini = gk104_fifo_fini, + .intr = gk104_fifo_intr, + .mmu_fault = &gv100_fifo_mmu_fault, .fault.access = gv100_fifo_fault_access, .fault.engine = gv100_fifo_fault_engine, .fault.reason = gv100_fifo_fault_reason, .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, + .pbdma = &gm200_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index b8031c212c8bd..ba3dbc86435e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -4,7 +4,9 @@ #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) #include struct nvkm_cgrp; +struct nvkm_memory; struct gk104_fifo; +struct gk104_fifo_chan; void nvkm_fifo_uevent(struct nvkm_fifo *); void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); @@ -23,8 +25,22 @@ struct nvkm_fifo_func { int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data); void (*init)(struct nvkm_fifo *); void (*fini)(struct nvkm_fifo *); + void (*intr)(struct nvkm_fifo *); - void (*fault)(struct nvkm_fifo *, struct nvkm_fault_data *); + void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit); + + const struct nvkm_fifo_func_mmu_fault { + void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *); + } *mmu_fault; + + struct { + const struct nvkm_enum *access; + const struct nvkm_enum *engine; + const struct nvkm_enum *reason; + const struct nvkm_enum *hubclient; + const struct nvkm_enum *gpcclient; + } fault; + int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *(*id_engine)(struct nvkm_fifo *, int engi); void (*pause)(struct nvkm_fifo *, unsigned long *); @@ -33,6 +49,22 @@ struct nvkm_fifo_func { void (*uevent_fini)(struct nvkm_fifo *); void (*recover_chan)(struct nvkm_fifo *, int chid); + const struct gk104_fifo_runlist_func { + u8 size; + void (*cgrp)(struct nvkm_fifo_cgrp *, + struct nvkm_memory *, u32 offset); + void (*chan)(struct gk104_fifo_chan *, + struct nvkm_memory *, u32 offset); + void (*commit)(struct gk104_fifo *, int runl, + struct nvkm_memory *, int entries); + } *runlist; + + const struct gk104_fifo_pbdma_func { + int (*nr)(struct gk104_fifo *); + void (*init)(struct gk104_fifo *); + void (*init_timeout)(struct gk104_fifo *); + } *pbdma; + struct nvkm_fifo_func_cgrp { struct nvkm_sclass user; const struct nvkm_cgrp_func *func; @@ -67,19 +99,29 @@ int nv50_fifo_chid_nr(struct nvkm_fifo *); extern const struct nvkm_chan_func g84_chan; -void gf100_fifo_intr_fault(struct nvkm_fifo *, int); +void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); int gk104_fifo_chid_nr(struct nvkm_fifo *); +void gk104_fifo_intr(struct nvkm_fifo *); +extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; +void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); +void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); extern const struct nvkm_cgrp_func gk110_cgrp; extern const struct nvkm_chan_func gk110_chan; +void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); +extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; extern const struct nvkm_chan_func gm107_chan; int gm200_fifo_chid_nr(struct nvkm_fifo *); +void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); + +extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; + int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, void *argv, u32 argc, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index db9b802bd67c7..274096316a1b1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -339,6 +339,11 @@ tu102_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) spin_unlock_irqrestore(&fifo->base.lock, flags); } +const struct nvkm_fifo_func_mmu_fault +tu102_fifo_mmu_fault = { + .recover = tu102_fifo_fault, +}; + static void tu102_fifo_intr_ctxsw_timeout(struct gk104_fifo *fifo) { @@ -433,31 +438,27 @@ tu102_fifo_intr(struct nvkm_fifo *base) } static const struct nvkm_fifo_func -tu102_fifo_ = { +tu102_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, + .chid_nr = gm200_fifo_chid_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = tu102_fifo_intr, - .fault = tu102_fifo_fault, - .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, - .recover_chan = tu102_fifo_recover_chan, -}; - -static const struct gk104_fifo_func -tu102_fifo = { - .chid_nr = gm200_fifo_chid_nr, - .pbdma = &tu102_fifo_pbdma, + .mmu_fault = &tu102_fifo_mmu_fault, .fault.access = gv100_fifo_fault_access, .fault.engine = tu102_fifo_fault_engine, .fault.reason = gv100_fifo_fault_reason, .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, + .engine_id = gk104_fifo_engine_id, + .id_engine = gk104_fifo_id_engine, + .uevent_init = gk104_fifo_uevent_init, + .uevent_fini = gk104_fifo_uevent_fini, + .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, + .pbdma = &tu102_fifo_pbdma, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new }, }; @@ -474,5 +475,5 @@ tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, INIT_WORK(&fifo->recover.work, tu102_fifo_recover_work); *pfifo = &fifo->base; - return nvkm_fifo_ctor(&tu102_fifo_, device, type, inst, &fifo->base); + return nvkm_fifo_ctor(&tu102_fifo, device, type, inst, &fifo->base); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index a1d93f28b634b..42b49043deefb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -162,8 +162,8 @@ nvkm_uchan_new(struct nvkm_fifo *fifo, struct nvkm_cgrp *cgrp, const struct nvkm nvkm_object_ctor(&nvkm_uchan, oclass, &uchan->object); *pobject = &uchan->object; - if (!fifo->func->chan.func) - ret = gk104_fifo(fifo)->func->chan.ctor(gk104_fifo(fifo), oclass, argv, argc, &object); + if (fifo->func->chan.ctor) + ret = fifo->func->chan.ctor(gk104_fifo(fifo), oclass, argv, argc, &object); else ret = fifo->func->chan.oclass->ctor(fifo, oclass, argv, argc, &object); if (!object) -- GitLab From 800ac1f8d708ea2b70ac55b3029687c1b4283431 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:23 +1000 Subject: [PATCH 1305/1927] drm/nouveau/fifo: add chid allocator We need to be able to allocate TSG IDs as well as channel IDs, also, Ampere has per-runlist channel IDs. - holds per-ID private data, which will be used for/to protect lookup - holds an nvkm_event which will be used for events tied to IDs - not used yet beyond setup, and switching use of "fifo->nr - 1" for channel ID mask to "chid->mask" Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 20 ++++- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/chid.c | 82 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/chid.h | 23 ++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 8 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 14 ++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 23 ++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 9 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 6 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 + 26 files changed, 209 insertions(+), 21 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 2f2059c5d4c9e..b688121f1a91e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -37,6 +37,9 @@ struct nvkm_fifo { const struct nvkm_fifo_func *func; struct nvkm_engine engine; + struct nvkm_chid *chid; + struct nvkm_chid *cgid; + DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR); int nr; struct list_head chan; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 76d2bf1f6d8b4..c985736bff2c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -1,5 +1,8 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/engine/fifo/base.o +nvkm-y += nvkm/engine/fifo/chan.o +nvkm-y += nvkm/engine/fifo/chid.o + nvkm-y += nvkm/engine/fifo/nv04.o nvkm-y += nvkm/engine/fifo/nv10.o nvkm-y += nvkm/engine/fifo/nv17.o @@ -18,7 +21,6 @@ nvkm-y += nvkm/engine/fifo/gv100.o nvkm-y += nvkm/engine/fifo/tu102.o nvkm-y += nvkm/engine/fifo/ga102.o -nvkm-y += nvkm/engine/fifo/chan.o nvkm-y += nvkm/engine/fifo/channv50.o nvkm-y += nvkm/engine/fifo/chang84.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 4e099bf0a3fa9..e6b0653625f5a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -23,6 +23,7 @@ */ #include "priv.h" #include "chan.h" +#include "chid.h" #include #include @@ -218,13 +219,15 @@ static int nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + switch (mthd) { - case NV_DEVICE_HOST_CHANNELS: *data = fifo->nr; return 0; + case NV_DEVICE_HOST_CHANNELS: *data = fifo->chid ? fifo->chid->nr : 0; return 0; default: if (fifo->func->info) return fifo->func->info(fifo, mthd, data); break; } + return -ENOSYS; } @@ -232,8 +235,18 @@ static int nvkm_fifo_oneinit(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + int ret; + + /* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */ + if (fifo->func->chid_nr) { + ret = fifo->func->chid_ctor(fifo, fifo->func->chid_nr(fifo)); + if (ret) + return ret; + } + if (fifo->func->oneinit) return fifo->func->oneinit(fifo); + return 0; } @@ -248,6 +261,10 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); void *data = fifo; + + nvkm_chid_unref(&fifo->cgid); + nvkm_chid_unref(&fifo->chid); + if (fifo->func->dtor) data = fifo->func->dtor(fifo); nvkm_event_fini(&fifo->kevent); @@ -289,7 +306,6 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, fifo->nr = NVKM_FIFO_CHID_NR; else fifo->nr = nr; - bitmap_clear(fifo->mask, 0, fifo->nr); if (func->uevent_init) { ret = nvkm_event_init(&nvkm_fifo_uevent_func, &fifo->engine.subdev, 1, 1, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 4d4905be4b92b..464c956aaca9d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -32,6 +32,10 @@ #include +const struct nvkm_event_func +nvkm_chan_event = { +}; + struct nvkm_fifo_chan_object { struct nvkm_oproxy oproxy; struct nvkm_fifo_chan *chan; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 24fbe6cf70e1b..a9c0e02b6f9f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -4,6 +4,8 @@ #define nvkm_chan(p) container_of((p), struct nvkm_chan, object) /*FIXME: remove later */ #include +extern const struct nvkm_event_func nvkm_chan_event; + struct nvkm_chan_func { void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c new file mode 100644 index 0000000000000..7c582bce7e245 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "chid.h" + +static void +nvkm_chid_del(struct kref *kref) +{ + struct nvkm_chid *chid = container_of(kref, typeof(*chid), kref); + + nvkm_event_fini(&chid->event); + + kvfree(chid->data); + kfree(chid); +} + +void +nvkm_chid_unref(struct nvkm_chid **pchid) +{ + struct nvkm_chid *chid = *pchid; + + if (!chid) + return; + + kref_put(&chid->kref, nvkm_chid_del); + *pchid = NULL; +} + +struct nvkm_chid * +nvkm_chid_ref(struct nvkm_chid *chid) +{ + if (chid) + kref_get(&chid->kref); + + return chid; +} + +int +nvkm_chid_new(const struct nvkm_event_func *func, struct nvkm_subdev *subdev, + int nr, int first, int count, struct nvkm_chid **pchid) +{ + struct nvkm_chid *chid; + int id; + + if (!(chid = *pchid = kzalloc(struct_size(chid, used, nr), GFP_KERNEL))) + return -ENOMEM; + + kref_init(&chid->kref); + chid->nr = nr; + chid->mask = chid->nr - 1; + spin_lock_init(&chid->lock); + + if (!(chid->data = kvzalloc(sizeof(*chid->data) * nr, GFP_KERNEL))) { + nvkm_chid_unref(pchid); + return -ENOMEM; + } + + for (id = 0; id < first; id++) + __set_bit(id, chid->used); + for (id = first + count; id < nr; id++) + __set_bit(id, chid->used); + + return nvkm_event_init(func, subdev, 1, nr, &chid->event); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h new file mode 100644 index 0000000000000..d359828dde809 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_CHID_H__ +#define __NVKM_CHID_H__ +#include + +struct nvkm_chid { + struct kref kref; + int nr; + u32 mask; + + struct nvkm_event event; + + void **data; + + spinlock_t lock; + unsigned long used[]; +}; + +int nvkm_chid_new(const struct nvkm_event_func *, struct nvkm_subdev *, + int nr, int first, int count, struct nvkm_chid **pchid); +struct nvkm_chid *nvkm_chid_ref(struct nvkm_chid *); +void nvkm_chid_unref(struct nvkm_chid **); +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 24732d355c831..dbb9df6f8eb51 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -119,6 +119,7 @@ g84_fifo = { .dtor = nv50_fifo_dtor, .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, + .chid_ctor = nv50_fifo_chid_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = g84_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 1a641b1fdfbf1..d8eeca8186372 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "chid.h" #include "gf100.h" #include "changf100.h" @@ -624,6 +625,12 @@ gf100_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ } +int +gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) +{ + return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr, &fifo->chid); +} + static int gf100_fifo_oneinit(struct nvkm_fifo *base) { @@ -681,6 +688,7 @@ gf100_fifo = { .dtor = gf100_fifo_dtor, .oneinit = gf100_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, + .chid_ctor = gf100_fifo_chid_ctor, .init = gf100_fifo_init, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 880ddfec22f7a..c4fa4fc68d467 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "chid.h" #include "gk104.h" #include "cgrp.h" @@ -1193,6 +1194,7 @@ gk104_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, + .chid_ctor = gf100_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 9621877400106..af6a004a62670 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -74,7 +74,7 @@ void gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit); void gk104_fifo_intr_runlist(struct gk104_fifo *fifo); void gk104_fifo_intr_engine(struct gk104_fifo *fifo); void *gk104_fifo_dtor(struct nvkm_fifo *base); -int gk104_fifo_oneinit(struct nvkm_fifo *base); +int gk104_fifo_oneinit(struct nvkm_fifo *); int gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data); void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 9dcc54f1d016b..b23d50d1b351f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -23,6 +23,7 @@ */ #include "cgrp.h" #include "chan.h" +#include "chid.h" #include "gk104.h" #include "changk104.h" @@ -56,11 +57,24 @@ gk110_fifo_runlist = { .commit = gk104_fifo_runlist_commit, }; +int +gk110_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) +{ + int ret; + + ret = nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr, &fifo->cgid); + if (ret) + return ret; + + return gf100_fifo_chid_ctor(fifo, nr); +} + static const struct nvkm_fifo_func gk110_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 28791baf77f8a..1dd937bdbbae0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -54,6 +54,7 @@ gk208_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gk208_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index b62e3bb0294ca..d2e78d2a868a5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -29,6 +29,7 @@ gk20a_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index f996f1c1141a0..32ef88a90d1e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -114,6 +114,7 @@ gm107_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gm107_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 8f3da21ce1e76..74891833926e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -51,6 +51,7 @@ gm200_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index dc590848c25c7..c1ad59d1223b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -86,6 +86,7 @@ gp100_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 36c02b5aac0f9..9e876bd871bb4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -301,6 +301,7 @@ gv100_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index f9f8371d37565..18fa5b933e9ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -23,6 +23,7 @@ */ #include "cgrp.h" #include "chan.h" +#include "chid.h" #include "nv04.h" #include "channv04.h" @@ -58,7 +59,6 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) struct nvkm_memory *fctx = device->imem->ramfc; const struct nv04_fifo_ramfc *c; unsigned long flags; - u32 mask = fifo->base.nr - 1; u32 data = chan->ramfc; u32 chid; @@ -67,7 +67,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) nvkm_wr32(device, NV03_PFIFO_CACHES, 0); /* if this channel is active, replace it with a null context */ - chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & mask; + chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; if (chid == chan->base.chid) { nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); @@ -91,7 +91,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, mask); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.chid->mask); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); } @@ -360,7 +360,7 @@ nv04_fifo_intr(struct nvkm_fifo *base) reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; nvkm_wr32(device, NV03_PFIFO_CACHES, 0); - chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & (fifo->base.nr - 1); + chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); if (stat & NV_PFIFO_INTR_CACHE_ERROR) { @@ -407,10 +407,9 @@ nv04_fifo_intr(struct nvkm_fifo *base) } void -nv04_fifo_init(struct nvkm_fifo *base) +nv04_fifo_init(struct nvkm_fifo *fifo) { - struct nv04_fifo *fifo = nv04_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_instmem *imem = device->imem; struct nvkm_ramht *ramht = imem->ramht; struct nvkm_memory *ramro = imem->ramro; @@ -425,7 +424,7 @@ nv04_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, NV03_PFIFO_RAMRO, nvkm_memory_addr(ramro) >> 8); nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.nr - 1); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); @@ -435,6 +434,13 @@ nv04_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, NV03_PFIFO_CACHES, 1); } +int +nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) +{ + /* The last CHID is reserved by HW as a "channel invalid" marker. */ + return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr - 1, &fifo->chid); +} + static int nv04_fifo_chid_nr(struct nvkm_fifo *fifo) { @@ -465,6 +471,7 @@ nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, static const struct nvkm_fifo_func nv04_fifo = { .chid_nr = nv04_fifo_chid_nr, + .chid_ctor = nv04_fifo_chid_ctor, .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h index 3f23bcde4a540..879c686675d66 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h @@ -19,5 +19,4 @@ struct nv04_fifo { int nv04_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, int nr, const struct nv04_fifo_ramfc *, struct nvkm_fifo **); -void nv04_fifo_init(struct nvkm_fifo *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index f07304e4b8c50..bc25520cb39b4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -56,6 +56,7 @@ nv10_fifo_chid_nr(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func nv10_fifo = { .chid_nr = nv10_fifo_chid_nr, + .chid_ctor = nv04_fifo_chid_ctor, .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 858c8ba158c94..dd254d88d008f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "chid.h" #include "nv04.h" #include "channv04.h" @@ -56,10 +57,9 @@ nv17_chan = { }; static void -nv17_fifo_init(struct nvkm_fifo *base) +nv17_fifo_init(struct nvkm_fifo *fifo) { - struct nv04_fifo *fifo = nv04_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_instmem *imem = device->imem; struct nvkm_ramht *ramht = imem->ramht; struct nvkm_memory *ramro = imem->ramro; @@ -75,7 +75,7 @@ nv17_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, NV03_PFIFO_RAMFC, nvkm_memory_addr(ramfc) >> 8 | 0x00010000); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.nr - 1); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); @@ -88,6 +88,7 @@ nv17_fifo_init(struct nvkm_fifo *base) static const struct nvkm_fifo_func nv17_fifo = { .chid_nr = nv10_fifo_chid_nr, + .chid_ctor = nv04_fifo_chid_ctor, .init = nv17_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index 30f924e3da3c8..c87d995e4fd1e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "chid.h" #include "nv04.h" #include "channv04.h" @@ -65,10 +66,9 @@ nv40_chan = { }; static void -nv40_fifo_init(struct nvkm_fifo *base) +nv40_fifo_init(struct nvkm_fifo *fifo) { - struct nv04_fifo *fifo = nv04_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_fb *fb = device->fb; struct nvkm_instmem *imem = device->imem; struct nvkm_ramht *ramht = imem->ramht; @@ -106,7 +106,7 @@ nv40_fifo_init(struct nvkm_fifo *base) break; } - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.nr - 1); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); nvkm_wr32(device, NV03_PFIFO_INTR_0, 0xffffffff); nvkm_wr32(device, NV03_PFIFO_INTR_EN_0, 0xffffffff); @@ -119,6 +119,7 @@ nv40_fifo_init(struct nvkm_fifo *base) static const struct nvkm_fifo_func nv40_fifo = { .chid_nr = nv10_fifo_chid_nr, + .chid_ctor = nv04_fifo_chid_ctor, .init = nv40_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 101015a1a7e94..13c514acc9d72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "chid.h" #include "nv50.h" #include "channv50.h" @@ -88,6 +89,13 @@ nv50_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002500, 0x00000001); } +int +nv50_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) +{ + /* CHID 0 is unusable (some kind of PIO channel?), 127 is "channel invalid". */ + return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 1, nr - 2, &fifo->chid); +} + int nv50_fifo_chid_nr(struct nvkm_fifo *fifo) { @@ -144,6 +152,7 @@ nv50_fifo = { .dtor = nv50_fifo_dtor, .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, + .chid_ctor = nv50_fifo_chid_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index ba3dbc86435e4..80280841cf814 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -21,6 +21,7 @@ struct nvkm_fifo_func { int (*oneinit)(struct nvkm_fifo *); int (*chid_nr)(struct nvkm_fifo *); + int (*chid_ctor)(struct nvkm_fifo *, int nr); int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data); void (*init)(struct nvkm_fifo *); @@ -86,6 +87,8 @@ struct nvkm_fifo_func { int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_fifo *); +int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); +void nv04_fifo_init(struct nvkm_fifo *); void nv04_fifo_intr(struct nvkm_fifo *); int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *nv04_fifo_id_engine(struct nvkm_fifo *, int); @@ -96,9 +99,11 @@ extern const struct nvkm_cgrp_func nv04_cgrp; int nv10_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_nr(struct nvkm_fifo *); +int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_chan_func g84_chan; +int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); int gk104_fifo_chid_nr(struct nvkm_fifo *); @@ -109,6 +114,7 @@ void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); +int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_cgrp_func gk110_cgrp; extern const struct nvkm_chan_func gk110_chan; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 274096316a1b1..fbc67b84a8b10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -442,6 +442,7 @@ tu102_fifo = { .dtor = gk104_fifo_dtor, .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, + .chid_ctor = gk110_fifo_chid_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, -- GitLab From 1c488ba96e59ce84bf1cfd321bf27252f4598bec Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:23 +1000 Subject: [PATCH 1306/1927] drm/nouveau/fifo: add runq Creates an nvkm_runq for each PBDMA, these will be associated with the relevant runlist(s) later. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 16 ++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 25 ++++++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 20 ++++----- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 8 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 8 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 7 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 11 ++++- .../gpu/drm/nouveau/nvkm/engine/fifo/runq.c | 45 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runq.h | 24 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 3 +- 17 files changed, 153 insertions(+), 27 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index b688121f1a91e..895eb262345e7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -40,6 +40,8 @@ struct nvkm_fifo { struct nvkm_chid *chid; struct nvkm_chid *cgid; + struct list_head runqs; + DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR); int nr; struct list_head chan; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index c985736bff2c4..447001db070e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -2,6 +2,7 @@ nvkm-y += nvkm/engine/fifo/base.o nvkm-y += nvkm/engine/fifo/chan.o nvkm-y += nvkm/engine/fifo/chid.o +nvkm-y += nvkm/engine/fifo/runq.o nvkm-y += nvkm/engine/fifo/nv04.o nvkm-y += nvkm/engine/fifo/nv10.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index e6b0653625f5a..806ca8986eb55 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -24,6 +24,7 @@ #include "priv.h" #include "chan.h" #include "chid.h" +#include "runq.h" #include #include @@ -235,7 +236,7 @@ static int nvkm_fifo_oneinit(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); - int ret; + int ret, nr, i; /* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */ if (fifo->func->chid_nr) { @@ -244,6 +245,14 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) return ret; } + /* Create runqueues for each PBDMA. */ + if (fifo->func->runq_nr) { + for (nr = fifo->func->runq_nr(fifo), i = 0; i < nr; i++) { + if (!nvkm_runq_new(fifo, i)) + return -ENOMEM; + } + } + if (fifo->func->oneinit) return fifo->func->oneinit(fifo); @@ -260,8 +269,12 @@ static void * nvkm_fifo_dtor(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + struct nvkm_runq *runq, *rtmp; void *data = fifo; + list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head) + nvkm_runq_del(runq); + nvkm_chid_unref(&fifo->cgid); nvkm_chid_unref(&fifo->chid); @@ -292,6 +305,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, int ret, nr; fifo->func = func; + INIT_LIST_HEAD(&fifo->runqs); spin_lock_init(&fifo->lock); mutex_init(&fifo->mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index d8eeca8186372..14d8bc3b59561 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -23,6 +23,7 @@ */ #include "chan.h" #include "chid.h" +#include "runq.h" #include "gf100.h" #include "changf100.h" @@ -100,6 +101,10 @@ gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit) nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); } +static const struct nvkm_runq_func +gf100_runq = { +}; + void gf100_fifo_runlist_commit(struct gf100_fifo *fifo) { @@ -625,6 +630,18 @@ gf100_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ } +int +gf100_fifo_runq_nr(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + u32 save; + + /* Determine number of PBDMAs by checking valid enable bits. */ + save = nvkm_mask(device, 0x000204, 0xffffffff, 0xffffffff); + save = nvkm_mask(device, 0x000204, 0xffffffff, save); + return hweight32(save); +} + int gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) { @@ -640,11 +657,7 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int ret; - /* Determine number of PBDMAs by checking valid enable bits. */ - nvkm_wr32(device, 0x002204, 0xffffffff); - fifo->pbdma_nr = hweight32(nvkm_rd32(device, 0x002204)); - nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr); - + fifo->pbdma_nr = fifo->base.func->runq_nr(&fifo->base); ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, false, &fifo->runlist.mem[0]); @@ -689,6 +702,7 @@ gf100_fifo = { .oneinit = gf100_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, + .runq_nr = gf100_fifo_runq_nr, .init = gf100_fifo_init, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, @@ -697,6 +711,7 @@ gf100_fifo = { .id_engine = gf100_fifo_id_engine, .uevent_init = gf100_fifo_uevent_init, .uevent_fini = gf100_fifo_uevent_fini, + .runq = &gf100_runq, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index c4fa4fc68d467..efebdcdbeefc1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -23,6 +23,7 @@ */ #include "chan.h" #include "chid.h" +#include "runq.h" #include "gk104.h" #include "cgrp.h" @@ -168,6 +169,10 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { {} }; +const struct nvkm_runq_func +gk104_runq = { +}; + void gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, struct nvkm_memory *mem, int nr) @@ -275,18 +280,8 @@ gk104_fifo_pbdma_init(struct gk104_fifo *fifo) nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1); } -int -gk104_fifo_pbdma_nr(struct gk104_fifo *fifo) -{ - struct nvkm_device *device = fifo->base.engine.subdev.device; - /* Determine number of PBDMAs by checking valid enable bits. */ - nvkm_wr32(device, 0x000204, 0xffffffff); - return hweight32(nvkm_rd32(device, 0x000204)); -} - const struct gk104_fifo_pbdma_func gk104_fifo_pbdma = { - .nr = gk104_fifo_pbdma_nr, .init = gk104_fifo_pbdma_init, }; @@ -1076,8 +1071,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) int pbid, ret, i, j; u32 *map; - fifo->pbdma_nr = fifo->func->pbdma->nr(fifo); - nvkm_debug(subdev, "%d PBDMA(s)\n", fifo->pbdma_nr); + fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base); /* Read PBDMA->runlist(s) mapping from HW. */ if (!(map = kcalloc(fifo->pbdma_nr, sizeof(*map), GFP_KERNEL))) @@ -1195,6 +1189,7 @@ gk104_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, + .runq_nr = gf100_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -1213,6 +1208,7 @@ gk104_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, .pbdma = &gk104_fifo_pbdma, + .runq = &gk104_runq, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index af6a004a62670..636091c7fcee2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -82,7 +82,6 @@ void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo); void gk104_fifo_uevent_init(struct nvkm_fifo *fifo); extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma; -int gk104_fifo_pbdma_nr(struct gk104_fifo *); void gk104_fifo_pbdma_init(struct gk104_fifo *); extern const struct nvkm_enum gk104_fifo_fault_access[]; extern const struct nvkm_enum gk104_fifo_fault_engine[]; @@ -106,7 +105,6 @@ extern const struct nvkm_enum gm107_fifo_fault_engine[]; extern const struct gk104_fifo_runlist_func gm107_fifo_runlist; extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma; -int gm200_fifo_pbdma_nr(struct gk104_fifo *); extern const struct nvkm_enum gp100_fifo_fault_engine[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index b23d50d1b351f..d6d86ad47ecbd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -75,6 +75,7 @@ gk110_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gf100_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -93,6 +94,7 @@ gk110_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk104_fifo_pbdma, + .runq = &gk104_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 1dd937bdbbae0..d81593722fa10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "runq.h" #include "gk104.h" #include "changk104.h" @@ -38,11 +39,14 @@ gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo) const struct gk104_fifo_pbdma_func gk208_fifo_pbdma = { - .nr = gk104_fifo_pbdma_nr, .init = gk104_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; +const struct nvkm_runq_func +gk208_runq = { +}; + static int gk208_fifo_chid_nr(struct nvkm_fifo *fifo) { @@ -55,6 +59,7 @@ gk208_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gk208_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gf100_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -73,6 +78,7 @@ gk208_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .runq = &gk208_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index d2e78d2a868a5..11227cf559416 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -30,6 +30,7 @@ gk20a_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gf100_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -48,6 +49,7 @@ gk20a_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .runq = &gk208_runq, .cgrp = {{ }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 32ef88a90d1e8..2dc89d3a0d0c2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -115,6 +115,7 @@ gm107_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gm107_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gf100_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -133,6 +134,7 @@ gm107_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .runq = &gk208_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 74891833926e0..f24551c701e1b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -27,15 +27,13 @@ #include int -gm200_fifo_pbdma_nr(struct gk104_fifo *fifo) +gm200_fifo_runq_nr(struct nvkm_fifo *fifo) { - struct nvkm_device *device = fifo->base.engine.subdev.device; - return nvkm_rd32(device, 0x002004) & 0x000000ff; + return nvkm_rd32(fifo->engine.subdev.device, 0x002004) & 0x000000ff; } const struct gk104_fifo_pbdma_func gm200_fifo_pbdma = { - .nr = gm200_fifo_pbdma_nr, .init = gk104_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; @@ -52,6 +50,7 @@ gm200_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gm200_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -70,6 +69,7 @@ gm200_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .runq = &gk208_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index c1ad59d1223b3..60bc5d3143245 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -87,6 +87,7 @@ gp100_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gm200_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -105,6 +106,7 @@ gp100_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .runq = &gk208_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 9e876bd871bb4..f7db1b4db0dce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -21,6 +21,7 @@ */ #include "chan.h" #include "cgrp.h" +#include "runq.h" #include "gk104.h" #include "changk104.h" @@ -33,6 +34,10 @@ static const struct nvkm_chan_func gv100_chan = { }; +const struct nvkm_runq_func +gv100_runq = { +}; + void gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan, struct nvkm_memory *memory, u32 offset) @@ -302,6 +307,7 @@ gv100_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gm200_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -319,6 +325,7 @@ gv100_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .runq = &gv100_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 80280841cf814..daa594669985b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -22,6 +22,7 @@ struct nvkm_fifo_func { int (*oneinit)(struct nvkm_fifo *); int (*chid_nr)(struct nvkm_fifo *); int (*chid_ctor)(struct nvkm_fifo *, int nr); + int (*runq_nr)(struct nvkm_fifo *); int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data); void (*init)(struct nvkm_fifo *); @@ -61,11 +62,12 @@ struct nvkm_fifo_func { } *runlist; const struct gk104_fifo_pbdma_func { - int (*nr)(struct gk104_fifo *); void (*init)(struct gk104_fifo *); void (*init_timeout)(struct gk104_fifo *); } *pbdma; + const struct nvkm_runq_func *runq; + struct nvkm_fifo_func_cgrp { struct nvkm_sclass user; const struct nvkm_cgrp_func *func; @@ -104,6 +106,7 @@ int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_chan_func g84_chan; int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); +int gf100_fifo_runq_nr(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); int gk104_fifo_chid_nr(struct nvkm_fifo *); @@ -113,19 +116,25 @@ void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); +extern const struct nvkm_runq_func gk104_runq; int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_cgrp_func gk110_cgrp; extern const struct nvkm_chan_func gk110_chan; +extern const struct nvkm_runq_func gk208_runq; + void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; extern const struct nvkm_chan_func gm107_chan; int gm200_fifo_chid_nr(struct nvkm_fifo *); +int gm200_fifo_runq_nr(struct nvkm_fifo *); void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); +extern const struct nvkm_runq_func gv100_runq; + extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c new file mode 100644 index 0000000000000..33bcf5fb3ef0b --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.c @@ -0,0 +1,45 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "runq.h" +#include "priv.h" + +void +nvkm_runq_del(struct nvkm_runq *runq) +{ + list_del(&runq->head); + kfree(runq); +} + +struct nvkm_runq * +nvkm_runq_new(struct nvkm_fifo *fifo, int pbid) +{ + struct nvkm_runq *runq; + + if (!(runq = kzalloc(sizeof(*runq), GFP_KERNEL))) + return NULL; + + runq->func = fifo->func->runq; + runq->fifo = fifo; + runq->id = pbid; + list_add_tail(&runq->head, &fifo->runqs); + return runq; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h new file mode 100644 index 0000000000000..1832a14488c3a --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVKM_RUNQ_H__ +#define __NVKM_RUNQ_H__ +#include + +struct nvkm_runq { + const struct nvkm_runq_func { + } *func; + struct nvkm_fifo *fifo; + int id; + + struct list_head head; +}; + +struct nvkm_runq *nvkm_runq_new(struct nvkm_fifo *, int pbid); +void nvkm_runq_del(struct nvkm_runq *); + +#define nvkm_runq_foreach_cond(runq,fifo,cond) nvkm_list_foreach(runq, &(fifo)->runqs, head, (cond)) + +#define RUNQ_PRINT(r,l,p,f,a...) \ + nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "PBDMA%d:"f, (r)->id, ##a) +#define RUNQ_ERROR(r,f,a...) RUNQ_PRINT((r), ERROR, err, " "f"\n", ##a) +#define RUNQ_DEBUG(r,f,a...) RUNQ_PRINT((r), DEBUG, info, " "f"\n", ##a) +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index fbc67b84a8b10..6d04316102352 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -99,7 +99,6 @@ tu102_fifo_pbdma_init(struct gk104_fifo *fifo) static const struct gk104_fifo_pbdma_func tu102_fifo_pbdma = { - .nr = gm200_fifo_pbdma_nr, .init = tu102_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; @@ -443,6 +442,7 @@ tu102_fifo = { .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, + .runq_nr = gm200_fifo_runq_nr, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -460,6 +460,7 @@ tu102_fifo = { .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, .pbdma = &tu102_fifo_pbdma, + .runq = &gv100_runq, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new }, }; -- GitLab From d94470e9d150aaf0fff26f09852da22ae951956f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:24 +1000 Subject: [PATCH 1307/1927] drm/nouveau/fifo: add common runlist/engine topology Creates an nvkm_runl for each runlist on the GPU, and an nvkm_engn for each engine that is reachable from a runlist. - basically what gk104- already does, but extended to all chips - adds per-runlist CHID allocators (Ampere) - splits g98/gt2xx out from g84 (different target engines) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/core/os.h | 15 +++ .../drm/nouveau/include/nvkm/engine/fifo.h | 2 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 14 +-- .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 19 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 37 +++++- .../gpu/drm/nouveau/nvkm/engine/fifo/g98.c | 77 ++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 35 ++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 102 +++++++++++----- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 9 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 9 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 9 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 17 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 29 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 13 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 17 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 5 + .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 19 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 112 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 55 +++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 9 ++ 26 files changed, 583 insertions(+), 43 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h index e8705216ae4e0..4486d98628493 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/os.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/os.h @@ -35,6 +35,21 @@ nvkm_blob_dtor(struct nvkm_blob *blob) blob->size = 0; } +#define nvkm_list_find_next(p,h,m,c) ({ \ + typeof(p) _p = NULL; \ + list_for_each_entry_continue(p, (h), m) { \ + if (c) { \ + _p = p; \ + break; \ + } \ + } \ + _p; \ +}) +#define nvkm_list_find(p,h,m,c) \ + (p = container_of((h), typeof(*p), m), nvkm_list_find_next(p, (h), m, (c))) +#define nvkm_list_foreach(p,h,m,c) \ + for (p = nvkm_list_find(p, (h), m, (c)); p; p = nvkm_list_find_next(p, (h), m, (c))) + /*FIXME: remove after */ #define nvkm_fifo_chan nvkm_chan #define nvkm_fifo_chan_func nvkm_chan_func diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 895eb262345e7..1e12697beae29 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -41,6 +41,7 @@ struct nvkm_fifo { struct nvkm_chid *cgid; struct list_head runqs; + struct list_head runls; DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR); int nr; @@ -71,6 +72,7 @@ int nv17_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int nv40_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int nv50_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int g84_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); +int g98_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gf100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gk104_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gk110_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 532231f1a0f53..4f5921a2aeb04 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -1095,7 +1095,7 @@ nv98_chipset = { .volt = { 0x00000001, nv40_volt_new }, .disp = { 0x00000001, g94_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, g84_gr_new }, .mspdec = { 0x00000001, g98_mspdec_new }, .msppp = { 0x00000001, g98_msppp_new }, @@ -1161,7 +1161,7 @@ nva3_chipset = { .ce = { 0x00000001, gt215_ce_new }, .disp = { 0x00000001, gt215_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, gt215_gr_new }, .mpeg = { 0x00000001, g84_mpeg_new }, .mspdec = { 0x00000001, gt215_mspdec_new }, @@ -1195,7 +1195,7 @@ nva5_chipset = { .ce = { 0x00000001, gt215_ce_new }, .disp = { 0x00000001, gt215_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, gt215_gr_new }, .mspdec = { 0x00000001, gt215_mspdec_new }, .msppp = { 0x00000001, gt215_msppp_new }, @@ -1228,7 +1228,7 @@ nva8_chipset = { .ce = { 0x00000001, gt215_ce_new }, .disp = { 0x00000001, gt215_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, gt215_gr_new }, .mspdec = { 0x00000001, gt215_mspdec_new }, .msppp = { 0x00000001, gt215_msppp_new }, @@ -1259,7 +1259,7 @@ nvaa_chipset = { .volt = { 0x00000001, nv40_volt_new }, .disp = { 0x00000001, mcp77_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, gt200_gr_new }, .mspdec = { 0x00000001, g98_mspdec_new }, .msppp = { 0x00000001, g98_msppp_new }, @@ -1291,7 +1291,7 @@ nvac_chipset = { .volt = { 0x00000001, nv40_volt_new }, .disp = { 0x00000001, mcp77_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, mcp79_gr_new }, .mspdec = { 0x00000001, g98_mspdec_new }, .msppp = { 0x00000001, g98_msppp_new }, @@ -1325,7 +1325,7 @@ nvaf_chipset = { .ce = { 0x00000001, gt215_ce_new }, .disp = { 0x00000001, mcp89_disp_new }, .dma = { 0x00000001, nv50_dma_new }, - .fifo = { 0x00000001, g84_fifo_new }, + .fifo = { 0x00000001, g98_fifo_new }, .gr = { 0x00000001, mcp89_gr_new }, .mspdec = { 0x00000001, gt215_mspdec_new }, .msppp = { 0x00000001, gt215_msppp_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index 447001db070e4..a92a88a72e596 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -2,6 +2,7 @@ nvkm-y += nvkm/engine/fifo/base.o nvkm-y += nvkm/engine/fifo/chan.o nvkm-y += nvkm/engine/fifo/chid.o +nvkm-y += nvkm/engine/fifo/runl.o nvkm-y += nvkm/engine/fifo/runq.o nvkm-y += nvkm/engine/fifo/nv04.o @@ -10,6 +11,7 @@ nvkm-y += nvkm/engine/fifo/nv17.o nvkm-y += nvkm/engine/fifo/nv40.o nvkm-y += nvkm/engine/fifo/nv50.o nvkm-y += nvkm/engine/fifo/g84.o +nvkm-y += nvkm/engine/fifo/g98.o nvkm-y += nvkm/engine/fifo/gf100.o nvkm-y += nvkm/engine/fifo/gk104.o nvkm-y += nvkm/engine/fifo/gk110.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 806ca8986eb55..60d5cbbe43810 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -24,6 +24,7 @@ #include "priv.h" #include "chan.h" #include "chid.h" +#include "runl.h" #include "runq.h" #include @@ -236,6 +237,8 @@ static int nvkm_fifo_oneinit(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + struct nvkm_runl *runl; + struct nvkm_engn *engn; int ret, nr, i; /* Initialise CHID/CGID allocator(s) on GPUs where they aren't per-runlist. */ @@ -253,6 +256,18 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) } } + /* Create runlists. */ + ret = fifo->func->runl_ctor(fifo); + if (ret) + return ret; + + nvkm_runl_foreach(runl, fifo) { + RUNL_DEBUG(runl, ""); + nvkm_runl_foreach_engn(engn, runl) { + ENGN_DEBUG(engn, ""); + } + } + if (fifo->func->oneinit) return fifo->func->oneinit(fifo); @@ -269,9 +284,12 @@ static void * nvkm_fifo_dtor(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + struct nvkm_runl *runl, *runt; struct nvkm_runq *runq, *rtmp; void *data = fifo; + list_for_each_entry_safe(runl, runt, &fifo->runls, head) + nvkm_runl_del(runl); list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head) nvkm_runq_del(runq); @@ -306,6 +324,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, fifo->func = func; INIT_LIST_HEAD(&fifo->runqs); + INIT_LIST_HEAD(&fifo->runls); spin_lock_init(&fifo->lock); mutex_init(&fifo->mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index dbb9df6f8eb51..3d242350fdee4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -22,6 +22,7 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "runl.h" #include "nv50.h" #include "channv50.h" @@ -32,21 +33,25 @@ const struct nvkm_chan_func g84_chan = { }; -static void +const struct nvkm_engn_func +g84_engn = { +}; + +void g84_fifo_uevent_fini(struct nvkm_fifo *fifo) { struct nvkm_device *device = fifo->engine.subdev.device; nvkm_mask(device, 0x002140, 0x40000000, 0x00000000); } -static void +void g84_fifo_uevent_init(struct nvkm_fifo *fifo) { struct nvkm_device *device = fifo->engine.subdev.device; nvkm_mask(device, 0x002140, 0x40000000, 0x40000000); } -static struct nvkm_engine * +struct nvkm_engine * g84_fifo_id_engine(struct nvkm_fifo *fifo, int engi) { struct nvkm_device *device = fifo->engine.subdev.device; @@ -92,7 +97,7 @@ g84_fifo_id_engine(struct nvkm_fifo *fifo, int engi) return nvkm_device_engine(fifo->engine.subdev.device, type, 0); } -static int +int g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { switch (engine->subdev.type) { @@ -114,12 +119,33 @@ g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) } } +static int +g84_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_runl *runl; + + runl = nvkm_runl_new(fifo, 0, 0, 0); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0); + nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0); + nvkm_runl_add(runl, 1, fifo->func->engn, NVKM_ENGINE_GR, 0); + nvkm_runl_add(runl, 2, fifo->func->engn, NVKM_ENGINE_MPEG, 0); + nvkm_runl_add(runl, 3, fifo->func->engn, NVKM_ENGINE_ME, 0); + nvkm_runl_add(runl, 4, fifo->func->engn, NVKM_ENGINE_VP, 0); + nvkm_runl_add(runl, 5, fifo->func->engn, NVKM_ENGINE_CIPHER, 0); + nvkm_runl_add(runl, 6, fifo->func->engn, NVKM_ENGINE_BSP, 0); + return 0; +} + static const struct nvkm_fifo_func g84_fifo = { .dtor = nv50_fifo_dtor, .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, + .runl_ctor = g84_fifo_runl_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = g84_fifo_engine_id, @@ -128,6 +154,9 @@ g84_fifo = { .start = nv04_fifo_start, .uevent_init = g84_fifo_uevent_init, .uevent_fini = g84_fifo_uevent_fini, + .runl = &nv50_runl, + .engn = &g84_engn, + .engn_sw = &nv50_engn_sw, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c new file mode 100644 index 0000000000000..39177ec0eca64 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c @@ -0,0 +1,77 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "chan.h" +#include "runl.h" + +#include "nv50.h" +#include "channv50.h" + +#include + +static int +g98_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_runl *runl; + + runl = nvkm_runl_new(fifo, 0, 0, 0); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0); + nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0); + nvkm_runl_add(runl, 1, fifo->func->engn, NVKM_ENGINE_GR, 0); + nvkm_runl_add(runl, 2, fifo->func->engn, NVKM_ENGINE_MSPPP, 0); + nvkm_runl_add(runl, 3, fifo->func->engn, NVKM_ENGINE_CE, 0); + nvkm_runl_add(runl, 4, fifo->func->engn, NVKM_ENGINE_MSPDEC, 0); + nvkm_runl_add(runl, 5, fifo->func->engn, NVKM_ENGINE_SEC, 0); + nvkm_runl_add(runl, 6, fifo->func->engn, NVKM_ENGINE_MSVLD, 0); + return 0; +} + +static const struct nvkm_fifo_func +g98_fifo = { + .dtor = nv50_fifo_dtor, + .oneinit = nv50_fifo_oneinit, + .chid_nr = nv50_fifo_chid_nr, + .chid_ctor = nv50_fifo_chid_ctor, + .runl_ctor = g98_fifo_runl_ctor, + .init = nv50_fifo_init, + .intr = nv04_fifo_intr, + .engine_id = g84_fifo_engine_id, + .id_engine = g84_fifo_id_engine, + .pause = nv04_fifo_pause, + .start = nv04_fifo_start, + .uevent_init = g84_fifo_uevent_init, + .uevent_fini = g84_fifo_uevent_fini, + .runl = &nv50_runl, + .engn = &g84_engn, + .engn_sw = &nv50_engn_sw, + .cgrp = {{ }, &nv04_cgrp }, + .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass }, +}; + +int +g98_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_fifo **pfifo) +{ + return nv50_fifo_new_(&g98_fifo, device, type, inst, pfifo); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 14d8bc3b59561..43cce02151d78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -23,6 +23,7 @@ */ #include "chan.h" #include "chid.h" +#include "runl.h" #include "runq.h" #include "gf100.h" @@ -55,6 +56,14 @@ static const struct nvkm_chan_func gf100_chan = { }; +static const struct nvkm_engn_func +gf100_engn = { +}; + +const struct nvkm_engn_func +gf100_engn_sw = { +}; + static const struct nvkm_bitfield gf100_fifo_pbdma_intr[] = { /* { 0x00008000, "" } seen with null ib push */ @@ -163,6 +172,10 @@ gf100_fifo_runlist_insert(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan) mutex_unlock(&fifo->base.mutex); } +static const struct nvkm_runl_func +gf100_runl = { +}; + static struct nvkm_engine * gf100_fifo_id_engine(struct nvkm_fifo *fifo, int engi) { @@ -630,6 +643,25 @@ gf100_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002628, 0x00000001); /* ENGINE_INTR_EN */ } +static int +gf100_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_runl *runl; + + runl = nvkm_runl_new(fifo, 0, 0, 0); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + nvkm_runl_add(runl, 0, fifo->func->engn, NVKM_ENGINE_GR, 0); + nvkm_runl_add(runl, 1, fifo->func->engn, NVKM_ENGINE_MSPDEC, 0); + nvkm_runl_add(runl, 2, fifo->func->engn, NVKM_ENGINE_MSPPP, 0); + nvkm_runl_add(runl, 3, fifo->func->engn, NVKM_ENGINE_MSVLD, 0); + nvkm_runl_add(runl, 4, fifo->func->engn, NVKM_ENGINE_CE, 0); + nvkm_runl_add(runl, 5, fifo->func->engn, NVKM_ENGINE_CE, 1); + nvkm_runl_add(runl, 15, &gf100_engn_sw, NVKM_ENGINE_SW, 0); + return 0; +} + int gf100_fifo_runq_nr(struct nvkm_fifo *fifo) { @@ -703,6 +735,7 @@ gf100_fifo = { .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, + .runl_ctor = gf100_fifo_runl_ctor, .init = gf100_fifo_init, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, @@ -711,7 +744,9 @@ gf100_fifo = { .id_engine = gf100_fifo_id_engine, .uevent_init = gf100_fifo_uevent_init, .uevent_fini = gf100_fifo_uevent_fini, + .runl = &gf100_runl, .runq = &gf100_runq, + .engn = &gf100_engn, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index efebdcdbeefc1..eaf61c07fafc9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -23,6 +23,7 @@ */ #include "chan.h" #include "chid.h" +#include "runl.h" #include "runq.h" #include "gk104.h" @@ -91,6 +92,14 @@ gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, status->chan == &status->next ? "*" : " "); } +const struct nvkm_engn_func +gk104_engn = { +}; + +const struct nvkm_engn_func +gk104_engn_ce = { +}; + void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo) { @@ -169,6 +178,12 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { {} }; +static u32 +gk104_runq_runm(struct nvkm_runq *runq) +{ + return nvkm_rd32(runq->fifo->engine.subdev.device, 0x002390 + (runq->id * 0x04)); +} + const struct nvkm_runq_func gk104_runq = { }; @@ -273,6 +288,10 @@ gk104_fifo_runlist = { .commit = gk104_fifo_runlist_commit, }; +static const struct nvkm_runl_func +gk104_runl = { +}; + void gk104_fifo_pbdma_init(struct gk104_fifo *fifo) { @@ -1054,6 +1073,52 @@ gk104_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x002140, 0x7fffffff); } +int +gk104_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_top_device *tdev; + struct nvkm_runl *runl; + struct nvkm_runq *runq; + const struct nvkm_engn_func *func; + + nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) { + runl = nvkm_runl_get(fifo, tdev->runlist, tdev->runlist); + if (!runl) { + runl = nvkm_runl_new(fifo, tdev->runlist, tdev->runlist, 0); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + nvkm_runq_foreach_cond(runq, fifo, gk104_runq_runm(runq) & BIT(runl->id)) { + if (WARN_ON(runl->runq_nr == ARRAY_SIZE(runl->runq))) + return -ENOMEM; + + runl->runq[runl->runq_nr++] = runq; + } + + } + + if (tdev->engine < 0) + continue; + + switch (tdev->type) { + case NVKM_ENGINE_CE: + func = fifo->func->engn_ce; + break; + case NVKM_ENGINE_GR: + nvkm_runl_add(runl, 15, &gf100_engn_sw, NVKM_ENGINE_SW, 0); + fallthrough; + default: + func = fifo->func->engn; + break; + } + + nvkm_runl_add(runl, tdev->engine, func, tdev->type, tdev->inst); + } + + return 0; +} + int gk104_fifo_chid_nr(struct nvkm_fifo *fifo) { @@ -1068,48 +1133,19 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) struct nvkm_device *device = subdev->device; struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); struct nvkm_top_device *tdev; - int pbid, ret, i, j; - u32 *map; + int ret, i, j; fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base); - /* Read PBDMA->runlist(s) mapping from HW. */ - if (!(map = kcalloc(fifo->pbdma_nr, sizeof(*map), GFP_KERNEL))) - return -ENOMEM; - - for (i = 0; i < fifo->pbdma_nr; i++) - map[i] = nvkm_rd32(device, 0x002390 + (i * 0x04)); - /* Determine runlist configuration from topology device info. */ list_for_each_entry(tdev, &device->top->device, head) { const int engn = tdev->engine; - char _en[16], *en; if (engn < 0) continue; - /* Determine which PBDMA handles requests for this engine. */ - for (j = 0, pbid = -1; j < fifo->pbdma_nr; j++) { - if (map[j] & BIT(tdev->runlist)) { - pbid = j; - break; - } - } - fifo->engine[engn].engine = nvkm_device_engine(device, tdev->type, tdev->inst); - if (!fifo->engine[engn].engine) { - snprintf(_en, sizeof(_en), "%s, %d", - nvkm_subdev_type[tdev->type], tdev->inst); - en = _en; - } else { - en = fifo->engine[engn].engine->subdev.name; - } - - nvkm_debug(subdev, "engine %2d: runlist %2d pbdma %2d (%s)\n", - tdev->engine, tdev->runlist, pbid, en); - fifo->engine[engn].runl = tdev->runlist; - fifo->engine[engn].pbid = pbid; fifo->engine_nr = max(fifo->engine_nr, engn + 1); fifo->runlist[tdev->runlist].engm |= BIT(engn); fifo->runlist[tdev->runlist].engm_sw |= BIT(engn); @@ -1118,8 +1154,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) fifo->runlist_nr = max(fifo->runlist_nr, tdev->runlist + 1); } - kfree(map); - for (i = 0; i < fifo->runlist_nr; i++) { for (j = 0; j < ARRAY_SIZE(fifo->runlist[i].mem); j++) { ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, @@ -1190,6 +1224,7 @@ gk104_fifo = { .chid_nr = gk104_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -1208,7 +1243,10 @@ gk104_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, .pbdma = &gk104_fifo_pbdma, + .runl = &gk104_runl, .runq = &gk104_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index d6d86ad47ecbd..08f5b06845106 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -24,6 +24,7 @@ #include "cgrp.h" #include "chan.h" #include "chid.h" +#include "runl.h" #include "gk104.h" #include "changk104.h" @@ -57,6 +58,10 @@ gk110_fifo_runlist = { .commit = gk104_fifo_runlist_commit, }; +const struct nvkm_runl_func +gk110_runl = { +}; + int gk110_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) { @@ -76,6 +81,7 @@ gk110_fifo = { .chid_nr = gk104_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -94,7 +100,10 @@ gk110_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk104_fifo_pbdma, + .runl = &gk110_runl, .runq = &gk104_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index d81593722fa10..c58343b5c49e7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -60,6 +60,7 @@ gk208_fifo = { .chid_nr = gk208_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -78,7 +79,10 @@ gk208_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .runl = &gk110_runl, .runq = &gk208_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 11227cf559416..9a1c1ca95121f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -31,6 +31,7 @@ gk20a_fifo = { .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -49,7 +50,10 @@ gk20a_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .runl = &gk110_runl, .runq = &gk208_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 2dc89d3a0d0c2..44dbebf06992b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -23,6 +23,7 @@ */ #include "priv.h" #include "chan.h" +#include "runl.h" #include "gk104.h" #include "changk104.h" @@ -51,6 +52,10 @@ gm107_fifo_runlist = { .commit = gk104_fifo_runlist_commit, }; +const struct nvkm_runl_func +gm107_runl = { +}; + const struct nvkm_enum gm107_fifo_fault_engine[] = { { 0x01, "DISPLAY" }, @@ -116,6 +121,7 @@ gm107_fifo = { .chid_nr = gm107_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -134,7 +140,10 @@ gm107_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .runl = &gm107_runl, .runq = &gk208_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index f24551c701e1b..e9f3c5f08ca80 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -51,6 +51,7 @@ gm200_fifo = { .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -69,7 +70,10 @@ gm200_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .runl = &gm107_runl, .runq = &gk208_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, .chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 60bc5d3143245..6fea0a5525df2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "runl.h" #include "gk104.h" #include "changk104.h" @@ -28,6 +29,10 @@ #include +static const struct nvkm_runl_func +gp100_runl = { +}; + const struct nvkm_enum gp100_fifo_fault_engine[] = { { 0x01, "DISPLAY" }, @@ -88,6 +93,7 @@ gp100_fifo = { .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -106,7 +112,10 @@ gp100_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .runl = &gp100_runl, .runq = &gk208_runq, + .engn = &gk104_engn, + .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index f7db1b4db0dce..21246a0f319dc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -21,6 +21,7 @@ */ #include "chan.h" #include "cgrp.h" +#include "runl.h" #include "runq.h" #include "gk104.h" @@ -34,6 +35,14 @@ static const struct nvkm_chan_func gv100_chan = { }; +const struct nvkm_engn_func +gv100_engn = { +}; + +const struct nvkm_engn_func +gv100_engn_ce = { +}; + const struct nvkm_runq_func gv100_runq = { }; @@ -70,6 +79,10 @@ gv100_fifo_runlist = { .commit = gk104_fifo_runlist_commit, }; +static const struct nvkm_runl_func +gv100_runl = { +}; + const struct nvkm_enum gv100_fifo_fault_gpcclient[] = { { 0x00, "T1_0" }, @@ -308,6 +321,7 @@ gv100_fifo = { .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -325,7 +339,10 @@ gv100_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .runl = &gv100_runl, .runq = &gv100_runq, + .engn = &gv100_engn, + .engn_ce = &gv100_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 18fa5b933e9ef..ce965d6aa874d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -24,6 +24,7 @@ #include "cgrp.h" #include "chan.h" #include "chid.h" +#include "runl.h" #include "nv04.h" #include "channv04.h" @@ -123,6 +124,10 @@ const struct nvkm_cgrp_func nv04_cgrp = { }; +const struct nvkm_engn_func +nv04_engn = { +}; + void nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags) __acquires(fifo->base.lock) @@ -173,6 +178,10 @@ __releases(fifo->base.lock) spin_unlock_irqrestore(&fifo->base.lock, flags); } +const struct nvkm_runl_func +nv04_runl = { +}; + struct nvkm_engine * nv04_fifo_id_engine(struct nvkm_fifo *fifo, int engi) { @@ -434,6 +443,22 @@ nv04_fifo_init(struct nvkm_fifo *fifo) nvkm_wr32(device, NV03_PFIFO_CACHES, 1); } +int +nv04_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_runl *runl; + + runl = nvkm_runl_new(fifo, 0, 0, 0); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_SW, 0); + nvkm_runl_add(runl, 0, fifo->func->engn_sw, NVKM_ENGINE_DMAOBJ, 0); + nvkm_runl_add(runl, 1, fifo->func->engn , NVKM_ENGINE_GR, 0); + nvkm_runl_add(runl, 2, fifo->func->engn , NVKM_ENGINE_MPEG, 0); /* NV31- */ + return 0; +} + int nv04_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) { @@ -472,12 +497,16 @@ static const struct nvkm_fifo_func nv04_fifo = { .chid_nr = nv04_fifo_chid_nr, .chid_ctor = nv04_fifo_chid_ctor, + .runl_ctor = nv04_fifo_runl_ctor, .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, + .runl = &nv04_runl, + .engn = &nv04_engn, + .engn_sw = &nv04_engn, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index bc25520cb39b4..84416440a8221 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -57,12 +57,16 @@ static const struct nvkm_fifo_func nv10_fifo = { .chid_nr = nv10_fifo_chid_nr, .chid_ctor = nv04_fifo_chid_ctor, + .runl_ctor = nv04_fifo_runl_ctor, .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, + .runl = &nv04_runl, + .engn = &nv04_engn, + .engn_sw = &nv04_engn, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, NV10_CHANNEL_DMA }, &nv10_chan, .oclass = &nv10_fifo_dma_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index dd254d88d008f..28ae10a78ee4b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -89,12 +89,16 @@ static const struct nvkm_fifo_func nv17_fifo = { .chid_nr = nv10_fifo_chid_nr, .chid_ctor = nv04_fifo_chid_ctor, + .runl_ctor = nv04_fifo_runl_ctor, .init = nv17_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, + .runl = &nv04_runl, + .engn = &nv04_engn, + .engn_sw = &nv04_engn, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, NV17_CHANNEL_DMA }, &nv17_chan, .oclass = &nv17_fifo_dma_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index c87d995e4fd1e..492609780c9bc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -23,6 +23,7 @@ */ #include "chan.h" #include "chid.h" +#include "runl.h" #include "nv04.h" #include "channv04.h" @@ -65,6 +66,14 @@ static const struct nvkm_chan_func nv40_chan = { }; +static const struct nvkm_engn_func +nv40_engn = { +}; + +static const struct nvkm_engn_func +nv40_engn_sw = { +}; + static void nv40_fifo_init(struct nvkm_fifo *fifo) { @@ -120,12 +129,16 @@ static const struct nvkm_fifo_func nv40_fifo = { .chid_nr = nv10_fifo_chid_nr, .chid_ctor = nv04_fifo_chid_ctor, + .runl_ctor = nv04_fifo_runl_ctor, .init = nv40_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, + .runl = &nv04_runl, + .engn = &nv40_engn, + .engn_sw = &nv40_engn_sw, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, NV40_CHANNEL_DMA }, &nv40_chan, .oclass = &nv40_fifo_dma_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 13c514acc9d72..e4a7ef5aa4616 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -23,6 +23,7 @@ */ #include "chan.h" #include "chid.h" +#include "runl.h" #include "nv50.h" #include "channv50.h" @@ -35,6 +36,14 @@ static const struct nvkm_chan_func nv50_chan = { }; +static const struct nvkm_engn_func +nv50_engn = { +}; + +const struct nvkm_engn_func +nv50_engn_sw = { +}; + static void nv50_fifo_runlist_update_locked(struct nv50_fifo *fifo) { @@ -65,6 +74,10 @@ nv50_fifo_runlist_update(struct nv50_fifo *fifo) mutex_unlock(&fifo->base.mutex); } +const struct nvkm_runl_func +nv50_runl = { +}; + void nv50_fifo_init(struct nvkm_fifo *base) { @@ -153,12 +166,16 @@ nv50_fifo = { .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, + .runl_ctor = nv04_fifo_runl_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, + .runl = &nv50_runl, + .engn = &nv50_engn, + .engn_sw = &nv50_engn_sw, .cgrp = {{ }, &nv04_cgrp }, .chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan, .oclass = &nv50_fifo_gpfifo_oclass }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h index 0111e7e5a4e38..9a23a84ea7f49 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h @@ -17,4 +17,9 @@ void *nv50_fifo_dtor(struct nvkm_fifo *); int nv50_fifo_oneinit(struct nvkm_fifo *); void nv50_fifo_init(struct nvkm_fifo *); void nv50_fifo_runlist_update(struct nv50_fifo *); + +int g84_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); +struct nvkm_engine *g84_fifo_id_engine(struct nvkm_fifo *, int); +void g84_fifo_uevent_init(struct nvkm_fifo *); +void g84_fifo_uevent_fini(struct nvkm_fifo *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index daa594669985b..93b8b726214e7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -23,6 +23,7 @@ struct nvkm_fifo_func { int (*chid_nr)(struct nvkm_fifo *); int (*chid_ctor)(struct nvkm_fifo *, int nr); int (*runq_nr)(struct nvkm_fifo *); + int (*runl_ctor)(struct nvkm_fifo *); int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data); void (*init)(struct nvkm_fifo *); @@ -66,7 +67,11 @@ struct nvkm_fifo_func { void (*init_timeout)(struct gk104_fifo *); } *pbdma; + const struct nvkm_runl_func *runl; const struct nvkm_runq_func *runq; + const struct nvkm_engn_func *engn; + const struct nvkm_engn_func *engn_sw; + const struct nvkm_engn_func *engn_ce; struct nvkm_fifo_func_cgrp { struct nvkm_sclass user; @@ -90,26 +95,34 @@ int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvk struct nvkm_fifo *); int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); +int nv04_fifo_runl_ctor(struct nvkm_fifo *); void nv04_fifo_init(struct nvkm_fifo *); void nv04_fifo_intr(struct nvkm_fifo *); int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *nv04_fifo_id_engine(struct nvkm_fifo *, int); void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); +extern const struct nvkm_runl_func nv04_runl; +extern const struct nvkm_engn_func nv04_engn; extern const struct nvkm_cgrp_func nv04_cgrp; int nv10_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); +extern const struct nvkm_runl_func nv50_runl; +extern const struct nvkm_engn_func nv50_engn_sw; +extern const struct nvkm_engn_func g84_engn; extern const struct nvkm_chan_func g84_chan; int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); int gf100_fifo_runq_nr(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); +extern const struct nvkm_engn_func gf100_engn_sw; int gk104_fifo_chid_nr(struct nvkm_fifo *); +int gk104_fifo_runl_ctor(struct nvkm_fifo *); void gk104_fifo_intr(struct nvkm_fifo *); extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); @@ -117,8 +130,11 @@ void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); extern const struct nvkm_runq_func gk104_runq; +extern const struct nvkm_engn_func gk104_engn; +extern const struct nvkm_engn_func gk104_engn_ce; int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); +extern const struct nvkm_runl_func gk110_runl; extern const struct nvkm_cgrp_func gk110_cgrp; extern const struct nvkm_chan_func gk110_chan; @@ -126,6 +142,7 @@ extern const struct nvkm_runq_func gk208_runq; void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; +extern const struct nvkm_runl_func gm107_runl; extern const struct nvkm_chan_func gm107_chan; int gm200_fifo_chid_nr(struct nvkm_fifo *); @@ -134,6 +151,8 @@ int gm200_fifo_runq_nr(struct nvkm_fifo *); void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); extern const struct nvkm_runq_func gv100_runq; +extern const struct nvkm_engn_func gv100_engn; +extern const struct nvkm_engn_func gv100_engn_ce; extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c new file mode 100644 index 0000000000000..4d73196741284 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -0,0 +1,112 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "runl.h" +#include "chan.h" +#include "chid.h" +#include "priv.h" + +void +nvkm_runl_del(struct nvkm_runl *runl) +{ + struct nvkm_engn *engn, *engt; + + list_for_each_entry_safe(engn, engt, &runl->engns, head) { + list_del(&engn->head); + kfree(engn); + } + + nvkm_chid_unref(&runl->chid); + nvkm_chid_unref(&runl->cgid); + + list_del(&runl->head); + kfree(runl); +} + +struct nvkm_engn * +nvkm_runl_add(struct nvkm_runl *runl, int engi, const struct nvkm_engn_func *func, + enum nvkm_subdev_type type, int inst) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + struct nvkm_engine *engine; + struct nvkm_engn *engn; + + engine = nvkm_device_engine(device, type, inst); + if (!engine) { + RUNL_DEBUG(runl, "engn %d.%d[%s] not found", engi, inst, nvkm_subdev_type[type]); + return NULL; + } + + if (!(engn = kzalloc(sizeof(*engn), GFP_KERNEL))) + return NULL; + + engn->func = func; + engn->runl = runl; + engn->id = engi; + engn->engine = engine; + list_add_tail(&engn->head, &runl->engns); + return engn; +} + +struct nvkm_runl * +nvkm_runl_get(struct nvkm_fifo *fifo, int runi, u32 addr) +{ + struct nvkm_runl *runl; + + nvkm_runl_foreach(runl, fifo) { + if ((runi >= 0 && runl->id == runi) || (runi < 0 && runl->addr == addr)) + return runl; + } + + return NULL; +} + +struct nvkm_runl * +nvkm_runl_new(struct nvkm_fifo *fifo, int runi, u32 addr, int id_nr) +{ + struct nvkm_subdev *subdev = &fifo->engine.subdev; + struct nvkm_runl *runl; + int ret; + + if (!(runl = kzalloc(sizeof(*runl), GFP_KERNEL))) + return NULL; + + runl->func = fifo->func->runl; + runl->fifo = fifo; + runl->id = runi; + runl->addr = addr; + INIT_LIST_HEAD(&runl->engns); + list_add_tail(&runl->head, &fifo->runls); + + if (!fifo->chid) { + if ((ret = nvkm_chid_new(&nvkm_chan_event, subdev, id_nr, 0, id_nr, &runl->cgid)) || + (ret = nvkm_chid_new(&nvkm_chan_event, subdev, id_nr, 0, id_nr, &runl->chid))) { + RUNL_ERROR(runl, "cgid/chid: %d", ret); + nvkm_runl_del(runl); + return NULL; + } + } else { + runl->cgid = nvkm_chid_ref(fifo->cgid); + runl->chid = nvkm_chid_ref(fifo->chid); + } + + return runl; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h new file mode 100644 index 0000000000000..9f27babc8caf3 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -0,0 +1,55 @@ +#ifndef __NVKM_RUNL_H__ +#define __NVKM_RUNL_H__ +#include +struct nvkm_cgrp; +struct nvkm_memory; +enum nvkm_subdev_type; + +struct nvkm_engn { + const struct nvkm_engn_func { + } *func; + struct nvkm_runl *runl; + int id; + + struct nvkm_engine *engine; + + struct list_head head; +}; + +#define ENGN_PRINT(e,l,p,f,a...) \ + RUNL_PRINT((e)->runl, l, p, "%02d[%8s]:"f, (e)->id, (e)->engine->subdev.name, ##a) +#define ENGN_DEBUG(e,f,a...) ENGN_PRINT((e), DEBUG, info, " "f"\n", ##a) + +struct nvkm_runl { + const struct nvkm_runl_func { + } *func; + struct nvkm_fifo *fifo; + int id; + u32 addr; + + struct nvkm_chid *cgid; + struct nvkm_chid *chid; + + struct list_head engns; + + struct nvkm_runq *runq[2]; + int runq_nr; + + struct list_head head; +}; + +struct nvkm_runl *nvkm_runl_new(struct nvkm_fifo *, int runi, u32 addr, int id_nr); +struct nvkm_runl *nvkm_runl_get(struct nvkm_fifo *, int runi, u32 addr); +struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_engn_func *, + enum nvkm_subdev_type, int inst); +void nvkm_runl_del(struct nvkm_runl *); + +#define nvkm_runl_foreach(runl,fifo) list_for_each_entry((runl), &(fifo)->runls, head) +#define nvkm_runl_foreach_engn(engn,runl) list_for_each_entry((engn), &(runl)->engns, head) + +#define RUNL_PRINT(r,l,p,f,a...) \ + nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "%06x:"f, (r)->addr, ##a) +#define RUNL_ERROR(r,f,a...) RUNL_PRINT((r), ERROR, err, " "f"\n", ##a) +#define RUNL_DEBUG(r,f,a...) RUNL_PRINT((r), DEBUG, info, " "f"\n", ##a) +#define RUNL_TRACE(r,f,a...) RUNL_PRINT((r), TRACE, info, " "f"\n", ##a) +#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 6d04316102352..ff13ae5bb45d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -20,6 +20,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include "chan.h" +#include "runl.h" #include "gk104.h" #include "cgrp.h" @@ -60,6 +61,10 @@ tu102_fifo_runlist = { .commit = tu102_fifo_runlist_commit, }; +static const struct nvkm_runl_func +tu102_runl = { +}; + static const struct nvkm_enum tu102_fifo_fault_engine[] = { { 0x01, "DISPLAY" }, @@ -443,6 +448,7 @@ tu102_fifo = { .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, + .runl_ctor = gk104_fifo_runl_ctor, .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, @@ -460,7 +466,10 @@ tu102_fifo = { .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, .pbdma = &tu102_fifo_pbdma, + .runl = &tu102_runl, .runq = &gv100_runq, + .engn = &gv100_engn, + .engn_ce = &gv100_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new }, }; -- GitLab From 6de125383a5cce5f0d9235a6d3a9ae83dc5d299e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:25 +1000 Subject: [PATCH 1308/1927] drm/nouveau/fifo: expose runlist topology info on all chipsets Previously only available from Kepler onwards. - also fixes the info() queries causing fifo init()/fini() unnecessarily Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nouveau_abi16.c | 26 +++++----- drivers/gpu/drm/nouveau/nouveau_chan.c | 7 ++- drivers/gpu/drm/nouveau/nouveau_chan.h | 5 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 36 +++++-------- drivers/gpu/drm/nouveau/nvkm/core/engine.c | 12 ++--- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 49 ++++++++++++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 52 ------------------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 - 17 files changed, 81 insertions(+), 116 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index 5bee655e7e63e..9237ef328b9fb 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -253,7 +253,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv); struct nouveau_abi16_chan *chan; struct nvif_device *device; - u64 engine; + u64 engine, runm; int ret; if (unlikely(!abi16)) @@ -263,6 +263,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) return nouveau_abi16_put(abi16, -ENODEV); device = &abi16->device; + engine = NV_DEVICE_HOST_RUNLIST_ENGINES_GR; /* hack to allow channel engine type specification on kepler */ if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) { @@ -276,19 +277,18 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) default: return nouveau_abi16_put(abi16, -ENOSYS); } - } else { - engine = NV_DEVICE_HOST_RUNLIST_ENGINES_GR; - } - if (engine != NV_DEVICE_HOST_RUNLIST_ENGINES_CE) - engine = nvif_fifo_runlist(device, engine); - else - engine = nvif_fifo_runlist_ce(device); - init->fb_ctxdma_handle = engine; - init->tt_ctxdma_handle = 0; + init->fb_ctxdma_handle = 0; + init->tt_ctxdma_handle = 0; + } } - if (init->fb_ctxdma_handle == ~0 || init->tt_ctxdma_handle == ~0) + if (engine != NV_DEVICE_HOST_RUNLIST_ENGINES_CE) + runm = nvif_fifo_runlist(device, engine); + else + runm = nvif_fifo_runlist_ce(device); + + if (!runm || init->fb_ctxdma_handle == ~0 || init->tt_ctxdma_handle == ~0) return nouveau_abi16_put(abi16, -EINVAL); /* allocate "abi16 channel" data and make up a handle for it */ @@ -300,8 +300,8 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS) list_add(&chan->head, &abi16->channels); /* create channel object and initialise dma and fence management */ - ret = nouveau_channel_new(drm, device, init->fb_ctxdma_handle, - init->tt_ctxdma_handle, false, &chan->chan); + ret = nouveau_channel_new(drm, device, false, runm, init->fb_ctxdma_handle, + init->tt_ctxdma_handle, &chan->chan); if (ret) goto done; diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 497d889c20b6b..3b5cbb67f15b1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -513,14 +513,13 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) int nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, - u32 arg0, u32 arg1, bool priv, - struct nouveau_channel **pchan) + bool priv, u64 runm, u32 vram, u32 gart, struct nouveau_channel **pchan) { struct nouveau_cli *cli = (void *)device->object.client; int ret; /* hack until fencenv50 is fixed, and agp access relaxed */ - ret = nouveau_channel_ind(drm, device, arg0, priv, pchan); + ret = nouveau_channel_ind(drm, device, runm, priv, pchan); if (ret) { NV_PRINTK(dbg, cli, "ib channel create, %d\n", ret); ret = nouveau_channel_dma(drm, device, pchan); @@ -530,7 +529,7 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, } } - ret = nouveau_channel_init(*pchan, arg0, arg1); + ret = nouveau_channel_init(*pchan, vram, gart); if (ret) { NV_PRINTK(err, cli, "channel failed to initialise, %d\n", ret); nouveau_channel_del(pchan); diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h index 195b38c847717..7ffee796a67ab 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.h +++ b/drivers/gpu/drm/nouveau/nouveau_chan.h @@ -56,9 +56,8 @@ struct nouveau_channel { int nouveau_channels_init(struct nouveau_drm *); -int nouveau_channel_new(struct nouveau_drm *, struct nvif_device *, - u32 arg0, u32 arg1, bool priv, - struct nouveau_channel **); +int nouveau_channel_new(struct nouveau_drm *, struct nvif_device *, bool priv, u64 runm, + u32 vram, u32 gart, struct nouveau_channel **); void nouveau_channel_del(struct nouveau_channel **); int nouveau_channel_idle(struct nouveau_channel *); diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 9df276bff3c7e..1812ffdaae8de 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -316,28 +316,19 @@ static void nouveau_accel_ce_init(struct nouveau_drm *drm) { struct nvif_device *device = &drm->client.device; + u64 runm; int ret = 0; /* Allocate channel that has access to a (preferably async) copy * engine, to use for TTM buffer moves. */ - if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) { - ret = nouveau_channel_new(drm, device, - nvif_fifo_runlist_ce(device), 0, - true, &drm->cechan); - } else - if (device->info.chipset >= 0xa3 && - device->info.chipset != 0xaa && - device->info.chipset != 0xac) { - /* Prior to Kepler, there's only a single runlist, so all - * engines can be accessed from any channel. - * - * We still want to use a separate channel though. - */ - ret = nouveau_channel_new(drm, device, NvDmaFB, NvDmaTT, false, - &drm->cechan); + runm = nvif_fifo_runlist_ce(device); + if (!runm) { + NV_DEBUG(drm, "no ce runlist\n"); + return; } + ret = nouveau_channel_new(drm, device, false, runm, NvDmaFB, NvDmaTT, &drm->cechan); if (ret) NV_ERROR(drm, "failed to create ce channel, %d\n", ret); } @@ -355,23 +346,20 @@ static void nouveau_accel_gr_init(struct nouveau_drm *drm) { struct nvif_device *device = &drm->client.device; - u32 arg0, arg1; + u64 runm; int ret; if (device->info.family >= NV_DEVICE_INFO_V0_AMPERE) return; /* Allocate channel that has access to the graphics engine. */ - if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) { - arg0 = nvif_fifo_runlist(device, NV_DEVICE_HOST_RUNLIST_ENGINES_GR); - arg1 = 1; - } else { - arg0 = NvDmaFB; - arg1 = NvDmaTT; + runm = nvif_fifo_runlist(device, NV_DEVICE_HOST_RUNLIST_ENGINES_GR); + if (!runm) { + NV_DEBUG(drm, "no gr runlist\n"); + return; } - ret = nouveau_channel_new(drm, device, arg0, arg1, false, - &drm->channel); + ret = nouveau_channel_new(drm, device, false, runm, NvDmaFB, NvDmaTT, &drm->channel); if (ret) { NV_ERROR(drm, "failed to create kernel channel, %d\n", ret); nouveau_accel_gr_fini(drm); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c index 558bd10e55188..964615a609971 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c @@ -80,14 +80,10 @@ static int nvkm_engine_info(struct nvkm_subdev *subdev, u64 mthd, u64 *data) { struct nvkm_engine *engine = nvkm_engine(subdev); - if (engine->func->info) { - if (!IS_ERR((engine = nvkm_engine_ref(engine)))) { - int ret = engine->func->info(engine, mthd, data); - nvkm_engine_unref(&engine); - return ret; - } - return PTR_ERR(engine); - } + + if (engine->func->info) + return engine->func->info(engine, mthd, data); + return -ENOSYS; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 60d5cbbe43810..2ccc0b3a3ecd6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -221,12 +221,57 @@ static int nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + struct nvkm_runl *runl; + struct nvkm_engn *engn; + int ret; + + ret = nvkm_subdev_oneinit(&fifo->engine.subdev); + if (ret) + return ret; switch (mthd) { case NV_DEVICE_HOST_CHANNELS: *data = fifo->chid ? fifo->chid->nr : 0; return 0; + case NV_DEVICE_HOST_RUNLISTS: + *data = 0; + nvkm_runl_foreach(runl, fifo) + *data |= BIT(runl->id); + return 0; + case NV_DEVICE_HOST_RUNLIST_ENGINES: + runl = nvkm_runl_get(fifo, *data, 0); + if (runl) { + *data = 0; + nvkm_runl_foreach_engn(engn, runl) { +#define CASE(n) case NVKM_ENGINE_##n: *data |= NV_DEVICE_HOST_RUNLIST_ENGINES_##n; break + switch (engn->engine->subdev.type) { + case NVKM_ENGINE_DMAOBJ: + break; + CASE(SW ); + CASE(GR ); + CASE(MPEG ); + CASE(ME ); + CASE(CIPHER); + CASE(BSP ); + CASE(VP ); + CASE(CE ); + CASE(SEC ); + CASE(MSVLD ); + CASE(MSPDEC); + CASE(MSPPP ); + CASE(MSENC ); + CASE(VIC ); + CASE(SEC2 ); + CASE(NVDEC ); + CASE(NVENC ); + default: + WARN_ON(1); + break; + } +#undef CASE + } + return 0; + } + return -EINVAL; default: - if (fifo->func->info) - return fifo->func->info(fifo, mthd, data); break; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index eaf61c07fafc9..b41481c984d10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -39,7 +39,6 @@ #include #include -#include static const struct nvkm_chan_func gk104_chan = { @@ -991,56 +990,6 @@ gk104_fifo_fini(struct nvkm_fifo *base) nvkm_mask(device, 0x002140, 0x10000000, 0x10000000); } -int -gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - switch (mthd) { - case NV_DEVICE_HOST_RUNLISTS: - *data = (1ULL << fifo->runlist_nr) - 1; - return 0; - case NV_DEVICE_HOST_RUNLIST_ENGINES: { - if (*data < fifo->runlist_nr) { - unsigned long engm = fifo->runlist[*data].engm; - struct nvkm_engine *engine; - int engn; - *data = 0; - for_each_set_bit(engn, &engm, fifo->engine_nr) { - if ((engine = fifo->engine[engn].engine)) { -#define CASE(n) case NVKM_ENGINE_##n: *data |= NV_DEVICE_HOST_RUNLIST_ENGINES_##n; break - switch (engine->subdev.type) { - CASE(SW ); - CASE(GR ); - CASE(MPEG ); - CASE(ME ); - CASE(CIPHER); - CASE(BSP ); - CASE(VP ); - CASE(CE ); - CASE(SEC ); - CASE(MSVLD ); - CASE(MSPDEC); - CASE(MSPPP ); - CASE(MSENC ); - CASE(VIC ); - CASE(SEC2 ); - CASE(NVDEC ); - CASE(NVENC ); - default: - WARN_ON(1); - break; - } - } - } - return 0; - } - } - return -EINVAL; - default: - return -EINVAL; - } -} - void gk104_fifo_init(struct nvkm_fifo *base) { @@ -1225,7 +1174,6 @@ gk104_fifo = { .chid_ctor = gf100_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 636091c7fcee2..403560aefbb78 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -75,7 +75,6 @@ void gk104_fifo_intr_runlist(struct gk104_fifo *fifo); void gk104_fifo_intr_engine(struct gk104_fifo *fifo); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); -int gk104_fifo_info(struct nvkm_fifo *base, u64 mthd, u64 *data); void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 08f5b06845106..64bdd3fd3ae66 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -82,7 +82,6 @@ gk110_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index c58343b5c49e7..3e074a976c440 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -61,7 +61,6 @@ gk208_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 9a1c1ca95121f..a494dacad37e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -32,7 +32,6 @@ gk20a_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 44dbebf06992b..c4a6972ee473e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -122,7 +122,6 @@ gm107_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index e9f3c5f08ca80..bc9893c4d297b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -52,7 +52,6 @@ gm200_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 6fea0a5525df2..4ac4afbf3147d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -94,7 +94,6 @@ gp100_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 21246a0f319dc..e1069c672438c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -322,7 +322,6 @@ gv100_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 93b8b726214e7..b0fa9d1df91da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -25,7 +25,6 @@ struct nvkm_fifo_func { int (*runq_nr)(struct nvkm_fifo *); int (*runl_ctor)(struct nvkm_fifo *); - int (*info)(struct nvkm_fifo *, u64 mthd, u64 *data); void (*init)(struct nvkm_fifo *); void (*fini)(struct nvkm_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index ff13ae5bb45d1..e5b9b1fc38667 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -449,7 +449,6 @@ tu102_fifo = { .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, - .info = gk104_fifo_info, .init = gk104_fifo_init, .fini = gk104_fifo_fini, .intr = tu102_fifo_intr, -- GitLab From eb39c613481fd2fe6b2f66ec2ca21f8fdcdd4cac Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:25 +1000 Subject: [PATCH 1309/1927] drm/nouveau/fifo: expose per-runlist CHID information DRM uses this to setup fence-related items. - nouveau_chan.runlist will always be "0" for the moment, not an issue as GPUs prior to ampere have system-wide channel IDs, Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/cl0080.h | 4 +- drivers/gpu/drm/nouveau/nouveau_chan.c | 47 +++++++++++++++++-- drivers/gpu/drm/nouveau/nouveau_chan.h | 2 + drivers/gpu/drm/nouveau/nouveau_drm.c | 1 + drivers/gpu/drm/nouveau/nouveau_drv.h | 8 +++- drivers/gpu/drm/nouveau/nouveau_fence.c | 8 +--- drivers/gpu/drm/nouveau/nv84_fence.c | 22 +++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 9 ++++ 8 files changed, 80 insertions(+), 21 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h index 59759c4fb62e2..8b5a240d57e47 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h +++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h @@ -68,7 +68,7 @@ struct nv_device_time_v0 { /* Returns the number of available runlists. */ #define NV_DEVICE_HOST_RUNLISTS NV_DEVICE_HOST(0x00000000) -/* Returns the number of available channels. */ +/* Returns the number of available channels (0 if per-runlist). */ #define NV_DEVICE_HOST_CHANNELS NV_DEVICE_HOST(0x00000001) /* Returns a mask of available engine types on runlist(data). */ @@ -90,4 +90,6 @@ struct nv_device_time_v0 { #define NV_DEVICE_HOST_RUNLIST_ENGINES_SEC2 0x00004000 #define NV_DEVICE_HOST_RUNLIST_ENGINES_NVDEC 0x00008000 #define NV_DEVICE_HOST_RUNLIST_ENGINES_NVENC 0x00010000 +/* Returns the number of available channels on runlist(data). */ +#define NV_DEVICE_HOST_RUNLIST_CHANNELS NV_DEVICE_HOST(0x00000101) #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 3b5cbb67f15b1..5b7042d94e808 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -543,6 +543,12 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, return ret; } +void +nouveau_channels_fini(struct nouveau_drm *drm) +{ + kfree(drm->runl); +} + int nouveau_channels_init(struct nouveau_drm *drm) { @@ -550,20 +556,53 @@ nouveau_channels_init(struct nouveau_drm *drm) struct nv_device_info_v1 m; struct { struct nv_device_info_v1_data channels; + struct nv_device_info_v1_data runlists; } v; } args = { .m.version = 1, .m.count = sizeof(args.v) / sizeof(args.v.channels), .v.channels.mthd = NV_DEVICE_HOST_CHANNELS, + .v.runlists.mthd = NV_DEVICE_HOST_RUNLISTS, }; struct nvif_object *device = &drm->client.device.object; - int ret; + int ret, i; ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args)); - if (ret || args.v.channels.mthd == NV_DEVICE_INFO_INVALID) + if (ret || + args.v.runlists.mthd == NV_DEVICE_INFO_INVALID || !args.v.runlists.data || + args.v.channels.mthd == NV_DEVICE_INFO_INVALID) return -ENODEV; - drm->chan.nr = args.v.channels.data; - drm->chan.context_base = dma_fence_context_alloc(drm->chan.nr); + drm->chan_nr = drm->chan_total = args.v.channels.data; + drm->runl_nr = fls64(args.v.runlists.data); + drm->runl = kcalloc(drm->runl_nr, sizeof(*drm->runl), GFP_KERNEL); + if (!drm->runl) + return -ENOMEM; + + if (drm->chan_nr == 0) { + for (i = 0; i < drm->runl_nr; i++) { + if (!(args.v.runlists.data & BIT(i))) + continue; + + args.v.channels.mthd = NV_DEVICE_HOST_RUNLIST_CHANNELS; + args.v.channels.data = i; + + ret = nvif_object_mthd(device, NV_DEVICE_V0_INFO, &args, sizeof(args)); + if (ret || args.v.channels.mthd == NV_DEVICE_INFO_INVALID) + return -ENODEV; + + drm->runl[i].chan_nr = args.v.channels.data; + drm->runl[i].chan_id_base = drm->chan_total; + drm->runl[i].context_base = dma_fence_context_alloc(drm->runl[i].chan_nr); + + drm->chan_total += drm->runl[i].chan_nr; + } + } else { + drm->runl[0].context_base = dma_fence_context_alloc(drm->chan_nr); + for (i = 1; i < drm->runl_nr; i++) + drm->runl[i].context_base = drm->runl[0].context_base; + + } + return 0; } diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h index 7ffee796a67ab..40467e8c96685 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.h +++ b/drivers/gpu/drm/nouveau/nouveau_chan.h @@ -16,6 +16,7 @@ struct nouveau_channel { struct nouveau_drm *drm; struct nouveau_vmm *vmm; + int runlist; int chid; u64 inst; u32 token; @@ -55,6 +56,7 @@ struct nouveau_channel { }; int nouveau_channels_init(struct nouveau_drm *); +void nouveau_channels_fini(struct nouveau_drm *); int nouveau_channel_new(struct nouveau_drm *, struct nvif_device *, bool priv, u64 runm, u32 vram, u32 gart, struct nouveau_channel **); diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 1812ffdaae8de..4a533b0804450 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -424,6 +424,7 @@ nouveau_accel_fini(struct nouveau_drm *drm) nouveau_accel_gr_fini(drm); if (drm->fence) nouveau_fence(drm)->dtor(drm); + nouveau_channels_fini(drm); } static void diff --git a/drivers/gpu/drm/nouveau/nouveau_drv.h b/drivers/gpu/drm/nouveau/nouveau_drv.h index 6fb636020f7fc..d6dd07bfa64aa 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drv.h +++ b/drivers/gpu/drm/nouveau/nouveau_drv.h @@ -174,10 +174,14 @@ struct nouveau_drm { void *fence; /* Global channel management. */ + int chan_total; /* Number of channels across all runlists. */ + int chan_nr; /* 0 if per-runlist CHIDs. */ + int runl_nr; struct { - int nr; + int chan_nr; + int chan_id_base; u64 context_base; - } chan; + } *runl; /* context for accelerated drm-internal operations */ struct nouveau_channel *cechan; diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c index c4c8af16af026..ee5e9d40c166f 100644 --- a/drivers/gpu/drm/nouveau/nouveau_fence.c +++ b/drivers/gpu/drm/nouveau/nouveau_fence.c @@ -77,10 +77,6 @@ nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm) fence->ops != &nouveau_fence_ops_uevent) return NULL; - if (fence->context < drm->chan.context_base || - fence->context >= drm->chan.context_base + drm->chan.nr) - return NULL; - return from_fence(fence); } @@ -184,7 +180,7 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha INIT_LIST_HEAD(&fctx->flip); INIT_LIST_HEAD(&fctx->pending); spin_lock_init(&fctx->lock); - fctx->context = chan->drm->chan.context_base + chan->chid; + fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid; if (chan == chan->drm->cechan) strcpy(fctx->name, "copy engine channel"); @@ -200,7 +196,7 @@ nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_cha args.host.version = 0; args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR; - ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", chan->chid, + ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", (chan->runlist << 16) | chan->chid, nouveau_fence_wait_uevent_handler, false, &args.base, sizeof(args), &fctx->event); diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index c3526a8622e3e..a01fd934c85e2 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -76,12 +76,18 @@ nv84_fence_sync32(struct nouveau_channel *chan, u64 virtual, u32 sequence) return ret; } +static inline u32 +nv84_fence_chid(struct nouveau_channel *chan) +{ + return chan->drm->runl[chan->runlist].chan_id_base + chan->chid; +} + static int nv84_fence_emit(struct nouveau_fence *fence) { struct nouveau_channel *chan = fence->channel; struct nv84_fence_chan *fctx = chan->fence; - u64 addr = fctx->vma->addr + chan->chid * 16; + u64 addr = fctx->vma->addr + nv84_fence_chid(chan) * 16; return fctx->base.emit32(chan, addr, fence->base.seqno); } @@ -91,7 +97,7 @@ nv84_fence_sync(struct nouveau_fence *fence, struct nouveau_channel *prev, struct nouveau_channel *chan) { struct nv84_fence_chan *fctx = chan->fence; - u64 addr = fctx->vma->addr + prev->chid * 16; + u64 addr = fctx->vma->addr + nv84_fence_chid(prev) * 16; return fctx->base.sync32(chan, addr, fence->base.seqno); } @@ -100,7 +106,7 @@ static u32 nv84_fence_read(struct nouveau_channel *chan) { struct nv84_fence_priv *priv = chan->drm->fence; - return nouveau_bo_rd32(priv->bo, chan->chid * 16/4); + return nouveau_bo_rd32(priv->bo, nv84_fence_chid(chan) * 16/4); } static void @@ -109,7 +115,7 @@ nv84_fence_context_del(struct nouveau_channel *chan) struct nv84_fence_priv *priv = chan->drm->fence; struct nv84_fence_chan *fctx = chan->fence; - nouveau_bo_wr32(priv->bo, chan->chid * 16 / 4, fctx->base.sequence); + nouveau_bo_wr32(priv->bo, nv84_fence_chid(chan) * 16 / 4, fctx->base.sequence); mutex_lock(&priv->mutex); nouveau_vma_del(&fctx->vma); mutex_unlock(&priv->mutex); @@ -152,9 +158,9 @@ nv84_fence_suspend(struct nouveau_drm *drm) struct nv84_fence_priv *priv = drm->fence; int i; - priv->suspend = vmalloc(array_size(sizeof(u32), drm->chan.nr)); + priv->suspend = vmalloc(array_size(sizeof(u32), drm->chan_total)); if (priv->suspend) { - for (i = 0; i < drm->chan.nr; i++) + for (i = 0; i < drm->chan_total; i++) priv->suspend[i] = nouveau_bo_rd32(priv->bo, i*4); } @@ -168,7 +174,7 @@ nv84_fence_resume(struct nouveau_drm *drm) int i; if (priv->suspend) { - for (i = 0; i < drm->chan.nr; i++) + for (i = 0; i < drm->chan_total; i++) nouveau_bo_wr32(priv->bo, i*4, priv->suspend[i]); vfree(priv->suspend); priv->suspend = NULL; @@ -216,7 +222,7 @@ nv84_fence_create(struct nouveau_drm *drm) * will lose CPU/GPU coherency! */ NOUVEAU_GEM_DOMAIN_GART | NOUVEAU_GEM_DOMAIN_COHERENT; - ret = nouveau_bo_new(&drm->client, 16 * drm->chan.nr, 0, + ret = nouveau_bo_new(&drm->client, 16 * drm->chan_total, 0, domain, 0, 0, NULL, NULL, &priv->bo); if (ret == 0) { ret = nouveau_bo_pin(priv->bo, domain, false); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 2ccc0b3a3ecd6..0f603d9f00cb1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -271,6 +271,15 @@ nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) return 0; } return -EINVAL; + case NV_DEVICE_HOST_RUNLIST_CHANNELS: + if (!fifo->chid) { + runl = nvkm_runl_get(fifo, *data, 0); + if (runl) { + *data = runl->chid->nr; + return 0; + } + } + return -EINVAL; default: break; } -- GitLab From 468fae7bb0a31094cff900e92a08f02263f793be Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:26 +1000 Subject: [PATCH 1310/1927] drm/nouveau/fifo: add cgrp, have all channels be part of one Engine context tracking will move to nvkm_cgrp in later commits, so we create SW-only channel groups on HW without support for them. - switches to nvkm_chid for TSG/channel ID allocation Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 5 +- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c | 94 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 23 ++++- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 68 ++++++++++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 5 + .../gpu/drm/nouveau/nvkm/engine/fifo/chid.c | 29 ++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/chid.h | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 9 ++ 13 files changed, 231 insertions(+), 18 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 1e12697beae29..62fe64dab11a2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -6,7 +6,6 @@ #include struct nvkm_fault_data; -#define NVKM_FIFO_CHID_NR 4096 #define NVKM_FIFO_ENGN_NR 16 struct nvkm_fifo_engn { @@ -17,13 +16,16 @@ struct nvkm_fifo_engn { struct nvkm_chan { const struct nvkm_chan_func *func; + char name[64]; + struct nvkm_cgrp *cgrp; + + union { int id; int chid; }; /*FIXME: remove later */ struct nvkm_fifo *fifo; u32 engm; struct nvkm_object object; struct list_head head; - u16 chid; struct nvkm_gpuobj *inst; struct nvkm_gpuobj *push; struct nvkm_vmm *vmm; @@ -43,7 +45,6 @@ struct nvkm_fifo { struct list_head runqs; struct list_head runls; - DECLARE_BITMAP(mask, NVKM_FIFO_CHID_NR); int nr; struct list_head chan; spinlock_t lock; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index a92a88a72e596..eb82da7e7d87c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -1,5 +1,6 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/engine/fifo/base.o +nvkm-y += nvkm/engine/fifo/cgrp.o nvkm-y += nvkm/engine/fifo/chan.o nvkm-y += nvkm/engine/fifo/chid.o nvkm-y += nvkm/engine/fifo/runl.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 0f603d9f00cb1..3e62eacc48adb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -389,10 +389,7 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, INIT_LIST_HEAD(&fifo->chan); nr = func->chid_nr(fifo); - if (WARN_ON(fifo->nr > NVKM_FIFO_CHID_NR)) - fifo->nr = NVKM_FIFO_CHID_NR; - else - fifo->nr = nr; + fifo->nr = nr; if (func->uevent_init) { ret = nvkm_event_init(&nvkm_fifo_uevent_func, &fifo->engine.subdev, 1, 1, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c new file mode 100644 index 0000000000000..e6ac178db8b50 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c @@ -0,0 +1,94 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "cgrp.h" +#include "chan.h" +#include "chid.h" +#include "runl.h" +#include "priv.h" + +#include + +static void +nvkm_cgrp_del(struct kref *kref) +{ + struct nvkm_cgrp *cgrp = container_of(kref, typeof(*cgrp), kref); + struct nvkm_runl *runl = cgrp->runl; + + if (runl->cgid) + nvkm_chid_put(runl->cgid, cgrp->id, &cgrp->lock); + + nvkm_vmm_unref(&cgrp->vmm); + kfree(cgrp); +} + +void +nvkm_cgrp_unref(struct nvkm_cgrp **pcgrp) +{ + struct nvkm_cgrp *cgrp = *pcgrp; + + if (!cgrp) + return; + + kref_put(&cgrp->kref, nvkm_cgrp_del); + *pcgrp = NULL; +} + +struct nvkm_cgrp * +nvkm_cgrp_ref(struct nvkm_cgrp *cgrp) +{ + if (cgrp) + kref_get(&cgrp->kref); + + return cgrp; +} + +int +nvkm_cgrp_new(struct nvkm_runl *runl, const char *name, struct nvkm_vmm *vmm, bool hw, + struct nvkm_cgrp **pcgrp) +{ + struct nvkm_cgrp *cgrp; + + if (!(cgrp = *pcgrp = kmalloc(sizeof(*cgrp), GFP_KERNEL))) + return -ENOMEM; + + cgrp->func = runl->fifo->func->cgrp.func; + strscpy(cgrp->name, name, sizeof(cgrp->name)); + cgrp->runl = runl; + cgrp->vmm = nvkm_vmm_ref(vmm); + cgrp->hw = hw; + cgrp->id = -1; + kref_init(&cgrp->kref); + cgrp->chans = NULL; + cgrp->chan_nr = 0; + spin_lock_init(&cgrp->lock); + + if (runl->cgid) { + cgrp->id = nvkm_chid_get(runl->cgid, cgrp); + if (cgrp->id < 0) { + RUNL_ERROR(runl, "!cgids"); + nvkm_cgrp_unref(pcgrp); + return -ENOSPC; + } + } + + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index 9a677a1acf9ba..d0510df9286a4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -2,13 +2,34 @@ #ifndef __NVKM_CGRP_H__ #define __NVKM_CGRP_H__ #include +struct nvkm_chan; struct nvkm_cgrp { const struct nvkm_cgrp_func { } *func; + char name[64]; + struct nvkm_runl *runl; + struct nvkm_vmm *vmm; + bool hw; int id; + struct kref kref; + + struct nvkm_chan *chans; + int chan_nr; + + spinlock_t lock; /* protects irq handler channel (group) lookup */ + struct list_head head; struct list_head chan; - int chan_nr; }; + +int nvkm_cgrp_new(struct nvkm_runl *, const char *name, struct nvkm_vmm *, bool hw, + struct nvkm_cgrp **); +struct nvkm_cgrp *nvkm_cgrp_ref(struct nvkm_cgrp *); +void nvkm_cgrp_unref(struct nvkm_cgrp **); + +#define CGRP_PRCLI(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) +#define CGRP_PRINT(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:"f, (c)->id, ##a) +#define CGRP_ERROR(c,f,a...) CGRP_PRCLI((c), ERROR, err, " "f"\n", ##a) +#define CGRP_TRACE(c,f,a...) CGRP_PRINT((c), TRACE, info, " "f"\n", ##a) #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 464c956aaca9d..e2178a8f78a7f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -22,6 +22,9 @@ * Authors: Ben Skeggs */ #include "chan.h" +#include "chid.h" +#include "cgrp.h" +#include "runl.h" #include "priv.h" #include @@ -312,6 +315,11 @@ nvkm_chan_del(struct nvkm_chan **pchan) if (!chan) return; + if (chan->cgrp) { + nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock); + nvkm_cgrp_unref(&chan->cgrp); + } + chan = nvkm_object_dtor(&chan->object); kfree(chan); } @@ -326,7 +334,6 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object) spin_lock_irqsave(&fifo->lock, flags); if (!list_empty(&chan->head)) { - __clear_bit(chan->chid, fifo->mask); list_del(&chan->head); } spin_unlock_irqrestore(&fifo->lock, flags); @@ -363,9 +370,22 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, struct nvkm_client *client = oclass->client; struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_dmaobj *dmaobj; + struct nvkm_cgrp *cgrp = NULL; + struct nvkm_runl *runl; + struct nvkm_engn *engn = NULL; + struct nvkm_vmm *vmm = NULL; unsigned long flags; int ret; + nvkm_runl_foreach(runl, fifo) { + engn = nvkm_runl_find_engn(engn, runl, engm & BIT(engn->id)); + if (engn) + break; + } + + if (!engn) + return -EINVAL; + /*FIXME: temp kludge to ease transition, remove later */ if (!(func = kmalloc(sizeof(*func), GFP_KERNEL))) return -ENOMEM; @@ -383,12 +403,38 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, func->submit_token = fn->submit_token; chan->func = func; + chan->id = -1; nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); chan->fifo = fifo; chan->engm = engm; INIT_LIST_HEAD(&chan->head); + /* Join channel group. + * + * GK110 and newer support channel groups (aka TSGs), where individual channels + * share a timeslice, and, engine context(s). + * + * As such, engine contexts are tracked in nvkm_cgrp and we need them even when + * channels aren't in an API channel group, and on HW that doesn't support TSGs. + */ + if (!cgrp) { + ret = nvkm_cgrp_new(runl, chan->name, vmm, fifo->func->cgrp.force, &chan->cgrp); + if (ret) { + RUNL_DEBUG(runl, "cgrp %d", ret); + return ret; + } + + cgrp = chan->cgrp; + } else { + if (cgrp->runl != runl || cgrp->vmm != vmm) { + RUNL_DEBUG(runl, "cgrp %d %d", cgrp->runl != runl, cgrp->vmm != vmm); + return -EINVAL; + } + + chan->cgrp = nvkm_cgrp_ref(cgrp); + } + /* instance memory */ ret = nvkm_gpuobj_new(device, size, align, zero, NULL, &chan->inst); if (ret) @@ -422,15 +468,23 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, chan->vmm = nvkm_vmm_ref(vmm); } - /* allocate channel id */ - spin_lock_irqsave(&fifo->lock, flags); - chan->chid = find_first_zero_bit(fifo->mask, NVKM_FIFO_CHID_NR); - if (chan->chid >= NVKM_FIFO_CHID_NR) { - spin_unlock_irqrestore(&fifo->lock, flags); + /* Allocate channel ID. */ + if (runl->cgid) { + chan->id = chan->cgrp->id; + runl->chid->data[chan->id] = chan; + set_bit(chan->id, runl->chid->used); + goto temp_hack_until_no_chid_eq_cgid_req; + } + + chan->id = nvkm_chid_get(runl->chid, chan); + if (chan->id < 0) { + RUNL_ERROR(runl, "!chids"); return -ENOSPC; } + +temp_hack_until_no_chid_eq_cgid_req: + spin_lock_irqsave(&fifo->lock, flags); list_add(&chan->head, &fifo->chan); - __set_bit(chan->chid, fifo->mask); spin_unlock_irqrestore(&fifo->lock, flags); /* determine address of this channel's user registers */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index a9c0e02b6f9f4..0f1added5c242 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -27,5 +27,10 @@ int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, const struct nvkm_oclass *, struct nvkm_fifo_chan *); void nvkm_chan_del(struct nvkm_chan **); +#define CHAN_PRCLI(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) +#define CHAN_PRINT(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:"f, (c)->id, ##a) +#define CHAN_ERROR(c,f,a...) CHAN_PRCLI((c), ERROR, err, " "f"\n", ##a) +#define CHAN_TRACE(c,f,a...) CHAN_PRINT((c), TRACE, info, " "f"\n", ##a) + int nvkm_fifo_chan_child_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c index 7c582bce7e245..23944d95efd5b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.c @@ -21,6 +21,35 @@ */ #include "chid.h" +void +nvkm_chid_put(struct nvkm_chid *chid, int id, spinlock_t *data_lock) +{ + if (id >= 0) { + spin_lock_irq(&chid->lock); + spin_lock(data_lock); + chid->data[id] = NULL; + spin_unlock(data_lock); + clear_bit(id, chid->used); + spin_unlock_irq(&chid->lock); + } +} + +int +nvkm_chid_get(struct nvkm_chid *chid, void *data) +{ + int id = -1, cid; + + spin_lock_irq(&chid->lock); + cid = find_first_zero_bit(chid->used, chid->nr); + if (cid < chid->nr) { + set_bit(cid, chid->used); + chid->data[cid] = data; + id = cid; + } + spin_unlock_irq(&chid->lock); + return id; +} + static void nvkm_chid_del(struct kref *kref) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h index d359828dde809..2a42efb18401d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chid.h @@ -20,4 +20,6 @@ int nvkm_chid_new(const struct nvkm_event_func *, struct nvkm_subdev *, int nr, int first, int count, struct nvkm_chid **pchid); struct nvkm_chid *nvkm_chid_ref(struct nvkm_chid *); void nvkm_chid_unref(struct nvkm_chid **); +int nvkm_chid_get(struct nvkm_chid *, void *data); +void nvkm_chid_put(struct nvkm_chid *, int id, spinlock_t *data_lock); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index ce965d6aa874d..52fe9d5a5a4ab 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -489,7 +489,6 @@ nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, if (ret) return ret; - set_bit(nr - 1, fifo->base.mask); /* inactive channel */ return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index e4a7ef5aa4616..4fcf49dbfd907 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -155,8 +155,6 @@ nv50_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, if (ret) return ret; - set_bit(0, fifo->base.mask); /* PIO channel */ - set_bit(127, fifo->base.mask); /* inactive channel */ return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 4d73196741284..1a15b0a9ab7ce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -38,6 +38,7 @@ nvkm_runl_del(struct nvkm_runl *runl) nvkm_chid_unref(&runl->cgid); list_del(&runl->head); + mutex_destroy(&runl->mutex); kfree(runl); } @@ -94,6 +95,8 @@ nvkm_runl_new(struct nvkm_fifo *fifo, int runi, u32 addr, int id_nr) runl->id = runi; runl->addr = addr; INIT_LIST_HEAD(&runl->engns); + INIT_LIST_HEAD(&runl->cgrps); + mutex_init(&runl->mutex); list_add_tail(&runl->head, &fifo->runls); if (!fifo->chid) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 9f27babc8caf3..cb0de870a118b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -35,6 +35,11 @@ struct nvkm_runl { struct nvkm_runq *runq[2]; int runq_nr; + struct list_head cgrps; + int cgrp_nr; + int chan_nr; + struct mutex mutex; + struct list_head head; }; @@ -44,8 +49,12 @@ struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_ enum nvkm_subdev_type, int inst); void nvkm_runl_del(struct nvkm_runl *); +#define nvkm_runl_find_engn(engn,runl,cond) nvkm_list_find(engn, &(runl)->engns, head, (cond)) + #define nvkm_runl_foreach(runl,fifo) list_for_each_entry((runl), &(fifo)->runls, head) #define nvkm_runl_foreach_engn(engn,runl) list_for_each_entry((engn), &(runl)->engns, head) +#define nvkm_runl_foreach_engn_cond(engn,runl,cond) \ + nvkm_list_foreach(engn, &(runl)->engns, head, (cond)) #define RUNL_PRINT(r,l,p,f,a...) \ nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "%06x:"f, (r)->addr, ##a) -- GitLab From 0fc72ee9d8d665484ecae652d114f577313c4cc6 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:26 +1000 Subject: [PATCH 1311/1927] drm/nouveau/fifo: use runlist engine info to lookup engine classes Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 46 ------------------ .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 47 ------------------- .../gpu/drm/nouveau/nvkm/engine/fifo/g98.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 10 ---- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 19 -------- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 3 -- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 40 ++++++++++++++-- 22 files changed, 35 insertions(+), 146 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 62fe64dab11a2..446d058f558a2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -22,7 +22,6 @@ struct nvkm_chan { union { int id; int chid; }; /*FIXME: remove later */ struct nvkm_fifo *fifo; - u32 engm; struct nvkm_object object; struct list_head head; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index e2178a8f78a7f..5fabf73730796 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -216,50 +216,6 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, return 0; } -static int -nvkm_fifo_chan_child_get(struct nvkm_object *object, int index, - struct nvkm_oclass *oclass) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - struct nvkm_fifo *fifo = chan->fifo; - struct nvkm_engine *engine; - u32 engm = chan->engm; - int engi, ret, c; - - for (; c = 0, engi = __ffs(engm), engm; engm &= ~(1ULL << engi)) { - if (!(engine = fifo->func->id_engine(fifo, engi))) - continue; - oclass->engine = engine; - oclass->base.oclass = 0; - - if (engine->func->fifo.sclass) { - ret = engine->func->fifo.sclass(oclass, index); - if (oclass->base.oclass) { - if (!oclass->base.ctor) - oclass->base.ctor = nvkm_object_new; - oclass->ctor = nvkm_fifo_chan_child_new; - return 0; - } - - index -= ret; - continue; - } - - while (engine->func->sclass[c].oclass) { - if (c++ == index) { - oclass->base = engine->func->sclass[index]; - if (!oclass->base.ctor) - oclass->base.ctor = nvkm_object_new; - oclass->ctor = nvkm_fifo_chan_child_new; - return 0; - } - } - index -= c; - } - - return -EINVAL; -} - static int nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) { @@ -355,7 +311,6 @@ nvkm_fifo_chan_func = { .init = nvkm_fifo_chan_init, .fini = nvkm_fifo_chan_fini, .map = nvkm_fifo_chan_map, - .sclass = nvkm_fifo_chan_child_get, .uevent = nvkm_fifo_chan_uevent, }; @@ -407,7 +362,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); chan->fifo = fifo; - chan->engm = engm; INIT_LIST_HEAD(&chan->head); /* Join channel group. diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 3d242350fdee4..122929b7d74fe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -51,52 +51,6 @@ g84_fifo_uevent_init(struct nvkm_fifo *fifo) nvkm_mask(device, 0x002140, 0x40000000, 0x40000000); } -struct nvkm_engine * -g84_fifo_id_engine(struct nvkm_fifo *fifo, int engi) -{ - struct nvkm_device *device = fifo->engine.subdev.device; - struct nvkm_engine *engine; - enum nvkm_subdev_type type; - - switch (engi) { - case G84_FIFO_ENGN_SW : type = NVKM_ENGINE_SW; break; - case G84_FIFO_ENGN_GR : type = NVKM_ENGINE_GR; break; - case G84_FIFO_ENGN_MPEG : - if ((engine = nvkm_device_engine(device, NVKM_ENGINE_MSPPP, 0))) - return engine; - type = NVKM_ENGINE_MPEG; - break; - case G84_FIFO_ENGN_ME : - if ((engine = nvkm_device_engine(device, NVKM_ENGINE_CE, 0))) - return engine; - type = NVKM_ENGINE_ME; - break; - case G84_FIFO_ENGN_VP : - if ((engine = nvkm_device_engine(device, NVKM_ENGINE_MSPDEC, 0))) - return engine; - type = NVKM_ENGINE_VP; - break; - case G84_FIFO_ENGN_CIPHER: - if ((engine = nvkm_device_engine(device, NVKM_ENGINE_VIC, 0))) - return engine; - if ((engine = nvkm_device_engine(device, NVKM_ENGINE_SEC, 0))) - return engine; - type = NVKM_ENGINE_CIPHER; - break; - case G84_FIFO_ENGN_BSP : - if ((engine = nvkm_device_engine(device, NVKM_ENGINE_MSVLD, 0))) - return engine; - type = NVKM_ENGINE_BSP; - break; - case G84_FIFO_ENGN_DMA : type = NVKM_ENGINE_DMAOBJ; break; - default: - WARN_ON(1); - return NULL; - } - - return nvkm_device_engine(fifo->engine.subdev.device, type, 0); -} - int g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { @@ -149,7 +103,6 @@ g84_fifo = { .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = g84_fifo_engine_id, - .id_engine = g84_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .uevent_init = g84_fifo_uevent_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c index 39177ec0eca64..118a773412fc2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c @@ -57,7 +57,6 @@ g98_fifo = { .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = g84_fifo_engine_id, - .id_engine = g84_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .uevent_init = g84_fifo_uevent_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 43cce02151d78..3d169f3ef5123 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -741,7 +741,6 @@ gf100_fifo = { .intr = gf100_fifo_intr, .mmu_fault = &gf100_fifo_mmu_fault, .engine_id = gf100_fifo_engine_id, - .id_engine = gf100_fifo_id_engine, .uevent_init = gf100_fifo_uevent_init, .uevent_fini = gf100_fifo_uevent_fini, .runl = &gf100_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index b41481c984d10..9d66aafc2dd3d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -303,15 +303,6 @@ gk104_fifo_pbdma = { .init = gk104_fifo_pbdma_init, }; -struct nvkm_engine * -gk104_fifo_id_engine(struct nvkm_fifo *base, int engi) -{ - if (engi == GK104_FIFO_ENGN_SW) - return nvkm_device_engine(base->engine.subdev.device, NVKM_ENGINE_SW, 0); - - return gk104_fifo(base)->engine[engi].engine; -} - int gk104_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { @@ -1185,7 +1176,6 @@ gk104_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 64bdd3fd3ae66..ed3aa170d429e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -93,7 +93,6 @@ gk110_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 3e074a976c440..7219dc92f4512 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -72,7 +72,6 @@ gk208_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index a494dacad37e0..9dfa93d914e00 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -43,7 +43,6 @@ gk20a_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index c4a6972ee473e..4b67fe1ed81a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -133,7 +133,6 @@ gm107_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index bc9893c4d297b..5c81f8b8d6c00 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -63,7 +63,6 @@ gm200_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 4ac4afbf3147d..d90ba4170d989 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -105,7 +105,6 @@ gp100_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index e1069c672438c..c2b32688cbd1e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -332,7 +332,6 @@ gv100_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 52fe9d5a5a4ab..120eb8b7611ff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -182,24 +182,6 @@ const struct nvkm_runl_func nv04_runl = { }; -struct nvkm_engine * -nv04_fifo_id_engine(struct nvkm_fifo *fifo, int engi) -{ - enum nvkm_subdev_type type; - - switch (engi) { - case NV04_FIFO_ENGN_SW : type = NVKM_ENGINE_SW; break; - case NV04_FIFO_ENGN_GR : type = NVKM_ENGINE_GR; break; - case NV04_FIFO_ENGN_MPEG: type = NVKM_ENGINE_MPEG; break; - case NV04_FIFO_ENGN_DMA : type = NVKM_ENGINE_DMAOBJ; break; - default: - WARN_ON(1); - return NULL; - } - - return nvkm_device_engine(fifo->engine.subdev.device, type, 0); -} - int nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { @@ -500,7 +482,6 @@ nv04_fifo = { .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, - .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index 84416440a8221..0f7c88ae5a1c6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -61,7 +61,6 @@ nv10_fifo = { .init = nv04_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, - .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 28ae10a78ee4b..59b752986ac1d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -93,7 +93,6 @@ nv17_fifo = { .init = nv17_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, - .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index 492609780c9bc..d0fc6c528825e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -133,7 +133,6 @@ nv40_fifo = { .init = nv40_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, - .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 4fcf49dbfd907..46013aea97d6c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -168,7 +168,6 @@ nv50_fifo = { .init = nv50_fifo_init, .intr = nv04_fifo_intr, .engine_id = nv04_fifo_engine_id, - .id_engine = nv04_fifo_id_engine, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv50_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h index 9a23a84ea7f49..b8f7201ba7f5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h @@ -19,7 +19,6 @@ void nv50_fifo_init(struct nvkm_fifo *); void nv50_fifo_runlist_update(struct nv50_fifo *); int g84_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); -struct nvkm_engine *g84_fifo_id_engine(struct nvkm_fifo *, int); void g84_fifo_uevent_init(struct nvkm_fifo *); void g84_fifo_uevent_fini(struct nvkm_fifo *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index b0fa9d1df91da..1b0a3027f297c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -44,7 +44,6 @@ struct nvkm_fifo_func { } fault; int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); - struct nvkm_engine *(*id_engine)(struct nvkm_fifo *, int engi); void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); void (*uevent_init)(struct nvkm_fifo *); @@ -98,7 +97,6 @@ int nv04_fifo_runl_ctor(struct nvkm_fifo *); void nv04_fifo_init(struct nvkm_fifo *); void nv04_fifo_intr(struct nvkm_fifo *); int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); -struct nvkm_engine *nv04_fifo_id_engine(struct nvkm_fifo *, int); void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); extern const struct nvkm_runl_func nv04_runl; @@ -127,7 +125,6 @@ extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); -struct nvkm_engine *gk104_fifo_id_engine(struct nvkm_fifo *, int); extern const struct nvkm_runq_func gk104_runq; extern const struct nvkm_engn_func gk104_engn; extern const struct nvkm_engn_func gk104_engn_ce; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index e5b9b1fc38667..f2c90c7dda7d3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -459,7 +459,6 @@ tu102_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .id_engine = gk104_fifo_id_engine, .uevent_init = gk104_fifo_uevent_init, .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = tu102_fifo_recover_chan, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 42b49043deefb..6f920e0e1008d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -22,6 +22,7 @@ #define nvkm_uchan(p) container_of((p), struct nvkm_uchan, object) #include "cgrp.h" #include "chan.h" +#include "runl.h" #include @@ -89,14 +90,43 @@ static int nvkm_uchan_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *oclass) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; + struct nvkm_engn *engn; int ret; - ret = chan->object.func->sclass(&chan->object, index, oclass); - if (ret) - return ret; + nvkm_runl_foreach_engn(engn, chan->cgrp->runl) { + struct nvkm_engine *engine = engn->engine; + int c = 0; + + oclass->engine = engine; + oclass->base.oclass = 0; + + if (engine->func->fifo.sclass) { + ret = engine->func->fifo.sclass(oclass, index); + if (oclass->base.oclass) { + if (!oclass->base.ctor) + oclass->base.ctor = nvkm_object_new; + oclass->ctor = nvkm_uchan_object_new; + return 0; + } + + index -= ret; + continue; + } + + while (engine->func->sclass[c].oclass) { + if (c++ == index) { + oclass->base = engine->func->sclass[index]; + if (!oclass->base.ctor) + oclass->base.ctor = nvkm_object_new; + oclass->ctor = nvkm_uchan_object_new; + return 0; + } + } + + index -= c; + } - oclass->ctor = nvkm_uchan_object_new; - return 0; + return -EINVAL; } static int -- GitLab From 2fc71a0566f63ac3cd43d7cf2d5efbbab6293c5f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:26 +1000 Subject: [PATCH 1312/1927] drm/nouveau/fifo: use explicit intr interfaces More control, and shallower call-chain to get to the point. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 27 +++++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 21 ++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 53 +++++++++---------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 3 -- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 34 ++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 8 +-- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 28 +++++----- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gf100.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gk104.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gp100.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gt215.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c | 2 +- 17 files changed, 105 insertions(+), 89 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 3e62eacc48adb..d93e71870d8d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -193,19 +193,16 @@ nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_dev return c; } -static void -nvkm_fifo_intr(struct nvkm_engine *engine) -{ - struct nvkm_fifo *fifo = nvkm_fifo(engine); - fifo->func->intr(fifo); -} - static int nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + + nvkm_inth_block(&fifo->engine.subdev.inth); + if (fifo->func->fini) fifo->func->fini(fifo); + return 0; } @@ -213,7 +210,10 @@ static int nvkm_fifo_init(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + fifo->func->init(fifo); + + nvkm_inth_allow(&fifo->engine.subdev.inth); return 0; } @@ -290,6 +290,8 @@ nvkm_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) static int nvkm_fifo_oneinit(struct nvkm_engine *engine) { + struct nvkm_subdev *subdev = &engine->subdev; + struct nvkm_device *device = subdev->device; struct nvkm_fifo *fifo = nvkm_fifo(engine); struct nvkm_runl *runl; struct nvkm_engn *engn; @@ -322,6 +324,16 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) } } + /* Register interrupt handler. */ + if (fifo->func->intr) { + ret = nvkm_inth_add(&device->mc->intr, NVKM_INTR_SUBDEV, NVKM_INTR_PRIO_NORMAL, + subdev, fifo->func->intr, &subdev->inth); + if (ret) { + nvkm_error(subdev, "intr %d\n", ret); + return ret; + } + } + if (fifo->func->oneinit) return fifo->func->oneinit(fifo); @@ -366,7 +378,6 @@ nvkm_fifo = { .info = nvkm_fifo_info, .init = nvkm_fifo_init, .fini = nvkm_fifo_fini, - .intr = nvkm_fifo_intr, .base.sclass = nvkm_fifo_class_get, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 3d169f3ef5123..1a7ec75ce8937 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -525,11 +526,11 @@ gf100_fifo_intr_engine(struct gf100_fifo *fifo) } } -static void -gf100_fifo_intr(struct nvkm_fifo *base) +static irqreturn_t +gf100_fifo_intr(struct nvkm_inth *inth) { - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 mask = nvkm_rd32(device, 0x002140); u32 stat = nvkm_rd32(device, 0x002100) & mask; @@ -542,7 +543,7 @@ gf100_fifo_intr(struct nvkm_fifo *base) } if (stat & 0x00000100) { - gf100_fifo_intr_sched(fifo); + gf100_fifo_intr_sched(gf100_fifo(fifo)); nvkm_wr32(device, 0x002100, 0x00000100); stat &= ~0x00000100; } @@ -565,7 +566,7 @@ gf100_fifo_intr(struct nvkm_fifo *base) u32 mask = nvkm_rd32(device, 0x00259c); while (mask) { u32 unit = __ffs(mask); - gf100_fifo_intr_mmu_fault_unit(&fifo->base, unit); + gf100_fifo_intr_mmu_fault_unit(fifo, unit); nvkm_wr32(device, 0x00259c, (1 << unit)); mask &= ~(1 << unit); } @@ -576,7 +577,7 @@ gf100_fifo_intr(struct nvkm_fifo *base) u32 mask = nvkm_rd32(device, 0x0025a0); while (mask) { u32 unit = __ffs(mask); - gf100_fifo_intr_pbdma(fifo, unit); + gf100_fifo_intr_pbdma(gf100_fifo(fifo), unit); nvkm_wr32(device, 0x0025a0, (1 << unit)); mask &= ~(1 << unit); } @@ -584,12 +585,12 @@ gf100_fifo_intr(struct nvkm_fifo *base) } if (stat & 0x40000000) { - gf100_fifo_intr_runlist(fifo); + gf100_fifo_intr_runlist(gf100_fifo(fifo)); stat &= ~0x40000000; } if (stat & 0x80000000) { - gf100_fifo_intr_engine(fifo); + gf100_fifo_intr_engine(gf100_fifo(fifo)); stat &= ~0x80000000; } @@ -598,6 +599,8 @@ gf100_fifo_intr(struct nvkm_fifo *base) nvkm_mask(device, 0x002140, stat, 0x00000000); nvkm_wr32(device, 0x002100, stat); } + + return IRQ_HANDLED; } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 9d66aafc2dd3d..07fa79194870d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -720,7 +721,7 @@ gk104_fifo_mmu_fault = { }; static const struct nvkm_enum -gk104_fifo_bind_reason[] = { +gk104_fifo_intr_bind_reason[] = { { 0x01, "BIND_NOT_UNBOUND" }, { 0x02, "SNOOP_WITHOUT_BAR1" }, { 0x03, "UNBIND_WHILE_RUNNING" }, @@ -731,14 +732,12 @@ gk104_fifo_bind_reason[] = { }; void -gk104_fifo_intr_bind(struct gk104_fifo *fifo) +gk104_fifo_intr_bind(struct nvkm_fifo *fifo) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 intr = nvkm_rd32(device, 0x00252c); + struct nvkm_subdev *subdev = &fifo->engine.subdev; + u32 intr = nvkm_rd32(subdev->device, 0x00252c); u32 code = intr & 0x000000ff; - const struct nvkm_enum *en = - nvkm_enum_find(gk104_fifo_bind_reason, code); + const struct nvkm_enum *en = nvkm_enum_find(gk104_fifo_intr_bind_reason, code); nvkm_error(subdev, "BIND_ERROR %02x [%s]\n", code, en ? en->name : ""); } @@ -802,21 +801,22 @@ gk104_fifo_intr_sched(struct gk104_fifo *fifo) } void -gk104_fifo_intr_chsw(struct gk104_fifo *fifo) +gk104_fifo_intr_chsw(struct nvkm_fifo *fifo) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 stat = nvkm_rd32(device, 0x00256c); + nvkm_error(subdev, "CHSW_ERROR %08x\n", stat); nvkm_wr32(device, 0x00256c, stat); } -void -gk104_fifo_intr_dropped_fault(struct gk104_fifo *fifo) +static void +gk104_fifo_intr_dropped_fault(struct nvkm_fifo *fifo) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0x00259c); + struct nvkm_subdev *subdev = &fifo->engine.subdev; + u32 stat = nvkm_rd32(subdev->device, 0x00259c); + nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat); } @@ -879,11 +879,11 @@ gk104_fifo_intr_engine(struct gk104_fifo *fifo) nvkm_fifo_uevent(&fifo->base); } -void -gk104_fifo_intr(struct nvkm_fifo *base) +irqreturn_t +gk104_fifo_intr(struct nvkm_inth *inth) { - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 mask = nvkm_rd32(device, 0x002140); u32 stat = nvkm_rd32(device, 0x002100) & mask; @@ -901,7 +901,7 @@ gk104_fifo_intr(struct nvkm_fifo *base) } if (stat & 0x00000100) { - gk104_fifo_intr_sched(fifo); + gk104_fifo_intr_sched(gk104_fifo(fifo)); nvkm_wr32(device, 0x002100, 0x00000100); stat &= ~0x00000100; } @@ -934,7 +934,7 @@ gk104_fifo_intr(struct nvkm_fifo *base) u32 mask = nvkm_rd32(device, 0x00259c); while (mask) { u32 unit = __ffs(mask); - fifo->func->intr_mmu_fault_unit(&fifo->base, unit); + fifo->func->intr_mmu_fault_unit(fifo, unit); nvkm_wr32(device, 0x00259c, (1 << unit)); mask &= ~(1 << unit); } @@ -945,8 +945,8 @@ gk104_fifo_intr(struct nvkm_fifo *base) u32 mask = nvkm_rd32(device, 0x0025a0); while (mask) { u32 unit = __ffs(mask); - gk104_fifo_intr_pbdma_0(fifo, unit); - gk104_fifo_intr_pbdma_1(fifo, unit); + gk104_fifo_intr_pbdma_0(gk104_fifo(fifo), unit); + gk104_fifo_intr_pbdma_1(gk104_fifo(fifo), unit); nvkm_wr32(device, 0x0025a0, (1 << unit)); mask &= ~(1 << unit); } @@ -954,13 +954,13 @@ gk104_fifo_intr(struct nvkm_fifo *base) } if (stat & 0x40000000) { - gk104_fifo_intr_runlist(fifo); + gk104_fifo_intr_runlist(gk104_fifo(fifo)); stat &= ~0x40000000; } if (stat & 0x80000000) { nvkm_wr32(device, 0x002100, 0x80000000); - gk104_fifo_intr_engine(fifo); + gk104_fifo_intr_engine(gk104_fifo(fifo)); stat &= ~0x80000000; } @@ -969,16 +969,15 @@ gk104_fifo_intr(struct nvkm_fifo *base) nvkm_mask(device, 0x002140, stat, 0x00000000); nvkm_wr32(device, 0x002100, stat); } + + return IRQ_HANDLED; } void gk104_fifo_fini(struct nvkm_fifo *base) { struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; flush_work(&fifo->recover.work); - /* allow mmu fault interrupts, even when we're not using fifo */ - nvkm_mask(device, 0x002140, 0x10000000, 0x10000000); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 403560aefbb78..23328a8c19b3e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -66,9 +66,6 @@ void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *); void gk104_fifo_runlist_update(struct gk104_fifo *, int runl); void gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, struct gk104_fifo_engine_status *status); -void gk104_fifo_intr_bind(struct gk104_fifo *fifo); -void gk104_fifo_intr_chsw(struct gk104_fifo *fifo); -void gk104_fifo_intr_dropped_fault(struct gk104_fifo *fifo); void gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit); void gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit); void gk104_fifo_intr_runlist(struct gk104_fifo *fifo); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 120eb8b7611ff..4f3fed0fcb500 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -236,9 +237,9 @@ nv04_fifo_swmthd(struct nvkm_device *device, u32 chid, u32 addr, u32 data) } static void -nv04_fifo_cache_error(struct nv04_fifo *fifo, u32 chid, u32 get) +nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; struct nvkm_fifo_chan *chan; unsigned long flags; @@ -263,12 +264,12 @@ nv04_fifo_cache_error(struct nv04_fifo *fifo, u32 chid, u32 get) if (!(pull0 & 0x00000100) || !nv04_fifo_swmthd(device, chid, mthd, data)) { - chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); + chan = nvkm_fifo_chan_chid(fifo, chid, &flags); nvkm_error(subdev, "CACHE_ERROR - " "ch %d [%s] subc %d mthd %04x data %08x\n", chid, chan ? chan->object.client->name : "unknown", (mthd >> 13) & 7, mthd & 0x1ffc, data); - nvkm_fifo_chan_put(&fifo->base, flags, &chan); + nvkm_fifo_chan_put(fifo, flags, &chan); } nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); @@ -287,9 +288,9 @@ nv04_fifo_cache_error(struct nv04_fifo *fifo, u32 chid, u32 get) } static void -nv04_fifo_dma_pusher(struct nv04_fifo *fifo, u32 chid) +nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 dma_get = nvkm_rd32(device, 0x003244); u32 dma_put = nvkm_rd32(device, 0x003240); @@ -299,7 +300,7 @@ nv04_fifo_dma_pusher(struct nv04_fifo *fifo, u32 chid) unsigned long flags; const char *name; - chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); + chan = nvkm_fifo_chan_chid(fifo, chid, &flags); name = chan ? chan->object.client->name : "unknown"; if (device->card_type == NV_50) { u32 ho_get = nvkm_rd32(device, 0x003328); @@ -331,18 +332,18 @@ nv04_fifo_dma_pusher(struct nv04_fifo *fifo, u32 chid) if (dma_get != dma_put) nvkm_wr32(device, 0x003244, dma_put); } - nvkm_fifo_chan_put(&fifo->base, flags, &chan); + nvkm_fifo_chan_put(fifo, flags, &chan); nvkm_wr32(device, 0x003228, 0x00000000); nvkm_wr32(device, 0x003220, 0x00000001); nvkm_wr32(device, 0x002100, NV_PFIFO_INTR_DMA_PUSHER); } -void -nv04_fifo_intr(struct nvkm_fifo *base) +irqreturn_t +nv04_fifo_intr(struct nvkm_inth *inth) { - struct nv04_fifo *fifo = nv04_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 mask = nvkm_rd32(device, NV03_PFIFO_INTR_EN_0); u32 stat = nvkm_rd32(device, NV03_PFIFO_INTR_0) & mask; @@ -351,16 +352,16 @@ nv04_fifo_intr(struct nvkm_fifo *base) reassign = nvkm_rd32(device, NV03_PFIFO_CACHES) & 1; nvkm_wr32(device, NV03_PFIFO_CACHES, 0); - chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; + chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask; get = nvkm_rd32(device, NV03_PFIFO_CACHE1_GET); if (stat & NV_PFIFO_INTR_CACHE_ERROR) { - nv04_fifo_cache_error(fifo, chid, get); + nv04_fifo_intr_cache_error(fifo, chid, get); stat &= ~NV_PFIFO_INTR_CACHE_ERROR; } if (stat & NV_PFIFO_INTR_DMA_PUSHER) { - nv04_fifo_dma_pusher(fifo, chid); + nv04_fifo_intr_dma_pusher(fifo, chid); stat &= ~NV_PFIFO_INTR_DMA_PUSHER; } @@ -383,7 +384,7 @@ nv04_fifo_intr(struct nvkm_fifo *base) if (stat & 0x40000000) { nvkm_wr32(device, 0x002100, 0x40000000); - nvkm_fifo_uevent(&fifo->base); + nvkm_fifo_uevent(fifo); stat &= ~0x40000000; } } @@ -395,6 +396,7 @@ nv04_fifo_intr(struct nvkm_fifo *base) } nvkm_wr32(device, NV03_PFIFO_CACHES, reassign); + return IRQ_HANDLED; } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 1b0a3027f297c..f314f107f2ebf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -28,7 +28,7 @@ struct nvkm_fifo_func { void (*init)(struct nvkm_fifo *); void (*fini)(struct nvkm_fifo *); - void (*intr)(struct nvkm_fifo *); + irqreturn_t (*intr)(struct nvkm_inth *); void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit); const struct nvkm_fifo_func_mmu_fault { @@ -95,7 +95,7 @@ int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvk int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); int nv04_fifo_runl_ctor(struct nvkm_fifo *); void nv04_fifo_init(struct nvkm_fifo *); -void nv04_fifo_intr(struct nvkm_fifo *); +irqreturn_t nv04_fifo_intr(struct nvkm_inth *); int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); @@ -120,7 +120,9 @@ extern const struct nvkm_engn_func gf100_engn_sw; int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_runl_ctor(struct nvkm_fifo *); -void gk104_fifo_intr(struct nvkm_fifo *); +irqreturn_t gk104_fifo_intr(struct nvkm_inth *); +void gk104_fifo_intr_chsw(struct nvkm_fifo *); +void gk104_fifo_intr_bind(struct nvkm_fifo *); extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); void gk104_fifo_recover_chan(struct nvkm_fifo *, int); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index f2c90c7dda7d3..ae399c6a192d0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -367,21 +368,20 @@ tu102_fifo_intr_ctxsw_timeout(struct gk104_fifo *fifo) } static void -tu102_fifo_intr_sched(struct gk104_fifo *fifo) +tu102_fifo_intr_sched(struct nvkm_fifo *fifo) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 intr = nvkm_rd32(device, 0x00254c); + struct nvkm_subdev *subdev = &fifo->engine.subdev; + u32 intr = nvkm_rd32(subdev->device, 0x00254c); u32 code = intr & 0x000000ff; nvkm_error(subdev, "SCHED_ERROR %02x\n", code); } -static void -tu102_fifo_intr(struct nvkm_fifo *base) +static irqreturn_t +tu102_fifo_intr(struct nvkm_inth *inth) { - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), engine.subdev.inth); + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 mask = nvkm_rd32(device, 0x002140); u32 stat = nvkm_rd32(device, 0x002100) & mask; @@ -393,7 +393,7 @@ tu102_fifo_intr(struct nvkm_fifo *base) } if (stat & 0x00000002) { - tu102_fifo_intr_ctxsw_timeout(fifo); + tu102_fifo_intr_ctxsw_timeout(gk104_fifo(fifo)); stat &= ~0x00000002; } @@ -415,8 +415,8 @@ tu102_fifo_intr(struct nvkm_fifo *base) while (mask) { u32 unit = __ffs(mask); - gk104_fifo_intr_pbdma_0(fifo, unit); - gk104_fifo_intr_pbdma_1(fifo, unit); + gk104_fifo_intr_pbdma_0(gk104_fifo(fifo), unit); + gk104_fifo_intr_pbdma_1(gk104_fifo(fifo), unit); nvkm_wr32(device, 0x0025a0, (1 << unit)); mask &= ~(1 << unit); } @@ -424,13 +424,13 @@ tu102_fifo_intr(struct nvkm_fifo *base) } if (stat & 0x40000000) { - gk104_fifo_intr_runlist(fifo); + gk104_fifo_intr_runlist(gk104_fifo(fifo)); stat &= ~0x40000000; } if (stat & 0x80000000) { nvkm_wr32(device, 0x002100, 0x80000000); - gk104_fifo_intr_engine(fifo); + gk104_fifo_intr_engine(gk104_fifo(fifo)); stat &= ~0x80000000; } @@ -439,6 +439,8 @@ tu102_fifo_intr(struct nvkm_fifo *base) nvkm_mask(device, 0x002140, stat, 0x00000000); nvkm_wr32(device, 0x002100, stat); } + + return IRQ_HANDLED; } static const struct nvkm_fifo_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c index 9feb7d167d236..f4ee99137b1f5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c @@ -41,7 +41,7 @@ g84_mc_intrs[] = { { NVKM_ENGINE_BSP , 0, 0, 0x00008000, true }, { NVKM_ENGINE_CIPHER, 0, 0, 0x00004000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_MPEG , 0, 0, 0x00000001, true }, { NVKM_SUBDEV_FB , 0, 0, 0x0002d101, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c index b0b328044e55a..f42684809f08f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c @@ -41,7 +41,7 @@ g98_mc_intrs[] = { { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, { NVKM_ENGINE_SEC , 0, 0, 0x00004000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_MSPPP , 0, 0, 0x00000001, true }, { NVKM_SUBDEV_FB , 0, 0, 0x0002d101, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c index d5dba44235cf3..5118e413d053b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c @@ -42,7 +42,7 @@ gf100_mc_intrs[] = { { NVKM_ENGINE_MSPDEC , 0, 0, 0x00020000, true }, { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_CE , 1, 0, 0x00000040, true }, { NVKM_ENGINE_CE , 0, 0, 0x00000020, true }, { NVKM_ENGINE_MSPPP , 0, 0, 0x00000001, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c index 023dfda735cda..9d482f3581386 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c @@ -33,7 +33,7 @@ gk104_mc_reset[] = { const struct nvkm_intr_data gk104_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_SUBDEV_PRIVRING, 0, 0, 0x40000000, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, { NVKM_SUBDEV_FB , 0, 0, 0x08002000, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c index bffde40bb237d..7606bed2ff6fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c @@ -26,7 +26,7 @@ const struct nvkm_intr_data gp100_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_SUBDEV_FAULT , 0, 0, 0x00000200, true }, { NVKM_SUBDEV_PRIVRING, 0, 0, 0x40000000, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c index 41e17a4703885..3d61836e42a37 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c @@ -41,7 +41,7 @@ gt215_mc_intrs[] = { { NVKM_ENGINE_MSPDEC, 0, 0, 0x00020000, true }, { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_MSPPP , 0, 0, 0x00000001, true }, { NVKM_SUBDEV_FB , 0, 0, 0x00429101, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c index 04809caf36d6a..8482a5550e5f9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c @@ -62,7 +62,7 @@ static const struct nvkm_intr_data nv04_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x01010000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, {} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c index 6a3e2f4753ef0..6d6278f434a48 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c @@ -27,7 +27,7 @@ static const struct nvkm_intr_data nv11_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x03010000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, {} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c index 2edc1cb0e379c..dbad7c111cebf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c @@ -35,7 +35,7 @@ const struct nvkm_intr_data nv17_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x03010000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_MPEG , 0, 0, 0x00000001, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, { NVKM_SUBDEV_TIMER, 0, 0, 0x00100000, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c index a42d20206e37e..d41099d35690f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c @@ -27,7 +27,7 @@ static const struct nvkm_intr_data nv50_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, - { NVKM_ENGINE_FIFO , 0, 0, 0x00000100, true }, + { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_MPEG , 0, 0, 0x00000001, true }, { NVKM_SUBDEV_FB , 0, 0, 0x00001101, true }, { NVKM_SUBDEV_BUS , 0, 0, 0x10000000, true }, -- GitLab From d67f3b96462922713e77cddb9d6f4965606918fc Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:27 +1000 Subject: [PATCH 1313/1927] drm/nouveau/fifo: tidy up non-stall intr handling - removes a layer of indirection in the intr handling - prevents non-stall ctrl racing with unknown intrs Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 7 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 34 ++--------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 3 - .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 33 +++++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/g98.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 59 ++++++++++++------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 27 ++------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 3 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 7 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 7 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 3 + 20 files changed, 93 insertions(+), 118 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 446d058f558a2..d3412c9162902 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -44,13 +44,16 @@ struct nvkm_fifo { struct list_head runqs; struct list_head runls; + struct { +#define NVKM_FIFO_NONSTALL_EVENT BIT(0) + struct nvkm_event event; + } nonstall; + int nr; struct list_head chan; spinlock_t lock; struct mutex mutex; -#define NVKM_FIFO_EVENT_NON_STALL_INTR BIT(0) - struct nvkm_event uevent; /* async user trigger */ #define NVKM_FIFO_EVENT_KILLED BIT(0) struct nvkm_event kevent; /* channel killed */ }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index d93e71870d8d8..d2a9e891939f5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -129,32 +129,6 @@ static const struct nvkm_event_func nvkm_fifo_kevent_func = { }; -static void -nvkm_fifo_uevent_fini(struct nvkm_event *event, int type, int index) -{ - struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), uevent); - fifo->func->uevent_fini(fifo); -} - -static void -nvkm_fifo_uevent_init(struct nvkm_event *event, int type, int index) -{ - struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), uevent); - fifo->func->uevent_init(fifo); -} - -static const struct nvkm_event_func -nvkm_fifo_uevent_func = { - .init = nvkm_fifo_uevent_init, - .fini = nvkm_fifo_uevent_fini, -}; - -void -nvkm_fifo_uevent(struct nvkm_fifo *fifo) -{ - nvkm_event_ntfy(&fifo->uevent, 0, NVKM_FIFO_EVENT_NON_STALL_INTR); -} - static int nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) @@ -365,7 +339,7 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) if (fifo->func->dtor) data = fifo->func->dtor(fifo); nvkm_event_fini(&fifo->kevent); - nvkm_event_fini(&fifo->uevent); + nvkm_event_fini(&fifo->nonstall.event); mutex_destroy(&fifo->mutex); return data; } @@ -402,9 +376,9 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, nr = func->chid_nr(fifo); fifo->nr = nr; - if (func->uevent_init) { - ret = nvkm_event_init(&nvkm_fifo_uevent_func, &fifo->engine.subdev, 1, 1, - &fifo->uevent); + if (func->nonstall) { + ret = nvkm_event_init(func->nonstall, &fifo->engine.subdev, 1, 1, + &fifo->nonstall.event); if (ret) return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 5fabf73730796..ea8148bad45e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -223,9 +223,6 @@ nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct n union nvif_chan_event_args *args = argv; switch (args->v0.type) { - case NVIF_CHAN_EVENT_V0_NON_STALL_INTR: - return nvkm_uevent_add(uevent, &chan->fifo->uevent, 0, - NVKM_FIFO_EVENT_NON_STALL_INTR, NULL); case NVIF_CHAN_EVENT_V0_KILLED: return nvkm_uevent_add(uevent, &chan->fifo->kevent, chan->chid, NVKM_FIFO_EVENT_KILLED, NULL); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 122929b7d74fe..afaa2dea4ef84 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -37,20 +37,34 @@ const struct nvkm_engn_func g84_engn = { }; -void -g84_fifo_uevent_fini(struct nvkm_fifo *fifo) +static void +g84_fifo_nonstall_block(struct nvkm_event *event, int type, int index) { - struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_mask(device, 0x002140, 0x40000000, 0x00000000); + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + nvkm_mask(fifo->engine.subdev.device, 0x002140, 0x40000000, 0x00000000); + spin_unlock_irqrestore(&fifo->lock, flags); } -void -g84_fifo_uevent_init(struct nvkm_fifo *fifo) +static void +g84_fifo_nonstall_allow(struct nvkm_event *event, int type, int index) { - struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_mask(device, 0x002140, 0x40000000, 0x40000000); + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + nvkm_mask(fifo->engine.subdev.device, 0x002140, 0x40000000, 0x40000000); + spin_unlock_irqrestore(&fifo->lock, flags); } +const struct nvkm_event_func +g84_fifo_nonstall = { + .init = g84_fifo_nonstall_allow, + .fini = g84_fifo_nonstall_block, +}; + int g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { @@ -105,8 +119,7 @@ g84_fifo = { .engine_id = g84_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .uevent_init = g84_fifo_uevent_init, - .uevent_fini = g84_fifo_uevent_fini, + .nonstall = &g84_fifo_nonstall, .runl = &nv50_runl, .engn = &g84_engn, .engn_sw = &nv50_engn_sw, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c index 118a773412fc2..a912e1ba4339c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c @@ -59,8 +59,7 @@ g98_fifo = { .engine_id = g84_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, - .uevent_init = g84_fifo_uevent_init, - .uevent_fini = g84_fifo_uevent_fini, + .nonstall = &g84_fifo_nonstall, .runl = &nv50_runl, .engn = &g84_engn, .engn_sw = &nv50_engn_sw, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 1a7ec75ce8937..3d091604fd0a1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -39,20 +39,6 @@ #include -static void -gf100_fifo_uevent_init(struct nvkm_fifo *fifo) -{ - struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_mask(device, 0x002140, 0x80000000, 0x80000000); -} - -static void -gf100_fifo_uevent_fini(struct nvkm_fifo *fifo) -{ - struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_mask(device, 0x002140, 0x80000000, 0x00000000); -} - static const struct nvkm_chan_func gf100_chan = { }; @@ -177,6 +163,34 @@ static const struct nvkm_runl_func gf100_runl = { }; +static void +gf100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index) +{ + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + nvkm_mask(fifo->engine.subdev.device, 0x002140, 0x80000000, 0x80000000); + spin_unlock_irqrestore(&fifo->lock, flags); +} + +void +gf100_fifo_nonstall_block(struct nvkm_event *event, int type, int index) +{ + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + nvkm_mask(fifo->engine.subdev.device, 0x002140, 0x80000000, 0x00000000); + spin_unlock_irqrestore(&fifo->lock, flags); +} + +const struct nvkm_event_func +gf100_fifo_nonstall = { + .init = gf100_fifo_nonstall_allow, + .fini = gf100_fifo_nonstall_block, +}; + static struct nvkm_engine * gf100_fifo_id_engine(struct nvkm_fifo *fifo, int engi) { @@ -490,9 +504,9 @@ gf100_fifo_intr_runlist(struct gf100_fifo *fifo) } static void -gf100_fifo_intr_engine_unit(struct gf100_fifo *fifo, int engn) +gf100_fifo_intr_engine_unit(struct nvkm_fifo *fifo, int engn) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 intr = nvkm_rd32(device, 0x0025a8 + (engn * 0x04)); u32 inte = nvkm_rd32(device, 0x002628); @@ -503,12 +517,11 @@ gf100_fifo_intr_engine_unit(struct gf100_fifo *fifo, int engn) for (unkn = 0; unkn < 8; unkn++) { u32 ints = (intr >> (unkn * 0x04)) & inte; if (ints & 0x1) { - nvkm_fifo_uevent(&fifo->base); + nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); ints &= ~1; } if (ints) { - nvkm_error(subdev, "ENGINE %d %d %01x", - engn, unkn, ints); + nvkm_error(subdev, "ENGINE %d %d %01x", engn, unkn, ints); nvkm_mask(device, 0x002628, ints, 0); } } @@ -519,9 +532,10 @@ gf100_fifo_intr_engine(struct gf100_fifo *fifo) { struct nvkm_device *device = fifo->base.engine.subdev.device; u32 mask = nvkm_rd32(device, 0x0025a4); + while (mask) { u32 unit = __ffs(mask); - gf100_fifo_intr_engine_unit(fifo, unit); + gf100_fifo_intr_engine_unit(&fifo->base, unit); mask &= ~(1 << unit); } } @@ -596,7 +610,9 @@ gf100_fifo_intr(struct nvkm_inth *inth) if (stat) { nvkm_error(subdev, "INTR %08x\n", stat); + spin_lock(&fifo->lock); nvkm_mask(device, 0x002140, stat, 0x00000000); + spin_unlock(&fifo->lock); nvkm_wr32(device, 0x002100, stat); } @@ -744,8 +760,7 @@ gf100_fifo = { .intr = gf100_fifo_intr, .mmu_fault = &gf100_fifo_mmu_fault, .engine_id = gf100_fifo_engine_id, - .uevent_init = gf100_fifo_uevent_init, - .uevent_fini = gf100_fifo_uevent_fini, + .nonstall = &gf100_fifo_nonstall, .runl = &gf100_runl, .runq = &gf100_runq, .engn = &gf100_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 07fa79194870d..9c25a27eeb2e0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -100,20 +100,6 @@ const struct nvkm_engn_func gk104_engn_ce = { }; -void -gk104_fifo_uevent_fini(struct nvkm_fifo *fifo) -{ - struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_mask(device, 0x002140, 0x80000000, 0x00000000); -} - -void -gk104_fifo_uevent_init(struct nvkm_fifo *fifo) -{ - struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_mask(device, 0x002140, 0x80000000, 0x80000000); -} - static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = { { 0x00000001, "HCE_RE_ILLEGAL_OP" }, { 0x00000002, "HCE_RE_ALIGNB" }, @@ -873,12 +859,6 @@ gk104_fifo_intr_runlist(struct gk104_fifo *fifo) } } -void -gk104_fifo_intr_engine(struct gk104_fifo *fifo) -{ - nvkm_fifo_uevent(&fifo->base); -} - irqreturn_t gk104_fifo_intr(struct nvkm_inth *inth) { @@ -960,13 +940,15 @@ gk104_fifo_intr(struct nvkm_inth *inth) if (stat & 0x80000000) { nvkm_wr32(device, 0x002100, 0x80000000); - gk104_fifo_intr_engine(gk104_fifo(fifo)); + nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); stat &= ~0x80000000; } if (stat) { nvkm_error(subdev, "INTR %08x\n", stat); + spin_lock(&fifo->lock); nvkm_mask(device, 0x002140, stat, 0x00000000); + spin_unlock(&fifo->lock); nvkm_wr32(device, 0x002100, stat); } @@ -1175,11 +1157,10 @@ gk104_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, .pbdma = &gk104_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gk104_runl, .runq = &gk104_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 23328a8c19b3e..8d72eaa555688 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -69,13 +69,10 @@ void gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, void gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit); void gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit); void gk104_fifo_intr_runlist(struct gk104_fifo *fifo); -void gk104_fifo_intr_engine(struct gk104_fifo *fifo); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); -void gk104_fifo_uevent_fini(struct nvkm_fifo *fifo); -void gk104_fifo_uevent_init(struct nvkm_fifo *fifo); extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma; void gk104_fifo_pbdma_init(struct gk104_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index ed3aa170d429e..9350fbcf0db97 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -93,11 +93,10 @@ gk110_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk104_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk104_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 7219dc92f4512..d50f32c530a59 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -72,10 +72,9 @@ gk208_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, + .nonstall = &gf100_fifo_nonstall, .pbdma = &gk208_fifo_pbdma, .runl = &gk110_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 9dfa93d914e00..add5e1a8de0ae 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -43,11 +43,10 @@ gk20a_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk208_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 4b67fe1ed81a2..50c6bec03f924 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -133,11 +133,10 @@ gm107_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gk208_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 5c81f8b8d6c00..497b0e9c7c32c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -63,11 +63,10 @@ gm200_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index d90ba4170d989..3fa8c0a1d1914 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -105,11 +105,10 @@ gp100_fifo = { .fault.hubclient = gk104_fifo_fault_hubclient, .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gp100_runl, .runq = &gk208_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index c2b32688cbd1e..a715dbdcd27eb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -332,11 +332,10 @@ gv100_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, .pbdma = &gm200_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &gv100_runl, .runq = &gv100_runq, .engn = &gv100_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 4f3fed0fcb500..b899127cfd728 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -384,7 +384,7 @@ nv04_fifo_intr(struct nvkm_inth *inth) if (stat & 0x40000000) { nvkm_wr32(device, 0x002100, 0x40000000); - nvkm_fifo_uevent(fifo); + nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); stat &= ~0x40000000; } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h index b8f7201ba7f5e..db5fb45a3aa50 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h @@ -19,6 +19,4 @@ void nv50_fifo_init(struct nvkm_fifo *); void nv50_fifo_runlist_update(struct nv50_fifo *); int g84_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); -void g84_fifo_uevent_init(struct nvkm_fifo *); -void g84_fifo_uevent_fini(struct nvkm_fifo *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index f314f107f2ebf..d9cb4967b799f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -8,7 +8,6 @@ struct nvkm_memory; struct gk104_fifo; struct gk104_fifo_chan; -void nvkm_fifo_uevent(struct nvkm_fifo *); void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid); @@ -46,8 +45,6 @@ struct nvkm_fifo_func { int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); - void (*uevent_init)(struct nvkm_fifo *); - void (*uevent_fini)(struct nvkm_fifo *); void (*recover_chan)(struct nvkm_fifo *, int chid); const struct gk104_fifo_runlist_func { @@ -65,6 +62,8 @@ struct nvkm_fifo_func { void (*init_timeout)(struct gk104_fifo *); } *pbdma; + const struct nvkm_event_func *nonstall; + const struct nvkm_runl_func *runl; const struct nvkm_runq_func *runq; const struct nvkm_engn_func *engn; @@ -110,12 +109,14 @@ int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func nv50_runl; extern const struct nvkm_engn_func nv50_engn_sw; +extern const struct nvkm_event_func g84_fifo_nonstall; extern const struct nvkm_engn_func g84_engn; extern const struct nvkm_chan_func g84_chan; int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); int gf100_fifo_runq_nr(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); +extern const struct nvkm_event_func gf100_fifo_nonstall; extern const struct nvkm_engn_func gf100_engn_sw; int gk104_fifo_chid_nr(struct nvkm_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index ae399c6a192d0..6551376cbdd43 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -430,13 +430,15 @@ tu102_fifo_intr(struct nvkm_inth *inth) if (stat & 0x80000000) { nvkm_wr32(device, 0x002100, 0x80000000); - gk104_fifo_intr_engine(gk104_fifo(fifo)); + nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); stat &= ~0x80000000; } if (stat) { nvkm_error(subdev, "INTR %08x\n", stat); + spin_lock(&fifo->lock); nvkm_mask(device, 0x002140, stat, 0x00000000); + spin_unlock(&fifo->lock); nvkm_wr32(device, 0x002100, stat); } @@ -461,11 +463,10 @@ tu102_fifo = { .fault.hubclient = gv100_fifo_fault_hubclient, .fault.gpcclient = gv100_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, - .uevent_init = gk104_fifo_uevent_init, - .uevent_fini = gk104_fifo_uevent_fini, .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, .pbdma = &tu102_fifo_pbdma, + .nonstall = &gf100_fifo_nonstall, .runl = &tu102_runl, .runq = &gv100_runq, .engn = &gv100_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 6f920e0e1008d..c4f404840806a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -39,6 +39,7 @@ static int nvkm_uchan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; + struct nvkm_runl *runl = chan->cgrp->runl; union nvif_chan_event_args *args = argv; if (!uevent) @@ -48,6 +49,8 @@ nvkm_uchan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_ switch (args->v0.type) { case NVIF_CHAN_EVENT_V0_NON_STALL_INTR: + return nvkm_uevent_add(uevent, &runl->fifo->nonstall.event, 0, + NVKM_FIFO_NONSTALL_EVENT, NULL); case NVIF_CHAN_EVENT_V0_KILLED: return chan->object.func->uevent(&chan->object, argv, argc, uevent); default: -- GitLab From 965c41d9116c713fe80191606096aa384761c8c3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 3 Oct 2022 13:19:08 +1000 Subject: [PATCH 1314/1927] drm/nouveau/fifo: tidy global PBDMA init Signed-off-by: Ben Skeggs --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 9 ++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 22 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 20 ++++++++--------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 4 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/runq.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 18 +++++++-------- 14 files changed, 52 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index d2a9e891939f5..a6cec417012f2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -184,6 +184,15 @@ static int nvkm_fifo_init(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + struct nvkm_runq *runq; + u32 mask = 0; + + if (fifo->func->init_pbdmas) { + nvkm_runq_foreach(runq, fifo) + mask |= BIT(runq->id); + + fifo->func->init_pbdmas(fifo, mask); + } fifo->func->init(fifo); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 3d091604fd0a1..d2b616b97f941 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -627,18 +627,16 @@ gf100_fifo_fini(struct nvkm_fifo *base) } static void -gf100_fifo_init(struct nvkm_fifo *base) +gf100_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) { - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; - int i; + struct nvkm_device *device = fifo->engine.subdev.device; /* Enable PBDMAs. */ - nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1); - nvkm_wr32(device, 0x002204, (1 << fifo->pbdma_nr) - 1); + nvkm_wr32(device, 0x000204, mask); + nvkm_wr32(device, 0x002204, mask); /* Assign engines to PBDMAs. */ - if (fifo->pbdma_nr >= 3) { + if ((mask & 7) == 7) { nvkm_wr32(device, 0x002208, ~(1 << 0)); /* PGRAPH */ nvkm_wr32(device, 0x00220c, ~(1 << 1)); /* PVP */ nvkm_wr32(device, 0x002210, ~(1 << 1)); /* PMSPP */ @@ -647,6 +645,15 @@ gf100_fifo_init(struct nvkm_fifo *base) nvkm_wr32(device, 0x00221c, ~(1 << 1)); /* PCE1 */ } +} + +static void +gf100_fifo_init(struct nvkm_fifo *base) +{ + struct gf100_fifo *fifo = gf100_fifo(base); + struct nvkm_device *device = fifo->base.engine.subdev.device; + int i; + /* PBDMA[n] */ for (i = 0; i < fifo->pbdma_nr; i++) { nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); @@ -756,6 +763,7 @@ gf100_fifo = { .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gf100_fifo_runl_ctor, .init = gf100_fifo_init, + .init_pbdmas = gf100_fifo_init_pbdmas, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, .mmu_fault = &gf100_fifo_mmu_fault, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 9c25a27eeb2e0..14c604a03f700 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -278,16 +278,8 @@ static const struct nvkm_runl_func gk104_runl = { }; -void -gk104_fifo_pbdma_init(struct gk104_fifo *fifo) -{ - struct nvkm_device *device = fifo->base.engine.subdev.device; - nvkm_wr32(device, 0x000204, (1 << fifo->pbdma_nr) - 1); -} - const struct gk104_fifo_pbdma_func gk104_fifo_pbdma = { - .init = gk104_fifo_pbdma_init, }; int @@ -962,6 +954,14 @@ gk104_fifo_fini(struct nvkm_fifo *base) flush_work(&fifo->recover.work); } +void +gk104_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + + nvkm_wr32(device, 0x000204, mask); +} + void gk104_fifo_init(struct nvkm_fifo *base) { @@ -969,9 +969,6 @@ gk104_fifo_init(struct nvkm_fifo *base) struct nvkm_device *device = fifo->base.engine.subdev.device; int i; - /* Enable PBDMAs. */ - fifo->func->pbdma->init(fifo); - /* PBDMA[n] */ for (i = 0; i < fifo->pbdma_nr; i++) { nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); @@ -1147,6 +1144,7 @@ gk104_fifo = { .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 8d72eaa555688..953177b90e9b5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -75,7 +75,6 @@ void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma; -void gk104_fifo_pbdma_init(struct gk104_fifo *); extern const struct nvkm_enum gk104_fifo_fault_access[]; extern const struct nvkm_enum gk104_fifo_fault_engine[]; extern const struct nvkm_enum gk104_fifo_fault_reason[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 9350fbcf0db97..85705008e0f24 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -83,6 +83,7 @@ gk110_fifo = { .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index d50f32c530a59..5224723ce4242 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -39,7 +39,6 @@ gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo) const struct gk104_fifo_pbdma_func gk208_fifo_pbdma = { - .init = gk104_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; @@ -62,6 +61,7 @@ gk208_fifo = { .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index add5e1a8de0ae..8ce3bb39f6515 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -33,6 +33,7 @@ gk20a_fifo = { .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 50c6bec03f924..9a608c02c6688 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -123,6 +123,7 @@ gm107_fifo = { .runq_nr = gf100_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 497b0e9c7c32c..1ecfbea3680af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -34,7 +34,6 @@ gm200_fifo_runq_nr(struct nvkm_fifo *fifo) const struct gk104_fifo_pbdma_func gm200_fifo_pbdma = { - .init = gk104_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; @@ -53,6 +52,7 @@ gm200_fifo = { .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 3fa8c0a1d1914..1e9c5ea9cd138 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -95,6 +95,7 @@ gp100_fifo = { .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gp100_fifo_intr_mmu_fault_unit, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index a715dbdcd27eb..cc9ddc65e8811 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -323,6 +323,7 @@ gv100_fifo = { .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = gk104_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .mmu_fault = &gv100_fifo_mmu_fault, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index d9cb4967b799f..8657e5c84c648 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -25,6 +25,8 @@ struct nvkm_fifo_func { int (*runl_ctor)(struct nvkm_fifo *); void (*init)(struct nvkm_fifo *); + void (*init_pbdmas)(struct nvkm_fifo *, u32 mask); + void (*fini)(struct nvkm_fifo *); irqreturn_t (*intr)(struct nvkm_inth *); @@ -58,7 +60,6 @@ struct nvkm_fifo_func { } *runlist; const struct gk104_fifo_pbdma_func { - void (*init)(struct gk104_fifo *); void (*init_timeout)(struct gk104_fifo *); } *pbdma; @@ -121,6 +122,7 @@ extern const struct nvkm_engn_func gf100_engn_sw; int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_runl_ctor(struct nvkm_fifo *); +void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32); irqreturn_t gk104_fifo_intr(struct nvkm_inth *); void gk104_fifo_intr_chsw(struct nvkm_fifo *); void gk104_fifo_intr_bind(struct nvkm_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h index 1832a14488c3a..3089d9a8e70e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h @@ -15,6 +15,7 @@ struct nvkm_runq { struct nvkm_runq *nvkm_runq_new(struct nvkm_fifo *, int pbid); void nvkm_runq_del(struct nvkm_runq *); +#define nvkm_runq_foreach(runq,fifo) list_for_each_entry((runq), &(fifo)->runqs, head) #define nvkm_runq_foreach_cond(runq,fifo,cond) nvkm_list_foreach(runq, &(fifo)->runqs, head, (cond)) #define RUNQ_PRINT(r,l,p,f,a...) \ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 6551376cbdd43..aa933231ec606 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -94,18 +94,8 @@ tu102_fifo_fault_engine[] = { {} }; -static void -tu102_fifo_pbdma_init(struct gk104_fifo *fifo) -{ - struct nvkm_device *device = fifo->base.engine.subdev.device; - const u32 mask = (1 << fifo->pbdma_nr) - 1; - /*XXX: this is a bit of a guess at this point in time. */ - nvkm_mask(device, 0xb65000, 0x80000fff, 0x80000000 | mask); -} - static const struct gk104_fifo_pbdma_func tu102_fifo_pbdma = { - .init = tu102_fifo_pbdma_init, .init_timeout = gk208_fifo_pbdma_init_timeout, }; @@ -445,6 +435,13 @@ tu102_fifo_intr(struct nvkm_inth *inth) return IRQ_HANDLED; } +static void +tu102_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) +{ + /* Not directly related to PBDMAs, but, enables doorbell to function. */ + nvkm_mask(fifo->engine.subdev.device, 0xb65000, 0x80000000, 0x80000000); +} + static const struct nvkm_fifo_func tu102_fifo = { .dtor = gk104_fifo_dtor, @@ -454,6 +451,7 @@ tu102_fifo = { .runq_nr = gm200_fifo_runq_nr, .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, + .init_pbdmas = tu102_fifo_init_pbdmas, .fini = gk104_fifo_fini, .intr = tu102_fifo_intr, .mmu_fault = &tu102_fifo_mmu_fault, -- GitLab From 324176e7c8b9f53b1dbf93981a00667121f61902 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Mon, 3 Oct 2022 13:20:07 +1000 Subject: [PATCH 1315/1927] drm/nouveau/fifo: program NV_PFIFO_FB_TIMEOUT on init NVGPU and RM both program this value. Fixes a bunch of random hangs running parallel piglit. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index d2b616b97f941..3c2f6ce622865 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -645,6 +645,7 @@ gf100_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) nvkm_wr32(device, 0x00221c, ~(1 << 1)); /* PCE1 */ } + nvkm_mask(device, 0x002a04, 0xbfffffff, 0xbfffffff); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 14c604a03f700..07fd5b9381849 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -960,6 +960,7 @@ gk104_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) struct nvkm_device *device = fifo->engine.subdev.device; nvkm_wr32(device, 0x000204, mask); + nvkm_mask(device, 0x002a04, 0xbfffffff, 0xbfffffff); } void -- GitLab From 87c860243149bd7f69ba76a2becfa480efa44974 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:28 +1000 Subject: [PATCH 1316/1927] drm/nouveau/fifo: move PBDMA init to runq - bumps pbdma timeout to value RM uses on newer HW - bumps fb timeout to max from boot default - one/both of these greatly improves stability on // piglit runs Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 3 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 21 +++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 2 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 36 +++++++------------ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 6 ---- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 15 +++----- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 6 ---- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 8 ++--- .../gpu/drm/nouveau/nvkm/engine/fifo/runq.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 6 ---- 15 files changed, 36 insertions(+), 74 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index a6cec417012f2..e51fb10a04f72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -192,6 +192,9 @@ nvkm_fifo_init(struct nvkm_engine *engine) mask |= BIT(runq->id); fifo->func->init_pbdmas(fifo, mask); + + nvkm_runq_foreach(runq, fifo) + runq->func->init(runq); } fifo->func->init(fifo); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 3c2f6ce622865..7deb88ec8a9f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -97,8 +97,19 @@ gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit) nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); } +void +gf100_runq_init(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + nvkm_mask(device, 0x04013c + (runq->id * 0x2000), 0x10000100, 0x00000000); + nvkm_wr32(device, 0x040108 + (runq->id * 0x2000), 0xffffffff); /* INTR */ + nvkm_wr32(device, 0x04010c + (runq->id * 0x2000), 0xfffffeff); /* INTREN */ +} + static const struct nvkm_runq_func gf100_runq = { + .init = gf100_runq_init, }; void @@ -653,14 +664,6 @@ gf100_fifo_init(struct nvkm_fifo *base) { struct gf100_fifo *fifo = gf100_fifo(base); struct nvkm_device *device = fifo->base.engine.subdev.device; - int i; - - /* PBDMA[n] */ - for (i = 0; i < fifo->pbdma_nr; i++) { - nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); - nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ - nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ - } nvkm_mask(device, 0x002200, 0x00000001, 0x00000001); nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); @@ -716,8 +719,6 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int ret; - fifo->pbdma_nr = fifo->base.func->runq_nr(&fifo->base); - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, false, &fifo->runlist.mem[0]); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index b8642490eb2f3..82b8467545a8e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -17,8 +17,6 @@ struct gf100_fifo { u64 mask; } recover; - int pbdma_nr; - struct { struct nvkm_memory *mem[2]; int active; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 07fd5b9381849..8b6bc79843c7e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -164,6 +164,17 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { {} }; +void +gk104_runq_init(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + gf100_runq_init(runq); + + nvkm_wr32(device, 0x040148 + (runq->id * 0x2000), 0xffffffff); /* HCE.INTR */ + nvkm_wr32(device, 0x04014c + (runq->id * 0x2000), 0xffffffff); /* HCE.INTREN */ +} + static u32 gk104_runq_runm(struct nvkm_runq *runq) { @@ -172,6 +183,7 @@ gk104_runq_runm(struct nvkm_runq *runq) const struct nvkm_runq_func gk104_runq = { + .init = gk104_runq_init, }; void @@ -278,10 +290,6 @@ static const struct nvkm_runl_func gk104_runl = { }; -const struct gk104_fifo_pbdma_func -gk104_fifo_pbdma = { -}; - int gk104_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { @@ -968,26 +976,9 @@ gk104_fifo_init(struct nvkm_fifo *base) { struct gk104_fifo *fifo = gk104_fifo(base); struct nvkm_device *device = fifo->base.engine.subdev.device; - int i; - - /* PBDMA[n] */ - for (i = 0; i < fifo->pbdma_nr; i++) { - nvkm_mask(device, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000); - nvkm_wr32(device, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */ - nvkm_wr32(device, 0x04010c + (i * 0x2000), 0xfffffeff); /* INTREN */ - } - - /* PBDMA[n].HCE */ - for (i = 0; i < fifo->pbdma_nr; i++) { - nvkm_wr32(device, 0x040148 + (i * 0x2000), 0xffffffff); /* INTR */ - nvkm_wr32(device, 0x04014c + (i * 0x2000), 0xffffffff); /* INTREN */ - } nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); - if (fifo->func->pbdma->init_timeout) - fifo->func->pbdma->init_timeout(fifo); - nvkm_wr32(device, 0x002100, 0xffffffff); nvkm_wr32(device, 0x002140, 0x7fffffff); } @@ -1054,8 +1045,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) struct nvkm_top_device *tdev; int ret, i, j; - fifo->pbdma_nr = fifo->func->runq_nr(&fifo->base); - /* Determine runlist configuration from topology device info. */ list_for_each_entry(tdev, &device->top->device, head) { const int engn = tdev->engine; @@ -1158,7 +1147,6 @@ gk104_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, - .pbdma = &gk104_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gk104_runl, .runq = &gk104_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 953177b90e9b5..99ae200b2f898 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -21,8 +21,6 @@ struct gk104_fifo { u32 runm; } recover; - int pbdma_nr; - struct { struct nvkm_engine *engine; int runl; @@ -74,7 +72,6 @@ int gk104_fifo_oneinit(struct nvkm_fifo *); void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); -extern const struct gk104_fifo_pbdma_func gk104_fifo_pbdma; extern const struct nvkm_enum gk104_fifo_fault_access[]; extern const struct nvkm_enum gk104_fifo_fault_engine[]; extern const struct nvkm_enum gk104_fifo_fault_reason[]; @@ -90,14 +87,11 @@ extern const struct gk104_fifo_runlist_func gk110_fifo_runlist; void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, struct nvkm_memory *, u32); -extern const struct gk104_fifo_pbdma_func gk208_fifo_pbdma; void gk208_fifo_pbdma_init_timeout(struct gk104_fifo *); extern const struct nvkm_enum gm107_fifo_fault_engine[]; extern const struct gk104_fifo_runlist_func gm107_fifo_runlist; -extern const struct gk104_fifo_pbdma_func gm200_fifo_pbdma; - extern const struct nvkm_enum gp100_fifo_fault_engine[]; extern const struct nvkm_enum gv100_fifo_fault_access[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 85705008e0f24..4143a07e3808a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -96,7 +96,6 @@ gk110_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, - .pbdma = &gk104_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk104_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 5224723ce4242..0b48fb1b3da4d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -28,22 +28,16 @@ #include void -gk208_fifo_pbdma_init_timeout(struct gk104_fifo *fifo) +gk208_runq_init(struct nvkm_runq *runq) { - struct nvkm_device *device = fifo->base.engine.subdev.device; - int i; + gk104_runq_init(runq); - for (i = 0; i < fifo->pbdma_nr; i++) - nvkm_wr32(device, 0x04012c + (i * 0x2000), 0x0000ffff); + nvkm_wr32(runq->fifo->engine.subdev.device, 0x04012c + (runq->id * 0x2000), 0x000f4240); } -const struct gk104_fifo_pbdma_func -gk208_fifo_pbdma = { - .init_timeout = gk208_fifo_pbdma_init_timeout, -}; - const struct nvkm_runq_func gk208_runq = { + .init = gk208_runq_init, }; static int @@ -75,7 +69,6 @@ gk208_fifo = { .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, - .pbdma = &gk208_fifo_pbdma, .runl = &gk110_runl, .runq = &gk208_runq, .engn = &gk104_engn, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 8ce3bb39f6515..3d3db802b824c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -46,7 +46,6 @@ gk20a_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, - .pbdma = &gk208_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 9a608c02c6688..a85f30768b746 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -136,7 +136,6 @@ gm107_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, - .pbdma = &gk208_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 1ecfbea3680af..131e4ae615908 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -32,11 +32,6 @@ gm200_fifo_runq_nr(struct nvkm_fifo *fifo) return nvkm_rd32(fifo->engine.subdev.device, 0x002004) & 0x000000ff; } -const struct gk104_fifo_pbdma_func -gm200_fifo_pbdma = { - .init_timeout = gk208_fifo_pbdma_init_timeout, -}; - int gm200_fifo_chid_nr(struct nvkm_fifo *fifo) { @@ -65,7 +60,6 @@ gm200_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, - .pbdma = &gm200_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 1e9c5ea9cd138..b05ede6dd3929 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -108,7 +108,6 @@ gp100_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, - .pbdma = &gm200_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gp100_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index cc9ddc65e8811..251d5e5826db8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -45,6 +45,7 @@ gv100_engn_ce = { const struct nvkm_runq_func gv100_runq = { + .init = gk208_runq_init, }; void @@ -335,7 +336,6 @@ gv100_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, - .pbdma = &gm200_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &gv100_runl, .runq = &gv100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 8657e5c84c648..c30d7ac078032 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -5,6 +5,7 @@ #include struct nvkm_cgrp; struct nvkm_memory; +struct nvkm_runq; struct gk104_fifo; struct gk104_fifo_chan; @@ -59,10 +60,6 @@ struct nvkm_fifo_func { struct nvkm_memory *, int entries); } *runlist; - const struct gk104_fifo_pbdma_func { - void (*init_timeout)(struct gk104_fifo *); - } *pbdma; - const struct nvkm_event_func *nonstall; const struct nvkm_runl_func *runl; @@ -118,6 +115,7 @@ int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); int gf100_fifo_runq_nr(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); extern const struct nvkm_event_func gf100_fifo_nonstall; +void gf100_runq_init(struct nvkm_runq *); extern const struct nvkm_engn_func gf100_engn_sw; int gk104_fifo_chid_nr(struct nvkm_fifo *); @@ -131,6 +129,7 @@ void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); extern const struct nvkm_runq_func gk104_runq; +void gk104_runq_init(struct nvkm_runq *); extern const struct nvkm_engn_func gk104_engn; extern const struct nvkm_engn_func gk104_engn_ce; @@ -140,6 +139,7 @@ extern const struct nvkm_cgrp_func gk110_cgrp; extern const struct nvkm_chan_func gk110_chan; extern const struct nvkm_runq_func gk208_runq; +void gk208_runq_init(struct nvkm_runq *); void gm107_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); extern const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h index 3089d9a8e70e8..e4909161c9bc6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h @@ -5,6 +5,7 @@ struct nvkm_runq { const struct nvkm_runq_func { + void (*init)(struct nvkm_runq *); } *func; struct nvkm_fifo *fifo; int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index aa933231ec606..5fcc4ebb9f5b4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -94,11 +94,6 @@ tu102_fifo_fault_engine[] = { {} }; -static const struct gk104_fifo_pbdma_func -tu102_fifo_pbdma = { - .init_timeout = gk208_fifo_pbdma_init_timeout, -}; - static void tu102_fifo_recover_work(struct work_struct *w) { @@ -463,7 +458,6 @@ tu102_fifo = { .engine_id = gk104_fifo_engine_id, .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, - .pbdma = &tu102_fifo_pbdma, .nonstall = &gf100_fifo_nonstall, .runl = &tu102_runl, .runq = &gv100_runq, -- GitLab From 923f1ff5274ce3072df55e5e3bbaa7db457fc35d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:29 +1000 Subject: [PATCH 1317/1927] drm/nouveau/fifo: move PBDMA intr to runq - merges gf100/gk104- NV_PFIFO_INTR_0_PBDMA and NV_PPBDMA_INTR_0 code Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 65 ++++++++----- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 92 ++++++------------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 3 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 6 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/runq.h | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 13 +-- 8 files changed, 84 insertions(+), 102 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 7deb88ec8a9f4..177ae02e78eea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -30,7 +30,6 @@ #include "changf100.h" #include -#include #include #include #include @@ -52,27 +51,28 @@ gf100_engn_sw = { }; static const struct nvkm_bitfield -gf100_fifo_pbdma_intr[] = { +gf100_runq_intr_0_names[] = { /* { 0x00008000, "" } seen with null ib push */ { 0x00200000, "ILLEGAL_MTHD" }, { 0x00800000, "EMPTY_SUBC" }, {} }; -static void -gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit) +bool +gf100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &runq->fifo->engine.subdev; struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000)); - u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000)); - u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000)); - u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0x7f; + u32 mask = nvkm_rd32(device, 0x04010c + (runq->id * 0x2000)); + u32 stat = nvkm_rd32(device, 0x040108 + (runq->id * 0x2000)) & mask; + u32 addr = nvkm_rd32(device, 0x0400c0 + (runq->id * 0x2000)); + u32 data = nvkm_rd32(device, 0x0400c4 + (runq->id * 0x2000)); + u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x2000)) & runq->fifo->chid->mask; u32 subc = (addr & 0x00070000) >> 16; u32 mthd = (addr & 0x00003ffc); - struct nvkm_fifo_chan *chan; - unsigned long flags; u32 show = stat; + struct nvkm_chan *chan; + unsigned long flags; char msg[128]; if (stat & 0x00800000) { @@ -83,18 +83,19 @@ gf100_fifo_intr_pbdma(struct gf100_fifo *fifo, int unit) } if (show) { - nvkm_snprintbf(msg, sizeof(msg), gf100_fifo_pbdma_intr, show); - chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); + nvkm_snprintbf(msg, sizeof(msg), runq->func->intr_0_names, show); + chan = nvkm_fifo_chan_chid(runq->fifo, chid, &flags); nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " "subc %d mthd %04x data %08x\n", - unit, show, msg, chid, chan ? chan->inst->addr : 0, + runq->id, show, msg, chid, chan ? chan->inst->addr : 0, chan ? chan->object.client->name : "unknown", subc, mthd, data); - nvkm_fifo_chan_put(&fifo->base, flags, &chan); + nvkm_fifo_chan_put(runq->fifo, flags, &chan); } - nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008); - nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); + nvkm_wr32(device, 0x0400c0 + (runq->id * 0x2000), 0x80600008); + nvkm_wr32(device, 0x040108 + (runq->id * 0x2000), stat); + return true; } void @@ -110,6 +111,8 @@ gf100_runq_init(struct nvkm_runq *runq) static const struct nvkm_runq_func gf100_runq = { .init = gf100_runq_init, + .intr = gf100_runq_intr, + .intr_0_names = gf100_runq_intr_0_names, }; void @@ -495,6 +498,24 @@ gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) nvkm_fifo_fault(fifo, &info); } +bool +gf100_fifo_intr_pbdma(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_runq *runq; + u32 mask = nvkm_rd32(device, 0x0025a0); + bool handled = false; + + nvkm_runq_foreach_cond(runq, fifo, mask & BIT(runq->id)) { + if (runq->func->intr(runq, NULL)) + handled = true; + + nvkm_wr32(device, 0x0025a0, BIT(runq->id)); + } + + return handled; +} + static void gf100_fifo_intr_runlist(struct gf100_fifo *fifo) { @@ -599,14 +620,8 @@ gf100_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x20000000) { - u32 mask = nvkm_rd32(device, 0x0025a0); - while (mask) { - u32 unit = __ffs(mask); - gf100_fifo_intr_pbdma(gf100_fifo(fifo), unit); - nvkm_wr32(device, 0x0025a0, (1 << unit)); - mask &= ~(1 << unit); - } - stat &= ~0x20000000; + if (gf100_fifo_intr_pbdma(fifo)) + stat &= ~0x20000000; } if (stat & 0x40000000) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 8b6bc79843c7e..6295389bfbea2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -37,7 +37,6 @@ #include #include #include -#include #include @@ -100,7 +99,8 @@ const struct nvkm_engn_func gk104_engn_ce = { }; -static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = { +static const struct nvkm_bitfield +gk104_runq_intr_1_names[] = { { 0x00000001, "HCE_RE_ILLEGAL_OP" }, { 0x00000002, "HCE_RE_ALIGNB" }, { 0x00000004, "HCE_PRIV" }, @@ -109,28 +109,30 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_1[] = { {} }; -void -gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit) +static bool +gk104_runq_intr_1(struct nvkm_runq *runq) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &runq->fifo->engine.subdev; struct nvkm_device *device = subdev->device; - u32 mask = nvkm_rd32(device, 0x04014c + (unit * 0x2000)); - u32 stat = nvkm_rd32(device, 0x040148 + (unit * 0x2000)) & mask; - u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff; + u32 mask = nvkm_rd32(device, 0x04014c + (runq->id * 0x2000)); + u32 stat = nvkm_rd32(device, 0x040148 + (runq->id * 0x2000)) & mask; + u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x2000)) & 0xfff; char msg[128]; if (stat) { - nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_1, stat); + nvkm_snprintbf(msg, sizeof(msg), gk104_runq_intr_1_names, stat); nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n", - unit, stat, msg, chid, - nvkm_rd32(device, 0x040150 + (unit * 0x2000)), - nvkm_rd32(device, 0x040154 + (unit * 0x2000))); + runq->id, stat, msg, chid, + nvkm_rd32(device, 0x040150 + (runq->id * 0x2000)), + nvkm_rd32(device, 0x040154 + (runq->id * 0x2000))); } - nvkm_wr32(device, 0x040148 + (unit * 0x2000), stat); + nvkm_wr32(device, 0x040148 + (runq->id * 0x2000), stat); + return true; } -static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { +const struct nvkm_bitfield +gk104_runq_intr_0_names[] = { { 0x00000001, "MEMREQ" }, { 0x00000002, "MEMACK_TIMEOUT" }, { 0x00000004, "MEMACK_EXTRA" }, @@ -164,6 +166,15 @@ static const struct nvkm_bitfield gk104_fifo_pbdma_intr_0[] = { {} }; +bool +gk104_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null) +{ + bool intr0 = gf100_runq_intr(runq, NULL); + bool intr1 = gk104_runq_intr_1(runq); + + return intr0 || intr1; +} + void gk104_runq_init(struct nvkm_runq *runq) { @@ -184,6 +195,8 @@ gk104_runq_runm(struct nvkm_runq *runq) const struct nvkm_runq_func gk104_runq = { .init = gk104_runq_init, + .intr = gk104_runq_intr, + .intr_0_names = gk104_runq_intr_0_names, }; void @@ -806,46 +819,6 @@ gk104_fifo_intr_dropped_fault(struct nvkm_fifo *fifo) nvkm_error(subdev, "DROPPED_MMU_FAULT %08x\n", stat); } -void -gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 mask = nvkm_rd32(device, 0x04010c + (unit * 0x2000)); - u32 stat = nvkm_rd32(device, 0x040108 + (unit * 0x2000)) & mask; - u32 addr = nvkm_rd32(device, 0x0400c0 + (unit * 0x2000)); - u32 data = nvkm_rd32(device, 0x0400c4 + (unit * 0x2000)); - u32 chid = nvkm_rd32(device, 0x040120 + (unit * 0x2000)) & 0xfff; - u32 subc = (addr & 0x00070000) >> 16; - u32 mthd = (addr & 0x00003ffc); - u32 show = stat; - struct nvkm_fifo_chan *chan; - unsigned long flags; - char msg[128]; - - if (stat & 0x00800000) { - if (device->sw) { - if (nvkm_sw_mthd(device->sw, chid, subc, mthd, data)) - show &= ~0x00800000; - } - } - - nvkm_wr32(device, 0x0400c0 + (unit * 0x2000), 0x80600008); - - if (show) { - nvkm_snprintbf(msg, sizeof(msg), gk104_fifo_pbdma_intr_0, show); - chan = nvkm_fifo_chan_chid(&fifo->base, chid, &flags); - nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " - "subc %d mthd %04x data %08x\n", - unit, show, msg, chid, chan ? chan->inst->addr : 0, - chan ? chan->object.client->name : "unknown", - subc, mthd, data); - nvkm_fifo_chan_put(&fifo->base, flags, &chan); - } - - nvkm_wr32(device, 0x040108 + (unit * 0x2000), stat); -} - void gk104_fifo_intr_runlist(struct gk104_fifo *fifo) { @@ -922,15 +895,8 @@ gk104_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x20000000) { - u32 mask = nvkm_rd32(device, 0x0025a0); - while (mask) { - u32 unit = __ffs(mask); - gk104_fifo_intr_pbdma_0(gk104_fifo(fifo), unit); - gk104_fifo_intr_pbdma_1(gk104_fifo(fifo), unit); - nvkm_wr32(device, 0x0025a0, (1 << unit)); - mask &= ~(1 << unit); - } - stat &= ~0x20000000; + if (gf100_fifo_intr_pbdma(fifo)) + stat &= ~0x20000000; } if (stat & 0x40000000) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 99ae200b2f898..f7e6c26a2a451 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -5,7 +5,6 @@ #include "priv.h" struct nvkm_fifo_cgrp; -#include #include #define gk104_fifo_func nvkm_fifo_func @@ -64,8 +63,6 @@ void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *); void gk104_fifo_runlist_update(struct gk104_fifo *, int runl); void gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, struct gk104_fifo_engine_status *status); -void gk104_fifo_intr_pbdma_0(struct gk104_fifo *fifo, int unit); -void gk104_fifo_intr_pbdma_1(struct gk104_fifo *fifo, int unit); void gk104_fifo_intr_runlist(struct gk104_fifo *fifo); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 0b48fb1b3da4d..87601c35581c9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -38,6 +38,8 @@ gk208_runq_init(struct nvkm_runq *runq) const struct nvkm_runq_func gk208_runq = { .init = gk208_runq_init, + .intr = gk104_runq_intr, + .intr_0_names = gk104_runq_intr_0_names, }; static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 251d5e5826db8..dc1586e2ab720 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -46,6 +46,8 @@ gv100_engn_ce = { const struct nvkm_runq_func gv100_runq = { .init = gk208_runq_init, + .intr = gk104_runq_intr, + .intr_0_names = gk104_runq_intr_0_names, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index c30d7ac078032..00f23c39978eb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -3,8 +3,10 @@ #define __NVKM_FIFO_PRIV_H__ #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) #include +#include struct nvkm_cgrp; struct nvkm_memory; +struct nvkm_runl; struct nvkm_runq; struct gk104_fifo; struct gk104_fifo_chan; @@ -113,9 +115,11 @@ extern const struct nvkm_chan_func g84_chan; int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); int gf100_fifo_runq_nr(struct nvkm_fifo *); +bool gf100_fifo_intr_pbdma(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); extern const struct nvkm_event_func gf100_fifo_nonstall; void gf100_runq_init(struct nvkm_runq *); +bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); extern const struct nvkm_engn_func gf100_engn_sw; int gk104_fifo_chid_nr(struct nvkm_fifo *); @@ -130,6 +134,8 @@ void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); extern const struct nvkm_runq_func gk104_runq; void gk104_runq_init(struct nvkm_runq *); +bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); +extern const struct nvkm_bitfield gk104_runq_intr_0_names[]; extern const struct nvkm_engn_func gk104_engn; extern const struct nvkm_engn_func gk104_engn_ce; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h index e4909161c9bc6..f939d3500cb6a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h @@ -2,10 +2,13 @@ #ifndef __NVKM_RUNQ_H__ #define __NVKM_RUNQ_H__ #include +struct nvkm_runl; struct nvkm_runq { const struct nvkm_runq_func { void (*init)(struct nvkm_runq *); + bool (*intr)(struct nvkm_runq *, struct nvkm_runl *); + const struct nvkm_bitfield *intr_0_names; } *func; struct nvkm_fifo *fifo; int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 5fcc4ebb9f5b4..2d14795b98420 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -395,17 +395,8 @@ tu102_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x20000000) { - u32 mask = nvkm_rd32(device, 0x0025a0); - - while (mask) { - u32 unit = __ffs(mask); - - gk104_fifo_intr_pbdma_0(gk104_fifo(fifo), unit); - gk104_fifo_intr_pbdma_1(gk104_fifo(fifo), unit); - nvkm_wr32(device, 0x0025a0, (1 << unit)); - mask &= ~(1 << unit); - } - stat &= ~0x20000000; + if (gf100_fifo_intr_pbdma(fifo)) + stat &= ~0x20000000; } if (stat & 0x40000000) { -- GitLab From e43c872c03a9ad56f5cbc52149b4454601aa6904 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:29 +1000 Subject: [PATCH 1318/1927] drm/nouveau/fifo: merge mmu fault handlers together After updating GF100 implementation from the GK104/TU102 ones, and using the new runlist/engine topology info, all three handlers become (almost) identical. - there's a temporary kludge to call through to the HW-specific recovery - engine fault mapping info determined at load time, not on every fault Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 110 +++++++++----- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 137 +++--------------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 14 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 32 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 40 ++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 22 +-- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 18 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 23 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 100 +------------ 14 files changed, 183 insertions(+), 335 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 177ae02e78eea..7fc4c37b407f0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -295,7 +295,7 @@ gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine, } static const struct nvkm_enum -gf100_fifo_fault_engine[] = { +gf100_fifo_mmu_fault_engine[] = { { 0x00, "PGRAPH", NULL, NVKM_ENGINE_GR }, { 0x03, "PEEPHOLE", NULL, NVKM_ENGINE_IFB }, { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, @@ -312,7 +312,7 @@ gf100_fifo_fault_engine[] = { }; static const struct nvkm_enum -gf100_fifo_fault_reason[] = { +gf100_fifo_mmu_fault_reason[] = { { 0x00, "PT_NOT_PRESENT" }, { 0x01, "PT_TOO_SHORT" }, { 0x02, "PAGE_NOT_PRESENT" }, @@ -326,7 +326,7 @@ gf100_fifo_fault_reason[] = { }; static const struct nvkm_enum -gf100_fifo_fault_hubclient[] = { +gf100_fifo_mmu_fault_hubclient[] = { { 0x01, "PCOPY0" }, { 0x02, "PCOPY1" }, { 0x04, "DISPATCH" }, @@ -345,7 +345,7 @@ gf100_fifo_fault_hubclient[] = { }; static const struct nvkm_enum -gf100_fifo_fault_gpcclient[] = { +gf100_fifo_mmu_fault_gpcclient[] = { { 0x01, "TEX" }, { 0x0c, "ESETUP" }, { 0x0e, "CTXCTL" }, @@ -353,29 +353,48 @@ gf100_fifo_fault_gpcclient[] = { {} }; -static void -gf100_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) +const struct nvkm_enum +gf100_fifo_mmu_fault_access[] = { + { 0x00, "READ" }, + { 0x01, "WRITE" }, + {} +}; + +void +gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) { - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; - const struct nvkm_enum *er, *eu, *ec; + const struct nvkm_enum *er, *ee, *ec, *ea; struct nvkm_engine *engine = NULL; struct nvkm_fifo_chan *chan; + struct nvkm_runl *runl; + struct nvkm_engn *engn; unsigned long flags; - char gpcid[8] = ""; + char ct[8] = "HUB/"; - er = nvkm_enum_find(gf100_fifo_fault_reason, info->reason); - eu = nvkm_enum_find(gf100_fifo_fault_engine, info->engine); + /* Lookup engine by MMU fault ID. */ + nvkm_runl_foreach(runl, fifo) { + engn = nvkm_runl_find_engn(engn, runl, engn->fault == info->engine); + if (engn) { + engine = engn->engine; + break; + } + } + + er = nvkm_enum_find(fifo->func->mmu_fault->reason, info->reason); + ee = nvkm_enum_find(fifo->func->mmu_fault->engine, info->engine); if (info->hub) { - ec = nvkm_enum_find(gf100_fifo_fault_hubclient, info->client); + ec = nvkm_enum_find(fifo->func->mmu_fault->hubclient, info->client); } else { - ec = nvkm_enum_find(gf100_fifo_fault_gpcclient, info->client); - snprintf(gpcid, sizeof(gpcid), "GPC%d/", info->gpc); + ec = nvkm_enum_find(fifo->func->mmu_fault->gpcclient, info->client); + snprintf(ct, sizeof(ct), "GPC%d/", info->gpc); } + ea = nvkm_enum_find(fifo->func->mmu_fault->access, info->access); - if (eu && eu->data2) { - switch (eu->data2) { + /* Handle BAR faults. */ + if (ee && ee->data2) { + switch (ee->data2) { case NVKM_SUBDEV_BAR: nvkm_bar_bar1_reset(device); break; @@ -386,30 +405,39 @@ gf100_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); break; default: - engine = nvkm_device_engine(device, eu->data2, eu->inst); break; } } - chan = nvkm_fifo_chan_inst(&fifo->base, info->inst, &flags); + chan = nvkm_fifo_chan_inst(fifo, info->inst, &flags); nvkm_error(subdev, - "%s fault at %010llx engine %02x [%s] client %02x [%s%s] " - "reason %02x [%s] on channel %d [%010llx %s]\n", - info->access ? "write" : "read", info->addr, - info->engine, eu ? eu->name : "", - info->client, gpcid, ec ? ec->name : "", - info->reason, er ? er->name : "", chan ? chan->chid : -1, - info->inst, chan ? chan->object.client->name : "unknown"); - + "fault %02x [%s] at %016llx engine %02x [%s] client %02x " + "[%s%s] reason %02x [%s] on channel %d [%010llx %s]\n", + info->access, ea ? ea->name : "", info->addr, + info->engine, ee ? ee->name : engine ? engine->subdev.name : "", + info->client, ct, ec ? ec->name : "", + info->reason, er ? er->name : "", + chan ? chan->id : -1, info->inst, chan ? chan->name : "unknown"); + + /* Handle host/engine faults. */ + if (fifo->func->recover_chan && chan) + fifo->func->recover_chan(fifo, chan->id); + else if (engine && chan) - gf100_fifo_recover(fifo, engine, (void *)chan); - nvkm_fifo_chan_put(&fifo->base, flags, &chan); + gf100_fifo_recover(gf100_fifo(fifo), engine, (void *)chan); + + nvkm_fifo_chan_put(fifo, flags, &chan); } static const struct nvkm_fifo_func_mmu_fault gf100_fifo_mmu_fault = { - .recover = gf100_fifo_fault, + .recover = gf100_fifo_mmu_fault_recover, + .access = gf100_fifo_mmu_fault_access, + .engine = gf100_fifo_mmu_fault_engine, + .reason = gf100_fifo_mmu_fault_reason, + .hubclient = gf100_fifo_mmu_fault_hubclient, + .gpcclient = gf100_fifo_mmu_fault_gpcclient, }; static const struct nvkm_enum @@ -498,6 +526,19 @@ gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) nvkm_fifo_fault(fifo, &info); } +void +gf100_fifo_intr_mmu_fault(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + unsigned long mask = nvkm_rd32(device, 0x00259c); + int unit; + + for_each_set_bit(unit, &mask, 32) { + fifo->func->intr_mmu_fault_unit(fifo, unit); + nvkm_wr32(device, 0x00259c, BIT(unit)); + } +} + bool gf100_fifo_intr_pbdma(struct nvkm_fifo *fifo) { @@ -609,13 +650,7 @@ gf100_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x10000000) { - u32 mask = nvkm_rd32(device, 0x00259c); - while (mask) { - u32 unit = __ffs(mask); - gf100_fifo_intr_mmu_fault_unit(fifo, unit); - nvkm_wr32(device, 0x00259c, (1 << unit)); - mask &= ~(1 << unit); - } + gf100_fifo_intr_mmu_fault(fifo); stat &= ~0x10000000; } @@ -783,6 +818,7 @@ gf100_fifo = { .init_pbdmas = gf100_fifo_init_pbdmas, .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, + .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .mmu_fault = &gf100_fifo_mmu_fault, .engine_id = gf100_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 6295389bfbea2..de8e3a17d79e2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -30,10 +30,8 @@ #include "cgrp.h" #include "changk104.h" -#include #include #include -#include #include #include #include @@ -474,7 +472,7 @@ gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn) if (!status.faulted && engine) { mmui = nvkm_top_fault_id(device, engine->subdev.type, engine->subdev.inst); if (mmui < 0) { - const struct nvkm_enum *en = fifo->func->fault.engine; + const struct nvkm_enum *en = fifo->func->mmu_fault->engine; for (; en && en->name; en++) { if (en->data2 == engine->subdev.type && en->inst == engine->subdev.inst) { @@ -512,15 +510,8 @@ gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn) schedule_work(&fifo->recover.work); } -const struct nvkm_enum -gk104_fifo_fault_access[] = { - { 0x0, "READ" }, - { 0x1, "WRITE" }, - {} -}; - -const struct nvkm_enum -gk104_fifo_fault_engine[] = { +static const struct nvkm_enum +gk104_fifo_mmu_fault_engine[] = { { 0x00, "GR", NULL, NVKM_ENGINE_GR }, { 0x01, "DISPLAY" }, { 0x02, "CAPTURE" }, @@ -528,14 +519,14 @@ gk104_fifo_fault_engine[] = { { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, { 0x06, "SCHED" }, - { 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO }, - { 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO }, - { 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO }, - { 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO }, - { 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO }, - { 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO }, - { 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO }, - { 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO }, + { 0x07, "HOST0" }, + { 0x08, "HOST1" }, + { 0x09, "HOST2" }, + { 0x0a, "HOST3" }, + { 0x0b, "HOST4" }, + { 0x0c, "HOST5" }, + { 0x0d, "HOST6" }, + { 0x0e, "HOST7" }, { 0x0f, "HOSTSR" }, { 0x10, "MSVLD", NULL, NVKM_ENGINE_MSVLD }, { 0x11, "MSPPP", NULL, NVKM_ENGINE_MSPPP }, @@ -551,7 +542,7 @@ gk104_fifo_fault_engine[] = { }; const struct nvkm_enum -gk104_fifo_fault_reason[] = { +gk104_fifo_mmu_fault_reason[] = { { 0x00, "PDE" }, { 0x01, "PDE_SIZE" }, { 0x02, "PTE" }, @@ -572,7 +563,7 @@ gk104_fifo_fault_reason[] = { }; const struct nvkm_enum -gk104_fifo_fault_hubclient[] = { +gk104_fifo_mmu_fault_hubclient[] = { { 0x00, "VIP" }, { 0x01, "CE0" }, { 0x02, "CE1" }, @@ -609,7 +600,7 @@ gk104_fifo_fault_hubclient[] = { }; const struct nvkm_enum -gk104_fifo_fault_gpcclient[] = { +gk104_fifo_mmu_fault_gpcclient[] = { { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" }, { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" }, { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" }, @@ -634,89 +625,14 @@ gk104_fifo_fault_gpcclient[] = { {} }; -void -gk104_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const struct nvkm_enum *er, *ee, *ec, *ea; - struct nvkm_engine *engine = NULL; - struct nvkm_fifo_chan *chan; - unsigned long flags; - const char *en = ""; - char ct[8] = "HUB/"; - - er = nvkm_enum_find(fifo->func->fault.reason, info->reason); - ee = nvkm_enum_find(fifo->func->fault.engine, info->engine); - if (info->hub) { - ec = nvkm_enum_find(fifo->func->fault.hubclient, info->client); - } else { - ec = nvkm_enum_find(fifo->func->fault.gpcclient, info->client); - snprintf(ct, sizeof(ct), "GPC%d/", info->gpc); - } - ea = nvkm_enum_find(fifo->func->fault.access, info->access); - - if (ee && ee->data2) { - switch (ee->data2) { - case NVKM_SUBDEV_BAR: - nvkm_bar_bar1_reset(device); - break; - case NVKM_SUBDEV_INSTMEM: - nvkm_bar_bar2_reset(device); - break; - case NVKM_ENGINE_IFB: - nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); - break; - default: - engine = nvkm_device_engine(device, ee->data2, 0); - break; - } - } - - if (ee == NULL) { - struct nvkm_subdev *subdev = nvkm_top_fault(device, info->engine); - if (subdev) { - if (subdev->func == &nvkm_engine) - engine = container_of(subdev, typeof(*engine), subdev); - en = engine->subdev.name; - } - } else { - en = ee->name; - } - - spin_lock_irqsave(&fifo->base.lock, flags); - chan = nvkm_fifo_chan_inst_locked(&fifo->base, info->inst); - - nvkm_error(subdev, - "fault %02x [%s] at %016llx engine %02x [%s] client %02x " - "[%s%s] reason %02x [%s] on channel %d [%010llx %s]\n", - info->access, ea ? ea->name : "", info->addr, - info->engine, ee ? ee->name : en, - info->client, ct, ec ? ec->name : "", - info->reason, er ? er->name : "", chan ? chan->chid : -1, - info->inst, chan ? chan->object.client->name : "unknown"); - - /* Kill the channel that caused the fault. */ - if (chan) - gk104_fifo_recover_chan(&fifo->base, chan->chid); - - /* Channel recovery will probably have already done this for the - * correct engine(s), but just in case we can't find the channel - * information... - */ - if (engine) { - int engn = fifo->base.func->engine_id(&fifo->base, engine); - if (engn >= 0 && engn != GK104_FIFO_ENGN_SW) - gk104_fifo_recover_engn(fifo, engn); - } - - spin_unlock_irqrestore(&fifo->base.lock, flags); -} - const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault = { - .recover = gk104_fifo_fault, + .recover = gf100_fifo_mmu_fault_recover, + .access = gf100_fifo_mmu_fault_access, + .engine = gk104_fifo_mmu_fault_engine, + .reason = gk104_fifo_mmu_fault_reason, + .hubclient = gk104_fifo_mmu_fault_hubclient, + .gpcclient = gk104_fifo_mmu_fault_gpcclient, }; static const struct nvkm_enum @@ -884,13 +800,7 @@ gk104_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x10000000) { - u32 mask = nvkm_rd32(device, 0x00259c); - while (mask) { - u32 unit = __ffs(mask); - fifo->func->intr_mmu_fault_unit(fifo, unit); - nvkm_wr32(device, 0x00259c, (1 << unit)); - mask &= ~(1 << unit); - } + gf100_fifo_intr_mmu_fault(fifo); stat &= ~0x10000000; } @@ -1105,11 +1015,6 @@ gk104_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .mmu_fault = &gk104_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gk104_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index f7e6c26a2a451..13f1efcc9e248 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -69,11 +69,6 @@ int gk104_fifo_oneinit(struct nvkm_fifo *); void gk104_fifo_init(struct nvkm_fifo *base); void gk104_fifo_fini(struct nvkm_fifo *base); -extern const struct nvkm_enum gk104_fifo_fault_access[]; -extern const struct nvkm_enum gk104_fifo_fault_engine[]; -extern const struct nvkm_enum gk104_fifo_fault_reason[]; -extern const struct nvkm_enum gk104_fifo_fault_hubclient[]; -extern const struct nvkm_enum gk104_fifo_fault_gpcclient[]; extern const struct gk104_fifo_runlist_func gk104_fifo_runlist; void gk104_fifo_runlist_chan(struct gk104_fifo_chan *, struct nvkm_memory *, u32); @@ -84,17 +79,8 @@ extern const struct gk104_fifo_runlist_func gk110_fifo_runlist; void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, struct nvkm_memory *, u32); -void gk208_fifo_pbdma_init_timeout(struct gk104_fifo *); - -extern const struct nvkm_enum gm107_fifo_fault_engine[]; extern const struct gk104_fifo_runlist_func gm107_fifo_runlist; -extern const struct nvkm_enum gp100_fifo_fault_engine[]; - -extern const struct nvkm_enum gv100_fifo_fault_access[]; -extern const struct nvkm_enum gv100_fifo_fault_reason[]; -extern const struct nvkm_enum gv100_fifo_fault_hubclient[]; -extern const struct nvkm_enum gv100_fifo_fault_gpcclient[]; void gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, struct nvkm_memory *, u32); void gv100_fifo_runlist_chan(struct gk104_fifo_chan *, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 4143a07e3808a..5271f55048542 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -88,11 +88,6 @@ gk110_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .mmu_fault = &gk104_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gk104_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 87601c35581c9..ab813aa6d1a16 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -62,11 +62,6 @@ gk208_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .mmu_fault = &gk104_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gk104_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 3d3db802b824c..0d633b805331f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -38,11 +38,6 @@ gk20a_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .mmu_fault = &gk104_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gk104_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index a85f30768b746..ecf4e6d948666 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -56,22 +56,22 @@ const struct nvkm_runl_func gm107_runl = { }; -const struct nvkm_enum -gm107_fifo_fault_engine[] = { +static const struct nvkm_enum +gm107_fifo_mmu_fault_engine[] = { { 0x01, "DISPLAY" }, { 0x02, "CAPTURE" }, { 0x03, "IFB", NULL, NVKM_ENGINE_IFB }, { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, { 0x06, "SCHED" }, - { 0x07, "HOST0", NULL, NVKM_ENGINE_FIFO }, - { 0x08, "HOST1", NULL, NVKM_ENGINE_FIFO }, - { 0x09, "HOST2", NULL, NVKM_ENGINE_FIFO }, - { 0x0a, "HOST3", NULL, NVKM_ENGINE_FIFO }, - { 0x0b, "HOST4", NULL, NVKM_ENGINE_FIFO }, - { 0x0c, "HOST5", NULL, NVKM_ENGINE_FIFO }, - { 0x0d, "HOST6", NULL, NVKM_ENGINE_FIFO }, - { 0x0e, "HOST7", NULL, NVKM_ENGINE_FIFO }, + { 0x07, "HOST0" }, + { 0x08, "HOST1" }, + { 0x09, "HOST2" }, + { 0x0a, "HOST3" }, + { 0x0b, "HOST4" }, + { 0x0c, "HOST5" }, + { 0x0d, "HOST6" }, + { 0x0e, "HOST7" }, { 0x0f, "HOSTSR" }, { 0x13, "PERF" }, { 0x17, "PMU" }, @@ -81,7 +81,12 @@ gm107_fifo_fault_engine[] = { const struct nvkm_fifo_func_mmu_fault gm107_fifo_mmu_fault = { - .recover = gk104_fifo_fault, + .recover = gf100_fifo_mmu_fault_recover, + .access = gf100_fifo_mmu_fault_access, + .engine = gm107_fifo_mmu_fault_engine, + .reason = gk104_fifo_mmu_fault_reason, + .hubclient = gk104_fifo_mmu_fault_hubclient, + .gpcclient = gk104_fifo_mmu_fault_gpcclient, }; void @@ -128,11 +133,6 @@ gm107_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, .mmu_fault = &gm107_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gm107_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 131e4ae615908..6fa96a4c3e239 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -52,11 +52,6 @@ gm200_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, .mmu_fault = &gm107_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gm107_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index b05ede6dd3929..0ca14b08ca3a6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -33,23 +33,23 @@ static const struct nvkm_runl_func gp100_runl = { }; -const struct nvkm_enum -gp100_fifo_fault_engine[] = { +static const struct nvkm_enum +gp100_fifo_mmu_fault_engine[] = { { 0x01, "DISPLAY" }, { 0x03, "IFB", NULL, NVKM_ENGINE_IFB }, { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM }, - { 0x06, "HOST0", NULL, NVKM_ENGINE_FIFO }, - { 0x07, "HOST1", NULL, NVKM_ENGINE_FIFO }, - { 0x08, "HOST2", NULL, NVKM_ENGINE_FIFO }, - { 0x09, "HOST3", NULL, NVKM_ENGINE_FIFO }, - { 0x0a, "HOST4", NULL, NVKM_ENGINE_FIFO }, - { 0x0b, "HOST5", NULL, NVKM_ENGINE_FIFO }, - { 0x0c, "HOST6", NULL, NVKM_ENGINE_FIFO }, - { 0x0d, "HOST7", NULL, NVKM_ENGINE_FIFO }, - { 0x0e, "HOST8", NULL, NVKM_ENGINE_FIFO }, - { 0x0f, "HOST9", NULL, NVKM_ENGINE_FIFO }, - { 0x10, "HOST10", NULL, NVKM_ENGINE_FIFO }, + { 0x06, "HOST0" }, + { 0x07, "HOST1" }, + { 0x08, "HOST2" }, + { 0x09, "HOST3" }, + { 0x0a, "HOST4" }, + { 0x0b, "HOST5" }, + { 0x0c, "HOST6" }, + { 0x0d, "HOST7" }, + { 0x0e, "HOST8" }, + { 0x0f, "HOST9" }, + { 0x10, "HOST10" }, { 0x13, "PERF" }, { 0x17, "PMU" }, { 0x18, "PTP" }, @@ -59,10 +59,15 @@ gp100_fifo_fault_engine[] = { static const struct nvkm_fifo_func_mmu_fault gp100_fifo_mmu_fault = { - .recover = gk104_fifo_fault, + .recover = gf100_fifo_mmu_fault_recover, + .access = gf100_fifo_mmu_fault_access, + .engine = gp100_fifo_mmu_fault_engine, + .reason = gk104_fifo_mmu_fault_reason, + .hubclient = gk104_fifo_mmu_fault_hubclient, + .gpcclient = gk104_fifo_mmu_fault_gpcclient, }; -void +static void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) { struct nvkm_device *device = fifo->engine.subdev.device; @@ -100,11 +105,6 @@ gp100_fifo = { .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gp100_fifo_intr_mmu_fault_unit, .mmu_fault = &gp100_fifo_mmu_fault, - .fault.access = gk104_fifo_fault_access, - .fault.engine = gp100_fifo_fault_engine, - .fault.reason = gk104_fifo_fault_reason, - .fault.hubclient = gk104_fifo_fault_hubclient, - .fault.gpcclient = gk104_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index dc1586e2ab720..49d7c9e7fb8ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -87,7 +87,7 @@ gv100_runl = { }; const struct nvkm_enum -gv100_fifo_fault_gpcclient[] = { +gv100_fifo_mmu_fault_gpcclient[] = { { 0x00, "T1_0" }, { 0x01, "T1_1" }, { 0x02, "T1_2" }, @@ -189,7 +189,7 @@ gv100_fifo_fault_gpcclient[] = { }; const struct nvkm_enum -gv100_fifo_fault_hubclient[] = { +gv100_fifo_mmu_fault_hubclient[] = { { 0x00, "VIP" }, { 0x01, "CE0" }, { 0x02, "CE1" }, @@ -251,7 +251,7 @@ gv100_fifo_fault_hubclient[] = { }; const struct nvkm_enum -gv100_fifo_fault_reason[] = { +gv100_fifo_mmu_fault_reason[] = { { 0x00, "PDE" }, { 0x01, "PDE_SIZE" }, { 0x02, "PTE" }, @@ -272,7 +272,7 @@ gv100_fifo_fault_reason[] = { }; static const struct nvkm_enum -gv100_fifo_fault_engine[] = { +gv100_fifo_mmu_fault_engine[] = { { 0x01, "DISPLAY" }, { 0x03, "PTP" }, { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, @@ -299,7 +299,7 @@ gv100_fifo_fault_engine[] = { }; const struct nvkm_enum -gv100_fifo_fault_access[] = { +gv100_fifo_mmu_fault_access[] = { { 0x0, "VIRT_READ" }, { 0x1, "VIRT_WRITE" }, { 0x2, "VIRT_ATOMIC" }, @@ -314,7 +314,12 @@ gv100_fifo_fault_access[] = { static const struct nvkm_fifo_func_mmu_fault gv100_fifo_mmu_fault = { - .recover = gk104_fifo_fault, + .recover = gf100_fifo_mmu_fault_recover, + .access = gv100_fifo_mmu_fault_access, + .engine = gv100_fifo_mmu_fault_engine, + .reason = gv100_fifo_mmu_fault_reason, + .hubclient = gv100_fifo_mmu_fault_hubclient, + .gpcclient = gv100_fifo_mmu_fault_gpcclient, }; static const struct nvkm_fifo_func @@ -330,11 +335,6 @@ gv100_fifo = { .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .mmu_fault = &gv100_fifo_mmu_fault, - .fault.access = gv100_fifo_fault_access, - .fault.engine = gv100_fifo_fault_engine, - .fault.reason = gv100_fifo_fault_reason, - .fault.hubclient = gv100_fifo_fault_hubclient, - .fault.gpcclient = gv100_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 00f23c39978eb..19c6e706f33e5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -37,15 +37,12 @@ struct nvkm_fifo_func { const struct nvkm_fifo_func_mmu_fault { void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *); - } *mmu_fault; - - struct { const struct nvkm_enum *access; const struct nvkm_enum *engine; const struct nvkm_enum *reason; const struct nvkm_enum *hubclient; const struct nvkm_enum *gpcclient; - } fault; + } *mmu_fault; int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); void (*pause)(struct nvkm_fifo *, unsigned long *); @@ -116,7 +113,10 @@ extern const struct nvkm_chan_func g84_chan; int gf100_fifo_chid_ctor(struct nvkm_fifo *, int); int gf100_fifo_runq_nr(struct nvkm_fifo *); bool gf100_fifo_intr_pbdma(struct nvkm_fifo *); +void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); +void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *); +extern const struct nvkm_enum gf100_fifo_mmu_fault_access[]; extern const struct nvkm_event_func gf100_fifo_nonstall; void gf100_runq_init(struct nvkm_runq *); bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); @@ -129,7 +129,9 @@ irqreturn_t gk104_fifo_intr(struct nvkm_inth *); void gk104_fifo_intr_chsw(struct nvkm_fifo *); void gk104_fifo_intr_bind(struct nvkm_fifo *); extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; -void gk104_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); +extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; +extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; +extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); extern const struct nvkm_runq_func gk104_runq; @@ -155,8 +157,10 @@ extern const struct nvkm_chan_func gm107_chan; int gm200_fifo_chid_nr(struct nvkm_fifo *); int gm200_fifo_runq_nr(struct nvkm_fifo *); -void gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); - +extern const struct nvkm_enum gv100_fifo_mmu_fault_access[]; +extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[]; +extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[]; +extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[]; extern const struct nvkm_runq_func gv100_runq; extern const struct nvkm_engn_func gv100_engn; extern const struct nvkm_engn_func gv100_engn_ce; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 1a15b0a9ab7ce..0b75565bfbc15 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -24,6 +24,8 @@ #include "chid.h" #include "priv.h" +#include + void nvkm_runl_del(struct nvkm_runl *runl) { @@ -46,7 +48,8 @@ struct nvkm_engn * nvkm_runl_add(struct nvkm_runl *runl, int engi, const struct nvkm_engn_func *func, enum nvkm_subdev_type type, int inst) { - struct nvkm_device *device = runl->fifo->engine.subdev.device; + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_engine *engine; struct nvkm_engn *engn; @@ -63,7 +66,25 @@ nvkm_runl_add(struct nvkm_runl *runl, int engi, const struct nvkm_engn_func *fun engn->runl = runl; engn->id = engi; engn->engine = engine; + engn->fault = -1; list_add_tail(&engn->head, &runl->engns); + + /* Lookup MMU engine ID for fault handling. */ + if (device->top) + engn->fault = nvkm_top_fault_id(device, engine->subdev.type, engine->subdev.inst); + + if (engn->fault < 0 && fifo->func->mmu_fault) { + const struct nvkm_enum *map = fifo->func->mmu_fault->engine; + + while (map->name) { + if (map->data2 == engine->subdev.type && map->inst == engine->subdev.inst) { + engn->fault = map->value; + break; + } + map++; + } + } + return engn; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index cb0de870a118b..b223da2d4399b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -13,6 +13,8 @@ struct nvkm_engn { struct nvkm_engine *engine; + int fault; + struct list_head head; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 2d14795b98420..47de0b97534a3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -26,12 +26,8 @@ #include "cgrp.h" #include "changk104.h" -#include #include -#include -#include #include -#include #include @@ -67,7 +63,7 @@ tu102_runl = { }; static const struct nvkm_enum -tu102_fifo_fault_engine[] = { +tu102_fifo_mmu_fault_engine[] = { { 0x01, "DISPLAY" }, { 0x03, "PTP" }, { 0x06, "PWR_PMU" }, @@ -247,91 +243,14 @@ tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn) schedule_work(&fifo->recover.work); } -static void -tu102_fifo_fault(struct nvkm_fifo *base, struct nvkm_fault_data *info) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const struct nvkm_enum *er, *ee, *ec, *ea; - struct nvkm_engine *engine = NULL; - struct nvkm_fifo_chan *chan; - unsigned long flags; - const char *en = ""; - char ct[8] = "HUB/"; - int engn; - - er = nvkm_enum_find(fifo->func->fault.reason, info->reason); - ee = nvkm_enum_find(fifo->func->fault.engine, info->engine); - if (info->hub) { - ec = nvkm_enum_find(fifo->func->fault.hubclient, info->client); - } else { - ec = nvkm_enum_find(fifo->func->fault.gpcclient, info->client); - snprintf(ct, sizeof(ct), "GPC%d/", info->gpc); - } - ea = nvkm_enum_find(fifo->func->fault.access, info->access); - - if (ee && ee->data2) { - switch (ee->data2) { - case NVKM_SUBDEV_BAR: - nvkm_bar_bar1_reset(device); - break; - case NVKM_SUBDEV_INSTMEM: - nvkm_bar_bar2_reset(device); - break; - case NVKM_ENGINE_IFB: - nvkm_mask(device, 0x001718, 0x00000000, 0x00000000); - break; - default: - engine = nvkm_device_engine(device, ee->data2, 0); - break; - } - } - - if (ee == NULL) { - struct nvkm_subdev *subdev = nvkm_top_fault(device, info->engine); - if (subdev) { - if (subdev->func == &nvkm_engine) - engine = container_of(subdev, typeof(*engine), subdev); - en = engine->subdev.name; - } - } else { - en = ee->name; - } - - spin_lock_irqsave(&fifo->base.lock, flags); - chan = nvkm_fifo_chan_inst_locked(&fifo->base, info->inst); - - nvkm_error(subdev, - "fault %02x [%s] at %016llx engine %02x [%s] client %02x " - "[%s%s] reason %02x [%s] on channel %d [%010llx %s]\n", - info->access, ea ? ea->name : "", info->addr, - info->engine, ee ? ee->name : en, - info->client, ct, ec ? ec->name : "", - info->reason, er ? er->name : "", chan ? chan->chid : -1, - info->inst, chan ? chan->object.client->name : "unknown"); - - /* Kill the channel that caused the fault. */ - if (chan) - tu102_fifo_recover_chan(&fifo->base, chan->chid); - - /* Channel recovery will probably have already done this for the - * correct engine(s), but just in case we can't find the channel - * information... - */ - for (engn = 0; engn < fifo->engine_nr && engine; engn++) { - if (fifo->engine[engn].engine == engine) { - tu102_fifo_recover_engn(fifo, engn); - break; - } - } - - spin_unlock_irqrestore(&fifo->base.lock, flags); -} - const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault = { - .recover = tu102_fifo_fault, + .recover = gf100_fifo_mmu_fault_recover, + .access = gv100_fifo_mmu_fault_access, + .engine = tu102_fifo_mmu_fault_engine, + .reason = gv100_fifo_mmu_fault_reason, + .hubclient = gv100_fifo_mmu_fault_hubclient, + .gpcclient = gv100_fifo_mmu_fault_gpcclient, }; static void @@ -441,11 +360,6 @@ tu102_fifo = { .fini = gk104_fifo_fini, .intr = tu102_fifo_intr, .mmu_fault = &tu102_fifo_mmu_fault, - .fault.access = gv100_fifo_fault_access, - .fault.engine = tu102_fifo_fault_engine, - .fault.reason = gv100_fifo_fault_reason, - .fault.hubclient = gv100_fifo_fault_hubclient, - .fault.gpcclient = gv100_fifo_fault_gpcclient, .engine_id = gk104_fifo_engine_id, .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, -- GitLab From c358f53871605a1a8d7ed6e544a05ea00e9c80cb Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:29 +1000 Subject: [PATCH 1319/1927] drm/nouveau/fifo: add new channel lookup interfaces - supports per-runlist CHIDs - channel group lock held across reference, rather than global lock v2: - remove unnecessary parenthesis Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/falcon.h | 2 +- .../drm/nouveau/include/nvkm/engine/fifo.h | 11 ++-- .../gpu/drm/nouveau/nvkm/engine/ce/gt215.c | 6 +- .../gpu/drm/nouveau/nvkm/engine/cipher/g84.c | 11 ++-- drivers/gpu/drm/nouveau/nvkm/engine/falcon.c | 6 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 57 ------------------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 50 ++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 10 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 17 +++--- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 3 - .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 47 +++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 3 + .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 10 ++-- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c | 10 ++-- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv25.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv2a.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.c | 10 ++-- .../gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c | 4 +- drivers/gpu/drm/nouveau/nvkm/engine/sec/g98.c | 6 +- drivers/gpu/drm/nouveau/nvkm/engine/sw/base.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c | 11 ++-- 28 files changed, 166 insertions(+), 132 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index b593407b9e36f..176f0921d82f3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -66,7 +66,7 @@ struct nvkm_falcon_func { u32 size; } data; void (*init)(struct nvkm_falcon *); - void (*intr)(struct nvkm_falcon *, struct nvkm_fifo_chan *); + void (*intr)(struct nvkm_falcon *, struct nvkm_chan *); u32 debug; u32 fbif; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index d3412c9162902..a47667d00f3a3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -34,6 +34,10 @@ struct nvkm_chan { struct nvkm_fifo_engn engn[NVKM_FIFO_ENGN_NR]; }; +struct nvkm_chan *nvkm_chan_get_chid(struct nvkm_engine *, int id, unsigned long *irqflags); +struct nvkm_chan *nvkm_chan_get_inst(struct nvkm_engine *, u64 inst, unsigned long *irqflags); +void nvkm_chan_put(struct nvkm_chan **, unsigned long irqflags); + struct nvkm_fifo { const struct nvkm_fifo_func *func; struct nvkm_engine engine; @@ -62,13 +66,6 @@ void nvkm_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); void nvkm_fifo_pause(struct nvkm_fifo *, unsigned long *); void nvkm_fifo_start(struct nvkm_fifo *, unsigned long *); -void nvkm_fifo_chan_put(struct nvkm_fifo *, unsigned long flags, - struct nvkm_fifo_chan **); -struct nvkm_fifo_chan * -nvkm_fifo_chan_inst(struct nvkm_fifo *, u64 inst, unsigned long *flags); -struct nvkm_fifo_chan * -nvkm_fifo_chan_chid(struct nvkm_fifo *, int chid, unsigned long *flags); - int nv04_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int nv10_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int nv17_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c index 09a112af2f893..c9bf6305c3ec8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gt215.c @@ -40,7 +40,7 @@ gt215_ce_isr_error_name[] = { }; void -gt215_ce_intr(struct nvkm_falcon *ce, struct nvkm_fifo_chan *chan) +gt215_ce_intr(struct nvkm_falcon *ce, struct nvkm_chan *chan) { struct nvkm_subdev *subdev = &ce->engine.subdev; struct nvkm_device *device = subdev->device; @@ -55,9 +55,9 @@ gt215_ce_intr(struct nvkm_falcon *ce, struct nvkm_fifo_chan *chan) nvkm_error(subdev, "DISPATCH_ERROR %04x [%s] ch %d [%010llx %s] " "subc %d mthd %04x data %08x\n", ssta, - en ? en->name : "", chan ? chan->chid : -1, + en ? en->name : "", chan ? chan->id : -1, chan ? chan->inst->addr : 0, - chan ? chan->object.client->name : "unknown", + chan ? chan->name : "unknown", subc, mthd, data); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/cipher/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/cipher/g84.c index be2a7181dc159..caca4f6398957 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/cipher/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/cipher/g84.c @@ -81,8 +81,7 @@ g84_cipher_intr(struct nvkm_engine *cipher) { struct nvkm_subdev *subdev = &cipher->subdev; struct nvkm_device *device = subdev->device; - struct nvkm_fifo *fifo = device->fifo; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; u32 stat = nvkm_rd32(device, 0x102130); u32 mthd = nvkm_rd32(device, 0x102190); u32 data = nvkm_rd32(device, 0x102194); @@ -90,16 +89,16 @@ g84_cipher_intr(struct nvkm_engine *cipher) unsigned long flags; char msg[128]; - chan = nvkm_fifo_chan_inst(fifo, (u64)inst << 12, &flags); + chan = nvkm_chan_get_inst(cipher, (u64)inst << 12, &flags); if (stat) { nvkm_snprintbf(msg, sizeof(msg), g84_cipher_intr_mask, stat); nvkm_error(subdev, "%08x [%s] ch %d [%010llx %s] " "mthd %04x data %08x\n", stat, msg, - chan ? chan->chid : -1, (u64)inst << 12, - chan ? chan->object.client->name : "unknown", + chan ? chan->id : -1, (u64)inst << 12, + chan ? chan->name : "unknown", mthd, data); } - nvkm_fifo_chan_put(fifo, flags, &chan); + nvkm_chan_put(&chan, flags); nvkm_wr32(device, 0x102130, stat); nvkm_wr32(device, 0x10200c, 0x10); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c index 43b7dec451796..d619b40a42c36 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/falcon.c @@ -65,10 +65,10 @@ nvkm_falcon_intr(struct nvkm_engine *engine) u32 dest = nvkm_rd32(device, base + 0x01c); u32 intr = nvkm_rd32(device, base + 0x008) & dest & ~(dest >> 16); u32 inst = nvkm_rd32(device, base + 0x050) & 0x3fffffff; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; unsigned long flags; - chan = nvkm_fifo_chan_inst(device->fifo, (u64)inst << 12, &flags); + chan = nvkm_chan_get_inst(engine, (u64)inst << 12, &flags); if (intr & 0x00000040) { if (falcon->func->intr) { @@ -89,7 +89,7 @@ nvkm_falcon_intr(struct nvkm_engine *engine) nvkm_wr32(device, base + 0x004, intr); } - nvkm_fifo_chan_put(device->fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index e51fb10a04f72..83da63bdf3c46 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -62,63 +62,6 @@ nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) return fifo->func->mmu_fault->recover(fifo, info); } -void -nvkm_fifo_chan_put(struct nvkm_fifo *fifo, unsigned long flags, - struct nvkm_fifo_chan **pchan) -{ - struct nvkm_fifo_chan *chan = *pchan; - if (likely(chan)) { - *pchan = NULL; - spin_unlock_irqrestore(&fifo->lock, flags); - } -} - -struct nvkm_fifo_chan * -nvkm_fifo_chan_inst_locked(struct nvkm_fifo *fifo, u64 inst) -{ - struct nvkm_fifo_chan *chan; - list_for_each_entry(chan, &fifo->chan, head) { - if (chan->inst->addr == inst) { - list_del(&chan->head); - list_add(&chan->head, &fifo->chan); - return chan; - } - } - return NULL; -} - -struct nvkm_fifo_chan * -nvkm_fifo_chan_inst(struct nvkm_fifo *fifo, u64 inst, unsigned long *rflags) -{ - struct nvkm_fifo_chan *chan; - unsigned long flags; - spin_lock_irqsave(&fifo->lock, flags); - if ((chan = nvkm_fifo_chan_inst_locked(fifo, inst))) { - *rflags = flags; - return chan; - } - spin_unlock_irqrestore(&fifo->lock, flags); - return NULL; -} - -struct nvkm_fifo_chan * -nvkm_fifo_chan_chid(struct nvkm_fifo *fifo, int chid, unsigned long *rflags) -{ - struct nvkm_fifo_chan *chan; - unsigned long flags; - spin_lock_irqsave(&fifo->lock, flags); - list_for_each_entry(chan, &fifo->chan, head) { - if (chan->chid == chid) { - list_del(&chan->head); - list_add(&chan->head, &fifo->chan); - *rflags = flags; - return chan; - } - } - spin_unlock_irqrestore(&fifo->lock, flags); - return NULL; -} - void nvkm_fifo_kevent(struct nvkm_fifo *fifo, int chid) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index ea8148bad45e4..bfbd5ec9a7206 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -302,6 +302,56 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object) return data; } +void +nvkm_chan_put(struct nvkm_chan **pchan, unsigned long irqflags) +{ + struct nvkm_chan *chan = *pchan; + + if (!chan) + return; + + *pchan = NULL; + spin_unlock_irqrestore(&chan->cgrp->lock, irqflags); +} + +struct nvkm_chan * +nvkm_chan_get_inst(struct nvkm_engine *engine, u64 inst, unsigned long *pirqflags) +{ + struct nvkm_fifo *fifo = engine->subdev.device->fifo; + struct nvkm_runl *runl; + struct nvkm_engn *engn; + struct nvkm_chan *chan; + + nvkm_runl_foreach(runl, fifo) { + nvkm_runl_foreach_engn(engn, runl) { + if (engine == &fifo->engine || engn->engine == engine) { + chan = nvkm_runl_chan_get_inst(runl, inst, pirqflags); + if (chan || engn->engine == engine) + return chan; + } + } + } + + return NULL; +} + +struct nvkm_chan * +nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags) +{ + struct nvkm_fifo *fifo = engine->subdev.device->fifo; + struct nvkm_runl *runl; + struct nvkm_engn *engn; + + nvkm_runl_foreach(runl, fifo) { + nvkm_runl_foreach_engn(engn, runl) { + if (fifo->chid || engn->engine == engine) + return nvkm_runl_chan_get_chid(runl, id, pirqflags); + } + } + + return NULL; +} + static const struct nvkm_object_func nvkm_fifo_chan_func = { .dtor = nvkm_fifo_chan_dtor, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 7fc4c37b407f0..7c1db7721ff61 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -84,13 +84,13 @@ gf100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null) if (show) { nvkm_snprintbf(msg, sizeof(msg), runq->func->intr_0_names, show); - chan = nvkm_fifo_chan_chid(runq->fifo, chid, &flags); + chan = nvkm_chan_get_chid(&runq->fifo->engine, chid, &flags); nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " "subc %d mthd %04x data %08x\n", runq->id, show, msg, chid, chan ? chan->inst->addr : 0, chan ? chan->object.client->name : "unknown", subc, mthd, data); - nvkm_fifo_chan_put(runq->fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } nvkm_wr32(device, 0x0400c0 + (runq->id * 0x2000), 0x80600008); @@ -367,9 +367,9 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf struct nvkm_device *device = subdev->device; const struct nvkm_enum *er, *ee, *ec, *ea; struct nvkm_engine *engine = NULL; - struct nvkm_fifo_chan *chan; struct nvkm_runl *runl; struct nvkm_engn *engn; + struct nvkm_chan *chan; unsigned long flags; char ct[8] = "HUB/"; @@ -409,7 +409,7 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf } } - chan = nvkm_fifo_chan_inst(fifo, info->inst, &flags); + chan = nvkm_chan_get_inst(&fifo->engine, info->inst, &flags); nvkm_error(subdev, "fault %02x [%s] at %016llx engine %02x [%s] client %02x " @@ -427,7 +427,7 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf if (engine && chan) gf100_fifo_recover(gf100_fifo(fifo), engine, (void *)chan); - nvkm_fifo_chan_put(fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } static const struct nvkm_fifo_func_mmu_fault diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index b899127cfd728..85f22fce11eda 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -30,7 +30,6 @@ #include "channv04.h" #include "regsnv04.h" -#include #include #include #include @@ -241,7 +240,7 @@ nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get) { struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; unsigned long flags; u32 pull0 = nvkm_rd32(device, 0x003250); u32 mthd, data; @@ -264,12 +263,12 @@ nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get) if (!(pull0 & 0x00000100) || !nv04_fifo_swmthd(device, chid, mthd, data)) { - chan = nvkm_fifo_chan_chid(fifo, chid, &flags); + chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); nvkm_error(subdev, "CACHE_ERROR - " "ch %d [%s] subc %d mthd %04x data %08x\n", - chid, chan ? chan->object.client->name : "unknown", + chid, chan ? chan->name : "unknown", (mthd >> 13) & 7, mthd & 0x1ffc, data); - nvkm_fifo_chan_put(fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0); @@ -296,12 +295,12 @@ nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid) u32 dma_put = nvkm_rd32(device, 0x003240); u32 push = nvkm_rd32(device, 0x003220); u32 state = nvkm_rd32(device, 0x003228); - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; unsigned long flags; const char *name; - chan = nvkm_fifo_chan_chid(fifo, chid, &flags); - name = chan ? chan->object.client->name : "unknown"; + chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); + name = chan ? chan->name : "unknown"; if (device->card_type == NV_50) { u32 ho_get = nvkm_rd32(device, 0x003328); u32 ho_put = nvkm_rd32(device, 0x003320); @@ -332,7 +331,7 @@ nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid) if (dma_get != dma_put) nvkm_wr32(device, 0x003244, dma_put); } - nvkm_fifo_chan_put(fifo, flags, &chan); + nvkm_chan_put(&chan, flags); nvkm_wr32(device, 0x003228, 0x00000000); nvkm_wr32(device, 0x003220, 0x00000001); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 19c6e706f33e5..7bc7b89ffd189 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -14,9 +14,6 @@ struct gk104_fifo_chan; void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid); -struct nvkm_fifo_chan * -nvkm_fifo_chan_inst_locked(struct nvkm_fifo *, u64 inst); - struct nvkm_fifo_chan_oclass; struct nvkm_fifo_func { void *(*dtor)(struct nvkm_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 0b75565bfbc15..e4984e1e7c650 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -20,12 +20,59 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include "runl.h" +#include "cgrp.h" #include "chan.h" #include "chid.h" #include "priv.h" +#include #include +struct nvkm_chan * +nvkm_runl_chan_get_inst(struct nvkm_runl *runl, u64 inst, unsigned long *pirqflags) +{ + struct nvkm_chid *chid = runl->chid; + struct nvkm_chan *chan; + unsigned long flags; + int id; + + spin_lock_irqsave(&chid->lock, flags); + for_each_set_bit(id, chid->used, chid->nr) { + chan = chid->data[id]; + if (likely(chan)) { + if (chan->inst->addr == inst) { + spin_lock(&chan->cgrp->lock); + *pirqflags = flags; + spin_unlock(&chid->lock); + return chan; + } + } + } + spin_unlock_irqrestore(&chid->lock, flags); + return NULL; +} + +struct nvkm_chan * +nvkm_runl_chan_get_chid(struct nvkm_runl *runl, int id, unsigned long *pirqflags) +{ + struct nvkm_chid *chid = runl->chid; + struct nvkm_chan *chan; + unsigned long flags; + + spin_lock_irqsave(&chid->lock, flags); + if (!WARN_ON(id >= chid->nr)) { + chan = chid->data[id]; + if (likely(chan)) { + spin_lock(&chan->cgrp->lock); + *pirqflags = flags; + spin_unlock(&chid->lock); + return chan; + } + } + spin_unlock_irqrestore(&chid->lock, flags); + return NULL; +} + void nvkm_runl_del(struct nvkm_runl *runl) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index b223da2d4399b..e618e326b3c34 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -51,6 +51,9 @@ struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_ enum nvkm_subdev_type, int inst); void nvkm_runl_del(struct nvkm_runl *); +struct nvkm_chan *nvkm_runl_chan_get_chid(struct nvkm_runl *, int chid, unsigned long *irqflags); +struct nvkm_chan *nvkm_runl_chan_get_inst(struct nvkm_runl *, u64 inst, unsigned long *irqflags); + #define nvkm_runl_find_engn(engn,runl,cond) nvkm_list_find(engn, &(runl)->engns, head, (cond)) #define nvkm_runl_foreach(runl,fifo) list_for_each_entry((runl), &(fifo)->runls, head) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index f16eabf4f6426..68da77df856ae 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1548,7 +1548,7 @@ gf100_gr_intr(struct nvkm_gr *base) struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; unsigned long flags; u64 inst = nvkm_rd32(device, 0x409b00) & 0x0fffffff; u32 stat = nvkm_rd32(device, 0x400100); @@ -1561,10 +1561,10 @@ gf100_gr_intr(struct nvkm_gr *base) const char *name = "unknown"; int chid = -1; - chan = nvkm_fifo_chan_inst(device->fifo, (u64)inst << 12, &flags); + chan = nvkm_chan_get_inst(&gr->base.engine, (u64)inst << 12, &flags); if (chan) { - name = chan->object.client->name; - chid = chan->chid; + name = chan->name; + chid = chan->id; } if (device->card_type < NV_E0 || subc < 4) @@ -1631,7 +1631,7 @@ gf100_gr_intr(struct nvkm_gr *base) } nvkm_wr32(device, 0x400500, 0x00010001); - nvkm_fifo_chan_put(device->fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c index 0bc1a238de435..81bd682c21021 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv04.c @@ -1192,7 +1192,7 @@ nv04_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv04_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; *ctx_reg(chan, NV04_PGRAPH_DEBUG_3) = 0xfad4ff31; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c index 942450b33bc60..7fe6e58f6bab6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv10.c @@ -1011,7 +1011,7 @@ nv10_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv10_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; NV_WRITE_CTX(0x00400e88, 0x08000000); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c index 6bff10cee71b9..75434f5de7add 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv20.c @@ -83,7 +83,7 @@ nv20_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv20_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; ret = nvkm_memory_new(gr->base.engine.subdev.device, @@ -182,7 +182,7 @@ nv20_gr_intr(struct nvkm_gr *base) struct nv20_gr *gr = nv20_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; u32 stat = nvkm_rd32(device, NV03_PGRAPH_INTR); u32 nsource = nvkm_rd32(device, NV03_PGRAPH_NSOURCE); u32 nstatus = nvkm_rd32(device, NV03_PGRAPH_NSTATUS); @@ -196,7 +196,7 @@ nv20_gr_intr(struct nvkm_gr *base) char msg[128], src[128], sta[128]; unsigned long flags; - chan = nvkm_fifo_chan_chid(device->fifo, chid, &flags); + chan = nvkm_chan_get_chid(&gr->base.engine, chid, &flags); nvkm_wr32(device, NV03_PGRAPH_INTR, stat); nvkm_wr32(device, NV04_PGRAPH_FIFO, 0x00000001); @@ -209,11 +209,11 @@ nv20_gr_intr(struct nvkm_gr *base) "nstatus %08x [%s] ch %d [%s] subc %d " "class %04x mthd %04x data %08x\n", show, msg, nsource, src, nstatus, sta, chid, - chan ? chan->object.client->name : "unknown", + chan ? chan->name : "unknown", subc, class, mthd, data); } - nvkm_fifo_chan_put(device->fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv25.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv25.c index f3a56f17d94ae..94685e4d4f87c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv25.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv25.c @@ -29,7 +29,7 @@ nv25_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv25_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; ret = nvkm_memory_new(gr->base.engine.subdev.device, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv2a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv2a.c index f268d2642d295..2d6273675291a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv2a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv2a.c @@ -29,7 +29,7 @@ nv2a_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv2a_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; ret = nvkm_memory_new(gr->base.engine.subdev.device, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c index e5737cdf2fa14..647bd6fede040 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c @@ -30,7 +30,7 @@ nv30_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv30_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; ret = nvkm_memory_new(gr->base.engine.subdev.device, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c index 1ab2da8ebf4eb..2eae3fe4ef4eb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c @@ -29,7 +29,7 @@ nv34_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv34_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; ret = nvkm_memory_new(gr->base.engine.subdev.device, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c index 591260f5676b7..657d7cdba369a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv35.c @@ -29,7 +29,7 @@ nv35_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, return -ENOMEM; nvkm_object_ctor(&nv35_gr_chan, oclass, &chan->object); chan->gr = gr; - chan->chid = fifoch->chid; + chan->chid = fifoch->id; *pobject = &chan->object; ret = nvkm_memory_new(gr->base.engine.subdev.device, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c index 67f3535ff97e1..d2df097a6cf63 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv40.c @@ -275,8 +275,8 @@ nv40_gr_intr(struct nvkm_gr *base) "nstatus %08x [%s] ch %d [%08x %s] subc %d " "class %04x mthd %04x data %08x\n", show, msg, nsource, src, nstatus, sta, - chan ? chan->fifo->chid : -1, inst << 4, - chan ? chan->fifo->object.client->name : "unknown", + chan ? chan->fifo->id : -1, inst << 4, + chan ? chan->fifo->name : "unknown", subc, class, mthd, data); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.c index 563a10097e953..1ba18a8e380f6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv50.c @@ -622,7 +622,7 @@ nv50_gr_intr(struct nvkm_gr *base) struct nv50_gr *gr = nv50_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; u32 stat = nvkm_rd32(device, 0x400100); u32 inst = nvkm_rd32(device, 0x40032c) & 0x0fffffff; u32 addr = nvkm_rd32(device, 0x400704); @@ -637,10 +637,10 @@ nv50_gr_intr(struct nvkm_gr *base) char msg[128]; int chid = -1; - chan = nvkm_fifo_chan_inst(device->fifo, (u64)inst << 12, &flags); + chan = nvkm_chan_get_inst(&gr->base.engine, (u64)inst << 12, &flags); if (chan) { - name = chan->object.client->name; - chid = chan->chid; + name = chan->name; + chid = chan->id; } if (show & 0x00100000) { @@ -672,7 +672,7 @@ nv50_gr_intr(struct nvkm_gr *base) if (nvkm_rd32(device, 0x400824) & (1 << 31)) nvkm_wr32(device, 0x400824, nvkm_rd32(device, 0x400824) & ~(1 << 31)); - nvkm_fifo_chan_put(device->fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c index b1054db4c1b85..cb0c3991b2ad9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv31.c @@ -213,8 +213,8 @@ nv31_mpeg_intr(struct nvkm_engine *engine) if (show) { nvkm_error(subdev, "ch %d [%s] %08x %08x %08x %08x\n", - mpeg->chan ? mpeg->chan->fifo->chid : -1, - mpeg->chan ? mpeg->chan->object.client->name : + mpeg->chan ? mpeg->chan->fifo->id : -1, + mpeg->chan ? mpeg->chan->fifo->name : "unknown", stat, type, mthd, data); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c index 521ce43a2871e..0890a279458ed 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/mpeg/nv44.c @@ -182,8 +182,8 @@ nv44_mpeg_intr(struct nvkm_engine *engine) if (show) { nvkm_error(subdev, "ch %d [%08x %s] %08x %08x %08x %08x\n", - chan ? chan->fifo->chid : -1, inst << 4, - chan ? chan->object.client->name : "unknown", + chan ? chan->fifo->id : -1, inst << 4, + chan ? chan->fifo->name : "unknown", stat, type, mthd, data); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec/g98.c index 1b87df03c8235..c15b2cbf506b2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec/g98.c @@ -40,7 +40,7 @@ static const struct nvkm_enum g98_sec_isr_error_name[] = { }; static void -g98_sec_intr(struct nvkm_falcon *sec, struct nvkm_fifo_chan *chan) +g98_sec_intr(struct nvkm_falcon *sec, struct nvkm_chan *chan) { struct nvkm_subdev *subdev = &sec->engine.subdev; struct nvkm_device *device = subdev->device; @@ -54,9 +54,9 @@ g98_sec_intr(struct nvkm_falcon *sec, struct nvkm_fifo_chan *chan) nvkm_error(subdev, "DISPATCH_ERROR %04x [%s] ch %d [%010llx %s] " "subc %d mthd %04x data %08x\n", ssta, - en ? en->name : "UNKNOWN", chan ? chan->chid : -1, + en ? en->name : "UNKNOWN", chan ? chan->id : -1, chan ? chan->inst->addr : 0, - chan ? chan->object.client->name : "unknown", + chan ? chan->name : "unknown", subc, mthd, data); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sw/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sw/base.c index 14871d0bd7463..a9d464db69749 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sw/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sw/base.c @@ -35,7 +35,7 @@ nvkm_sw_mthd(struct nvkm_sw *sw, int chid, int subc, u32 mthd, u32 data) spin_lock_irqsave(&sw->engine.lock, flags); list_for_each_entry(chan, &sw->chan, head) { - if (chan->fifo->chid == chid) { + if (chan->fifo->id == chid) { handled = nvkm_sw_chan_mthd(chan, subc, mthd, data); list_del(&chan->head); list_add(&chan->head, &sw->chan); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c index 95fd8f8340102..48641735168e7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c @@ -137,8 +137,7 @@ nv50_fb_intr(struct nvkm_fb *base) struct nv50_fb *fb = nv50_fb(base); struct nvkm_subdev *subdev = &fb->base.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_fifo *fifo = device->fifo; - struct nvkm_fifo_chan *chan; + struct nvkm_chan *chan; const struct nvkm_enum *en, *re, *cl, *sc; u32 trap[6], idx, inst; u8 st0, st1, st2, st3; @@ -178,18 +177,18 @@ nv50_fb_intr(struct nvkm_fb *base) else if (en && en->data) sc = nvkm_enum_find(en->data, st3); else sc = NULL; - chan = nvkm_fifo_chan_inst(fifo, inst, &flags); + chan = nvkm_chan_get_inst(&device->fifo->engine, inst, &flags); nvkm_error(subdev, "trapped %s at %02x%04x%04x on channel %d [%08x %s] " "engine %02x [%s] client %02x [%s] " "subclient %02x [%s] reason %08x [%s]\n", (trap[5] & 0x00000100) ? "read" : "write", trap[5] & 0xff, trap[4] & 0xffff, trap[3] & 0xffff, - chan ? chan->chid : -1, inst, - chan ? chan->object.client->name : "unknown", + chan ? chan->id : -1, inst, + chan ? chan->name : "unknown", st0, en ? en->name : "", st2, cl ? cl->name : "", st3, sc ? sc->name : "", st1, re ? re->name : ""); - nvkm_fifo_chan_put(fifo, flags, &chan); + nvkm_chan_put(&chan, flags); } static int -- GitLab From f48dd2936138882d7755cbbc5d9984015c75980c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:30 +1000 Subject: [PATCH 1320/1927] drm/nouveau/fifo: add new engine context tracking Channel groups have somewhat more complicated requirements than what we currently support. An engine context is shared between all channels in a channel group, VEID/subctx support (later) brings per-VEID components, and we need to track an individual channel's engine context pointers. This commit adds the structures and refcounting to support the above, wrapping the prior implementation for the moment. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 3 +- drivers/gpu/drm/nouveau/nvkm/core/oproxy.c | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c | 110 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 23 ++++ .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 83 +++++++++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 14 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 66 +++++++++++ 7 files changed, 296 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index a47667d00f3a3..5a5c374804002 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -11,7 +11,6 @@ struct nvkm_fault_data; struct nvkm_fifo_engn { struct nvkm_object *object; int refcount; - int usecount; }; struct nvkm_chan { @@ -21,6 +20,8 @@ struct nvkm_chan { union { int id; int chid; }; /*FIXME: remove later */ + struct list_head cctxs; + struct nvkm_fifo *fifo; struct nvkm_object object; diff --git a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c index 3ffd4845d9e5b..3385528da6509 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/oproxy.c @@ -47,7 +47,12 @@ nvkm_oproxy_map(struct nvkm_object *object, void *argv, u32 argc, static int nvkm_oproxy_unmap(struct nvkm_object *object) { - return nvkm_object_unmap(nvkm_oproxy(object)->object); + struct nvkm_oproxy *oproxy = nvkm_oproxy(object); + + if (unlikely(!oproxy->object)) + return 0; + + return nvkm_object_unmap(oproxy->object); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c index e6ac178db8b50..ed838609f15d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c @@ -27,6 +27,112 @@ #include +static void +nvkm_cgrp_ectx_put(struct nvkm_cgrp *cgrp, struct nvkm_ectx **pectx) +{ + struct nvkm_ectx *ectx = *pectx; + + if (ectx) { + struct nvkm_engn *engn = ectx->engn; + + if (refcount_dec_and_test(&ectx->refs)) { + CGRP_TRACE(cgrp, "dtor ectx %d[%s]", engn->id, engn->engine->subdev.name); + list_del(&ectx->head); + kfree(ectx); + } + + *pectx = NULL; + } +} + +static int +nvkm_cgrp_ectx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_ectx **pectx, + struct nvkm_chan *chan, struct nvkm_client *client) +{ + struct nvkm_ectx *ectx; + int ret = 0; + + /* Look for an existing context for this engine in the channel group. */ + ectx = nvkm_list_find(ectx, &cgrp->ectxs, head, ectx->engn == engn); + if (ectx) { + refcount_inc(&ectx->refs); + *pectx = ectx; + return 0; + } + + /* Nope - create a fresh one. */ + CGRP_TRACE(cgrp, "ctor ectx %d[%s]", engn->id, engn->engine->subdev.name); + if (!(ectx = *pectx = kzalloc(sizeof(*ectx), GFP_KERNEL))) + return -ENOMEM; + + ectx->engn = engn; + refcount_set(&ectx->refs, 1); + list_add_tail(&ectx->head, &cgrp->ectxs); + return ret; +} + +void +nvkm_cgrp_vctx_put(struct nvkm_cgrp *cgrp, struct nvkm_vctx **pvctx) +{ + struct nvkm_vctx *vctx = *pvctx; + + if (vctx) { + struct nvkm_engn *engn = vctx->ectx->engn; + + if (refcount_dec_and_test(&vctx->refs)) { + CGRP_TRACE(cgrp, "dtor vctx %d[%s]", engn->id, engn->engine->subdev.name); + + nvkm_cgrp_ectx_put(cgrp, &vctx->ectx); + if (vctx->vmm) { + atomic_dec(&vctx->vmm->engref[engn->engine->subdev.type]); + nvkm_vmm_unref(&vctx->vmm); + } + list_del(&vctx->head); + kfree(vctx); + } + + *pvctx = NULL; + } +} + +int +nvkm_cgrp_vctx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_chan *chan, + struct nvkm_vctx **pvctx, struct nvkm_client *client) +{ + struct nvkm_ectx *ectx; + struct nvkm_vctx *vctx; + int ret; + + /* Look for an existing sub-context for this engine+VEID in the channel group. */ + vctx = nvkm_list_find(vctx, &cgrp->vctxs, head, + vctx->ectx->engn == engn && vctx->vmm == chan->vmm); + if (vctx) { + refcount_inc(&vctx->refs); + *pvctx = vctx; + return 0; + } + + /* Nope - create a fresh one. But, context first. */ + ret = nvkm_cgrp_ectx_get(cgrp, engn, &ectx, chan, client); + if (ret) { + CGRP_ERROR(cgrp, "ectx %d[%s]: %d", engn->id, engn->engine->subdev.name, ret); + return ret; + } + + /* Now, create the sub-context. */ + CGRP_TRACE(cgrp, "ctor vctx %d[%s]", engn->id, engn->engine->subdev.name); + if (!(vctx = *pvctx = kzalloc(sizeof(*vctx), GFP_KERNEL))) { + nvkm_cgrp_ectx_put(cgrp, &ectx); + return -ENOMEM; + } + + vctx->ectx = ectx; + vctx->vmm = nvkm_vmm_ref(chan->vmm); + refcount_set(&vctx->refs, 1); + list_add_tail(&vctx->head, &cgrp->vctxs); + return ret; +} + static void nvkm_cgrp_del(struct kref *kref) { @@ -36,6 +142,7 @@ nvkm_cgrp_del(struct kref *kref) if (runl->cgid) nvkm_chid_put(runl->cgid, cgrp->id, &cgrp->lock); + mutex_destroy(&cgrp->mutex); nvkm_vmm_unref(&cgrp->vmm); kfree(cgrp); } @@ -80,6 +187,9 @@ nvkm_cgrp_new(struct nvkm_runl *runl, const char *name, struct nvkm_vmm *vmm, bo cgrp->chans = NULL; cgrp->chan_nr = 0; spin_lock_init(&cgrp->lock); + INIT_LIST_HEAD(&cgrp->ectxs); + INIT_LIST_HEAD(&cgrp->vctxs); + mutex_init(&cgrp->mutex); if (runl->cgid) { cgrp->id = nvkm_chid_get(runl->cgid, cgrp); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index d0510df9286a4..50ba7c7ee0a8f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -3,6 +3,22 @@ #define __NVKM_CGRP_H__ #include struct nvkm_chan; +struct nvkm_client; + +struct nvkm_vctx { + struct nvkm_ectx *ectx; + struct nvkm_vmm *vmm; + refcount_t refs; + + struct list_head head; +}; + +struct nvkm_ectx { + struct nvkm_engn *engn; + refcount_t refs; + + struct list_head head; +}; struct nvkm_cgrp { const struct nvkm_cgrp_func { @@ -19,6 +35,10 @@ struct nvkm_cgrp { spinlock_t lock; /* protects irq handler channel (group) lookup */ + struct list_head ectxs; + struct list_head vctxs; + struct mutex mutex; + struct list_head head; struct list_head chan; }; @@ -27,6 +47,9 @@ int nvkm_cgrp_new(struct nvkm_runl *, const char *name, struct nvkm_vmm *, bool struct nvkm_cgrp **); struct nvkm_cgrp *nvkm_cgrp_ref(struct nvkm_cgrp *); void nvkm_cgrp_unref(struct nvkm_cgrp **); +int nvkm_cgrp_vctx_get(struct nvkm_cgrp *, struct nvkm_engn *, struct nvkm_chan *, + struct nvkm_vctx **, struct nvkm_client *); +void nvkm_cgrp_vctx_put(struct nvkm_cgrp *, struct nvkm_vctx **); #define CGRP_PRCLI(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) #define CGRP_PRINT(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:"f, (c)->id, ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index bfbd5ec9a7206..f250b0f9e2740 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -65,9 +65,6 @@ nvkm_fifo_chan_child_fini(struct nvkm_oproxy *base, bool suspend) const char *name = engine->subdev.name; int ret = 0; - if (--engn->usecount) - return 0; - if (chan->func->engine_fini) { ret = chan->func->engine_fini(chan, engine, suspend); if (ret) { @@ -98,9 +95,6 @@ nvkm_fifo_chan_child_init(struct nvkm_oproxy *base) const char *name = engine->subdev.name; int ret; - if (engn->usecount++) - return 0; - if (engn->object) { ret = nvkm_object_init(engn->object); if (ret) @@ -144,8 +138,6 @@ nvkm_fifo_chan_child_del(struct nvkm_oproxy *base) static const struct nvkm_oproxy_func nvkm_fifo_chan_child_func = { .dtor[0] = nvkm_fifo_chan_child_del, - .init[0] = nvkm_fifo_chan_child_init, - .fini[0] = nvkm_fifo_chan_child_fini, }; int @@ -216,6 +208,80 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, return 0; } +void +nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct nvkm_cctx *cctx) +{ + /* Update context pointer. */ + if (cctx) + nvkm_fifo_chan_child_init(nvkm_oproxy(oproxy->object)); + else + nvkm_fifo_chan_child_fini(nvkm_oproxy(oproxy->object), false); +} + +void +nvkm_chan_cctx_put(struct nvkm_chan *chan, struct nvkm_cctx **pcctx) +{ + struct nvkm_cctx *cctx = *pcctx; + + if (cctx) { + struct nvkm_engn *engn = cctx->vctx->ectx->engn; + + if (refcount_dec_and_mutex_lock(&cctx->refs, &chan->cgrp->mutex)) { + CHAN_TRACE(chan, "dtor cctx %d[%s]", engn->id, engn->engine->subdev.name); + nvkm_cgrp_vctx_put(chan->cgrp, &cctx->vctx); + list_del(&cctx->head); + kfree(cctx); + mutex_unlock(&chan->cgrp->mutex); + } + + *pcctx = NULL; + } +} + +int +nvkm_chan_cctx_get(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx **pcctx, + struct nvkm_client *client) +{ + struct nvkm_cgrp *cgrp = chan->cgrp; + struct nvkm_vctx *vctx; + struct nvkm_cctx *cctx; + int ret; + + /* Look for an existing channel context for this engine+VEID. */ + mutex_lock(&cgrp->mutex); + cctx = nvkm_list_find(cctx, &chan->cctxs, head, + cctx->vctx->ectx->engn == engn && cctx->vctx->vmm == chan->vmm); + if (cctx) { + refcount_inc(&cctx->refs); + *pcctx = cctx; + mutex_unlock(&chan->cgrp->mutex); + return 0; + } + + /* Nope - create a fresh one. But, sub-context first. */ + ret = nvkm_cgrp_vctx_get(cgrp, engn, chan, &vctx, client); + if (ret) { + CHAN_ERROR(chan, "vctx %d[%s]: %d", engn->id, engn->engine->subdev.name, ret); + goto done; + } + + /* Now, create the channel context - to track engine binding. */ + CHAN_TRACE(chan, "ctor cctx %d[%s]", engn->id, engn->engine->subdev.name); + if (!(cctx = *pcctx = kzalloc(sizeof(*cctx), GFP_KERNEL))) { + nvkm_cgrp_vctx_put(cgrp, &vctx); + ret = -ENOMEM; + goto done; + } + + cctx->vctx = vctx; + refcount_set(&cctx->refs, 1); + refcount_set(&cctx->uses, 0); + list_add_tail(&cctx->head, &chan->cctxs); +done: + mutex_unlock(&cgrp->mutex); + return ret; +} + static int nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) { @@ -409,6 +475,7 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); chan->fifo = fifo; + INIT_LIST_HEAD(&chan->cctxs); INIT_LIST_HEAD(&chan->head); /* Join channel group. diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 0f1added5c242..bf1c3b580defd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -3,9 +3,18 @@ #define __NVKM_CHAN_H__ #define nvkm_chan(p) container_of((p), struct nvkm_chan, object) /*FIXME: remove later */ #include +struct nvkm_engn; extern const struct nvkm_event_func nvkm_chan_event; +struct nvkm_cctx { + struct nvkm_vctx *vctx; + refcount_t refs; + refcount_t uses; + + struct list_head head; +}; + struct nvkm_chan_func { void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); @@ -26,6 +35,11 @@ int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, u32 engm, int bar, u32 base, u32 user, const struct nvkm_oclass *, struct nvkm_fifo_chan *); void nvkm_chan_del(struct nvkm_chan **); +int nvkm_chan_cctx_get(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx **, + struct nvkm_client * /*TODO: remove need for this */); +void nvkm_chan_cctx_put(struct nvkm_chan *, struct nvkm_cctx **); +struct nvkm_oproxy; +void nvkm_chan_cctx_bind(struct nvkm_chan *, struct nvkm_oproxy *, struct nvkm_cctx *); #define CHAN_PRCLI(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) #define CHAN_PRINT(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:"f, (c)->id, ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index c4f404840806a..8681ff0458877 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -63,10 +63,61 @@ nvkm_uchan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_ struct nvkm_uobj { struct nvkm_oproxy oproxy; struct nvkm_chan *chan; + struct nvkm_cctx *cctx; }; +static int +nvkm_uchan_object_fini_1(struct nvkm_oproxy *oproxy, bool suspend) +{ + struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); + struct nvkm_chan *chan = uobj->chan; + struct nvkm_cctx *cctx = uobj->cctx; + + /* Unbind engine context from channel, if no longer required. */ + if (refcount_dec_and_mutex_lock(&cctx->uses, &chan->cgrp->mutex)) { + nvkm_chan_cctx_bind(chan, oproxy, NULL); + mutex_unlock(&chan->cgrp->mutex); + } + + return 0; +} + +static int +nvkm_uchan_object_init_0(struct nvkm_oproxy *oproxy) +{ + struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); + struct nvkm_chan *chan = uobj->chan; + struct nvkm_cctx *cctx = uobj->cctx; + int ret = 0; + + /* Bind engine context to channel, if it hasn't been already. */ + if (!refcount_inc_not_zero(&cctx->uses)) { + mutex_lock(&chan->cgrp->mutex); + if (!refcount_inc_not_zero(&cctx->uses)) { + if (ret == 0) { + nvkm_chan_cctx_bind(chan, oproxy, cctx); + refcount_set(&cctx->uses, 1); + } + } + mutex_unlock(&chan->cgrp->mutex); + } + + return ret; +} + +static void +nvkm_uchan_object_dtor(struct nvkm_oproxy *oproxy) +{ + struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); + + nvkm_chan_cctx_put(uobj->chan, &uobj->cctx); +} + static const struct nvkm_oproxy_func nvkm_uchan_object = { + .dtor[1] = nvkm_uchan_object_dtor, + .init[0] = nvkm_uchan_object_init_0, + .fini[1] = nvkm_uchan_object_fini_1, }; static int @@ -74,9 +125,18 @@ nvkm_uchan_object_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) { struct nvkm_chan *chan = nvkm_uchan(oclass->parent)->chan; + struct nvkm_cgrp *cgrp = chan->cgrp; + struct nvkm_engn *engn; struct nvkm_uobj *uobj; struct nvkm_oclass _oclass; + int ret; + /* Lookup host engine state for target engine. */ + engn = nvkm_runl_find_engn(engn, cgrp->runl, engn->engine == oclass->engine); + if (WARN_ON(!engn)) + return -EINVAL; + + /* Allocate SW object. */ if (!(uobj = kzalloc(sizeof(*uobj), GFP_KERNEL))) return -ENOMEM; @@ -84,6 +144,12 @@ nvkm_uchan_object_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, uobj->chan = chan; *pobject = &uobj->oproxy.base; + /* Ref. channel context for target engine.*/ + ret = nvkm_chan_cctx_get(chan, engn, &uobj->cctx, oclass->client); + if (ret) + return ret; + + /* Allocate HW object. */ _oclass = *oclass; _oclass.parent = &chan->object; return nvkm_fifo_chan_child_new(&_oclass, argv, argc, &uobj->oproxy.object); -- GitLab From 4a492fd5d26298c82c555f603fe4aa38cf512464 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:30 +1000 Subject: [PATCH 1321/1927] drm/nouveau/fifo: add runlist wait() - adds g8x/turing registers, which were missing before - switches fermi to polled wait, like later hw (see: 4f2fc25c0f8bc...) Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 4 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 5 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 23 +++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 33 +++++++++++-------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 2 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 27 +++++++++++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 3 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 9 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 5 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 14 ++++++-- 15 files changed, 102 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 5a5c374804002..a4cc7f07bb896 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -54,6 +54,10 @@ struct nvkm_fifo { struct nvkm_event event; } nonstall; + struct { + u32 chan_msec; + } timeout; + int nr; struct list_head chan; spinlock_t lock; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 83da63bdf3c46..e08d8d14a19ab 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -319,6 +319,11 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, fifo->func = func; INIT_LIST_HEAD(&fifo->runqs); INIT_LIST_HEAD(&fifo->runls); + /*TODO: Needs to be >CTXSW_TIMEOUT, so RC can recover before this is hit. + * CTXSW_TIMEOUT HW default seems to differ between GPUs, so just a + * large number for now until we support changing it. + */ + fifo->timeout.chan_msec = 10000; spin_lock_init(&fifo->lock); mutex_init(&fifo->mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 7c1db7721ff61..70a2609479b91 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -115,12 +115,19 @@ gf100_runq = { .intr_0_names = gf100_runq_intr_0_names, }; +static bool +gf100_runl_pending(struct nvkm_runl *runl) +{ + return nvkm_rd32(runl->fifo->engine.subdev.device, 0x00227c) & 0x00100000; +} + void gf100_fifo_runlist_commit(struct gf100_fifo *fifo) { struct gf100_fifo_chan *chan; struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; + struct nvkm_runl *runl = nvkm_runl_first(&fifo->base); struct nvkm_memory *cur; int nr = 0; int target; @@ -150,10 +157,7 @@ gf100_fifo_runlist_commit(struct gf100_fifo *fifo) (target << 28)); nvkm_wr32(device, 0x002274, 0x01f00000 | nr); - if (wait_event_timeout(fifo->runlist.wait, - !(nvkm_rd32(device, 0x00227c) & 0x00100000), - msecs_to_jiffies(2000)) == 0) - nvkm_error(subdev, "runlist update timeout\n"); + runl->func->wait(runl); mutex_unlock(&fifo->base.mutex); } @@ -175,6 +179,8 @@ gf100_fifo_runlist_insert(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan) static const struct nvkm_runl_func gf100_runl = { + .wait = nv50_runl_wait, + .pending = gf100_runl_pending, }; static void @@ -558,14 +564,13 @@ gf100_fifo_intr_pbdma(struct nvkm_fifo *fifo) } static void -gf100_fifo_intr_runlist(struct gf100_fifo *fifo) +gf100_fifo_intr_runlist(struct nvkm_fifo *fifo) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 intr = nvkm_rd32(device, 0x002a00); if (intr & 0x10000000) { - wake_up(&fifo->runlist.wait); nvkm_wr32(device, 0x002a00, 0x10000000); intr &= ~0x10000000; } @@ -660,7 +665,7 @@ gf100_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x40000000) { - gf100_fifo_intr_runlist(gf100_fifo(fifo)); + gf100_fifo_intr_runlist(fifo); stat &= ~0x40000000; } @@ -779,8 +784,6 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) if (ret) return ret; - init_waitqueue_head(&fifo->runlist.wait); - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000, 0x1000, false, &fifo->user.mem); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index 82b8467545a8e..8e0c1a5e3c578 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -20,7 +20,6 @@ struct gf100_fifo { struct { struct nvkm_memory *mem[2]; int active; - wait_queue_head_t wait; } runlist; struct { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index de8e3a17d79e2..c816654dc2459 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -197,12 +197,21 @@ gk104_runq = { .intr_0_names = gk104_runq_intr_0_names, }; +bool +gk104_runl_pending(struct nvkm_runl *runl) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + + return nvkm_rd32(device, 0x002284 + (runl->id * 0x08)) & 0x00100000; +} + void gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, struct nvkm_memory *mem, int nr) { struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; + struct nvkm_runl *rl = nvkm_runl_get(&fifo->base, runl, 0); int target; switch (nvkm_memory_target(mem)) { @@ -217,11 +226,7 @@ gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, (target << 28)); nvkm_wr32(device, 0x002274, (runl << 20) | nr); - if (nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x002284 + (runl * 0x08)) & 0x00100000)) - break; - ) < 0) - nvkm_error(subdev, "runlist %d update timeout\n", runl); + rl->func->wait(rl); } void @@ -299,6 +304,8 @@ gk104_fifo_runlist = { static const struct nvkm_runl_func gk104_runl = { + .wait = nv50_runl_wait, + .pending = gk104_runl_pending, }; int @@ -736,15 +743,14 @@ gk104_fifo_intr_dropped_fault(struct nvkm_fifo *fifo) } void -gk104_fifo_intr_runlist(struct gk104_fifo *fifo) +gk104_fifo_intr_runlist(struct nvkm_fifo *fifo) { - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_runl *runl; u32 mask = nvkm_rd32(device, 0x002a00); - while (mask) { - int runl = __ffs(mask); - wake_up(&fifo->runlist[runl].wait); - nvkm_wr32(device, 0x002a00, 1 << runl); - mask &= ~(1 << runl); + + nvkm_runl_foreach_cond(runl, fifo, mask & BIT(runl->id)) { + nvkm_wr32(device, 0x002a00, BIT(runl->id)); } } @@ -810,7 +816,7 @@ gk104_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x40000000) { - gk104_fifo_intr_runlist(gk104_fifo(fifo)); + gk104_fifo_intr_runlist(fifo); stat &= ~0x40000000; } @@ -949,7 +955,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) return ret; } - init_waitqueue_head(&fifo->runlist[i].wait); INIT_LIST_HEAD(&fifo->runlist[i].cgrp); INIT_LIST_HEAD(&fifo->runlist[i].chan); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 13f1efcc9e248..7cff1529ae350 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -30,7 +30,6 @@ struct gk104_fifo { struct { struct nvkm_memory *mem[2]; int next; - wait_queue_head_t wait; struct list_head cgrp; struct list_head chan; u32 engm; @@ -63,7 +62,6 @@ void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *); void gk104_fifo_runlist_update(struct gk104_fifo *, int runl); void gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, struct gk104_fifo_engine_status *status); -void gk104_fifo_intr_runlist(struct gk104_fifo *fifo); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); void gk104_fifo_init(struct nvkm_fifo *base); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 5271f55048542..bcc78b35d4e24 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -60,6 +60,8 @@ gk110_fifo_runlist = { const struct nvkm_runl_func gk110_runl = { + .wait = nv50_runl_wait, + .pending = gk104_runl_pending, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index ecf4e6d948666..b6a2210f42c2a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -54,6 +54,8 @@ gm107_fifo_runlist = { const struct nvkm_runl_func gm107_runl = { + .wait = nv50_runl_wait, + .pending = gk104_runl_pending, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 0ca14b08ca3a6..b810175209d13 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -31,6 +31,8 @@ static const struct nvkm_runl_func gp100_runl = { + .wait = nv50_runl_wait, + .pending = gk104_runl_pending, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 49d7c9e7fb8ad..7ad04e97a82de 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -84,6 +84,8 @@ gv100_fifo_runlist = { static const struct nvkm_runl_func gv100_runl = { + .wait = nv50_runl_wait, + .pending = gk104_runl_pending, }; const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 46013aea97d6c..16e59d8ea3256 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -25,11 +25,12 @@ #include "chid.h" #include "runl.h" +#include +#include + #include "nv50.h" #include "channv50.h" -#include - #include static const struct nvkm_chan_func @@ -74,8 +75,30 @@ nv50_fifo_runlist_update(struct nv50_fifo *fifo) mutex_unlock(&fifo->base.mutex); } +static bool +nv50_runl_pending(struct nvkm_runl *runl) +{ + return nvkm_rd32(runl->fifo->engine.subdev.device, 0x0032ec) & 0x00000100; +} + +int +nv50_runl_wait(struct nvkm_runl *runl) +{ + struct nvkm_fifo *fifo = runl->fifo; + + nvkm_msec(fifo->engine.subdev.device, fifo->timeout.chan_msec, + if (!nvkm_runl_update_pending(runl)) + return 0; + usleep_range(1, 2); + ); + + return -ETIMEDOUT; +} + const struct nvkm_runl_func nv50_runl = { + .wait = nv50_runl_wait, + .pending = nv50_runl_pending, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 7bc7b89ffd189..bab3cfbc5fcf5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -101,6 +101,7 @@ int nv10_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func nv50_runl; +int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; extern const struct nvkm_event_func g84_fifo_nonstall; @@ -123,6 +124,7 @@ int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_runl_ctor(struct nvkm_fifo *); void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32); irqreturn_t gk104_fifo_intr(struct nvkm_inth *); +void gk104_fifo_intr_runlist(struct nvkm_fifo *); void gk104_fifo_intr_chsw(struct nvkm_fifo *); void gk104_fifo_intr_bind(struct nvkm_fifo *); extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; @@ -131,6 +133,7 @@ extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); +bool gk104_runl_pending(struct nvkm_runl *); extern const struct nvkm_runq_func gk104_runq; void gk104_runq_init(struct nvkm_runq *); bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index e4984e1e7c650..0e1d703456b69 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -73,6 +73,15 @@ nvkm_runl_chan_get_chid(struct nvkm_runl *runl, int id, unsigned long *pirqflags return NULL; } +bool +nvkm_runl_update_pending(struct nvkm_runl *runl) +{ + if (!runl->func->pending(runl)) + return false; + + return true; +} + void nvkm_runl_del(struct nvkm_runl *runl) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index e618e326b3c34..7682731c3af63 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -24,6 +24,8 @@ struct nvkm_engn { struct nvkm_runl { const struct nvkm_runl_func { + int (*wait)(struct nvkm_runl *); + bool (*pending)(struct nvkm_runl *); } *func; struct nvkm_fifo *fifo; int id; @@ -50,13 +52,16 @@ struct nvkm_runl *nvkm_runl_get(struct nvkm_fifo *, int runi, u32 addr); struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_engn_func *, enum nvkm_subdev_type, int inst); void nvkm_runl_del(struct nvkm_runl *); +bool nvkm_runl_update_pending(struct nvkm_runl *); struct nvkm_chan *nvkm_runl_chan_get_chid(struct nvkm_runl *, int chid, unsigned long *irqflags); struct nvkm_chan *nvkm_runl_chan_get_inst(struct nvkm_runl *, u64 inst, unsigned long *irqflags); #define nvkm_runl_find_engn(engn,runl,cond) nvkm_list_find(engn, &(runl)->engns, head, (cond)) +#define nvkm_runl_first(fifo) list_first_entry(&(fifo)->runls, struct nvkm_runl, head) #define nvkm_runl_foreach(runl,fifo) list_for_each_entry((runl), &(fifo)->runls, head) +#define nvkm_runl_foreach_cond(runl,fifo,cond) nvkm_list_foreach(runl, &(fifo)->runls, head, (cond)) #define nvkm_runl_foreach_engn(engn,runl) list_for_each_entry((engn), &(runl)->engns, head) #define nvkm_runl_foreach_engn_cond(engn,runl,cond) \ nvkm_list_foreach(engn, &(runl)->engns, head, (cond)) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 47de0b97534a3..f0564fa400b77 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -35,6 +35,14 @@ static const struct nvkm_chan_func tu102_chan = { }; +static bool +tu102_runl_pending(struct nvkm_runl *runl) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + + return nvkm_rd32(device, 0x002b0c + (runl->id * 0x10)) & 0x00008000; +} + static void tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, struct nvkm_memory *mem, int nr) @@ -46,8 +54,6 @@ tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, nvkm_wr32(device, 0x002b00 + (runl * 0x10), lower_32_bits(addr)); nvkm_wr32(device, 0x002b04 + (runl * 0x10), upper_32_bits(addr)); nvkm_wr32(device, 0x002b08 + (runl * 0x10), nr); - - /*XXX: how to wait? can you even wait? */ } static const struct gk104_fifo_runlist_func @@ -60,6 +66,8 @@ tu102_fifo_runlist = { static const struct nvkm_runl_func tu102_runl = { + .wait = nv50_runl_wait, + .pending = tu102_runl_pending, }; static const struct nvkm_enum @@ -319,7 +327,7 @@ tu102_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x40000000) { - gk104_fifo_intr_runlist(gk104_fifo(fifo)); + gk104_fifo_intr_runlist(fifo); stat &= ~0x40000000; } -- GitLab From 3a6bc9c242e10c203a5b083af7823b50b5d63010 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:31 +1000 Subject: [PATCH 1322/1927] drm/nouveau/fifo: add runlist block()/allow() Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 13 +++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 14 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 14 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 2 ++ .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 4 --- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 18 ++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 28 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 6 ++++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 2 ++ 13 files changed, 95 insertions(+), 14 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index f250b0f9e2740..748f3f199b9f5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -211,11 +211,24 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, void nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct nvkm_cctx *cctx) { + struct nvkm_cgrp *cgrp = chan->cgrp; + struct nvkm_runl *runl = cgrp->runl; + + /* Prevent any channel in channel group from being rescheduled, kick them + * off host and any engine(s) they're loaded on. + */ + if (cgrp->hw) + nvkm_runl_block(runl); + /* Update context pointer. */ if (cctx) nvkm_fifo_chan_child_init(nvkm_oproxy(oproxy->object)); else nvkm_fifo_chan_child_fini(nvkm_oproxy(oproxy->object), false); + + /* Resume normal operation. */ + if (cgrp->hw) + nvkm_runl_allow(runl); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 70a2609479b91..fcfc241c8a993 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -115,6 +115,18 @@ gf100_runq = { .intr_0_names = gf100_runq_intr_0_names, }; +static void +gf100_runl_allow(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, 0x002630, engm, 0x00000000); +} + +static void +gf100_runl_block(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, 0x002630, engm, engm); +} + static bool gf100_runl_pending(struct nvkm_runl *runl) { @@ -181,6 +193,8 @@ static const struct nvkm_runl_func gf100_runl = { .wait = nv50_runl_wait, .pending = gf100_runl_pending, + .block = gf100_runl_block, + .allow = gf100_runl_allow, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index c816654dc2459..12aebf83f0909 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -197,6 +197,18 @@ gk104_runq = { .intr_0_names = gk104_runq_intr_0_names, }; +void +gk104_runl_allow(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, 0x002630, BIT(runl->id), 0x00000000); +} + +void +gk104_runl_block(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, 0x002630, BIT(runl->id), BIT(runl->id)); +} + bool gk104_runl_pending(struct nvkm_runl *runl) { @@ -306,6 +318,8 @@ static const struct nvkm_runl_func gk104_runl = { .wait = nv50_runl_wait, .pending = gk104_runl_pending, + .block = gk104_runl_block, + .allow = gk104_runl_allow, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index bcc78b35d4e24..134de3c71ad59 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -62,6 +62,8 @@ const struct nvkm_runl_func gk110_runl = { .wait = nv50_runl_wait, .pending = gk104_runl_pending, + .block = gk104_runl_block, + .allow = gk104_runl_allow, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index b6a2210f42c2a..d3b2aa701f532 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -56,6 +56,8 @@ const struct nvkm_runl_func gm107_runl = { .wait = nv50_runl_wait, .pending = gk104_runl_pending, + .block = gk104_runl_block, + .allow = gk104_runl_allow, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index b810175209d13..4dd3fb04d7e53 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -33,6 +33,8 @@ static const struct nvkm_runl_func gp100_runl = { .wait = nv50_runl_wait, .pending = gk104_runl_pending, + .block = gk104_runl_block, + .allow = gk104_runl_allow, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 34a8e792c0e89..6e74fdc0dc240 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -37,15 +37,12 @@ gv100_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *chan) static int gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid) { - struct nvkm_subdev *subdev = &chan->base.fifo->engine.subdev; - struct nvkm_device *device = subdev->device; const u32 mask = ce ? 0x00020000 : 0x00010000; const u32 data = valid ? mask : 0x00000000; int ret; /* Block runlist to prevent the channel from being rescheduled. */ mutex_lock(&chan->fifo->base.mutex); - nvkm_mask(device, 0x002630, BIT(chan->runl), BIT(chan->runl)); /* Preempt the channel. */ ret = gk104_fifo_gpfifo_kick_locked(chan); @@ -57,7 +54,6 @@ gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid } /* Resume runlist. */ - nvkm_mask(device, 0x002630, BIT(chan->runl), 0); mutex_unlock(&chan->fifo->base.mutex); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 7ad04e97a82de..c147c26b80ca8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -86,6 +86,8 @@ static const struct nvkm_runl_func gv100_runl = { .wait = nv50_runl_wait, .pending = gk104_runl_pending, + .block = gk104_runl_block, + .allow = gk104_runl_allow, }; const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 85f22fce11eda..446e65dcb0369 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -129,14 +129,13 @@ nv04_engn = { }; void -nv04_fifo_pause(struct nvkm_fifo *base, unsigned long *pflags) -__acquires(fifo->base.lock) +nv04_fifo_pause(struct nvkm_fifo *fifo, unsigned long *pflags) +__acquires(fifo->lock) { - struct nv04_fifo *fifo = nv04_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; unsigned long flags; - spin_lock_irqsave(&fifo->base.lock, flags); + spin_lock_irqsave(&fifo->lock, flags); *pflags = flags; nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000000); @@ -165,17 +164,16 @@ __acquires(fifo->base.lock) } void -nv04_fifo_start(struct nvkm_fifo *base, unsigned long *pflags) -__releases(fifo->base.lock) +nv04_fifo_start(struct nvkm_fifo *fifo, unsigned long *pflags) +__releases(fifo->lock) { - struct nv04_fifo *fifo = nv04_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; unsigned long flags = *pflags; nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0x00000001); nvkm_wr32(device, NV03_PFIFO_CACHES, 0x00000001); - spin_unlock_irqrestore(&fifo->base.lock, flags); + spin_unlock_irqrestore(&fifo->lock, flags); } const struct nvkm_runl_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index bab3cfbc5fcf5..16fe77ee4c867 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -134,6 +134,8 @@ extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); bool gk104_runl_pending(struct nvkm_runl *); +void gk104_runl_block(struct nvkm_runl *, u32); +void gk104_runl_allow(struct nvkm_runl *, u32); extern const struct nvkm_runq_func gk104_runq; void gk104_runq_init(struct nvkm_runq *); bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 0e1d703456b69..438e884b81005 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -82,6 +82,34 @@ nvkm_runl_update_pending(struct nvkm_runl *runl) return true; } +void +nvkm_runl_allow(struct nvkm_runl *runl) +{ + struct nvkm_fifo *fifo = runl->fifo; + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + if (!--runl->blocked) { + RUNL_TRACE(runl, "running"); + runl->func->allow(runl, ~0); + } + spin_unlock_irqrestore(&fifo->lock, flags); +} + +void +nvkm_runl_block(struct nvkm_runl *runl) +{ + struct nvkm_fifo *fifo = runl->fifo; + unsigned long flags; + + spin_lock_irqsave(&fifo->lock, flags); + if (!runl->blocked++) { + RUNL_TRACE(runl, "stopped"); + runl->func->block(runl, ~0); + } + spin_unlock_irqrestore(&fifo->lock, flags); +} + void nvkm_runl_del(struct nvkm_runl *runl) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 7682731c3af63..aebd8a750d38c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -26,6 +26,8 @@ struct nvkm_runl { const struct nvkm_runl_func { int (*wait)(struct nvkm_runl *); bool (*pending)(struct nvkm_runl *); + void (*block)(struct nvkm_runl *, u32 engm); + void (*allow)(struct nvkm_runl *, u32 engm); } *func; struct nvkm_fifo *fifo; int id; @@ -44,6 +46,8 @@ struct nvkm_runl { int chan_nr; struct mutex mutex; + int blocked; + struct list_head head; }; @@ -52,6 +56,8 @@ struct nvkm_runl *nvkm_runl_get(struct nvkm_fifo *, int runi, u32 addr); struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_engn_func *, enum nvkm_subdev_type, int inst); void nvkm_runl_del(struct nvkm_runl *); +void nvkm_runl_block(struct nvkm_runl *); +void nvkm_runl_allow(struct nvkm_runl *); bool nvkm_runl_update_pending(struct nvkm_runl *); struct nvkm_chan *nvkm_runl_chan_get_chid(struct nvkm_runl *, int chid, unsigned long *irqflags); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index f0564fa400b77..7d3c9d8e54a74 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -68,6 +68,8 @@ static const struct nvkm_runl_func tu102_runl = { .wait = nv50_runl_wait, .pending = tu102_runl_pending, + .block = gk104_runl_block, + .allow = gk104_runl_allow, }; static const struct nvkm_enum -- GitLab From 62742b5ef314c622ae9d848938223071ba360706 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:31 +1000 Subject: [PATCH 1323/1927] drm/nouveau/fifo: add chan bind()/unbind() - stops programming (non-existent) runl id field on bind(), from maxwell Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 3 ++ .../drm/nouveau/nvkm/engine/fifo/chang84.c | 15 +------- .../drm/nouveau/nvkm/engine/fifo/channv50.c | 6 ++-- .../drm/nouveau/nvkm/engine/fifo/channv50.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 13 +++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 35 ++++++++++++++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 28 +++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 ++ .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 7 ---- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 6 ---- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 19 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 4 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 2 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 6 ++++ 17 files changed, 115 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index bf1c3b580defd..fe398ed544cf0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -16,6 +16,9 @@ struct nvkm_cctx { }; struct nvkm_chan_func { + void (*bind)(struct nvkm_chan *); + void (*unbind)(struct nvkm_chan *); + void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); void (*fini)(struct nvkm_fifo_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 903b54c07750e..100631a44a85e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -166,23 +166,10 @@ g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, return nvkm_ramht_insert(chan->ramht, object, 0, 4, handle, context); } -static void -g84_fifo_chan_init(struct nvkm_fifo_chan *base) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nv50_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u64 addr = chan->ramfc->addr >> 8; - u32 chid = chan->base.chid; - - nvkm_wr32(device, 0x002600 + (chid * 4), 0x80000000 | addr); - nv50_fifo_runlist_update(fifo); -} - static const struct nvkm_fifo_chan_func g84_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .init = g84_fifo_chan_init, + .init = nv50_fifo_chan_init, .fini = nv50_fifo_chan_fini, .engine_ctor = g84_fifo_chan_engine_ctor, .engine_dtor = nv50_fifo_chan_engine_dtor, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index c44d7c81dd52a..0fc97c4c341ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -194,19 +194,17 @@ nv50_fifo_chan_fini(struct nvkm_fifo_chan *base) /* remove channel from runlist, fifo will unload context */ nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x00000000); nv50_fifo_runlist_update(fifo); - nvkm_wr32(device, 0x002600 + (chid * 4), 0x00000000); } -static void +void nv50_fifo_chan_init(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); struct nv50_fifo *fifo = chan->fifo; struct nvkm_device *device = fifo->base.engine.subdev.device; - u64 addr = chan->ramfc->addr >> 12; u32 chid = chan->base.chid; - nvkm_wr32(device, 0x002600 + (chid * 4), 0x80000000 | addr); + nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x80000000); nv50_fifo_runlist_update(fifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index 3a95730d7ff50..5d5d9f3d99280 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -40,6 +40,7 @@ struct nv50_fifo_chan { int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *); +void nv50_fifo_chan_init(struct nvkm_fifo_chan *); void nv50_fifo_chan_fini(struct nvkm_fifo_chan *); struct nvkm_gpuobj **nv50_fifo_chan_engine(struct nv50_fifo_chan *, struct nvkm_engine *); void nv50_fifo_chan_engine_dtor(struct nvkm_fifo_chan *, struct nvkm_engine *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index afaa2dea4ef84..2777f03ffc299 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -21,16 +21,29 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" #include "chan.h" #include "runl.h" +#include + #include "nv50.h" #include "channv50.h" #include +static void +g84_chan_bind(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x002600 + (chan->id * 4), nv50_fifo_chan(chan)->ramfc->addr >> 8); +} + const struct nvkm_chan_func g84_chan = { + .bind = g84_chan_bind, + .unbind = nv50_chan_unbind, }; const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index fcfc241c8a993..50109f4f3860d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" @@ -38,8 +39,32 @@ #include +static void gf100_fifo_intr_engine(struct nvkm_fifo *); + +static void +gf100_chan_unbind(struct nvkm_chan *chan) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; + + /*TODO: Is this cargo-culted, or necessary? RM does *something* here... Why? */ + gf100_fifo_intr_engine(fifo); + + nvkm_wr32(device, 0x003000 + (chan->id * 8), 0x00000000); +} + +static void +gf100_chan_bind(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x003000 + (chan->id * 8), 0xc0000000 | chan->inst->addr >> 12); +} + static const struct nvkm_chan_func gf100_chan = { + .bind = gf100_chan_bind, + .unbind = gf100_chan_unbind, }; static const struct nvkm_engn_func @@ -619,15 +644,15 @@ gf100_fifo_intr_engine_unit(struct nvkm_fifo *fifo, int engn) } } -void -gf100_fifo_intr_engine(struct gf100_fifo *fifo) +static void +gf100_fifo_intr_engine(struct nvkm_fifo *fifo) { - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; u32 mask = nvkm_rd32(device, 0x0025a4); while (mask) { u32 unit = __ffs(mask); - gf100_fifo_intr_engine_unit(&fifo->base, unit); + gf100_fifo_intr_engine_unit(fifo, unit); mask &= ~(1 << unit); } } @@ -684,7 +709,7 @@ gf100_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x80000000) { - gf100_fifo_intr_engine(gf100_fifo(fifo)); + gf100_fifo_intr_engine(fifo); stat &= ~0x80000000; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index 8e0c1a5e3c578..16268e81077d7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -28,7 +28,6 @@ struct gf100_fifo { } user; }; -void gf100_fifo_intr_engine(struct gf100_fifo *); void gf100_fifo_runlist_insert(struct gf100_fifo *, struct gf100_fifo_chan *); void gf100_fifo_runlist_remove(struct gf100_fifo *, struct gf100_fifo_chan *); void gf100_fifo_runlist_commit(struct gf100_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 12aebf83f0909..fafe9453ab0c3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -38,8 +38,36 @@ #include +void +gk104_chan_unbind(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x800000 + (chan->id * 8), 0x00000000); +} + +void +gk104_chan_bind_inst(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x800000 + (chan->id * 8), 0x80000000 | chan->inst->addr >> 12); +} + +void +gk104_chan_bind(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x800004 + (chan->id * 8), 0x000f0000, runl->id << 16); + gk104_chan_bind_inst(chan); +} + static const struct nvkm_chan_func gk104_chan = { + .bind = gk104_chan_bind, + .unbind = gk104_chan_unbind, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 134de3c71ad59..1724937a2f5fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -35,6 +35,8 @@ const struct nvkm_chan_func gk110_chan = { + .bind = gk104_chan_bind, + .unbind = gk104_chan_unbind, }; const struct nvkm_cgrp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index d3b2aa701f532..742404bf84152 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -34,6 +34,8 @@ const struct nvkm_chan_func gm107_chan = { + .bind = gk104_chan_bind_inst, + .unbind = gk104_chan_unbind, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 5d6e3a7b8f1e4..93739b34bfcdc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -157,10 +157,6 @@ gf100_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) nvkm_mask(device, 0x003004 + coff, 0x00000001, 0x00000000); gf100_fifo_runlist_commit(fifo); } - - gf100_fifo_intr_engine(fifo); - - nvkm_wr32(device, 0x003000 + coff, 0x00000000); } static void @@ -169,11 +165,8 @@ gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base) struct gf100_fifo_chan *chan = gf100_fifo_chan(base); struct gf100_fifo *fifo = chan->fifo; struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 addr = chan->base.inst->addr >> 12; u32 coff = chan->base.chid * 8; - nvkm_wr32(device, 0x003000 + coff, 0xc0000000 | addr); - if (list_empty(&chan->head) && !chan->killed) { gf100_fifo_runlist_insert(fifo, chan); nvkm_wr32(device, 0x003004 + coff, 0x001f0001); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 6415e80a1fbf1..c2b5eaa9cdea4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -208,8 +208,6 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) gk104_fifo_gpfifo_kick(chan); gk104_fifo_runlist_update(fifo, chan->runl); } - - nvkm_wr32(device, 0x800000 + coff, 0x00000000); } void @@ -218,12 +216,8 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) struct gk104_fifo_chan *chan = gk104_fifo_chan(base); struct gk104_fifo *fifo = chan->fifo; struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 addr = chan->base.inst->addr >> 12; u32 coff = chan->base.chid * 8; - nvkm_mask(device, 0x800004 + coff, 0x000f0000, chan->runl << 16); - nvkm_wr32(device, 0x800000 + coff, 0x80000000 | addr); - if (list_empty(&chan->head) && !chan->killed) { gk104_fifo_runlist_insert(fifo, chan); nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index c147c26b80ca8..eb59527544b28 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -33,6 +33,8 @@ static const struct nvkm_chan_func gv100_chan = { + .bind = gk104_chan_bind_inst, + .unbind = gk104_chan_unbind, }; const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 16e59d8ea3256..d7adb057e4ca8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" @@ -33,8 +34,26 @@ #include +void +nv50_chan_unbind(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x002600 + (chan->id * 4), 0x00000000); +} + +static void +nv50_chan_bind(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x002600 + (chan->id * 4), nv50_fifo_chan(chan)->ramfc->addr >> 12); +} + static const struct nvkm_chan_func nv50_chan = { + .bind = nv50_chan_bind, + .unbind = nv50_chan_unbind, }; static const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 16fe77ee4c867..f023aa0a559e3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -103,6 +103,7 @@ int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func nv50_runl; int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; +void nv50_chan_unbind(struct nvkm_chan *); extern const struct nvkm_event_func g84_fifo_nonstall; extern const struct nvkm_engn_func g84_engn; @@ -142,6 +143,9 @@ bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); extern const struct nvkm_bitfield gk104_runq_intr_0_names[]; extern const struct nvkm_engn_func gk104_engn; extern const struct nvkm_engn_func gk104_engn_ce; +void gk104_chan_bind(struct nvkm_chan *); +void gk104_chan_bind_inst(struct nvkm_chan *); +void gk104_chan_unbind(struct nvkm_chan *); int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func gk110_runl; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 7d3c9d8e54a74..b23fc330aa6cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -33,6 +33,8 @@ static const struct nvkm_chan_func tu102_chan = { + .bind = gk104_chan_bind_inst, + .unbind = gk104_chan_unbind, }; static bool diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 8681ff0458877..c1d2035e00cf2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -217,6 +217,9 @@ nvkm_uchan_fini(struct nvkm_object *object, bool suspend) if (ret && suspend) return ret; + if (chan->func->unbind) + chan->func->unbind(chan); + return 0; } @@ -225,6 +228,9 @@ nvkm_uchan_init(struct nvkm_object *object) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; + if (chan->func->bind) + chan->func->bind(chan); + return chan->object.func->init(&chan->object); } -- GitLab From 67059b9fb8997f3d4515d72052c331503b00274b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:32 +1000 Subject: [PATCH 1324/1927] drm/nouveau/fifo: add chan start()/stop() - nvkm_chan_error() built on top, stops channel and sends 'killed' event - removes an odd double-bashing of channel enable regs on kepler and up - pokes doorbell on turing and up, after enabling channel Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 13 +--- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 73 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 7 +- .../drm/nouveau/nvkm/engine/fifo/channv04.h | 2 - .../drm/nouveau/nvkm/engine/fifo/channv50.c | 7 -- .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 2 - .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 22 +++++- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 24 ++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 + .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 6 -- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 7 -- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 9 +-- .../nouveau/nvkm/engine/fifo/gpfifotu102.c | 8 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 9 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 27 ++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 18 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 25 +++++-- .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 11 ++- 27 files changed, 191 insertions(+), 108 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index a4cc7f07bb896..798fbd3b530cb 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -20,6 +20,10 @@ struct nvkm_chan { union { int id; int chid; }; /*FIXME: remove later */ + spinlock_t lock; + atomic_t blocked; + atomic_t errored; + struct list_head cctxs; struct nvkm_fifo *fifo; @@ -62,9 +66,6 @@ struct nvkm_fifo { struct list_head chan; spinlock_t lock; struct mutex mutex; - -#define NVKM_FIFO_EVENT_KILLED BIT(0) - struct nvkm_event kevent; /* channel killed */ }; void nvkm_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index e08d8d14a19ab..549e4b1fcbe14 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -62,16 +62,6 @@ nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info) return fifo->func->mmu_fault->recover(fifo, info); } -void -nvkm_fifo_kevent(struct nvkm_fifo *fifo, int chid) -{ - nvkm_event_ntfy(&fifo->kevent, chid, NVKM_FIFO_EVENT_KILLED); -} - -static const struct nvkm_event_func -nvkm_fifo_kevent_func = { -}; - static int nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) @@ -293,7 +283,6 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) if (fifo->func->dtor) data = fifo->func->dtor(fifo); - nvkm_event_fini(&fifo->kevent); nvkm_event_fini(&fifo->nonstall.event); mutex_destroy(&fifo->mutex); return data; @@ -343,5 +332,5 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, return ret; } - return nvkm_event_init(&nvkm_fifo_kevent_func, &fifo->engine.subdev, 1, nr, &fifo->kevent); + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 748f3f199b9f5..4f11448ed7209 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -24,6 +24,7 @@ #include "chan.h" #include "chid.h" #include "cgrp.h" +#include "chid.h" #include "runl.h" #include "priv.h" @@ -219,6 +220,8 @@ nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct n */ if (cgrp->hw) nvkm_runl_block(runl); + else + nvkm_chan_block(chan); /* Update context pointer. */ if (cctx) @@ -229,6 +232,8 @@ nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct n /* Resume normal operation. */ if (cgrp->hw) nvkm_runl_allow(runl); + else + nvkm_chan_allow(chan); } void @@ -295,23 +300,6 @@ nvkm_chan_cctx_get(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_c return ret; } -static int -nvkm_fifo_chan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_uevent *uevent) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - union nvif_chan_event_args *args = argv; - - switch (args->v0.type) { - case NVIF_CHAN_EVENT_V0_KILLED: - return nvkm_uevent_add(uevent, &chan->fifo->kevent, chan->chid, - NVKM_FIFO_EVENT_KILLED, NULL); - default: - break; - } - - return -ENOSYS; -} - static int nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) @@ -327,7 +315,8 @@ static int nvkm_fifo_chan_fini(struct nvkm_object *object, bool suspend) { struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - chan->func->fini(chan); + if (chan->func->fini) + chan->func->fini(chan); return 0; } @@ -335,10 +324,51 @@ static int nvkm_fifo_chan_init(struct nvkm_object *object) { struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - chan->func->init(chan); + if (chan->func->init) + chan->func->init(chan); return 0; } +static void +nvkm_chan_block_locked(struct nvkm_chan *chan) +{ + CHAN_TRACE(chan, "block %d", atomic_read(&chan->blocked)); + if (atomic_inc_return(&chan->blocked) == 1) + chan->func->stop(chan); +} + +void +nvkm_chan_error(struct nvkm_chan *chan, bool preempt) +{ + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + if (atomic_inc_return(&chan->errored) == 1) { + CHAN_ERROR(chan, "errored - disabling channel"); + nvkm_chan_block_locked(chan); + nvkm_event_ntfy(&chan->cgrp->runl->chid->event, chan->id, NVKM_CHAN_EVENT_ERRORED); + } + spin_unlock_irqrestore(&chan->lock, flags); +} + +void +nvkm_chan_block(struct nvkm_chan *chan) +{ + spin_lock_irq(&chan->lock); + nvkm_chan_block_locked(chan); + spin_unlock_irq(&chan->lock); +} + +void +nvkm_chan_allow(struct nvkm_chan *chan) +{ + spin_lock_irq(&chan->lock); + CHAN_TRACE(chan, "allow %d", atomic_read(&chan->blocked)); + if (atomic_dec_and_test(&chan->blocked)) + chan->func->start(chan); + spin_unlock_irq(&chan->lock); +} + void nvkm_chan_del(struct nvkm_chan **pchan) { @@ -437,7 +467,6 @@ nvkm_fifo_chan_func = { .init = nvkm_fifo_chan_init, .fini = nvkm_fifo_chan_fini, .map = nvkm_fifo_chan_map, - .uevent = nvkm_fifo_chan_uevent, }; int @@ -481,10 +510,12 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, func->engine_fini = fn->engine_fini; func->object_ctor = fn->object_ctor; func->object_dtor = fn->object_dtor; - func->submit_token = fn->submit_token; chan->func = func; chan->id = -1; + spin_lock_init(&chan->lock); + atomic_set(&chan->blocked, 1); + atomic_set(&chan->errored, 0); nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); chan->fifo = fifo; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index fe398ed544cf0..d573d9fdb4d80 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -18,6 +18,9 @@ struct nvkm_cctx { struct nvkm_chan_func { void (*bind)(struct nvkm_chan *); void (*unbind)(struct nvkm_chan *); + void (*start)(struct nvkm_chan *); + void (*stop)(struct nvkm_chan *); + u32 (*doorbell_handle)(struct nvkm_chan *); void *(*dtor)(struct nvkm_fifo_chan *); void (*init)(struct nvkm_fifo_chan *); @@ -30,7 +33,6 @@ struct nvkm_chan_func { bool suspend); int (*object_ctor)(struct nvkm_fifo_chan *, struct nvkm_object *); void (*object_dtor)(struct nvkm_fifo_chan *, int); - u32 (*submit_token)(struct nvkm_fifo_chan *); }; int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, @@ -38,6 +40,9 @@ int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, u32 engm, int bar, u32 base, u32 user, const struct nvkm_oclass *, struct nvkm_fifo_chan *); void nvkm_chan_del(struct nvkm_chan **); +void nvkm_chan_allow(struct nvkm_chan *); +void nvkm_chan_block(struct nvkm_chan *); +void nvkm_chan_error(struct nvkm_chan *, bool preempt); int nvkm_chan_cctx_get(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx **, struct nvkm_client * /*TODO: remove need for this */); void nvkm_chan_cctx_put(struct nvkm_chan *, struct nvkm_cctx **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h index 727bc8976b401..837f9f7f69cee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h @@ -18,8 +18,6 @@ struct nv04_fifo_chan { extern const struct nvkm_fifo_chan_func nv04_fifo_dma_func; void *nv04_fifo_dma_dtor(struct nvkm_fifo_chan *); -void nv04_fifo_dma_init(struct nvkm_fifo_chan *); -void nv04_fifo_dma_fini(struct nvkm_fifo_chan *); void nv04_fifo_dma_object_dtor(struct nvkm_fifo_chan *, int); extern const struct nvkm_fifo_chan_oclass nv04_fifo_dma_oclass; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 0fc97c4c341ad..7669d17406c69 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -188,11 +188,7 @@ nv50_fifo_chan_fini(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); struct nv50_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 chid = chan->base.chid; - /* remove channel from runlist, fifo will unload context */ - nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x00000000); nv50_fifo_runlist_update(fifo); } @@ -201,10 +197,7 @@ nv50_fifo_chan_init(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); struct nv50_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 chid = chan->base.chid; - nvkm_mask(device, 0x002600 + (chid * 4), 0x80000000, 0x80000000); nv50_fifo_runlist_update(fifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index ed2ae9e699874..d515cf08fc3bb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -88,8 +88,6 @@ nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) const struct nvkm_fifo_chan_func nv04_fifo_dma_func = { .dtor = nv04_fifo_dma_dtor, - .init = nv04_fifo_dma_init, - .fini = nv04_fifo_dma_fini, .object_ctor = nv04_fifo_dma_object_ctor, .object_dtor = nv04_fifo_dma_object_dtor, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c index c42c2e01d8011..2f0dceaf3b018 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c @@ -176,8 +176,6 @@ nv40_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, static const struct nvkm_fifo_chan_func nv40_fifo_dma_func = { .dtor = nv04_fifo_dma_dtor, - .init = nv04_fifo_dma_init, - .fini = nv04_fifo_dma_fini, .engine_ctor = nv40_fifo_dma_engine_ctor, .engine_dtor = nv40_fifo_dma_engine_dtor, .engine_init = nv40_fifo_dma_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 2777f03ffc299..91b43f17120b7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -44,6 +44,8 @@ const struct nvkm_chan_func g84_chan = { .bind = g84_chan_bind, .unbind = nv50_chan_unbind, + .start = nv50_chan_start, + .stop = nv50_chan_stop, }; const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 50109f4f3860d..e47ca113e9e85 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -39,6 +39,22 @@ #include +static void +gf100_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x003004 + (chan->id * 8), 0x00000001, 0x00000000); +} + +static void +gf100_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x003004 + (chan->id * 8), 0x001f0001); +} + static void gf100_fifo_intr_engine(struct nvkm_fifo *); static void @@ -65,6 +81,8 @@ static const struct nvkm_chan_func gf100_chan = { .bind = gf100_chan_bind, .unbind = gf100_chan_unbind, + .start = gf100_chan_start, + .stop = gf100_chan_stop, }; static const struct nvkm_engn_func @@ -321,7 +339,6 @@ gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine, struct gf100_fifo_chan *chan) { struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; u32 chid = chan->base.chid; int engi = gf100_fifo_engine_id(&fifo->base, engine); @@ -329,14 +346,13 @@ gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine, engine->subdev.name, chid); assert_spin_locked(&fifo->base.lock); - nvkm_mask(device, 0x003004 + (chid * 0x08), 0x00000001, 0x00000000); + nvkm_chan_error(&chan->base, false); list_del_init(&chan->head); chan->killed = true; if (engi >= 0 && engi != GF100_FIFO_ENGN_SW) fifo->recover.mask |= BIT(engi); schedule_work(&fifo->recover.work); - nvkm_fifo_kevent(&fifo->base, chid); } static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index fafe9453ab0c3..5ba922cf3c2c8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -38,6 +38,22 @@ #include +void +gk104_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x800004 + (chan->id * 8), 0x00000800, 0x00000800); +} + +void +gk104_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x800004 + (chan->id * 8), 0x00000400, 0x00000400); +} + void gk104_chan_unbind(struct nvkm_chan *chan) { @@ -68,6 +84,8 @@ static const struct nvkm_chan_func gk104_chan = { .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, }; void @@ -469,13 +487,9 @@ gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid) chan = gk104_fifo_recover_chid(fifo, runl, chid); if (chan) { chan->killed = true; - nvkm_fifo_kevent(&fifo->base, chid); + nvkm_chan_error(&chan->base, false); } - /* Disable channel. */ - nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); - nvkm_warn(subdev, "channel %d: killed\n", chid); - /* Block channel assignments from changing during recovery. */ gk104_fifo_recover_runl(fifo, runl); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 1724937a2f5fb..3582b9f55611b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -37,6 +37,8 @@ const struct nvkm_chan_func gk110_chan = { .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, }; const struct nvkm_cgrp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 742404bf84152..2924ce10c3373 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -36,6 +36,8 @@ const struct nvkm_chan_func gm107_chan = { .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 93739b34bfcdc..29783d271f1e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -149,12 +149,9 @@ gf100_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); struct gf100_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (!list_empty(&chan->head) && !chan->killed) { gf100_fifo_runlist_remove(fifo, chan); - nvkm_mask(device, 0x003004 + coff, 0x00000001, 0x00000000); gf100_fifo_runlist_commit(fifo); } } @@ -164,12 +161,9 @@ gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base) { struct gf100_fifo_chan *chan = gf100_fifo_chan(base); struct gf100_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (list_empty(&chan->head) && !chan->killed) { gf100_fifo_runlist_insert(fifo, chan); - nvkm_wr32(device, 0x003004 + coff, 0x001f0001); gf100_fifo_runlist_commit(fifo); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index c2b5eaa9cdea4..1847b1b7af8bb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -199,12 +199,9 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); struct gk104_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (!list_empty(&chan->head)) { gk104_fifo_runlist_remove(fifo, chan); - nvkm_mask(device, 0x800004 + coff, 0x00000800, 0x00000800); gk104_fifo_gpfifo_kick(chan); gk104_fifo_runlist_update(fifo, chan->runl); } @@ -215,14 +212,10 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); struct gk104_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 coff = chan->base.chid * 8; if (list_empty(&chan->head) && !chan->killed) { gk104_fifo_runlist_insert(fifo, chan); - nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); gk104_fifo_runlist_update(fifo, chan->runl); - nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 6e74fdc0dc240..a901ce269e06a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -28,12 +28,6 @@ #include #include -static u32 -gv100_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *chan) -{ - return chan->chid; -} - static int gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid) { @@ -125,7 +119,6 @@ gv100_fifo_gpfifo = { .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, .engine_fini = gv100_fifo_gpfifo_engine_fini, - .submit_token = gv100_fifo_gpfifo_submit_token, }; int @@ -159,7 +152,7 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, *chid = chan->base.chid; *inst = chan->base.inst->addr; - *token = chan->base.func->submit_token(&chan->base); + *token = chan->base.func->doorbell_handle(&chan->base); /* Hack to support GPUs where even individual channels should be * part of a channel group. diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index 4d35bee0ab54b..1c48a109a2b8a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -28,13 +28,6 @@ #include #include -static u32 -tu102_fifo_gpfifo_submit_token(struct nvkm_fifo_chan *base) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - return (chan->runl << 16) | chan->base.chid; -} - static const struct nvkm_fifo_chan_func tu102_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, @@ -44,7 +37,6 @@ tu102_fifo_gpfifo = { .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, .engine_fini = gv100_fifo_gpfifo_engine_fini, - .submit_token = tu102_fifo_gpfifo_submit_token, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index eb59527544b28..0e9e070201b16 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -31,10 +31,19 @@ #include +static u32 +gv100_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return chan->id; +} + static const struct nvkm_chan_func gv100_chan = { .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, + .start = gk104_chan_start, + .stop = gk104_chan_stop, + .doorbell_handle = gv100_chan_doorbell_handle, }; const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 446e65dcb0369..a0a28a81f7f4c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -52,15 +52,14 @@ nv04_fifo_ramfc[] = { }; void -nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) +nv04_chan_stop(struct nvkm_chan *chan) { - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; + struct nv04_fifo *fifo = nv04_fifo(chan->cgrp->runl->fifo); struct nvkm_device *device = fifo->base.engine.subdev.device; struct nvkm_memory *fctx = device->imem->ramfc; const struct nv04_fifo_ramfc *c; unsigned long flags; - u32 data = chan->ramfc; + u32 data = nv04_fifo_chan(chan)->ramfc; u32 chid; /* prevent fifo context switches */ @@ -69,7 +68,7 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) /* if this channel is active, replace it with a null context */ chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; - if (chid == chan->base.chid) { + if (chid == chan->id) { nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); @@ -98,26 +97,26 @@ nv04_fifo_dma_fini(struct nvkm_fifo_chan *base) } /* restore normal operation, after disabling dma mode */ - nvkm_mask(device, NV04_PFIFO_MODE, 1 << chan->base.chid, 0); + nvkm_mask(device, NV04_PFIFO_MODE, BIT(chan->id), 0); nvkm_wr32(device, NV03_PFIFO_CACHES, 1); spin_unlock_irqrestore(&fifo->base.lock, flags); } void -nv04_fifo_dma_init(struct nvkm_fifo_chan *base) +nv04_chan_start(struct nvkm_chan *chan) { - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - u32 mask = 1 << chan->base.chid; + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; unsigned long flags; - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_mask(device, NV04_PFIFO_MODE, mask, mask); - spin_unlock_irqrestore(&fifo->base.lock, flags); + + spin_lock_irqsave(&fifo->lock, flags); + nvkm_mask(fifo->engine.subdev.device, NV04_PFIFO_MODE, BIT(chan->id), BIT(chan->id)); + spin_unlock_irqrestore(&fifo->lock, flags); } static const struct nvkm_chan_func nv04_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; const struct nvkm_cgrp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index 0f7c88ae5a1c6..8d302cb8ba8c5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -45,6 +45,8 @@ nv10_fifo_ramfc[] = { static const struct nvkm_chan_func nv10_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 59b752986ac1d..7b1b90d285c14 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -54,6 +54,8 @@ nv17_fifo_ramfc[] = { static const struct nvkm_chan_func nv17_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index d0fc6c528825e..19f7c0407e70e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -64,6 +64,8 @@ nv40_fifo_ramfc[] = { static const struct nvkm_chan_func nv40_chan = { + .start = nv04_chan_start, + .stop = nv04_chan_stop, }; static const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index d7adb057e4ca8..38d36749862c0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -34,6 +34,22 @@ #include +void +nv50_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x00000000); +} + +void +nv50_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + nvkm_mask(device, 0x002600 + (chan->id * 4), 0x80000000, 0x80000000); +} + void nv50_chan_unbind(struct nvkm_chan *chan) { @@ -54,6 +70,8 @@ static const struct nvkm_chan_func nv50_chan = { .bind = nv50_chan_bind, .unbind = nv50_chan_unbind, + .start = nv50_chan_start, + .stop = nv50_chan_stop, }; static const struct nvkm_engn_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index f023aa0a559e3..d9de5ab264521 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -11,7 +11,6 @@ struct nvkm_runq; struct gk104_fifo; struct gk104_fifo_chan; -void nvkm_fifo_kevent(struct nvkm_fifo *, int chid); void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid); struct nvkm_fifo_chan_oclass; @@ -95,6 +94,8 @@ void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); extern const struct nvkm_runl_func nv04_runl; extern const struct nvkm_engn_func nv04_engn; extern const struct nvkm_cgrp_func nv04_cgrp; +void nv04_chan_start(struct nvkm_chan *); +void nv04_chan_stop(struct nvkm_chan *); int nv10_fifo_chid_nr(struct nvkm_fifo *); @@ -104,6 +105,8 @@ extern const struct nvkm_runl_func nv50_runl; int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; void nv50_chan_unbind(struct nvkm_chan *); +void nv50_chan_start(struct nvkm_chan *); +void nv50_chan_stop(struct nvkm_chan *); extern const struct nvkm_event_func g84_fifo_nonstall; extern const struct nvkm_engn_func g84_engn; @@ -146,6 +149,8 @@ extern const struct nvkm_engn_func gk104_engn_ce; void gk104_chan_bind(struct nvkm_chan *); void gk104_chan_bind_inst(struct nvkm_chan *); void gk104_chan_unbind(struct nvkm_chan *); +void gk104_chan_start(struct nvkm_chan *); +void gk104_chan_stop(struct nvkm_chan *); int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func gk110_runl; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index aebd8a750d38c..35dce7c831242 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -34,6 +34,7 @@ struct nvkm_runl { u32 addr; struct nvkm_chid *cgid; +#define NVKM_CHAN_EVENT_ERRORED BIT(0) struct nvkm_chid *chid; struct list_head engns; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index b23fc330aa6cb..4ae8544a22cf5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -28,13 +28,32 @@ #include #include +#include #include +static u32 +tu102_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return (chan->cgrp->runl->id << 16) | chan->id; +} + +static void +tu102_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + + gk104_chan_start(chan); + nvkm_wr32(device, device->vfn->addr.user + 0x0090, chan->func->doorbell_handle(chan)); +} + static const struct nvkm_chan_func tu102_chan = { .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, + .start = tu102_chan_start, + .stop = gk104_chan_stop, + .doorbell_handle = tu102_chan_doorbell_handle, }; static bool @@ -202,13 +221,9 @@ tu102_fifo_recover_chan(struct nvkm_fifo *base, int chid) chan = tu102_fifo_recover_chid(fifo, runl, chid); if (chan) { chan->killed = true; - nvkm_fifo_kevent(&fifo->base, chid); + nvkm_chan_error(&chan->base, false); } - /* Disable channel. */ - nvkm_wr32(device, 0x800004 + (chid * 0x08), stat | 0x00000800); - nvkm_warn(subdev, "channel %d: killed\n", chid); - /* Block channel assignments from changing during recovery. */ tu102_fifo_recover_runl(fifo, runl); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index c1d2035e00cf2..096e09f231777 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -22,6 +22,7 @@ #define nvkm_uchan(p) container_of((p), struct nvkm_uchan, object) #include "cgrp.h" #include "chan.h" +#include "chid.h" #include "runl.h" #include @@ -52,7 +53,8 @@ nvkm_uchan_uevent(struct nvkm_object *object, void *argv, u32 argc, struct nvkm_ return nvkm_uevent_add(uevent, &runl->fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT, NULL); case NVIF_CHAN_EVENT_V0_KILLED: - return chan->object.func->uevent(&chan->object, argv, argc, uevent); + return nvkm_uevent_add(uevent, &runl->chid->event, chan->id, + NVKM_CHAN_EVENT_ERRORED, NULL); default: break; } @@ -213,6 +215,8 @@ nvkm_uchan_fini(struct nvkm_object *object, bool suspend) struct nvkm_chan *chan = nvkm_uchan(object)->chan; int ret; + nvkm_chan_block(chan); + ret = chan->object.func->fini(&chan->object, suspend); if (ret && suspend) return ret; @@ -228,9 +232,14 @@ nvkm_uchan_init(struct nvkm_object *object) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; + if (atomic_read(&chan->errored)) + return 0; + if (chan->func->bind) chan->func->bind(chan); + nvkm_chan_allow(chan); + return chan->object.func->init(&chan->object); } -- GitLab From acff941535639a1d244ad9a20dbf3266d39915ed Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:33 +1000 Subject: [PATCH 1325/1927] drm/nouveau/fifo: add chan/cgrp preempt() Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 11 ----- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 30 +++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 3 ++ .../drm/nouveau/nvkm/engine/fifo/changk104.h | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 14 ++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 23 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 + .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 17 ------- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 45 +------------------ .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 12 +---- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 6 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 12 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 2 + .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 1 + 19 files changed, 102 insertions(+), 86 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 549e4b1fcbe14..1c3c3495a2504 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -33,17 +33,6 @@ #include #include -void -nvkm_fifo_recover_chan(struct nvkm_fifo *fifo, int chid) -{ - unsigned long flags; - if (WARN_ON(!fifo->func->recover_chan)) - return; - spin_lock_irqsave(&fifo->lock, flags); - fifo->func->recover_chan(fifo, chid); - spin_unlock_irqrestore(&fifo->lock, flags); -} - void nvkm_fifo_pause(struct nvkm_fifo *fifo, unsigned long *flags) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index 50ba7c7ee0a8f..1440c72ad7ddb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -22,6 +22,7 @@ struct nvkm_ectx { struct nvkm_cgrp { const struct nvkm_cgrp_func { + void (*preempt)(struct nvkm_cgrp *); } *func; char name[64]; struct nvkm_runl *runl; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 4f11448ed7209..9b8f346d52b1a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -222,6 +222,7 @@ nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct n nvkm_runl_block(runl); else nvkm_chan_block(chan); + nvkm_chan_preempt(chan, true); /* Update context pointer. */ if (cctx) @@ -300,6 +301,33 @@ nvkm_chan_cctx_get(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_c return ret; } +int +nvkm_chan_preempt_locked(struct nvkm_chan *chan, bool wait) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + + CHAN_TRACE(chan, "preempt"); + chan->func->preempt(chan); + if (!wait) + return 0; + + return nvkm_runl_preempt_wait(runl); +} + +int +nvkm_chan_preempt(struct nvkm_chan *chan, bool wait) +{ + int ret; + + if (!chan->func->preempt) + return 0; + + mutex_lock(&chan->cgrp->runl->mutex); + ret = nvkm_chan_preempt_locked(chan, wait); + mutex_unlock(&chan->cgrp->runl->mutex); + return ret; +} + static int nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) @@ -346,6 +374,8 @@ nvkm_chan_error(struct nvkm_chan *chan, bool preempt) if (atomic_inc_return(&chan->errored) == 1) { CHAN_ERROR(chan, "errored - disabling channel"); nvkm_chan_block_locked(chan); + if (preempt) + chan->func->preempt(chan); nvkm_event_ntfy(&chan->cgrp->runl->chid->event, chan->id, NVKM_CHAN_EVENT_ERRORED); } spin_unlock_irqrestore(&chan->lock, flags); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index d573d9fdb4d80..2ad385914bec8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -20,6 +20,7 @@ struct nvkm_chan_func { void (*unbind)(struct nvkm_chan *); void (*start)(struct nvkm_chan *); void (*stop)(struct nvkm_chan *); + void (*preempt)(struct nvkm_chan *); u32 (*doorbell_handle)(struct nvkm_chan *); void *(*dtor)(struct nvkm_fifo_chan *); @@ -43,6 +44,8 @@ void nvkm_chan_del(struct nvkm_chan **); void nvkm_chan_allow(struct nvkm_chan *); void nvkm_chan_block(struct nvkm_chan *); void nvkm_chan_error(struct nvkm_chan *, bool preempt); +int nvkm_chan_preempt(struct nvkm_chan *, bool wait); +int nvkm_chan_preempt_locked(struct nvkm_chan *, bool wait); int nvkm_chan_cctx_get(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx **, struct nvkm_client * /*TODO: remove need for this */); void nvkm_chan_cctx_put(struct nvkm_chan *, struct nvkm_cctx **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 9713daee6c766..7a83b8667443a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -33,8 +33,6 @@ int gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *, struct nvkm_engine *, struct nvkm_object *); void gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *, struct nvkm_engine *); -int gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *); -int gk104_fifo_gpfifo_kick_locked(struct gk104_fifo_chan *); int gv100_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index e47ca113e9e85..4c3338c4d47a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -39,6 +39,12 @@ #include +void +gf100_chan_preempt(struct nvkm_chan *chan) +{ + nvkm_wr32(chan->cgrp->runl->fifo->engine.subdev.device, 0x002634, chan->id); +} + static void gf100_chan_stop(struct nvkm_chan *chan) { @@ -83,6 +89,7 @@ gf100_chan = { .unbind = gf100_chan_unbind, .start = gf100_chan_start, .stop = gf100_chan_stop, + .preempt = gf100_chan_preempt, }; static const struct nvkm_engn_func @@ -158,6 +165,12 @@ gf100_runq = { .intr_0_names = gf100_runq_intr_0_names, }; +bool +gf100_runl_preempt_pending(struct nvkm_runl *runl) +{ + return nvkm_rd32(runl->fifo->engine.subdev.device, 0x002634) & 0x00100000; +} + static void gf100_runl_allow(struct nvkm_runl *runl, u32 engm) { @@ -238,6 +251,7 @@ gf100_runl = { .pending = gf100_runl_pending, .block = gf100_runl_block, .allow = gf100_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 5ba922cf3c2c8..bcaa0efc19a97 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -86,6 +86,7 @@ gk104_chan = { .unbind = gk104_chan_unbind, .start = gk104_chan_start, .stop = gk104_chan_stop, + .preempt = gf100_chan_preempt, }; void @@ -366,6 +367,7 @@ gk104_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 3582b9f55611b..a88e24ba956d0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -30,19 +30,41 @@ #include "changk104.h" #include +#include #include +void +gk110_chan_preempt(struct nvkm_chan *chan) +{ + struct nvkm_cgrp *cgrp = chan->cgrp; + + if (cgrp->hw) { + cgrp->func->preempt(cgrp); + return; + } + + gf100_chan_preempt(chan); +} + const struct nvkm_chan_func gk110_chan = { .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, .stop = gk104_chan_stop, + .preempt = gk110_chan_preempt, }; +static void +gk110_cgrp_preempt(struct nvkm_cgrp *cgrp) +{ + nvkm_wr32(cgrp->runl->fifo->engine.subdev.device, 0x002634, 0x01000000 | cgrp->id); +} + const struct nvkm_cgrp_func gk110_cgrp = { + .preempt = gk110_cgrp_preempt, }; void @@ -68,6 +90,7 @@ gk110_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 2924ce10c3373..a98ea71df2dec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -38,6 +38,7 @@ gm107_chan = { .unbind = gk104_chan_unbind, .start = gk104_chan_start, .stop = gk104_chan_stop, + .preempt = gk110_chan_preempt, }; static void @@ -62,6 +63,7 @@ gm107_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 4dd3fb04d7e53..ddac252508fbf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -35,6 +35,7 @@ gp100_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 29783d271f1e8..77a4f2346f04b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -62,26 +62,9 @@ gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, { const u32 offset = gf100_fifo_gpfifo_engine_addr(engine); struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct nvkm_subdev *subdev = &chan->fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; struct nvkm_gpuobj *inst = chan->base.inst; int ret = 0; - mutex_lock(&chan->fifo->base.mutex); - nvkm_wr32(device, 0x002634, chan->base.chid); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x002634) == chan->base.chid) - break; - ) < 0) { - nvkm_error(subdev, "channel %d [%s] kick timeout\n", - chan->base.chid, chan->base.object.client->name); - ret = -ETIMEDOUT; - } - mutex_unlock(&chan->fifo->base.mutex); - - if (ret && suspend) - return ret; - if (offset) { nvkm_kmap(inst); nvkm_wo32(inst, offset + 0x00, 0x00000000); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 1847b1b7af8bb..9440cf57a1b83 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -33,43 +33,6 @@ #include #include -int -gk104_fifo_gpfifo_kick_locked(struct gk104_fifo_chan *chan) -{ - struct gk104_fifo *fifo = chan->fifo; - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_client *client = chan->base.object.client; - struct nvkm_fifo_cgrp *cgrp = chan->cgrp; - int ret = 0; - - if (cgrp) - nvkm_wr32(device, 0x002634, cgrp->id | 0x01000000); - else - nvkm_wr32(device, 0x002634, chan->base.chid); - if (nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x002634) & 0x00100000)) - break; - ) < 0) { - nvkm_error(subdev, "%s %d [%s] kick timeout\n", - cgrp ? "tsg" : "channel", - cgrp ? cgrp->id : chan->base.chid, client->name); - nvkm_fifo_recover_chan(&fifo->base, chan->base.chid); - ret = -ETIMEDOUT; - } - return ret; -} - -int -gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan) -{ - int ret; - mutex_lock(&chan->base.fifo->mutex); - ret = gk104_fifo_gpfifo_kick_locked(chan); - mutex_unlock(&chan->base.fifo->mutex); - return ret; -} - static u32 gk104_fifo_gpfifo_engine_addr(struct nvkm_engine *engine) { @@ -110,11 +73,6 @@ gk104_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, struct gk104_fifo_chan *chan = gk104_fifo_chan(base); struct nvkm_gpuobj *inst = chan->base.inst; u32 offset = gk104_fifo_gpfifo_engine_addr(engine); - int ret; - - ret = gk104_fifo_gpfifo_kick(chan); - if (ret && suspend) - return ret; if (offset) { nvkm_kmap(inst); @@ -127,7 +85,7 @@ gk104_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, nvkm_done(inst); } - return ret; + return 0; } static int @@ -202,7 +160,6 @@ gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) if (!list_empty(&chan->head)) { gk104_fifo_runlist_remove(fifo, chan); - gk104_fifo_gpfifo_kick(chan); gk104_fifo_runlist_update(fifo, chan->runl); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index a901ce269e06a..66d6079ff210c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -33,23 +33,15 @@ gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid { const u32 mask = ce ? 0x00020000 : 0x00010000; const u32 data = valid ? mask : 0x00000000; - int ret; - - /* Block runlist to prevent the channel from being rescheduled. */ - mutex_lock(&chan->fifo->base.mutex); - /* Preempt the channel. */ - ret = gk104_fifo_gpfifo_kick_locked(chan); - if (ret == 0) { + if (1) { /* Update engine context validity. */ nvkm_kmap(chan->base.inst); nvkm_mo32(chan->base.inst, 0x0ac, mask, data); nvkm_done(chan->base.inst); } - /* Resume runlist. */ - mutex_unlock(&chan->fifo->base.mutex); - return ret; + return 0; } int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 0e9e070201b16..2e61a7b384376 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -43,6 +43,7 @@ gv100_chan = { .unbind = gk104_chan_unbind, .start = gk104_chan_start, .stop = gk104_chan_stop, + .preempt = gk110_chan_preempt, .doorbell_handle = gv100_chan_doorbell_handle, }; @@ -99,6 +100,7 @@ gv100_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index d9de5ab264521..1a0d94bb1aaa4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -11,8 +11,6 @@ struct nvkm_runq; struct gk104_fifo; struct gk104_fifo_chan; -void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid); - struct nvkm_fifo_chan_oclass; struct nvkm_fifo_func { void *(*dtor)(struct nvkm_fifo *); @@ -107,6 +105,7 @@ extern const struct nvkm_engn_func nv50_engn_sw; void nv50_chan_unbind(struct nvkm_chan *); void nv50_chan_start(struct nvkm_chan *); void nv50_chan_stop(struct nvkm_chan *); +void nv50_chan_preempt(struct nvkm_chan *); extern const struct nvkm_event_func g84_fifo_nonstall; extern const struct nvkm_engn_func g84_engn; @@ -120,9 +119,11 @@ void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *); extern const struct nvkm_enum gf100_fifo_mmu_fault_access[]; extern const struct nvkm_event_func gf100_fifo_nonstall; +bool gf100_runl_preempt_pending(struct nvkm_runl *); void gf100_runq_init(struct nvkm_runq *); bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); extern const struct nvkm_engn_func gf100_engn_sw; +void gf100_chan_preempt(struct nvkm_chan *); int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_runl_ctor(struct nvkm_fifo *); @@ -156,6 +157,7 @@ int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func gk110_runl; extern const struct nvkm_cgrp_func gk110_cgrp; extern const struct nvkm_chan_func gk110_chan; +void gk110_chan_preempt(struct nvkm_chan *); extern const struct nvkm_runq_func gk208_runq; void gk208_runq_init(struct nvkm_runq *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 438e884b81005..325c4de1b7d29 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -26,6 +26,7 @@ #include "priv.h" #include +#include #include struct nvkm_chan * @@ -73,6 +74,17 @@ nvkm_runl_chan_get_chid(struct nvkm_runl *runl, int id, unsigned long *pirqflags return NULL; } +int +nvkm_runl_preempt_wait(struct nvkm_runl *runl) +{ + return nvkm_msec(runl->fifo->engine.subdev.device, runl->fifo->timeout.chan_msec, + if (!runl->func->preempt_pending(runl)) + break; + + usleep_range(1, 2); + ) < 0 ? -ETIMEDOUT : 0; +} + bool nvkm_runl_update_pending(struct nvkm_runl *runl) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 35dce7c831242..68d6854e6d0fb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -28,6 +28,7 @@ struct nvkm_runl { bool (*pending)(struct nvkm_runl *); void (*block)(struct nvkm_runl *, u32 engm); void (*allow)(struct nvkm_runl *, u32 engm); + bool (*preempt_pending)(struct nvkm_runl *); } *func; struct nvkm_fifo *fifo; int id; @@ -60,6 +61,7 @@ void nvkm_runl_del(struct nvkm_runl *); void nvkm_runl_block(struct nvkm_runl *); void nvkm_runl_allow(struct nvkm_runl *); bool nvkm_runl_update_pending(struct nvkm_runl *); +int nvkm_runl_preempt_wait(struct nvkm_runl *); struct nvkm_chan *nvkm_runl_chan_get_chid(struct nvkm_runl *, int chid, unsigned long *irqflags); struct nvkm_chan *nvkm_runl_chan_get_inst(struct nvkm_runl *, u64 inst, unsigned long *irqflags); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 4ae8544a22cf5..641c1ff0aa5d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -53,6 +53,7 @@ tu102_chan = { .unbind = gk104_chan_unbind, .start = tu102_chan_start, .stop = gk104_chan_stop, + .preempt = gk110_chan_preempt, .doorbell_handle = tu102_chan_doorbell_handle, }; @@ -91,6 +92,7 @@ tu102_runl = { .pending = tu102_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt_pending = gf100_runl_preempt_pending, }; static const struct nvkm_enum diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 096e09f231777..1fe7bd6a9aa8c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -216,6 +216,7 @@ nvkm_uchan_fini(struct nvkm_object *object, bool suspend) int ret; nvkm_chan_block(chan); + nvkm_chan_preempt(chan, true); ret = chan->object.func->fini(&chan->object, suspend); if (ret && suspend) -- GitLab From 520db0405e9daed6b96b69149673491d80849fe7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:33 +1000 Subject: [PATCH 1326/1927] drm/nouveau/fifo: kill channel on a selection of PBDMA errors A bunch of these can be handled in such a way that the channel can continue, however, any of these are a pretty decent sign something has gone horribly wrong, and the safest option is to disable the channel. This is a bit of a hack, we will want to handle these individually and dump relevant debug info for each at some point. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 4c3338c4d47a8..ff28b5a4c36f9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -30,7 +30,6 @@ #include "gf100.h" #include "changf100.h" -#include #include #include #include @@ -138,8 +137,9 @@ gf100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null) nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] " "subc %d mthd %04x data %08x\n", runq->id, show, msg, chid, chan ? chan->inst->addr : 0, - chan ? chan->object.client->name : "unknown", - subc, mthd, data); + chan ? chan->name : "unknown", subc, mthd, data); + if ((stat & 0xc67fe000) && chan) + nvkm_chan_error(chan, true); nvkm_chan_put(&chan, flags); } -- GitLab From 0b1bb1296f288bb7164d143ca82dc958f87cbff6 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:33 +1000 Subject: [PATCH 1327/1927] drm/nouveau/fifo: kill channel on NV_PPBDMA_INTR_1_CTXNOTVALID Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 6 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 23 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runq.h | 1 + 3 files changed, 30 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index bcaa0efc19a97..41b265b683cd7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -164,6 +164,12 @@ gk104_runq_intr_1(struct nvkm_runq *runq) u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x2000)) & 0xfff; char msg[128]; + if (stat & 0x80000000) { + if (runq->func->intr_1_ctxnotvalid && + runq->func->intr_1_ctxnotvalid(runq, chid)) + stat &= ~0x80000000; + } + if (stat) { nvkm_snprintbf(msg, sizeof(msg), gk104_runq_intr_1_names, stat); nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d %08x %08x\n", diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 2e61a7b384376..e68f3b7767b08 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -55,11 +55,34 @@ const struct nvkm_engn_func gv100_engn_ce = { }; +static bool +gv100_runq_intr_1_ctxnotvalid(struct nvkm_runq *runq, int chid) +{ + struct nvkm_fifo *fifo = runq->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_chan *chan; + unsigned long flags; + + RUNQ_ERROR(runq, "CTXNOTVALID chid:%d", chid); + + chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags); + if (WARN_ON_ONCE(!chan)) + return false; + + nvkm_chan_error(chan, true); + nvkm_chan_put(&chan, flags); + + nvkm_mask(device, 0x0400ac + (runq->id * 0x2000), 0x00030000, 0x00030000); + nvkm_wr32(device, 0x040148 + (runq->id * 0x2000), 0x80000000); + return true; +} + const struct nvkm_runq_func gv100_runq = { .init = gk208_runq_init, .intr = gk104_runq_intr, .intr_0_names = gk104_runq_intr_0_names, + .intr_1_ctxnotvalid = gv100_runq_intr_1_ctxnotvalid, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h index f939d3500cb6a..011fbf69bb35e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h @@ -9,6 +9,7 @@ struct nvkm_runq { void (*init)(struct nvkm_runq *); bool (*intr)(struct nvkm_runq *, struct nvkm_runl *); const struct nvkm_bitfield *intr_0_names; + bool (*intr_1_ctxnotvalid)(struct nvkm_runq *, int chid); } *func; struct nvkm_fifo *fifo; int id; -- GitLab From 4d60100a23ec5b98e43277d82e5de53c359cf02c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:34 +1000 Subject: [PATCH 1328/1927] drm/nouveau/fifo: add common channel recovery That sure was fun to untangle. - handled per-runlist, rather than globally - more straight-forward process in general - various potential SW/HW races have been fixed - fixes lockdep issues that were present in >=gk104's prior implementation - volta recovery now actually stands a chance of working - volta/turing waiting for PBDMA idle before engine reset - turing using hw-provided TSG info for CTXSW_TIMEOUT Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 5 +- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c | 13 + .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 11 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 286 ++++++++------- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 341 ++++-------------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 23 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 3 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 28 +- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 16 +- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 185 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 18 + .../gpu/drm/nouveau/nvkm/engine/fifo/runq.h | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 210 +++-------- 19 files changed, 559 insertions(+), 605 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 1c3c3495a2504..078a97ab5e375 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -93,11 +93,12 @@ static int nvkm_fifo_fini(struct nvkm_engine *engine, bool suspend) { struct nvkm_fifo *fifo = nvkm_fifo(engine); + struct nvkm_runl *runl; nvkm_inth_block(&fifo->engine.subdev.inth); - if (fifo->func->fini) - fifo->func->fini(fifo); + nvkm_runl_foreach(runl, fifo) + nvkm_runl_fini(runl); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c index ed838609f15d8..fd9e6144981e3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c @@ -168,6 +168,18 @@ nvkm_cgrp_ref(struct nvkm_cgrp *cgrp) return cgrp; } +void +nvkm_cgrp_put(struct nvkm_cgrp **pcgrp, unsigned long irqflags) +{ + struct nvkm_cgrp *cgrp = *pcgrp; + + if (!cgrp) + return; + + *pcgrp = NULL; + spin_unlock_irqrestore(&cgrp->lock, irqflags); +} + int nvkm_cgrp_new(struct nvkm_runl *runl, const char *name, struct nvkm_vmm *vmm, bool hw, struct nvkm_cgrp **pcgrp) @@ -190,6 +202,7 @@ nvkm_cgrp_new(struct nvkm_runl *runl, const char *name, struct nvkm_vmm *vmm, bo INIT_LIST_HEAD(&cgrp->ectxs); INIT_LIST_HEAD(&cgrp->vctxs); mutex_init(&cgrp->mutex); + atomic_set(&cgrp->rc, NVKM_CGRP_RC_NONE); if (runl->cgid) { cgrp->id = nvkm_chid_get(runl->cgid, cgrp); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index 1440c72ad7ddb..e7ce66fecfd43 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -40,6 +40,11 @@ struct nvkm_cgrp { struct list_head vctxs; struct mutex mutex; +#define NVKM_CGRP_RC_NONE 0 +#define NVKM_CGRP_RC_PENDING 1 +#define NVKM_CGRP_RC_RUNNING 2 + atomic_t rc; + struct list_head head; struct list_head chan; }; @@ -52,6 +57,12 @@ int nvkm_cgrp_vctx_get(struct nvkm_cgrp *, struct nvkm_engn *, struct nvkm_chan struct nvkm_vctx **, struct nvkm_client *); void nvkm_cgrp_vctx_put(struct nvkm_cgrp *, struct nvkm_vctx **); +void nvkm_cgrp_put(struct nvkm_cgrp **, unsigned long irqflags); + +#define nvkm_cgrp_foreach_chan(chan,cgrp) for ((chan) = (cgrp)->chans; (chan); (chan) = NULL) +#define nvkm_cgrp_foreach_chan_safe(chan,ctmp,cgrp) \ + (void)(ctmp); nvkm_cgrp_foreach_chan((chan), (cgrp)) + #define CGRP_PRCLI(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) #define CGRP_PRINT(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:"f, (c)->id, ##a) #define CGRP_ERROR(c,f,a...) CGRP_PRCLI((c), ERROR, err, " "f"\n", ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index ff28b5a4c36f9..4fc9e80b5f692 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -91,8 +91,98 @@ gf100_chan = { .preempt = gf100_chan_preempt, }; +bool +gf100_engn_mmu_fault_triggered(struct nvkm_engn *engn) +{ + struct nvkm_runl *runl = engn->runl; + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; + u32 data = nvkm_rd32(device, 0x002a30 + (engn->id * 4)); + + ENGN_DEBUG(engn, "%08x: mmu fault triggered", data); + if (!(data & 0x00000100)) + return false; + + spin_lock(&fifo->lock); + nvkm_mask(device, 0x002a30 + (engn->id * 4), 0x00000100, 0x00000000); + if (atomic_dec_and_test(&runl->rc_triggered)) + nvkm_mask(device, 0x002140, 0x00000100, 0x00000100); + spin_unlock(&fifo->lock); + return true; +} + +void +gf100_engn_mmu_fault_trigger(struct nvkm_engn *engn) +{ + struct nvkm_runl *runl = engn->runl; + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; + + ENGN_DEBUG(engn, "triggering mmu fault on 0x%02x", engn->fault); + spin_lock(&fifo->lock); + if (atomic_inc_return(&runl->rc_triggered) == 1) + nvkm_mask(device, 0x002140, 0x00000100, 0x00000000); + nvkm_wr32(device, 0x002100, 0x00000100); + nvkm_wr32(device, 0x002a30 + (engn->id * 4), 0x00000100 | engn->fault); + spin_unlock(&fifo->lock); +} + +/*TODO: clean all this up. */ +struct gf100_engn_status { + bool busy; + bool save; + bool unk0; + bool unk1; + u8 chid; +}; + +static void +gf100_engn_status(struct nvkm_engn *engn, struct gf100_engn_status *status) +{ + u32 stat = nvkm_rd32(engn->engine->subdev.device, 0x002640 + (engn->id * 4)); + + status->busy = (stat & 0x10000000); + status->save = (stat & 0x00100000); + status->unk0 = (stat & 0x00004000); + status->unk1 = (stat & 0x00001000); + status->chid = (stat & 0x0000007f); + + ENGN_DEBUG(engn, "%08x: busy %d save %d unk0 %d unk1 %d chid %d", + stat, status->busy, status->save, status->unk0, status->unk1, status->chid); +} + +static int +gf100_engn_cxid(struct nvkm_engn *engn, bool *cgid) +{ + struct gf100_engn_status status; + + gf100_engn_status(engn, &status); + if (status.busy) { + *cgid = false; + return status.chid; + } + + return -ENODEV; +} + +static bool +gf100_engn_chsw(struct nvkm_engn *engn) +{ + struct gf100_engn_status status; + + gf100_engn_status(engn, &status); + if (status.busy && (status.unk0 || status.unk1)) + return true; + + return false; +} + static const struct nvkm_engn_func gf100_engn = { + .chsw = gf100_engn_chsw, + .cxid = gf100_engn_cxid, + .mmu_fault_trigger = gf100_engn_mmu_fault_trigger, + .mmu_fault_triggered = gf100_engn_mmu_fault_triggered, }; const struct nvkm_engn_func @@ -138,6 +228,8 @@ gf100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null) "subc %d mthd %04x data %08x\n", runq->id, show, msg, chid, chan ? chan->inst->addr : 0, chan ? chan->name : "unknown", subc, mthd, data); + + /*TODO: use proper procedure for clearing each exception / debug output */ if ((stat & 0xc67fe000) && chan) nvkm_chan_error(chan, true); nvkm_chan_put(&chan, flags); @@ -171,6 +263,12 @@ gf100_runl_preempt_pending(struct nvkm_runl *runl) return nvkm_rd32(runl->fifo->engine.subdev.device, 0x002634) & 0x00100000; } +static void +gf100_runl_fault_clear(struct nvkm_runl *runl) +{ + nvkm_mask(runl->fifo->engine.subdev.device, 0x00262c, 0x00000000, 0x00000000); +} + static void gf100_runl_allow(struct nvkm_runl *runl, u32 engm) { @@ -251,6 +349,7 @@ gf100_runl = { .pending = gf100_runl_pending, .block = gf100_runl_block, .allow = gf100_runl_allow, + .fault_clear = gf100_runl_fault_clear, .preempt_pending = gf100_runl_preempt_pending, }; @@ -282,28 +381,6 @@ gf100_fifo_nonstall = { .fini = gf100_fifo_nonstall_block, }; -static struct nvkm_engine * -gf100_fifo_id_engine(struct nvkm_fifo *fifo, int engi) -{ - enum nvkm_subdev_type type; - int inst; - - switch (engi) { - case GF100_FIFO_ENGN_GR : type = NVKM_ENGINE_GR ; inst = 0; break; - case GF100_FIFO_ENGN_MSPDEC: type = NVKM_ENGINE_MSPDEC; inst = 0; break; - case GF100_FIFO_ENGN_MSPPP : type = NVKM_ENGINE_MSPPP ; inst = 0; break; - case GF100_FIFO_ENGN_MSVLD : type = NVKM_ENGINE_MSVLD ; inst = 0; break; - case GF100_FIFO_ENGN_CE0 : type = NVKM_ENGINE_CE ; inst = 0; break; - case GF100_FIFO_ENGN_CE1 : type = NVKM_ENGINE_CE ; inst = 1; break; - case GF100_FIFO_ENGN_SW : type = NVKM_ENGINE_SW ; inst = 0; break; - default: - WARN_ON(1); - return NULL; - } - - return nvkm_device_engine(fifo->engine.subdev.device, type, inst); -} - static int gf100_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) { @@ -320,62 +397,13 @@ gf100_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) } } -static void -gf100_fifo_recover_work(struct work_struct *w) -{ - struct gf100_fifo *fifo = container_of(w, typeof(*fifo), recover.work); - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_engine *engine; - unsigned long flags; - u32 engm, engn, todo; - - spin_lock_irqsave(&fifo->base.lock, flags); - engm = fifo->recover.mask; - fifo->recover.mask = 0ULL; - spin_unlock_irqrestore(&fifo->base.lock, flags); - - nvkm_mask(device, 0x002630, engm, engm); - - for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT_ULL(engn)) { - if ((engine = gf100_fifo_id_engine(&fifo->base, engn))) { - nvkm_subdev_fini(&engine->subdev, false); - WARN_ON(nvkm_subdev_init(&engine->subdev)); - } - } - - gf100_fifo_runlist_commit(fifo); - nvkm_wr32(device, 0x00262c, engm); - nvkm_mask(device, 0x002630, engm, 0x00000000); -} - -static void -gf100_fifo_recover(struct gf100_fifo *fifo, struct nvkm_engine *engine, - struct gf100_fifo_chan *chan) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - u32 chid = chan->base.chid; - int engi = gf100_fifo_engine_id(&fifo->base, engine); - - nvkm_error(subdev, "%s engine fault on channel %d, recovering...\n", - engine->subdev.name, chid); - assert_spin_locked(&fifo->base.lock); - - nvkm_chan_error(&chan->base, false); - list_del_init(&chan->head); - chan->killed = true; - - if (engi >= 0 && engi != GF100_FIFO_ENGN_SW) - fifo->recover.mask |= BIT(engi); - schedule_work(&fifo->recover.work); -} - static const struct nvkm_enum gf100_fifo_mmu_fault_engine[] = { { 0x00, "PGRAPH", NULL, NVKM_ENGINE_GR }, { 0x03, "PEEPHOLE", NULL, NVKM_ENGINE_IFB }, { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR }, { 0x05, "BAR3", NULL, NVKM_SUBDEV_INSTMEM }, - { 0x07, "PFIFO", NULL, NVKM_ENGINE_FIFO }, + { 0x07, "PFIFO" }, { 0x10, "PMSVLD", NULL, NVKM_ENGINE_MSVLD }, { 0x11, "PMSPPP", NULL, NVKM_ENGINE_MSPPP }, { 0x13, "PCOUNTER" }, @@ -452,6 +480,13 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf nvkm_runl_foreach(runl, fifo) { engn = nvkm_runl_find_engn(engn, runl, engn->fault == info->engine); if (engn) { + /* Fault triggered by CTXSW_TIMEOUT recovery procedure. */ + if (engn->func->mmu_fault_triggered && + engn->func->mmu_fault_triggered(engn)) { + nvkm_runl_rc_engn(runl, engn); + return; + } + engine = engn->engine; break; } @@ -496,11 +531,8 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf chan ? chan->id : -1, info->inst, chan ? chan->name : "unknown"); /* Handle host/engine faults. */ - if (fifo->func->recover_chan && chan) - fifo->func->recover_chan(fifo, chan->id); - else - if (engine && chan) - gf100_fifo_recover(gf100_fifo(fifo), engine, (void *)chan); + if (chan) + nvkm_runl_rc_cgrp(chan->cgrp); nvkm_chan_put(&chan, flags); } @@ -515,56 +547,72 @@ gf100_fifo_mmu_fault = { .gpcclient = gf100_fifo_mmu_fault_gpcclient, }; -static const struct nvkm_enum -gf100_fifo_sched_reason[] = { - { 0x0a, "CTXSW_TIMEOUT" }, - {} -}; - -static void -gf100_fifo_intr_sched_ctxsw(struct gf100_fifo *fifo) +void +gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *fifo, u32 engm) { - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_engine *engine; - struct gf100_fifo_chan *chan; - unsigned long flags; - u32 engn; - - spin_lock_irqsave(&fifo->base.lock, flags); - for (engn = 0; engn < 6; engn++) { - u32 stat = nvkm_rd32(device, 0x002640 + (engn * 0x04)); - u32 busy = (stat & 0x80000000); - u32 save = (stat & 0x00100000); /* maybe? */ - u32 unk0 = (stat & 0x00040000); - u32 unk1 = (stat & 0x00001000); - u32 chid = (stat & 0x0000007f); - (void)save; - - if (busy && unk0 && unk1) { - list_for_each_entry(chan, &fifo->chan, head) { - if (chan->base.chid == chid) { - engine = gf100_fifo_id_engine(&fifo->base, engn); - if (!engine) - break; - gf100_fifo_recover(fifo, engine, chan); - break; + struct nvkm_runl *runl; + struct nvkm_engn *engn, *engn2; + bool cgid, cgid2; + int id, id2; + + nvkm_runl_foreach(runl, fifo) { + /* Stop the runlist, and go through all engines serving it. */ + nvkm_runl_block(runl); + nvkm_runl_foreach_engn_cond(engn, runl, engm & BIT(engn->id)) { + /* Determine what channel (group) the engine is on. */ + id = engn->func->cxid(engn, &cgid); + if (id >= 0) { + /* Trigger MMU fault on any engine(s) on that channel (group). */ + nvkm_runl_foreach_engn_cond(engn2, runl, engn2->func->cxid) { + id2 = engn2->func->cxid(engn2, &cgid2); + if (cgid2 == cgid && id2 == id) + engn2->func->mmu_fault_trigger(engn2); } } } + nvkm_runl_allow(runl); /* HW will keep runlist blocked via ERROR_SCHED_DISABLE. */ } - spin_unlock_irqrestore(&fifo->base.lock, flags); } static void -gf100_fifo_intr_sched(struct gf100_fifo *fifo) +gf100_fifo_intr_sched_ctxsw(struct nvkm_fifo *fifo) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; + struct nvkm_runl *runl; + struct nvkm_engn *engn; + u32 engm = 0; + + /* Look for any engines that are busy, and awaiting chsw ack. */ + nvkm_runl_foreach(runl, fifo) { + nvkm_runl_foreach_engn_cond(engn, runl, engn->func->chsw) { + if (WARN_ON(engn->fault < 0) || !engn->func->chsw(engn)) + continue; + + engm |= BIT(engn->id); + } + } + + if (!engm) + return; + + fifo->func->intr_ctxsw_timeout(fifo, engm); +} + +static const struct nvkm_enum +gf100_fifo_intr_sched_names[] = { + { 0x0a, "CTXSW_TIMEOUT" }, + {} +}; + +void +gf100_fifo_intr_sched(struct nvkm_fifo *fifo) +{ + struct nvkm_subdev *subdev = &fifo->engine.subdev; struct nvkm_device *device = subdev->device; u32 intr = nvkm_rd32(device, 0x00254c); u32 code = intr & 0x000000ff; const struct nvkm_enum *en; - en = nvkm_enum_find(gf100_fifo_sched_reason, code); + en = nvkm_enum_find(gf100_fifo_intr_sched_names, code); nvkm_error(subdev, "SCHED_ERROR %02x [%s]\n", code, en ? en->name : ""); @@ -704,7 +752,7 @@ gf100_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x00000100) { - gf100_fifo_intr_sched(gf100_fifo(fifo)); + gf100_fifo_intr_sched(fifo); nvkm_wr32(device, 0x002100, 0x00000100); stat &= ~0x00000100; } @@ -754,13 +802,6 @@ gf100_fifo_intr(struct nvkm_inth *inth) return IRQ_HANDLED; } -static void -gf100_fifo_fini(struct nvkm_fifo *base) -{ - struct gf100_fifo *fifo = gf100_fifo(base); - flush_work(&fifo->recover.work); -} - static void gf100_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) { @@ -888,9 +929,9 @@ gf100_fifo = { .runl_ctor = gf100_fifo_runl_ctor, .init = gf100_fifo_init, .init_pbdmas = gf100_fifo_init_pbdmas, - .fini = gf100_fifo_fini, .intr = gf100_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gf100_fifo_mmu_fault, .engine_id = gf100_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, @@ -910,7 +951,6 @@ gf100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) return -ENOMEM; INIT_LIST_HEAD(&fifo->chan); - INIT_WORK(&fifo->recover.work, gf100_fifo_recover_work); *pfifo = &fifo->base; return nvkm_fifo_ctor(&gf100_fifo, device, type, inst, &fifo->base); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index 16268e81077d7..6d7771f505c6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -12,11 +12,6 @@ struct gf100_fifo { struct list_head chan; - struct { - struct work_struct work; - u64 mask; - } recover; - struct { struct nvkm_memory *mem[2]; int active; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 41b265b683cd7..d8cb2626b1884 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -89,14 +88,23 @@ gk104_chan = { .preempt = gf100_chan_preempt, }; -void -gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, - struct gk104_fifo_engine_status *status) +/*TODO: clean this up */ +struct gk104_engn_status { + bool busy; + bool faulted; + bool chsw; + bool save; + bool load; + struct { + bool tsg; + u32 id; + } prev, next, *chan; +}; + +static void +gk104_engn_status(struct nvkm_engn *engn, struct gk104_engn_status *status) { - struct nvkm_engine *engine = fifo->engine[engn].engine; - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0x002640 + (engn * 0x08)); + u32 stat = nvkm_rd32(engn->runl->fifo->engine.subdev.device, 0x002640 + (engn->id * 0x08)); status->busy = !!(stat & 0x80000000); status->faulted = !!(stat & 0x40000000); @@ -111,7 +119,7 @@ gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, if (status->busy && status->chsw) { if (status->load && status->save) { - if (engine && nvkm_engine_chsw_load(engine)) + if (nvkm_engine_chsw_load(engn->engine)) status->chan = &status->next; else status->chan = &status->prev; @@ -126,24 +134,64 @@ gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, status->chan = &status->prev; } - nvkm_debug(subdev, "engine %02d: busy %d faulted %d chsw %d " - "save %d load %d %sid %d%s-> %sid %d%s\n", - engn, status->busy, status->faulted, - status->chsw, status->save, status->load, + ENGN_DEBUG(engn, "%08x: busy %d faulted %d chsw %d save %d load %d %sid %d%s-> %sid %d%s", + stat, status->busy, status->faulted, status->chsw, status->save, status->load, status->prev.tsg ? "tsg" : "ch", status->prev.id, status->chan == &status->prev ? "*" : " ", status->next.tsg ? "tsg" : "ch", status->next.id, status->chan == &status->next ? "*" : " "); } +int +gk104_engn_cxid(struct nvkm_engn *engn, bool *cgid) +{ + struct gk104_engn_status status; + + gk104_engn_status(engn, &status); + if (status.chan) { + *cgid = status.chan->tsg; + return status.chan->id; + } + + return -ENODEV; +} + +bool +gk104_engn_chsw(struct nvkm_engn *engn) +{ + struct gk104_engn_status status; + + gk104_engn_status(engn, &status); + if (status.busy && status.chsw) + return true; + + return false; +} + const struct nvkm_engn_func gk104_engn = { + .chsw = gk104_engn_chsw, + .cxid = gk104_engn_cxid, + .mmu_fault_trigger = gf100_engn_mmu_fault_trigger, + .mmu_fault_triggered = gf100_engn_mmu_fault_triggered, }; const struct nvkm_engn_func gk104_engn_ce = { + .chsw = gk104_engn_chsw, + .cxid = gk104_engn_cxid, + .mmu_fault_trigger = gf100_engn_mmu_fault_trigger, + .mmu_fault_triggered = gf100_engn_mmu_fault_triggered, }; +bool +gk104_runq_idle(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + return !(nvkm_rd32(device, 0x003080 + (runq->id * 4)) & 0x0000e000); +} + static const struct nvkm_bitfield gk104_runq_intr_1_names[] = { { 0x00000001, "HCE_RE_ILLEGAL_OP" }, @@ -248,8 +296,15 @@ gk104_runq = { .init = gk104_runq_init, .intr = gk104_runq_intr, .intr_0_names = gk104_runq_intr_0_names, + .idle = gk104_runq_idle, }; +void +gk104_runl_fault_clear(struct nvkm_runl *runl) +{ + nvkm_wr32(runl->fifo->engine.subdev.device, 0x00262c, BIT(runl->id)); +} + void gk104_runl_allow(struct nvkm_runl *runl, u32 engm) { @@ -373,6 +428,7 @@ gk104_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .fault_clear = gk104_runl_fault_clear, .preempt_pending = gf100_runl_preempt_pending, }; @@ -394,193 +450,6 @@ gk104_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) return -1; } -static void -gk104_fifo_recover_work(struct work_struct *w) -{ - struct gk104_fifo *fifo = container_of(w, typeof(*fifo), recover.work); - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_engine *engine; - unsigned long flags; - u32 engm, runm, todo; - int engn, runl; - - spin_lock_irqsave(&fifo->base.lock, flags); - runm = fifo->recover.runm; - engm = fifo->recover.engm; - fifo->recover.engm = 0; - fifo->recover.runm = 0; - spin_unlock_irqrestore(&fifo->base.lock, flags); - - nvkm_mask(device, 0x002630, runm, runm); - - for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT(engn)) { - if ((engine = fifo->engine[engn].engine)) { - nvkm_subdev_fini(&engine->subdev, false); - WARN_ON(nvkm_subdev_init(&engine->subdev)); - } - } - - for (todo = runm; runl = __ffs(todo), todo; todo &= ~BIT(runl)) - gk104_fifo_runlist_update(fifo, runl); - - nvkm_wr32(device, 0x00262c, runm); - nvkm_mask(device, 0x002630, runm, 0x00000000); -} - -static void gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn); - -static void -gk104_fifo_recover_runl(struct gk104_fifo *fifo, int runl) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const u32 runm = BIT(runl); - - assert_spin_locked(&fifo->base.lock); - if (fifo->recover.runm & runm) - return; - fifo->recover.runm |= runm; - - /* Block runlist to prevent channel assignment(s) from changing. */ - nvkm_mask(device, 0x002630, runm, runm); - - /* Schedule recovery. */ - nvkm_warn(subdev, "runlist %d: scheduled for recovery\n", runl); - schedule_work(&fifo->recover.work); -} - -static struct gk104_fifo_chan * -gk104_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid) -{ - struct gk104_fifo_chan *chan; - struct nvkm_fifo_cgrp *cgrp; - - list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { - if (chan->base.chid == chid) { - list_del_init(&chan->head); - return chan; - } - } - - list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) { - if (cgrp->id == chid) { - chan = list_first_entry(&cgrp->chan, typeof(*chan), head); - list_del_init(&chan->head); - if (!--cgrp->chan_nr) - list_del_init(&cgrp->head); - return chan; - } - } - - return NULL; -} - -void -gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const u32 stat = nvkm_rd32(device, 0x800004 + (chid * 0x08)); - const u32 runl = (stat & 0x000f0000) >> 16; - const bool used = (stat & 0x00000001); - unsigned long engn, engm = fifo->runlist[runl].engm; - struct gk104_fifo_chan *chan; - - assert_spin_locked(&fifo->base.lock); - if (!used) - return; - - /* Lookup SW state for channel, and mark it as dead. */ - chan = gk104_fifo_recover_chid(fifo, runl, chid); - if (chan) { - chan->killed = true; - nvkm_chan_error(&chan->base, false); - } - - /* Block channel assignments from changing during recovery. */ - gk104_fifo_recover_runl(fifo, runl); - - /* Schedule recovery for any engines the channel is on. */ - for_each_set_bit(engn, &engm, fifo->engine_nr) { - struct gk104_fifo_engine_status status; - gk104_fifo_engine_status(fifo, engn, &status); - if (!status.chan || status.chan->id != chid) - continue; - gk104_fifo_recover_engn(fifo, engn); - } -} - -static void -gk104_fifo_recover_engn(struct gk104_fifo *fifo, int engn) -{ - struct nvkm_engine *engine = fifo->engine[engn].engine; - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const u32 runl = fifo->engine[engn].runl; - const u32 engm = BIT(engn); - struct gk104_fifo_engine_status status; - int mmui = -1; - - assert_spin_locked(&fifo->base.lock); - if (fifo->recover.engm & engm) - return; - fifo->recover.engm |= engm; - - /* Block channel assignments from changing during recovery. */ - gk104_fifo_recover_runl(fifo, runl); - - /* Determine which channel (if any) is currently on the engine. */ - gk104_fifo_engine_status(fifo, engn, &status); - if (status.chan) { - /* The channel is not longer viable, kill it. */ - gk104_fifo_recover_chan(&fifo->base, status.chan->id); - } - - /* Determine MMU fault ID for the engine, if we're not being - * called from the fault handler already. - */ - if (!status.faulted && engine) { - mmui = nvkm_top_fault_id(device, engine->subdev.type, engine->subdev.inst); - if (mmui < 0) { - const struct nvkm_enum *en = fifo->func->mmu_fault->engine; - for (; en && en->name; en++) { - if (en->data2 == engine->subdev.type && - en->inst == engine->subdev.inst) { - mmui = en->value; - break; - } - } - } - WARN_ON(mmui < 0); - } - - /* Trigger a MMU fault for the engine. - * - * No good idea why this is needed, but nvgpu does something similar, - * and it makes recovery from CTXSW_TIMEOUT a lot more reliable. - */ - if (mmui >= 0) { - nvkm_wr32(device, 0x002a30 + (engn * 0x04), 0x00000100 | mmui); - - /* Wait for fault to trigger. */ - nvkm_msec(device, 2000, - gk104_fifo_engine_status(fifo, engn, &status); - if (status.faulted) - break; - ); - - /* Release MMU fault trigger, and ACK the fault. */ - nvkm_wr32(device, 0x002a30 + (engn * 0x04), 0x00000000); - nvkm_wr32(device, 0x00259c, BIT(mmui)); - nvkm_wr32(device, 0x002100, 0x10000000); - } - - /* Schedule recovery. */ - nvkm_warn(subdev, "engine %d: scheduled for recovery\n", engn); - schedule_work(&fifo->recover.work); -} - static const struct nvkm_enum gk104_fifo_mmu_fault_engine[] = { { 0x00, "GR", NULL, NVKM_ENGINE_GR }, @@ -728,64 +597,6 @@ gk104_fifo_intr_bind(struct nvkm_fifo *fifo) nvkm_error(subdev, "BIND_ERROR %02x [%s]\n", code, en ? en->name : ""); } -static const struct nvkm_enum -gk104_fifo_sched_reason[] = { - { 0x0a, "CTXSW_TIMEOUT" }, - {} -}; - -static void -gk104_fifo_intr_sched_ctxsw(struct gk104_fifo *fifo) -{ - struct nvkm_device *device = fifo->base.engine.subdev.device; - unsigned long flags, engm = 0; - u32 engn; - - /* We need to ACK the SCHED_ERROR here, and prevent it reasserting, - * as MMU_FAULT cannot be triggered while it's pending. - */ - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_mask(device, 0x002140, 0x00000100, 0x00000000); - nvkm_wr32(device, 0x002100, 0x00000100); - - for (engn = 0; engn < fifo->engine_nr; engn++) { - struct gk104_fifo_engine_status status; - - gk104_fifo_engine_status(fifo, engn, &status); - if (!status.busy || !status.chsw) - continue; - - engm |= BIT(engn); - } - - for_each_set_bit(engn, &engm, fifo->engine_nr) - gk104_fifo_recover_engn(fifo, engn); - - nvkm_mask(device, 0x002140, 0x00000100, 0x00000100); - spin_unlock_irqrestore(&fifo->base.lock, flags); -} - -static void -gk104_fifo_intr_sched(struct gk104_fifo *fifo) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 intr = nvkm_rd32(device, 0x00254c); - u32 code = intr & 0x000000ff; - const struct nvkm_enum *en = - nvkm_enum_find(gk104_fifo_sched_reason, code); - - nvkm_error(subdev, "SCHED_ERROR %02x [%s]\n", code, en ? en->name : ""); - - switch (code) { - case 0x0a: - gk104_fifo_intr_sched_ctxsw(fifo); - break; - default: - break; - } -} - void gk104_fifo_intr_chsw(struct nvkm_fifo *fifo) { @@ -840,7 +651,7 @@ gk104_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x00000100) { - gk104_fifo_intr_sched(gk104_fifo(fifo)); + gf100_fifo_intr_sched(fifo); nvkm_wr32(device, 0x002100, 0x00000100); stat &= ~0x00000100; } @@ -901,13 +712,6 @@ gk104_fifo_intr(struct nvkm_inth *inth) return IRQ_HANDLED; } -void -gk104_fifo_fini(struct nvkm_fifo *base) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - flush_work(&fifo->recover.work); -} - void gk104_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) { @@ -999,7 +803,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) continue; fifo->engine[engn].engine = nvkm_device_engine(device, tdev->type, tdev->inst); - fifo->engine[engn].runl = tdev->runlist; fifo->engine_nr = max(fifo->engine_nr, engn + 1); fifo->runlist[tdev->runlist].engm |= BIT(engn); fifo->runlist[tdev->runlist].engm_sw |= BIT(engn); @@ -1064,7 +867,6 @@ gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) return -ENOMEM; fifo->func = func; - INIT_WORK(&fifo->recover.work, gk104_fifo_recover_work); *pfifo = &fifo->base; return nvkm_fifo_ctor(func, device, type, inst, &fifo->base); @@ -1080,12 +882,11 @@ gk104_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gk104_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk104_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 7cff1529ae350..64d9b1e857e10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -14,16 +14,8 @@ struct gk104_fifo { const struct gk104_fifo_func *func; struct nvkm_fifo base; - struct { - struct work_struct work; - u32 engm; - u32 runm; - } recover; - struct { struct nvkm_engine *engine; - int runl; - int pbid; } engine[16]; int engine_nr; @@ -43,29 +35,14 @@ struct gk104_fifo { } user; }; -struct gk104_fifo_engine_status { - bool busy; - bool faulted; - bool chsw; - bool save; - bool load; - struct { - bool tsg; - u32 id; - } prev, next, *chan; -}; - int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int index, int nr, struct nvkm_fifo **); void gk104_fifo_runlist_insert(struct gk104_fifo *, struct gk104_fifo_chan *); void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *); void gk104_fifo_runlist_update(struct gk104_fifo *, int runl); -void gk104_fifo_engine_status(struct gk104_fifo *fifo, int engn, - struct gk104_fifo_engine_status *status); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); void gk104_fifo_init(struct nvkm_fifo *base); -void gk104_fifo_fini(struct nvkm_fifo *base); extern const struct gk104_fifo_runlist_func gk104_fifo_runlist; void gk104_fifo_runlist_chan(struct gk104_fifo_chan *, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index a88e24ba956d0..f27b7ea23e153 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -90,6 +90,7 @@ gk110_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .fault_clear = gk104_runl_fault_clear, .preempt_pending = gf100_runl_preempt_pending, }; @@ -115,12 +116,11 @@ gk110_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index ab813aa6d1a16..9886bd38a2120 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -40,6 +40,7 @@ gk208_runq = { .init = gk208_runq_init, .intr = gk104_runq_intr, .intr_0_names = gk104_runq_intr_0_names, + .idle = gk104_runq_idle, }; static int @@ -58,12 +59,11 @@ gk208_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 0d633b805331f..9177383d5f79f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -34,12 +34,11 @@ gk20a_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index a98ea71df2dec..bf8671bf3892c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -63,6 +63,7 @@ gm107_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .fault_clear = gk104_runl_fault_clear, .preempt_pending = gf100_runl_preempt_pending, }; @@ -139,12 +140,11 @@ gm107_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gm107_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 6fa96a4c3e239..13c293aba6f2d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -48,12 +48,11 @@ gm200_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gm107_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index ddac252508fbf..7698d640a6f72 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -35,6 +35,7 @@ gp100_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .fault_clear = gk104_runl_fault_clear, .preempt_pending = gf100_runl_preempt_pending, }; @@ -106,12 +107,11 @@ gp100_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, .intr_mmu_fault_unit = gp100_fifo_intr_mmu_fault_unit, + .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gp100_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gm107_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gp100_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index e68f3b7767b08..4ff2c75f53ab1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -20,6 +20,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #include "chan.h" +#include "chid.h" #include "cgrp.h" #include "runl.h" #include "runq.h" @@ -49,10 +50,14 @@ gv100_chan = { const struct nvkm_engn_func gv100_engn = { + .chsw = gk104_engn_chsw, + .cxid = gk104_engn_cxid, }; const struct nvkm_engn_func gv100_engn_ce = { + .chsw = gk104_engn_chsw, + .cxid = gk104_engn_cxid, }; static bool @@ -83,8 +88,15 @@ gv100_runq = { .intr = gk104_runq_intr, .intr_0_names = gk104_runq_intr_0_names, .intr_1_ctxnotvalid = gv100_runq_intr_1_ctxnotvalid, + .idle = gk104_runq_idle, }; +void +gv100_runl_preempt(struct nvkm_runl *runl) +{ + nvkm_wr32(runl->fifo->engine.subdev.device, 0x002638, BIT(runl->id)); +} + void gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan, struct nvkm_memory *memory, u32 offset) @@ -123,6 +135,7 @@ gv100_runl = { .pending = gk104_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt = gv100_runl_preempt, .preempt_pending = gf100_runl_preempt_pending, }; @@ -362,6 +375,18 @@ gv100_fifo_mmu_fault = { .gpcclient = gv100_fifo_mmu_fault_gpcclient, }; +static void +gv100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *fifo, u32 engm) +{ + struct nvkm_runl *runl; + struct nvkm_engn *engn; + + nvkm_runl_foreach(runl, fifo) { + nvkm_runl_foreach_engn_cond(engn, runl, engm & BIT(engn->id)) + nvkm_runl_rc_engn(runl, engn); + } +} + static const struct nvkm_fifo_func gv100_fifo = { .dtor = gk104_fifo_dtor, @@ -372,11 +397,10 @@ gv100_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = gk104_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = gk104_fifo_intr, + .intr_ctxsw_timeout = gv100_fifo_intr_ctxsw_timeout, .mmu_fault = &gv100_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = gk104_fifo_recover_chan, .runlist = &gv100_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gv100_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 1a0d94bb1aaa4..48153d8bba328 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -5,6 +5,7 @@ #include #include struct nvkm_cgrp; +struct nvkm_engn; struct nvkm_memory; struct nvkm_runl; struct nvkm_runq; @@ -24,10 +25,9 @@ struct nvkm_fifo_func { void (*init)(struct nvkm_fifo *); void (*init_pbdmas)(struct nvkm_fifo *, u32 mask); - void (*fini)(struct nvkm_fifo *); - irqreturn_t (*intr)(struct nvkm_inth *); void (*intr_mmu_fault_unit)(struct nvkm_fifo *, int unit); + void (*intr_ctxsw_timeout)(struct nvkm_fifo *, u32 engm); const struct nvkm_fifo_func_mmu_fault { void (*recover)(struct nvkm_fifo *, struct nvkm_fault_data *); @@ -41,7 +41,6 @@ struct nvkm_fifo_func { int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); - void (*recover_chan)(struct nvkm_fifo *, int chid); const struct gk104_fifo_runlist_func { u8 size; @@ -116,12 +115,16 @@ int gf100_fifo_runq_nr(struct nvkm_fifo *); bool gf100_fifo_intr_pbdma(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault(struct nvkm_fifo *); void gf100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *, int); +void gf100_fifo_intr_sched(struct nvkm_fifo *); +void gf100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *, u32); void gf100_fifo_mmu_fault_recover(struct nvkm_fifo *, struct nvkm_fault_data *); extern const struct nvkm_enum gf100_fifo_mmu_fault_access[]; extern const struct nvkm_event_func gf100_fifo_nonstall; bool gf100_runl_preempt_pending(struct nvkm_runl *); void gf100_runq_init(struct nvkm_runq *); bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); +void gf100_engn_mmu_fault_trigger(struct nvkm_engn *); +bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *); extern const struct nvkm_engn_func gf100_engn_sw; void gf100_chan_preempt(struct nvkm_chan *); @@ -136,16 +139,19 @@ extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; -void gk104_fifo_recover_chan(struct nvkm_fifo *, int); int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); bool gk104_runl_pending(struct nvkm_runl *); void gk104_runl_block(struct nvkm_runl *, u32); void gk104_runl_allow(struct nvkm_runl *, u32); +void gk104_runl_fault_clear(struct nvkm_runl *); extern const struct nvkm_runq_func gk104_runq; void gk104_runq_init(struct nvkm_runq *); bool gk104_runq_intr(struct nvkm_runq *, struct nvkm_runl *); extern const struct nvkm_bitfield gk104_runq_intr_0_names[]; +bool gk104_runq_idle(struct nvkm_runq *); extern const struct nvkm_engn_func gk104_engn; +bool gk104_engn_chsw(struct nvkm_engn *); +int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); extern const struct nvkm_engn_func gk104_engn_ce; void gk104_chan_bind(struct nvkm_chan *); void gk104_chan_bind_inst(struct nvkm_chan *); @@ -174,10 +180,12 @@ extern const struct nvkm_enum gv100_fifo_mmu_fault_access[]; extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[]; extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[]; extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[]; +void gv100_runl_preempt(struct nvkm_runl *); extern const struct nvkm_runq_func gv100_runq; extern const struct nvkm_engn_func gv100_engn; extern const struct nvkm_engn_func gv100_engn_ce; +void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 325c4de1b7d29..27c688d11464a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -24,11 +24,164 @@ #include "chan.h" #include "chid.h" #include "priv.h" +#include "runq.h" #include #include #include +struct nvkm_cgrp * +nvkm_engn_cgrp_get(struct nvkm_engn *engn, unsigned long *pirqflags) +{ + struct nvkm_cgrp *cgrp = NULL; + struct nvkm_chan *chan; + bool cgid; + int id; + + id = engn->func->cxid(engn, &cgid); + if (id < 0) + return NULL; + + if (!cgid) { + chan = nvkm_runl_chan_get_chid(engn->runl, id, pirqflags); + if (chan) + cgrp = chan->cgrp; + } else { + cgrp = nvkm_runl_cgrp_get_cgid(engn->runl, id, pirqflags); + } + + WARN_ON(!cgrp); + return cgrp; +} + +#include "gf100.h" +#include "gk104.h" + +static void +nvkm_runl_rc(struct nvkm_runl *runl) +{ + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_cgrp *cgrp, *gtmp; + struct nvkm_chan *chan, *ctmp; + struct nvkm_engn *engn; + unsigned long flags; + int rc, state, i; + bool reset; + + /* Runlist is blocked before scheduling recovery - fetch count. */ + BUG_ON(!mutex_is_locked(&runl->mutex)); + rc = atomic_xchg(&runl->rc_pending, 0); + if (!rc) + return; + + /* Look for channel groups flagged for RC. */ + nvkm_runl_foreach_cgrp_safe(cgrp, gtmp, runl) { + state = atomic_cmpxchg(&cgrp->rc, NVKM_CGRP_RC_PENDING, NVKM_CGRP_RC_RUNNING); + if (state == NVKM_CGRP_RC_PENDING) { + /* Disable all channels in them, and remove from runlist. */ + nvkm_cgrp_foreach_chan_safe(chan, ctmp, cgrp) + nvkm_chan_error(chan, false); + } + } + + /* On GPUs with runlist preempt, wait for PBDMA(s) servicing runlist to go idle. */ + if (runl->func->preempt) { + for (i = 0; i < runl->runq_nr; i++) { + struct nvkm_runq *runq = runl->runq[i]; + + if (runq) { + nvkm_msec(fifo->engine.subdev.device, 2000, + if (runq->func->idle(runq)) + break; + ); + } + } + } + + /* Look for engines that are still on flagged channel groups - reset them. */ + nvkm_runl_foreach_engn_cond(engn, runl, engn->func->cxid) { + cgrp = nvkm_engn_cgrp_get(engn, &flags); + if (!cgrp) { + ENGN_DEBUG(engn, "cxid not valid"); + continue; + } + + reset = atomic_read(&cgrp->rc) == NVKM_CGRP_RC_RUNNING; + nvkm_cgrp_put(&cgrp, flags); + if (!reset) { + ENGN_DEBUG(engn, "cxid not in recovery"); + continue; + } + + ENGN_DEBUG(engn, "resetting..."); + nvkm_subdev_fini(&engn->engine->subdev, false); + WARN_ON(nvkm_subdev_init(&engn->engine->subdev)); + } + + /* Submit runlist update, and clear any remaining exception state. */ + if (runl->fifo->engine.subdev.device->card_type < NV_E0) + gf100_fifo_runlist_commit(gf100_fifo(runl->fifo)); + else + gk104_fifo_runlist_update(gk104_fifo(runl->fifo), runl->id); + if (runl->func->fault_clear) + runl->func->fault_clear(runl); + + /* Unblock runlist processing. */ + while (rc--) + nvkm_runl_allow(runl); +} + +static void +nvkm_runl_rc_runl(struct nvkm_runl *runl) +{ + RUNL_ERROR(runl, "rc scheduled"); + + nvkm_runl_block(runl); + if (runl->func->preempt) + runl->func->preempt(runl); + + atomic_inc(&runl->rc_pending); + schedule_work(&runl->work); +} + +void +nvkm_runl_rc_cgrp(struct nvkm_cgrp *cgrp) +{ + if (atomic_cmpxchg(&cgrp->rc, NVKM_CGRP_RC_NONE, NVKM_CGRP_RC_PENDING) != NVKM_CGRP_RC_NONE) + return; + + CGRP_ERROR(cgrp, "rc scheduled"); + nvkm_runl_rc_runl(cgrp->runl); +} + +void +nvkm_runl_rc_engn(struct nvkm_runl *runl, struct nvkm_engn *engn) +{ + struct nvkm_cgrp *cgrp; + unsigned long flags; + + /* Lookup channel group currently on engine. */ + cgrp = nvkm_engn_cgrp_get(engn, &flags); + if (!cgrp) { + ENGN_DEBUG(engn, "rc skipped, not on channel"); + return; + } + + nvkm_runl_rc_cgrp(cgrp); + nvkm_cgrp_put(&cgrp, flags); +} + +static void +nvkm_runl_work(struct work_struct *work) +{ + struct nvkm_runl *runl = container_of(work, typeof(*runl), work); + + mutex_lock(&runl->mutex); + nvkm_runl_rc(runl); + mutex_unlock(&runl->mutex); + +} + struct nvkm_chan * nvkm_runl_chan_get_inst(struct nvkm_runl *runl, u64 inst, unsigned long *pirqflags) { @@ -74,6 +227,27 @@ nvkm_runl_chan_get_chid(struct nvkm_runl *runl, int id, unsigned long *pirqflags return NULL; } +struct nvkm_cgrp * +nvkm_runl_cgrp_get_cgid(struct nvkm_runl *runl, int id, unsigned long *pirqflags) +{ + struct nvkm_chid *cgid = runl->cgid; + struct nvkm_cgrp *cgrp; + unsigned long flags; + + spin_lock_irqsave(&cgid->lock, flags); + if (!WARN_ON(id >= cgid->nr)) { + cgrp = cgid->data[id]; + if (likely(cgrp)) { + spin_lock(&cgrp->lock); + *pirqflags = flags; + spin_unlock(&cgid->lock); + return cgrp; + } + } + spin_unlock_irqrestore(&cgid->lock, flags); + return NULL; +} + int nvkm_runl_preempt_wait(struct nvkm_runl *runl) { @@ -81,6 +255,7 @@ nvkm_runl_preempt_wait(struct nvkm_runl *runl) if (!runl->func->preempt_pending(runl)) break; + nvkm_runl_rc(runl); usleep_range(1, 2); ) < 0 ? -ETIMEDOUT : 0; } @@ -91,6 +266,7 @@ nvkm_runl_update_pending(struct nvkm_runl *runl) if (!runl->func->pending(runl)) return false; + nvkm_runl_rc(runl); return true; } @@ -122,6 +298,12 @@ nvkm_runl_block(struct nvkm_runl *runl) spin_unlock_irqrestore(&fifo->lock, flags); } +void +nvkm_runl_fini(struct nvkm_runl *runl) +{ + flush_work(&runl->work); +} + void nvkm_runl_del(struct nvkm_runl *runl) { @@ -214,6 +396,9 @@ nvkm_runl_new(struct nvkm_fifo *fifo, int runi, u32 addr, int id_nr) INIT_LIST_HEAD(&runl->engns); INIT_LIST_HEAD(&runl->cgrps); mutex_init(&runl->mutex); + INIT_WORK(&runl->work, nvkm_runl_work); + atomic_set(&runl->rc_triggered, 0); + atomic_set(&runl->rc_pending, 0); list_add_tail(&runl->head, &fifo->runls); if (!fifo->chid) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 68d6854e6d0fb..47bffc7bb7c0c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -7,6 +7,10 @@ enum nvkm_subdev_type; struct nvkm_engn { const struct nvkm_engn_func { + bool (*chsw)(struct nvkm_engn *); + int (*cxid)(struct nvkm_engn *, bool *cgid); + void (*mmu_fault_trigger)(struct nvkm_engn *); + bool (*mmu_fault_triggered)(struct nvkm_engn *); } *func; struct nvkm_runl *runl; int id; @@ -28,6 +32,8 @@ struct nvkm_runl { bool (*pending)(struct nvkm_runl *); void (*block)(struct nvkm_runl *, u32 engm); void (*allow)(struct nvkm_runl *, u32 engm); + void (*fault_clear)(struct nvkm_runl *); + void (*preempt)(struct nvkm_runl *); bool (*preempt_pending)(struct nvkm_runl *); } *func; struct nvkm_fifo *fifo; @@ -50,6 +56,10 @@ struct nvkm_runl { int blocked; + struct work_struct work; + atomic_t rc_triggered; + atomic_t rc_pending; + struct list_head head; }; @@ -58,11 +68,16 @@ struct nvkm_runl *nvkm_runl_get(struct nvkm_fifo *, int runi, u32 addr); struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_engn_func *, enum nvkm_subdev_type, int inst); void nvkm_runl_del(struct nvkm_runl *); +void nvkm_runl_fini(struct nvkm_runl *); void nvkm_runl_block(struct nvkm_runl *); void nvkm_runl_allow(struct nvkm_runl *); bool nvkm_runl_update_pending(struct nvkm_runl *); int nvkm_runl_preempt_wait(struct nvkm_runl *); +void nvkm_runl_rc_engn(struct nvkm_runl *, struct nvkm_engn *); +void nvkm_runl_rc_cgrp(struct nvkm_cgrp *); + +struct nvkm_cgrp *nvkm_runl_cgrp_get_cgid(struct nvkm_runl *, int cgid, unsigned long *irqflags); struct nvkm_chan *nvkm_runl_chan_get_chid(struct nvkm_runl *, int chid, unsigned long *irqflags); struct nvkm_chan *nvkm_runl_chan_get_inst(struct nvkm_runl *, u64 inst, unsigned long *irqflags); @@ -74,6 +89,9 @@ struct nvkm_chan *nvkm_runl_chan_get_inst(struct nvkm_runl *, u64 inst, unsigned #define nvkm_runl_foreach_engn(engn,runl) list_for_each_entry((engn), &(runl)->engns, head) #define nvkm_runl_foreach_engn_cond(engn,runl,cond) \ nvkm_list_foreach(engn, &(runl)->engns, head, (cond)) +#define nvkm_runl_foreach_cgrp(cgrp,runl) list_for_each_entry((cgrp), &(runl)->cgrps, head) +#define nvkm_runl_foreach_cgrp_safe(cgrp,gtmp,runl) \ + list_for_each_entry_safe((cgrp), (gtmp), &(runl)->cgrps, head) #define RUNL_PRINT(r,l,p,f,a...) \ nvkm_printk__(&(r)->fifo->engine.subdev, NV_DBG_##l, p, "%06x:"f, (r)->addr, ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h index 011fbf69bb35e..2cb4836e8b314 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runq.h @@ -10,6 +10,7 @@ struct nvkm_runq { bool (*intr)(struct nvkm_runq *, struct nvkm_runl *); const struct nvkm_bitfield *intr_0_names; bool (*intr_1_ctxnotvalid)(struct nvkm_runq *, int chid); + bool (*idle)(struct nvkm_runq *); } *func; struct nvkm_fifo *fifo; int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 641c1ff0aa5d8..724a99a5597f4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -92,6 +92,7 @@ tu102_runl = { .pending = tu102_runl_pending, .block = gk104_runl_block, .allow = gk104_runl_allow, + .preempt = gv100_runl_preempt, .preempt_pending = gf100_runl_preempt_pending, }; @@ -123,155 +124,6 @@ tu102_fifo_mmu_fault_engine[] = { {} }; -static void -tu102_fifo_recover_work(struct work_struct *w) -{ - struct gk104_fifo *fifo = container_of(w, typeof(*fifo), recover.work); - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_engine *engine; - unsigned long flags; - u32 engm, runm, todo; - int engn, runl; - - spin_lock_irqsave(&fifo->base.lock, flags); - runm = fifo->recover.runm; - engm = fifo->recover.engm; - fifo->recover.engm = 0; - fifo->recover.runm = 0; - spin_unlock_irqrestore(&fifo->base.lock, flags); - - nvkm_mask(device, 0x002630, runm, runm); - - for (todo = engm; engn = __ffs(todo), todo; todo &= ~BIT(engn)) { - if ((engine = fifo->engine[engn].engine)) { - nvkm_subdev_fini(&engine->subdev, false); - WARN_ON(nvkm_subdev_init(&engine->subdev)); - } - } - - for (todo = runm; runl = __ffs(todo), todo; todo &= ~BIT(runl)) - gk104_fifo_runlist_update(fifo, runl); - - nvkm_mask(device, 0x002630, runm, 0x00000000); -} - -static void tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn); - -static void -tu102_fifo_recover_runl(struct gk104_fifo *fifo, int runl) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const u32 runm = BIT(runl); - - assert_spin_locked(&fifo->base.lock); - if (fifo->recover.runm & runm) - return; - fifo->recover.runm |= runm; - - /* Block runlist to prevent channel assignment(s) from changing. */ - nvkm_mask(device, 0x002630, runm, runm); - - /* Schedule recovery. */ - nvkm_warn(subdev, "runlist %d: scheduled for recovery\n", runl); - schedule_work(&fifo->recover.work); -} - -static struct gk104_fifo_chan * -tu102_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid) -{ - struct gk104_fifo_chan *chan; - struct nvkm_fifo_cgrp *cgrp; - - list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { - if (chan->base.chid == chid) { - list_del_init(&chan->head); - return chan; - } - } - - list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) { - if (cgrp->id == chid) { - chan = list_first_entry(&cgrp->chan, typeof(*chan), head); - list_del_init(&chan->head); - if (!--cgrp->chan_nr) - list_del_init(&cgrp->head); - return chan; - } - } - - return NULL; -} - -static void -tu102_fifo_recover_chan(struct nvkm_fifo *base, int chid) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const u32 stat = nvkm_rd32(device, 0x800004 + (chid * 0x08)); - const u32 runl = (stat & 0x000f0000) >> 16; - const bool used = (stat & 0x00000001); - unsigned long engn, engm = fifo->runlist[runl].engm; - struct gk104_fifo_chan *chan; - - assert_spin_locked(&fifo->base.lock); - if (!used) - return; - - /* Lookup SW state for channel, and mark it as dead. */ - chan = tu102_fifo_recover_chid(fifo, runl, chid); - if (chan) { - chan->killed = true; - nvkm_chan_error(&chan->base, false); - } - - /* Block channel assignments from changing during recovery. */ - tu102_fifo_recover_runl(fifo, runl); - - /* Schedule recovery for any engines the channel is on. */ - for_each_set_bit(engn, &engm, fifo->engine_nr) { - struct gk104_fifo_engine_status status; - - gk104_fifo_engine_status(fifo, engn, &status); - if (!status.chan || status.chan->id != chid) - continue; - tu102_fifo_recover_engn(fifo, engn); - } -} - -static void -tu102_fifo_recover_engn(struct gk104_fifo *fifo, int engn) -{ - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - const u32 runl = fifo->engine[engn].runl; - const u32 engm = BIT(engn); - struct gk104_fifo_engine_status status; - - assert_spin_locked(&fifo->base.lock); - if (fifo->recover.engm & engm) - return; - fifo->recover.engm |= engm; - - /* Block channel assignments from changing during recovery. */ - tu102_fifo_recover_runl(fifo, runl); - - /* Determine which channel (if any) is currently on the engine. */ - gk104_fifo_engine_status(fifo, engn, &status); - if (status.chan) { - /* The channel is not longer viable, kill it. */ - tu102_fifo_recover_chan(&fifo->base, status.chan->id); - } - - /* Preempt the runlist */ - nvkm_wr32(device, 0x2638, BIT(runl)); - - /* Schedule recovery. */ - nvkm_warn(subdev, "engine %d: scheduled for recovery\n", engn); - schedule_work(&fifo->recover.work); -} - const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault = { .recover = gf100_fifo_mmu_fault_recover, @@ -282,22 +134,55 @@ tu102_fifo_mmu_fault = { .gpcclient = gv100_fifo_mmu_fault_gpcclient, }; -static void -tu102_fifo_intr_ctxsw_timeout(struct gk104_fifo *fifo) +void +tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *engn, u32 info) { - struct nvkm_device *device = fifo->base.engine.subdev.device; - unsigned long flags, engm; - u32 engn; + struct nvkm_runl *runl = engn->runl; + struct nvkm_cgrp *cgrp; + unsigned long flags; + + /* Check that engine hasn't become unstuck since timeout raised. */ + ENGN_DEBUG(engn, "CTXSW_TIMEOUT %08x", info); + if (info & 0xc0000000) + return; - spin_lock_irqsave(&fifo->base.lock, flags); + /* Determine channel group the engine is stuck on, and schedule recovery. */ + switch (info & 0x0000c000) { + case 0x00004000: /* LOAD */ + cgrp = nvkm_runl_cgrp_get_cgid(runl, info & 0x3fff0000, &flags); + break; + case 0x00008000: /* SAVE */ + case 0x0000c000: /* SWITCH */ + cgrp = nvkm_runl_cgrp_get_cgid(runl, info & 0x00003fff, &flags); + break; + default: + cgrp = NULL; + break; + } - engm = nvkm_rd32(device, 0x2a30); - nvkm_wr32(device, 0x2a30, engm); + if (!WARN_ON(!cgrp)) { + nvkm_runl_rc_cgrp(cgrp); + nvkm_cgrp_put(&cgrp, flags); + } +} - for_each_set_bit(engn, &engm, 32) - tu102_fifo_recover_engn(fifo, engn); +static void +tu102_fifo_intr_ctxsw_timeout(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_runl *runl; + struct nvkm_engn *engn; + u32 engm = nvkm_rd32(device, 0x002a30); + u32 info; + + nvkm_runl_foreach(runl, fifo) { + nvkm_runl_foreach_engn_cond(engn, runl, engm & BIT(engn->id)) { + info = nvkm_rd32(device, 0x003200 + (engn->id * 4)); + tu102_fifo_intr_ctxsw_timeout_info(engn, info); + } + } - spin_unlock_irqrestore(&fifo->base.lock, flags); + nvkm_wr32(device, 0x002a30, engm); } static void @@ -326,7 +211,7 @@ tu102_fifo_intr(struct nvkm_inth *inth) } if (stat & 0x00000002) { - tu102_fifo_intr_ctxsw_timeout(gk104_fifo(fifo)); + tu102_fifo_intr_ctxsw_timeout(fifo); stat &= ~0x00000002; } @@ -386,11 +271,9 @@ tu102_fifo = { .runl_ctor = gk104_fifo_runl_ctor, .init = gk104_fifo_init, .init_pbdmas = tu102_fifo_init_pbdmas, - .fini = gk104_fifo_fini, .intr = tu102_fifo_intr, .mmu_fault = &tu102_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .recover_chan = tu102_fifo_recover_chan, .runlist = &tu102_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &tu102_runl, @@ -410,7 +293,6 @@ tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) return -ENOMEM; fifo->func = &tu102_fifo; - INIT_WORK(&fifo->recover.work, tu102_fifo_recover_work); *pfifo = &fifo->base; return nvkm_fifo_ctor(&tu102_fifo, device, type, inst, &fifo->base); -- GitLab From b084fff210bfd00de5cdef1802291272c77f581d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:35 +1000 Subject: [PATCH 1329/1927] drm/nouveau/fifo: add common runlist control - less dependence on waiting for runlist updates, on GPUs that allow it - supports runqueue selector in RAMRL entries - completes switch to common runl/cgrp/chan topology info Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 88 ++++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 5 +- .../drm/nouveau/nvkm/engine/fifo/chang84.c | 2 - .../drm/nouveau/nvkm/engine/fifo/changf100.h | 3 - .../drm/nouveau/nvkm/engine/fifo/changk104.h | 6 - .../drm/nouveau/nvkm/engine/fifo/channv50.c | 20 --- .../drm/nouveau/nvkm/engine/fifo/channv50.h | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/g98.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 69 ++------ .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 12 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 119 ++------------ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 24 --- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 17 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 21 +-- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 15 +- .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 27 --- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 40 ----- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 15 -- .../nouveau/nvkm/engine/fifo/gpfifotu102.c | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 31 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 154 ++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 18 +- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 26 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 11 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 28 ++-- .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 11 +- 35 files changed, 284 insertions(+), 505 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 798fbd3b530cb..2d34f1944d64a 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -17,6 +17,7 @@ struct nvkm_chan { const struct nvkm_chan_func *func; char name[64]; struct nvkm_cgrp *cgrp; + int runq; union { int id; int chid; }; /*FIXME: remove later */ @@ -63,7 +64,6 @@ struct nvkm_fifo { } timeout; int nr; - struct list_head chan; spinlock_t lock; struct mutex mutex; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 078a97ab5e375..0cbfb9225e6cf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -310,8 +310,6 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, if (ret) return ret; - INIT_LIST_HEAD(&fifo->chan); - nr = func->chid_nr(fifo); fifo->nr = nr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c index fd9e6144981e3..ac42dfc9d4cc4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c @@ -196,7 +196,7 @@ nvkm_cgrp_new(struct nvkm_runl *runl, const char *name, struct nvkm_vmm *vmm, bo cgrp->hw = hw; cgrp->id = -1; kref_init(&cgrp->kref); - cgrp->chans = NULL; + INIT_LIST_HEAD(&cgrp->chans); cgrp->chan_nr = 0; spin_lock_init(&cgrp->lock); INIT_LIST_HEAD(&cgrp->ectxs); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index e7ce66fecfd43..be9fbe9fab373 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -31,7 +31,7 @@ struct nvkm_cgrp { int id; struct kref kref; - struct nvkm_chan *chans; + struct list_head chans; int chan_nr; spinlock_t lock; /* protects irq handler channel (group) lookup */ @@ -46,7 +46,6 @@ struct nvkm_cgrp { atomic_t rc; struct list_head head; - struct list_head chan; }; int nvkm_cgrp_new(struct nvkm_runl *, const char *name, struct nvkm_vmm *, bool hw, @@ -59,9 +58,9 @@ void nvkm_cgrp_vctx_put(struct nvkm_cgrp *, struct nvkm_vctx **); void nvkm_cgrp_put(struct nvkm_cgrp **, unsigned long irqflags); -#define nvkm_cgrp_foreach_chan(chan,cgrp) for ((chan) = (cgrp)->chans; (chan); (chan) = NULL) +#define nvkm_cgrp_foreach_chan(chan,cgrp) list_for_each_entry((chan), &(cgrp)->chans, head) #define nvkm_cgrp_foreach_chan_safe(chan,ctmp,cgrp) \ - (void)(ctmp); nvkm_cgrp_foreach_chan((chan), (cgrp)) + list_for_each_entry_safe((chan), (ctmp), &(cgrp)->chans, head) #define CGRP_PRCLI(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) #define CGRP_PRINT(c,l,p,f,a...) RUNL_PRINT((c)->runl, l, p, "%04x:"f, (c)->id, ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 9b8f346d52b1a..3832a9aad80ae 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -339,22 +339,60 @@ nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, return 0; } -static int -nvkm_fifo_chan_fini(struct nvkm_object *object, bool suspend) +void +nvkm_chan_remove_locked(struct nvkm_chan *chan) { - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - if (chan->func->fini) - chan->func->fini(chan); - return 0; + struct nvkm_cgrp *cgrp = chan->cgrp; + struct nvkm_runl *runl = cgrp->runl; + + if (list_empty(&chan->head)) + return; + + CHAN_TRACE(chan, "remove"); + if (!--cgrp->chan_nr) { + runl->cgrp_nr--; + list_del(&cgrp->head); + } + runl->chan_nr--; + list_del_init(&chan->head); + atomic_set(&runl->changed, 1); } -static int -nvkm_fifo_chan_init(struct nvkm_object *object) +void +nvkm_chan_remove(struct nvkm_chan *chan, bool preempt) { - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - if (chan->func->init) - chan->func->init(chan); - return 0; + struct nvkm_runl *runl = chan->cgrp->runl; + + mutex_lock(&runl->mutex); + if (preempt && chan->func->preempt) + nvkm_chan_preempt_locked(chan, true); + nvkm_chan_remove_locked(chan); + nvkm_runl_update_locked(runl, true); + mutex_unlock(&runl->mutex); +} + +void +nvkm_chan_insert(struct nvkm_chan *chan) +{ + struct nvkm_cgrp *cgrp = chan->cgrp; + struct nvkm_runl *runl = cgrp->runl; + + mutex_lock(&runl->mutex); + if (WARN_ON(!list_empty(&chan->head))) { + mutex_unlock(&runl->mutex); + return; + } + + CHAN_TRACE(chan, "insert"); + list_add_tail(&chan->head, &cgrp->chans); + runl->chan_nr++; + if (!cgrp->chan_nr++) { + list_add_tail(&cgrp->head, &cgrp->runl->cgrps); + runl->cgrp_nr++; + } + atomic_set(&runl->changed, 1); + nvkm_runl_update_locked(runl, true); + mutex_unlock(&runl->mutex); } static void @@ -420,15 +458,7 @@ static void * nvkm_fifo_chan_dtor(struct nvkm_object *object) { struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - struct nvkm_fifo *fifo = chan->fifo; void *data = chan->func->dtor(chan); - unsigned long flags; - - spin_lock_irqsave(&fifo->lock, flags); - if (!list_empty(&chan->head)) { - list_del(&chan->head); - } - spin_unlock_irqrestore(&fifo->lock, flags); if (chan->vmm) { nvkm_vmm_part(chan->vmm, chan->inst->memory); @@ -494,8 +524,6 @@ nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags) static const struct nvkm_object_func nvkm_fifo_chan_func = { .dtor = nvkm_fifo_chan_dtor, - .init = nvkm_fifo_chan_init, - .fini = nvkm_fifo_chan_fini, .map = nvkm_fifo_chan_map, }; @@ -514,7 +542,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, struct nvkm_runl *runl; struct nvkm_engn *engn = NULL; struct nvkm_vmm *vmm = NULL; - unsigned long flags; int ret; nvkm_runl_foreach(runl, fifo) { @@ -532,8 +559,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, *func = *fifo->func->chan.func; func->dtor = fn->dtor; - func->init = fn->init; - func->fini = fn->fini; func->engine_ctor = fn->engine_ctor; func->engine_dtor = fn->engine_dtor; func->engine_init = fn->engine_init; @@ -611,23 +636,14 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, } /* Allocate channel ID. */ - if (runl->cgid) { - chan->id = chan->cgrp->id; - runl->chid->data[chan->id] = chan; - set_bit(chan->id, runl->chid->used); - goto temp_hack_until_no_chid_eq_cgid_req; - } - chan->id = nvkm_chid_get(runl->chid, chan); if (chan->id < 0) { RUNL_ERROR(runl, "!chids"); return -ENOSPC; } -temp_hack_until_no_chid_eq_cgid_req: - spin_lock_irqsave(&fifo->lock, flags); - list_add(&chan->head, &fifo->chan); - spin_unlock_irqrestore(&fifo->lock, flags); + if (cgrp->id < 0) + cgrp->id = chan->id; /* determine address of this channel's user registers */ chan->addr = device->func->resource_addr(device, bar) + diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 2ad385914bec8..804441f30f440 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -24,8 +24,6 @@ struct nvkm_chan_func { u32 (*doorbell_handle)(struct nvkm_chan *); void *(*dtor)(struct nvkm_fifo_chan *); - void (*init)(struct nvkm_fifo_chan *); - void (*fini)(struct nvkm_fifo_chan *); int (*engine_ctor)(struct nvkm_fifo_chan *, struct nvkm_engine *, struct nvkm_object *); void (*engine_dtor)(struct nvkm_fifo_chan *, struct nvkm_engine *); @@ -44,6 +42,9 @@ void nvkm_chan_del(struct nvkm_chan **); void nvkm_chan_allow(struct nvkm_chan *); void nvkm_chan_block(struct nvkm_chan *); void nvkm_chan_error(struct nvkm_chan *, bool preempt); +void nvkm_chan_insert(struct nvkm_chan *); +void nvkm_chan_remove(struct nvkm_chan *, bool preempt); +void nvkm_chan_remove_locked(struct nvkm_chan *); int nvkm_chan_preempt(struct nvkm_chan *, bool wait); int nvkm_chan_preempt_locked(struct nvkm_chan *, bool wait); int nvkm_chan_cctx_get(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx **, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 100631a44a85e..64566b5cc23f3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -169,8 +169,6 @@ g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, static const struct nvkm_fifo_chan_func g84_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .init = nv50_fifo_chan_init, - .fini = nv50_fifo_chan_fini, .engine_ctor = g84_fifo_chan_engine_ctor, .engine_dtor = nv50_fifo_chan_engine_dtor, .engine_init = g84_fifo_chan_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h index f7ac1061fa849..3c7b6c0d8cf12 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h @@ -9,9 +9,6 @@ struct gf100_fifo_chan { struct nvkm_fifo_chan base; struct gf100_fifo *fifo; - struct list_head head; - bool killed; - #define GF100_FIFO_ENGN_GR 0 #define GF100_FIFO_ENGN_MSPDEC 1 #define GF100_FIFO_ENGN_MSPPP 2 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 7a83b8667443a..95e788f31590d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -10,10 +10,6 @@ struct gk104_fifo_chan { struct gk104_fifo *fifo; int runl; - struct nvkm_fifo_cgrp *cgrp; - struct list_head head; - bool killed; - #define GK104_FIFO_ENGN_SW 15 struct gk104_fifo_engn { struct nvkm_gpuobj *inst; @@ -26,8 +22,6 @@ extern const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func; int gk104_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); void *gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *); -void gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *); -void gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *); struct gk104_fifo_engn *gk104_fifo_gpfifo_engine(struct gk104_fifo_chan *, struct nvkm_engine *); int gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *, struct nvkm_engine *, struct nvkm_object *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 7669d17406c69..5d6f82236c50b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -183,24 +183,6 @@ nv50_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, return nvkm_ramht_insert(chan->ramht, object, 0, 4, handle, context); } -void -nv50_fifo_chan_fini(struct nvkm_fifo_chan *base) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nv50_fifo *fifo = chan->fifo; - - nv50_fifo_runlist_update(fifo); -} - -void -nv50_fifo_chan_init(struct nvkm_fifo_chan *base) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nv50_fifo *fifo = chan->fifo; - - nv50_fifo_runlist_update(fifo); -} - void * nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) { @@ -216,8 +198,6 @@ nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) static const struct nvkm_fifo_chan_func nv50_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .init = nv50_fifo_chan_init, - .fini = nv50_fifo_chan_fini, .engine_ctor = nv50_fifo_chan_engine_ctor, .engine_dtor = nv50_fifo_chan_engine_dtor, .engine_init = nv50_fifo_chan_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index 5d5d9f3d99280..ce7d61e8d297d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -40,8 +40,6 @@ struct nv50_fifo_chan { int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *); -void nv50_fifo_chan_init(struct nvkm_fifo_chan *); -void nv50_fifo_chan_fini(struct nvkm_fifo_chan *); struct nvkm_gpuobj **nv50_fifo_chan_engine(struct nv50_fifo_chan *, struct nvkm_engine *); void nv50_fifo_chan_engine_dtor(struct nvkm_fifo_chan *, struct nvkm_engine *); void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *, int); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 91b43f17120b7..18ec982f4f54d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -125,7 +125,6 @@ g84_fifo_runl_ctor(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func g84_fifo = { .dtor = nv50_fifo_dtor, - .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, .runl_ctor = g84_fifo_runl_ctor, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c index a912e1ba4339c..8016a2228e92d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c @@ -50,7 +50,6 @@ g98_fifo_runl_ctor(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func g98_fifo = { .dtor = nv50_fifo_dtor, - .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, .runl_ctor = g98_fifo_runl_ctor, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 4fc9e80b5f692..767507e0e2390 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -287,64 +287,38 @@ gf100_runl_pending(struct nvkm_runl *runl) return nvkm_rd32(runl->fifo->engine.subdev.device, 0x00227c) & 0x00100000; } -void -gf100_fifo_runlist_commit(struct gf100_fifo *fifo) +static void +gf100_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) { - struct gf100_fifo_chan *chan; - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_runl *runl = nvkm_runl_first(&fifo->base); - struct nvkm_memory *cur; - int nr = 0; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u64 addr = nvkm_memory_addr(memory) + start; int target; - mutex_lock(&fifo->base.mutex); - cur = fifo->runlist.mem[fifo->runlist.active]; - fifo->runlist.active = !fifo->runlist.active; - - nvkm_kmap(cur); - list_for_each_entry(chan, &fifo->chan, head) { - nvkm_wo32(cur, (nr * 8) + 0, chan->base.chid); - nvkm_wo32(cur, (nr * 8) + 4, 0x00000004); - nr++; - } - nvkm_done(cur); - - switch (nvkm_memory_target(cur)) { + switch (nvkm_memory_target(memory)) { case NVKM_MEM_TARGET_VRAM: target = 0; break; case NVKM_MEM_TARGET_NCOH: target = 3; break; default: - mutex_unlock(&fifo->base.mutex); WARN_ON(1); return; } - nvkm_wr32(device, 0x002270, (nvkm_memory_addr(cur) >> 12) | - (target << 28)); - nvkm_wr32(device, 0x002274, 0x01f00000 | nr); - - runl->func->wait(runl); - mutex_unlock(&fifo->base.mutex); -} - -void -gf100_fifo_runlist_remove(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan) -{ - mutex_lock(&fifo->base.mutex); - list_del_init(&chan->head); - mutex_unlock(&fifo->base.mutex); + nvkm_wr32(device, 0x002270, (target << 28) | (addr >> 12)); + nvkm_wr32(device, 0x002274, 0x01f00000 | count); } -void -gf100_fifo_runlist_insert(struct gf100_fifo *fifo, struct gf100_fifo_chan *chan) +static void +gf100_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) { - mutex_lock(&fifo->base.mutex); - list_add_tail(&chan->head, &fifo->chan); - mutex_unlock(&fifo->base.mutex); + nvkm_wo32(memory, offset + 0, chan->id); + nvkm_wo32(memory, offset + 4, 0x00000004); } static const struct nvkm_runl_func gf100_runl = { + .size = 8, + .update = nv50_runl_update, + .insert_chan = gf100_runl_insert_chan, + .commit = gf100_runl_commit, .wait = nv50_runl_wait, .pending = gf100_runl_pending, .block = gf100_runl_block, @@ -884,16 +858,6 @@ gf100_fifo_oneinit(struct nvkm_fifo *base) struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); int ret; - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, - false, &fifo->runlist.mem[0]); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, - false, &fifo->runlist.mem[1]); - if (ret) - return ret; - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000, 0x1000, false, &fifo->user.mem); if (ret) @@ -914,8 +878,6 @@ gf100_fifo_dtor(struct nvkm_fifo *base) struct nvkm_device *device = fifo->base.engine.subdev.device; nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar); nvkm_memory_unref(&fifo->user.mem); - nvkm_memory_unref(&fifo->runlist.mem[0]); - nvkm_memory_unref(&fifo->runlist.mem[1]); return fifo; } @@ -950,7 +912,6 @@ gf100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) return -ENOMEM; - INIT_LIST_HEAD(&fifo->chan); *pfifo = &fifo->base; return nvkm_fifo_ctor(&gf100_fifo, device, type, inst, &fifo->base); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index 6d7771f505c6d..42a950c004830 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -6,24 +6,12 @@ #include -struct gf100_fifo_chan; struct gf100_fifo { struct nvkm_fifo base; - struct list_head chan; - - struct { - struct nvkm_memory *mem[2]; - int active; - } runlist; - struct { struct nvkm_memory *mem; struct nvkm_vma *bar; } user; }; - -void gf100_fifo_runlist_insert(struct gf100_fifo *, struct gf100_fifo_chan *); -void gf100_fifo_runlist_remove(struct gf100_fifo *, struct gf100_fifo_chan *); -void gf100_fifo_runlist_commit(struct gf100_fifo *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index d8cb2626b1884..b5e698a223b6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -326,15 +326,14 @@ gk104_runl_pending(struct nvkm_runl *runl) } void -gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, - struct nvkm_memory *mem, int nr) +gk104_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) { - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_runl *rl = nvkm_runl_get(&fifo->base, runl, 0); + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; + u64 addr = nvkm_memory_addr(memory) + start; int target; - switch (nvkm_memory_target(mem)) { + switch (nvkm_memory_target(memory)) { case NVKM_MEM_TARGET_VRAM: target = 0; break; case NVKM_MEM_TARGET_NCOH: target = 3; break; default: @@ -342,88 +341,25 @@ gk104_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, return; } - nvkm_wr32(device, 0x002270, (nvkm_memory_addr(mem) >> 12) | - (target << 28)); - nvkm_wr32(device, 0x002274, (runl << 20) | nr); - - rl->func->wait(rl); -} - -void -gk104_fifo_runlist_update(struct gk104_fifo *fifo, int runl) -{ - const struct gk104_fifo_runlist_func *func = fifo->func->runlist; - struct gk104_fifo_chan *chan; - struct nvkm_memory *mem; - struct nvkm_fifo_cgrp *cgrp; - int nr = 0; - - mutex_lock(&fifo->base.mutex); - mem = fifo->runlist[runl].mem[fifo->runlist[runl].next]; - fifo->runlist[runl].next = !fifo->runlist[runl].next; - - nvkm_kmap(mem); - list_for_each_entry(chan, &fifo->runlist[runl].chan, head) { - func->chan(chan, mem, nr++ * func->size); - } - - list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) { - func->cgrp(cgrp, mem, nr++ * func->size); - list_for_each_entry(chan, &cgrp->chan, head) { - func->chan(chan, mem, nr++ * func->size); - } - } - nvkm_done(mem); - - func->commit(fifo, runl, mem, nr); - mutex_unlock(&fifo->base.mutex); -} - -void -gk104_fifo_runlist_remove(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan) -{ - struct nvkm_fifo_cgrp *cgrp = chan->cgrp; - mutex_lock(&fifo->base.mutex); - if (!list_empty(&chan->head)) { - list_del_init(&chan->head); - if (cgrp && !--cgrp->chan_nr) - list_del_init(&cgrp->head); - } - mutex_unlock(&fifo->base.mutex); -} - -void -gk104_fifo_runlist_insert(struct gk104_fifo *fifo, struct gk104_fifo_chan *chan) -{ - struct nvkm_fifo_cgrp *cgrp = chan->cgrp; - mutex_lock(&fifo->base.mutex); - if (cgrp) { - if (!cgrp->chan_nr++) - list_add_tail(&cgrp->head, &fifo->runlist[chan->runl].cgrp); - list_add_tail(&chan->head, &cgrp->chan); - } else { - list_add_tail(&chan->head, &fifo->runlist[chan->runl].chan); - } - mutex_unlock(&fifo->base.mutex); + spin_lock_irq(&fifo->lock); + nvkm_wr32(device, 0x002270, (target << 28) | (addr >> 12)); + nvkm_wr32(device, 0x002274, (runl->id << 20) | count); + spin_unlock_irq(&fifo->lock); } void -gk104_fifo_runlist_chan(struct gk104_fifo_chan *chan, - struct nvkm_memory *memory, u32 offset) +gk104_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) { - nvkm_wo32(memory, offset + 0, chan->base.chid); + nvkm_wo32(memory, offset + 0, chan->id); nvkm_wo32(memory, offset + 4, 0x00000000); } -const struct gk104_fifo_runlist_func -gk104_fifo_runlist = { - .size = 8, - .chan = gk104_fifo_runlist_chan, - .commit = gk104_fifo_runlist_commit, -}; - static const struct nvkm_runl_func gk104_runl = { + .size = 8, + .update = nv50_runl_update, + .insert_chan = gk104_runl_insert_chan, + .commit = gk104_runl_commit, .wait = nv50_runl_wait, .pending = gk104_runl_pending, .block = gk104_runl_block, @@ -793,7 +729,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) struct nvkm_device *device = subdev->device; struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); struct nvkm_top_device *tdev; - int ret, i, j; + int ret; /* Determine runlist configuration from topology device info. */ list_for_each_entry(tdev, &device->top->device, head) { @@ -811,21 +747,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) fifo->runlist_nr = max(fifo->runlist_nr, tdev->runlist + 1); } - for (i = 0; i < fifo->runlist_nr; i++) { - for (j = 0; j < ARRAY_SIZE(fifo->runlist[i].mem); j++) { - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - fifo->base.nr * 2/* TSG+chan */ * - fifo->func->runlist->size, - 0x1000, false, - &fifo->runlist[i].mem[j]); - if (ret) - return ret; - } - - INIT_LIST_HEAD(&fifo->runlist[i].cgrp); - INIT_LIST_HEAD(&fifo->runlist[i].chan); - } - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, fifo->base.nr * 0x200, 0x1000, true, &fifo->user.mem); @@ -845,16 +766,9 @@ gk104_fifo_dtor(struct nvkm_fifo *base) { struct gk104_fifo *fifo = gk104_fifo(base); struct nvkm_device *device = fifo->base.engine.subdev.device; - int i; nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar); nvkm_memory_unref(&fifo->user.mem); - - for (i = 0; i < fifo->runlist_nr; i++) { - nvkm_memory_unref(&fifo->runlist[i].mem[1]); - nvkm_memory_unref(&fifo->runlist[i].mem[0]); - } - return fifo; } @@ -887,7 +801,6 @@ gk104_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gk104_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk104_runl, .runq = &gk104_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 64d9b1e857e10..3915b3b9e6bf4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -20,10 +20,6 @@ struct gk104_fifo { int engine_nr; struct { - struct nvkm_memory *mem[2]; - int next; - struct list_head cgrp; - struct list_head chan; u32 engm; u32 engm_sw; } runlist[16]; @@ -37,27 +33,7 @@ struct gk104_fifo { int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int index, int nr, struct nvkm_fifo **); -void gk104_fifo_runlist_insert(struct gk104_fifo *, struct gk104_fifo_chan *); -void gk104_fifo_runlist_remove(struct gk104_fifo *, struct gk104_fifo_chan *); -void gk104_fifo_runlist_update(struct gk104_fifo *, int runl); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); void gk104_fifo_init(struct nvkm_fifo *base); - -extern const struct gk104_fifo_runlist_func gk104_fifo_runlist; -void gk104_fifo_runlist_chan(struct gk104_fifo_chan *, - struct nvkm_memory *, u32); -void gk104_fifo_runlist_commit(struct gk104_fifo *, int runl, - struct nvkm_memory *, int); - -extern const struct gk104_fifo_runlist_func gk110_fifo_runlist; -void gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, - struct nvkm_memory *, u32); - -extern const struct gk104_fifo_runlist_func gm107_fifo_runlist; - -void gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *, - struct nvkm_memory *, u32); -void gv100_fifo_runlist_chan(struct gk104_fifo_chan *, - struct nvkm_memory *, u32); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index f27b7ea23e153..812fefff09255 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -68,24 +68,20 @@ gk110_cgrp = { }; void -gk110_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp, - struct nvkm_memory *memory, u32 offset) +gk110_runl_insert_cgrp(struct nvkm_cgrp *cgrp, struct nvkm_memory *memory, u64 offset) { nvkm_wo32(memory, offset + 0, (cgrp->chan_nr << 26) | (128 << 18) | (3 << 14) | 0x00002000 | cgrp->id); nvkm_wo32(memory, offset + 4, 0x00000000); } -const struct gk104_fifo_runlist_func -gk110_fifo_runlist = { - .size = 8, - .cgrp = gk110_fifo_runlist_cgrp, - .chan = gk104_fifo_runlist_chan, - .commit = gk104_fifo_runlist_commit, -}; - const struct nvkm_runl_func gk110_runl = { + .size = 8, + .update = nv50_runl_update, + .insert_cgrp = gk110_runl_insert_cgrp, + .insert_chan = gk104_runl_insert_chan, + .commit = gk104_runl_commit, .wait = nv50_runl_wait, .pending = gk104_runl_pending, .block = gk104_runl_block, @@ -121,7 +117,6 @@ gk110_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk104_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 9886bd38a2120..81d184473f799 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -64,7 +64,6 @@ gk208_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 9177383d5f79f..47b7d93f5a042 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -39,7 +39,6 @@ gk20a_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gk110_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index bf8671bf3892c..c0eca7da7d098 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -42,23 +42,19 @@ gm107_chan = { }; static void -gm107_fifo_runlist_chan(struct gk104_fifo_chan *chan, - struct nvkm_memory *memory, u32 offset) +gm107_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) { - nvkm_wo32(memory, offset + 0, chan->base.chid); - nvkm_wo32(memory, offset + 4, chan->base.inst->addr >> 12); + nvkm_wo32(memory, offset + 0, chan->id); + nvkm_wo32(memory, offset + 4, chan->inst->addr >> 12); } -const struct gk104_fifo_runlist_func -gm107_fifo_runlist = { - .size = 8, - .cgrp = gk110_fifo_runlist_cgrp, - .chan = gm107_fifo_runlist_chan, - .commit = gk104_fifo_runlist_commit, -}; - const struct nvkm_runl_func gm107_runl = { + .size = 8, + .update = nv50_runl_update, + .insert_cgrp = gk110_runl_insert_cgrp, + .insert_chan = gm107_runl_insert_chan, + .commit = gk104_runl_commit, .wait = nv50_runl_wait, .pending = gk104_runl_pending, .block = gk104_runl_block, @@ -145,7 +141,6 @@ gm107_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gm107_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gm107_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 13c293aba6f2d..8be59812797a0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -53,7 +53,6 @@ gm200_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gm107_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gm107_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 7698d640a6f72..a353c6b860ef8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -25,12 +25,26 @@ #include "gk104.h" #include "changk104.h" +#include #include #include +static void +gp100_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) +{ + nvkm_wo32(memory, offset + 0, chan->id | chan->runq << 14); + nvkm_wo32(memory, offset + 4, chan->inst->addr >> 12); +} + static const struct nvkm_runl_func gp100_runl = { + .runqs = 2, + .size = 8, + .update = nv50_runl_update, + .insert_cgrp = gk110_runl_insert_cgrp, + .insert_chan = gp100_runl_insert_chan, + .commit = gk104_runl_commit, .wait = nv50_runl_wait, .pending = gk104_runl_pending, .block = gk104_runl_block, @@ -112,7 +126,6 @@ gp100_fifo = { .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gp100_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gm107_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gp100_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 77a4f2346f04b..871f199ddccf6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -127,30 +127,6 @@ gf100_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, return nvkm_memory_map(engn->inst, 0, chan->base.vmm, engn->vma, NULL, 0); } -static void -gf100_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) -{ - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct gf100_fifo *fifo = chan->fifo; - - if (!list_empty(&chan->head) && !chan->killed) { - gf100_fifo_runlist_remove(fifo, chan); - gf100_fifo_runlist_commit(fifo); - } -} - -static void -gf100_fifo_gpfifo_init(struct nvkm_fifo_chan *base) -{ - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct gf100_fifo *fifo = chan->fifo; - - if (list_empty(&chan->head) && !chan->killed) { - gf100_fifo_runlist_insert(fifo, chan); - gf100_fifo_runlist_commit(fifo); - } -} - static void * gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { @@ -160,8 +136,6 @@ gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) static const struct nvkm_fifo_chan_func gf100_fifo_gpfifo_func = { .dtor = gf100_fifo_gpfifo_dtor, - .init = gf100_fifo_gpfifo_init, - .fini = gf100_fifo_gpfifo_fini, .engine_ctor = gf100_fifo_gpfifo_engine_ctor, .engine_dtor = gf100_fifo_gpfifo_engine_dtor, .engine_init = gf100_fifo_gpfifo_engine_init, @@ -197,7 +171,6 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return -ENOMEM; *pobject = &chan->base.object; chan->fifo = fifo; - INIT_LIST_HEAD(&chan->head); ret = nvkm_fifo_chan_ctor(&gf100_fifo_gpfifo_func, &fifo->base, 0x1000, 0x1000, true, args->v0.vmm, 0, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 9440cf57a1b83..aa3ea7f7f8e19 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -152,43 +152,16 @@ gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, return nvkm_memory_map(engn->inst, 0, chan->base.vmm, engn->vma, NULL, 0); } -void -gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct gk104_fifo *fifo = chan->fifo; - - if (!list_empty(&chan->head)) { - gk104_fifo_runlist_remove(fifo, chan); - gk104_fifo_runlist_update(fifo, chan->runl); - } -} - -void -gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct gk104_fifo *fifo = chan->fifo; - - if (list_empty(&chan->head) && !chan->killed) { - gk104_fifo_runlist_insert(fifo, chan); - gk104_fifo_runlist_update(fifo, chan->runl); - } -} - void * gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - kfree(chan->cgrp); return chan; } const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func = { .dtor = gk104_fifo_gpfifo_dtor, - .init = gk104_fifo_gpfifo_init, - .fini = gk104_fifo_gpfifo_fini, .engine_ctor = gk104_fifo_gpfifo_engine_ctor, .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gk104_fifo_gpfifo_engine_init, @@ -215,7 +188,6 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, *pobject = &chan->base.object; chan->fifo = fifo; chan->runl = runlist; - INIT_LIST_HEAD(&chan->head); ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, 0x1000, 0x1000, true, vmm, 0, fifo->runlist[runlist].engm_sw, @@ -227,18 +199,6 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, *chid = chan->base.chid; *inst = chan->base.inst->addr; - /* Hack to support GPUs where even individual channels should be - * part of a channel group. - */ - if (fifo->func->cgrp.force) { - if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL))) - return -ENOMEM; - chan->cgrp->id = chan->base.chid; - INIT_LIST_HEAD(&chan->cgrp->head); - INIT_LIST_HEAD(&chan->cgrp->chan); - chan->cgrp->chan_nr = 0; - } - /* Clear channel control registers. */ usermem = chan->base.chid * 0x200; ilength = order_base_2(ilength / 8); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 66d6079ff210c..428901abdb813 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -105,8 +105,6 @@ gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, static const struct nvkm_fifo_chan_func gv100_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, - .init = gk104_fifo_gpfifo_init, - .fini = gk104_fifo_gpfifo_fini, .engine_ctor = gk104_fifo_gpfifo_engine_ctor, .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, @@ -134,7 +132,6 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, *pobject = &chan->base.object; chan->fifo = fifo; chan->runl = runlist; - INIT_LIST_HEAD(&chan->head); ret = nvkm_fifo_chan_ctor(func, &fifo->base, 0x1000, 0x1000, true, vmm, 0, fifo->runlist[runlist].engm, 1, fifo->user.bar->addr, 0x200, @@ -146,18 +143,6 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, *inst = chan->base.inst->addr; *token = chan->base.func->doorbell_handle(&chan->base); - /* Hack to support GPUs where even individual channels should be - * part of a channel group. - */ - if (fifo->func->cgrp.force) { - if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL))) - return -ENOMEM; - chan->cgrp->id = chan->base.chid; - INIT_LIST_HEAD(&chan->cgrp->head); - INIT_LIST_HEAD(&chan->cgrp->chan); - chan->cgrp->chan_nr = 0; - } - /* Clear channel control registers. */ usermem = chan->base.chid * 0x200; ilength = order_base_2(ilength / 8); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index 1c48a109a2b8a..6562bdf39cb0d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -31,8 +31,6 @@ static const struct nvkm_fifo_chan_func tu102_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, - .init = gk104_fifo_gpfifo_init, - .fini = gk104_fifo_gpfifo_fini, .engine_ctor = gk104_fifo_gpfifo_engine_ctor, .engine_dtor = gk104_fifo_gpfifo_engine_dtor, .engine_init = gv100_fifo_gpfifo_engine_init, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 4ff2c75f53ab1..29a83ed3404f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -98,22 +98,20 @@ gv100_runl_preempt(struct nvkm_runl *runl) } void -gv100_fifo_runlist_chan(struct gk104_fifo_chan *chan, - struct nvkm_memory *memory, u32 offset) +gv100_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) { - struct nvkm_memory *usermem = chan->fifo->user.mem; - const u64 user = nvkm_memory_addr(usermem) + (chan->base.chid * 0x200); - const u64 inst = chan->base.inst->addr; + struct nvkm_memory *usermem = gk104_fifo(chan->cgrp->runl->fifo)->user.mem; + const u64 user = nvkm_memory_addr(usermem) + (chan->id * 0x200); + const u64 inst = chan->inst->addr; - nvkm_wo32(memory, offset + 0x0, lower_32_bits(user)); + nvkm_wo32(memory, offset + 0x0, lower_32_bits(user) | chan->runq << 1); nvkm_wo32(memory, offset + 0x4, upper_32_bits(user)); - nvkm_wo32(memory, offset + 0x8, lower_32_bits(inst) | chan->base.chid); + nvkm_wo32(memory, offset + 0x8, lower_32_bits(inst) | chan->id); nvkm_wo32(memory, offset + 0xc, upper_32_bits(inst)); } void -gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp, - struct nvkm_memory *memory, u32 offset) +gv100_runl_insert_cgrp(struct nvkm_cgrp *cgrp, struct nvkm_memory *memory, u64 offset) { nvkm_wo32(memory, offset + 0x0, (128 << 24) | (3 << 16) | 0x00000001); nvkm_wo32(memory, offset + 0x4, cgrp->chan_nr); @@ -121,16 +119,14 @@ gv100_fifo_runlist_cgrp(struct nvkm_fifo_cgrp *cgrp, nvkm_wo32(memory, offset + 0xc, 0x00000000); } -static const struct gk104_fifo_runlist_func -gv100_fifo_runlist = { - .size = 16, - .cgrp = gv100_fifo_runlist_cgrp, - .chan = gv100_fifo_runlist_chan, - .commit = gk104_fifo_runlist_commit, -}; - static const struct nvkm_runl_func gv100_runl = { + .runqs = 2, + .size = 16, + .update = nv50_runl_update, + .insert_cgrp = gv100_runl_insert_cgrp, + .insert_chan = gv100_runl_insert_chan, + .commit = gk104_runl_commit, .wait = nv50_runl_wait, .pending = gk104_runl_pending, .block = gk104_runl_block, @@ -401,7 +397,6 @@ gv100_fifo = { .intr_ctxsw_timeout = gv100_fifo_intr_ctxsw_timeout, .mmu_fault = &gv100_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &gv100_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &gv100_runl, .runq = &gv100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 38d36749862c0..6a069e84006c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -82,36 +82,6 @@ const struct nvkm_engn_func nv50_engn_sw = { }; -static void -nv50_fifo_runlist_update_locked(struct nv50_fifo *fifo) -{ - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_memory *cur; - int i, p; - - cur = fifo->runlist[fifo->cur_runlist]; - fifo->cur_runlist = !fifo->cur_runlist; - - nvkm_kmap(cur); - for (i = 0, p = 0; i < fifo->base.nr; i++) { - if (nvkm_rd32(device, 0x002600 + (i * 4)) & 0x80000000) - nvkm_wo32(cur, p++ * 4, i); - } - nvkm_done(cur); - - nvkm_wr32(device, 0x0032f4, nvkm_memory_addr(cur) >> 12); - nvkm_wr32(device, 0x0032ec, p); - nvkm_wr32(device, 0x002500, 0x00000101); -} - -void -nv50_fifo_runlist_update(struct nv50_fifo *fifo) -{ - mutex_lock(&fifo->base.mutex); - nv50_fifo_runlist_update_locked(fifo); - mutex_unlock(&fifo->base.mutex); -} - static bool nv50_runl_pending(struct nvkm_runl *runl) { @@ -132,17 +102,112 @@ nv50_runl_wait(struct nvkm_runl *runl) return -ETIMEDOUT; } +static void +nv50_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u64 addr = nvkm_memory_addr(memory) + start; + + nvkm_wr32(device, 0x0032f4, addr >> 12); + nvkm_wr32(device, 0x0032ec, count); +} + +static void +nv50_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) +{ + nvkm_wo32(memory, offset, chan->id); +} + +static struct nvkm_memory * +nv50_runl_alloc(struct nvkm_runl *runl, u32 *offset) +{ + const u32 segment = ALIGN((runl->cgrp_nr + runl->chan_nr) * runl->func->size, 0x1000); + const u32 maxsize = (runl->cgid ? runl->cgid->nr : 0) + runl->chid->nr; + int ret; + + if (unlikely(!runl->mem)) { + ret = nvkm_memory_new(runl->fifo->engine.subdev.device, NVKM_MEM_TARGET_INST, + maxsize * 2 * runl->func->size, 0, false, &runl->mem); + if (ret) { + RUNL_ERROR(runl, "alloc %d\n", ret); + return ERR_PTR(ret); + } + } else { + if (runl->offset + segment >= nvkm_memory_size(runl->mem)) { + ret = runl->func->wait(runl); + if (ret) { + RUNL_DEBUG(runl, "rewind timeout"); + return ERR_PTR(ret); + } + + runl->offset = 0; + } + } + + *offset = runl->offset; + runl->offset += segment; + return runl->mem; +} + +int +nv50_runl_update(struct nvkm_runl *runl) +{ + struct nvkm_memory *memory; + struct nvkm_cgrp *cgrp; + struct nvkm_chan *chan; + u32 start, offset, count; + + /*TODO: prio, interleaving. */ + + RUNL_TRACE(runl, "RAMRL: update cgrps:%d chans:%d", runl->cgrp_nr, runl->chan_nr); + memory = nv50_runl_alloc(runl, &start); + if (IS_ERR(memory)) + return PTR_ERR(memory); + + RUNL_TRACE(runl, "RAMRL: update start:%08x", start); + offset = start; + + nvkm_kmap(memory); + nvkm_runl_foreach_cgrp(cgrp, runl) { + if (cgrp->hw) { + CGRP_TRACE(cgrp, " RAMRL+%08x: chans:%d", offset, cgrp->chan_nr); + runl->func->insert_cgrp(cgrp, memory, offset); + offset += runl->func->size; + } + + nvkm_cgrp_foreach_chan(chan, cgrp) { + CHAN_TRACE(chan, "RAMRL+%08x: [%s]", offset, chan->name); + runl->func->insert_chan(chan, memory, offset); + offset += runl->func->size; + } + } + nvkm_done(memory); + + /*TODO: look into using features on newer HW to guarantee forward progress. */ + list_rotate_left(&runl->cgrps); + + count = (offset - start) / runl->func->size; + RUNL_TRACE(runl, "RAMRL: commit start:%08x count:%d", start, count); + + runl->func->commit(runl, memory, start, count); + return 0; +} + const struct nvkm_runl_func nv50_runl = { + .size = 4, + .update = nv50_runl_update, + .insert_chan = nv50_runl_insert_chan, + .commit = nv50_runl_commit, .wait = nv50_runl_wait, .pending = nv50_runl_pending, }; void -nv50_fifo_init(struct nvkm_fifo *base) +nv50_fifo_init(struct nvkm_fifo *fifo) { - struct nv50_fifo *fifo = nv50_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_runl *runl = nvkm_runl_first(fifo); + struct nvkm_device *device = fifo->engine.subdev.device; int i; nvkm_mask(device, 0x000200, 0x00000100, 0x00000000); @@ -155,7 +220,9 @@ nv50_fifo_init(struct nvkm_fifo *base) for (i = 0; i < 128; i++) nvkm_wr32(device, 0x002600 + (i * 4), 0x00000000); - nv50_fifo_runlist_update_locked(fifo); + + atomic_set(&runl->changed, 1); + runl->func->update(runl); nvkm_wr32(device, 0x003200, 0x00000001); nvkm_wr32(device, 0x003250, 0x00000001); @@ -175,28 +242,10 @@ nv50_fifo_chid_nr(struct nvkm_fifo *fifo) return 128; } -int -nv50_fifo_oneinit(struct nvkm_fifo *base) -{ - struct nv50_fifo *fifo = nv50_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; - int ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000, - false, &fifo->runlist[0]); - if (ret) - return ret; - - return nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 4, 0x1000, - false, &fifo->runlist[1]); -} - void * nv50_fifo_dtor(struct nvkm_fifo *base) { struct nv50_fifo *fifo = nv50_fifo(base); - nvkm_memory_unref(&fifo->runlist[1]); - nvkm_memory_unref(&fifo->runlist[0]); return fifo; } @@ -221,7 +270,6 @@ nv50_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, static const struct nvkm_fifo_func nv50_fifo = { .dtor = nv50_fifo_dtor, - .oneinit = nv50_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, .runl_ctor = nv04_fifo_runl_ctor, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h index db5fb45a3aa50..1573ea430cf34 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h @@ -6,17 +6,12 @@ struct nv50_fifo { struct nvkm_fifo base; - struct nvkm_memory *runlist[2]; - int cur_runlist; }; int nv50_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_fifo **); void *nv50_fifo_dtor(struct nvkm_fifo *); -int nv50_fifo_oneinit(struct nvkm_fifo *); -void nv50_fifo_init(struct nvkm_fifo *); -void nv50_fifo_runlist_update(struct nv50_fifo *); int g84_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 48153d8bba328..78c6d2218398f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -10,7 +10,6 @@ struct nvkm_memory; struct nvkm_runl; struct nvkm_runq; struct gk104_fifo; -struct gk104_fifo_chan; struct nvkm_fifo_chan_oclass; struct nvkm_fifo_func { @@ -42,16 +41,6 @@ struct nvkm_fifo_func { void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); - const struct gk104_fifo_runlist_func { - u8 size; - void (*cgrp)(struct nvkm_fifo_cgrp *, - struct nvkm_memory *, u32 offset); - void (*chan)(struct gk104_fifo_chan *, - struct nvkm_memory *, u32 offset); - void (*commit)(struct gk104_fifo *, int runl, - struct nvkm_memory *, int entries); - } *runlist; - const struct nvkm_event_func *nonstall; const struct nvkm_runl_func *runl; @@ -98,7 +87,9 @@ int nv10_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_nr(struct nvkm_fifo *); int nv50_fifo_chid_ctor(struct nvkm_fifo *, int); +void nv50_fifo_init(struct nvkm_fifo *); extern const struct nvkm_runl_func nv50_runl; +int nv50_runl_update(struct nvkm_runl *); int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; void nv50_chan_unbind(struct nvkm_chan *); @@ -140,6 +131,8 @@ extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); +void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); +void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int); bool gk104_runl_pending(struct nvkm_runl *); void gk104_runl_block(struct nvkm_runl *, u32); void gk104_runl_allow(struct nvkm_runl *, u32); @@ -162,6 +155,7 @@ void gk104_chan_stop(struct nvkm_chan *); int gk110_fifo_chid_ctor(struct nvkm_fifo *, int); extern const struct nvkm_runl_func gk110_runl; extern const struct nvkm_cgrp_func gk110_cgrp; +void gk110_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); extern const struct nvkm_chan_func gk110_chan; void gk110_chan_preempt(struct nvkm_chan *); @@ -180,6 +174,8 @@ extern const struct nvkm_enum gv100_fifo_mmu_fault_access[]; extern const struct nvkm_enum gv100_fifo_mmu_fault_reason[]; extern const struct nvkm_enum gv100_fifo_mmu_fault_hubclient[]; extern const struct nvkm_enum gv100_fifo_mmu_fault_gpcclient[]; +void gv100_runl_insert_cgrp(struct nvkm_cgrp *, struct nvkm_memory *, u64); +void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); void gv100_runl_preempt(struct nvkm_runl *); extern const struct nvkm_runq_func gv100_runq; extern const struct nvkm_engn_func gv100_engn; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index 27c688d11464a..e654c1c4f769e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -54,9 +54,6 @@ nvkm_engn_cgrp_get(struct nvkm_engn *engn, unsigned long *pirqflags) return cgrp; } -#include "gf100.h" -#include "gk104.h" - static void nvkm_runl_rc(struct nvkm_runl *runl) { @@ -79,8 +76,10 @@ nvkm_runl_rc(struct nvkm_runl *runl) state = atomic_cmpxchg(&cgrp->rc, NVKM_CGRP_RC_PENDING, NVKM_CGRP_RC_RUNNING); if (state == NVKM_CGRP_RC_PENDING) { /* Disable all channels in them, and remove from runlist. */ - nvkm_cgrp_foreach_chan_safe(chan, ctmp, cgrp) + nvkm_cgrp_foreach_chan_safe(chan, ctmp, cgrp) { nvkm_chan_error(chan, false); + nvkm_chan_remove_locked(chan); + } } } @@ -119,16 +118,14 @@ nvkm_runl_rc(struct nvkm_runl *runl) } /* Submit runlist update, and clear any remaining exception state. */ - if (runl->fifo->engine.subdev.device->card_type < NV_E0) - gf100_fifo_runlist_commit(gf100_fifo(runl->fifo)); - else - gk104_fifo_runlist_update(gk104_fifo(runl->fifo), runl->id); + runl->func->update(runl); if (runl->func->fault_clear) runl->func->fault_clear(runl); /* Unblock runlist processing. */ while (rc--) nvkm_runl_allow(runl); + runl->func->wait(runl); } static void @@ -270,6 +267,16 @@ nvkm_runl_update_pending(struct nvkm_runl *runl) return true; } +void +nvkm_runl_update_locked(struct nvkm_runl *runl, bool wait) +{ + if (atomic_xchg(&runl->changed, 0) && runl->func->update) { + runl->func->update(runl); + if (wait) + runl->func->wait(runl); + } +} + void nvkm_runl_allow(struct nvkm_runl *runl) { @@ -309,6 +316,8 @@ nvkm_runl_del(struct nvkm_runl *runl) { struct nvkm_engn *engn, *engt; + nvkm_memory_unref(&runl->mem); + list_for_each_entry_safe(engn, engt, &runl->engns, head) { list_del(&engn->head); kfree(engn); @@ -395,6 +404,7 @@ nvkm_runl_new(struct nvkm_fifo *fifo, int runi, u32 addr, int id_nr) runl->addr = addr; INIT_LIST_HEAD(&runl->engns); INIT_LIST_HEAD(&runl->cgrps); + atomic_set(&runl->changed, 0); mutex_init(&runl->mutex); INIT_WORK(&runl->work, nvkm_runl_work); atomic_set(&runl->rc_triggered, 0); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 47bffc7bb7c0c..c3b469356d5d9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -2,6 +2,7 @@ #define __NVKM_RUNL_H__ #include struct nvkm_cgrp; +struct nvkm_chan; struct nvkm_memory; enum nvkm_subdev_type; @@ -28,6 +29,12 @@ struct nvkm_engn { struct nvkm_runl { const struct nvkm_runl_func { + int runqs; + u8 size; + int (*update)(struct nvkm_runl *); + void (*insert_cgrp)(struct nvkm_cgrp *, struct nvkm_memory *, u64 offset); + void (*insert_chan)(struct nvkm_chan *, struct nvkm_memory *, u64 offset); + void (*commit)(struct nvkm_runl *, struct nvkm_memory *, u32 start, int count); int (*wait)(struct nvkm_runl *); bool (*pending)(struct nvkm_runl *); void (*block)(struct nvkm_runl *, u32 engm); @@ -52,6 +59,9 @@ struct nvkm_runl { struct list_head cgrps; int cgrp_nr; int chan_nr; + atomic_t changed; + struct nvkm_memory *mem; + u32 offset; struct mutex mutex; int blocked; @@ -71,6 +81,7 @@ void nvkm_runl_del(struct nvkm_runl *); void nvkm_runl_fini(struct nvkm_runl *); void nvkm_runl_block(struct nvkm_runl *); void nvkm_runl_allow(struct nvkm_runl *); +void nvkm_runl_update_locked(struct nvkm_runl *, bool wait); bool nvkm_runl_update_pending(struct nvkm_runl *); int nvkm_runl_preempt_wait(struct nvkm_runl *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 724a99a5597f4..babadc8641dcf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -66,28 +66,25 @@ tu102_runl_pending(struct nvkm_runl *runl) } static void -tu102_fifo_runlist_commit(struct gk104_fifo *fifo, int runl, - struct nvkm_memory *mem, int nr) +tu102_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) { - struct nvkm_device *device = fifo->base.engine.subdev.device; - u64 addr = nvkm_memory_addr(mem); + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u64 addr = nvkm_memory_addr(memory) + start; /*XXX: target? */ - nvkm_wr32(device, 0x002b00 + (runl * 0x10), lower_32_bits(addr)); - nvkm_wr32(device, 0x002b04 + (runl * 0x10), upper_32_bits(addr)); - nvkm_wr32(device, 0x002b08 + (runl * 0x10), nr); + nvkm_wr32(device, 0x002b00 + (runl->id * 0x10), lower_32_bits(addr)); + nvkm_wr32(device, 0x002b04 + (runl->id * 0x10), upper_32_bits(addr)); + nvkm_wr32(device, 0x002b08 + (runl->id * 0x10), count); } -static const struct gk104_fifo_runlist_func -tu102_fifo_runlist = { - .size = 16, - .cgrp = gv100_fifo_runlist_cgrp, - .chan = gv100_fifo_runlist_chan, - .commit = tu102_fifo_runlist_commit, -}; - static const struct nvkm_runl_func tu102_runl = { + .runqs = 2, + .size = 16, + .update = nv50_runl_update, + .insert_cgrp = gv100_runl_insert_cgrp, + .insert_chan = gv100_runl_insert_chan, + .commit = tu102_runl_commit, .wait = nv50_runl_wait, .pending = tu102_runl_pending, .block = gk104_runl_block, @@ -274,7 +271,6 @@ tu102_fifo = { .intr = tu102_fifo_intr, .mmu_fault = &tu102_fifo_mmu_fault, .engine_id = gk104_fifo_engine_id, - .runlist = &tu102_fifo_runlist, .nonstall = &gf100_fifo_nonstall, .runl = &tu102_runl, .runq = &gv100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 1fe7bd6a9aa8c..c68ed6f71d4da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -213,14 +213,9 @@ static int nvkm_uchan_fini(struct nvkm_object *object, bool suspend) { struct nvkm_chan *chan = nvkm_uchan(object)->chan; - int ret; nvkm_chan_block(chan); - nvkm_chan_preempt(chan, true); - - ret = chan->object.func->fini(&chan->object, suspend); - if (ret && suspend) - return ret; + nvkm_chan_remove(chan, true); if (chan->func->unbind) chan->func->unbind(chan); @@ -240,8 +235,8 @@ nvkm_uchan_init(struct nvkm_object *object) chan->func->bind(chan); nvkm_chan_allow(chan); - - return chan->object.func->init(&chan->object); + nvkm_chan_insert(chan); + return 0; } static void * -- GitLab From d3e7a4392c82ec2d3c573cdc0fbcc843f3d76b12 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:35 +1000 Subject: [PATCH 1330/1927] drm/nouveau/fifo: add RAMIN info to nvkm_chan_func Currently provided by {chan,dma,gpfifo}*.c, and those are going away. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 4 +- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 41 +++++++++++-------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 6 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 8 ++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 6 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 7 ++++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 3 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 + 16 files changed, 64 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 2d34f1944d64a..02a725d7bf395 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -19,6 +19,8 @@ struct nvkm_chan { struct nvkm_cgrp *cgrp; int runq; + struct nvkm_gpuobj *inst; + struct nvkm_vmm *vmm; union { int id; int chid; }; /*FIXME: remove later */ spinlock_t lock; @@ -31,9 +33,7 @@ struct nvkm_chan { struct nvkm_object object; struct list_head head; - struct nvkm_gpuobj *inst; struct nvkm_gpuobj *push; - struct nvkm_vmm *vmm; u64 addr; u32 size; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 3832a9aad80ae..5b6377ec0df03 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -602,39 +602,44 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, chan->cgrp = nvkm_cgrp_ref(cgrp); } - /* instance memory */ - ret = nvkm_gpuobj_new(device, size, align, zero, NULL, &chan->inst); - if (ret) + /* Allocate instance block. */ + ret = nvkm_gpuobj_new(device, func->inst->size, 0x1000, func->inst->zero, NULL, + &chan->inst); + if (ret) { + RUNL_DEBUG(runl, "inst %d", ret); return ret; - - /* allocate push buffer ctxdma instance */ - if (push) { - dmaobj = nvkm_dmaobj_search(client, push); - if (IS_ERR(dmaobj)) - return PTR_ERR(dmaobj); - - ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, - &chan->push); - if (ret) - return ret; } - /* channel address space */ - if (hvmm) { + /* Initialise virtual address-space. */ + if (func->inst->vmm) { struct nvkm_vmm *vmm = nvkm_uvmm_search(client, hvmm); if (IS_ERR(vmm)) return PTR_ERR(vmm); - if (vmm->mmu != device->mmu) + if (WARN_ON(vmm->mmu != device->mmu)) return -EINVAL; ret = nvkm_vmm_join(vmm, chan->inst->memory); - if (ret) + if (ret) { + RUNL_DEBUG(runl, "vmm %d", ret); return ret; + } chan->vmm = nvkm_vmm_ref(vmm); } + /* allocate push buffer ctxdma instance */ + if (push) { + dmaobj = nvkm_dmaobj_search(client, push); + if (IS_ERR(dmaobj)) + return PTR_ERR(dmaobj); + + ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, + &chan->push); + if (ret) + return ret; + } + /* Allocate channel ID. */ chan->id = nvkm_chid_get(runl->chid, chan); if (chan->id < 0) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 804441f30f440..0b764d28117ea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -16,6 +16,12 @@ struct nvkm_cctx { }; struct nvkm_chan_func { + const struct nvkm_chan_func_inst { + u32 size; + bool zero; + bool vmm; + } *inst; + void (*bind)(struct nvkm_chan *); void (*unbind)(struct nvkm_chan *); void (*start)(struct nvkm_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 18ec982f4f54d..beb51dcb88637 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -42,6 +42,7 @@ g84_chan_bind(struct nvkm_chan *chan) const struct nvkm_chan_func g84_chan = { + .inst = &nv50_chan_inst, .bind = g84_chan_bind, .unbind = nv50_chan_unbind, .start = nv50_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 767507e0e2390..8d1ffeb10c451 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -82,8 +82,16 @@ gf100_chan_bind(struct nvkm_chan *chan) nvkm_wr32(device, 0x003000 + (chan->id * 8), 0xc0000000 | chan->inst->addr >> 12); } +const struct nvkm_chan_func_inst +gf100_chan_inst = { + .size = 0x1000, + .zero = true, + .vmm = true, +}; + static const struct nvkm_chan_func gf100_chan = { + .inst = &gf100_chan_inst, .bind = gf100_chan_bind, .unbind = gf100_chan_unbind, .start = gf100_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index b5e698a223b6d..a7e2b58f5029d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -81,6 +81,7 @@ gk104_chan_bind(struct nvkm_chan *chan) static const struct nvkm_chan_func gk104_chan = { + .inst = &gf100_chan_inst, .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 812fefff09255..5cc48a367c0db 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -49,6 +49,7 @@ gk110_chan_preempt(struct nvkm_chan *chan) const struct nvkm_chan_func gk110_chan = { + .inst = &gf100_chan_inst, .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index c0eca7da7d098..f8bc89ac35abf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -34,6 +34,7 @@ const struct nvkm_chan_func gm107_chan = { + .inst = &gf100_chan_inst, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 29a83ed3404f1..db998c9de3ad9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -40,6 +40,7 @@ gv100_chan_doorbell_handle(struct nvkm_chan *chan) static const struct nvkm_chan_func gv100_chan = { + .inst = &gf100_chan_inst, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index a0a28a81f7f4c..aeb24f838e3f3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -113,8 +113,14 @@ nv04_chan_start(struct nvkm_chan *chan) spin_unlock_irqrestore(&fifo->lock, flags); } +const struct nvkm_chan_func_inst +nv04_chan_inst = { + .size = 0x1000, +}; + static const struct nvkm_chan_func nv04_chan = { + .inst = &nv04_chan_inst, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index 8d302cb8ba8c5..b9111fcfbcad3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -45,6 +45,7 @@ nv10_fifo_ramfc[] = { static const struct nvkm_chan_func nv10_chan = { + .inst = &nv04_chan_inst, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index 7b1b90d285c14..e0ad8fabb2d6a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -54,6 +54,7 @@ nv17_fifo_ramfc[] = { static const struct nvkm_chan_func nv17_chan = { + .inst = &nv04_chan_inst, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index 19f7c0407e70e..c2d892a7400b1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -64,6 +64,7 @@ nv40_fifo_ramfc[] = { static const struct nvkm_chan_func nv40_chan = { + .inst = &nv04_chan_inst, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 6a069e84006c4..384772d522afd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -66,8 +66,15 @@ nv50_chan_bind(struct nvkm_chan *chan) nvkm_wr32(device, 0x002600 + (chan->id * 4), nv50_fifo_chan(chan)->ramfc->addr >> 12); } +const struct nvkm_chan_func_inst +nv50_chan_inst = { + .size = 0x10000, + .vmm = true, +}; + static const struct nvkm_chan_func nv50_chan = { + .inst = &nv50_chan_inst, .bind = nv50_chan_bind, .unbind = nv50_chan_unbind, .start = nv50_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 78c6d2218398f..bf8067e97ba2b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -80,6 +80,7 @@ void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); extern const struct nvkm_runl_func nv04_runl; extern const struct nvkm_engn_func nv04_engn; extern const struct nvkm_cgrp_func nv04_cgrp; +extern const struct nvkm_chan_func_inst nv04_chan_inst; void nv04_chan_start(struct nvkm_chan *); void nv04_chan_stop(struct nvkm_chan *); @@ -92,6 +93,7 @@ extern const struct nvkm_runl_func nv50_runl; int nv50_runl_update(struct nvkm_runl *); int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; +extern const struct nvkm_chan_func_inst nv50_chan_inst; void nv50_chan_unbind(struct nvkm_chan *); void nv50_chan_start(struct nvkm_chan *); void nv50_chan_stop(struct nvkm_chan *); @@ -117,6 +119,7 @@ bool gf100_runq_intr(struct nvkm_runq *, struct nvkm_runl *); void gf100_engn_mmu_fault_trigger(struct nvkm_engn *); bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *); extern const struct nvkm_engn_func gf100_engn_sw; +extern const struct nvkm_chan_func_inst gf100_chan_inst; void gf100_chan_preempt(struct nvkm_chan *); int gk104_fifo_chid_nr(struct nvkm_fifo *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index babadc8641dcf..75e4c0db40bcd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -49,6 +49,7 @@ tu102_chan_start(struct nvkm_chan *chan) static const struct nvkm_chan_func tu102_chan = { + .inst = &gf100_chan_inst, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = tu102_chan_start, -- GitLab From fbe9f4337c1381710f17df0d74dd588d3d9784b2 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:36 +1000 Subject: [PATCH 1331/1927] drm/nouveau/fifo: add USERD info to nvkm_chan_func And use it to cleanup multiple implementations of almost the same thing. - prepares for non-polled / client-provided USERD - only zeroes relevant "registers", rather than entire USERD Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 12 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 25 ++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 27 ++++----- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 7 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 58 +++++++++---------- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 5 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 36 +++++------- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 6 -- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 + .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 14 +---- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 13 +---- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 12 +--- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 11 +++- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 8 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 8 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 8 +++ .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 6 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 10 +++- 23 files changed, 157 insertions(+), 115 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 02a725d7bf395..95cb33b532287 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -23,6 +23,11 @@ struct nvkm_chan { struct nvkm_vmm *vmm; union { int id; int chid; }; /*FIXME: remove later */ + struct { + struct nvkm_memory *mem; + u32 base; + } userd; + spinlock_t lock; atomic_t blocked; atomic_t errored; @@ -34,8 +39,6 @@ struct nvkm_chan { struct list_head head; struct nvkm_gpuobj *push; - u64 addr; - u32 size; struct nvkm_fifo_engn engn[NVKM_FIFO_ENGN_NR]; }; @@ -63,6 +66,11 @@ struct nvkm_fifo { u32 chan_msec; } timeout; + struct { + struct nvkm_memory *mem; + struct nvkm_vma *bar1; + } userd; + int nr; spinlock_t lock; struct mutex mutex; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 0cbfb9225e6cf..c77af1b3ffa06 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -28,7 +28,9 @@ #include "runq.h" #include +#include #include +#include #include #include @@ -243,6 +245,25 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) } } + /* Allocate USERD + BAR1 polling area. */ + if (fifo->func->chan.func->userd->bar == 1) { + struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device); + + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, fifo->chid->nr * + fifo->func->chan.func->userd->size, 0, true, + &fifo->userd.mem); + if (ret) + return ret; + + ret = nvkm_vmm_get(bar1, 12, nvkm_memory_size(fifo->userd.mem), &fifo->userd.bar1); + if (ret) + return ret; + + ret = nvkm_memory_map(fifo->userd.mem, 0, bar1, fifo->userd.bar1, NULL, 0); + if (ret) + return ret; + } + if (fifo->func->oneinit) return fifo->func->oneinit(fifo); @@ -263,6 +284,10 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) struct nvkm_runq *runq, *rtmp; void *data = fifo; + if (fifo->userd.bar1) + nvkm_vmm_put(nvkm_bar_bar1_vmm(engine->subdev.device), &fifo->userd.bar1); + nvkm_memory_unref(&fifo->userd.mem); + list_for_each_entry_safe(runl, runt, &fifo->runls, head) nvkm_runl_del(runl); list_for_each_entry_safe(runq, rtmp, &fifo->runqs, head) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 5b6377ec0df03..0df089b692a9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -328,17 +328,6 @@ nvkm_chan_preempt(struct nvkm_chan *chan, bool wait) return ret; } -static int -nvkm_fifo_chan_map(struct nvkm_object *object, void *argv, u32 argc, - enum nvkm_object_map *type, u64 *addr, u64 *size) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - *type = NVKM_OBJECT_MAP_IO; - *addr = chan->addr; - *size = chan->size; - return 0; -} - void nvkm_chan_remove_locked(struct nvkm_chan *chan) { @@ -445,6 +434,8 @@ nvkm_chan_del(struct nvkm_chan **pchan) if (!chan) return; + nvkm_memory_unref(&chan->userd.mem); + if (chan->cgrp) { nvkm_chid_put(chan->cgrp->runl->chid, chan->id, &chan->cgrp->lock); nvkm_cgrp_unref(&chan->cgrp); @@ -524,7 +515,6 @@ nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags) static const struct nvkm_object_func nvkm_fifo_chan_func = { .dtor = nvkm_fifo_chan_dtor, - .map = nvkm_fifo_chan_map, }; int @@ -650,9 +640,14 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, if (cgrp->id < 0) cgrp->id = chan->id; - /* determine address of this channel's user registers */ - chan->addr = device->func->resource_addr(device, bar) + - base + user * chan->chid; - chan->size = user; + /* Initialise USERD. */ + if (1) { + chan->userd.mem = nvkm_memory_ref(fifo->userd.mem); + chan->userd.base = chan->id * chan->func->userd->size; + } + + if (chan->func->userd->clear) + chan->func->userd->clear(chan); + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 0b764d28117ea..54ef137bdfac0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -22,6 +22,13 @@ struct nvkm_chan_func { bool vmm; } *inst; + const struct nvkm_chan_func_userd { + int bar; + u32 base; + u32 size; + void (*clear)(struct nvkm_chan *); + } *userd; + void (*bind)(struct nvkm_chan *); void (*unbind)(struct nvkm_chan *); void (*start)(struct nvkm_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index beb51dcb88637..76ba98b4d3bf7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -43,6 +43,7 @@ g84_chan_bind(struct nvkm_chan *chan) const struct nvkm_chan_func g84_chan = { .inst = &nv50_chan_inst, + .userd = &nv50_chan_userd, .bind = g84_chan_bind, .unbind = nv50_chan_unbind, .start = nv50_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 8d1ffeb10c451..afee42440ff55 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -82,6 +82,30 @@ gf100_chan_bind(struct nvkm_chan *chan) nvkm_wr32(device, 0x003000 + (chan->id * 8), 0xc0000000 | chan->inst->addr >> 12); } +void +gf100_chan_userd_clear(struct nvkm_chan *chan) +{ + nvkm_kmap(chan->userd.mem); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x040, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x044, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x048, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x04c, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x050, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x058, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x05c, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x060, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x088, 0x00000000); + nvkm_wo32(chan->userd.mem, chan->userd.base + 0x08c, 0x00000000); + nvkm_done(chan->userd.mem); +} + +static const struct nvkm_chan_func_userd +gf100_chan_userd = { + .bar = 1, + .size = 0x1000, + .clear = gf100_chan_userd_clear, +}; + const struct nvkm_chan_func_inst gf100_chan_inst = { .size = 0x1000, @@ -92,6 +116,7 @@ gf100_chan_inst = { static const struct nvkm_chan_func gf100_chan = { .inst = &gf100_chan_inst, + .userd = &gf100_chan_userd, .bind = gf100_chan_bind, .unbind = gf100_chan_unbind, .start = gf100_chan_start, @@ -807,13 +832,12 @@ gf100_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) } static void -gf100_fifo_init(struct nvkm_fifo *base) +gf100_fifo_init(struct nvkm_fifo *fifo) { - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; nvkm_mask(device, 0x002200, 0x00000001, 0x00000001); - nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); + nvkm_wr32(device, 0x002254, 0x10000000 | fifo->userd.bar1->addr >> 12); nvkm_wr32(device, 0x002100, 0xffffffff); nvkm_wr32(device, 0x002140, 0x7fffffff); @@ -857,42 +881,16 @@ gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr, &fifo->chid); } -static int -gf100_fifo_oneinit(struct nvkm_fifo *base) -{ - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); - int ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 128 * 0x1000, - 0x1000, false, &fifo->user.mem); - if (ret) - return ret; - - ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem), - &fifo->user.bar); - if (ret) - return ret; - - return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); -} - static void * gf100_fifo_dtor(struct nvkm_fifo *base) { struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; - nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar); - nvkm_memory_unref(&fifo->user.mem); return fifo; } static const struct nvkm_fifo_func gf100_fifo = { .dtor = gf100_fifo_dtor, - .oneinit = gf100_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h index 42a950c004830..6027ee7dce5ad 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h @@ -8,10 +8,5 @@ struct gf100_fifo { struct nvkm_fifo base; - - struct { - struct nvkm_memory *mem; - struct nvkm_vma *bar; - } user; }; #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index a7e2b58f5029d..61966d433bce2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -31,7 +31,6 @@ #include "changk104.h" #include -#include #include #include @@ -79,9 +78,17 @@ gk104_chan_bind(struct nvkm_chan *chan) gk104_chan_bind_inst(chan); } +const struct nvkm_chan_func_userd +gk104_chan_userd = { + .bar = 1, + .size = 0x200, + .clear = gf100_chan_userd_clear, +}; + static const struct nvkm_chan_func gk104_chan = { .inst = &gf100_chan_inst, + .userd = &gk104_chan_userd, .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, @@ -659,12 +666,12 @@ gk104_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) } void -gk104_fifo_init(struct nvkm_fifo *base) +gk104_fifo_init(struct nvkm_fifo *fifo) { - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_device *device = fifo->engine.subdev.device; - nvkm_wr32(device, 0x002254, 0x10000000 | fifo->user.bar->addr >> 12); + if (fifo->func->chan.func->userd->bar == 1) + nvkm_wr32(device, 0x002254, 0x10000000 | fifo->userd.bar1->addr >> 12); nvkm_wr32(device, 0x002100, 0xffffffff); nvkm_wr32(device, 0x002140, 0x7fffffff); @@ -728,9 +735,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) struct gk104_fifo *fifo = gk104_fifo(base); struct nvkm_subdev *subdev = &fifo->base.engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_vmm *bar = nvkm_bar_bar1_vmm(device); struct nvkm_top_device *tdev; - int ret; /* Determine runlist configuration from topology device info. */ list_for_each_entry(tdev, &device->top->device, head) { @@ -748,28 +753,13 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) fifo->runlist_nr = max(fifo->runlist_nr, tdev->runlist + 1); } - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - fifo->base.nr * 0x200, 0x1000, true, - &fifo->user.mem); - if (ret) - return ret; - - ret = nvkm_vmm_get(bar, 12, nvkm_memory_size(fifo->user.mem), - &fifo->user.bar); - if (ret) - return ret; - - return nvkm_memory_map(fifo->user.mem, 0, bar, fifo->user.bar, NULL, 0); + return 0; } void * gk104_fifo_dtor(struct nvkm_fifo *base) { struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_device *device = fifo->base.engine.subdev.device; - - nvkm_vmm_put(nvkm_bar_bar1_vmm(device), &fifo->user.bar); - nvkm_memory_unref(&fifo->user.mem); return fifo; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 3915b3b9e6bf4..3bcf0720f89da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -24,16 +24,10 @@ struct gk104_fifo { u32 engm_sw; } runlist[16]; int runlist_nr; - - struct { - struct nvkm_memory *mem; - struct nvkm_vma *bar; - } user; }; int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int index, int nr, struct nvkm_fifo **); void *gk104_fifo_dtor(struct nvkm_fifo *base); int gk104_fifo_oneinit(struct nvkm_fifo *); -void gk104_fifo_init(struct nvkm_fifo *base); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 5cc48a367c0db..7ff7a435d8900 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -50,6 +50,7 @@ gk110_chan_preempt(struct nvkm_chan *chan) const struct nvkm_chan_func gk110_chan = { .inst = &gf100_chan_inst, + .userd = &gk104_chan_userd, .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index f8bc89ac35abf..cc2638018db75 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -35,6 +35,7 @@ const struct nvkm_chan_func gm107_chan = { .inst = &gf100_chan_inst, + .userd = &gk104_chan_userd, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 871f199ddccf6..1e459afc36ea4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -153,7 +153,7 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, struct nvkm_object *parent = oclass->parent; struct gf100_fifo_chan *chan; u64 usermem, ioffset, ilength; - int ret = -ENOSYS, i; + int ret = -ENOSYS; nvif_ioctl(parent, "create channel gpfifo size %d\n", size); if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { @@ -181,25 +181,17 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, BIT(GF100_FIFO_ENGN_CE0) | BIT(GF100_FIFO_ENGN_CE1) | BIT(GF100_FIFO_ENGN_SW), - 1, fifo->user.bar->addr, 0x1000, + 0, 0, 0, oclass, &chan->base); if (ret) return ret; args->v0.chid = chan->base.chid; - /* clear channel control registers */ - - usermem = chan->base.chid * 0x1000; + usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; ioffset = args->v0.ioffset; ilength = order_base_2(args->v0.ilength / 8); - nvkm_kmap(fifo->user.mem); - for (i = 0; i < 0x1000; i += 4) - nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000); - nvkm_done(fifo->user.mem); - usermem = nvkm_memory_addr(fifo->user.mem) + usermem; - /* RAMFC */ nvkm_kmap(chan->base.inst); nvkm_wo32(chan->base.inst, 0x08, lower_32_bits(usermem)); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index aa3ea7f7f8e19..b02ec7c545cfe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -175,7 +175,7 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, struct nvkm_object **pobject) { struct gk104_fifo_chan *chan; - int runlist = ffs(*runlists) -1, ret, i; + int runlist = ffs(*runlists) -1, ret; u64 usermem; if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) @@ -191,7 +191,7 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, 0x1000, 0x1000, true, vmm, 0, fifo->runlist[runlist].engm_sw, - 1, fifo->user.bar->addr, 0x200, + 0, 0, 0, oclass, &chan->base); if (ret) return ret; @@ -199,16 +199,9 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, *chid = chan->base.chid; *inst = chan->base.inst->addr; - /* Clear channel control registers. */ - usermem = chan->base.chid * 0x200; + usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; ilength = order_base_2(ilength / 8); - nvkm_kmap(fifo->user.mem); - for (i = 0; i < 0x200; i += 4) - nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000); - nvkm_done(fifo->user.mem); - usermem = nvkm_memory_addr(fifo->user.mem) + usermem; - /* RAMFC */ nvkm_kmap(chan->base.inst); nvkm_wo32(chan->base.inst, 0x08, lower_32_bits(usermem)); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 428901abdb813..317787954a3ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -119,7 +119,7 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, struct nvkm_object **pobject) { struct gk104_fifo_chan *chan; - int runlist = ffs(*runlists) -1, ret, i; + int runlist = ffs(*runlists) -1, ret; u64 usermem; if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) @@ -134,7 +134,7 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, chan->runl = runlist; ret = nvkm_fifo_chan_ctor(func, &fifo->base, 0x1000, 0x1000, true, vmm, - 0, fifo->runlist[runlist].engm, 1, fifo->user.bar->addr, 0x200, + 0, fifo->runlist[runlist].engm, 0, 0, 0, oclass, &chan->base); if (ret) return ret; @@ -144,15 +144,9 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, *token = chan->base.func->doorbell_handle(&chan->base); /* Clear channel control registers. */ - usermem = chan->base.chid * 0x200; + usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; ilength = order_base_2(ilength / 8); - nvkm_kmap(fifo->user.mem); - for (i = 0; i < 0x200; i += 4) - nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000); - nvkm_done(fifo->user.mem); - usermem = nvkm_memory_addr(fifo->user.mem) + usermem; - /* RAMFC */ nvkm_kmap(chan->base.inst); nvkm_wo32(chan->base.inst, 0x008, lower_32_bits(usermem)); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index db998c9de3ad9..9611bd73c1142 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -38,9 +38,17 @@ gv100_chan_doorbell_handle(struct nvkm_chan *chan) return chan->id; } +const struct nvkm_chan_func_userd +gv100_chan_userd = { + .bar = 1, /*FIXME: hw doesn't have poller, flip to user-allocated in uapi commit. */ + .size = 0x200, + .clear = gf100_chan_userd_clear, +}; + static const struct nvkm_chan_func gv100_chan = { .inst = &gf100_chan_inst, + .userd = &gv100_chan_userd, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = gk104_chan_start, @@ -101,8 +109,7 @@ gv100_runl_preempt(struct nvkm_runl *runl) void gv100_runl_insert_chan(struct nvkm_chan *chan, struct nvkm_memory *memory, u64 offset) { - struct nvkm_memory *usermem = gk104_fifo(chan->cgrp->runl->fifo)->user.mem; - const u64 user = nvkm_memory_addr(usermem) + (chan->id * 0x200); + const u64 user = nvkm_memory_addr(chan->userd.mem) + chan->userd.base; const u64 inst = chan->inst->addr; nvkm_wo32(memory, offset + 0x0, lower_32_bits(user) | chan->runq << 1); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index aeb24f838e3f3..f4207100f20cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -113,6 +113,13 @@ nv04_chan_start(struct nvkm_chan *chan) spin_unlock_irqrestore(&fifo->lock, flags); } +const struct nvkm_chan_func_userd +nv04_chan_userd = { + .bar = 0, + .base = 0x800000, + .size = 0x010000, +}; + const struct nvkm_chan_func_inst nv04_chan_inst = { .size = 0x1000, @@ -121,6 +128,7 @@ nv04_chan_inst = { static const struct nvkm_chan_func nv04_chan = { .inst = &nv04_chan_inst, + .userd = &nv04_chan_userd, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index b9111fcfbcad3..bbad5ffa179bb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -46,6 +46,7 @@ nv10_fifo_ramfc[] = { static const struct nvkm_chan_func nv10_chan = { .inst = &nv04_chan_inst, + .userd = &nv04_chan_userd, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index e0ad8fabb2d6a..b0dfa6f1a1d3f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -55,6 +55,7 @@ nv17_fifo_ramfc[] = { static const struct nvkm_chan_func nv17_chan = { .inst = &nv04_chan_inst, + .userd = &nv04_chan_userd, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index c2d892a7400b1..c2dc2d7abc9db 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -62,9 +62,17 @@ nv40_fifo_ramfc[] = { {} }; +static const struct nvkm_chan_func_userd +nv40_chan_userd = { + .bar = 0, + .base = 0xc00000, + .size = 0x001000, +}; + static const struct nvkm_chan_func nv40_chan = { .inst = &nv04_chan_inst, + .userd = &nv40_chan_userd, .start = nv04_chan_start, .stop = nv04_chan_stop, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 384772d522afd..24b4580481802 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -66,6 +66,13 @@ nv50_chan_bind(struct nvkm_chan *chan) nvkm_wr32(device, 0x002600 + (chan->id * 4), nv50_fifo_chan(chan)->ramfc->addr >> 12); } +const struct nvkm_chan_func_userd +nv50_chan_userd = { + .bar = 0, + .base = 0xc00000, + .size = 0x002000, +}; + const struct nvkm_chan_func_inst nv50_chan_inst = { .size = 0x10000, @@ -75,6 +82,7 @@ nv50_chan_inst = { static const struct nvkm_chan_func nv50_chan = { .inst = &nv50_chan_inst, + .userd = &nv50_chan_userd, .bind = nv50_chan_bind, .unbind = nv50_chan_unbind, .start = nv50_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index bf8067e97ba2b..c56deb78e9cc0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -81,6 +81,7 @@ extern const struct nvkm_runl_func nv04_runl; extern const struct nvkm_engn_func nv04_engn; extern const struct nvkm_cgrp_func nv04_cgrp; extern const struct nvkm_chan_func_inst nv04_chan_inst; +extern const struct nvkm_chan_func_userd nv04_chan_userd; void nv04_chan_start(struct nvkm_chan *); void nv04_chan_stop(struct nvkm_chan *); @@ -94,6 +95,7 @@ int nv50_runl_update(struct nvkm_runl *); int nv50_runl_wait(struct nvkm_runl *); extern const struct nvkm_engn_func nv50_engn_sw; extern const struct nvkm_chan_func_inst nv50_chan_inst; +extern const struct nvkm_chan_func_userd nv50_chan_userd; void nv50_chan_unbind(struct nvkm_chan *); void nv50_chan_start(struct nvkm_chan *); void nv50_chan_stop(struct nvkm_chan *); @@ -120,10 +122,12 @@ void gf100_engn_mmu_fault_trigger(struct nvkm_engn *); bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *); extern const struct nvkm_engn_func gf100_engn_sw; extern const struct nvkm_chan_func_inst gf100_chan_inst; +void gf100_chan_userd_clear(struct nvkm_chan *); void gf100_chan_preempt(struct nvkm_chan *); int gk104_fifo_chid_nr(struct nvkm_fifo *); int gk104_fifo_runl_ctor(struct nvkm_fifo *); +void gk104_fifo_init(struct nvkm_fifo *); void gk104_fifo_init_pbdmas(struct nvkm_fifo *, u32); irqreturn_t gk104_fifo_intr(struct nvkm_inth *); void gk104_fifo_intr_runlist(struct nvkm_fifo *); @@ -149,6 +153,7 @@ extern const struct nvkm_engn_func gk104_engn; bool gk104_engn_chsw(struct nvkm_engn *); int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); extern const struct nvkm_engn_func gk104_engn_ce; +extern const struct nvkm_chan_func_userd gk104_chan_userd; void gk104_chan_bind(struct nvkm_chan *); void gk104_chan_bind_inst(struct nvkm_chan *); void gk104_chan_unbind(struct nvkm_chan *); @@ -183,6 +188,7 @@ void gv100_runl_preempt(struct nvkm_runl *); extern const struct nvkm_runq_func gv100_runq; extern const struct nvkm_engn_func gv100_engn; extern const struct nvkm_engn_func gv100_engn_ce; +extern const struct nvkm_chan_func_userd gv100_chan_userd; void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 75e4c0db40bcd..42d2975b82aa5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -50,6 +50,7 @@ tu102_chan_start(struct nvkm_chan *chan) static const struct nvkm_chan_func tu102_chan = { .inst = &gf100_chan_inst, + .userd = &gv100_chan_userd, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = tu102_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index c68ed6f71d4da..74584a29454a9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -204,9 +204,17 @@ static int nvkm_uchan_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) { + struct nvkm_device *device = object->engine->subdev.device; struct nvkm_chan *chan = nvkm_uchan(object)->chan; - return chan->object.func->map(&chan->object, argv, argc, type, addr, size); + if (chan->func->userd->bar < 0) + return -ENOSYS; + + *type = NVKM_OBJECT_MAP_IO; + *addr = device->func->resource_addr(device, chan->func->userd->bar) + + chan->func->userd->base + chan->userd.base; + *size = chan->func->userd->size; + return 0; } static int -- GitLab From 3647c53bd7dada9053b46a5ca27eb3f7c5472416 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:36 +1000 Subject: [PATCH 1332/1927] drm/nouveau/fifo: add RAMFC info to nvkm_chan_func - adds support for specifying SUBDEVICE_ID for channel - rounds non-power-of-two GPFIFO sizes down, rather than up Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 7 ++ .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 22 +++-- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 15 ++++ .../drm/nouveau/nvkm/engine/fifo/chang84.c | 26 +----- .../drm/nouveau/nvkm/engine/fifo/channv50.c | 26 +----- .../drm/nouveau/nvkm/engine/fifo/channv50.h | 3 - .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 27 +----- .../drm/nouveau/nvkm/engine/fifo/dmanv10.c | 16 +--- .../drm/nouveau/nvkm/engine/fifo/dmanv17.c | 16 +--- .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 17 +--- .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 60 ++++++++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 34 +++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 34 +++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 + .../drm/nouveau/nvkm/engine/fifo/gpfifog84.c | 18 +--- .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 25 +----- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 24 +---- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 23 +---- .../drm/nouveau/nvkm/engine/fifo/gpfifonv50.c | 16 +--- .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 32 +++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 90 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.h | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 59 +++++++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 66 ++++++++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 84 +++++++++++------ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 54 ++++++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 + 29 files changed, 488 insertions(+), 321 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 95cb33b532287..1616a329db9e2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -28,6 +28,13 @@ struct nvkm_chan { u32 base; } userd; + u32 ramfc_offset; + struct nvkm_gpuobj *ramfc; + struct nvkm_gpuobj *cache; + struct nvkm_gpuobj *eng; + struct nvkm_gpuobj *pgd; + struct nvkm_ramht *ramht; + spinlock_t lock; atomic_t blocked; atomic_t errored; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 0df089b692a9e..00680b3ca9abd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -29,8 +29,8 @@ #include "priv.h" #include -#include #include +#include #include #include @@ -434,6 +434,15 @@ nvkm_chan_del(struct nvkm_chan **pchan) if (!chan) return; + if (chan->func->ramfc->clear) + chan->func->ramfc->clear(chan); + + nvkm_ramht_del(&chan->ramht); + nvkm_gpuobj_del(&chan->pgd); + nvkm_gpuobj_del(&chan->eng); + nvkm_gpuobj_del(&chan->cache); + nvkm_gpuobj_del(&chan->ramfc); + nvkm_memory_unref(&chan->userd.mem); if (chan->cgrp) { @@ -618,16 +627,17 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, chan->vmm = nvkm_vmm_ref(vmm); } - /* allocate push buffer ctxdma instance */ - if (push) { + /* Allocate HW ctxdma for push buffer. */ + if (func->ramfc->ctxdma) { dmaobj = nvkm_dmaobj_search(client, push); if (IS_ERR(dmaobj)) return PTR_ERR(dmaobj); - ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, - &chan->push); - if (ret) + ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, &chan->push); + if (ret) { + RUNL_DEBUG(runl, "bind %d", ret); return ret; + } } /* Allocate channel ID. */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 54ef137bdfac0..6e41eba0a7e2c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -29,6 +29,21 @@ struct nvkm_chan_func { void (*clear)(struct nvkm_chan *); } *userd; + const struct nvkm_chan_func_ramfc { + const struct nvkm_ramfc_layout { + unsigned bits:6; + unsigned ctxs:5; + unsigned ctxp:8; + unsigned regs:5; + unsigned regp; + } *layout; + int (*write)(struct nvkm_chan *, u64 offset, u64 length, u32 devm, bool priv); + void (*clear)(struct nvkm_chan *); + bool ctxdma; + u32 devm; + bool priv; + } *ramfc; + void (*bind)(struct nvkm_chan *); void (*unbind)(struct nvkm_chan *); void (*start)(struct nvkm_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 64566b5cc23f3..f5e3557b9e50f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -182,7 +182,6 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, const struct nvkm_oclass *oclass, struct nv50_fifo_chan *chan) { - struct nvkm_device *device = fifo->base.engine.subdev.device; int ret; if (!vmm) @@ -206,28 +205,5 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, BIT(G84_FIFO_ENGN_DMA), 0, 0xc00000, 0x2000, oclass, &chan->base); chan->fifo = fifo; - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x0200, 0, true, chan->base.inst, - &chan->eng); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x4000, 0, false, chan->base.inst, - &chan->pgd); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x1000, 0x400, true, chan->base.inst, - &chan->cache); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x100, 0x100, true, chan->base.inst, - &chan->ramfc); - if (ret) - return ret; - - return nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht); + return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 5d6f82236c50b..0bf59c4e92b96 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -187,11 +187,6 @@ void * nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) { struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - nvkm_ramht_del(&chan->ramht); - nvkm_gpuobj_del(&chan->pgd); - nvkm_gpuobj_del(&chan->eng); - nvkm_gpuobj_del(&chan->cache); - nvkm_gpuobj_del(&chan->ramfc); return chan; } @@ -211,7 +206,6 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, const struct nvkm_oclass *oclass, struct nv50_fifo_chan *chan) { - struct nvkm_device *device = fifo->base.engine.subdev.device; int ret; if (!vmm) @@ -225,23 +219,5 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, BIT(NV50_FIFO_ENGN_DMA), 0, 0xc00000, 0x2000, oclass, &chan->base); chan->fifo = fifo; - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x0200, 0x1000, true, chan->base.inst, - &chan->ramfc); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x1200, 0, true, chan->base.inst, - &chan->eng); - if (ret) - return ret; - - ret = nvkm_gpuobj_new(device, 0x4000, 0, false, chan->base.inst, - &chan->pgd); - if (ret) - return ret; - - return nvkm_ramht_new(device, 0x8000, 16, chan->base.inst, &chan->ramht); + return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index ce7d61e8d297d..343356a0e5ef2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -9,10 +9,7 @@ struct nv50_fifo_chan { struct nv50_fifo *fifo; struct nvkm_fifo_chan base; - struct nvkm_gpuobj *ramfc; - struct nvkm_gpuobj *cache; struct nvkm_gpuobj *eng; - struct nvkm_gpuobj *pgd; struct nvkm_ramht *ramht; #define NV50_FIFO_ENGN_SW 0 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index d515cf08fc3bb..4220120d8bb8a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -73,15 +73,6 @@ void * nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) { struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_instmem *imem = fifo->base.engine.subdev.device->imem; - const struct nv04_fifo_ramfc *c = fifo->ramfc; - - nvkm_kmap(imem->ramfc); - do { - nvkm_wo32(imem->ramfc, chan->ramfc + c->ctxp, 0x00000000); - } while ((++c)->bits); - nvkm_done(imem->ramfc); return chan; } @@ -102,8 +93,6 @@ nv04_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, } *args = data; struct nv04_fifo *fifo = nv04_fifo(base); struct nv04_fifo_chan *chan = NULL; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_instmem *imem = device->imem; int ret = -ENOSYS; nvif_ioctl(parent, "create channel dma size %d\n", size); @@ -131,20 +120,8 @@ nv04_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return ret; args->v0.chid = chan->base.chid; - chan->ramfc = chan->base.chid * 32; - - nvkm_kmap(imem->ramfc); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x00, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x04, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x08, chan->base.push->addr >> 4); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x10, - NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | - NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | -#ifdef __BIG_ENDIAN - NV_PFIFO_CACHE1_BIG_ENDIAN | -#endif - NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); - nvkm_done(imem->ramfc); + + chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c index f45655072f9d0..aa5893c11203e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c @@ -41,8 +41,6 @@ nv10_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, } *args = data; struct nv04_fifo *fifo = nv04_fifo(base); struct nv04_fifo_chan *chan = NULL; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_instmem *imem = device->imem; int ret = -ENOSYS; nvif_ioctl(parent, "create channel dma size %d\n", size); @@ -70,20 +68,8 @@ nv10_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return ret; args->v0.chid = chan->base.chid; - chan->ramfc = chan->base.chid * 32; - nvkm_kmap(imem->ramfc); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x00, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x04, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x0c, chan->base.push->addr >> 4); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x14, - NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | - NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | -#ifdef __BIG_ENDIAN - NV_PFIFO_CACHE1_BIG_ENDIAN | -#endif - NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); - nvkm_done(imem->ramfc); + chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c index cd3f9f84ab9c2..0a7d145ce5aa3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c @@ -41,8 +41,6 @@ nv17_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, } *args = data; struct nv04_fifo *fifo = nv04_fifo(base); struct nv04_fifo_chan *chan = NULL; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_instmem *imem = device->imem; int ret = -ENOSYS; nvif_ioctl(parent, "create channel dma size %d\n", size); @@ -71,20 +69,8 @@ nv17_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return ret; args->v0.chid = chan->base.chid; - chan->ramfc = chan->base.chid * 64; - nvkm_kmap(imem->ramfc); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x00, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x04, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x0c, chan->base.push->addr >> 4); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x14, - NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | - NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | -#ifdef __BIG_ENDIAN - NV_PFIFO_CACHE1_BIG_ENDIAN | -#endif - NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); - nvkm_done(imem->ramfc); + chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c index 2f0dceaf3b018..5e7104457a02f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c @@ -194,8 +194,6 @@ nv40_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, } *args = data; struct nv04_fifo *fifo = nv04_fifo(base); struct nv04_fifo_chan *chan = NULL; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_instmem *imem = device->imem; int ret = -ENOSYS; nvif_ioctl(parent, "create channel dma size %d\n", size); @@ -224,21 +222,8 @@ nv40_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, return ret; args->v0.chid = chan->base.chid; - chan->ramfc = chan->base.chid * 128; - nvkm_kmap(imem->ramfc); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x00, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x04, args->v0.offset); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x0c, chan->base.push->addr >> 4); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x18, 0x30000000 | - NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | - NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | -#ifdef __BIG_ENDIAN - NV_PFIFO_CACHE1_BIG_ENDIAN | -#endif - NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); - nvkm_wo32(imem->ramfc, chan->ramfc + 0x3c, 0x0001ffff); - nvkm_done(imem->ramfc); + chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 76ba98b4d3bf7..6f3f5caaa7485 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -25,7 +25,7 @@ #include "chan.h" #include "runl.h" -#include +#include #include "nv50.h" #include "channv50.h" @@ -37,13 +37,69 @@ g84_chan_bind(struct nvkm_chan *chan) { struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; - nvkm_wr32(device, 0x002600 + (chan->id * 4), nv50_fifo_chan(chan)->ramfc->addr >> 8); + nvkm_wr32(device, 0x002600 + (chan->id * 4), chan->ramfc->addr >> 8); } +static int +g84_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + const u32 limit2 = ilog2(length / 8); + int ret; + + ret = nvkm_gpuobj_new(device, 0x0200, 0, true, chan->inst, &chan->eng); + if (ret) + return ret; + + ret = nvkm_gpuobj_new(device, 0x4000, 0, false, chan->inst, &chan->pgd); + if (ret) + return ret; + + ret = nvkm_gpuobj_new(device, 0x1000, 0x400, true, chan->inst, &chan->cache); + if (ret) + return ret; + + ret = nvkm_gpuobj_new(device, 0x100, 0x100, true, chan->inst, &chan->ramfc); + if (ret) + return ret; + + ret = nvkm_ramht_new(device, 0x8000, 16, chan->inst, &chan->ramht); + if (ret) + return ret; + + nv50_fifo_chan(chan)->eng = chan->eng; + nv50_fifo_chan(chan)->ramht = chan->ramht; + + nvkm_kmap(chan->ramfc); + nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); + nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); + nvkm_wo32(chan->ramfc, 0x48, chan->push->node->offset >> 4); + nvkm_wo32(chan->ramfc, 0x50, lower_32_bits(offset)); + nvkm_wo32(chan->ramfc, 0x54, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); + nvkm_wo32(chan->ramfc, 0x78, 0x00000000); + nvkm_wo32(chan->ramfc, 0x7c, 0x30000000 | devm); + nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | + (4 << 24) /* SEARCH_FULL */ | + (chan->ramht->gpuobj->node->offset >> 4)); + nvkm_wo32(chan->ramfc, 0x88, chan->cache->addr >> 10); + nvkm_wo32(chan->ramfc, 0x98, chan->inst->addr >> 12); + nvkm_done(chan->ramfc); + return 0; +} + +static const struct nvkm_chan_func_ramfc +g84_chan_ramfc = { + .write = g84_chan_ramfc_write, + .ctxdma = true, + .devm = 0xfff, +}; + const struct nvkm_chan_func g84_chan = { .inst = &nv50_chan_inst, .userd = &nv50_chan_userd, + .ramfc = &g84_chan_ramfc, .bind = g84_chan_bind, .unbind = nv50_chan_unbind, .start = nv50_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index afee42440ff55..351b35d580fd2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -82,6 +82,39 @@ gf100_chan_bind(struct nvkm_chan *chan) nvkm_wr32(device, 0x003000 + (chan->id * 8), 0xc0000000 | chan->inst->addr >> 12); } +static int +gf100_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + const u64 userd = nvkm_memory_addr(chan->userd.mem) + chan->userd.base; + const u32 limit2 = ilog2(length / 8); + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x08, lower_32_bits(userd)); + nvkm_wo32(chan->inst, 0x0c, upper_32_bits(userd)); + nvkm_wo32(chan->inst, 0x10, 0x0000face); + nvkm_wo32(chan->inst, 0x30, 0xfffff902); + nvkm_wo32(chan->inst, 0x48, lower_32_bits(offset)); + nvkm_wo32(chan->inst, 0x4c, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->inst, 0x54, 0x00000002); + nvkm_wo32(chan->inst, 0x84, 0x20400000); + nvkm_wo32(chan->inst, 0x94, 0x30000000 | devm); + nvkm_wo32(chan->inst, 0x9c, 0x00000100); + nvkm_wo32(chan->inst, 0xa4, 0x1f1f1f1f); + nvkm_wo32(chan->inst, 0xa8, 0x1f1f1f1f); + nvkm_wo32(chan->inst, 0xac, 0x0000001f); + nvkm_wo32(chan->inst, 0xb8, 0xf8000000); + nvkm_wo32(chan->inst, 0xf8, 0x10003080); /* 0x002310 */ + nvkm_wo32(chan->inst, 0xfc, 0x10000010); /* 0x002350 */ + nvkm_done(chan->inst); + return 0; +} + +static const struct nvkm_chan_func_ramfc +gf100_chan_ramfc = { + .write = gf100_chan_ramfc_write, + .devm = 0xfff, +}; + void gf100_chan_userd_clear(struct nvkm_chan *chan) { @@ -117,6 +150,7 @@ static const struct nvkm_chan_func gf100_chan = { .inst = &gf100_chan_inst, .userd = &gf100_chan_userd, + .ramfc = &gf100_chan_ramfc, .bind = gf100_chan_bind, .unbind = gf100_chan_unbind, .start = gf100_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 61966d433bce2..b75e3d089da31 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -78,6 +78,39 @@ gk104_chan_bind(struct nvkm_chan *chan) gk104_chan_bind_inst(chan); } +static int +gk104_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + const u64 userd = nvkm_memory_addr(chan->userd.mem) + chan->userd.base; + const u32 limit2 = ilog2(length / 8); + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x08, lower_32_bits(userd)); + nvkm_wo32(chan->inst, 0x0c, upper_32_bits(userd)); + nvkm_wo32(chan->inst, 0x10, 0x0000face); + nvkm_wo32(chan->inst, 0x30, 0xfffff902); + nvkm_wo32(chan->inst, 0x48, lower_32_bits(offset)); + nvkm_wo32(chan->inst, 0x4c, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->inst, 0x84, 0x20400000); + nvkm_wo32(chan->inst, 0x94, 0x30000000 | devm); + nvkm_wo32(chan->inst, 0x9c, 0x00000100); + nvkm_wo32(chan->inst, 0xac, 0x0000001f); + nvkm_wo32(chan->inst, 0xe4, priv ? 0x00000020 : 0x00000000); + nvkm_wo32(chan->inst, 0xe8, chan->id); + nvkm_wo32(chan->inst, 0xb8, 0xf8000000); + nvkm_wo32(chan->inst, 0xf8, 0x10003080); /* 0x002310 */ + nvkm_wo32(chan->inst, 0xfc, 0x10000010); /* 0x002350 */ + nvkm_done(chan->inst); + return 0; +} + +const struct nvkm_chan_func_ramfc +gk104_chan_ramfc = { + .write = gk104_chan_ramfc_write, + .devm = 0xfff, + .priv = true, +}; + const struct nvkm_chan_func_userd gk104_chan_userd = { .bar = 1, @@ -89,6 +122,7 @@ static const struct nvkm_chan_func gk104_chan = { .inst = &gf100_chan_inst, .userd = &gk104_chan_userd, + .ramfc = &gk104_chan_ramfc, .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 7ff7a435d8900..2a617215d9247 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -51,6 +51,7 @@ const struct nvkm_chan_func gk110_chan = { .inst = &gf100_chan_inst, .userd = &gk104_chan_userd, + .ramfc = &gk104_chan_ramfc, .bind = gk104_chan_bind, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index cc2638018db75..3764245fbca10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -36,6 +36,7 @@ const struct nvkm_chan_func gm107_chan = { .inst = &gf100_chan_inst, .userd = &gk104_chan_userd, + .ramfc = &gk104_chan_ramfc, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c index 3157a76332311..31022441ee8fc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c @@ -65,23 +65,9 @@ g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, args->v0.chid = chan->base.chid; ioffset = args->v0.ioffset; - ilength = order_base_2(args->v0.ilength / 8); + ilength = args->v0.ilength; - nvkm_kmap(chan->ramfc); - nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); - nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); - nvkm_wo32(chan->ramfc, 0x48, chan->base.push->node->offset >> 4); - nvkm_wo32(chan->ramfc, 0x50, lower_32_bits(ioffset)); - nvkm_wo32(chan->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16)); - nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); - nvkm_wo32(chan->ramfc, 0x78, 0x00000000); - nvkm_wo32(chan->ramfc, 0x7c, 0x30000001); - nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | - (4 << 24) /* SEARCH_FULL */ | - (chan->ramht->gpuobj->node->offset >> 4)); - nvkm_wo32(chan->ramfc, 0x88, chan->cache->addr >> 10); - nvkm_wo32(chan->ramfc, 0x98, chan->base.inst->addr >> 12); - nvkm_done(chan->ramfc); + chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 1e459afc36ea4..c43cd0c24a6de 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -190,28 +190,9 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; ioffset = args->v0.ioffset; - ilength = order_base_2(args->v0.ilength / 8); - - /* RAMFC */ - nvkm_kmap(chan->base.inst); - nvkm_wo32(chan->base.inst, 0x08, lower_32_bits(usermem)); - nvkm_wo32(chan->base.inst, 0x0c, upper_32_bits(usermem)); - nvkm_wo32(chan->base.inst, 0x10, 0x0000face); - nvkm_wo32(chan->base.inst, 0x30, 0xfffff902); - nvkm_wo32(chan->base.inst, 0x48, lower_32_bits(ioffset)); - nvkm_wo32(chan->base.inst, 0x4c, upper_32_bits(ioffset) | - (ilength << 16)); - nvkm_wo32(chan->base.inst, 0x54, 0x00000002); - nvkm_wo32(chan->base.inst, 0x84, 0x20400000); - nvkm_wo32(chan->base.inst, 0x94, 0x30000001); - nvkm_wo32(chan->base.inst, 0x9c, 0x00000100); - nvkm_wo32(chan->base.inst, 0xa4, 0x1f1f1f1f); - nvkm_wo32(chan->base.inst, 0xa8, 0x1f1f1f1f); - nvkm_wo32(chan->base.inst, 0xac, 0x0000001f); - nvkm_wo32(chan->base.inst, 0xb8, 0xf8000000); - nvkm_wo32(chan->base.inst, 0xf8, 0x10003080); /* 0x002310 */ - nvkm_wo32(chan->base.inst, 0xfc, 0x10000010); /* 0x002350 */ - nvkm_done(chan->base.inst); + ilength = args->v0.ilength; + + chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index b02ec7c545cfe..77de170c8144d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -176,7 +176,6 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, { struct gk104_fifo_chan *chan; int runlist = ffs(*runlists) -1, ret; - u64 usermem; if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) return -EINVAL; @@ -199,28 +198,7 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, *chid = chan->base.chid; *inst = chan->base.inst->addr; - usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; - ilength = order_base_2(ilength / 8); - - /* RAMFC */ - nvkm_kmap(chan->base.inst); - nvkm_wo32(chan->base.inst, 0x08, lower_32_bits(usermem)); - nvkm_wo32(chan->base.inst, 0x0c, upper_32_bits(usermem)); - nvkm_wo32(chan->base.inst, 0x10, 0x0000face); - nvkm_wo32(chan->base.inst, 0x30, 0xfffff902); - nvkm_wo32(chan->base.inst, 0x48, lower_32_bits(ioffset)); - nvkm_wo32(chan->base.inst, 0x4c, upper_32_bits(ioffset) | - (ilength << 16)); - nvkm_wo32(chan->base.inst, 0x84, 0x20400000); - nvkm_wo32(chan->base.inst, 0x94, 0x30000001); - nvkm_wo32(chan->base.inst, 0x9c, 0x00000100); - nvkm_wo32(chan->base.inst, 0xac, 0x0000001f); - nvkm_wo32(chan->base.inst, 0xe4, priv ? 0x00000020 : 0x00000000); - nvkm_wo32(chan->base.inst, 0xe8, chan->base.chid); - nvkm_wo32(chan->base.inst, 0xb8, 0xf8000000); - nvkm_wo32(chan->base.inst, 0xf8, 0x10003080); /* 0x002310 */ - nvkm_wo32(chan->base.inst, 0xfc, 0x10000010); /* 0x002350 */ - nvkm_done(chan->base.inst); + chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), priv); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 317787954a3ee..9a2308be73a2e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -120,7 +120,6 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, { struct gk104_fifo_chan *chan; int runlist = ffs(*runlists) -1, ret; - u64 usermem; if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) return -EINVAL; @@ -143,27 +142,7 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, *inst = chan->base.inst->addr; *token = chan->base.func->doorbell_handle(&chan->base); - /* Clear channel control registers. */ - usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; - ilength = order_base_2(ilength / 8); - - /* RAMFC */ - nvkm_kmap(chan->base.inst); - nvkm_wo32(chan->base.inst, 0x008, lower_32_bits(usermem)); - nvkm_wo32(chan->base.inst, 0x00c, upper_32_bits(usermem)); - nvkm_wo32(chan->base.inst, 0x010, 0x0000face); - nvkm_wo32(chan->base.inst, 0x030, 0x7ffff902); - nvkm_wo32(chan->base.inst, 0x048, lower_32_bits(ioffset)); - nvkm_wo32(chan->base.inst, 0x04c, upper_32_bits(ioffset) | - (ilength << 16)); - nvkm_wo32(chan->base.inst, 0x084, 0x20400000); - nvkm_wo32(chan->base.inst, 0x094, 0x30000001); - nvkm_wo32(chan->base.inst, 0x0e4, priv ? 0x00000020 : 0x00000000); - nvkm_wo32(chan->base.inst, 0x0e8, chan->base.chid); - nvkm_wo32(chan->base.inst, 0x0f4, 0x00001000); - nvkm_wo32(chan->base.inst, 0x0f8, 0x10003080); - nvkm_mo32(chan->base.inst, 0x218, 0x00000000, 0x00000000); - nvkm_done(chan->base.inst); + chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), priv); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c index 410fc9e6c1f99..88ea7c29941ed 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c @@ -65,21 +65,9 @@ nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, args->v0.chid = chan->base.chid; ioffset = args->v0.ioffset; - ilength = order_base_2(args->v0.ilength / 8); + ilength = args->v0.ilength; - nvkm_kmap(chan->ramfc); - nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); - nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); - nvkm_wo32(chan->ramfc, 0x48, chan->base.push->node->offset >> 4); - nvkm_wo32(chan->ramfc, 0x50, lower_32_bits(ioffset)); - nvkm_wo32(chan->ramfc, 0x54, upper_32_bits(ioffset) | (ilength << 16)); - nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); - nvkm_wo32(chan->ramfc, 0x78, 0x00000000); - nvkm_wo32(chan->ramfc, 0x7c, 0x30000001); - nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | - (4 << 24) /* SEARCH_FULL */ | - (chan->ramht->gpuobj->node->offset >> 4)); - nvkm_done(chan->ramfc); + chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), false); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index 9611bd73c1142..ffca153a9a282 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -38,6 +38,37 @@ gv100_chan_doorbell_handle(struct nvkm_chan *chan) return chan->id; } +static int +gv100_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + const u64 userd = nvkm_memory_addr(chan->userd.mem) + chan->userd.base; + const u32 limit2 = ilog2(length / 8); + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x008, lower_32_bits(userd)); + nvkm_wo32(chan->inst, 0x00c, upper_32_bits(userd)); + nvkm_wo32(chan->inst, 0x010, 0x0000face); + nvkm_wo32(chan->inst, 0x030, 0x7ffff902); + nvkm_wo32(chan->inst, 0x048, lower_32_bits(offset)); + nvkm_wo32(chan->inst, 0x04c, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->inst, 0x084, 0x20400000); + nvkm_wo32(chan->inst, 0x094, 0x30000000 | devm); + nvkm_wo32(chan->inst, 0x0e4, priv ? 0x00000020 : 0x00000000); + nvkm_wo32(chan->inst, 0x0e8, chan->id); + nvkm_wo32(chan->inst, 0x0f4, 0x00001000 | (priv ? 0x00000100 : 0x00000000)); + nvkm_wo32(chan->inst, 0x0f8, 0x10003080); + nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000); + nvkm_done(chan->inst); + return 0; +} + +const struct nvkm_chan_func_ramfc +gv100_chan_ramfc = { + .write = gv100_chan_ramfc_write, + .devm = 0xfff, + .priv = true, +}; + const struct nvkm_chan_func_userd gv100_chan_userd = { .bar = 1, /*FIXME: hw doesn't have poller, flip to user-allocated in uapi commit. */ @@ -49,6 +80,7 @@ static const struct nvkm_chan_func gv100_chan = { .inst = &gf100_chan_inst, .userd = &gv100_chan_userd, + .ramfc = &gv100_chan_ramfc, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = gk104_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index f4207100f20cb..cf8369b8b8819 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -38,42 +38,29 @@ #include -static const struct nv04_fifo_ramfc -nv04_fifo_ramfc[] = { - { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, - { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, - { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, - { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, - { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, - { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, - { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, - { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, - {} -}; - void nv04_chan_stop(struct nvkm_chan *chan) { - struct nv04_fifo *fifo = nv04_fifo(chan->cgrp->runl->fifo); - struct nvkm_device *device = fifo->base.engine.subdev.device; + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; struct nvkm_memory *fctx = device->imem->ramfc; - const struct nv04_fifo_ramfc *c; + const struct nvkm_ramfc_layout *c; unsigned long flags; - u32 data = nv04_fifo_chan(chan)->ramfc; + u32 data = chan->ramfc_offset; u32 chid; /* prevent fifo context switches */ - spin_lock_irqsave(&fifo->base.lock, flags); + spin_lock_irqsave(&fifo->lock, flags); nvkm_wr32(device, NV03_PFIFO_CACHES, 0); /* if this channel is active, replace it with a null context */ - chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->base.chid->mask; + chid = nvkm_rd32(device, NV03_PFIFO_CACHE1_PUSH1) & fifo->chid->mask; if (chid == chan->id) { nvkm_mask(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0x00000001, 0); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 0); nvkm_mask(device, NV04_PFIFO_CACHE1_PULL0, 0x00000001, 0); - c = fifo->ramfc; + c = chan->func->ramfc->layout; nvkm_kmap(fctx); do { u32 rm = ((1ULL << c->bits) - 1) << c->regs; @@ -84,14 +71,14 @@ nv04_chan_stop(struct nvkm_chan *chan) } while ((++c)->bits); nvkm_done(fctx); - c = fifo->ramfc; + c = chan->func->ramfc->layout; do { nvkm_wr32(device, c->regp, 0x00000000); } while ((++c)->bits); nvkm_wr32(device, NV03_PFIFO_CACHE1_GET, 0); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUT, 0); - nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->base.chid->mask); + nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH1, fifo->chid->mask); nvkm_wr32(device, NV03_PFIFO_CACHE1_PUSH0, 1); nvkm_wr32(device, NV04_PFIFO_CACHE1_PULL0, 1); } @@ -99,7 +86,7 @@ nv04_chan_stop(struct nvkm_chan *chan) /* restore normal operation, after disabling dma mode */ nvkm_mask(device, NV04_PFIFO_MODE, BIT(chan->id), 0); nvkm_wr32(device, NV03_PFIFO_CACHES, 1); - spin_unlock_irqrestore(&fifo->base.lock, flags); + spin_unlock_irqrestore(&fifo->lock, flags); } void @@ -113,6 +100,59 @@ nv04_chan_start(struct nvkm_chan *chan) spin_unlock_irqrestore(&fifo->lock, flags); } +void +nv04_chan_ramfc_clear(struct nvkm_chan *chan) +{ + struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; + const struct nvkm_ramfc_layout *c = chan->func->ramfc->layout; + + nvkm_kmap(ramfc); + do { + nvkm_wo32(ramfc, chan->ramfc_offset + c->ctxp, 0x00000000); + } while ((++c)->bits); + nvkm_done(ramfc); +} + +static int +nv04_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; + const u32 base = chan->id * 32; + + chan->ramfc_offset = base; + + nvkm_kmap(ramfc); + nvkm_wo32(ramfc, base + 0x00, offset); + nvkm_wo32(ramfc, base + 0x04, offset); + nvkm_wo32(ramfc, base + 0x08, chan->push->addr >> 4); + nvkm_wo32(ramfc, base + 0x10, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + nvkm_done(ramfc); + return 0; +} + +static const struct nvkm_chan_func_ramfc +nv04_chan_ramfc = { + .layout = (const struct nvkm_ramfc_layout[]) { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 16, 0, 0x08, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 16, 16, 0x08, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_PULL1 }, + {} + }, + .write = nv04_chan_ramfc_write, + .clear = nv04_chan_ramfc_clear, + .ctxdma = true, +}; + const struct nvkm_chan_func_userd nv04_chan_userd = { .bar = 0, @@ -129,6 +169,7 @@ static const struct nvkm_chan_func nv04_chan = { .inst = &nv04_chan_inst, .userd = &nv04_chan_userd, + .ramfc = &nv04_chan_ramfc, .start = nv04_chan_start, .stop = nv04_chan_stop, }; @@ -476,7 +517,6 @@ nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) return -ENOMEM; - fifo->ramfc = ramfc; *pfifo = &fifo->base; ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); @@ -507,5 +547,5 @@ int nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv04_fifo, device, type, inst, 16, nv04_fifo_ramfc, pfifo); + return nv04_fifo_new_(&nv04_fifo, device, type, inst, 0, NULL, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h index 879c686675d66..93248ab24a817 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h @@ -4,17 +4,10 @@ #define nv04_fifo(p) container_of((p), struct nv04_fifo, base) #include "priv.h" -struct nv04_fifo_ramfc { - unsigned bits:6; - unsigned ctxs:5; - unsigned ctxp:8; - unsigned regs:5; - unsigned regp; -}; +#define nv04_fifo_ramfc nvkm_ramfc_layout struct nv04_fifo { struct nvkm_fifo base; - const struct nv04_fifo_ramfc *ramfc; }; int nv04_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index bbad5ffa179bb..c7dbcd236630b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -21,7 +21,12 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" #include "chan.h" +#include "runl.h" + +#include +#include #include "nv04.h" #include "channv04.h" @@ -29,24 +34,52 @@ #include -static const struct nv04_fifo_ramfc -nv10_fifo_ramfc[] = { - { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, - { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, - { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, - { 16, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, - { 16, 16, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, - { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_STATE }, - { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, - { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_ENGINE }, - { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_PULL1 }, - {} +static int +nv10_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; + const u32 base = chan->id * 32; + + chan->ramfc_offset = base; + + nvkm_kmap(ramfc); + nvkm_wo32(ramfc, base + 0x00, offset); + nvkm_wo32(ramfc, base + 0x04, offset); + nvkm_wo32(ramfc, base + 0x0c, chan->push->addr >> 4); + nvkm_wo32(ramfc, base + 0x14, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + nvkm_done(ramfc); + return 0; +} + +static const struct nvkm_chan_func_ramfc +nv10_chan_ramfc = { + .layout = (const struct nvkm_ramfc_layout[]) { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, + { 16, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 16, 16, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_PULL1 }, + {} + }, + .write = nv10_chan_ramfc_write, + .clear = nv04_chan_ramfc_clear, + .ctxdma = true, }; static const struct nvkm_chan_func nv10_chan = { .inst = &nv04_chan_inst, .userd = &nv04_chan_userd, + .ramfc = &nv10_chan_ramfc, .start = nv04_chan_start, .stop = nv04_chan_stop, }; @@ -78,5 +111,5 @@ int nv10_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv10_fifo, device, type, inst, 32, nv10_fifo_ramfc, pfifo); + return nv04_fifo_new_(&nv10_fifo, device, type, inst, 0, NULL, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index b0dfa6f1a1d3f..a829b0b08f9b8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -21,8 +21,10 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" #include "chan.h" #include "chid.h" +#include "runl.h" #include "nv04.h" #include "channv04.h" @@ -33,29 +35,57 @@ #include -static const struct nv04_fifo_ramfc -nv17_fifo_ramfc[] = { - { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, - { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, - { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, - { 16, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, - { 16, 16, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, - { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_STATE }, - { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, - { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_ENGINE }, - { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_PULL1 }, - { 32, 0, 0x20, 0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE }, - { 32, 0, 0x24, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP }, - { 32, 0, 0x28, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT }, - { 32, 0, 0x2c, 0, NV10_PFIFO_CACHE1_SEMAPHORE }, - { 32, 0, 0x30, 0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE }, - {} +static int +nv17_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; + const u32 base = chan->id * 64; + + chan->ramfc_offset = base; + + nvkm_kmap(ramfc); + nvkm_wo32(ramfc, base + 0x00, offset); + nvkm_wo32(ramfc, base + 0x04, offset); + nvkm_wo32(ramfc, base + 0x0c, chan->push->addr >> 4); + nvkm_wo32(ramfc, base + 0x14, NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + nvkm_done(ramfc); + return 0; +} + +static const struct nvkm_chan_func_ramfc +nv17_chan_ramfc = { + .layout = (const struct nvkm_ramfc_layout[]) { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, + { 16, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 16, 16, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 32, 0, 0x18, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_PULL1 }, + { 32, 0, 0x20, 0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE }, + { 32, 0, 0x24, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP }, + { 32, 0, 0x28, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT }, + { 32, 0, 0x2c, 0, NV10_PFIFO_CACHE1_SEMAPHORE }, + { 32, 0, 0x30, 0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE }, + {} + }, + .write = nv17_chan_ramfc_write, + .clear = nv04_chan_ramfc_clear, + .ctxdma = true, }; static const struct nvkm_chan_func nv17_chan = { .inst = &nv04_chan_inst, .userd = &nv04_chan_userd, + .ramfc = &nv17_chan_ramfc, .start = nv04_chan_start, .stop = nv04_chan_stop, }; @@ -110,5 +140,5 @@ int nv17_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv17_fifo, device, type, inst, 32, nv17_fifo_ramfc, pfifo); + return nv04_fifo_new_(&nv17_fifo, device, type, inst, 0, NULL, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index c2dc2d7abc9db..faf19ec3b323c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" @@ -35,31 +36,61 @@ #include -static const struct nv04_fifo_ramfc -nv40_fifo_ramfc[] = { - { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, - { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, - { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, - { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, - { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, - { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_STATE }, - { 28, 0, 0x18, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, - { 2, 28, 0x18, 28, 0x002058 }, - { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_ENGINE }, - { 32, 0, 0x20, 0, NV04_PFIFO_CACHE1_PULL1 }, - { 32, 0, 0x24, 0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE }, - { 32, 0, 0x28, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP }, - { 32, 0, 0x2c, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT }, - { 32, 0, 0x30, 0, NV10_PFIFO_CACHE1_SEMAPHORE }, - { 32, 0, 0x34, 0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE }, - { 32, 0, 0x38, 0, NV40_PFIFO_GRCTX_INSTANCE }, - { 17, 0, 0x3c, 0, NV04_PFIFO_DMA_TIMESLICE }, - { 32, 0, 0x40, 0, 0x0032e4 }, - { 32, 0, 0x44, 0, 0x0032e8 }, - { 32, 0, 0x4c, 0, 0x002088 }, - { 32, 0, 0x50, 0, 0x003300 }, - { 32, 0, 0x54, 0, 0x00330c }, - {} +static int +nv40_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_memory *ramfc = chan->cgrp->runl->fifo->engine.subdev.device->imem->ramfc; + const u32 base = chan->id * 128; + + chan->ramfc_offset = base; + nv04_fifo_chan(chan)->ramfc = base; + + nvkm_kmap(ramfc); + nvkm_wo32(ramfc, base + 0x00, offset); + nvkm_wo32(ramfc, base + 0x04, offset); + nvkm_wo32(ramfc, base + 0x0c, chan->push->addr >> 4); + nvkm_wo32(ramfc, base + 0x18, 0x30000000 | + NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES | + NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES | +#ifdef __BIG_ENDIAN + NV_PFIFO_CACHE1_BIG_ENDIAN | +#endif + NV_PFIFO_CACHE1_DMA_FETCH_MAX_REQS_8); + nvkm_wo32(ramfc, base + 0x3c, 0x0001ffff); + nvkm_done(ramfc); + return 0; +} + +static const struct nvkm_chan_func_ramfc +nv40_chan_ramfc = { + .layout = (const struct nvkm_ramfc_layout[]) { + { 32, 0, 0x00, 0, NV04_PFIFO_CACHE1_DMA_PUT }, + { 32, 0, 0x04, 0, NV04_PFIFO_CACHE1_DMA_GET }, + { 32, 0, 0x08, 0, NV10_PFIFO_CACHE1_REF_CNT }, + { 32, 0, 0x0c, 0, NV04_PFIFO_CACHE1_DMA_INSTANCE }, + { 32, 0, 0x10, 0, NV04_PFIFO_CACHE1_DMA_DCOUNT }, + { 32, 0, 0x14, 0, NV04_PFIFO_CACHE1_DMA_STATE }, + { 28, 0, 0x18, 0, NV04_PFIFO_CACHE1_DMA_FETCH }, + { 2, 28, 0x18, 28, 0x002058 }, + { 32, 0, 0x1c, 0, NV04_PFIFO_CACHE1_ENGINE }, + { 32, 0, 0x20, 0, NV04_PFIFO_CACHE1_PULL1 }, + { 32, 0, 0x24, 0, NV10_PFIFO_CACHE1_ACQUIRE_VALUE }, + { 32, 0, 0x28, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMESTAMP }, + { 32, 0, 0x2c, 0, NV10_PFIFO_CACHE1_ACQUIRE_TIMEOUT }, + { 32, 0, 0x30, 0, NV10_PFIFO_CACHE1_SEMAPHORE }, + { 32, 0, 0x34, 0, NV10_PFIFO_CACHE1_DMA_SUBROUTINE }, + { 32, 0, 0x38, 0, NV40_PFIFO_GRCTX_INSTANCE }, + { 17, 0, 0x3c, 0, NV04_PFIFO_DMA_TIMESLICE }, + { 32, 0, 0x40, 0, 0x0032e4 }, + { 32, 0, 0x44, 0, 0x0032e8 }, + { 32, 0, 0x4c, 0, 0x002088 }, + { 32, 0, 0x50, 0, 0x003300 }, + { 32, 0, 0x54, 0, 0x00330c }, + {} + }, + .write = nv40_chan_ramfc_write, + .clear = nv04_chan_ramfc_clear, + .ctxdma = true, }; static const struct nvkm_chan_func_userd @@ -73,6 +104,7 @@ static const struct nvkm_chan_func nv40_chan = { .inst = &nv04_chan_inst, .userd = &nv40_chan_userd, + .ramfc = &nv40_chan_ramfc, .start = nv04_chan_start, .stop = nv04_chan_stop, }; @@ -157,5 +189,5 @@ int nv40_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv40_fifo, device, type, inst, 32, nv40_fifo_ramfc, pfifo); + return nv04_fifo_new_(&nv40_fifo, device, type, inst, 0, NULL, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 24b4580481802..7d5c0d0269678 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -26,7 +26,7 @@ #include "chid.h" #include "runl.h" -#include +#include #include #include "nv50.h" @@ -63,9 +63,58 @@ nv50_chan_bind(struct nvkm_chan *chan) { struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; - nvkm_wr32(device, 0x002600 + (chan->id * 4), nv50_fifo_chan(chan)->ramfc->addr >> 12); + nvkm_wr32(device, 0x002600 + (chan->id * 4), chan->ramfc->addr >> 12); } +static int +nv50_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; + const u32 limit2 = ilog2(length / 8); + int ret; + + ret = nvkm_gpuobj_new(device, 0x0200, 0x1000, true, chan->inst, &chan->ramfc); + if (ret) + return ret; + + ret = nvkm_gpuobj_new(device, 0x1200, 0, true, chan->inst, &chan->eng); + if (ret) + return ret; + + ret = nvkm_gpuobj_new(device, 0x4000, 0, false, chan->inst, &chan->pgd); + if (ret) + return ret; + + ret = nvkm_ramht_new(device, 0x8000, 16, chan->inst, &chan->ramht); + if (ret) + return ret; + + nv50_fifo_chan(chan)->eng = chan->eng; + nv50_fifo_chan(chan)->ramht = chan->ramht; + + nvkm_kmap(chan->ramfc); + nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); + nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); + nvkm_wo32(chan->ramfc, 0x48, chan->push->node->offset >> 4); + nvkm_wo32(chan->ramfc, 0x50, lower_32_bits(offset)); + nvkm_wo32(chan->ramfc, 0x54, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->ramfc, 0x60, 0x7fffffff); + nvkm_wo32(chan->ramfc, 0x78, 0x00000000); + nvkm_wo32(chan->ramfc, 0x7c, 0x30000000 | devm); + nvkm_wo32(chan->ramfc, 0x80, ((chan->ramht->bits - 9) << 27) | + (4 << 24) /* SEARCH_FULL */ | + (chan->ramht->gpuobj->node->offset >> 4)); + nvkm_done(chan->ramfc); + return 0; +} + +static const struct nvkm_chan_func_ramfc +nv50_chan_ramfc = { + .write = nv50_chan_ramfc_write, + .ctxdma = true, + .devm = 0xfff, +}; + const struct nvkm_chan_func_userd nv50_chan_userd = { .bar = 0, @@ -83,6 +132,7 @@ static const struct nvkm_chan_func nv50_chan = { .inst = &nv50_chan_inst, .userd = &nv50_chan_userd, + .ramfc = &nv50_chan_ramfc, .bind = nv50_chan_bind, .unbind = nv50_chan_unbind, .start = nv50_chan_start, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index c56deb78e9cc0..43f248a109f65 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -82,6 +82,7 @@ extern const struct nvkm_engn_func nv04_engn; extern const struct nvkm_cgrp_func nv04_cgrp; extern const struct nvkm_chan_func_inst nv04_chan_inst; extern const struct nvkm_chan_func_userd nv04_chan_userd; +void nv04_chan_ramfc_clear(struct nvkm_chan *); void nv04_chan_start(struct nvkm_chan *); void nv04_chan_stop(struct nvkm_chan *); @@ -154,6 +155,7 @@ bool gk104_engn_chsw(struct nvkm_engn *); int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); extern const struct nvkm_engn_func gk104_engn_ce; extern const struct nvkm_chan_func_userd gk104_chan_userd; +extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc; void gk104_chan_bind(struct nvkm_chan *); void gk104_chan_bind_inst(struct nvkm_chan *); void gk104_chan_unbind(struct nvkm_chan *); @@ -189,6 +191,7 @@ extern const struct nvkm_runq_func gv100_runq; extern const struct nvkm_engn_func gv100_engn; extern const struct nvkm_engn_func gv100_engn_ce; extern const struct nvkm_chan_func_userd gv100_chan_userd; +extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 42d2975b82aa5..a8452eee2949d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -51,6 +51,7 @@ static const struct nvkm_chan_func tu102_chan = { .inst = &gf100_chan_inst, .userd = &gv100_chan_userd, + .ramfc = &gv100_chan_ramfc, .bind = gk104_chan_bind_inst, .unbind = gk104_chan_unbind, .start = tu102_chan_start, -- GitLab From 8ab849d6dd4c2eb8880096e53e91dfb6ca37b589 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:37 +1000 Subject: [PATCH 1333/1927] drm/nouveau/fifo: add new engine context handling Builds on the context tracking that was added earlier. - marks engine context PTEs as 'priv' where possible Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/fifo.h | 9 -- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 5 +- .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c | 35 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h | 6 + .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 128 ++--------------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 9 +- .../drm/nouveau/nvkm/engine/fifo/chang84.c | 109 --------------- .../drm/nouveau/nvkm/engine/fifo/changf100.h | 4 - .../drm/nouveau/nvkm/engine/fifo/changk104.h | 13 -- .../drm/nouveau/nvkm/engine/fifo/channv04.h | 2 - .../drm/nouveau/nvkm/engine/fifo/channv50.c | 131 ------------------ .../drm/nouveau/nvkm/engine/fifo/channv50.h | 4 - .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 119 ---------------- .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 75 ++++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/g98.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 60 +++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 81 ++++++++--- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 5 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 1 - .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 100 ------------- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 123 ---------------- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 78 ----------- .../nouveau/nvkm/engine/fifo/gpfifotu102.c | 4 - .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 43 +++++- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 15 -- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 46 +++++- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 58 +++++++- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 2 - .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 4 + .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 25 +++- 39 files changed, 382 insertions(+), 930 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 1616a329db9e2..da8abcbeb965c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -8,11 +8,6 @@ struct nvkm_fault_data; #define NVKM_FIFO_ENGN_NR 16 -struct nvkm_fifo_engn { - struct nvkm_object *object; - int refcount; -}; - struct nvkm_chan { const struct nvkm_chan_func *func; char name[64]; @@ -41,13 +36,10 @@ struct nvkm_chan { struct list_head cctxs; - struct nvkm_fifo *fifo; struct nvkm_object object; struct list_head head; struct nvkm_gpuobj *push; - - struct nvkm_fifo_engn engn[NVKM_FIFO_ENGN_NR]; }; struct nvkm_chan *nvkm_chan_get_chid(struct nvkm_engine *, int id, unsigned long *irqflags); @@ -78,7 +70,6 @@ struct nvkm_fifo { struct nvkm_vma *bar1; } userd; - int nr; spinlock_t lock; struct mutex mutex; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index c77af1b3ffa06..9362afe37f3ec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -318,7 +318,7 @@ int nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo *fifo) { - int ret, nr; + int ret; fifo->func = func; INIT_LIST_HEAD(&fifo->runqs); @@ -335,9 +335,6 @@ nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, if (ret) return ret; - nr = func->chid_nr(fifo); - fifo->nr = nr; - if (func->nonstall) { ret = nvkm_event_init(func->nonstall, &fifo->engine.subdev, 1, 1, &fifo->nonstall.event); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c index ac42dfc9d4cc4..ea53fb3d5d06f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.c @@ -25,6 +25,7 @@ #include "runl.h" #include "priv.h" +#include #include static void @@ -37,6 +38,7 @@ nvkm_cgrp_ectx_put(struct nvkm_cgrp *cgrp, struct nvkm_ectx **pectx) if (refcount_dec_and_test(&ectx->refs)) { CGRP_TRACE(cgrp, "dtor ectx %d[%s]", engn->id, engn->engine->subdev.name); + nvkm_object_del(&ectx->object); list_del(&ectx->head); kfree(ectx); } @@ -49,6 +51,11 @@ static int nvkm_cgrp_ectx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_ectx **pectx, struct nvkm_chan *chan, struct nvkm_client *client) { + struct nvkm_engine *engine = engn->engine; + struct nvkm_oclass cclass = { + .client = client, + .engine = engine, + }; struct nvkm_ectx *ectx; int ret = 0; @@ -67,7 +74,18 @@ nvkm_cgrp_ectx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_e ectx->engn = engn; refcount_set(&ectx->refs, 1); + refcount_set(&ectx->uses, 0); list_add_tail(&ectx->head, &cgrp->ectxs); + + /* Allocate the HW structures. */ + if (engine->func->fifo.cclass) + ret = engine->func->fifo.cclass(chan, &cclass, &ectx->object); + else if (engine->func->cclass) + ret = nvkm_object_new_(engine->func->cclass, &cclass, NULL, 0, &ectx->object); + + if (ret) + nvkm_cgrp_ectx_put(cgrp, pectx); + return ret; } @@ -81,6 +99,8 @@ nvkm_cgrp_vctx_put(struct nvkm_cgrp *cgrp, struct nvkm_vctx **pvctx) if (refcount_dec_and_test(&vctx->refs)) { CGRP_TRACE(cgrp, "dtor vctx %d[%s]", engn->id, engn->engine->subdev.name); + nvkm_vmm_put(vctx->vmm, &vctx->vma); + nvkm_gpuobj_del(&vctx->inst); nvkm_cgrp_ectx_put(cgrp, &vctx->ectx); if (vctx->vmm) { @@ -130,6 +150,21 @@ nvkm_cgrp_vctx_get(struct nvkm_cgrp *cgrp, struct nvkm_engn *engn, struct nvkm_c vctx->vmm = nvkm_vmm_ref(chan->vmm); refcount_set(&vctx->refs, 1); list_add_tail(&vctx->head, &cgrp->vctxs); + + /* MMU on some GPUs needs to know engine usage for TLB invalidation. */ + if (vctx->vmm) + atomic_inc(&vctx->vmm->engref[engn->engine->subdev.type]); + + /* Allocate the HW structures. */ + if (engn->func->bind) { + ret = nvkm_object_bind(vctx->ectx->object, NULL, 0, &vctx->inst); + if (ret == 0 && engn->func->ctor) + ret = engn->func->ctor(engn, vctx); + } + + if (ret) + nvkm_cgrp_vctx_put(cgrp, pvctx); + return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h index be9fbe9fab373..5f6abd59a6ff0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/cgrp.h @@ -10,12 +10,18 @@ struct nvkm_vctx { struct nvkm_vmm *vmm; refcount_t refs; + struct nvkm_gpuobj *inst; + struct nvkm_vma *vma; + struct list_head head; }; struct nvkm_ectx { struct nvkm_engn *engn; refcount_t refs; + refcount_t uses; + + struct nvkm_object *object; struct list_head head; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 00680b3ca9abd..a35439dda054a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -28,7 +28,6 @@ #include "runl.h" #include "priv.h" -#include #include #include #include @@ -46,94 +45,15 @@ struct nvkm_fifo_chan_object { int hash; }; -static struct nvkm_fifo_engn * -nvkm_fifo_chan_engn(struct nvkm_fifo_chan *chan, struct nvkm_engine *engine) -{ - int engi = chan->fifo->func->engine_id(chan->fifo, engine); - if (engi >= 0) - return &chan->engn[engi]; - return NULL; -} - -static int -nvkm_fifo_chan_child_fini(struct nvkm_oproxy *base, bool suspend) -{ - struct nvkm_fifo_chan_object *object = - container_of(base, typeof(*object), oproxy); - struct nvkm_engine *engine = object->oproxy.object->engine; - struct nvkm_fifo_chan *chan = object->chan; - struct nvkm_fifo_engn *engn = nvkm_fifo_chan_engn(chan, engine); - const char *name = engine->subdev.name; - int ret = 0; - - if (chan->func->engine_fini) { - ret = chan->func->engine_fini(chan, engine, suspend); - if (ret) { - nvif_error(&chan->object, - "detach %s failed, %d\n", name, ret); - return ret; - } - } - - if (engn->object) { - ret = nvkm_object_fini(engn->object, suspend); - if (ret && suspend) - return ret; - } - - nvif_trace(&chan->object, "detached %s\n", name); - return ret; -} - -static int -nvkm_fifo_chan_child_init(struct nvkm_oproxy *base) -{ - struct nvkm_fifo_chan_object *object = - container_of(base, typeof(*object), oproxy); - struct nvkm_engine *engine = object->oproxy.object->engine; - struct nvkm_fifo_chan *chan = object->chan; - struct nvkm_fifo_engn *engn = nvkm_fifo_chan_engn(chan, engine); - const char *name = engine->subdev.name; - int ret; - - if (engn->object) { - ret = nvkm_object_init(engn->object); - if (ret) - return ret; - } - - if (chan->func->engine_init) { - ret = chan->func->engine_init(chan, engine); - if (ret) { - nvif_error(&chan->object, - "attach %s failed, %d\n", name, ret); - return ret; - } - } - - nvif_trace(&chan->object, "attached %s\n", name); - return 0; -} - static void nvkm_fifo_chan_child_del(struct nvkm_oproxy *base) { struct nvkm_fifo_chan_object *object = container_of(base, typeof(*object), oproxy); - struct nvkm_engine *engine = object->oproxy.base.engine; struct nvkm_fifo_chan *chan = object->chan; - struct nvkm_fifo_engn *engn = nvkm_fifo_chan_engn(chan, engine); if (chan->func->object_dtor) chan->func->object_dtor(chan, object->hash); - - if (!--engn->refcount) { - if (chan->func->engine_dtor) - chan->func->engine_dtor(chan, engine); - nvkm_object_del(&engn->object); - if (chan->vmm) - atomic_dec(&chan->vmm->engref[engine->subdev.type]); - } } static const struct nvkm_oproxy_func @@ -147,7 +67,8 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, { struct nvkm_engine *engine = oclass->engine; struct nvkm_fifo_chan *chan = nvkm_fifo_chan(oclass->parent); - struct nvkm_fifo_engn *engn = nvkm_fifo_chan_engn(chan, engine); + struct nvkm_ectx *engn = nvkm_list_find(engn, &chan->cgrp->ectxs, head, + engn->engn->engine == engine); struct nvkm_fifo_chan_object *object; int ret = 0; @@ -157,33 +78,6 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, object->chan = chan; *pobject = &object->oproxy.base; - if (!engn->refcount++) { - struct nvkm_oclass cclass = { - .client = oclass->client, - .engine = oclass->engine, - }; - - if (chan->vmm) - atomic_inc(&chan->vmm->engref[engine->subdev.type]); - - if (engine->func->fifo.cclass) { - ret = engine->func->fifo.cclass(chan, &cclass, - &engn->object); - } else - if (engine->func->cclass) { - ret = nvkm_object_new_(engine->func->cclass, &cclass, - NULL, 0, &engn->object); - } - if (ret) - return ret; - - if (chan->func->engine_ctor) { - ret = chan->func->engine_ctor(chan, oclass->engine, - engn->object); - if (ret) - return ret; - } - } ret = oclass->base.ctor(&(const struct nvkm_oclass) { .base = oclass->base, @@ -210,10 +104,16 @@ nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, } void -nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct nvkm_cctx *cctx) +nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx *cctx) { struct nvkm_cgrp *cgrp = chan->cgrp; struct nvkm_runl *runl = cgrp->runl; + struct nvkm_engine *engine = engn->engine; + + if (!engn->func->bind) + return; + + CHAN_TRACE(chan, "%sbind cctx %d[%s]", cctx ? "" : "un", engn->id, engine->subdev.name); /* Prevent any channel in channel group from being rescheduled, kick them * off host and any engine(s) they're loaded on. @@ -225,10 +125,7 @@ nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_oproxy *oproxy, struct n nvkm_chan_preempt(chan, true); /* Update context pointer. */ - if (cctx) - nvkm_fifo_chan_child_init(nvkm_oproxy(oproxy->object)); - else - nvkm_fifo_chan_child_fini(nvkm_oproxy(oproxy->object), false); + engn->func->bind(engn, cctx, chan); /* Resume normal operation. */ if (cgrp->hw) @@ -558,10 +455,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, *func = *fifo->func->chan.func; func->dtor = fn->dtor; - func->engine_ctor = fn->engine_ctor; - func->engine_dtor = fn->engine_dtor; - func->engine_init = fn->engine_init; - func->engine_fini = fn->engine_fini; func->object_ctor = fn->object_ctor; func->object_dtor = fn->object_dtor; @@ -572,7 +465,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, atomic_set(&chan->errored, 0); nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); - chan->fifo = fifo; INIT_LIST_HEAD(&chan->cctxs); INIT_LIST_HEAD(&chan->head); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 6e41eba0a7e2c..bb142593d8b65 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -52,12 +52,6 @@ struct nvkm_chan_func { u32 (*doorbell_handle)(struct nvkm_chan *); void *(*dtor)(struct nvkm_fifo_chan *); - int (*engine_ctor)(struct nvkm_fifo_chan *, struct nvkm_engine *, - struct nvkm_object *); - void (*engine_dtor)(struct nvkm_fifo_chan *, struct nvkm_engine *); - int (*engine_init)(struct nvkm_fifo_chan *, struct nvkm_engine *); - int (*engine_fini)(struct nvkm_fifo_chan *, struct nvkm_engine *, - bool suspend); int (*object_ctor)(struct nvkm_fifo_chan *, struct nvkm_object *); void (*object_dtor)(struct nvkm_fifo_chan *, int); }; @@ -78,8 +72,7 @@ int nvkm_chan_preempt_locked(struct nvkm_chan *, bool wait); int nvkm_chan_cctx_get(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx **, struct nvkm_client * /*TODO: remove need for this */); void nvkm_chan_cctx_put(struct nvkm_chan *, struct nvkm_cctx **); -struct nvkm_oproxy; -void nvkm_chan_cctx_bind(struct nvkm_chan *, struct nvkm_oproxy *, struct nvkm_cctx *); +void nvkm_chan_cctx_bind(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cctx *); #define CHAN_PRCLI(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:[%s]"f, (c)->id, (c)->name, ##a) #define CHAN_PRINT(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:"f, (c)->id, ##a) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index f5e3557b9e50f..359d8b5b124c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -30,111 +30,6 @@ #include -static int -g84_fifo_chan_engine_addr(struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW : return -1; - case NVKM_ENGINE_GR : return 0x0020; - case NVKM_ENGINE_VP : - case NVKM_ENGINE_MSPDEC: return 0x0040; - case NVKM_ENGINE_MPEG : - case NVKM_ENGINE_MSPPP : return 0x0060; - case NVKM_ENGINE_BSP : - case NVKM_ENGINE_MSVLD : return 0x0080; - case NVKM_ENGINE_CIPHER: - case NVKM_ENGINE_SEC : return 0x00a0; - case NVKM_ENGINE_CE : return 0x00c0; - default: - WARN_ON(1); - return -1; - } -} - -static int -g84_fifo_chan_engine_fini(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, bool suspend) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nv50_fifo *fifo = chan->fifo; - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - u32 engn, save; - int offset; - bool done; - - offset = g84_fifo_chan_engine_addr(engine); - if (offset < 0) - return 0; - - engn = fifo->base.func->engine_id(&fifo->base, engine) - 1; - save = nvkm_mask(device, 0x002520, 0x0000003f, 1 << engn); - nvkm_wr32(device, 0x0032fc, chan->base.inst->addr >> 12); - done = nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x0032fc) != 0xffffffff) - break; - ) >= 0; - nvkm_wr32(device, 0x002520, save); - if (!done) { - nvkm_error(subdev, "channel %d [%s] unload timeout\n", - chan->base.chid, chan->base.object.client->name); - if (suspend) - return -EBUSY; - } - - nvkm_kmap(chan->eng); - nvkm_wo32(chan->eng, offset + 0x00, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x04, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x08, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x0c, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x10, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x14, 0x00000000); - nvkm_done(chan->eng); - return 0; -} - - -static int -g84_fifo_chan_engine_init(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nvkm_gpuobj *engn = *nv50_fifo_chan_engine(chan, engine); - u64 limit, start; - int offset; - - offset = g84_fifo_chan_engine_addr(engine); - if (offset < 0) - return 0; - limit = engn->addr + engn->size - 1; - start = engn->addr; - - nvkm_kmap(chan->eng); - nvkm_wo32(chan->eng, offset + 0x00, 0x00190000); - nvkm_wo32(chan->eng, offset + 0x04, lower_32_bits(limit)); - nvkm_wo32(chan->eng, offset + 0x08, lower_32_bits(start)); - nvkm_wo32(chan->eng, offset + 0x0c, upper_32_bits(limit) << 24 | - upper_32_bits(start)); - nvkm_wo32(chan->eng, offset + 0x10, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x14, 0x00000000); - nvkm_done(chan->eng); - return 0; -} - -static int -g84_fifo_chan_engine_ctor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, - struct nvkm_object *object) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - - if (g84_fifo_chan_engine_addr(engine) < 0) - return 0; - - return nvkm_object_bind(object, NULL, 0, nv50_fifo_chan_engine(chan, engine)); -} - static int g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, struct nvkm_object *object) @@ -169,10 +64,6 @@ g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, static const struct nvkm_fifo_chan_func g84_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .engine_ctor = g84_fifo_chan_engine_ctor, - .engine_dtor = nv50_fifo_chan_engine_dtor, - .engine_init = g84_fifo_chan_engine_init, - .engine_fini = g84_fifo_chan_engine_fini, .object_ctor = g84_fifo_chan_object_ctor, .object_dtor = nv50_fifo_chan_object_dtor, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h index 3c7b6c0d8cf12..739cd750742e3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h @@ -16,10 +16,6 @@ struct gf100_fifo_chan { #define GF100_FIFO_ENGN_CE0 4 #define GF100_FIFO_ENGN_CE1 5 #define GF100_FIFO_ENGN_SW 15 - struct gf100_fifo_engn { - struct nvkm_gpuobj *inst; - struct nvkm_vma *vma; - } engn[NVKM_FIFO_ENGN_NR]; }; extern const struct nvkm_fifo_chan_oclass gf100_fifo_gpfifo_oclass; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 95e788f31590d..2043f06f500b2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -11,10 +11,6 @@ struct gk104_fifo_chan { int runl; #define GK104_FIFO_ENGN_SW 15 - struct gk104_fifo_engn { - struct nvkm_gpuobj *inst; - struct nvkm_vma *vma; - } engn[NVKM_FIFO_ENGN_NR]; }; extern const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func; @@ -22,11 +18,6 @@ extern const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func; int gk104_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); void *gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *); -struct gk104_fifo_engn *gk104_fifo_gpfifo_engine(struct gk104_fifo_chan *, struct nvkm_engine *); -int gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *, struct nvkm_engine *, - struct nvkm_object *); -void gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *, - struct nvkm_engine *); int gv100_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); @@ -34,10 +25,6 @@ int gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *, struct gk104_fifo *, u64 *, u16 *, u64, u64, u64, u64 *, bool, u32 *, const struct nvkm_oclass *, struct nvkm_object **); -int gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *, - struct nvkm_engine *); -int gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *, - struct nvkm_engine *, bool); int tu102_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, void *data, u32 size, struct nvkm_object **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h index 837f9f7f69cee..5ebfa2241750a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h @@ -8,12 +8,10 @@ struct nv04_fifo_chan { struct nvkm_fifo_chan base; struct nv04_fifo *fifo; - u32 ramfc; #define NV04_FIFO_ENGN_SW 0 #define NV04_FIFO_ENGN_GR 1 #define NV04_FIFO_ENGN_MPEG 2 #define NV04_FIFO_ENGN_DMA 3 - struct nvkm_gpuobj *engn[NVKM_FIFO_ENGN_NR]; }; extern const struct nvkm_fifo_chan_func nv04_fifo_dma_func; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index 0bf59c4e92b96..cb56060414995 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -28,133 +28,6 @@ #include #include -static int -nv50_fifo_chan_engine_addr(struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW : return -1; - case NVKM_ENGINE_GR : return 0x0000; - case NVKM_ENGINE_MPEG : return 0x0060; - default: - WARN_ON(1); - return -1; - } -} - -struct nvkm_gpuobj ** -nv50_fifo_chan_engine(struct nv50_fifo_chan *chan, struct nvkm_engine *engine) -{ - int engi = chan->base.fifo->func->engine_id(chan->base.fifo, engine); - if (engi >= 0) - return &chan->engn[engi]; - return NULL; -} - -static int -nv50_fifo_chan_engine_fini(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, bool suspend) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nv50_fifo *fifo = chan->fifo; - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - int offset, ret = 0; - u32 me; - - offset = nv50_fifo_chan_engine_addr(engine); - if (offset < 0) - return 0; - - /* HW bug workaround: - * - * PFIFO will hang forever if the connected engines don't report - * that they've processed the context switch request. - * - * In order for the kickoff to work, we need to ensure all the - * connected engines are in a state where they can answer. - * - * Newer chipsets don't seem to suffer from this issue, and well, - * there's also a "ignore these engines" bitmask reg we can use - * if we hit the issue there.. - */ - me = nvkm_mask(device, 0x00b860, 0x00000001, 0x00000001); - - /* do the kickoff... */ - nvkm_wr32(device, 0x0032fc, chan->base.inst->addr >> 12); - if (nvkm_msec(device, 2000, - if (nvkm_rd32(device, 0x0032fc) != 0xffffffff) - break; - ) < 0) { - nvkm_error(subdev, "channel %d [%s] unload timeout\n", - chan->base.chid, chan->base.object.client->name); - if (suspend) - ret = -EBUSY; - } - nvkm_wr32(device, 0x00b860, me); - - if (ret == 0) { - nvkm_kmap(chan->eng); - nvkm_wo32(chan->eng, offset + 0x00, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x04, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x08, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x0c, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x10, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x14, 0x00000000); - nvkm_done(chan->eng); - } - - return ret; -} - -static int -nv50_fifo_chan_engine_init(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - struct nvkm_gpuobj *engn = *nv50_fifo_chan_engine(chan, engine); - u64 limit, start; - int offset; - - offset = nv50_fifo_chan_engine_addr(engine); - if (offset < 0) - return 0; - limit = engn->addr + engn->size - 1; - start = engn->addr; - - nvkm_kmap(chan->eng); - nvkm_wo32(chan->eng, offset + 0x00, 0x00190000); - nvkm_wo32(chan->eng, offset + 0x04, lower_32_bits(limit)); - nvkm_wo32(chan->eng, offset + 0x08, lower_32_bits(start)); - nvkm_wo32(chan->eng, offset + 0x0c, upper_32_bits(limit) << 24 | - upper_32_bits(start)); - nvkm_wo32(chan->eng, offset + 0x10, 0x00000000); - nvkm_wo32(chan->eng, offset + 0x14, 0x00000000); - nvkm_done(chan->eng); - return 0; -} - -void -nv50_fifo_chan_engine_dtor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - nvkm_gpuobj_del(nv50_fifo_chan_engine(chan, engine)); -} - -static int -nv50_fifo_chan_engine_ctor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, - struct nvkm_object *object) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - - if (nv50_fifo_chan_engine_addr(engine) < 0) - return 0; - - return nvkm_object_bind(object, NULL, 0, nv50_fifo_chan_engine(chan, engine)); -} - void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *base, int cookie) { @@ -193,10 +66,6 @@ nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) static const struct nvkm_fifo_chan_func nv50_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .engine_ctor = nv50_fifo_chan_engine_ctor, - .engine_dtor = nv50_fifo_chan_engine_dtor, - .engine_init = nv50_fifo_chan_engine_init, - .engine_fini = nv50_fifo_chan_engine_fini, .object_ctor = nv50_fifo_chan_object_ctor, .object_dtor = nv50_fifo_chan_object_dtor, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index 343356a0e5ef2..ced832c7f425c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -9,7 +9,6 @@ struct nv50_fifo_chan { struct nv50_fifo *fifo; struct nvkm_fifo_chan base; - struct nvkm_gpuobj *eng; struct nvkm_ramht *ramht; #define NV50_FIFO_ENGN_SW 0 @@ -31,14 +30,11 @@ struct nv50_fifo_chan { #define G84_FIFO_ENGN_BSP 6 #define G84_FIFO_ENGN_MSVLD 6 #define G84_FIFO_ENGN_DMA 7 - struct nvkm_gpuobj *engn[NVKM_FIFO_ENGN_NR]; }; int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *); -struct nvkm_gpuobj **nv50_fifo_chan_engine(struct nv50_fifo_chan *, struct nvkm_engine *); -void nv50_fifo_chan_engine_dtor(struct nvkm_fifo_chan *, struct nvkm_engine *); void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *, int); int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c index 5e7104457a02f..ca6fd8a5af6b9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c @@ -31,121 +31,6 @@ #include #include -static bool -nv40_fifo_dma_engine(struct nvkm_engine *engine, u32 *reg, u32 *ctx) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW: - return false; - case NVKM_ENGINE_GR: - *reg = 0x0032e0; - *ctx = 0x38; - return true; - case NVKM_ENGINE_MPEG: - if (engine->subdev.device->chipset < 0x44) - return false; - *reg = 0x00330c; - *ctx = 0x54; - return true; - default: - WARN_ON(1); - return false; - } -} - -static struct nvkm_gpuobj ** -nv40_fifo_dma_engn(struct nv04_fifo_chan *chan, struct nvkm_engine *engine) -{ - int engi = chan->base.fifo->func->engine_id(chan->base.fifo, engine); - if (engi >= 0) - return &chan->engn[engi]; - return NULL; -} - -static int -nv40_fifo_dma_engine_fini(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, bool suspend) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_instmem *imem = device->imem; - unsigned long flags; - u32 reg, ctx; - int chid; - - if (!nv40_fifo_dma_engine(engine, ®, &ctx)) - return 0; - - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_mask(device, 0x002500, 0x00000001, 0x00000000); - - chid = nvkm_rd32(device, 0x003204) & (fifo->base.nr - 1); - if (chid == chan->base.chid) - nvkm_wr32(device, reg, 0x00000000); - nvkm_kmap(imem->ramfc); - nvkm_wo32(imem->ramfc, chan->ramfc + ctx, 0x00000000); - nvkm_done(imem->ramfc); - - nvkm_mask(device, 0x002500, 0x00000001, 0x00000001); - spin_unlock_irqrestore(&fifo->base.lock, flags); - return 0; -} - -static int -nv40_fifo_dma_engine_init(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nv04_fifo *fifo = chan->fifo; - struct nvkm_device *device = fifo->base.engine.subdev.device; - struct nvkm_instmem *imem = device->imem; - unsigned long flags; - u32 inst, reg, ctx; - int chid; - - if (!nv40_fifo_dma_engine(engine, ®, &ctx)) - return 0; - inst = (*nv40_fifo_dma_engn(chan, engine))->addr >> 4; - - spin_lock_irqsave(&fifo->base.lock, flags); - nvkm_mask(device, 0x002500, 0x00000001, 0x00000000); - - chid = nvkm_rd32(device, 0x003204) & (fifo->base.nr - 1); - if (chid == chan->base.chid) - nvkm_wr32(device, reg, inst); - nvkm_kmap(imem->ramfc); - nvkm_wo32(imem->ramfc, chan->ramfc + ctx, inst); - nvkm_done(imem->ramfc); - - nvkm_mask(device, 0x002500, 0x00000001, 0x00000001); - spin_unlock_irqrestore(&fifo->base.lock, flags); - return 0; -} - -static void -nv40_fifo_dma_engine_dtor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - nvkm_gpuobj_del(nv40_fifo_dma_engn(chan, engine)); -} - -static int -nv40_fifo_dma_engine_ctor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, - struct nvkm_object *object) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - u32 reg, ctx; - - if (!nv40_fifo_dma_engine(engine, ®, &ctx)) - return 0; - - return nvkm_object_bind(object, NULL, 0, nv40_fifo_dma_engn(chan, engine)); -} - static int nv40_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, struct nvkm_object *object) @@ -176,10 +61,6 @@ nv40_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, static const struct nvkm_fifo_chan_func nv40_fifo_dma_func = { .dtor = nv04_fifo_dma_dtor, - .engine_ctor = nv40_fifo_dma_engine_ctor, - .engine_dtor = nv40_fifo_dma_engine_dtor, - .engine_init = nv40_fifo_dma_engine_init, - .engine_fini = nv40_fifo_dma_engine_fini, .object_ctor = nv40_fifo_dma_object_ctor, .object_dtor = nv04_fifo_dma_object_dtor, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 6f3f5caaa7485..2bc835f2ef744 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -26,6 +26,7 @@ #include "runl.h" #include +#include #include "nv50.h" #include "channv50.h" @@ -67,7 +68,6 @@ g84_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, b if (ret) return ret; - nv50_fifo_chan(chan)->eng = chan->eng; nv50_fifo_chan(chan)->ramht = chan->ramht; nvkm_kmap(chan->ramfc); @@ -106,8 +106,58 @@ g84_chan = { .stop = nv50_chan_stop, }; +static void +g84_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + struct nvkm_subdev *subdev = &chan->cgrp->runl->fifo->engine.subdev; + struct nvkm_device *device = subdev->device; + u64 start = 0, limit = 0; + u32 flags = 0, ptr0, save; + + switch (engn->engine->subdev.type) { + case NVKM_ENGINE_GR : ptr0 = 0x0020; break; + case NVKM_ENGINE_VP : + case NVKM_ENGINE_MSPDEC: ptr0 = 0x0040; break; + case NVKM_ENGINE_MPEG : + case NVKM_ENGINE_MSPPP : ptr0 = 0x0060; break; + case NVKM_ENGINE_BSP : + case NVKM_ENGINE_MSVLD : ptr0 = 0x0080; break; + case NVKM_ENGINE_CIPHER: + case NVKM_ENGINE_SEC : ptr0 = 0x00a0; break; + case NVKM_ENGINE_CE : ptr0 = 0x00c0; break; + default: + WARN_ON(1); + return; + } + + if (!cctx) { + save = nvkm_mask(device, 0x002520, 0x0000003f, BIT(engn->id - 1)); + nvkm_wr32(device, 0x0032fc, chan->inst->addr >> 12); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x0032fc) != 0xffffffff) + break; + ); + nvkm_wr32(device, 0x002520, save); + } else { + flags = 0x00190000; + start = cctx->vctx->inst->addr; + limit = start + cctx->vctx->inst->size - 1; + } + + nvkm_kmap(chan->eng); + nvkm_wo32(chan->eng, ptr0 + 0x00, flags); + nvkm_wo32(chan->eng, ptr0 + 0x04, lower_32_bits(limit)); + nvkm_wo32(chan->eng, ptr0 + 0x08, lower_32_bits(start)); + nvkm_wo32(chan->eng, ptr0 + 0x0c, upper_32_bits(limit) << 24 | + lower_32_bits(start)); + nvkm_wo32(chan->eng, ptr0 + 0x10, 0x00000000); + nvkm_wo32(chan->eng, ptr0 + 0x14, 0x00000000); + nvkm_done(chan->eng); +} + const struct nvkm_engn_func g84_engn = { + .bind = g84_ectx_bind, }; static void @@ -138,28 +188,6 @@ g84_fifo_nonstall = { .fini = g84_fifo_nonstall_block, }; -int -g84_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_SW : return G84_FIFO_ENGN_SW; - case NVKM_ENGINE_GR : return G84_FIFO_ENGN_GR; - case NVKM_ENGINE_MPEG : - case NVKM_ENGINE_MSPPP : return G84_FIFO_ENGN_MPEG; - case NVKM_ENGINE_CE : return G84_FIFO_ENGN_CE0; - case NVKM_ENGINE_VP : - case NVKM_ENGINE_MSPDEC: return G84_FIFO_ENGN_VP; - case NVKM_ENGINE_CIPHER: - case NVKM_ENGINE_SEC : return G84_FIFO_ENGN_CIPHER; - case NVKM_ENGINE_BSP : - case NVKM_ENGINE_MSVLD : return G84_FIFO_ENGN_BSP; - case NVKM_ENGINE_DMAOBJ: return G84_FIFO_ENGN_DMA; - default: - WARN_ON(1); - return -1; - } -} - static int g84_fifo_runl_ctor(struct nvkm_fifo *fifo) { @@ -188,7 +216,6 @@ g84_fifo = { .runl_ctor = g84_fifo_runl_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, - .engine_id = g84_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .nonstall = &g84_fifo_nonstall, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c index 8016a2228e92d..b2634f21d3207 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c @@ -55,7 +55,6 @@ g98_fifo = { .runl_ctor = g98_fifo_runl_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, - .engine_id = g84_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .nonstall = &g84_fifo_nonstall, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 351b35d580fd2..2e2626168bf29 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -158,6 +158,47 @@ gf100_chan = { .preempt = gf100_chan_preempt, }; +static void +gf100_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + u64 addr = 0ULL; + u32 ptr0; + + switch (engn->engine->subdev.type) { + case NVKM_ENGINE_SW : return; + case NVKM_ENGINE_GR : ptr0 = 0x0210; break; + case NVKM_ENGINE_CE : ptr0 = 0x0230 + (engn->engine->subdev.inst * 0x10); break; + case NVKM_ENGINE_MSPDEC: ptr0 = 0x0250; break; + case NVKM_ENGINE_MSPPP : ptr0 = 0x0260; break; + case NVKM_ENGINE_MSVLD : ptr0 = 0x0270; break; + default: + WARN_ON(1); + return; + } + + if (cctx) { + addr = cctx->vctx->vma->addr; + addr |= 4ULL; + } + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, ptr0 + 0, lower_32_bits(addr)); + nvkm_wo32(chan->inst, ptr0 + 4, upper_32_bits(addr)); + nvkm_done(chan->inst); +} + +static int +gf100_ectx_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx) +{ + int ret; + + ret = nvkm_vmm_get(vctx->vmm, 12, vctx->inst->size, &vctx->vma); + if (ret) + return ret; + + return nvkm_memory_map(vctx->inst, 0, vctx->vmm, vctx->vma, NULL, 0); +} + bool gf100_engn_mmu_fault_triggered(struct nvkm_engn *engn) { @@ -250,6 +291,8 @@ gf100_engn = { .cxid = gf100_engn_cxid, .mmu_fault_trigger = gf100_engn_mmu_fault_trigger, .mmu_fault_triggered = gf100_engn_mmu_fault_triggered, + .ctor = gf100_ectx_ctor, + .bind = gf100_ectx_bind, }; const struct nvkm_engn_func @@ -422,22 +465,6 @@ gf100_fifo_nonstall = { .fini = gf100_fifo_nonstall_block, }; -static int -gf100_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_GR : return GF100_FIFO_ENGN_GR; - case NVKM_ENGINE_MSPDEC: return GF100_FIFO_ENGN_MSPDEC; - case NVKM_ENGINE_MSPPP : return GF100_FIFO_ENGN_MSPPP; - case NVKM_ENGINE_MSVLD : return GF100_FIFO_ENGN_MSVLD; - case NVKM_ENGINE_CE : return GF100_FIFO_ENGN_CE0 + engine->subdev.inst; - case NVKM_ENGINE_SW : return GF100_FIFO_ENGN_SW; - default: - WARN_ON(1); - return -1; - } -} - static const struct nvkm_enum gf100_fifo_mmu_fault_engine[] = { { 0x00, "PGRAPH", NULL, NVKM_ENGINE_GR }, @@ -935,7 +962,6 @@ gf100_fifo = { .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gf100_fifo_mmu_fault, - .engine_id = gf100_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gf100_runl, .runq = &gf100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index b75e3d089da31..6130f492f48a3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -35,6 +35,7 @@ #include #include +#include void gk104_chan_stop(struct nvkm_chan *chan) @@ -130,6 +131,63 @@ gk104_chan = { .preempt = gf100_chan_preempt, }; +static void +gk104_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + u32 ptr0, ptr1 = 0; + u64 addr = 0ULL; + + switch (engn->engine->subdev.type) { + case NVKM_ENGINE_SW : return; + case NVKM_ENGINE_GR : ptr0 = 0x0210; break; + case NVKM_ENGINE_SEC : ptr0 = 0x0220; break; + case NVKM_ENGINE_MSPDEC: ptr0 = 0x0250; break; + case NVKM_ENGINE_MSPPP : ptr0 = 0x0260; break; + case NVKM_ENGINE_MSVLD : ptr0 = 0x0270; break; + case NVKM_ENGINE_VIC : ptr0 = 0x0280; break; + case NVKM_ENGINE_MSENC : ptr0 = 0x0290; break; + case NVKM_ENGINE_NVDEC : + ptr1 = 0x0270; + ptr0 = 0x0210; + break; + case NVKM_ENGINE_NVENC : + if (!engn->engine->subdev.inst) + ptr1 = 0x0290; + ptr0 = 0x0210; + break; + default: + WARN_ON(1); + return; + } + + if (cctx) { + addr = cctx->vctx->vma->addr; + addr |= 4ULL; + } + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, ptr0 + 0, lower_32_bits(addr)); + nvkm_wo32(chan->inst, ptr0 + 4, upper_32_bits(addr)); + if (ptr1) { + nvkm_wo32(chan->inst, ptr1 + 0, lower_32_bits(addr)); + nvkm_wo32(chan->inst, ptr1 + 4, upper_32_bits(addr)); + } + nvkm_done(chan->inst); +} + +int +gk104_ectx_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx) +{ + struct gf100_vmm_map_v0 args = { .priv = 1 }; + int ret; + + ret = nvkm_vmm_get(vctx->vmm, 12, vctx->inst->size, &vctx->vma); + if (ret) + return ret; + + return nvkm_memory_map(vctx->inst, 0, vctx->vmm, vctx->vma, &args, sizeof(args)); +} + /*TODO: clean this up */ struct gk104_engn_status { bool busy; @@ -216,6 +274,8 @@ gk104_engn = { .cxid = gk104_engn_cxid, .mmu_fault_trigger = gf100_engn_mmu_fault_trigger, .mmu_fault_triggered = gf100_engn_mmu_fault_triggered, + .ctor = gk104_ectx_ctor, + .bind = gk104_ectx_bind, }; const struct nvkm_engn_func @@ -410,24 +470,6 @@ gk104_runl = { .preempt_pending = gf100_runl_preempt_pending, }; -int -gk104_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - int engn; - - if (engine->subdev.type == NVKM_ENGINE_SW) - return GK104_FIFO_ENGN_SW; - - for (engn = 0; engn < fifo->engine_nr && engine; engn++) { - if (fifo->engine[engn].engine == engine) - return engn; - } - - WARN_ON(1); - return -1; -} - static const struct nvkm_enum gk104_fifo_mmu_fault_engine[] = { { 0x00, "GR", NULL, NVKM_ENGINE_GR }, @@ -778,8 +820,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base) if (engn < 0) continue; - fifo->engine[engn].engine = nvkm_device_engine(device, tdev->type, tdev->inst); - fifo->engine_nr = max(fifo->engine_nr, engn + 1); fifo->runlist[tdev->runlist].engm |= BIT(engn); fifo->runlist[tdev->runlist].engm_sw |= BIT(engn); if (tdev->type == NVKM_ENGINE_GR) @@ -825,7 +865,6 @@ gk104_fifo = { .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gk104_runl, .runq = &gk104_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h index 3bcf0720f89da..97fc9ccc2930d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h @@ -14,11 +14,6 @@ struct gk104_fifo { const struct gk104_fifo_func *func; struct nvkm_fifo base; - struct { - struct nvkm_engine *engine; - } engine[16]; - int engine_nr; - struct { u32 engm; u32 engm_sw; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 2a617215d9247..2afbaf42572d3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -119,7 +119,6 @@ gk110_fifo = { .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk104_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 81d184473f799..3727be30d7f4f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -63,7 +63,6 @@ gk208_fifo = { .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index 47b7d93f5a042..eda1446dce27c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -38,7 +38,6 @@ gk20a_fifo = { .intr_mmu_fault_unit = gf100_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gk104_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gk110_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 3764245fbca10..1089062313cc6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -143,7 +143,6 @@ gm107_fifo = { .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gm107_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 8be59812797a0..286c1098b6659 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -52,7 +52,6 @@ gm200_fifo = { .intr_mmu_fault_unit = gm107_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gm107_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gm107_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index a353c6b860ef8..45d536d448f0c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -125,7 +125,6 @@ gp100_fifo = { .intr_mmu_fault_unit = gp100_fifo_intr_mmu_fault_unit, .intr_ctxsw_timeout = gf100_fifo_intr_ctxsw_timeout, .mmu_fault = &gp100_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gp100_runl, .runq = &gk208_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index c43cd0c24a6de..6ba4335d40bcb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -31,102 +31,6 @@ #include #include -static u32 -gf100_fifo_gpfifo_engine_addr(struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_SW : return 0; - case NVKM_ENGINE_GR : return 0x0210; - case NVKM_ENGINE_CE : return 0x0230 + (engine->subdev.inst * 0x10); - case NVKM_ENGINE_MSPDEC: return 0x0250; - case NVKM_ENGINE_MSPPP : return 0x0260; - case NVKM_ENGINE_MSVLD : return 0x0270; - default: - WARN_ON(1); - return 0; - } -} - -static struct gf100_fifo_engn * -gf100_fifo_gpfifo_engine(struct gf100_fifo_chan *chan, struct nvkm_engine *engine) -{ - int engi = chan->base.fifo->func->engine_id(chan->base.fifo, engine); - if (engi >= 0) - return &chan->engn[engi]; - return NULL; -} - -static int -gf100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, bool suspend) -{ - const u32 offset = gf100_fifo_gpfifo_engine_addr(engine); - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct nvkm_gpuobj *inst = chan->base.inst; - int ret = 0; - - if (offset) { - nvkm_kmap(inst); - nvkm_wo32(inst, offset + 0x00, 0x00000000); - nvkm_wo32(inst, offset + 0x04, 0x00000000); - nvkm_done(inst); - } - - return ret; -} - -static int -gf100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - const u32 offset = gf100_fifo_gpfifo_engine_addr(engine); - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct gf100_fifo_engn *engn = gf100_fifo_gpfifo_engine(chan, engine); - struct nvkm_gpuobj *inst = chan->base.inst; - - if (offset) { - nvkm_kmap(inst); - nvkm_wo32(inst, offset + 0x00, lower_32_bits(engn->vma->addr) | 4); - nvkm_wo32(inst, offset + 0x04, upper_32_bits(engn->vma->addr)); - nvkm_done(inst); - } - - return 0; -} - -static void -gf100_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct gf100_fifo_engn *engn = gf100_fifo_gpfifo_engine(chan, engine); - nvkm_vmm_put(chan->base.vmm, &engn->vma); - nvkm_gpuobj_del(&engn->inst); -} - -static int -gf100_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, - struct nvkm_object *object) -{ - struct gf100_fifo_chan *chan = gf100_fifo_chan(base); - struct gf100_fifo_engn *engn = gf100_fifo_gpfifo_engine(chan, engine); - int ret; - - if (!gf100_fifo_gpfifo_engine_addr(engine)) - return 0; - - ret = nvkm_object_bind(object, NULL, 0, &engn->inst); - if (ret) - return ret; - - ret = nvkm_vmm_get(chan->base.vmm, 12, engn->inst->size, &engn->vma); - if (ret) - return ret; - - return nvkm_memory_map(engn->inst, 0, chan->base.vmm, engn->vma, NULL, 0); -} - static void * gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { @@ -136,10 +40,6 @@ gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) static const struct nvkm_fifo_chan_func gf100_fifo_gpfifo_func = { .dtor = gf100_fifo_gpfifo_dtor, - .engine_ctor = gf100_fifo_gpfifo_engine_ctor, - .engine_dtor = gf100_fifo_gpfifo_engine_dtor, - .engine_init = gf100_fifo_gpfifo_engine_init, - .engine_fini = gf100_fifo_gpfifo_engine_fini, }; static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 77de170c8144d..28a7a5eae72cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -33,125 +33,6 @@ #include #include -static u32 -gk104_fifo_gpfifo_engine_addr(struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_SW : - case NVKM_ENGINE_CE : return 0; - case NVKM_ENGINE_GR : return 0x0210; - case NVKM_ENGINE_SEC : return 0x0220; - case NVKM_ENGINE_MSPDEC: return 0x0250; - case NVKM_ENGINE_MSPPP : return 0x0260; - case NVKM_ENGINE_MSVLD : return 0x0270; - case NVKM_ENGINE_VIC : return 0x0280; - case NVKM_ENGINE_MSENC : return 0x0290; - case NVKM_ENGINE_NVDEC : return 0x02100270; - case NVKM_ENGINE_NVENC : - if (engine->subdev.inst) - return 0x0210; - return 0x02100290; - default: - WARN_ON(1); - return 0; - } -} - -struct gk104_fifo_engn * -gk104_fifo_gpfifo_engine(struct gk104_fifo_chan *chan, struct nvkm_engine *engine) -{ - int engi = chan->base.fifo->func->engine_id(chan->base.fifo, engine); - if (engi >= 0) - return &chan->engn[engi]; - return NULL; -} - -static int -gk104_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, bool suspend) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct nvkm_gpuobj *inst = chan->base.inst; - u32 offset = gk104_fifo_gpfifo_engine_addr(engine); - - if (offset) { - nvkm_kmap(inst); - nvkm_wo32(inst, (offset & 0xffff) + 0x00, 0x00000000); - nvkm_wo32(inst, (offset & 0xffff) + 0x04, 0x00000000); - if ((offset >>= 16)) { - nvkm_wo32(inst, offset + 0x00, 0x00000000); - nvkm_wo32(inst, offset + 0x04, 0x00000000); - } - nvkm_done(inst); - } - - return 0; -} - -static int -gk104_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct gk104_fifo_engn *engn = gk104_fifo_gpfifo_engine(chan, engine); - struct nvkm_gpuobj *inst = chan->base.inst; - u32 offset = gk104_fifo_gpfifo_engine_addr(engine); - - if (offset) { - u32 datalo = lower_32_bits(engn->vma->addr) | 0x00000004; - u32 datahi = upper_32_bits(engn->vma->addr); - nvkm_kmap(inst); - nvkm_wo32(inst, (offset & 0xffff) + 0x00, datalo); - nvkm_wo32(inst, (offset & 0xffff) + 0x04, datahi); - if ((offset >>= 16)) { - nvkm_wo32(inst, offset + 0x00, datalo); - nvkm_wo32(inst, offset + 0x04, datahi); - } - nvkm_done(inst); - } - - return 0; -} - -void -gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct gk104_fifo_engn *engn = gk104_fifo_gpfifo_engine(chan, engine); - nvkm_vmm_put(chan->base.vmm, &engn->vma); - nvkm_gpuobj_del(&engn->inst); -} - -int -gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, - struct nvkm_object *object) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct gk104_fifo_engn *engn = gk104_fifo_gpfifo_engine(chan, engine); - int ret; - - if (!gk104_fifo_gpfifo_engine_addr(engine)) { - if (engine->subdev.type != NVKM_ENGINE_CE || - engine->subdev.device->card_type < GV100) - return 0; - } - - ret = nvkm_object_bind(object, NULL, 0, &engn->inst); - if (ret) - return ret; - - if (!gk104_fifo_gpfifo_engine_addr(engine)) - return 0; - - ret = nvkm_vmm_get(chan->base.vmm, 12, engn->inst->size, &engn->vma); - if (ret) - return ret; - - return nvkm_memory_map(engn->inst, 0, chan->base.vmm, engn->vma, NULL, 0); -} - void * gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) { @@ -162,10 +43,6 @@ gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func = { .dtor = gk104_fifo_gpfifo_dtor, - .engine_ctor = gk104_fifo_gpfifo_engine_ctor, - .engine_dtor = gk104_fifo_gpfifo_engine_dtor, - .engine_init = gk104_fifo_gpfifo_engine_init, - .engine_fini = gk104_fifo_gpfifo_engine_fini, }; static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 9a2308be73a2e..3c2c76bcbe790 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -28,87 +28,9 @@ #include #include -static int -gv100_fifo_gpfifo_engine_valid(struct gk104_fifo_chan *chan, bool ce, bool valid) -{ - const u32 mask = ce ? 0x00020000 : 0x00010000; - const u32 data = valid ? mask : 0x00000000; - - if (1) { - /* Update engine context validity. */ - nvkm_kmap(chan->base.inst); - nvkm_mo32(chan->base.inst, 0x0ac, mask, data); - nvkm_done(chan->base.inst); - } - - return 0; -} - -int -gv100_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine, bool suspend) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct nvkm_gpuobj *inst = chan->base.inst; - int ret; - - if (engine->subdev.type == NVKM_ENGINE_CE) { - ret = gv100_fifo_gpfifo_engine_valid(chan, true, false); - if (ret && suspend) - return ret; - - nvkm_kmap(inst); - nvkm_wo32(chan->base.inst, 0x220, 0x00000000); - nvkm_wo32(chan->base.inst, 0x224, 0x00000000); - nvkm_done(inst); - return ret; - } - - ret = gv100_fifo_gpfifo_engine_valid(chan, false, false); - if (ret && suspend) - return ret; - - nvkm_kmap(inst); - nvkm_wo32(inst, 0x0210, 0x00000000); - nvkm_wo32(inst, 0x0214, 0x00000000); - nvkm_done(inst); - return ret; -} - -int -gv100_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, - struct nvkm_engine *engine) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - struct gk104_fifo_engn *engn = gk104_fifo_gpfifo_engine(chan, engine); - struct nvkm_gpuobj *inst = chan->base.inst; - - if (engine->subdev.type == NVKM_ENGINE_CE) { - const u64 bar2 = nvkm_memory_bar2(engn->inst->memory); - - nvkm_kmap(inst); - nvkm_wo32(chan->base.inst, 0x220, lower_32_bits(bar2)); - nvkm_wo32(chan->base.inst, 0x224, upper_32_bits(bar2)); - nvkm_done(inst); - - return gv100_fifo_gpfifo_engine_valid(chan, true, true); - } - - nvkm_kmap(inst); - nvkm_wo32(inst, 0x210, lower_32_bits(engn->vma->addr) | 0x00000004); - nvkm_wo32(inst, 0x214, upper_32_bits(engn->vma->addr)); - nvkm_done(inst); - - return gv100_fifo_gpfifo_engine_valid(chan, false, true); -} - static const struct nvkm_fifo_chan_func gv100_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, - .engine_ctor = gk104_fifo_gpfifo_engine_ctor, - .engine_dtor = gk104_fifo_gpfifo_engine_dtor, - .engine_init = gv100_fifo_gpfifo_engine_init, - .engine_fini = gv100_fifo_gpfifo_engine_fini, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c index 6562bdf39cb0d..62a0f9664a9d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c @@ -31,10 +31,6 @@ static const struct nvkm_fifo_chan_func tu102_fifo_gpfifo = { .dtor = gk104_fifo_gpfifo_dtor, - .engine_ctor = gk104_fifo_gpfifo_engine_ctor, - .engine_dtor = gk104_fifo_gpfifo_engine_dtor, - .engine_init = gv100_fifo_gpfifo_engine_init, - .engine_fini = gv100_fifo_gpfifo_engine_fini, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index ffca153a9a282..d55dfc93c61e4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -89,16 +89,58 @@ gv100_chan = { .doorbell_handle = gv100_chan_doorbell_handle, }; +void +gv100_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + u64 addr = 0ULL; + + if (cctx) { + addr = cctx->vctx->vma->addr; + addr |= 4ULL; + } + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x210, lower_32_bits(addr)); + nvkm_wo32(chan->inst, 0x214, upper_32_bits(addr)); + nvkm_mo32(chan->inst, 0x0ac, 0x00010000, cctx ? 0x00010000 : 0x00000000); + nvkm_done(chan->inst); +} + const struct nvkm_engn_func gv100_engn = { .chsw = gk104_engn_chsw, .cxid = gk104_engn_cxid, + .ctor = gk104_ectx_ctor, + .bind = gv100_ectx_bind, }; +void +gv100_ectx_ce_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + const u64 bar2 = cctx ? nvkm_memory_bar2(cctx->vctx->inst->memory) : 0ULL; + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x220, lower_32_bits(bar2)); + nvkm_wo32(chan->inst, 0x224, upper_32_bits(bar2)); + nvkm_mo32(chan->inst, 0x0ac, 0x00020000, cctx ? 0x00020000 : 0x00000000); + nvkm_done(chan->inst); +} + +int +gv100_ectx_ce_ctor(struct nvkm_engn *engn, struct nvkm_vctx *vctx) +{ + if (nvkm_memory_bar2(vctx->inst->memory) == ~0ULL) + return -EFAULT; + + return 0; +} + const struct nvkm_engn_func gv100_engn_ce = { .chsw = gk104_engn_chsw, .cxid = gk104_engn_cxid, + .ctor = gv100_ectx_ce_ctor, + .bind = gv100_ectx_ce_bind, }; static bool @@ -436,7 +478,6 @@ gv100_fifo = { .intr = gk104_fifo_intr, .intr_ctxsw_timeout = gv100_fifo_intr_ctxsw_timeout, .mmu_fault = &gv100_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &gv100_runl, .runq = &gv100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index cf8369b8b8819..08de6ee612ce0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -234,20 +234,6 @@ const struct nvkm_runl_func nv04_runl = { }; -int -nv04_fifo_engine_id(struct nvkm_fifo *base, struct nvkm_engine *engine) -{ - switch (engine->subdev.type) { - case NVKM_ENGINE_SW : return NV04_FIFO_ENGN_SW; - case NVKM_ENGINE_GR : return NV04_FIFO_ENGN_GR; - case NVKM_ENGINE_MPEG : return NV04_FIFO_ENGN_MPEG; - case NVKM_ENGINE_DMAOBJ: return NV04_FIFO_ENGN_DMA; - default: - WARN_ON(1); - return 0; - } -} - static const char * nv_dma_state_err(u32 state) { @@ -533,7 +519,6 @@ nv04_fifo = { .runl_ctor = nv04_fifo_runl_ctor, .init = nv04_fifo_init, .intr = nv04_fifo_intr, - .engine_id = nv04_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index c7dbcd236630b..9658ac3dac91a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -97,7 +97,6 @@ nv10_fifo = { .runl_ctor = nv04_fifo_runl_ctor, .init = nv04_fifo_init, .intr = nv04_fifo_intr, - .engine_id = nv04_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index a829b0b08f9b8..fb0b106eb09bb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -126,7 +126,6 @@ nv17_fifo = { .runl_ctor = nv04_fifo_runl_ctor, .init = nv17_fifo_init, .intr = nv04_fifo_intr, - .engine_id = nv04_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index faf19ec3b323c..f27ca3e7bb27b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -43,7 +43,6 @@ nv40_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, const u32 base = chan->id * 128; chan->ramfc_offset = base; - nv04_fifo_chan(chan)->ramfc = base; nvkm_kmap(ramfc); nvkm_wo32(ramfc, base + 0x00, offset); @@ -109,8 +108,52 @@ nv40_chan = { .stop = nv04_chan_stop, }; +static void +nv40_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_memory *ramfc = device->imem->ramfc; + u32 inst = 0x00000000, reg, ctx; + int chid; + + switch (engn->engine->subdev.type) { + case NVKM_ENGINE_GR: + reg = 0x0032e0; + ctx = 0x38; + break; + case NVKM_ENGINE_MPEG: + if (WARN_ON(device->chipset < 0x44)) + return; + reg = 0x00330c; + ctx = 0x54; + break; + default: + WARN_ON(1); + return; + } + + if (cctx) + inst = cctx->vctx->inst->addr >> 4; + + spin_lock_irq(&fifo->lock); + nvkm_mask(device, 0x002500, 0x00000001, 0x00000000); + + chid = nvkm_rd32(device, 0x003204) & (fifo->chid->nr - 1); + if (chid == chan->id) + nvkm_wr32(device, reg, inst); + + nvkm_kmap(ramfc); + nvkm_wo32(ramfc, chan->ramfc_offset + ctx, inst); + nvkm_done(ramfc); + + nvkm_mask(device, 0x002500, 0x00000001, 0x00000001); + spin_unlock_irq(&fifo->lock); +} + static const struct nvkm_engn_func nv40_engn = { + .bind = nv40_ectx_bind, }; static const struct nvkm_engn_func @@ -175,7 +218,6 @@ nv40_fifo = { .runl_ctor = nv04_fifo_runl_ctor, .init = nv40_fifo_init, .intr = nv04_fifo_intr, - .engine_id = nv04_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv04_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 7d5c0d0269678..f068a27d93383 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -89,7 +89,6 @@ nv50_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, if (ret) return ret; - nv50_fifo_chan(chan)->eng = chan->eng; nv50_fifo_chan(chan)->ramht = chan->ramht; nvkm_kmap(chan->ramfc); @@ -139,8 +138,64 @@ nv50_chan = { .stop = nv50_chan_stop, }; +static void +nv50_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) +{ + struct nvkm_subdev *subdev = &chan->cgrp->runl->fifo->engine.subdev; + struct nvkm_device *device = subdev->device; + u64 start = 0, limit = 0; + u32 flags = 0, ptr0, save; + + switch (engn->engine->subdev.type) { + case NVKM_ENGINE_GR : ptr0 = 0x0000; break; + case NVKM_ENGINE_MPEG : ptr0 = 0x0060; break; + default: + WARN_ON(1); + return; + } + + if (!cctx) { + /* HW bug workaround: + * + * PFIFO will hang forever if the connected engines don't report + * that they've processed the context switch request. + * + * In order for the kickoff to work, we need to ensure all the + * connected engines are in a state where they can answer. + * + * Newer chipsets don't seem to suffer from this issue, and well, + * there's also a "ignore these engines" bitmask reg we can use + * if we hit the issue there.. + */ + save = nvkm_mask(device, 0x00b860, 0x00000001, 0x00000001); + + /* Tell engines to save out contexts. */ + nvkm_wr32(device, 0x0032fc, chan->inst->addr >> 12); + nvkm_msec(device, 2000, + if (nvkm_rd32(device, 0x0032fc) != 0xffffffff) + break; + ); + nvkm_wr32(device, 0x00b860, save); + } else { + flags = 0x00190000; + start = cctx->vctx->inst->addr; + limit = start + cctx->vctx->inst->size - 1; + } + + nvkm_kmap(chan->eng); + nvkm_wo32(chan->eng, ptr0 + 0x00, flags); + nvkm_wo32(chan->eng, ptr0 + 0x04, lower_32_bits(limit)); + nvkm_wo32(chan->eng, ptr0 + 0x08, lower_32_bits(start)); + nvkm_wo32(chan->eng, ptr0 + 0x0c, upper_32_bits(limit) << 24 | + lower_32_bits(start)); + nvkm_wo32(chan->eng, ptr0 + 0x10, 0x00000000); + nvkm_wo32(chan->eng, ptr0 + 0x14, 0x00000000); + nvkm_done(chan->eng); +} + static const struct nvkm_engn_func nv50_engn = { + .bind = nv50_ectx_bind, }; const struct nvkm_engn_func @@ -340,7 +395,6 @@ nv50_fifo = { .runl_ctor = nv04_fifo_runl_ctor, .init = nv50_fifo_init, .intr = nv04_fifo_intr, - .engine_id = nv04_fifo_engine_id, .pause = nv04_fifo_pause, .start = nv04_fifo_start, .runl = &nv50_runl, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h index 1573ea430cf34..3c6d67906730c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h @@ -12,6 +12,4 @@ int nv50_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvk struct nvkm_fifo **); void *nv50_fifo_dtor(struct nvkm_fifo *); - -int g84_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 43f248a109f65..13b2185189841 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -4,11 +4,13 @@ #define nvkm_fifo(p) container_of((p), struct nvkm_fifo, engine) #include #include +struct nvkm_cctx; struct nvkm_cgrp; struct nvkm_engn; struct nvkm_memory; struct nvkm_runl; struct nvkm_runq; +struct nvkm_vctx; struct gk104_fifo; struct nvkm_fifo_chan_oclass; @@ -37,7 +39,6 @@ struct nvkm_fifo_func { const struct nvkm_enum *gpcclient; } *mmu_fault; - int (*engine_id)(struct nvkm_fifo *, struct nvkm_engine *); void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); @@ -74,7 +75,6 @@ int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); int nv04_fifo_runl_ctor(struct nvkm_fifo *); void nv04_fifo_init(struct nvkm_fifo *); irqreturn_t nv04_fifo_intr(struct nvkm_inth *); -int nv04_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); void nv04_fifo_pause(struct nvkm_fifo *, unsigned long *); void nv04_fifo_start(struct nvkm_fifo *, unsigned long *); extern const struct nvkm_runl_func nv04_runl; @@ -138,7 +138,6 @@ extern const struct nvkm_fifo_func_mmu_fault gk104_fifo_mmu_fault; extern const struct nvkm_enum gk104_fifo_mmu_fault_reason[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_hubclient[]; extern const struct nvkm_enum gk104_fifo_mmu_fault_gpcclient[]; -int gk104_fifo_engine_id(struct nvkm_fifo *, struct nvkm_engine *); void gk104_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); void gk104_runl_commit(struct nvkm_runl *, struct nvkm_memory *, u32, int); bool gk104_runl_pending(struct nvkm_runl *); @@ -153,6 +152,7 @@ bool gk104_runq_idle(struct nvkm_runq *); extern const struct nvkm_engn_func gk104_engn; bool gk104_engn_chsw(struct nvkm_engn *); int gk104_engn_cxid(struct nvkm_engn *, bool *cgid); +int gk104_ectx_ctor(struct nvkm_engn *, struct nvkm_vctx *); extern const struct nvkm_engn_func gk104_engn_ce; extern const struct nvkm_chan_func_userd gk104_chan_userd; extern const struct nvkm_chan_func_ramfc gk104_chan_ramfc; @@ -189,7 +189,10 @@ void gv100_runl_insert_chan(struct nvkm_chan *, struct nvkm_memory *, u64); void gv100_runl_preempt(struct nvkm_runl *); extern const struct nvkm_runq_func gv100_runq; extern const struct nvkm_engn_func gv100_engn; +void gv100_ectx_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); extern const struct nvkm_engn_func gv100_engn_ce; +int gv100_ectx_ce_ctor(struct nvkm_engn *, struct nvkm_vctx *); +void gv100_ectx_ce_bind(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); extern const struct nvkm_chan_func_userd gv100_chan_userd; extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index c3b469356d5d9..86bca72c68bc8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -1,9 +1,11 @@ #ifndef __NVKM_RUNL_H__ #define __NVKM_RUNL_H__ #include +struct nvkm_cctx; struct nvkm_cgrp; struct nvkm_chan; struct nvkm_memory; +struct nvkm_vctx; enum nvkm_subdev_type; struct nvkm_engn { @@ -12,6 +14,8 @@ struct nvkm_engn { int (*cxid)(struct nvkm_engn *, bool *cgid); void (*mmu_fault_trigger)(struct nvkm_engn *); bool (*mmu_fault_triggered)(struct nvkm_engn *); + int (*ctor)(struct nvkm_engn *, struct nvkm_vctx *); + void (*bind)(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); } *func; struct nvkm_runl *runl; int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index a8452eee2949d..608db9b8dc118 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -273,7 +273,6 @@ tu102_fifo = { .init_pbdmas = tu102_fifo_init_pbdmas, .intr = tu102_fifo_intr, .mmu_fault = &tu102_fifo_mmu_fault, - .engine_id = gk104_fifo_engine_id, .nonstall = &gf100_fifo_nonstall, .runl = &tu102_runl, .runq = &gv100_runq, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 74584a29454a9..f35bdd245233f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -25,6 +25,7 @@ #include "chid.h" #include "runl.h" +#include #include #include @@ -74,10 +75,17 @@ nvkm_uchan_object_fini_1(struct nvkm_oproxy *oproxy, bool suspend) struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); struct nvkm_chan *chan = uobj->chan; struct nvkm_cctx *cctx = uobj->cctx; + struct nvkm_ectx *ectx = cctx->vctx->ectx; + + if (!ectx->object) + return 0; /* Unbind engine context from channel, if no longer required. */ if (refcount_dec_and_mutex_lock(&cctx->uses, &chan->cgrp->mutex)) { - nvkm_chan_cctx_bind(chan, oproxy, NULL); + nvkm_chan_cctx_bind(chan, ectx->engn, NULL); + + if (refcount_dec_and_test(&ectx->uses)) + nvkm_object_fini(ectx->object, false); mutex_unlock(&chan->cgrp->mutex); } @@ -90,14 +98,24 @@ nvkm_uchan_object_init_0(struct nvkm_oproxy *oproxy) struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); struct nvkm_chan *chan = uobj->chan; struct nvkm_cctx *cctx = uobj->cctx; + struct nvkm_ectx *ectx = cctx->vctx->ectx; int ret = 0; + if (!ectx->object) + return 0; + /* Bind engine context to channel, if it hasn't been already. */ if (!refcount_inc_not_zero(&cctx->uses)) { mutex_lock(&chan->cgrp->mutex); if (!refcount_inc_not_zero(&cctx->uses)) { + if (!refcount_inc_not_zero(&ectx->uses)) { + ret = nvkm_object_init(ectx->object); + if (ret == 0) + refcount_set(&ectx->uses, 1); + } + if (ret == 0) { - nvkm_chan_cctx_bind(chan, oproxy, cctx); + nvkm_chan_cctx_bind(chan, ectx->engn, cctx); refcount_set(&cctx->uses, 1); } } @@ -112,6 +130,9 @@ nvkm_uchan_object_dtor(struct nvkm_oproxy *oproxy) { struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); + if (!uobj->cctx) + return; + nvkm_chan_cctx_put(uobj->chan, &uobj->cctx); } -- GitLab From 7ac293328122075a2afc40a4089e7afc6cbc26eb Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:37 +1000 Subject: [PATCH 1334/1927] drm/nouveau/fifo: add new engine object handling Simplifies the GPU-specific code, completing the switch to newer HALs. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 66 ------------------- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 4 -- .../drm/nouveau/nvkm/engine/fifo/chang84.c | 34 ---------- .../drm/nouveau/nvkm/engine/fifo/changf100.h | 1 - .../drm/nouveau/nvkm/engine/fifo/changk104.h | 1 - .../drm/nouveau/nvkm/engine/fifo/channv04.h | 2 - .../drm/nouveau/nvkm/engine/fifo/channv50.c | 31 --------- .../drm/nouveau/nvkm/engine/fifo/channv50.h | 4 -- .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 41 ------------ .../drm/nouveau/nvkm/engine/fifo/dmanv10.c | 1 - .../drm/nouveau/nvkm/engine/fifo/dmanv17.c | 1 - .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 31 --------- .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 4 +- .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 1 - .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 1 - .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 27 ++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 18 +++++ .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 18 ++++- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 29 ++++++-- 22 files changed, 94 insertions(+), 228 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index a35439dda054a..4f99e5bf4e7ff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -39,70 +39,6 @@ const struct nvkm_event_func nvkm_chan_event = { }; -struct nvkm_fifo_chan_object { - struct nvkm_oproxy oproxy; - struct nvkm_fifo_chan *chan; - int hash; -}; - -static void -nvkm_fifo_chan_child_del(struct nvkm_oproxy *base) -{ - struct nvkm_fifo_chan_object *object = - container_of(base, typeof(*object), oproxy); - struct nvkm_fifo_chan *chan = object->chan; - - if (chan->func->object_dtor) - chan->func->object_dtor(chan, object->hash); -} - -static const struct nvkm_oproxy_func -nvkm_fifo_chan_child_func = { - .dtor[0] = nvkm_fifo_chan_child_del, -}; - -int -nvkm_fifo_chan_child_new(const struct nvkm_oclass *oclass, void *data, u32 size, - struct nvkm_object **pobject) -{ - struct nvkm_engine *engine = oclass->engine; - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(oclass->parent); - struct nvkm_ectx *engn = nvkm_list_find(engn, &chan->cgrp->ectxs, head, - engn->engn->engine == engine); - struct nvkm_fifo_chan_object *object; - int ret = 0; - - if (!(object = kzalloc(sizeof(*object), GFP_KERNEL))) - return -ENOMEM; - nvkm_oproxy_ctor(&nvkm_fifo_chan_child_func, oclass, &object->oproxy); - object->chan = chan; - *pobject = &object->oproxy.base; - - - ret = oclass->base.ctor(&(const struct nvkm_oclass) { - .base = oclass->base, - .engn = oclass->engn, - .handle = oclass->handle, - .object = oclass->object, - .client = oclass->client, - .parent = engn->object ? - engn->object : - oclass->parent, - .engine = engine, - }, data, size, &object->oproxy.object); - if (ret) - return ret; - - if (chan->func->object_ctor) { - object->hash = - chan->func->object_ctor(chan, object->oproxy.object); - if (object->hash < 0) - return object->hash; - } - - return 0; -} - void nvkm_chan_cctx_bind(struct nvkm_chan *chan, struct nvkm_engn *engn, struct nvkm_cctx *cctx) { @@ -455,8 +391,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, *func = *fifo->func->chan.func; func->dtor = fn->dtor; - func->object_ctor = fn->object_ctor; - func->object_dtor = fn->object_dtor; chan->func = func; chan->id = -1; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index bb142593d8b65..836f9ecb13c64 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -52,8 +52,6 @@ struct nvkm_chan_func { u32 (*doorbell_handle)(struct nvkm_chan *); void *(*dtor)(struct nvkm_fifo_chan *); - int (*object_ctor)(struct nvkm_fifo_chan *, struct nvkm_object *); - void (*object_dtor)(struct nvkm_fifo_chan *, int); }; int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, @@ -78,6 +76,4 @@ void nvkm_chan_cctx_bind(struct nvkm_chan *, struct nvkm_engn *, struct nvkm_cct #define CHAN_PRINT(c,l,p,f,a...) CGRP_PRINT((c)->cgrp, l, p, "%04x:"f, (c)->id, ##a) #define CHAN_ERROR(c,f,a...) CHAN_PRCLI((c), ERROR, err, " "f"\n", ##a) #define CHAN_TRACE(c,f,a...) CHAN_PRINT((c), TRACE, info, " "f"\n", ##a) - -int nvkm_fifo_chan_child_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c index 359d8b5b124c4..fc266af2cf536 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c @@ -30,42 +30,9 @@ #include -static int -g84_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, - struct nvkm_object *object) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - u32 handle = object->handle; - u32 context; - - switch (object->engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW : context = 0x00000000; break; - case NVKM_ENGINE_GR : context = 0x00100000; break; - case NVKM_ENGINE_MPEG : - case NVKM_ENGINE_MSPPP : context = 0x00200000; break; - case NVKM_ENGINE_ME : - case NVKM_ENGINE_CE : context = 0x00300000; break; - case NVKM_ENGINE_VP : - case NVKM_ENGINE_MSPDEC: context = 0x00400000; break; - case NVKM_ENGINE_CIPHER: - case NVKM_ENGINE_SEC : - case NVKM_ENGINE_VIC : context = 0x00500000; break; - case NVKM_ENGINE_BSP : - case NVKM_ENGINE_MSVLD : context = 0x00600000; break; - default: - WARN_ON(1); - return -EINVAL; - } - - return nvkm_ramht_insert(chan->ramht, object, 0, 4, handle, context); -} - static const struct nvkm_fifo_chan_func g84_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .object_ctor = g84_fifo_chan_object_ctor, - .object_dtor = nv50_fifo_chan_object_dtor, }; int @@ -95,6 +62,5 @@ g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, BIT(G84_FIFO_ENGN_MSVLD) | BIT(G84_FIFO_ENGN_DMA), 0, 0xc00000, 0x2000, oclass, &chan->base); - chan->fifo = fifo; return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h index 739cd750742e3..e94c16b0efb6b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h @@ -7,7 +7,6 @@ struct gf100_fifo_chan { struct nvkm_fifo_chan base; - struct gf100_fifo *fifo; #define GF100_FIFO_ENGN_GR 0 #define GF100_FIFO_ENGN_MSPDEC 1 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h index 2043f06f500b2..3450f6349db3a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h @@ -7,7 +7,6 @@ struct gk104_fifo_chan { struct nvkm_fifo_chan base; - struct gk104_fifo *fifo; int runl; #define GK104_FIFO_ENGN_SW 15 diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h index 5ebfa2241750a..5a1717888978e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h @@ -7,7 +7,6 @@ struct nv04_fifo_chan { struct nvkm_fifo_chan base; - struct nv04_fifo *fifo; #define NV04_FIFO_ENGN_SW 0 #define NV04_FIFO_ENGN_GR 1 #define NV04_FIFO_ENGN_MPEG 2 @@ -16,7 +15,6 @@ struct nv04_fifo_chan { extern const struct nvkm_fifo_chan_func nv04_fifo_dma_func; void *nv04_fifo_dma_dtor(struct nvkm_fifo_chan *); -void nv04_fifo_dma_object_dtor(struct nvkm_fifo_chan *, int); extern const struct nvkm_fifo_chan_oclass nv04_fifo_dma_oclass; extern const struct nvkm_fifo_chan_oclass nv10_fifo_dma_oclass; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c index cb56060414995..0ad6354b9cccc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c @@ -28,34 +28,6 @@ #include #include -void -nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *base, int cookie) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - nvkm_ramht_remove(chan->ramht, cookie); -} - -static int -nv50_fifo_chan_object_ctor(struct nvkm_fifo_chan *base, - struct nvkm_object *object) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - u32 handle = object->handle; - u32 context; - - switch (object->engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW : context = 0x00000000; break; - case NVKM_ENGINE_GR : context = 0x00100000; break; - case NVKM_ENGINE_MPEG : context = 0x00200000; break; - default: - WARN_ON(1); - return -EINVAL; - } - - return nvkm_ramht_insert(chan->ramht, object, 0, 4, handle, context); -} - void * nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) { @@ -66,8 +38,6 @@ nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) static const struct nvkm_fifo_chan_func nv50_fifo_chan_func = { .dtor = nv50_fifo_chan_dtor, - .object_ctor = nv50_fifo_chan_object_ctor, - .object_dtor = nv50_fifo_chan_object_dtor, }; int @@ -87,6 +57,5 @@ nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, BIT(NV50_FIFO_ENGN_MPEG) | BIT(NV50_FIFO_ENGN_DMA), 0, 0xc00000, 0x2000, oclass, &chan->base); - chan->fifo = fifo; return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h index ced832c7f425c..ea4ec3bc4b8ab 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h @@ -6,11 +6,8 @@ #include "nv50.h" struct nv50_fifo_chan { - struct nv50_fifo *fifo; struct nvkm_fifo_chan base; - struct nvkm_ramht *ramht; - #define NV50_FIFO_ENGN_SW 0 #define NV50_FIFO_ENGN_GR 1 #define NV50_FIFO_ENGN_MPEG 2 @@ -35,7 +32,6 @@ struct nv50_fifo_chan { int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *); -void nv50_fifo_chan_object_dtor(struct nvkm_fifo_chan *, int); int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, const struct nvkm_oclass *, struct nv50_fifo_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c index 4220120d8bb8a..41f7f22ed935e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c @@ -31,44 +31,6 @@ #include #include -void -nv04_fifo_dma_object_dtor(struct nvkm_fifo_chan *base, int cookie) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nvkm_instmem *imem = chan->fifo->base.engine.subdev.device->imem; - - mutex_lock(&chan->fifo->base.mutex); - nvkm_ramht_remove(imem->ramht, cookie); - mutex_unlock(&chan->fifo->base.mutex); -} - -static int -nv04_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, - struct nvkm_object *object) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nvkm_instmem *imem = chan->fifo->base.engine.subdev.device->imem; - u32 context = 0x80000000 | chan->base.chid << 24; - u32 handle = object->handle; - int hash; - - switch (object->engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW : context |= 0x00000000; break; - case NVKM_ENGINE_GR : context |= 0x00010000; break; - case NVKM_ENGINE_MPEG : context |= 0x00020000; break; - default: - WARN_ON(1); - return -EINVAL; - } - - mutex_lock(&chan->fifo->base.mutex); - hash = nvkm_ramht_insert(imem->ramht, object, chan->base.chid, 4, - handle, context); - mutex_unlock(&chan->fifo->base.mutex); - return hash; -} - void * nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) { @@ -79,8 +41,6 @@ nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) const struct nvkm_fifo_chan_func nv04_fifo_dma_func = { .dtor = nv04_fifo_dma_dtor, - .object_ctor = nv04_fifo_dma_object_ctor, - .object_dtor = nv04_fifo_dma_object_dtor, }; static int @@ -115,7 +75,6 @@ nv04_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, BIT(NV04_FIFO_ENGN_GR) | BIT(NV04_FIFO_ENGN_DMA), 0, 0x800000, 0x10000, oclass, &chan->base); - chan->fifo = fifo; if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c index aa5893c11203e..49a000fabc5ae 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c @@ -63,7 +63,6 @@ nv10_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, BIT(NV04_FIFO_ENGN_GR) | BIT(NV04_FIFO_ENGN_DMA), 0, 0x800000, 0x10000, oclass, &chan->base); - chan->fifo = fifo; if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c index 0a7d145ce5aa3..cad8c5ec8dbb4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c @@ -64,7 +64,6 @@ nv17_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, BIT(NV04_FIFO_ENGN_MPEG) | /* NV31- */ BIT(NV04_FIFO_ENGN_DMA), 0, 0x800000, 0x10000, oclass, &chan->base); - chan->fifo = fifo; if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c index ca6fd8a5af6b9..e922e4a5ab1ac 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c @@ -31,38 +31,8 @@ #include #include -static int -nv40_fifo_dma_object_ctor(struct nvkm_fifo_chan *base, - struct nvkm_object *object) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - struct nvkm_instmem *imem = chan->fifo->base.engine.subdev.device->imem; - u32 context = chan->base.chid << 23; - u32 handle = object->handle; - int hash; - - switch (object->engine->subdev.type) { - case NVKM_ENGINE_DMAOBJ: - case NVKM_ENGINE_SW : context |= 0x00000000; break; - case NVKM_ENGINE_GR : context |= 0x00100000; break; - case NVKM_ENGINE_MPEG : context |= 0x00200000; break; - default: - WARN_ON(1); - return -EINVAL; - } - - mutex_lock(&chan->fifo->base.mutex); - hash = nvkm_ramht_insert(imem->ramht, object, chan->base.chid, 4, - handle, context); - mutex_unlock(&chan->fifo->base.mutex); - return hash; -} - static const struct nvkm_fifo_chan_func nv40_fifo_dma_func = { - .dtor = nv04_fifo_dma_dtor, - .object_ctor = nv40_fifo_dma_object_ctor, - .object_dtor = nv04_fifo_dma_object_dtor, }; static int @@ -98,7 +68,6 @@ nv40_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, BIT(NV04_FIFO_ENGN_MPEG) | BIT(NV04_FIFO_ENGN_DMA), 0, 0xc00000, 0x1000, oclass, &chan->base); - chan->fifo = fifo; if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index 2bc835f2ef744..bc72c536f60d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -68,8 +68,6 @@ g84_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, b if (ret) return ret; - nv50_fifo_chan(chan)->ramht = chan->ramht; - nvkm_kmap(chan->ramfc); nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); @@ -158,6 +156,8 @@ g84_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan * const struct nvkm_engn_func g84_engn = { .bind = g84_ectx_bind, + .ramht_add = nv50_eobj_ramht_add, + .ramht_del = nv50_eobj_ramht_del, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c index 6ba4335d40bcb..f87a54e2f5758 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c @@ -70,7 +70,6 @@ gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) return -ENOMEM; *pobject = &chan->base.object; - chan->fifo = fifo; ret = nvkm_fifo_chan_ctor(&gf100_fifo_gpfifo_func, &fifo->base, 0x1000, 0x1000, true, args->v0.vmm, 0, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c index 28a7a5eae72cb..5036edb62dbd2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c @@ -62,7 +62,6 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) return -ENOMEM; *pobject = &chan->base.object; - chan->fifo = fifo; chan->runl = runlist; ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c index 3c2c76bcbe790..a97ec53d7e6ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c @@ -51,7 +51,6 @@ gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) return -ENOMEM; *pobject = &chan->base.object; - chan->fifo = fifo; chan->runl = runlist; ret = nvkm_fifo_chan_ctor(func, &fifo->base, 0x1000, 0x1000, true, vmm, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 08de6ee612ce0..73ae205fc6baf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -178,8 +178,35 @@ const struct nvkm_cgrp_func nv04_cgrp = { }; +void +nv04_eobj_ramht_del(struct nvkm_chan *chan, int hash) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_instmem *imem = fifo->engine.subdev.device->imem; + + mutex_lock(&fifo->mutex); + nvkm_ramht_remove(imem->ramht, hash); + mutex_unlock(&fifo->mutex); +} + +static int +nv04_eobj_ramht_add(struct nvkm_engn *engn, struct nvkm_object *eobj, struct nvkm_chan *chan) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_instmem *imem = fifo->engine.subdev.device->imem; + u32 context = 0x80000000 | chan->id << 24 | engn->id << 16; + int hash; + + mutex_lock(&fifo->mutex); + hash = nvkm_ramht_insert(imem->ramht, eobj, chan->id, 4, eobj->handle, context); + mutex_unlock(&fifo->mutex); + return hash; +} + const struct nvkm_engn_func nv04_engn = { + .ramht_add = nv04_eobj_ramht_add, + .ramht_del = nv04_eobj_ramht_del, }; void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index f27ca3e7bb27b..7574be50d9595 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -108,6 +108,20 @@ nv40_chan = { .stop = nv04_chan_stop, }; +static int +nv40_eobj_ramht_add(struct nvkm_engn *engn, struct nvkm_object *eobj, struct nvkm_chan *chan) +{ + struct nvkm_fifo *fifo = chan->cgrp->runl->fifo; + struct nvkm_instmem *imem = fifo->engine.subdev.device->imem; + u32 context = chan->id << 23 | engn->id << 20; + int hash; + + mutex_lock(&fifo->mutex); + hash = nvkm_ramht_insert(imem->ramht, eobj, chan->id, 4, eobj->handle, context); + mutex_unlock(&fifo->mutex); + return hash; +} + static void nv40_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan *chan) { @@ -154,10 +168,14 @@ nv40_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan static const struct nvkm_engn_func nv40_engn = { .bind = nv40_ectx_bind, + .ramht_add = nv40_eobj_ramht_add, + .ramht_del = nv04_eobj_ramht_del, }; static const struct nvkm_engn_func nv40_engn_sw = { + .ramht_add = nv40_eobj_ramht_add, + .ramht_del = nv04_eobj_ramht_del, }; static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index f068a27d93383..829c52a44a36d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -34,6 +34,18 @@ #include +void +nv50_eobj_ramht_del(struct nvkm_chan *chan, int hash) +{ + nvkm_ramht_remove(chan->ramht, hash); +} + +int +nv50_eobj_ramht_add(struct nvkm_engn *engn, struct nvkm_object *eobj, struct nvkm_chan *chan) +{ + return nvkm_ramht_insert(chan->ramht, eobj, 0, 4, eobj->handle, engn->id << 20); +} + void nv50_chan_stop(struct nvkm_chan *chan) { @@ -89,8 +101,6 @@ nv50_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, if (ret) return ret; - nv50_fifo_chan(chan)->ramht = chan->ramht; - nvkm_kmap(chan->ramfc); nvkm_wo32(chan->ramfc, 0x3c, 0x403f6078); nvkm_wo32(chan->ramfc, 0x44, 0x01003fff); @@ -196,10 +206,14 @@ nv50_ectx_bind(struct nvkm_engn *engn, struct nvkm_cctx *cctx, struct nvkm_chan static const struct nvkm_engn_func nv50_engn = { .bind = nv50_ectx_bind, + .ramht_add = nv50_eobj_ramht_add, + .ramht_del = nv50_eobj_ramht_del, }; const struct nvkm_engn_func nv50_engn_sw = { + .ramht_add = nv50_eobj_ramht_add, + .ramht_del = nv50_eobj_ramht_del, }; static bool diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 13b2185189841..a5df5e2d9d4b4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -85,6 +85,7 @@ extern const struct nvkm_chan_func_userd nv04_chan_userd; void nv04_chan_ramfc_clear(struct nvkm_chan *); void nv04_chan_start(struct nvkm_chan *); void nv04_chan_stop(struct nvkm_chan *); +void nv04_eobj_ramht_del(struct nvkm_chan *, int); int nv10_fifo_chid_nr(struct nvkm_fifo *); @@ -101,6 +102,8 @@ void nv50_chan_unbind(struct nvkm_chan *); void nv50_chan_start(struct nvkm_chan *); void nv50_chan_stop(struct nvkm_chan *); void nv50_chan_preempt(struct nvkm_chan *); +int nv50_eobj_ramht_add(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *); +void nv50_eobj_ramht_del(struct nvkm_chan *, int); extern const struct nvkm_event_func g84_fifo_nonstall; extern const struct nvkm_engn_func g84_engn; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 86bca72c68bc8..5159b64377c5a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -5,6 +5,7 @@ struct nvkm_cctx; struct nvkm_cgrp; struct nvkm_chan; struct nvkm_memory; +struct nvkm_object; struct nvkm_vctx; enum nvkm_subdev_type; @@ -16,6 +17,8 @@ struct nvkm_engn { bool (*mmu_fault_triggered)(struct nvkm_engn *); int (*ctor)(struct nvkm_engn *, struct nvkm_vctx *); void (*bind)(struct nvkm_engn *, struct nvkm_cctx *, struct nvkm_chan *); + int (*ramht_add)(struct nvkm_engn *, struct nvkm_object *, struct nvkm_chan *); + void (*ramht_del)(struct nvkm_chan *, int hash); } *func; struct nvkm_runl *runl; int id; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index f35bdd245233f..1a89c9313662a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -67,6 +67,7 @@ struct nvkm_uobj { struct nvkm_oproxy oproxy; struct nvkm_chan *chan; struct nvkm_cctx *cctx; + int hash; }; static int @@ -129,10 +130,15 @@ static void nvkm_uchan_object_dtor(struct nvkm_oproxy *oproxy) { struct nvkm_uobj *uobj = container_of(oproxy, typeof(*uobj), oproxy); + struct nvkm_engn *engn; if (!uobj->cctx) return; + engn = uobj->cctx->vctx->ectx->engn; + if (engn->func->ramht_del) + engn->func->ramht_del(uobj->chan, uobj->hash); + nvkm_chan_cctx_put(uobj->chan, &uobj->cctx); } @@ -151,7 +157,6 @@ nvkm_uchan_object_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_cgrp *cgrp = chan->cgrp; struct nvkm_engn *engn; struct nvkm_uobj *uobj; - struct nvkm_oclass _oclass; int ret; /* Lookup host engine state for target engine. */ @@ -173,9 +178,25 @@ nvkm_uchan_object_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, return ret; /* Allocate HW object. */ - _oclass = *oclass; - _oclass.parent = &chan->object; - return nvkm_fifo_chan_child_new(&_oclass, argv, argc, &uobj->oproxy.object); + ret = oclass->base.ctor(&(const struct nvkm_oclass) { + .base = oclass->base, + .engn = oclass->engn, + .handle = oclass->handle, + .object = oclass->object, + .client = oclass->client, + .parent = uobj->cctx->vctx->ectx->object ?: oclass->parent, + .engine = engn->engine, + }, argv, argc, &uobj->oproxy.object); + if (ret) + return ret; + + if (engn->func->ramht_add) { + uobj->hash = engn->func->ramht_add(engn, uobj->oproxy.object, uobj->chan); + if (uobj->hash < 0) + return uobj->hash; + } + + return 0; } static int -- GitLab From 06db7fded6dec88772a65c5a39af12ba4dc2ad38 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:38 +1000 Subject: [PATCH 1335/1927] drm/nouveau/fifo: add new channel classes Exposes a bunch of the new features that became possible as a result of the earlier commits. DRM will build on this in the future to add support for features such as SCG ("async compute") and multi-device rendering, as part of the work necessary to be able to write a half- decent vulkan driver - finally. For the moment, this just crudely ports DRM to the API changes. - channel class interfaces now the same for all HW classes - channel group class exposed (SCG) - channel runqueue selector exposed (SCG) - channel sub-device id control exposed (multi-device rendering) - channel names in logging will reflect creating process, not fd owner - explicit USERD allocation required by VOLTA_CHANNEL_GPFIFO_A and newer - drm is smarter about determining the appropriate channel class to use Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvif/cl006b.h | 12 - drivers/gpu/drm/nouveau/include/nvif/cl506e.h | 13 - drivers/gpu/drm/nouveau/include/nvif/cl506f.h | 14 -- drivers/gpu/drm/nouveau/include/nvif/cl826e.h | 13 - drivers/gpu/drm/nouveau/include/nvif/cl826f.h | 14 -- drivers/gpu/drm/nouveau/include/nvif/cl906f.h | 13 - drivers/gpu/drm/nouveau/include/nvif/cla06f.h | 15 -- drivers/gpu/drm/nouveau/include/nvif/class.h | 28 +-- drivers/gpu/drm/nouveau/include/nvif/clc36f.h | 16 -- drivers/gpu/drm/nouveau/include/nvif/if0020.h | 32 +++ drivers/gpu/drm/nouveau/include/nvif/if0021.h | 16 ++ .../drm/nouveau/include/nvkm/engine/fifo.h | 7 +- drivers/gpu/drm/nouveau/nouveau_abi16.c | 1 - drivers/gpu/drm/nouveau/nouveau_chan.c | 222 ++++++++---------- drivers/gpu/drm/nouveau/nouveau_chan.h | 3 + drivers/gpu/drm/nouveau/nouveau_dma.c | 8 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 1 - .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 16 +- .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 30 ++- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.c | 97 ++++---- .../gpu/drm/nouveau/nvkm/engine/fifo/chan.h | 13 +- .../drm/nouveau/nvkm/engine/fifo/chang84.c | 66 ------ .../drm/nouveau/nvkm/engine/fifo/changf100.h | 21 -- .../drm/nouveau/nvkm/engine/fifo/changk104.h | 30 --- .../drm/nouveau/nvkm/engine/fifo/channv04.h | 23 -- .../drm/nouveau/nvkm/engine/fifo/channv50.c | 61 ----- .../drm/nouveau/nvkm/engine/fifo/channv50.h | 41 ---- .../drm/nouveau/nvkm/engine/fifo/dmanv04.c | 90 ------- .../drm/nouveau/nvkm/engine/fifo/dmanv10.c | 78 ------ .../drm/nouveau/nvkm/engine/fifo/dmanv17.c | 79 ------- .../drm/nouveau/nvkm/engine/fifo/dmanv40.c | 83 ------- .../gpu/drm/nouveau/nvkm/engine/fifo/g84.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/g98.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/ga102.c | 18 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.c | 23 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gf100.h | 12 - .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.c | 59 +---- .../gpu/drm/nouveau/nvkm/engine/fifo/gk104.h | 28 --- .../gpu/drm/nouveau/nvkm/engine/fifo/gk110.c | 10 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk208.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm107.c | 8 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gm200.c | 9 +- .../gpu/drm/nouveau/nvkm/engine/fifo/gp100.c | 9 +- .../drm/nouveau/nvkm/engine/fifo/gpfifog84.c | 77 ------ .../nouveau/nvkm/engine/fifo/gpfifogf100.c | 101 -------- .../nouveau/nvkm/engine/fifo/gpfifogk104.c | 110 --------- .../nouveau/nvkm/engine/fifo/gpfifogv100.c | 100 -------- .../drm/nouveau/nvkm/engine/fifo/gpfifonv50.c | 77 ------ .../nouveau/nvkm/engine/fifo/gpfifotu102.c | 66 ------ .../gpu/drm/nouveau/nvkm/engine/fifo/gv100.c | 13 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.c | 26 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv04.h | 15 -- .../gpu/drm/nouveau/nvkm/engine/fifo/nv10.c | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv17.c | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv40.c | 7 +- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.c | 34 +-- .../gpu/drm/nouveau/nvkm/engine/fifo/nv50.h | 15 -- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 17 +- .../gpu/drm/nouveau/nvkm/engine/fifo/tu102.c | 19 +- .../gpu/drm/nouveau/nvkm/engine/fifo/ucgrp.c | 125 ++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/uchan.c | 101 +++++++- .../gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c | 2 +- 63 files changed, 524 insertions(+), 1763 deletions(-) delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl006b.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl506e.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl506f.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl826e.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl826f.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cl906f.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/cla06f.h delete mode 100644 drivers/gpu/drm/nouveau/include/nvif/clc36f.h create mode 100644 drivers/gpu/drm/nouveau/include/nvif/if0021.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h delete mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/ucgrp.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl006b.h b/drivers/gpu/drm/nouveau/include/nvif/cl006b.h deleted file mode 100644 index c960c449e430b..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl006b.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL006B_H__ -#define __NVIF_CL006B_H__ - -struct nv03_channel_dma_v0 { - __u8 version; - __u8 chid; - __u8 pad02[2]; - __u32 offset; - __u64 pushbuf; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl506e.h b/drivers/gpu/drm/nouveau/include/nvif/cl506e.h deleted file mode 100644 index 9df289c7a84f7..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl506e.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL506E_H__ -#define __NVIF_CL506E_H__ - -struct nv50_channel_dma_v0 { - __u8 version; - __u8 chid; - __u8 pad02[6]; - __u64 vmm; - __u64 pushbuf; - __u64 offset; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl506f.h b/drivers/gpu/drm/nouveau/include/nvif/cl506f.h deleted file mode 100644 index 327c96a994bbe..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl506f.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL506F_H__ -#define __NVIF_CL506F_H__ - -struct nv50_channel_gpfifo_v0 { - __u8 version; - __u8 chid; - __u8 pad02[2]; - __u32 ilength; - __u64 ioffset; - __u64 pushbuf; - __u64 vmm; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h b/drivers/gpu/drm/nouveau/include/nvif/cl826e.h deleted file mode 100644 index ef3033b836f0e..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl826e.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL826E_H__ -#define __NVIF_CL826E_H__ - -struct g82_channel_dma_v0 { - __u8 version; - __u8 chid; - __u8 pad02[6]; - __u64 vmm; - __u64 pushbuf; - __u64 offset; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h b/drivers/gpu/drm/nouveau/include/nvif/cl826f.h deleted file mode 100644 index c5d5e56b04cc0..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl826f.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL826F_H__ -#define __NVIF_CL826F_H__ - -struct g82_channel_gpfifo_v0 { - __u8 version; - __u8 chid; - __u8 pad02[2]; - __u32 ilength; - __u64 ioffset; - __u64 pushbuf; - __u64 vmm; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h b/drivers/gpu/drm/nouveau/include/nvif/cl906f.h deleted file mode 100644 index 5ccc8fd8c458e..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cl906f.h +++ /dev/null @@ -1,13 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CL906F_H__ -#define __NVIF_CL906F_H__ - -struct fermi_channel_gpfifo_v0 { - __u8 version; - __u8 chid; - __u8 pad02[2]; - __u32 ilength; - __u64 ioffset; - __u64 vmm; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h b/drivers/gpu/drm/nouveau/include/nvif/cla06f.h deleted file mode 100644 index 10449accd3e84..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/cla06f.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CLA06F_H__ -#define __NVIF_CLA06F_H__ - -struct kepler_channel_gpfifo_a_v0 { - __u8 version; - __u8 priv; - __u16 chid; - __u32 ilength; - __u64 ioffset; - __u64 runlist; - __u64 vmm; - __u64 inst; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index e9d9f314a744e..ceea074b064b5 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -70,23 +70,23 @@ #define MAXWELL_FAULT_BUFFER_A /* clb069.h */ 0x0000b069 #define VOLTA_FAULT_BUFFER_A /* clb069.h */ 0x0000c369 -#define NV03_CHANNEL_DMA /* cl506b.h */ 0x0000006b -#define NV10_CHANNEL_DMA /* cl506b.h */ 0x0000006e -#define NV17_CHANNEL_DMA /* cl506b.h */ 0x0000176e -#define NV40_CHANNEL_DMA /* cl506b.h */ 0x0000406e +#define NV03_CHANNEL_DMA /* if0020.h */ 0x0000006b +#define NV10_CHANNEL_DMA /* if0020.h */ 0x0000006e +#define NV17_CHANNEL_DMA /* if0020.h */ 0x0000176e +#define NV40_CHANNEL_DMA /* if0020.h */ 0x0000406e #define KEPLER_CHANNEL_GROUP_A /* if0021.h */ 0x0000a06c -#define NV50_CHANNEL_GPFIFO /* cl506f.h */ 0x0000506f -#define G82_CHANNEL_GPFIFO /* cl826f.h */ 0x0000826f -#define FERMI_CHANNEL_GPFIFO /* cl906f.h */ 0x0000906f -#define KEPLER_CHANNEL_GPFIFO_A /* cla06f.h */ 0x0000a06f -#define KEPLER_CHANNEL_GPFIFO_B /* cla06f.h */ 0x0000a16f -#define MAXWELL_CHANNEL_GPFIFO_A /* cla06f.h */ 0x0000b06f -#define PASCAL_CHANNEL_GPFIFO_A /* cla06f.h */ 0x0000c06f -#define VOLTA_CHANNEL_GPFIFO_A /* clc36f.h */ 0x0000c36f -#define TURING_CHANNEL_GPFIFO_A /* clc36f.h */ 0x0000c46f -#define AMPERE_CHANNEL_GPFIFO_B /* clc36f.h */ 0x0000c76f +#define NV50_CHANNEL_GPFIFO /* if0020.h */ 0x0000506f +#define G82_CHANNEL_GPFIFO /* if0020.h */ 0x0000826f +#define FERMI_CHANNEL_GPFIFO /* if0020.h */ 0x0000906f +#define KEPLER_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000a06f +#define KEPLER_CHANNEL_GPFIFO_B /* if0020.h */ 0x0000a16f +#define MAXWELL_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000b06f +#define PASCAL_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c06f +#define VOLTA_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c36f +#define TURING_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c46f +#define AMPERE_CHANNEL_GPFIFO_B /* if0020.h */ 0x0000c76f #define NV50_DISP /* if0010.h */ 0x00005070 #define G82_DISP /* if0010.h */ 0x00008270 diff --git a/drivers/gpu/drm/nouveau/include/nvif/clc36f.h b/drivers/gpu/drm/nouveau/include/nvif/clc36f.h deleted file mode 100644 index cdf6708e1d9a6..0000000000000 --- a/drivers/gpu/drm/nouveau/include/nvif/clc36f.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NVIF_CLC36F_H__ -#define __NVIF_CLC36F_H__ - -struct volta_channel_gpfifo_a_v0 { - __u8 version; - __u8 priv; - __u16 chid; - __u32 ilength; - __u64 ioffset; - __u64 runlist; - __u64 vmm; - __u64 inst; - __u32 token; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0020.h b/drivers/gpu/drm/nouveau/include/nvif/if0020.h index 1893b8aa0abb4..085e0ae8a4506 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/if0020.h +++ b/drivers/gpu/drm/nouveau/include/nvif/if0020.h @@ -2,6 +2,38 @@ #ifndef __NVIF_IF0020_H__ #define __NVIF_IF0020_H__ +union nvif_chan_args { + struct nvif_chan_v0 { + __u8 version; + __u8 namelen; + __u8 runlist; + __u8 runq; + __u8 priv; + __u8 pad05; + __u16 devm; + __u64 vmm; + + __u64 ctxdma; + __u64 offset; + __u64 length; + + __u64 huserd; + __u64 ouserd; + + __u32 token; + __u16 chid; + __u8 pad3e; +#define NVIF_CHAN_V0_INST_APER_VRAM 0 +#define NVIF_CHAN_V0_INST_APER_HOST 1 +#define NVIF_CHAN_V0_INST_APER_NCOH 2 +#define NVIF_CHAN_V0_INST_APER_INST 0xff + __u8 aper; + __u64 inst; + + __u8 name[]; + } v0; +}; + union nvif_chan_event_args { struct nvif_chan_event_v0 { __u8 version; diff --git a/drivers/gpu/drm/nouveau/include/nvif/if0021.h b/drivers/gpu/drm/nouveau/include/nvif/if0021.h new file mode 100644 index 0000000000000..5013def90455a --- /dev/null +++ b/drivers/gpu/drm/nouveau/include/nvif/if0021.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +#ifndef __NVIF_IF0021_H__ +#define __NVIF_IF0021_H__ + +union nvif_cgrp_args { + struct nvif_cgrp_v0 { + __u8 version; + __u8 namelen; + __u8 runlist; + __u8 pad03[3]; + __u16 cgid; + __u64 vmm; + __u8 name[]; + } v0; +}; +#endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index da8abcbeb965c..c71f412bbb7ff 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -16,7 +16,8 @@ struct nvkm_chan { struct nvkm_gpuobj *inst; struct nvkm_vmm *vmm; - union { int id; int chid; }; /*FIXME: remove later */ + struct nvkm_gpuobj *push; + int id; struct { struct nvkm_memory *mem; @@ -35,11 +36,7 @@ struct nvkm_chan { atomic_t errored; struct list_head cctxs; - - struct nvkm_object object; - struct list_head head; - struct nvkm_gpuobj *push; }; struct nvkm_chan *nvkm_chan_get_chid(struct nvkm_engine *, int id, unsigned long *irqflags); diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c index 9237ef328b9fb..82dab51d8aebc 100644 --- a/drivers/gpu/drm/nouveau/nouveau_abi16.c +++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include "nouveau_drv.h" diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index 5b7042d94e808..efd6cf46921b1 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -25,13 +25,7 @@ #include #include -#include -#include -#include -#include -#include #include -#include #include "nouveau_drv.h" #include "nouveau_dma.h" @@ -101,6 +95,7 @@ nouveau_channel_del(struct nouveau_channel **pchan) nvif_object_dtor(&chan->vram); nvif_event_dtor(&chan->kill); nvif_object_dtor(&chan->user); + nvif_mem_dtor(&chan->mem_userd); nvif_object_dtor(&chan->push.ctxdma); nouveau_vma_del(&chan->push.vma); nouveau_bo_unmap(chan->push.buffer); @@ -250,134 +245,112 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nvif_device *device, } static int -nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device, - u64 runlist, bool priv, struct nouveau_channel **pchan) +nouveau_channel_ctor(struct nouveau_drm *drm, struct nvif_device *device, bool priv, u64 runm, + struct nouveau_channel **pchan) { - static const u16 oclasses[] = { AMPERE_CHANNEL_GPFIFO_B, - TURING_CHANNEL_GPFIFO_A, - VOLTA_CHANNEL_GPFIFO_A, - PASCAL_CHANNEL_GPFIFO_A, - MAXWELL_CHANNEL_GPFIFO_A, - KEPLER_CHANNEL_GPFIFO_B, - KEPLER_CHANNEL_GPFIFO_A, - FERMI_CHANNEL_GPFIFO, - G82_CHANNEL_GPFIFO, - NV50_CHANNEL_GPFIFO, - 0 }; - const u16 *oclass = oclasses; - union { - struct nv50_channel_gpfifo_v0 nv50; - struct fermi_channel_gpfifo_v0 fermi; - struct kepler_channel_gpfifo_a_v0 kepler; - struct volta_channel_gpfifo_a_v0 volta; + static const struct { + s32 oclass; + int version; + } hosts[] = { + { AMPERE_CHANNEL_GPFIFO_B, 0 }, + { TURING_CHANNEL_GPFIFO_A, 0 }, + { VOLTA_CHANNEL_GPFIFO_A, 0 }, + { PASCAL_CHANNEL_GPFIFO_A, 0 }, + { MAXWELL_CHANNEL_GPFIFO_A, 0 }, + { KEPLER_CHANNEL_GPFIFO_B, 0 }, + { KEPLER_CHANNEL_GPFIFO_A, 0 }, + { FERMI_CHANNEL_GPFIFO , 0 }, + { G82_CHANNEL_GPFIFO , 0 }, + { NV50_CHANNEL_GPFIFO , 0 }, + { NV40_CHANNEL_DMA , 0 }, + { NV17_CHANNEL_DMA , 0 }, + { NV10_CHANNEL_DMA , 0 }, + { NV03_CHANNEL_DMA , 0 }, + {} + }; + struct { + struct nvif_chan_v0 chan; + char name[TASK_COMM_LEN+16]; } args; + struct nouveau_cli *cli = (void *)device->object.client; struct nouveau_channel *chan; - u32 size; - int ret; + const u64 plength = 0x10000; + const u64 ioffset = plength; + const u64 ilength = 0x02000; + char name[TASK_COMM_LEN]; + int cid, ret; + u64 size; + + cid = nvif_mclass(&device->object, hosts); + if (cid < 0) + return cid; + + if (hosts[cid].oclass < NV50_CHANNEL_GPFIFO) + size = plength; + else + size = ioffset + ilength; /* allocate dma push buffer */ - ret = nouveau_channel_prep(drm, device, 0x12000, &chan); + ret = nouveau_channel_prep(drm, device, size, &chan); *pchan = chan; if (ret) return ret; /* create channel object */ - do { - if (oclass[0] >= VOLTA_CHANNEL_GPFIFO_A) { - args.volta.version = 0; - args.volta.ilength = 0x02000; - args.volta.ioffset = 0x10000 + chan->push.addr; - args.volta.runlist = runlist; - args.volta.vmm = nvif_handle(&chan->vmm->vmm.object); - args.volta.priv = priv; - size = sizeof(args.volta); - } else - if (oclass[0] >= KEPLER_CHANNEL_GPFIFO_A) { - args.kepler.version = 0; - args.kepler.ilength = 0x02000; - args.kepler.ioffset = 0x10000 + chan->push.addr; - args.kepler.runlist = runlist; - args.kepler.vmm = nvif_handle(&chan->vmm->vmm.object); - args.kepler.priv = priv; - size = sizeof(args.kepler); - } else - if (oclass[0] >= FERMI_CHANNEL_GPFIFO) { - args.fermi.version = 0; - args.fermi.ilength = 0x02000; - args.fermi.ioffset = 0x10000 + chan->push.addr; - args.fermi.vmm = nvif_handle(&chan->vmm->vmm.object); - size = sizeof(args.fermi); - } else { - args.nv50.version = 0; - args.nv50.ilength = 0x02000; - args.nv50.ioffset = 0x10000 + chan->push.addr; - args.nv50.pushbuf = nvif_handle(&chan->push.ctxdma); - args.nv50.vmm = nvif_handle(&chan->vmm->vmm.object); - size = sizeof(args.nv50); - } - - ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, - *oclass++, &args, size, &chan->user); - if (ret == 0) { - if (chan->user.oclass >= VOLTA_CHANNEL_GPFIFO_A) { - chan->chid = args.volta.chid; - chan->inst = args.volta.inst; - chan->token = args.volta.token; - } else - if (chan->user.oclass >= KEPLER_CHANNEL_GPFIFO_A) { - chan->chid = args.kepler.chid; - chan->inst = args.kepler.inst; - } else - if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) { - chan->chid = args.fermi.chid; - } else { - chan->chid = args.nv50.chid; - } + args.chan.version = 0; + args.chan.namelen = sizeof(args.name); + args.chan.runlist = __ffs64(runm); + args.chan.runq = 0; + args.chan.priv = priv; + args.chan.devm = BIT(0); + if (hosts[cid].oclass < NV50_CHANNEL_GPFIFO) { + args.chan.vmm = 0; + args.chan.ctxdma = nvif_handle(&chan->push.ctxdma); + args.chan.offset = chan->push.addr; + args.chan.length = 0; + } else { + args.chan.vmm = nvif_handle(&chan->vmm->vmm.object); + if (hosts[cid].oclass < FERMI_CHANNEL_GPFIFO) + args.chan.ctxdma = nvif_handle(&chan->push.ctxdma); + else + args.chan.ctxdma = 0; + args.chan.offset = ioffset + chan->push.addr; + args.chan.length = ilength; + } + args.chan.huserd = 0; + args.chan.ouserd = 0; + + /* allocate userd */ + if (hosts[cid].oclass >= VOLTA_CHANNEL_GPFIFO_A) { + ret = nvif_mem_ctor(&cli->mmu, "abi16ChanUSERD", NVIF_CLASS_MEM_GF100, + NVIF_MEM_VRAM | NVIF_MEM_COHERENT | NVIF_MEM_MAPPABLE, + 0, PAGE_SIZE, NULL, 0, &chan->mem_userd); + if (ret) return ret; - } - } while (*oclass); - nouveau_channel_del(pchan); - return ret; -} + args.chan.huserd = nvif_handle(&chan->mem_userd.object); + args.chan.ouserd = 0; -static int -nouveau_channel_dma(struct nouveau_drm *drm, struct nvif_device *device, - struct nouveau_channel **pchan) -{ - static const u16 oclasses[] = { NV40_CHANNEL_DMA, - NV17_CHANNEL_DMA, - NV10_CHANNEL_DMA, - NV03_CHANNEL_DMA, - 0 }; - const u16 *oclass = oclasses; - struct nv03_channel_dma_v0 args; - struct nouveau_channel *chan; - int ret; + chan->userd = &chan->mem_userd.object; + } else { + chan->userd = &chan->user; + } - /* allocate dma push buffer */ - ret = nouveau_channel_prep(drm, device, 0x10000, &chan); - *pchan = chan; - if (ret) - return ret; + get_task_comm(name, current); + snprintf(args.name, sizeof(args.name), "%s[%d]", name, task_pid_nr(current)); - /* create channel object */ - args.version = 0; - args.pushbuf = nvif_handle(&chan->push.ctxdma); - args.offset = chan->push.addr; - - do { - ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, - *oclass++, &args, sizeof(args), - &chan->user); - if (ret == 0) { - chan->chid = args.chid; - return ret; - } - } while (ret && *oclass); + ret = nvif_object_ctor(&device->object, "abi16ChanUser", 0, hosts[cid].oclass, + &args, sizeof(args), &chan->user); + if (ret) { + nouveau_channel_del(pchan); + return ret; + } - nouveau_channel_del(pchan); - return ret; + chan->runlist = args.chan.runlist; + chan->chid = args.chan.chid; + chan->inst = args.chan.inst; + chan->token = args.chan.token; + return 0; } static int @@ -388,7 +361,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) struct nv_dma_v0 args = {}; int ret, i; - ret = nvif_object_map(&chan->user, NULL, 0); + ret = nvif_object_map(chan->userd, NULL, 0); if (ret) return ret; @@ -518,15 +491,10 @@ nouveau_channel_new(struct nouveau_drm *drm, struct nvif_device *device, struct nouveau_cli *cli = (void *)device->object.client; int ret; - /* hack until fencenv50 is fixed, and agp access relaxed */ - ret = nouveau_channel_ind(drm, device, runm, priv, pchan); + ret = nouveau_channel_ctor(drm, device, priv, runm, pchan); if (ret) { - NV_PRINTK(dbg, cli, "ib channel create, %d\n", ret); - ret = nouveau_channel_dma(drm, device, pchan); - if (ret) { - NV_PRINTK(dbg, cli, "dma channel create, %d\n", ret); - return ret; - } + NV_PRINTK(dbg, cli, "channel create, %d\n", ret); + return ret; } ret = nouveau_channel_init(*pchan, vram, gart); diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.h b/drivers/gpu/drm/nouveau/nouveau_chan.h index 40467e8c96685..e06a8ffed31a8 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.h +++ b/drivers/gpu/drm/nouveau/nouveau_chan.h @@ -16,6 +16,9 @@ struct nouveau_channel { struct nouveau_drm *drm; struct nouveau_vmm *vmm; + struct nvif_mem mem_userd; + struct nvif_object *userd; + int runlist; int chid; u64 inst; diff --git a/drivers/gpu/drm/nouveau/nouveau_dma.c b/drivers/gpu/drm/nouveau/nouveau_dma.c index ddb75d80bc53a..b90cac6d5772b 100644 --- a/drivers/gpu/drm/nouveau/nouveau_dma.c +++ b/drivers/gpu/drm/nouveau/nouveau_dma.c @@ -42,9 +42,9 @@ READ_GET(struct nouveau_channel *chan, uint64_t *prev_get, int *timeout) { uint64_t val; - val = nvif_rd32(&chan->user, chan->user_get); + val = nvif_rd32(chan->userd, chan->user_get); if (chan->user_get_hi) - val |= (uint64_t)nvif_rd32(&chan->user, chan->user_get_hi) << 32; + val |= (uint64_t)nvif_rd32(chan->userd, chan->user_get_hi) << 32; /* reset counter as long as GET is still advancing, this is * to avoid misdetecting a GPU lockup if the GPU happens to @@ -86,7 +86,7 @@ nv50_dma_push(struct nouveau_channel *chan, u64 offset, int length) /* Flush writes. */ nouveau_bo_rd32(pb, 0); - nvif_wr32(&chan->user, 0x8c, chan->dma.ib_put); + nvif_wr32(chan->userd, 0x8c, chan->dma.ib_put); if (user->func && user->func->doorbell) user->func->doorbell(user, chan->token); chan->dma.ib_free--; @@ -98,7 +98,7 @@ nv50_dma_push_wait(struct nouveau_channel *chan, int count) uint32_t cnt = 0, prev_get = 0; while (chan->dma.ib_free < count) { - uint32_t get = nvif_rd32(&chan->user, 0x88); + uint32_t get = nvif_rd32(chan->userd, 0x88); if (get != prev_get) { prev_get = get; cnt = 0; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 4a533b0804450..61be972f83852 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -50,7 +50,6 @@ #include #include -#include #include "nouveau_drv.h" #include "nouveau_dma.h" diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index eb82da7e7d87c..f9aad4cf82713 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -25,19 +25,5 @@ nvkm-y += nvkm/engine/fifo/gv100.o nvkm-y += nvkm/engine/fifo/tu102.o nvkm-y += nvkm/engine/fifo/ga102.o -nvkm-y += nvkm/engine/fifo/channv50.o -nvkm-y += nvkm/engine/fifo/chang84.o - -nvkm-y += nvkm/engine/fifo/dmanv04.o -nvkm-y += nvkm/engine/fifo/dmanv10.o -nvkm-y += nvkm/engine/fifo/dmanv17.o -nvkm-y += nvkm/engine/fifo/dmanv40.o - -nvkm-y += nvkm/engine/fifo/gpfifonv50.o -nvkm-y += nvkm/engine/fifo/gpfifog84.o -nvkm-y += nvkm/engine/fifo/gpfifogf100.o -nvkm-y += nvkm/engine/fifo/gpfifogk104.o -nvkm-y += nvkm/engine/fifo/gpfifogv100.o -nvkm-y += nvkm/engine/fifo/gpfifotu102.o - +nvkm-y += nvkm/engine/fifo/ucgrp.o nvkm-y += nvkm/engine/fifo/uchan.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 9362afe37f3ec..61534a8a33d7f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -59,6 +59,9 @@ nvkm_fifo_class_new(struct nvkm_device *device, const struct nvkm_oclass *oclass { struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); + if (oclass->engn == &fifo->func->cgrp.user) + return nvkm_ucgrp_new(fifo, oclass, argv, argc, pobject); + if (oclass->engn == &fifo->func->chan.user) return nvkm_uchan_new(fifo, NULL, oclass, argv, argc, pobject); @@ -75,9 +78,20 @@ static int nvkm_fifo_class_get(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) { struct nvkm_fifo *fifo = nvkm_fifo(oclass->engine); + const struct nvkm_fifo_func_cgrp *cgrp = &fifo->func->cgrp; const struct nvkm_fifo_func_chan *chan = &fifo->func->chan; int c = 0; + /* *_CHANNEL_GROUP_* */ + if (cgrp->user.oclass) { + if (c++ == index) { + oclass->base = cgrp->user; + oclass->engn = &fifo->func->cgrp.user; + *class = &nvkm_fifo_class; + return 0; + } + } + /* *_CHANNEL_DMA, *_CHANNEL_GPFIFO_* */ if (chan->user.oclass) { if (c++ == index) { @@ -264,9 +278,6 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) return ret; } - if (fifo->func->oneinit) - return fifo->func->oneinit(fifo); - return 0; } @@ -282,7 +293,6 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) struct nvkm_fifo *fifo = nvkm_fifo(engine); struct nvkm_runl *runl, *runt; struct nvkm_runq *runq, *rtmp; - void *data = fifo; if (fifo->userd.bar1) nvkm_vmm_put(nvkm_bar_bar1_vmm(engine->subdev.device), &fifo->userd.bar1); @@ -296,11 +306,9 @@ nvkm_fifo_dtor(struct nvkm_engine *engine) nvkm_chid_unref(&fifo->cgid); nvkm_chid_unref(&fifo->chid); - if (fifo->func->dtor) - data = fifo->func->dtor(fifo); nvkm_event_fini(&fifo->nonstall.event); mutex_destroy(&fifo->mutex); - return data; + return fifo; } static const struct nvkm_engine_func @@ -315,11 +323,15 @@ nvkm_fifo = { }; int -nvkm_fifo_ctor(const struct nvkm_fifo_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_fifo *fifo) +nvkm_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { + struct nvkm_fifo *fifo; int ret; + if (!(fifo = *pfifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) + return -ENOMEM; + fifo->func = func; INIT_LIST_HEAD(&fifo->runqs); INIT_LIST_HEAD(&fifo->runls); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c index 4f99e5bf4e7ff..b7c9d6115bce3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c @@ -28,7 +28,6 @@ #include "runl.h" #include "priv.h" -#include #include #include #include @@ -283,16 +282,6 @@ nvkm_chan_del(struct nvkm_chan **pchan) nvkm_cgrp_unref(&chan->cgrp); } - chan = nvkm_object_dtor(&chan->object); - kfree(chan); -} - -static void * -nvkm_fifo_chan_dtor(struct nvkm_object *object) -{ - struct nvkm_fifo_chan *chan = nvkm_fifo_chan(object); - void *data = chan->func->dtor(chan); - if (chan->vmm) { nvkm_vmm_part(chan->vmm, chan->inst->memory); nvkm_vmm_unref(&chan->vmm); @@ -300,8 +289,7 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object) nvkm_gpuobj_del(&chan->push); nvkm_gpuobj_del(&chan->inst); - kfree(chan->func); - return data; + kfree(chan); } void @@ -354,51 +342,42 @@ nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags) return NULL; } -static const struct nvkm_object_func -nvkm_fifo_chan_func = { - .dtor = nvkm_fifo_chan_dtor, -}; - int -nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, - struct nvkm_fifo *fifo, u32 size, u32 align, bool zero, - u64 hvmm, u64 push, u32 engm, int bar, u32 base, - u32 user, const struct nvkm_oclass *oclass, - struct nvkm_fifo_chan *chan) +nvkm_chan_new_(const struct nvkm_chan_func *func, struct nvkm_runl *runl, int runq, + struct nvkm_cgrp *cgrp, const char *name, bool priv, u32 devm, struct nvkm_vmm *vmm, + struct nvkm_dmaobj *dmaobj, u64 offset, u64 length, + struct nvkm_memory *userd, u64 ouserd, struct nvkm_chan **pchan) { - struct nvkm_chan_func *func; - struct nvkm_client *client = oclass->client; + struct nvkm_fifo *fifo = runl->fifo; struct nvkm_device *device = fifo->engine.subdev.device; - struct nvkm_dmaobj *dmaobj; - struct nvkm_cgrp *cgrp = NULL; - struct nvkm_runl *runl; - struct nvkm_engn *engn = NULL; - struct nvkm_vmm *vmm = NULL; + struct nvkm_chan *chan; int ret; - nvkm_runl_foreach(runl, fifo) { - engn = nvkm_runl_find_engn(engn, runl, engm & BIT(engn->id)); - if (engn) - break; - } - - if (!engn) + /* Validate arguments against class requirements. */ + if ((runq && runq >= runl->func->runqs) || + (!func->inst->vmm != !vmm) || + ((func->userd->bar < 0) == !userd) || + (!func->ramfc->ctxdma != !dmaobj) || + ((func->ramfc->devm < devm) && devm != BIT(0)) || + (!func->ramfc->priv && priv)) { + RUNL_DEBUG(runl, "args runq:%d:%d vmm:%d:%p userd:%d:%p " + "push:%d:%p devm:%08x:%08x priv:%d:%d", + runl->func->runqs, runq, func->inst->vmm, vmm, + func->userd->bar < 0, userd, func->ramfc->ctxdma, dmaobj, + func->ramfc->devm, devm, func->ramfc->priv, priv); return -EINVAL; + } - /*FIXME: temp kludge to ease transition, remove later */ - if (!(func = kmalloc(sizeof(*func), GFP_KERNEL))) + if (!(chan = *pchan = kzalloc(sizeof(*chan), GFP_KERNEL))) return -ENOMEM; - *func = *fifo->func->chan.func; - func->dtor = fn->dtor; - chan->func = func; + strscpy(chan->name, name, sizeof(chan->name)); + chan->runq = runq; chan->id = -1; spin_lock_init(&chan->lock); atomic_set(&chan->blocked, 1); atomic_set(&chan->errored, 0); - - nvkm_object_ctor(&nvkm_fifo_chan_func, oclass, &chan->object); INIT_LIST_HEAD(&chan->cctxs); INIT_LIST_HEAD(&chan->head); @@ -437,10 +416,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, /* Initialise virtual address-space. */ if (func->inst->vmm) { - struct nvkm_vmm *vmm = nvkm_uvmm_search(client, hvmm); - if (IS_ERR(vmm)) - return PTR_ERR(vmm); - if (WARN_ON(vmm->mmu != device->mmu)) return -EINVAL; @@ -455,10 +430,6 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, /* Allocate HW ctxdma for push buffer. */ if (func->ramfc->ctxdma) { - dmaobj = nvkm_dmaobj_search(client, push); - if (IS_ERR(dmaobj)) - return PTR_ERR(dmaobj); - ret = nvkm_object_bind(&dmaobj->object, chan->inst, -16, &chan->push); if (ret) { RUNL_DEBUG(runl, "bind %d", ret); @@ -477,7 +448,20 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, cgrp->id = chan->id; /* Initialise USERD. */ - if (1) { + if (func->userd->bar < 0) { + if (ouserd + chan->func->userd->size >= nvkm_memory_size(userd)) { + RUNL_DEBUG(runl, "ouserd %llx", ouserd); + return -EINVAL; + } + + ret = nvkm_memory_kmap(userd, &chan->userd.mem); + if (ret) { + RUNL_DEBUG(runl, "userd %d", ret); + return ret; + } + + chan->userd.base = ouserd; + } else { chan->userd.mem = nvkm_memory_ref(fifo->userd.mem); chan->userd.base = chan->id * chan->func->userd->size; } @@ -485,5 +469,12 @@ nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *fn, if (chan->func->userd->clear) chan->func->userd->clear(chan); + /* Initialise RAMFC. */ + ret = chan->func->ramfc->write(chan, offset, length, devm, priv); + if (ret) { + RUNL_DEBUG(runl, "ramfc %d", ret); + return ret; + } + return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h index 836f9ecb13c64..85b94f699128d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.h @@ -1,9 +1,10 @@ /* SPDX-License-Identifier: MIT */ #ifndef __NVKM_CHAN_H__ #define __NVKM_CHAN_H__ -#define nvkm_chan(p) container_of((p), struct nvkm_chan, object) /*FIXME: remove later */ #include +struct nvkm_dmaobj; struct nvkm_engn; +struct nvkm_runl; extern const struct nvkm_event_func nvkm_chan_event; @@ -50,14 +51,12 @@ struct nvkm_chan_func { void (*stop)(struct nvkm_chan *); void (*preempt)(struct nvkm_chan *); u32 (*doorbell_handle)(struct nvkm_chan *); - - void *(*dtor)(struct nvkm_fifo_chan *); }; -int nvkm_fifo_chan_ctor(const struct nvkm_fifo_chan_func *, struct nvkm_fifo *, - u32 size, u32 align, bool zero, u64 vm, u64 push, - u32 engm, int bar, u32 base, u32 user, - const struct nvkm_oclass *, struct nvkm_fifo_chan *); +int nvkm_chan_new_(const struct nvkm_chan_func *, struct nvkm_runl *, int runq, struct nvkm_cgrp *, + const char *name, bool priv, u32 devm, struct nvkm_vmm *, struct nvkm_dmaobj *, + u64 offset, u64 length, struct nvkm_memory *userd, u64 userd_bar1, + struct nvkm_chan **); void nvkm_chan_del(struct nvkm_chan **); void nvkm_chan_allow(struct nvkm_chan *); void nvkm_chan_block(struct nvkm_chan *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c deleted file mode 100644 index fc266af2cf536..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chang84.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv50.h" - -#include -#include -#include -#include - -#include - -static const struct nvkm_fifo_chan_func -g84_fifo_chan_func = { - .dtor = nv50_fifo_chan_dtor, -}; - -int -g84_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, - const struct nvkm_oclass *oclass, - struct nv50_fifo_chan *chan) -{ - int ret; - - if (!vmm) - return -EINVAL; - - ret = nvkm_fifo_chan_ctor(&g84_fifo_chan_func, &fifo->base, - 0x10000, 0x1000, false, vmm, push, - BIT(G84_FIFO_ENGN_SW) | - BIT(G84_FIFO_ENGN_GR) | - BIT(G84_FIFO_ENGN_MPEG) | - BIT(G84_FIFO_ENGN_MSPPP) | - BIT(G84_FIFO_ENGN_ME) | - BIT(G84_FIFO_ENGN_CE0) | - BIT(G84_FIFO_ENGN_VP) | - BIT(G84_FIFO_ENGN_MSPDEC) | - BIT(G84_FIFO_ENGN_CIPHER) | - BIT(G84_FIFO_ENGN_SEC) | - BIT(G84_FIFO_ENGN_VIC) | - BIT(G84_FIFO_ENGN_BSP) | - BIT(G84_FIFO_ENGN_MSVLD) | - BIT(G84_FIFO_ENGN_DMA), - 0, 0xc00000, 0x2000, oclass, &chan->base); - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h deleted file mode 100644 index e94c16b0efb6b..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changf100.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __GF100_FIFO_CHAN_H__ -#define __GF100_FIFO_CHAN_H__ -#define gf100_fifo_chan(p) container_of((p), struct gf100_fifo_chan, base) -#include "chan.h" -#include "gf100.h" - -struct gf100_fifo_chan { - struct nvkm_fifo_chan base; - -#define GF100_FIFO_ENGN_GR 0 -#define GF100_FIFO_ENGN_MSPDEC 1 -#define GF100_FIFO_ENGN_MSPPP 2 -#define GF100_FIFO_ENGN_MSVLD 3 -#define GF100_FIFO_ENGN_CE0 4 -#define GF100_FIFO_ENGN_CE1 5 -#define GF100_FIFO_ENGN_SW 15 -}; - -extern const struct nvkm_fifo_chan_oclass gf100_fifo_gpfifo_oclass; -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h deleted file mode 100644 index 3450f6349db3a..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h +++ /dev/null @@ -1,30 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __GK104_FIFO_CHAN_H__ -#define __GK104_FIFO_CHAN_H__ -#define gk104_fifo_chan(p) container_of((p), struct gk104_fifo_chan, base) -#include "chan.h" -#include "gk104.h" - -struct gk104_fifo_chan { - struct nvkm_fifo_chan base; - int runl; - -#define GK104_FIFO_ENGN_SW 15 -}; - -extern const struct nvkm_fifo_chan_func gk104_fifo_gpfifo_func; - -int gk104_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, - void *data, u32 size, struct nvkm_object **); -void *gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *); - -int gv100_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, - void *data, u32 size, struct nvkm_object **); -int gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *, - struct gk104_fifo *, u64 *, u16 *, u64, u64, u64, - u64 *, bool, u32 *, const struct nvkm_oclass *, - struct nvkm_object **); - -int tu102_fifo_gpfifo_new(struct gk104_fifo *, const struct nvkm_oclass *, - void *data, u32 size, struct nvkm_object **); -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h deleted file mode 100644 index 5a1717888978e..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv04.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NV04_FIFO_CHAN_H__ -#define __NV04_FIFO_CHAN_H__ -#define nv04_fifo_chan(p) container_of((p), struct nv04_fifo_chan, base) -#include "chan.h" -#include "nv04.h" - -struct nv04_fifo_chan { - struct nvkm_fifo_chan base; -#define NV04_FIFO_ENGN_SW 0 -#define NV04_FIFO_ENGN_GR 1 -#define NV04_FIFO_ENGN_MPEG 2 -#define NV04_FIFO_ENGN_DMA 3 -}; - -extern const struct nvkm_fifo_chan_func nv04_fifo_dma_func; -void *nv04_fifo_dma_dtor(struct nvkm_fifo_chan *); - -extern const struct nvkm_fifo_chan_oclass nv04_fifo_dma_oclass; -extern const struct nvkm_fifo_chan_oclass nv10_fifo_dma_oclass; -extern const struct nvkm_fifo_chan_oclass nv17_fifo_dma_oclass; -extern const struct nvkm_fifo_chan_oclass nv40_fifo_dma_oclass; -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c deleted file mode 100644 index 0ad6354b9cccc..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.c +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv50.h" - -#include -#include -#include -#include - -void * -nv50_fifo_chan_dtor(struct nvkm_fifo_chan *base) -{ - struct nv50_fifo_chan *chan = nv50_fifo_chan(base); - return chan; -} - -static const struct nvkm_fifo_chan_func -nv50_fifo_chan_func = { - .dtor = nv50_fifo_chan_dtor, -}; - -int -nv50_fifo_chan_ctor(struct nv50_fifo *fifo, u64 vmm, u64 push, - const struct nvkm_oclass *oclass, - struct nv50_fifo_chan *chan) -{ - int ret; - - if (!vmm) - return -EINVAL; - - ret = nvkm_fifo_chan_ctor(&nv50_fifo_chan_func, &fifo->base, - 0x10000, 0x1000, false, vmm, push, - BIT(NV50_FIFO_ENGN_SW) | - BIT(NV50_FIFO_ENGN_GR) | - BIT(NV50_FIFO_ENGN_MPEG) | - BIT(NV50_FIFO_ENGN_DMA), - 0, 0xc00000, 0x2000, oclass, &chan->base); - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h deleted file mode 100644 index ea4ec3bc4b8ab..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/channv50.h +++ /dev/null @@ -1,41 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NV50_FIFO_CHAN_H__ -#define __NV50_FIFO_CHAN_H__ -#define nv50_fifo_chan(p) container_of((p), struct nv50_fifo_chan, base) -#include "chan.h" -#include "nv50.h" - -struct nv50_fifo_chan { - struct nvkm_fifo_chan base; - -#define NV50_FIFO_ENGN_SW 0 -#define NV50_FIFO_ENGN_GR 1 -#define NV50_FIFO_ENGN_MPEG 2 -#define NV50_FIFO_ENGN_DMA 3 - -#define G84_FIFO_ENGN_SW 0 -#define G84_FIFO_ENGN_GR 1 -#define G84_FIFO_ENGN_MPEG 2 -#define G84_FIFO_ENGN_MSPPP 2 -#define G84_FIFO_ENGN_ME 3 -#define G84_FIFO_ENGN_CE0 3 -#define G84_FIFO_ENGN_VP 4 -#define G84_FIFO_ENGN_MSPDEC 4 -#define G84_FIFO_ENGN_CIPHER 5 -#define G84_FIFO_ENGN_SEC 5 -#define G84_FIFO_ENGN_VIC 5 -#define G84_FIFO_ENGN_BSP 6 -#define G84_FIFO_ENGN_MSVLD 6 -#define G84_FIFO_ENGN_DMA 7 -}; - -int nv50_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, - const struct nvkm_oclass *, struct nv50_fifo_chan *); -void *nv50_fifo_chan_dtor(struct nvkm_fifo_chan *); - -int g84_fifo_chan_ctor(struct nv50_fifo *, u64 vmm, u64 push, - const struct nvkm_oclass *, struct nv50_fifo_chan *); - -extern const struct nvkm_fifo_chan_oclass nv50_fifo_gpfifo_oclass; -extern const struct nvkm_fifo_chan_oclass g84_fifo_gpfifo_oclass; -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c deleted file mode 100644 index 41f7f22ed935e..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv04.c +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv04.h" -#include "regsnv04.h" - -#include -#include -#include - -#include -#include - -void * -nv04_fifo_dma_dtor(struct nvkm_fifo_chan *base) -{ - struct nv04_fifo_chan *chan = nv04_fifo_chan(base); - return chan; -} - -const struct nvkm_fifo_chan_func -nv04_fifo_dma_func = { - .dtor = nv04_fifo_dma_dtor, -}; - -static int -nv04_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct nv03_channel_dma_v0 v0; - } *args = data; - struct nv04_fifo *fifo = nv04_fifo(base); - struct nv04_fifo_chan *chan = NULL; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel dma size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx " - "offset %08x\n", args->v0.version, - args->v0.pushbuf, args->v0.offset); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nvkm_fifo_chan_ctor(&nv04_fifo_dma_func, &fifo->base, - 0x1000, 0x1000, false, 0, args->v0.pushbuf, - BIT(NV04_FIFO_ENGN_SW) | - BIT(NV04_FIFO_ENGN_GR) | - BIT(NV04_FIFO_ENGN_DMA), - 0, 0x800000, 0x10000, oclass, &chan->base); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -nv04_fifo_dma_oclass = { - .ctor = nv04_fifo_dma_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c deleted file mode 100644 index 49a000fabc5ae..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv10.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv04.h" -#include "regsnv04.h" - -#include -#include -#include - -#include -#include - -static int -nv10_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct nv03_channel_dma_v0 v0; - } *args = data; - struct nv04_fifo *fifo = nv04_fifo(base); - struct nv04_fifo_chan *chan = NULL; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel dma size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx " - "offset %08x\n", args->v0.version, - args->v0.pushbuf, args->v0.offset); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nvkm_fifo_chan_ctor(&nv04_fifo_dma_func, &fifo->base, - 0x1000, 0x1000, false, 0, args->v0.pushbuf, - BIT(NV04_FIFO_ENGN_SW) | - BIT(NV04_FIFO_ENGN_GR) | - BIT(NV04_FIFO_ENGN_DMA), - 0, 0x800000, 0x10000, oclass, &chan->base); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -nv10_fifo_dma_oclass = { - .ctor = nv10_fifo_dma_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c deleted file mode 100644 index cad8c5ec8dbb4..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv17.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv04.h" -#include "regsnv04.h" - -#include -#include -#include - -#include -#include - -static int -nv17_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct nv03_channel_dma_v0 v0; - } *args = data; - struct nv04_fifo *fifo = nv04_fifo(base); - struct nv04_fifo_chan *chan = NULL; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel dma size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx " - "offset %08x\n", args->v0.version, - args->v0.pushbuf, args->v0.offset); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nvkm_fifo_chan_ctor(&nv04_fifo_dma_func, &fifo->base, - 0x1000, 0x1000, false, 0, args->v0.pushbuf, - BIT(NV04_FIFO_ENGN_SW) | - BIT(NV04_FIFO_ENGN_GR) | - BIT(NV04_FIFO_ENGN_MPEG) | /* NV31- */ - BIT(NV04_FIFO_ENGN_DMA), - 0, 0x800000, 0x10000, oclass, &chan->base); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -nv17_fifo_dma_oclass = { - .ctor = nv17_fifo_dma_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c deleted file mode 100644 index e922e4a5ab1ac..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/dmanv40.c +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv04.h" -#include "regsnv04.h" - -#include -#include -#include - -#include -#include - -static const struct nvkm_fifo_chan_func -nv40_fifo_dma_func = { -}; - -static int -nv40_fifo_dma_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct nv03_channel_dma_v0 v0; - } *args = data; - struct nv04_fifo *fifo = nv04_fifo(base); - struct nv04_fifo_chan *chan = NULL; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel dma size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel dma vers %d pushbuf %llx " - "offset %08x\n", args->v0.version, - args->v0.pushbuf, args->v0.offset); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nvkm_fifo_chan_ctor(&nv40_fifo_dma_func, &fifo->base, - 0x1000, 0x1000, false, 0, args->v0.pushbuf, - BIT(NV04_FIFO_ENGN_SW) | - BIT(NV04_FIFO_ENGN_GR) | - BIT(NV04_FIFO_ENGN_MPEG) | - BIT(NV04_FIFO_ENGN_DMA), - 0, 0xc00000, 0x1000, oclass, &chan->base); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - chan->base.func->ramfc->write(&chan->base, args->v0.offset, 0, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -nv40_fifo_dma_oclass = { - .ctor = nv40_fifo_dma_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c index bc72c536f60d8..6b229a3fbd97e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g84.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "runl.h" @@ -28,9 +29,6 @@ #include #include -#include "nv50.h" -#include "channv50.h" - #include static void @@ -210,7 +208,6 @@ g84_fifo_runl_ctor(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func g84_fifo = { - .dtor = nv50_fifo_dtor, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, .runl_ctor = g84_fifo_runl_ctor, @@ -223,12 +220,12 @@ g84_fifo = { .engn = &g84_engn, .engn_sw = &nv50_engn_sw, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass }, + .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan }, }; int g84_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv50_fifo_new_(&g84_fifo, device, type, inst, pfifo); + return nvkm_fifo_new_(&g84_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c index b2634f21d3207..c6ca050c38bf6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/g98.c @@ -19,12 +19,10 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "priv.h" #include "chan.h" #include "runl.h" -#include "nv50.h" -#include "channv50.h" - #include static int @@ -49,7 +47,6 @@ g98_fifo_runl_ctor(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func g98_fifo = { - .dtor = nv50_fifo_dtor, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, .runl_ctor = g98_fifo_runl_ctor, @@ -62,12 +59,12 @@ g98_fifo = { .engn = &g84_engn, .engn_sw = &nv50_engn_sw, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan, .oclass = &g84_fifo_gpfifo_oclass }, + .chan = {{ 0, 0, G82_CHANNEL_GPFIFO }, &g84_chan }, }; int g98_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv50_fifo_new_(&g98_fifo, device, type, inst, pfifo); + return nvkm_fifo_new_(&g98_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c index e228ca3a52032..b0c7b562bfc84 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c @@ -29,7 +29,7 @@ #include #include -#include +#include #include struct ga102_fifo { @@ -148,9 +148,10 @@ static int ga102_chan_new(struct nvkm_device *device, const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) { - struct volta_channel_gpfifo_a_v0 *args = argv; + struct nvif_chan_v0 *args = argv; struct nvkm_top_device *tdev; struct nvkm_vmm *vmm; + struct nvkm_memory *userd; struct ga102_chan *chan; int ret; @@ -194,9 +195,9 @@ ga102_chan_new(struct nvkm_device *device, nvkm_kmap(chan->inst); nvkm_wo32(chan->inst, 0x010, 0x0000face); nvkm_wo32(chan->inst, 0x030, 0x7ffff902); - nvkm_wo32(chan->inst, 0x048, lower_32_bits(args->ioffset)); - nvkm_wo32(chan->inst, 0x04c, upper_32_bits(args->ioffset) | - (order_base_2(args->ilength / 8) << 16)); + nvkm_wo32(chan->inst, 0x048, lower_32_bits(args->offset)); + nvkm_wo32(chan->inst, 0x04c, upper_32_bits(args->offset) | + (order_base_2(args->length / 8) << 16)); nvkm_wo32(chan->inst, 0x084, 0x20400000); nvkm_wo32(chan->inst, 0x094, 0x30000001); nvkm_wo32(chan->inst, 0x0ac, 0x00020000); @@ -209,7 +210,12 @@ ga102_chan_new(struct nvkm_device *device, nvkm_wo32(chan->inst, 0x224, upper_32_bits(nvkm_memory_bar2(chan->mthd))); nvkm_done(chan->inst); - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->user); + userd = nvkm_umem_search(oclass->client, args->huserd); + if (IS_ERR(userd)) + return PTR_ERR(userd); + + ret = nvkm_memory_kmap(userd, &chan->user); + nvkm_memory_unref(&userd); if (ret) return ret; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c index 2e2626168bf29..5bb65258c36d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c @@ -21,19 +21,18 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" #include "runq.h" -#include "gf100.h" -#include "changf100.h" - #include #include #include #include +#include #include #include @@ -942,16 +941,8 @@ gf100_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) return nvkm_chid_new(&nvkm_chan_event, &fifo->engine.subdev, nr, 0, nr, &fifo->chid); } -static void * -gf100_fifo_dtor(struct nvkm_fifo *base) -{ - struct gf100_fifo *fifo = gf100_fifo(base); - return fifo; -} - static const struct nvkm_fifo_func gf100_fifo = { - .dtor = gf100_fifo_dtor, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, @@ -967,18 +958,12 @@ gf100_fifo = { .runq = &gf100_runq, .engn = &gf100_engn, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan, .oclass = &gf100_fifo_gpfifo_oclass }, + .chan = {{ 0, 0, FERMI_CHANNEL_GPFIFO }, &gf100_chan }, }; int gf100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - struct gf100_fifo *fifo; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - *pfifo = &fifo->base; - - return nvkm_fifo_ctor(&gf100_fifo, device, type, inst, &fifo->base); + return nvkm_fifo_new_(&gf100_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h deleted file mode 100644 index 6027ee7dce5ad..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.h +++ /dev/null @@ -1,12 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __GF100_FIFO_H__ -#define __GF100_FIFO_H__ -#define gf100_fifo(p) container_of((p), struct gf100_fifo, base) -#include "priv.h" - -#include - -struct gf100_fifo { - struct nvkm_fifo base; -}; -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c index 6130f492f48a3..d8a4d773a58ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c @@ -21,17 +21,16 @@ * * Authors: Ben Skeggs */ +#include "priv.h" +#include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" #include "runq.h" -#include "gk104.h" -#include "cgrp.h" -#include "changk104.h" - #include #include +#include #include #include @@ -805,56 +804,8 @@ gk104_fifo_chid_nr(struct nvkm_fifo *fifo) return 4096; } -int -gk104_fifo_oneinit(struct nvkm_fifo *base) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - struct nvkm_subdev *subdev = &fifo->base.engine.subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_top_device *tdev; - - /* Determine runlist configuration from topology device info. */ - list_for_each_entry(tdev, &device->top->device, head) { - const int engn = tdev->engine; - - if (engn < 0) - continue; - - fifo->runlist[tdev->runlist].engm |= BIT(engn); - fifo->runlist[tdev->runlist].engm_sw |= BIT(engn); - if (tdev->type == NVKM_ENGINE_GR) - fifo->runlist[tdev->runlist].engm_sw |= BIT(GK104_FIFO_ENGN_SW); - fifo->runlist_nr = max(fifo->runlist_nr, tdev->runlist + 1); - } - - return 0; -} - -void * -gk104_fifo_dtor(struct nvkm_fifo *base) -{ - struct gk104_fifo *fifo = gk104_fifo(base); - return fifo; -} - -int -gk104_fifo_new_(const struct gk104_fifo_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, int nr, struct nvkm_fifo **pfifo) -{ - struct gk104_fifo *fifo; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - fifo->func = func; - *pfifo = &fifo->base; - - return nvkm_fifo_ctor(func, device, type, inst, &fifo->base); -} - static const struct nvkm_fifo_func gk104_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, .chid_ctor = gf100_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, @@ -871,12 +822,12 @@ gk104_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk104_chan }, }; int gk104_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk104_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gk104_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h deleted file mode 100644 index 97fc9ccc2930d..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __GK104_FIFO_H__ -#define __GK104_FIFO_H__ -#define gk104_fifo(p) container_of((p), struct gk104_fifo, base) -#include "priv.h" -struct nvkm_fifo_cgrp; - -#include - -#define gk104_fifo_func nvkm_fifo_func - -struct gk104_fifo_chan; -struct gk104_fifo { - const struct gk104_fifo_func *func; - struct nvkm_fifo base; - - struct { - u32 engm; - u32 engm_sw; - } runlist[16]; - int runlist_nr; -}; - -int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, - int index, int nr, struct nvkm_fifo **); -void *gk104_fifo_dtor(struct nvkm_fifo *base); -int gk104_fifo_oneinit(struct nvkm_fifo *); -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c index 2afbaf42572d3..a8ff21cf7712e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk110.c @@ -21,14 +21,12 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" -#include "gk104.h" -#include "changk104.h" - #include #include @@ -107,8 +105,6 @@ gk110_fifo_chid_ctor(struct nvkm_fifo *fifo, int nr) static const struct nvkm_fifo_func gk110_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gk104_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, @@ -125,12 +121,12 @@ gk110_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, - .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gk110_chan }, }; int gk110_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk110_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gk110_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c index 3727be30d7f4f..8fa2b0be141a5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk208.c @@ -21,9 +21,8 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "runq.h" -#include "gk104.h" -#include "changk104.h" #include @@ -51,8 +50,6 @@ gk208_fifo_chid_nr(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func gk208_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gk208_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, @@ -69,12 +66,12 @@ gk208_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, - .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan }, }; int gk208_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk208_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gk208_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c index eda1446dce27c..b63ca836130f9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk20a.c @@ -19,15 +19,12 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ -#include "gk104.h" -#include "changk104.h" +#include "priv.h" #include static const struct nvkm_fifo_func gk20a_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, @@ -44,12 +41,12 @@ gk20a_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ }, &gk110_cgrp }, - .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_A }, &gk110_chan }, }; int gk20a_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gk20a_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gk20a_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c index 1089062313cc6..5ba60021b510e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm107.c @@ -24,8 +24,6 @@ #include "priv.h" #include "chan.h" #include "runl.h" -#include "gk104.h" -#include "changk104.h" #include #include @@ -131,8 +129,6 @@ gm107_fifo_chid_nr(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func gm107_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gm107_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gf100_fifo_runq_nr, @@ -149,12 +145,12 @@ gm107_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, - .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, KEPLER_CHANNEL_GPFIFO_B }, &gm107_chan }, }; int gm107_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gm107_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gm107_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c index 286c1098b6659..d92d1ac391917 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gm200.c @@ -21,8 +21,7 @@ * * Authors: Ben Skeggs */ -#include "gk104.h" -#include "changk104.h" +#include "priv.h" #include @@ -40,8 +39,6 @@ gm200_fifo_chid_nr(struct nvkm_fifo *fifo) static const struct nvkm_fifo_func gm200_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, @@ -58,12 +55,12 @@ gm200_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp }, - .chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, MAXWELL_CHANNEL_GPFIFO_A }, &gm107_chan }, }; int gm200_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gm200_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gm200_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c index 45d536d448f0c..65bdb6a7d5172 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c @@ -21,9 +21,8 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "runl.h" -#include "gk104.h" -#include "changk104.h" #include #include @@ -113,8 +112,6 @@ gp100_fifo_intr_mmu_fault_unit(struct nvkm_fifo *fifo, int unit) static const struct nvkm_fifo_func gp100_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, @@ -131,12 +128,12 @@ gp100_fifo = { .engn = &gk104_engn, .engn_ce = &gk104_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, - .chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan, .ctor = &gk104_fifo_gpfifo_new }, + .chan = {{ 0, 0, PASCAL_CHANNEL_GPFIFO_A }, &gm107_chan }, }; int gp100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gp100_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gp100_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c deleted file mode 100644 index 31022441ee8fc..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifog84.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv50.h" - -#include -#include - -#include -#include - -static int -g84_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct g82_channel_gpfifo_v0 v0; - } *args = data; - struct nv50_fifo *fifo = nv50_fifo(base); - struct nv50_fifo_chan *chan; - u64 ioffset, ilength; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel gpfifo size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " - "pushbuf %llx ioffset %016llx " - "ilength %08x\n", - args->v0.version, args->v0.vmm, args->v0.pushbuf, - args->v0.ioffset, args->v0.ilength); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = g84_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, - oclass, chan); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - ioffset = args->v0.ioffset; - ilength = args->v0.ilength; - - chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -g84_fifo_gpfifo_oclass = { - .ctor = g84_fifo_gpfifo_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c deleted file mode 100644 index f87a54e2f5758..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogf100.c +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "changf100.h" - -#include -#include -#include -#include - -#include -#include - -static void * -gf100_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) -{ - return gf100_fifo_chan(base); -} - -static const struct nvkm_fifo_chan_func -gf100_fifo_gpfifo_func = { - .dtor = gf100_fifo_gpfifo_dtor, -}; - -static int -gf100_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - union { - struct fermi_channel_gpfifo_v0 v0; - } *args = data; - struct gf100_fifo *fifo = gf100_fifo(base); - struct nvkm_object *parent = oclass->parent; - struct gf100_fifo_chan *chan; - u64 usermem, ioffset, ilength; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel gpfifo size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " - "ioffset %016llx ilength %08x\n", - args->v0.version, args->v0.vmm, args->v0.ioffset, - args->v0.ilength); - if (!args->v0.vmm) - return -EINVAL; - } else - return ret; - - /* allocate channel */ - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nvkm_fifo_chan_ctor(&gf100_fifo_gpfifo_func, &fifo->base, - 0x1000, 0x1000, true, args->v0.vmm, 0, - BIT(GF100_FIFO_ENGN_GR) | - BIT(GF100_FIFO_ENGN_MSPDEC) | - BIT(GF100_FIFO_ENGN_MSPPP) | - BIT(GF100_FIFO_ENGN_MSVLD) | - BIT(GF100_FIFO_ENGN_CE0) | - BIT(GF100_FIFO_ENGN_CE1) | - BIT(GF100_FIFO_ENGN_SW), - 0, 0, 0, - oclass, &chan->base); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - - usermem = nvkm_memory_addr(chan->base.userd.mem) + chan->base.userd.base; - ioffset = args->v0.ioffset; - ilength = args->v0.ilength; - - chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -gf100_fifo_gpfifo_oclass = { - .ctor = gf100_fifo_gpfifo_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c deleted file mode 100644 index 5036edb62dbd2..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "changk104.h" -#include "cgrp.h" - -#include -#include -#include -#include -#include - -#include -#include - -void * -gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) -{ - struct gk104_fifo_chan *chan = gk104_fifo_chan(base); - return chan; -} - -const struct nvkm_fifo_chan_func -gk104_fifo_gpfifo_func = { - .dtor = gk104_fifo_gpfifo_dtor, -}; - -static int -gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid, - u64 vmm, u64 ioffset, u64 ilength, u64 *inst, bool priv, - const struct nvkm_oclass *oclass, - struct nvkm_object **pobject) -{ - struct gk104_fifo_chan *chan; - int runlist = ffs(*runlists) -1, ret; - - if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) - return -EINVAL; - *runlists = BIT_ULL(runlist); - - /* Allocate the channel. */ - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - chan->runl = runlist; - - ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, - 0x1000, 0x1000, true, vmm, 0, fifo->runlist[runlist].engm_sw, - 0, 0, 0, - oclass, &chan->base); - if (ret) - return ret; - - *chid = chan->base.chid; - *inst = chan->base.inst->addr; - - chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), priv); - return 0; -} - -int -gk104_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct kepler_channel_gpfifo_a_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel gpfifo size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " - "ioffset %016llx ilength %08x " - "runlist %016llx priv %d\n", - args->v0.version, args->v0.vmm, args->v0.ioffset, - args->v0.ilength, args->v0.runlist, args->v0.priv); - return gk104_fifo_gpfifo_new_(fifo, - &args->v0.runlist, - &args->v0.chid, - args->v0.vmm, - args->v0.ioffset, - args->v0.ilength, - &args->v0.inst, - args->v0.priv, - oclass, pobject); - } - - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c deleted file mode 100644 index a97ec53d7e6ee..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogv100.c +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2018 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#include "changk104.h" -#include "cgrp.h" - -#include -#include - -#include -#include - -static const struct nvkm_fifo_chan_func -gv100_fifo_gpfifo = { - .dtor = gk104_fifo_gpfifo_dtor, -}; - -int -gv100_fifo_gpfifo_new_(const struct nvkm_fifo_chan_func *func, - struct gk104_fifo *fifo, u64 *runlists, u16 *chid, - u64 vmm, u64 ioffset, u64 ilength, u64 *inst, bool priv, - u32 *token, const struct nvkm_oclass *oclass, - struct nvkm_object **pobject) -{ - struct gk104_fifo_chan *chan; - int runlist = ffs(*runlists) -1, ret; - - if (!vmm || runlist < 0 || runlist >= fifo->runlist_nr) - return -EINVAL; - *runlists = BIT_ULL(runlist); - - /* Allocate the channel. */ - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - chan->runl = runlist; - - ret = nvkm_fifo_chan_ctor(func, &fifo->base, 0x1000, 0x1000, true, vmm, - 0, fifo->runlist[runlist].engm, 0, 0, 0, - oclass, &chan->base); - if (ret) - return ret; - - *chid = chan->base.chid; - *inst = chan->base.inst->addr; - *token = chan->base.func->doorbell_handle(&chan->base); - - chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), priv); - return 0; -} - -int -gv100_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct volta_channel_gpfifo_a_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel gpfifo size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " - "ioffset %016llx ilength %08x " - "runlist %016llx priv %d\n", - args->v0.version, args->v0.vmm, args->v0.ioffset, - args->v0.ilength, args->v0.runlist, args->v0.priv); - return gv100_fifo_gpfifo_new_(&gv100_fifo_gpfifo, fifo, - &args->v0.runlist, - &args->v0.chid, - args->v0.vmm, - args->v0.ioffset, - args->v0.ilength, - &args->v0.inst, - args->v0.priv, - &args->v0.token, - oclass, pobject); - } - - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c deleted file mode 100644 index 88ea7c29941ed..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifonv50.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2012 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: Ben Skeggs - */ -#include "channv50.h" - -#include -#include - -#include -#include - -static int -nv50_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct nv50_channel_gpfifo_v0 v0; - } *args = data; - struct nv50_fifo *fifo = nv50_fifo(base); - struct nv50_fifo_chan *chan; - u64 ioffset, ilength; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel gpfifo size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " - "pushbuf %llx ioffset %016llx " - "ilength %08x\n", - args->v0.version, args->v0.vmm, args->v0.pushbuf, - args->v0.ioffset, args->v0.ilength); - if (!args->v0.pushbuf) - return -EINVAL; - } else - return ret; - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - *pobject = &chan->base.object; - - ret = nv50_fifo_chan_ctor(fifo, args->v0.vmm, args->v0.pushbuf, - oclass, chan); - if (ret) - return ret; - - args->v0.chid = chan->base.chid; - ioffset = args->v0.ioffset; - ilength = args->v0.ilength; - - chan->base.func->ramfc->write(&chan->base, ioffset, ilength, BIT(0), false); - return 0; -} - -const struct nvkm_fifo_chan_oclass -nv50_fifo_gpfifo_oclass = { - .ctor = nv50_fifo_gpfifo_new, -}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c deleted file mode 100644 index 62a0f9664a9d1..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifotu102.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2018 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#include "changk104.h" -#include "cgrp.h" - -#include -#include - -#include -#include - -static const struct nvkm_fifo_chan_func -tu102_fifo_gpfifo = { - .dtor = gk104_fifo_gpfifo_dtor, -}; - -int -tu102_fifo_gpfifo_new(struct gk104_fifo *fifo, const struct nvkm_oclass *oclass, - void *data, u32 size, struct nvkm_object **pobject) -{ - struct nvkm_object *parent = oclass->parent; - union { - struct volta_channel_gpfifo_a_v0 v0; - } *args = data; - int ret = -ENOSYS; - - nvif_ioctl(parent, "create channel gpfifo size %d\n", size); - if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { - nvif_ioctl(parent, "create channel gpfifo vers %d vmm %llx " - "ioffset %016llx ilength %08x " - "runlist %016llx priv %d\n", - args->v0.version, args->v0.vmm, args->v0.ioffset, - args->v0.ilength, args->v0.runlist, args->v0.priv); - return gv100_fifo_gpfifo_new_(&tu102_fifo_gpfifo, fifo, - &args->v0.runlist, - &args->v0.chid, - args->v0.vmm, - args->v0.ioffset, - args->v0.ilength, - &args->v0.inst, - args->v0.priv, - &args->v0.token, - oclass, pobject); - } - - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c index d55dfc93c61e4..33066c8cdc648 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gv100.c @@ -19,16 +19,15 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "priv.h" #include "chan.h" #include "chid.h" #include "cgrp.h" #include "runl.h" #include "runq.h" -#include "gk104.h" -#include "changk104.h" - #include +#include #include @@ -71,7 +70,7 @@ gv100_chan_ramfc = { const struct nvkm_chan_func_userd gv100_chan_userd = { - .bar = 1, /*FIXME: hw doesn't have poller, flip to user-allocated in uapi commit. */ + .bar = -1, .size = 0x200, .clear = gf100_chan_userd_clear, }; @@ -467,8 +466,6 @@ gv100_fifo_intr_ctxsw_timeout(struct nvkm_fifo *fifo, u32 engm) static const struct nvkm_fifo_func gv100_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, @@ -484,12 +481,12 @@ gv100_fifo = { .engn = &gv100_engn, .engn_ce = &gv100_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, - .chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan, .ctor = gv100_fifo_gpfifo_new }, + .chan = {{ 0, 0, VOLTA_CHANNEL_GPFIFO_A }, &gv100_chan }, }; int gv100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return gk104_fifo_new_(&gv100_fifo, device, type, inst, 0, pfifo); + return nvkm_fifo_new_(&gv100_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c index 73ae205fc6baf..674faf002b20b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c @@ -21,13 +21,12 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" -#include "nv04.h" -#include "channv04.h" #include "regsnv04.h" #include @@ -520,25 +519,6 @@ nv04_fifo_chid_nr(struct nvkm_fifo *fifo) return 16; } -int -nv04_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, int nr, const struct nv04_fifo_ramfc *ramfc, - struct nvkm_fifo **pfifo) -{ - struct nv04_fifo *fifo; - int ret; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - *pfifo = &fifo->base; - - ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); - if (ret) - return ret; - - return 0; -} - static const struct nvkm_fifo_func nv04_fifo = { .chid_nr = nv04_fifo_chid_nr, @@ -552,12 +532,12 @@ nv04_fifo = { .engn = &nv04_engn, .engn_sw = &nv04_engn, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan, .oclass = &nv04_fifo_dma_oclass }, + .chan = {{ 0, 0, NV03_CHANNEL_DMA }, &nv04_chan }, }; int nv04_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv04_fifo, device, type, inst, 0, NULL, pfifo); + return nvkm_fifo_new_(&nv04_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h deleted file mode 100644 index 93248ab24a817..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NV04_FIFO_H__ -#define __NV04_FIFO_H__ -#define nv04_fifo(p) container_of((p), struct nv04_fifo, base) -#include "priv.h" - -#define nv04_fifo_ramfc nvkm_ramfc_layout - -struct nv04_fifo { - struct nvkm_fifo base; -}; - -int nv04_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, - int nr, const struct nv04_fifo_ramfc *, struct nvkm_fifo **); -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c index 9658ac3dac91a..a4bcf6b0a7e2c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv10.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "runl.h" @@ -28,8 +29,6 @@ #include #include -#include "nv04.h" -#include "channv04.h" #include "regsnv04.h" #include @@ -103,12 +102,12 @@ nv10_fifo = { .engn = &nv04_engn, .engn_sw = &nv04_engn, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, NV10_CHANNEL_DMA }, &nv10_chan, .oclass = &nv10_fifo_dma_oclass }, + .chan = {{ 0, 0, NV10_CHANNEL_DMA }, &nv10_chan }, }; int nv10_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv10_fifo, device, type, inst, 0, NULL, pfifo); + return nvkm_fifo_new_(&nv10_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c index fb0b106eb09bb..c70f44fd4f3bb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv17.c @@ -21,13 +21,12 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" -#include "nv04.h" -#include "channv04.h" #include "regsnv04.h" #include @@ -132,12 +131,12 @@ nv17_fifo = { .engn = &nv04_engn, .engn_sw = &nv04_engn, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, NV17_CHANNEL_DMA }, &nv17_chan, .oclass = &nv17_fifo_dma_oclass }, + .chan = {{ 0, 0, NV17_CHANNEL_DMA }, &nv17_chan }, }; int nv17_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv17_fifo, device, type, inst, 0, NULL, pfifo); + return nvkm_fifo_new_(&nv17_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c index 7574be50d9595..e50a94b6d7f86 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv40.c @@ -21,13 +21,12 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" #include "runl.h" -#include "nv04.h" -#include "channv04.h" #include "regsnv04.h" #include @@ -242,12 +241,12 @@ nv40_fifo = { .engn = &nv40_engn, .engn_sw = &nv40_engn_sw, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, NV40_CHANNEL_DMA }, &nv40_chan, .oclass = &nv40_fifo_dma_oclass }, + .chan = {{ 0, 0, NV40_CHANNEL_DMA }, &nv40_chan }, }; int nv40_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv04_fifo_new_(&nv40_fifo, device, type, inst, 0, NULL, pfifo); + return nvkm_fifo_new_(&nv40_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c index 829c52a44a36d..954b5f3a7d57a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.c @@ -21,6 +21,7 @@ * * Authors: Ben Skeggs */ +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" @@ -29,9 +30,6 @@ #include #include -#include "nv50.h" -#include "channv50.h" - #include void @@ -376,34 +374,8 @@ nv50_fifo_chid_nr(struct nvkm_fifo *fifo) return 128; } -void * -nv50_fifo_dtor(struct nvkm_fifo *base) -{ - struct nv50_fifo *fifo = nv50_fifo(base); - return fifo; -} - -int -nv50_fifo_new_(const struct nvkm_fifo_func *func, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) -{ - struct nv50_fifo *fifo; - int ret; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - *pfifo = &fifo->base; - - ret = nvkm_fifo_ctor(func, device, type, inst, &fifo->base); - if (ret) - return ret; - - return 0; -} - static const struct nvkm_fifo_func nv50_fifo = { - .dtor = nv50_fifo_dtor, .chid_nr = nv50_fifo_chid_nr, .chid_ctor = nv50_fifo_chid_ctor, .runl_ctor = nv04_fifo_runl_ctor, @@ -415,12 +387,12 @@ nv50_fifo = { .engn = &nv50_engn, .engn_sw = &nv50_engn_sw, .cgrp = {{ }, &nv04_cgrp }, - .chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan, .oclass = &nv50_fifo_gpfifo_oclass }, + .chan = {{ 0, 0, NV50_CHANNEL_GPFIFO }, &nv50_chan }, }; int nv50_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - return nv50_fifo_new_(&nv50_fifo, device, type, inst, pfifo); + return nvkm_fifo_new_(&nv50_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h deleted file mode 100644 index 3c6d67906730c..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv50.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -#ifndef __NV50_FIFO_H__ -#define __NV50_FIFO_H__ -#define nv50_fifo(p) container_of((p), struct nv50_fifo, base) -#include "priv.h" - -struct nv50_fifo { - struct nvkm_fifo base; -}; - -int nv50_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, - struct nvkm_fifo **); - -void *nv50_fifo_dtor(struct nvkm_fifo *); -#endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index a5df5e2d9d4b4..24125e2d5d4f8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -11,13 +11,8 @@ struct nvkm_memory; struct nvkm_runl; struct nvkm_runq; struct nvkm_vctx; -struct gk104_fifo; -struct nvkm_fifo_chan_oclass; struct nvkm_fifo_func { - void *(*dtor)(struct nvkm_fifo *); - - int (*oneinit)(struct nvkm_fifo *); int (*chid_nr)(struct nvkm_fifo *); int (*chid_ctor)(struct nvkm_fifo *, int nr); int (*runq_nr)(struct nvkm_fifo *); @@ -59,17 +54,11 @@ struct nvkm_fifo_func { struct nvkm_fifo_func_chan { struct nvkm_sclass user; const struct nvkm_chan_func *func; - const struct nvkm_fifo_chan_oclass { - int (*ctor)(struct nvkm_fifo *, const struct nvkm_oclass *, - void *data, u32 size, struct nvkm_object **); - } *oclass; - int (*ctor)(struct gk104_fifo *, const struct nvkm_oclass *, void *, u32, - struct nvkm_object **); } chan; }; -int nvkm_fifo_ctor(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, - struct nvkm_fifo *); +int nvkm_fifo_new_(const struct nvkm_fifo_func *, struct nvkm_device *, enum nvkm_subdev_type, int, + struct nvkm_fifo **); int nv04_fifo_chid_ctor(struct nvkm_fifo *, int); int nv04_fifo_runl_ctor(struct nvkm_fifo *); @@ -204,4 +193,6 @@ extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, void *argv, u32 argc, struct nvkm_object **); +int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc, + struct nvkm_object **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c index 608db9b8dc118..ea9e151dbb488 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/tu102.c @@ -19,13 +19,11 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ +#include "priv.h" +#include "cgrp.h" #include "chan.h" #include "runl.h" -#include "gk104.h" -#include "cgrp.h" -#include "changk104.h" - #include #include #include @@ -263,8 +261,6 @@ tu102_fifo_init_pbdmas(struct nvkm_fifo *fifo, u32 mask) static const struct nvkm_fifo_func tu102_fifo = { - .dtor = gk104_fifo_dtor, - .oneinit = gk104_fifo_oneinit, .chid_nr = gm200_fifo_chid_nr, .chid_ctor = gk110_fifo_chid_ctor, .runq_nr = gm200_fifo_runq_nr, @@ -279,19 +275,12 @@ tu102_fifo = { .engn = &gv100_engn, .engn_ce = &gv100_engn_ce, .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &gk110_cgrp, .force = true }, - .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan, .ctor = tu102_fifo_gpfifo_new }, + .chan = {{ 0, 0, TURING_CHANNEL_GPFIFO_A }, &tu102_chan }, }; int tu102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - struct gk104_fifo *fifo; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - fifo->func = &tu102_fifo; - *pfifo = &fifo->base; - - return nvkm_fifo_ctor(&tu102_fifo, device, type, inst, &fifo->base); + return nvkm_fifo_new_(&tu102_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ucgrp.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ucgrp.c new file mode 100644 index 0000000000000..52c594dfb1b8d --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ucgrp.c @@ -0,0 +1,125 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#define nvkm_ucgrp(p) container_of((p), struct nvkm_ucgrp, object) +#include "priv.h" +#include "cgrp.h" +#include "runl.h" + +#include + +#include + +struct nvkm_ucgrp { + struct nvkm_object object; + struct nvkm_cgrp *cgrp; +}; + +static int +nvkm_ucgrp_chan_new(const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + struct nvkm_cgrp *cgrp = nvkm_ucgrp(oclass->parent)->cgrp; + + return nvkm_uchan_new(cgrp->runl->fifo, cgrp, oclass, argv, argc, pobject); +} + +static int +nvkm_ucgrp_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *oclass) +{ + struct nvkm_cgrp *cgrp = nvkm_ucgrp(object)->cgrp; + struct nvkm_fifo *fifo = cgrp->runl->fifo; + const struct nvkm_fifo_func_chan *chan = &fifo->func->chan; + int c = 0; + + /* *_CHANNEL_GPFIFO_* */ + if (chan->user.oclass) { + if (c++ == index) { + oclass->base = chan->user; + oclass->ctor = nvkm_ucgrp_chan_new; + return 0; + } + } + + return -EINVAL; +} + +static void * +nvkm_ucgrp_dtor(struct nvkm_object *object) +{ + struct nvkm_ucgrp *ucgrp = nvkm_ucgrp(object); + + nvkm_cgrp_unref(&ucgrp->cgrp); + return ucgrp; +} + +static const struct nvkm_object_func +nvkm_ucgrp = { + .dtor = nvkm_ucgrp_dtor, + .sclass = nvkm_ucgrp_sclass, +}; + +int +nvkm_ucgrp_new(struct nvkm_fifo *fifo, const struct nvkm_oclass *oclass, void *argv, u32 argc, + struct nvkm_object **pobject) +{ + union nvif_cgrp_args *args = argv; + struct nvkm_runl *runl; + struct nvkm_vmm *vmm; + struct nvkm_ucgrp *ucgrp; + int ret; + + if (argc < sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + argc -= sizeof(args->v0); + + if (args->v0.namelen != argc) + return -EINVAL; + + /* Lookup objects referenced in args. */ + runl = nvkm_runl_get(fifo, args->v0.runlist, 0); + if (!runl) + return -EINVAL; + + vmm = nvkm_uvmm_search(oclass->client, args->v0.vmm); + if (IS_ERR(vmm)) + return PTR_ERR(vmm); + + /* Allocate channel group. */ + if (!(ucgrp = kzalloc(sizeof(*ucgrp), GFP_KERNEL))) { + ret = -ENOMEM; + goto done; + } + + nvkm_object_ctor(&nvkm_ucgrp, oclass, &ucgrp->object); + *pobject = &ucgrp->object; + + ret = nvkm_cgrp_new(runl, args->v0.name, vmm, true, &ucgrp->cgrp); + if (ret) + goto done; + + /* Return channel group info to caller. */ + args->v0.cgid = ucgrp->cgrp->id; + +done: + nvkm_vmm_unref(&vmm); + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c index 1a89c9313662a..1dac95ae7b43e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/uchan.c @@ -20,6 +20,7 @@ * OTHER DEALINGS IN THE SOFTWARE. */ #define nvkm_uchan(p) container_of((p), struct nvkm_uchan, object) +#include "priv.h" #include "cgrp.h" #include "chan.h" #include "chid.h" @@ -27,11 +28,11 @@ #include #include +#include +#include #include -#include "gk104.h" - struct nvkm_uchan { struct nvkm_object object; struct nvkm_chan *chan; @@ -204,12 +205,20 @@ nvkm_uchan_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *ocl { struct nvkm_chan *chan = nvkm_uchan(object)->chan; struct nvkm_engn *engn; - int ret; + int ret, runq = 0; nvkm_runl_foreach_engn(engn, chan->cgrp->runl) { struct nvkm_engine *engine = engn->engine; int c = 0; + /* Each runqueue, on runlists with multiple, has its own LCE. */ + if (engn->runl->func->runqs) { + if (engine->subdev.type == NVKM_ENGINE_CE) { + if (chan->runq != runq++) + continue; + } + } + oclass->engine = engine; oclass->base.oclass = 0; @@ -246,8 +255,8 @@ static int nvkm_uchan_map(struct nvkm_object *object, void *argv, u32 argc, enum nvkm_object_map *type, u64 *addr, u64 *size) { - struct nvkm_device *device = object->engine->subdev.device; struct nvkm_chan *chan = nvkm_uchan(object)->chan; + struct nvkm_device *device = chan->cgrp->runl->fifo->engine.subdev.device; if (chan->func->userd->bar < 0) return -ENOSYS; @@ -312,23 +321,89 @@ int nvkm_uchan_new(struct nvkm_fifo *fifo, struct nvkm_cgrp *cgrp, const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) { - struct nvkm_object *object = NULL; + union nvif_chan_args *args = argv; + struct nvkm_runl *runl; + struct nvkm_vmm *vmm = NULL; + struct nvkm_dmaobj *ctxdma = NULL; + struct nvkm_memory *userd = NULL; struct nvkm_uchan *uchan; + struct nvkm_chan *chan; int ret; - if (!(uchan = kzalloc(sizeof(*uchan), GFP_KERNEL))) - return -ENOMEM; + if (argc < sizeof(args->v0) || args->v0.version != 0) + return -ENOSYS; + argc -= sizeof(args->v0); + + if (args->v0.namelen != argc) + return -EINVAL; + + /* Lookup objects referenced in args. */ + runl = nvkm_runl_get(fifo, args->v0.runlist, 0); + if (!runl) + return -EINVAL; + + if (args->v0.vmm) { + vmm = nvkm_uvmm_search(oclass->client, args->v0.vmm); + if (IS_ERR(vmm)) + return PTR_ERR(vmm); + } + + if (args->v0.ctxdma) { + ctxdma = nvkm_dmaobj_search(oclass->client, args->v0.ctxdma); + if (IS_ERR(ctxdma)) { + ret = PTR_ERR(ctxdma); + goto done; + } + } + + if (args->v0.huserd) { + userd = nvkm_umem_search(oclass->client, args->v0.huserd); + if (IS_ERR(userd)) { + ret = PTR_ERR(userd); + userd = NULL; + goto done; + } + } + + /* Allocate channel. */ + if (!(uchan = kzalloc(sizeof(*uchan), GFP_KERNEL))) { + ret = -ENOMEM; + goto done; + } nvkm_object_ctor(&nvkm_uchan, oclass, &uchan->object); *pobject = &uchan->object; - if (fifo->func->chan.ctor) - ret = fifo->func->chan.ctor(gk104_fifo(fifo), oclass, argv, argc, &object); + ret = nvkm_chan_new_(fifo->func->chan.func, runl, args->v0.runq, cgrp, args->v0.name, + args->v0.priv != 0, args->v0.devm, vmm, ctxdma, args->v0.offset, + args->v0.length, userd, args->v0.ouserd, &uchan->chan); + if (ret) + goto done; + + chan = uchan->chan; + + /* Return channel info to caller. */ + if (chan->func->doorbell_handle) + args->v0.token = chan->func->doorbell_handle(chan); else - ret = fifo->func->chan.oclass->ctor(fifo, oclass, argv, argc, &object); - if (!object) - return ret; + args->v0.token = ~0; + + args->v0.chid = chan->id; + + switch (nvkm_memory_target(chan->inst->memory)) { + case NVKM_MEM_TARGET_INST: args->v0.aper = NVIF_CHAN_V0_INST_APER_INST; break; + case NVKM_MEM_TARGET_VRAM: args->v0.aper = NVIF_CHAN_V0_INST_APER_VRAM; break; + case NVKM_MEM_TARGET_HOST: args->v0.aper = NVIF_CHAN_V0_INST_APER_HOST; break; + case NVKM_MEM_TARGET_NCOH: args->v0.aper = NVIF_CHAN_V0_INST_APER_NCOH; break; + default: + WARN_ON(1); + ret = -EFAULT; + break; + } - uchan->chan = container_of(object, typeof(*uchan->chan), object); + args->v0.inst = nvkm_memory_addr(chan->inst->memory); +done: + nvkm_memory_unref(&userd); + nvkm_vmm_unref(&vmm); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c index 186b4e63e559b..524cd3c0e3fec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/uvmm.c @@ -39,7 +39,7 @@ nvkm_uvmm_search(struct nvkm_client *client, u64 handle) if (IS_ERR(object)) return (void *)object; - return nvkm_uvmm(object)->vmm; + return nvkm_vmm_ref(nvkm_uvmm(object)->vmm); } static int -- GitLab From 05d271c32e12c16d18b096c797eef809fd6c9215 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:38 +1000 Subject: [PATCH 1336/1927] drm/nouveau/ce/ga100-: initial support - replaces the hacked-up version that existed solely to support TTM - noop until the next commit, adding proper support for ampere host v2. fixup for ga103 early merge Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + .../gpu/drm/nouveau/include/nvkm/engine/ce.h | 2 + drivers/gpu/drm/nouveau/nouveau_bo.c | 3 + drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/engine/ce/ga100.c | 82 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/ce/ga102.c | 44 ++++++++++ drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h | 4 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 6 ++ 8 files changed, 144 insertions(+) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/ce/ga102.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index ceea074b064b5..36c59c7afd954 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -216,6 +216,7 @@ #define PASCAL_DMA_COPY_B 0x0000c1b5 #define VOLTA_DMA_COPY_A 0x0000c3b5 #define TURING_DMA_COPY_A 0x0000c5b5 +#define AMPERE_DMA_COPY_A 0x0000c6b5 #define AMPERE_DMA_COPY_B 0x0000c7b5 #define FERMI_DECOMPRESS 0x000090b8 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h index cfd2da8e66fed..b616a1e8ca02f 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h @@ -12,4 +12,6 @@ int gp100_ce_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct n int gp102_ce_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_engine **); int gv100_ce_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_engine **); int tu102_ce_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_engine **); +int ga100_ce_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_engine **); +int ga102_ce_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_engine **); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 813937ad1dc2e..a11871e3119ce 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -856,6 +856,9 @@ nouveau_bo_move_init(struct nouveau_drm *drm) int (*init)(struct nouveau_channel *, u32 handle); } _methods[] = { { "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, + { "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, + { "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, + { "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild index ba88613e1e46b..8bf1635ffabc0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild @@ -8,3 +8,5 @@ nvkm-y += nvkm/engine/ce/gp100.o nvkm-y += nvkm/engine/ce/gp102.o nvkm-y += nvkm/engine/ce/gv100.o nvkm-y += nvkm/engine/ce/tu102.o +nvkm-y += nvkm/engine/ce/ga100.o +nvkm-y += nvkm/engine/ce/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c new file mode 100644 index 0000000000000..6648ed62daa68 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga100.c @@ -0,0 +1,82 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include + +#include + +static irqreturn_t +ga100_ce_intr(struct nvkm_inth *inth) +{ + struct nvkm_subdev *subdev = container_of(inth, typeof(*subdev), inth); + + /*TODO*/ + nvkm_error(subdev, "intr\n"); + return IRQ_NONE; +} + +int +ga100_ce_fini(struct nvkm_engine *engine, bool suspend) +{ + nvkm_inth_block(&engine->subdev.inth); + return 0; +} + +int +ga100_ce_init(struct nvkm_engine *engine) +{ + nvkm_inth_allow(&engine->subdev.inth); + return 0; +} + +int +ga100_ce_oneinit(struct nvkm_engine *engine) +{ + struct nvkm_subdev *subdev = &engine->subdev; + struct nvkm_device *device = subdev->device; + u32 vector; + + vector = nvkm_rd32(device, 0x10442c + (subdev->inst * 0x80)) & 0x00000fff; + + return nvkm_inth_add(&device->vfn->intr, vector, NVKM_INTR_PRIO_NORMAL, + subdev, ga100_ce_intr, &subdev->inth); +} + +static const struct nvkm_engine_func +ga100_ce = { + .oneinit = ga100_ce_oneinit, + .init = ga100_ce_init, + .fini = ga100_ce_fini, + .cclass = &gv100_ce_cclass, + .sclass = { + { -1, -1, AMPERE_DMA_COPY_A }, + {} + } +}; + +int +ga100_ce_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_engine **pengine) +{ + return nvkm_engine_new_(&ga100_ce, device, type, inst, true, pengine); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga102.c new file mode 100644 index 0000000000000..9f3448ad625f3 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/ga102.c @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include + +static const struct nvkm_engine_func +ga102_ce = { + .oneinit = ga100_ce_oneinit, + .init = ga100_ce_init, + .fini = ga100_ce_fini, + .cclass = &gv100_ce_cclass, + .sclass = { + { -1, -1, AMPERE_DMA_COPY_A }, + { -1, -1, AMPERE_DMA_COPY_B }, + {} + } +}; + +int +ga102_ce_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_engine **pengine) +{ + return nvkm_engine_new_(&ga102_ce, device, type, inst, true, pengine); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h index cd53b93664d64..c4c046916fa6e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h @@ -8,4 +8,8 @@ void gk104_ce_intr(struct nvkm_engine *); void gp100_ce_intr(struct nvkm_engine *); extern const struct nvkm_object_func gv100_ce_cclass; + +int ga100_ce_oneinit(struct nvkm_engine *); +int ga100_ce_init(struct nvkm_engine *); +int ga100_ce_fini(struct nvkm_engine *, bool); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 4f5921a2aeb04..490fac3a4a3a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2589,6 +2589,7 @@ nv170_chipset = { .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, + .ce = { 0x000003ff, ga100_ce_new }, }; static const struct nvkm_device_chip @@ -2609,6 +2610,7 @@ nv172_chipset = { .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, + .ce = { 0x0000001f, ga102_ce_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2632,6 +2634,7 @@ nv173_chipset = { .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, + .ce = { 0x0000001f, ga102_ce_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2655,6 +2658,7 @@ nv174_chipset = { .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, + .ce = { 0x0000001f, ga102_ce_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2678,6 +2682,7 @@ nv176_chipset = { .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, + .ce = { 0x0000001f, ga102_ce_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, @@ -2701,6 +2706,7 @@ nv177_chipset = { .timer = { 0x00000001, gk20a_timer_new }, .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, + .ce = { 0x0000001f, ga102_ce_new }, .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, -- GitLab From 7f4f35ea5b080e6aeb159b4be023a44a527906ec Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:39 +1000 Subject: [PATCH 1337/1927] drm/nouveau/fifo/ga100-: initial support - replaces the hacked-up version that existed solely to support TTM v2. remove earlier hack preventing use of non-stall intr for fences Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/include/nvif/class.h | 1 + .../drm/nouveau/include/nvkm/engine/fifo.h | 2 + drivers/gpu/drm/nouveau/nouveau_chan.c | 4 +- drivers/gpu/drm/nouveau/nouveau_drm.c | 4 +- drivers/gpu/drm/nouveau/nv84_fence.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/device/base.c | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/engine/fifo/base.c | 19 +- .../gpu/drm/nouveau/nvkm/engine/fifo/ga100.c | 550 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/fifo/ga102.c | 280 +-------- .../gpu/drm/nouveau/nvkm/engine/fifo/priv.h | 11 + .../gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 3 + .../gpu/drm/nouveau/nvkm/engine/fifo/runl.h | 8 +- 13 files changed, 610 insertions(+), 276 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 36c59c7afd954..15529d2c85f96 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -86,6 +86,7 @@ #define PASCAL_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c06f #define VOLTA_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c36f #define TURING_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c46f +#define AMPERE_CHANNEL_GPFIFO_A /* if0020.h */ 0x0000c56f #define AMPERE_CHANNEL_GPFIFO_B /* if0020.h */ 0x0000c76f #define NV50_DISP /* if0010.h */ 0x00005070 diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index c71f412bbb7ff..4825403a26885 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -56,6 +56,7 @@ struct nvkm_fifo { struct { #define NVKM_FIFO_NONSTALL_EVENT BIT(0) struct nvkm_event event; + struct nvkm_inth intr; } nonstall; struct { @@ -92,5 +93,6 @@ int gm200_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gp100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int gv100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int tu102_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); +int ga100_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int ga102_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index efd6cf46921b1..e648ecd0c1a03 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -253,6 +253,7 @@ nouveau_channel_ctor(struct nouveau_drm *drm, struct nvif_device *device, bool p int version; } hosts[] = { { AMPERE_CHANNEL_GPFIFO_B, 0 }, + { AMPERE_CHANNEL_GPFIFO_A, 0 }, { TURING_CHANNEL_GPFIFO_A, 0 }, { VOLTA_CHANNEL_GPFIFO_A, 0 }, { PASCAL_CHANNEL_GPFIFO_A, 0 }, @@ -365,8 +366,7 @@ nouveau_channel_init(struct nouveau_channel *chan, u32 vram, u32 gart) if (ret) return ret; - if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO && - chan->user.oclass < AMPERE_CHANNEL_GPFIFO_B) { + if (chan->user.oclass >= FERMI_CHANNEL_GPFIFO) { struct { struct nvif_event_v0 base; struct nvif_chan_event_v0 host; diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c index 61be972f83852..a19f18b251f31 100644 --- a/drivers/gpu/drm/nouveau/nouveau_drm.c +++ b/drivers/gpu/drm/nouveau/nouveau_drm.c @@ -348,9 +348,6 @@ nouveau_accel_gr_init(struct nouveau_drm *drm) u64 runm; int ret; - if (device->info.family >= NV_DEVICE_INFO_V0_AMPERE) - return; - /* Allocate channel that has access to the graphics engine. */ runm = nvif_fifo_runlist(device, NV_DEVICE_HOST_RUNLIST_ENGINES_GR); if (!runm) { @@ -473,6 +470,7 @@ nouveau_accel_init(struct nouveau_drm *drm) case PASCAL_CHANNEL_GPFIFO_A: case VOLTA_CHANNEL_GPFIFO_A: case TURING_CHANNEL_GPFIFO_A: + case AMPERE_CHANNEL_GPFIFO_A: case AMPERE_CHANNEL_GPFIFO_B: ret = nvc0_fence_create(drm); break; diff --git a/drivers/gpu/drm/nouveau/nv84_fence.c b/drivers/gpu/drm/nouveau/nv84_fence.c index a01fd934c85e2..812b8c62eeba1 100644 --- a/drivers/gpu/drm/nouveau/nv84_fence.c +++ b/drivers/gpu/drm/nouveau/nv84_fence.c @@ -210,7 +210,7 @@ nv84_fence_create(struct nouveau_drm *drm) priv->base.context_new = nv84_fence_context_new; priv->base.context_del = nv84_fence_context_del; - priv->base.uevent = drm->client.device.info.family < NV_DEVICE_INFO_V0_AMPERE; + priv->base.uevent = true; mutex_init(&priv->mutex); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 490fac3a4a3a8..852c18aec4cdb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2590,6 +2590,7 @@ nv170_chipset = { .top = { 0x00000001, ga100_top_new }, .vfn = { 0x00000001, ga100_vfn_new }, .ce = { 0x000003ff, ga100_ce_new }, + .fifo = { 0x00000001, ga100_fifo_new }, }; static const struct nvkm_device_chip diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild index f9aad4cf82713..5a074b9970abe 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild @@ -23,6 +23,7 @@ nvkm-y += nvkm/engine/fifo/gm200.o nvkm-y += nvkm/engine/fifo/gp100.o nvkm-y += nvkm/engine/fifo/gv100.o nvkm-y += nvkm/engine/fifo/tu102.o +nvkm-y += nvkm/engine/fifo/ga100.o nvkm-y += nvkm/engine/fifo/ga102.o nvkm-y += nvkm/engine/fifo/ucgrp.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index 61534a8a33d7f..abce1ecfd4d44 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -124,6 +124,7 @@ nvkm_fifo_init(struct nvkm_engine *engine) { struct nvkm_fifo *fifo = nvkm_fifo(engine); struct nvkm_runq *runq; + struct nvkm_runl *runl; u32 mask = 0; if (fifo->func->init_pbdmas) { @@ -136,7 +137,13 @@ nvkm_fifo_init(struct nvkm_engine *engine) runq->func->init(runq); } - fifo->func->init(fifo); + nvkm_runl_foreach(runl, fifo) { + if (runl->func->init) + runl->func->init(runl); + } + + if (fifo->func->init) + fifo->func->init(fifo); nvkm_inth_allow(&fifo->engine.subdev.inth); return 0; @@ -243,7 +250,7 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) return ret; nvkm_runl_foreach(runl, fifo) { - RUNL_DEBUG(runl, ""); + RUNL_DEBUG(runl, "chan:%06x", runl->chan); nvkm_runl_foreach_engn(engn, runl) { ENGN_DEBUG(engn, ""); } @@ -259,6 +266,14 @@ nvkm_fifo_oneinit(struct nvkm_engine *engine) } } + /* Initialise non-stall intr handling. */ + if (fifo->func->nonstall_ctor) { + ret = fifo->func->nonstall_ctor(fifo); + if (ret) { + nvkm_error(subdev, "nonstall %d\n", ret); + } + } + /* Allocate USERD + BAR1 polling area. */ if (fifo->func->chan.func->userd->bar == 1) { struct nvkm_vmm *bar1 = nvkm_bar_bar1_vmm(device); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c new file mode 100644 index 0000000000000..12a5d99d5e772 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga100.c @@ -0,0 +1,550 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" +#include "cgrp.h" +#include "chan.h" +#include "chid.h" +#include "runl.h" +#include "runq.h" + +#include +#include +#include + +#include + +/*TODO: allocate? */ +#define GA100_FIFO_NONSTALL_VECTOR 0 + +static u32 +ga100_chan_doorbell_handle(struct nvkm_chan *chan) +{ + return (chan->cgrp->runl->doorbell << 16) | chan->id; +} + +static void +ga100_chan_stop(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + + nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0x00000003); +} + +static void +ga100_chan_start(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + const int gfid = 0; + + nvkm_wr32(device, runl->chan + (chan->id * 4), 0x00000002); + nvkm_wr32(device, runl->addr + 0x0090, (gfid << 16) | chan->id); /* INTERNAL_DOORBELL. */ +} + +static void +ga100_chan_unbind(struct nvkm_chan *chan) +{ + struct nvkm_runl *runl = chan->cgrp->runl; + + nvkm_wr32(runl->fifo->engine.subdev.device, runl->chan + (chan->id * 4), 0xffffffff); +} + +static int +ga100_chan_ramfc_write(struct nvkm_chan *chan, u64 offset, u64 length, u32 devm, bool priv) +{ + const u32 limit2 = ilog2(length / 8); + + nvkm_kmap(chan->inst); + nvkm_wo32(chan->inst, 0x010, 0x0000face); + nvkm_wo32(chan->inst, 0x030, 0x7ffff902); + nvkm_wo32(chan->inst, 0x048, lower_32_bits(offset)); + nvkm_wo32(chan->inst, 0x04c, upper_32_bits(offset) | (limit2 << 16)); + nvkm_wo32(chan->inst, 0x084, 0x20400000); + nvkm_wo32(chan->inst, 0x094, 0x30000000 | devm); + nvkm_wo32(chan->inst, 0x0e4, priv ? 0x00000020 : 0x00000000); + nvkm_wo32(chan->inst, 0x0e8, chan->id); + nvkm_wo32(chan->inst, 0x0f4, 0x00001000 | (priv ? 0x00000100 : 0x00000000)); + nvkm_wo32(chan->inst, 0x0f8, 0x80000000 | GA100_FIFO_NONSTALL_VECTOR); + nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000); + nvkm_done(chan->inst); + return 0; +} + +static const struct nvkm_chan_func_ramfc +ga100_chan_ramfc = { + .write = ga100_chan_ramfc_write, + .devm = 0xfff, + .priv = true, +}; + +const struct nvkm_chan_func +ga100_chan = { + .inst = &gf100_chan_inst, + .userd = &gv100_chan_userd, + .ramfc = &ga100_chan_ramfc, + .unbind = ga100_chan_unbind, + .start = ga100_chan_start, + .stop = ga100_chan_stop, + .preempt = gk110_chan_preempt, + .doorbell_handle = ga100_chan_doorbell_handle, +}; + +static void +ga100_cgrp_preempt(struct nvkm_cgrp *cgrp) +{ + struct nvkm_runl *runl = cgrp->runl; + + nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x01000000 | cgrp->id); +} + +const struct nvkm_cgrp_func +ga100_cgrp = { + .preempt = ga100_cgrp_preempt, +}; + +static int +ga100_engn_cxid(struct nvkm_engn *engn, bool *cgid) +{ + struct nvkm_runl *runl = engn->runl; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u32 stat = nvkm_rd32(device, runl->addr + 0x200 + engn->id * 0x40); + + ENGN_DEBUG(engn, "status %08x", stat); + *cgid = true; + + switch ((stat & 0x0000e000) >> 13) { + case 0 /* INVALID */: return -ENODEV; + case 1 /* VALID */: + case 5 /* SAVE */: return (stat & 0x00000fff); + case 6 /* LOAD */: return (stat & 0x0fff0000) >> 16; + case 7 /* SWITCH */: + if (nvkm_engine_chsw_load(engn->engine)) + return (stat & 0x0fff0000) >> 16; + return (stat & 0x00000fff); + default: + WARN_ON(1); + break; + } + + return -ENODEV; +} + +const struct nvkm_engn_func +ga100_engn = { + .cxid = ga100_engn_cxid, + .ctor = gk104_ectx_ctor, + .bind = gv100_ectx_bind, +}; + +const struct nvkm_engn_func +ga100_engn_ce = { + .cxid = ga100_engn_cxid, + .ctor = gv100_ectx_ce_ctor, + .bind = gv100_ectx_ce_bind, +}; + +static bool +ga100_runq_idle(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + return !(nvkm_rd32(device, 0x04015c + (runq->id * 0x800)) & 0x0000e000); +} + +static bool +ga100_runq_intr_1(struct nvkm_runq *runq, struct nvkm_runl *runl) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + u32 inte = nvkm_rd32(device, 0x040180 + (runq->id * 0x800)); + u32 intr = nvkm_rd32(device, 0x040148 + (runq->id * 0x800)); + u32 stat = intr & inte; + + if (!stat) { + RUNQ_DEBUG(runq, "inte1 %08x %08x", intr, inte); + return false; + } + + if (stat & 0x80000000) { + u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask; + struct nvkm_chan *chan; + unsigned long flags; + + RUNQ_ERROR(runq, "CTXNOTVALID chid:%d", chid); + chan = nvkm_runl_chan_get_chid(runl, chid, &flags); + if (chan) { + nvkm_chan_error(chan, true); + nvkm_chan_put(&chan, flags); + } + + nvkm_mask(device, 0x0400ac + (runq->id * 0x800), 0x00030000, 0x00030000); + stat &= ~0x80000000; + } + + if (stat) { + RUNQ_ERROR(runq, "intr1 %08x", stat); + nvkm_wr32(device, 0x0401a0 + (runq->id * 0x800), stat); + } + + nvkm_wr32(device, 0x040148 + (runq->id * 0x800), intr); + return true; +} + +static bool +ga100_runq_intr_0(struct nvkm_runq *runq, struct nvkm_runl *runl) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + u32 inte = nvkm_rd32(device, 0x040170 + (runq->id * 0x800)); + u32 intr = nvkm_rd32(device, 0x040108 + (runq->id * 0x800)); + u32 stat = intr & inte; + + if (!stat) { + RUNQ_DEBUG(runq, "inte0 %08x %08x", intr, inte); + return false; + } + + /*TODO: expand on this when fixing up gf100's version. */ + if (stat & 0xc6afe000) { + u32 chid = nvkm_rd32(device, 0x040120 + (runq->id * 0x0800)) & runl->chid->mask; + struct nvkm_chan *chan; + unsigned long flags; + + RUNQ_ERROR(runq, "intr0 %08x", stat); + chan = nvkm_runl_chan_get_chid(runl, chid, &flags); + if (chan) { + nvkm_chan_error(chan, true); + nvkm_chan_put(&chan, flags); + } + + stat &= ~0xc6afe000; + } + + if (stat) { + RUNQ_ERROR(runq, "intr0 %08x", stat); + nvkm_wr32(device, 0x040190 + (runq->id * 0x800), stat); + } + + nvkm_wr32(device, 0x040108 + (runq->id * 0x800), intr); + return true; +} + +static bool +ga100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *runl) +{ + bool intr0 = ga100_runq_intr_0(runq, runl); + bool intr1 = ga100_runq_intr_1(runq, runl); + + return intr0 || intr1; +} + +static void +ga100_runq_init(struct nvkm_runq *runq) +{ + struct nvkm_device *device = runq->fifo->engine.subdev.device; + + nvkm_wr32(device, 0x040108 + (runq->id * 0x800), 0xffffffff); /* INTR_0 */ + nvkm_wr32(device, 0x040148 + (runq->id * 0x800), 0xffffffff); /* INTR_1 */ + nvkm_wr32(device, 0x040170 + (runq->id * 0x800), 0xffffffff); /* INTR_0_EN_SET_TREE */ + nvkm_wr32(device, 0x040180 + (runq->id * 0x800), 0xffffffff); /* INTR_1_EN_SET_TREE */ +} + +const struct nvkm_runq_func +ga100_runq = { + .init = ga100_runq_init, + .intr = ga100_runq_intr, + .idle = ga100_runq_idle, +}; + +static bool +ga100_runl_preempt_pending(struct nvkm_runl *runl) +{ + return nvkm_rd32(runl->fifo->engine.subdev.device, runl->addr + 0x098) & 0x00100000; +} + +static void +ga100_runl_preempt(struct nvkm_runl *runl) +{ + nvkm_wr32(runl->fifo->engine.subdev.device, runl->addr + 0x098, 0x00000000); +} + +static void +ga100_runl_allow(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000000); +} + +static void +ga100_runl_block(struct nvkm_runl *runl, u32 engm) +{ + nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x094, 0x00000001, 0x00000001); +} + +static bool +ga100_runl_pending(struct nvkm_runl *runl) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + + return nvkm_rd32(device, runl->addr + 0x08c) & 0x00008000; +} + +static void +ga100_runl_commit(struct nvkm_runl *runl, struct nvkm_memory *memory, u32 start, int count) +{ + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u64 addr = nvkm_memory_addr(memory) + start; + + nvkm_wr32(device, runl->addr + 0x080, lower_32_bits(addr)); + nvkm_wr32(device, runl->addr + 0x084, upper_32_bits(addr)); + nvkm_wr32(device, runl->addr + 0x088, count); +} + +static irqreturn_t +ga100_runl_intr(struct nvkm_inth *inth) +{ + struct nvkm_runl *runl = container_of(inth, typeof(*runl), inth); + struct nvkm_engn *engn; + struct nvkm_device *device = runl->fifo->engine.subdev.device; + u32 inte = nvkm_rd32(device, runl->addr + 0x120); + u32 intr = nvkm_rd32(device, runl->addr + 0x100); + u32 stat = intr & inte; + u32 info; + + if (!stat) { + RUNL_DEBUG(runl, "inte %08x %08x", intr, inte); + return IRQ_NONE; + } + + if (stat & 0x00000007) { + nvkm_runl_foreach_engn_cond(engn, runl, stat & BIT(engn->id)) { + info = nvkm_rd32(device, runl->addr + 0x224 + (engn->id * 0x40)); + + tu102_fifo_intr_ctxsw_timeout_info(engn, info); + + nvkm_wr32(device, runl->addr + 0x100, BIT(engn->id)); + stat &= ~BIT(engn->id); + } + } + + if (stat & 0x00000300) { + nvkm_wr32(device, runl->addr + 0x100, stat & 0x00000300); + stat &= ~0x00000300; + } + + if (stat & 0x00010000) { + if (runl->runq[0]) { + if (runl->runq[0]->func->intr(runl->runq[0], runl)) + stat &= ~0x00010000; + } + } + + if (stat & 0x00020000) { + if (runl->runq[1]) { + if (runl->runq[1]->func->intr(runl->runq[1], runl)) + stat &= ~0x00020000; + } + } + + if (stat) { + RUNL_ERROR(runl, "intr %08x", stat); + nvkm_wr32(device, runl->addr + 0x140, stat); + } + + nvkm_wr32(device, runl->addr + 0x180, 0x00000001); + return IRQ_HANDLED; +} + +static void +ga100_runl_fini(struct nvkm_runl *runl) +{ + nvkm_mask(runl->fifo->engine.subdev.device, runl->addr + 0x300, 0x80000000, 0x00000000); + nvkm_inth_block(&runl->inth); +} + +static void +ga100_runl_init(struct nvkm_runl *runl) +{ + struct nvkm_fifo *fifo = runl->fifo; + struct nvkm_runq *runq; + struct nvkm_device *device = fifo->engine.subdev.device; + int i; + + /* Submit NULL runlist and preempt. */ + nvkm_wr32(device, runl->addr + 0x088, 0x00000000); + runl->func->preempt(runl); + + /* Enable doorbell. */ + nvkm_mask(device, runl->addr + 0x300, 0x80000000, 0x80000000); + + nvkm_wr32(device, runl->addr + 0x100, 0xffffffff); /* INTR_0 */ + nvkm_wr32(device, runl->addr + 0x140, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(0) */ + nvkm_wr32(device, runl->addr + 0x120, 0x000f1307); /* INTR_0_EN_SET_TREE(0) */ + nvkm_wr32(device, runl->addr + 0x148, 0xffffffff); /* INTR_0_EN_CLEAR_TREE(1) */ + nvkm_wr32(device, runl->addr + 0x128, 0x00000000); /* INTR_0_EN_SET_TREE(1) */ + + /* Init PBDMA(s). */ + for (i = 0; i < runl->runq_nr; i++) { + runq = runl->runq[i]; + runq->func->init(runq); + } + + nvkm_inth_allow(&runl->inth); +} + +const struct nvkm_runl_func +ga100_runl = { + .init = ga100_runl_init, + .fini = ga100_runl_fini, + .size = 16, + .update = nv50_runl_update, + .insert_cgrp = gv100_runl_insert_cgrp, + .insert_chan = gv100_runl_insert_chan, + .commit = ga100_runl_commit, + .wait = nv50_runl_wait, + .pending = ga100_runl_pending, + .block = ga100_runl_block, + .allow = ga100_runl_allow, + .preempt = ga100_runl_preempt, + .preempt_pending = ga100_runl_preempt_pending, +}; + +static int +ga100_runl_new(struct nvkm_fifo *fifo, int id, u32 addr, struct nvkm_runl **prunl) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_runl *runl; + u32 chcfg = nvkm_rd32(device, addr + 0x004); + u32 chnum = 1 << (chcfg & 0x0000000f); + u32 chaddr = (chcfg & 0xfffffff0); + u32 dbcfg = nvkm_rd32(device, addr + 0x008); + u32 vector = nvkm_rd32(device, addr + 0x160); + int i, ret; + + runl = *prunl = nvkm_runl_new(fifo, id, addr, chnum); + if (IS_ERR(runl)) + return PTR_ERR(runl); + + for (i = 0; i < 2; i++) { + u32 pbcfg = nvkm_rd32(device, addr + 0x010 + (i * 0x04)); + if (pbcfg & 0x80000000) { + runl->runq[runl->runq_nr] = + nvkm_runq_new(fifo, ((pbcfg & 0x03fffc00) - 0x040000) / 0x800); + if (!runl->runq[runl->runq_nr]) + return -ENOMEM; + + runl->runq_nr++; + } + } + + ret = nvkm_inth_add(&device->vfn->intr, vector & 0x00000fff, NVKM_INTR_PRIO_NORMAL, + &fifo->engine.subdev, ga100_runl_intr, &runl->inth); + if (ret) + return ret; + + runl->chan = chaddr; + runl->doorbell = dbcfg >> 16; + return 0; +} + +static irqreturn_t +ga100_fifo_nonstall_intr(struct nvkm_inth *inth) +{ + struct nvkm_fifo *fifo = container_of(inth, typeof(*fifo), nonstall.intr); + + nvkm_event_ntfy(&fifo->nonstall.event, 0, NVKM_FIFO_NONSTALL_EVENT); + return IRQ_HANDLED; +} + +static void +ga100_fifo_nonstall_block(struct nvkm_event *event, int type, int index) +{ + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + + nvkm_inth_block(&fifo->nonstall.intr); +} + +static void +ga100_fifo_nonstall_allow(struct nvkm_event *event, int type, int index) +{ + struct nvkm_fifo *fifo = container_of(event, typeof(*fifo), nonstall.event); + + nvkm_inth_allow(&fifo->nonstall.intr); +} + +const struct nvkm_event_func +ga100_fifo_nonstall = { + .init = ga100_fifo_nonstall_allow, + .fini = ga100_fifo_nonstall_block, +}; + +int +ga100_fifo_nonstall_ctor(struct nvkm_fifo *fifo) +{ + return nvkm_inth_add(&fifo->engine.subdev.device->vfn->intr, GA100_FIFO_NONSTALL_VECTOR, + NVKM_INTR_PRIO_NORMAL, &fifo->engine.subdev, ga100_fifo_nonstall_intr, + &fifo->nonstall.intr); +} + +int +ga100_fifo_runl_ctor(struct nvkm_fifo *fifo) +{ + struct nvkm_device *device = fifo->engine.subdev.device; + struct nvkm_top_device *tdev; + struct nvkm_runl *runl; + int id = 0, ret; + + nvkm_list_foreach(tdev, &device->top->device, head, tdev->runlist >= 0) { + runl = nvkm_runl_get(fifo, -1, tdev->runlist); + if (!runl) { + ret = ga100_runl_new(fifo, id++, tdev->runlist, &runl); + if (ret) + return ret; + } + + if (tdev->engine < 0) + continue; + + nvkm_runl_add(runl, tdev->engine, (tdev->type == NVKM_ENGINE_CE) ? + fifo->func->engn_ce : fifo->func->engn, tdev->type, tdev->inst); + } + + return 0; +} + +static const struct nvkm_fifo_func +ga100_fifo = { + .runl_ctor = ga100_fifo_runl_ctor, + .mmu_fault = &tu102_fifo_mmu_fault, + .nonstall_ctor = ga100_fifo_nonstall_ctor, + .nonstall = &ga100_fifo_nonstall, + .runl = &ga100_runl, + .runq = &ga100_runq, + .engn = &ga100_engn, + .engn_ce = &ga100_engn_ce, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true }, + .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_A }, &ga100_chan }, +}; + +int +ga100_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_fifo **pfifo) +{ + return nvkm_fifo_new_(&ga100_fifo, device, type, inst, pfifo); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c index b0c7b562bfc84..2cdf5da339b60 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/ga102.c @@ -19,281 +19,27 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#define ga102_fifo(p) container_of((p), struct ga102_fifo, base.engine) -#define ga102_chan(p) container_of((p), struct ga102_chan, object) -#include +#include "priv.h" -#include -#include -#include -#include - -#include -#include #include -struct ga102_fifo { - struct nvkm_fifo base; -}; - -struct ga102_chan { - struct nvkm_object object; - - struct { - u32 runl; - u32 chan; - } ctrl; - - struct nvkm_memory *mthd; - struct nvkm_memory *inst; - struct nvkm_memory *user; - struct nvkm_memory *runl; - - struct nvkm_vmm *vmm; -}; - -static int -ga102_chan_sclass(struct nvkm_object *object, int index, struct nvkm_oclass *oclass) -{ - if (index == 0) { - oclass->ctor = nvkm_object_new; - oclass->base = (struct nvkm_sclass) { -1, -1, AMPERE_DMA_COPY_B }; - return 0; - } - - return -EINVAL; -} - -static int -ga102_chan_map(struct nvkm_object *object, void *argv, u32 argc, - enum nvkm_object_map *type, u64 *addr, u64 *size) -{ - struct ga102_chan *chan = ga102_chan(object); - struct nvkm_device *device = chan->object.engine->subdev.device; - u64 bar2 = nvkm_memory_bar2(chan->user); - - if (bar2 == ~0ULL) - return -EFAULT; - - *type = NVKM_OBJECT_MAP_IO; - *addr = device->func->resource_addr(device, 3) + bar2; - *size = 0x1000; - return 0; -} - -static int -ga102_chan_fini(struct nvkm_object *object, bool suspend) -{ - struct ga102_chan *chan = ga102_chan(object); - struct nvkm_device *device = chan->object.engine->subdev.device; - - nvkm_wr32(device, chan->ctrl.chan, 0x00000003); - - nvkm_wr32(device, chan->ctrl.runl + 0x098, 0x01000000); - nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, chan->ctrl.runl + 0x098) & 0x00100000)) - break; - ); - - nvkm_wr32(device, chan->ctrl.runl + 0x088, 0); - - nvkm_wr32(device, chan->ctrl.chan, 0xffffffff); - return 0; -} - -static int -ga102_chan_init(struct nvkm_object *object) -{ - struct ga102_chan *chan = ga102_chan(object); - struct nvkm_device *device = chan->object.engine->subdev.device; - - nvkm_mask(device, chan->ctrl.runl + 0x300, 0x80000000, 0x80000000); - - nvkm_wr32(device, chan->ctrl.runl + 0x080, lower_32_bits(nvkm_memory_addr(chan->runl))); - nvkm_wr32(device, chan->ctrl.runl + 0x084, upper_32_bits(nvkm_memory_addr(chan->runl))); - nvkm_wr32(device, chan->ctrl.runl + 0x088, 2); - - nvkm_wr32(device, chan->ctrl.chan, 0x00000002); - nvkm_wr32(device, chan->ctrl.runl + 0x0090, 0); - return 0; -} - -static void * -ga102_chan_dtor(struct nvkm_object *object) -{ - struct ga102_chan *chan = ga102_chan(object); - - if (chan->vmm) { - nvkm_vmm_part(chan->vmm, chan->inst); - nvkm_vmm_unref(&chan->vmm); - } - - nvkm_memory_unref(&chan->runl); - nvkm_memory_unref(&chan->user); - nvkm_memory_unref(&chan->inst); - nvkm_memory_unref(&chan->mthd); - return chan; -} - -static const struct nvkm_object_func -ga102_chan = { - .dtor = ga102_chan_dtor, - .init = ga102_chan_init, - .fini = ga102_chan_fini, - .map = ga102_chan_map, - .sclass = ga102_chan_sclass, -}; - -static int -ga102_chan_new(struct nvkm_device *device, - const struct nvkm_oclass *oclass, void *argv, u32 argc, struct nvkm_object **pobject) -{ - struct nvif_chan_v0 *args = argv; - struct nvkm_top_device *tdev; - struct nvkm_vmm *vmm; - struct nvkm_memory *userd; - struct ga102_chan *chan; - int ret; - - if (argc != sizeof(*args)) - return -ENOSYS; - - vmm = nvkm_uvmm_search(oclass->client, args->vmm); - if (IS_ERR(vmm)) - return PTR_ERR(vmm); - - if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) - return -ENOMEM; - - nvkm_object_ctor(&ga102_chan, oclass, &chan->object); - *pobject = &chan->object; - - list_for_each_entry(tdev, &device->top->device, head) { - if (tdev->type == NVKM_ENGINE_CE) { - chan->ctrl.runl = tdev->runlist; - break; - } - } - - if (!chan->ctrl.runl) - return -ENODEV; - - chan->ctrl.chan = nvkm_rd32(device, chan->ctrl.runl + 0x004) & 0xfffffff0; - - args->chid = 0; - args->inst = 0; - args->token = nvkm_rd32(device, chan->ctrl.runl + 0x008) & 0xffff0000; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->mthd); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->inst); - if (ret) - return ret; - - nvkm_kmap(chan->inst); - nvkm_wo32(chan->inst, 0x010, 0x0000face); - nvkm_wo32(chan->inst, 0x030, 0x7ffff902); - nvkm_wo32(chan->inst, 0x048, lower_32_bits(args->offset)); - nvkm_wo32(chan->inst, 0x04c, upper_32_bits(args->offset) | - (order_base_2(args->length / 8) << 16)); - nvkm_wo32(chan->inst, 0x084, 0x20400000); - nvkm_wo32(chan->inst, 0x094, 0x30000001); - nvkm_wo32(chan->inst, 0x0ac, 0x00020000); - nvkm_wo32(chan->inst, 0x0e4, 0x00000000); - nvkm_wo32(chan->inst, 0x0e8, 0); - nvkm_wo32(chan->inst, 0x0f4, 0x00001000); - nvkm_wo32(chan->inst, 0x0f8, 0x10003080); - nvkm_mo32(chan->inst, 0x218, 0x00000000, 0x00000000); - nvkm_wo32(chan->inst, 0x220, lower_32_bits(nvkm_memory_bar2(chan->mthd))); - nvkm_wo32(chan->inst, 0x224, upper_32_bits(nvkm_memory_bar2(chan->mthd))); - nvkm_done(chan->inst); - - userd = nvkm_umem_search(oclass->client, args->huserd); - if (IS_ERR(userd)) - return PTR_ERR(userd); - - ret = nvkm_memory_kmap(userd, &chan->user); - nvkm_memory_unref(&userd); - if (ret) - return ret; - - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, 0x1000, 0x1000, true, &chan->runl); - if (ret) - return ret; - - nvkm_kmap(chan->runl); - nvkm_wo32(chan->runl, 0x00, 0x80030001); - nvkm_wo32(chan->runl, 0x04, 1); - nvkm_wo32(chan->runl, 0x08, 0); - nvkm_wo32(chan->runl, 0x0c, 0x00000000); - nvkm_wo32(chan->runl, 0x10, lower_32_bits(nvkm_memory_addr(chan->user))); - nvkm_wo32(chan->runl, 0x14, upper_32_bits(nvkm_memory_addr(chan->user))); - nvkm_wo32(chan->runl, 0x18, lower_32_bits(nvkm_memory_addr(chan->inst))); - nvkm_wo32(chan->runl, 0x1c, upper_32_bits(nvkm_memory_addr(chan->inst))); - nvkm_done(chan->runl); - - ret = nvkm_vmm_join(vmm, chan->inst); - if (ret) - return ret; - - chan->vmm = nvkm_vmm_ref(vmm); - return 0; -} - -static const struct nvkm_device_oclass -ga102_chan_oclass = { - .ctor = ga102_chan_new, -}; - -static int -ga102_fifo_sclass(struct nvkm_oclass *oclass, int index, const struct nvkm_device_oclass **class) -{ - if (index == 0) { - oclass->base = (struct nvkm_sclass) { 0, 0, AMPERE_CHANNEL_GPFIFO_B }; - *class = &ga102_chan_oclass; - return 0; - } - - return 1; -} - -static int -ga102_fifo_info(struct nvkm_engine *engine, u64 mthd, u64 *data) -{ - switch (mthd) { - case NV_DEVICE_HOST_CHANNELS: *data = 1; return 0; - default: - break; - } - - return -ENOSYS; -} - -static void * -ga102_fifo_dtor(struct nvkm_engine *engine) -{ - return ga102_fifo(engine); -} - -static const struct nvkm_engine_func +static const struct nvkm_fifo_func ga102_fifo = { - .dtor = ga102_fifo_dtor, - .info = ga102_fifo_info, - .base.sclass = ga102_fifo_sclass, + .runl_ctor = ga100_fifo_runl_ctor, + .mmu_fault = &tu102_fifo_mmu_fault, + .nonstall_ctor = ga100_fifo_nonstall_ctor, + .nonstall = &ga100_fifo_nonstall, + .runl = &ga100_runl, + .runq = &ga100_runq, + .engn = &ga100_engn, + .engn_ce = &ga100_engn_ce, + .cgrp = {{ 0, 0, KEPLER_CHANNEL_GROUP_A }, &ga100_cgrp, .force = true }, + .chan = {{ 0, 0, AMPERE_CHANNEL_GPFIFO_B }, &ga100_chan }, }; int ga102_fifo_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fifo **pfifo) { - struct ga102_fifo *fifo; - - if (!(fifo = kzalloc(sizeof(*fifo), GFP_KERNEL))) - return -ENOMEM; - - nvkm_engine_ctor(&ga102_fifo, device, type, inst, true, &fifo->base.engine); - *pfifo = &fifo->base; - return 0; + return nvkm_fifo_new_(&ga102_fifo, device, type, inst, pfifo); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h index 24125e2d5d4f8..4d448be19224a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h @@ -37,6 +37,7 @@ struct nvkm_fifo_func { void (*pause)(struct nvkm_fifo *, unsigned long *); void (*start)(struct nvkm_fifo *, unsigned long *); + int (*nonstall_ctor)(struct nvkm_fifo *); const struct nvkm_event_func *nonstall; const struct nvkm_runl_func *runl; @@ -191,6 +192,16 @@ extern const struct nvkm_chan_func_ramfc gv100_chan_ramfc; void tu102_fifo_intr_ctxsw_timeout_info(struct nvkm_engn *, u32 info); extern const struct nvkm_fifo_func_mmu_fault tu102_fifo_mmu_fault; +int ga100_fifo_runl_ctor(struct nvkm_fifo *); +int ga100_fifo_nonstall_ctor(struct nvkm_fifo *); +extern const struct nvkm_event_func ga100_fifo_nonstall; +extern const struct nvkm_runl_func ga100_runl; +extern const struct nvkm_runq_func ga100_runq; +extern const struct nvkm_engn_func ga100_engn; +extern const struct nvkm_engn_func ga100_engn_ce; +extern const struct nvkm_cgrp_func ga100_cgrp; +extern const struct nvkm_chan_func ga100_chan; + int nvkm_uchan_new(struct nvkm_fifo *, struct nvkm_cgrp *, const struct nvkm_oclass *, void *argv, u32 argc, struct nvkm_object **); int nvkm_ucgrp_new(struct nvkm_fifo *, const struct nvkm_oclass *, void *argv, u32 argc, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index e654c1c4f769e..fe51c075d6c41 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -308,6 +308,9 @@ nvkm_runl_block(struct nvkm_runl *runl) void nvkm_runl_fini(struct nvkm_runl *runl) { + if (runl->func->fini) + runl->func->fini(runl); + flush_work(&runl->work); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h index 5159b64377c5a..c93d21bb7bd5c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h @@ -1,6 +1,6 @@ #ifndef __NVKM_RUNL_H__ #define __NVKM_RUNL_H__ -#include +#include struct nvkm_cctx; struct nvkm_cgrp; struct nvkm_chan; @@ -36,6 +36,8 @@ struct nvkm_engn { struct nvkm_runl { const struct nvkm_runl_func { + void (*init)(struct nvkm_runl *); + void (*fini)(struct nvkm_runl *); int runqs; u8 size; int (*update)(struct nvkm_runl *); @@ -53,6 +55,8 @@ struct nvkm_runl { struct nvkm_fifo *fifo; int id; u32 addr; + u32 chan; + u16 doorbell; struct nvkm_chid *cgid; #define NVKM_CHAN_EVENT_ERRORED BIT(0) @@ -63,6 +67,8 @@ struct nvkm_runl { struct nvkm_runq *runq[2]; int runq_nr; + struct nvkm_inth inth; + struct list_head cgrps; int cgrp_nr; int chan_nr; -- GitLab From 74f9dcb0df10567c1124b0cece2c6229a8c77cab Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:47 +1000 Subject: [PATCH 1338/1927] drm/nouveau/gsp: add funcs Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvkm/core/layout.h | 4 ++-- drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c | 4 +++- drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c | 11 ++++++++--- drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h | 6 +++++- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h b/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h index a344ddb77c808..58108dea5aeb6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/layout.h @@ -1,9 +1,10 @@ /* SPDX-License-Identifier: MIT */ +NVKM_LAYOUT_ONCE(NVKM_SUBDEV_TOP , struct nvkm_top , top) +NVKM_LAYOUT_ONCE(NVKM_SUBDEV_GSP , struct nvkm_gsp , gsp) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_VFN , struct nvkm_vfn , vfn) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_PCI , struct nvkm_pci , pci) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_VBIOS , struct nvkm_bios , bios) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_DEVINIT , struct nvkm_devinit , devinit) -NVKM_LAYOUT_ONCE(NVKM_SUBDEV_TOP , struct nvkm_top , top) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_PRIVRING, struct nvkm_subdev , privring) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_GPIO , struct nvkm_gpio , gpio) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_I2C , struct nvkm_i2c , i2c) @@ -24,7 +25,6 @@ NVKM_LAYOUT_ONCE(NVKM_SUBDEV_VOLT , struct nvkm_volt , volt) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_ICCSENSE, struct nvkm_iccsense, iccsense) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_THERM , struct nvkm_therm , therm) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_CLK , struct nvkm_clk , clk) -NVKM_LAYOUT_ONCE(NVKM_SUBDEV_GSP , struct nvkm_gsp , gsp) NVKM_LAYOUT_INST(NVKM_SUBDEV_IOCTRL , struct nvkm_subdev , ioctrl, 3) NVKM_LAYOUT_ONCE(NVKM_SUBDEV_FLA , struct nvkm_subdev , fla) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h index cf42a59d4e58f..ccf886ec1f84a 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h @@ -5,7 +5,9 @@ #include struct nvkm_gsp { + const struct nvkm_gsp_func *func; struct nvkm_subdev subdev; + struct nvkm_falcon falcon; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c index 22574886b8194..591ac95c26699 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/base.c @@ -53,5 +53,7 @@ nvkm_gsp_new_(const struct nvkm_gsp_fwif *fwif, struct nvkm_device *device, if (IS_ERR(fwif)) return PTR_ERR(fwif); - return nvkm_falcon_ctor(fwif->flcn, &gsp->subdev, gsp->subdev.name, 0, &gsp->falcon); + gsp->func = fwif->func; + + return nvkm_falcon_ctor(gsp->func->flcn, &gsp->subdev, gsp->subdev.name, 0, &gsp->falcon); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c index 6c4ef62a746a5..bc1138799ce50 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c @@ -36,6 +36,11 @@ gv100_gsp_flcn = { .disable = nvkm_falcon_v1_disable, }; +static const struct nvkm_gsp_func +gv100_gsp = { + .flcn = &gv100_gsp_flcn, +}; + static int gv100_gsp_nofw(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif) { @@ -43,8 +48,8 @@ gv100_gsp_nofw(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif) } static struct nvkm_gsp_fwif -gv100_gsp[] = { - { -1, gv100_gsp_nofw, &gv100_gsp_flcn }, +gv100_gsps[] = { + { -1, gv100_gsp_nofw, &gv100_gsp }, {} }; @@ -52,5 +57,5 @@ int gv100_gsp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gsp **pgsp) { - return nvkm_gsp_new_(gv100_gsp, device, type, inst, pgsp); + return nvkm_gsp_new_(gv100_gsps, device, type, inst, pgsp); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h index 19381ddd38d4d..89749a40203c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/priv.h @@ -4,10 +4,14 @@ #include enum nvkm_acr_lsf_id; +struct nvkm_gsp_func { + const struct nvkm_falcon_func *flcn; +}; + struct nvkm_gsp_fwif { int version; int (*load)(struct nvkm_gsp *, int ver, const struct nvkm_gsp_fwif *); - const struct nvkm_falcon_func *flcn; + const struct nvkm_gsp_func *func; }; int nvkm_gsp_new_(const struct nvkm_gsp_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int, -- GitLab From b7f44ef7320a50dd6718df06f8750489c1b1d5ea Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:48 +1000 Subject: [PATCH 1339/1927] drm/nouveau/pmu: move preinit() falcon reset to devinit Cleanup before falcon changes. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c | 8 +++++++- drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c | 12 ------------ 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c index a308b9bde4497..f30718d7e61af 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/devinit/gm200.c @@ -26,6 +26,7 @@ #include #include #include +#include #include static void @@ -85,13 +86,18 @@ pmu_load(struct nv50_devinit *init, u8 type, bool post, struct nvkm_subdev *subdev = &init->base.subdev; struct nvkm_bios *bios = subdev->device->bios; struct nvbios_pmuR pmu; + int ret; if (!nvbios_pmuRm(bios, type, &pmu)) return -EINVAL; - if (!post) + if (!post || !subdev->device->pmu) return 0; + ret = nvkm_falcon_reset(&subdev->device->pmu->falcon); + if (ret) + return ret; + pmu_code(init, pmu.boot_addr_pmu, pmu.boot_addr, pmu.boot_size, false); pmu_code(init, pmu.code_addr_pmu, pmu.code_addr, pmu.code_size, true); pmu_data(init, pmu.data_addr_pmu, pmu.data_addr, pmu.data_size); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c index 455e95a89259f..e10f46c7e817f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c @@ -99,9 +99,6 @@ nvkm_pmu_reset(struct nvkm_pmu *pmu) { struct nvkm_device *device = pmu->subdev.device; - if (!pmu->func->enabled(pmu)) - return; - /* Reset. */ if (pmu->func->reset) pmu->func->reset(pmu); @@ -113,14 +110,6 @@ nvkm_pmu_reset(struct nvkm_pmu *pmu) ); } -static int -nvkm_pmu_preinit(struct nvkm_subdev *subdev) -{ - struct nvkm_pmu *pmu = nvkm_pmu(subdev); - nvkm_pmu_reset(pmu); - return 0; -} - static int nvkm_pmu_init(struct nvkm_subdev *subdev) { @@ -160,7 +149,6 @@ nvkm_pmu_dtor(struct nvkm_subdev *subdev) static const struct nvkm_subdev_func nvkm_pmu = { .dtor = nvkm_pmu_dtor, - .preinit = nvkm_pmu_preinit, .init = nvkm_pmu_init, .fini = nvkm_pmu_fini, .intr = nvkm_pmu_intr, -- GitLab From ccdc043123d2a485e173e5e2627598151b7850b3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:48 +1000 Subject: [PATCH 1340/1927] drm/nouveau/pmu: move init() falcon reset to non-nvfw code Cleanup before falcon changes. - fixes (attempt at?) reset of pmu while rtos is running, on gm20b v2: - remove extra whitespace Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/subdev/pmu/base.c | 31 ++---------------- .../gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c | 2 -- .../gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c | 7 ++-- .../gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c | 7 ---- .../gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c | 14 ++------ .../gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c | 32 +++++++++++-------- .../gpu/drm/nouveau/nvkm/subdev/pmu/priv.h | 3 +- 8 files changed, 27 insertions(+), 70 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c index e10f46c7e817f..51808a4e75ac6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c @@ -81,6 +81,9 @@ nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend) { struct nvkm_pmu *pmu = nvkm_pmu(subdev); + if (!subdev->use.enabled) + return 0; + if (pmu->func->fini) pmu->func->fini(pmu); @@ -94,42 +97,14 @@ nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend) return 0; } -static void -nvkm_pmu_reset(struct nvkm_pmu *pmu) -{ - struct nvkm_device *device = pmu->subdev.device; - - /* Reset. */ - if (pmu->func->reset) - pmu->func->reset(pmu); - - /* Wait for IMEM/DMEM scrubbing to be complete. */ - nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x10a10c) & 0x00000006)) - break; - ); -} - static int nvkm_pmu_init(struct nvkm_subdev *subdev) { struct nvkm_pmu *pmu = nvkm_pmu(subdev); - struct nvkm_device *device = pmu->subdev.device; if (!pmu->func->init) return 0; - if (pmu->func->enabled(pmu)) { - /* Inhibit interrupts, and wait for idle. */ - nvkm_wr32(device, 0x10a014, 0x0000ffff); - nvkm_msec(device, 2000, - if (!nvkm_rd32(device, 0x10a04c)) - break; - ); - - nvkm_pmu_reset(pmu); - } - return pmu->func->init(pmu); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c index a67a42e73f084..b5e52b35f5d09 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gk20a.c @@ -197,7 +197,6 @@ gk20a_dvfs_data= { static const struct nvkm_pmu_func gk20a_pmu = { .flcn = >215_pmu_flcn, - .enabled = gf100_pmu_enabled, .init = gk20a_pmu_init, .fini = gk20a_pmu_fini, .reset = gf100_pmu_reset, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c index 40439e329aa9f..34e8320421f5a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c @@ -55,11 +55,9 @@ gm200_pmu_flcn = { static const struct nvkm_pmu_func gm200_pmu = { .flcn = &gm200_pmu_flcn, - .enabled = gf100_pmu_enabled, .reset = gf100_pmu_reset, }; - int gm200_pmu_nofw(struct nvkm_pmu *pmu, int ver, const struct nvkm_pmu_fwif *fwif) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c index 612310d5d4812..2b73848d4aed7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c @@ -166,7 +166,7 @@ gm20b_pmu_acr_init_wpr(struct nvkm_pmu *pmu) gm20b_pmu_acr_init_wpr_callback, pmu, 0); } -int +static int gm20b_pmu_initmsg(struct nvkm_pmu *pmu) { struct nv_pmu_init_msg msg; @@ -192,7 +192,7 @@ gm20b_pmu_initmsg(struct nvkm_pmu *pmu) return gm20b_pmu_acr_init_wpr(pmu); } -void +static void gm20b_pmu_recv(struct nvkm_pmu *pmu) { if (!pmu->initmsg_received) { @@ -209,10 +209,9 @@ gm20b_pmu_recv(struct nvkm_pmu *pmu) nvkm_falcon_msgq_recv(pmu->msgq); } -static const struct nvkm_pmu_func +const struct nvkm_pmu_func gm20b_pmu = { .flcn = &gm200_pmu_flcn, - .enabled = gf100_pmu_enabled, .intr = gt215_pmu_intr, .recv = gm20b_pmu_recv, .initmsg = gm20b_pmu_initmsg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c index 1a6f9c3af5ecd..9fd1116ebe27b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c @@ -31,16 +31,9 @@ gp102_pmu_reset(struct nvkm_pmu *pmu) nvkm_mask(device, 0x10a3c0, 0x00000001, 0x00000000); } -static bool -gp102_pmu_enabled(struct nvkm_pmu *pmu) -{ - return !(nvkm_rd32(pmu->subdev.device, 0x10a3c0) & 0x00000001); -} - static const struct nvkm_pmu_func gp102_pmu = { .flcn = &gm200_pmu_flcn, - .enabled = gp102_pmu_enabled, .reset = gp102_pmu_reset, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c index 94cfb1791af6e..9fba37ec05e09 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c @@ -76,16 +76,6 @@ gp10b_pmu_acr = { .bootstrap_multiple_falcons = gp10b_pmu_acr_bootstrap_multiple_falcons, }; -static const struct nvkm_pmu_func -gp10b_pmu = { - .flcn = &gm200_pmu_flcn, - .enabled = gf100_pmu_enabled, - .intr = gt215_pmu_intr, - .recv = gm20b_pmu_recv, - .initmsg = gm20b_pmu_initmsg, - .reset = gp102_pmu_reset, -}; - #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC) MODULE_FIRMWARE("nvidia/gp10b/pmu/desc.bin"); MODULE_FIRMWARE("nvidia/gp10b/pmu/image.bin"); @@ -94,8 +84,8 @@ MODULE_FIRMWARE("nvidia/gp10b/pmu/sig.bin"); static const struct nvkm_pmu_fwif gp10b_pmu_fwif[] = { - { 0, gm20b_pmu_load, &gp10b_pmu, &gp10b_pmu_acr }, - { -1, gm200_pmu_nofw, &gp10b_pmu }, + { 0, gm20b_pmu_load, &gm20b_pmu, &gp10b_pmu_acr }, + { -1, gm200_pmu_nofw, &gm20b_pmu }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c index b0407b86bc10d..6d1a966be081c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c @@ -184,6 +184,7 @@ static void gt215_pmu_reset(struct nvkm_pmu *pmu) { struct nvkm_device *device = pmu->subdev.device; + nvkm_mask(device, 0x022210, 0x00000001, 0x00000000); nvkm_mask(device, 0x022210, 0x00000001, 0x00000001); nvkm_rd32(device, 0x022210); @@ -201,6 +202,23 @@ gt215_pmu_init(struct nvkm_pmu *pmu) struct nvkm_device *device = pmu->subdev.device; int i; + /* Inhibit interrupts, and wait for idle. */ + if (pmu->func->enabled(pmu)) { + nvkm_wr32(device, 0x10a014, 0x0000ffff); + nvkm_msec(device, 2000, + if (!nvkm_rd32(device, 0x10a04c)) + break; + ); + } + + pmu->func->reset(pmu); + + /* Wait for IMEM/DMEM scrubbing to be complete. */ + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x10a10c) & 0x00000006)) + break; + ); + /* upload data segment */ nvkm_wr32(device, 0x10a1c0, 0x01000000); for (i = 0; i < pmu->func->data.size / 4; i++) @@ -243,20 +261,6 @@ gt215_pmu_init(struct nvkm_pmu *pmu) const struct nvkm_falcon_func gt215_pmu_flcn = { - .debug = 0xc08, - .fbif = 0xe00, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = nvkm_falcon_v1_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, - .start = nvkm_falcon_v1_start, - .enable = nvkm_falcon_v1_enable, - .disable = nvkm_falcon_v1_disable, - .cmdq = { 0x4a0, 0x4b0, 4 }, - .msgq = { 0x4c8, 0x4cc, 0 }, }; static const struct nvkm_pmu_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h index 21abf31f44420..1e7815b1a6499 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h @@ -47,12 +47,11 @@ void gk110_pmu_pgob(struct nvkm_pmu *, bool); extern const struct nvkm_falcon_func gm200_pmu_flcn; +extern const struct nvkm_pmu_func gm20b_pmu; void gm20b_pmu_acr_bld_patch(struct nvkm_acr *, u32, s64); void gm20b_pmu_acr_bld_write(struct nvkm_acr *, u32, struct nvkm_acr_lsfw *); int gm20b_pmu_acr_boot(struct nvkm_falcon *); int gm20b_pmu_acr_bootstrap_falcon(struct nvkm_falcon *, enum nvkm_acr_lsf_id); -void gm20b_pmu_recv(struct nvkm_pmu *); -int gm20b_pmu_initmsg(struct nvkm_pmu *); struct nvkm_pmu_fwif { int version; -- GitLab From a9d90860c42a6a014b3a0596c76dff016fc2ce27 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:49 +1000 Subject: [PATCH 1341/1927] drm/nouveau/pmu/gm20b,gp10b: boot RTOS from PMU init Cleanup before falcon changes. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/engine/falcon.h | 8 +- .../gpu/drm/nouveau/include/nvkm/subdev/acr.h | 1 + drivers/gpu/drm/nouveau/nvkm/falcon/base.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/acr/base.c | 80 +++++++++++-------- .../gpu/drm/nouveau/nvkm/subdev/pmu/base.c | 7 -- .../gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c | 49 +++++++++--- .../gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c | 1 - .../gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/pmu/priv.h | 1 - 9 files changed, 92 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index 176f0921d82f3..780e25fdd1199 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -16,7 +16,7 @@ enum nvkm_falcon_dmaidx { struct nvkm_falcon { const struct nvkm_falcon_func *func; - const struct nvkm_subdev *owner; + struct nvkm_subdev *owner; const char *name; u32 addr; @@ -24,7 +24,7 @@ struct nvkm_falcon { struct mutex dmem_mutex; bool oneinit; - const struct nvkm_subdev *user; + struct nvkm_subdev *user; u8 version; u8 secret; @@ -50,8 +50,8 @@ struct nvkm_falcon { struct nvkm_engine engine; }; -int nvkm_falcon_get(struct nvkm_falcon *, const struct nvkm_subdev *); -void nvkm_falcon_put(struct nvkm_falcon *, const struct nvkm_subdev *); +int nvkm_falcon_get(struct nvkm_falcon *, struct nvkm_subdev *); +void nvkm_falcon_put(struct nvkm_falcon *, struct nvkm_subdev *); int nvkm_falcon_new_(const struct nvkm_falcon_func *, struct nvkm_device *, enum nvkm_subdev_type, int inst, bool enable, u32 addr, struct nvkm_engine **); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h index c0b254f7f0b5b..3a269bb1d41b4 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h @@ -50,6 +50,7 @@ struct nvkm_acr { struct nvkm_vmm *vmm; bool done; + struct nvkm_acr_lsf *rtos; const struct firmware *wpr_fw; bool wpr_comp; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index f3f90c1063dd0..5a5b96dad6403 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -169,7 +169,7 @@ nvkm_falcon_oneinit(struct nvkm_falcon *falcon) } void -nvkm_falcon_put(struct nvkm_falcon *falcon, const struct nvkm_subdev *user) +nvkm_falcon_put(struct nvkm_falcon *falcon, struct nvkm_subdev *user) { if (unlikely(!falcon)) return; @@ -183,7 +183,7 @@ nvkm_falcon_put(struct nvkm_falcon *falcon, const struct nvkm_subdev *user) } int -nvkm_falcon_get(struct nvkm_falcon *falcon, const struct nvkm_subdev *user) +nvkm_falcon_get(struct nvkm_falcon *falcon, struct nvkm_subdev *user) { int ret = 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c index af6cac696d434..777b6cb33b478 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c @@ -63,10 +63,30 @@ nvkm_acr_hsf_boot(struct nvkm_acr *acr, const char *name) return 0; } +static struct nvkm_acr_lsf * +nvkm_acr_rtos(struct nvkm_acr *acr) +{ + struct nvkm_acr_lsf *lsf; + + if (acr) { + list_for_each_entry(lsf, &acr->lsf, head) { + if (lsf->func->bootstrap_falcon) + return lsf; + } + } + + return NULL; +} + static void nvkm_acr_unload(struct nvkm_acr *acr) { if (acr->done) { + if (acr->rtos) { + nvkm_subdev_unref(acr->rtos->falcon->owner); + acr->rtos = NULL; + } + nvkm_acr_hsf_boot(acr, "unload"); acr->done = false; } @@ -76,6 +96,7 @@ static int nvkm_acr_load(struct nvkm_acr *acr) { struct nvkm_subdev *subdev = &acr->subdev; + struct nvkm_acr_lsf *rtos = nvkm_acr_rtos(acr); struct nvkm_acr_lsf *lsf; u64 start, limit; int ret; @@ -100,6 +121,14 @@ nvkm_acr_load(struct nvkm_acr *acr) acr->done = true; + if (rtos) { + ret = nvkm_subdev_ref(rtos->falcon->owner); + if (ret) + return ret; + + acr->rtos = rtos; + } + list_for_each_entry(lsf, &acr->lsf, head) { if (lsf->func->boot) { ret = lsf->func->boot(lsf->falcon); @@ -118,33 +147,17 @@ nvkm_acr_reload(struct nvkm_acr *acr) return nvkm_acr_load(acr); } -static struct nvkm_acr_lsf * -nvkm_acr_falcon(struct nvkm_device *device) -{ - struct nvkm_acr *acr = device->acr; - struct nvkm_acr_lsf *lsf; - - if (acr) { - list_for_each_entry(lsf, &acr->lsf, head) { - if (lsf->func->bootstrap_falcon) - return lsf; - } - } - - return NULL; -} - int nvkm_acr_bootstrap_falcons(struct nvkm_device *device, unsigned long mask) { - struct nvkm_acr_lsf *acrflcn = nvkm_acr_falcon(device); struct nvkm_acr *acr = device->acr; + struct nvkm_acr_lsf *rtos = nvkm_acr_rtos(acr); unsigned long id; /* If there's no LS FW managing bootstrapping of other LS falcons, * we depend on the HS firmware being able to do it instead. */ - if (!acrflcn) { + if (!rtos) { /* Which isn't possible everywhere... */ if ((mask & acr->func->bootstrap_falcons) == mask) { int ret = nvkm_acr_reload(acr); @@ -156,16 +169,14 @@ nvkm_acr_bootstrap_falcons(struct nvkm_device *device, unsigned long mask) return -ENOSYS; } - if ((mask & acrflcn->func->bootstrap_falcons) != mask) + if ((mask & rtos->func->bootstrap_falcons) != mask) return -ENOSYS; - if (acrflcn->func->bootstrap_multiple_falcons) { - return acrflcn->func-> - bootstrap_multiple_falcons(acrflcn->falcon, mask); - } + if (rtos->func->bootstrap_multiple_falcons) + return rtos->func->bootstrap_multiple_falcons(rtos->falcon, mask); for_each_set_bit(id, &mask, NVKM_ACR_LSF_NUM) { - int ret = acrflcn->func->bootstrap_falcon(acrflcn->falcon, id); + int ret = rtos->func->bootstrap_falcon(rtos->falcon, id); if (ret) return ret; } @@ -189,6 +200,9 @@ nvkm_acr_managed_falcon(struct nvkm_device *device, enum nvkm_acr_lsf_id id) static int nvkm_acr_fini(struct nvkm_subdev *subdev, bool suspend) { + if (!subdev->use.enabled) + return 0; + nvkm_acr_unload(nvkm_acr(subdev)); return 0; } @@ -196,10 +210,12 @@ nvkm_acr_fini(struct nvkm_subdev *subdev, bool suspend) static int nvkm_acr_init(struct nvkm_subdev *subdev) { - if (!nvkm_acr_falcon(subdev->device)) + struct nvkm_acr *acr = nvkm_acr(subdev); + + if (!nvkm_acr_rtos(acr)) return 0; - return nvkm_acr_load(nvkm_acr(subdev)); + return nvkm_acr_load(acr); } static void @@ -218,7 +234,7 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev) struct nvkm_acr *acr = nvkm_acr(subdev); struct nvkm_acr_hsfw *hsfw; struct nvkm_acr_lsfw *lsfw, *lsft; - struct nvkm_acr_lsf *lsf; + struct nvkm_acr_lsf *lsf, *rtos; u32 wpr_size = 0; u64 falcons; int ret, i; @@ -260,10 +276,10 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev) } /* Ensure the falcon that'll provide ACR functions is booted first. */ - lsf = nvkm_acr_falcon(device); - if (lsf) { - falcons = lsf->func->bootstrap_falcons; - list_move(&lsf->head, &acr->lsf); + rtos = nvkm_acr_rtos(acr); + if (rtos) { + falcons = rtos->func->bootstrap_falcons; + list_move(&rtos->head, &acr->lsf); } else { falcons = acr->func->bootstrap_falcons; } @@ -301,7 +317,7 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev) nvkm_wobj(acr->wpr, 0, acr->wpr_fw->data, acr->wpr_fw->size); if (!acr->wpr_fw || acr->wpr_comp) - acr->func->wpr_build(acr, nvkm_acr_falcon(device)); + acr->func->wpr_build(acr, rtos); acr->func->wpr_patch(acr, (s64)acr->wpr_start - acr->wpr_prev); if (acr->wpr_fw && acr->wpr_comp) { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c index 51808a4e75ac6..8f2f50ad4ded1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/base.c @@ -87,13 +87,6 @@ nvkm_pmu_fini(struct nvkm_subdev *subdev, bool suspend) if (pmu->func->fini) pmu->func->fini(pmu); - flush_work(&pmu->recv.work); - - reinit_completion(&pmu->wpr_ready); - - nvkm_falcon_cmdq_fini(pmu->lpq); - nvkm_falcon_cmdq_fini(pmu->hpq); - pmu->initmsg_received = false; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c index 2b73848d4aed7..a72403777329e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm20b.c @@ -62,16 +62,6 @@ gm20b_pmu_acr_bootstrap_falcon(struct nvkm_falcon *falcon, return ret; } -int -gm20b_pmu_acr_boot(struct nvkm_falcon *falcon) -{ - struct nv_pmu_args args = { .secure_mode = true }; - const u32 addr_args = falcon->data.limit - sizeof(struct nv_pmu_args); - nvkm_falcon_load_dmem(falcon, &args, addr_args, sizeof(args), 0); - nvkm_falcon_start(falcon); - return 0; -} - void gm20b_pmu_acr_bld_patch(struct nvkm_acr *acr, u32 bld, s64 adjust) { @@ -125,7 +115,6 @@ gm20b_pmu_acr = { .bld_size = sizeof(struct loader_config), .bld_write = gm20b_pmu_acr_bld_write, .bld_patch = gm20b_pmu_acr_bld_patch, - .boot = gm20b_pmu_acr_boot, .bootstrap_falcons = BIT_ULL(NVKM_ACR_LSF_PMU) | BIT_ULL(NVKM_ACR_LSF_FECS) | BIT_ULL(NVKM_ACR_LSF_GPCCS), @@ -198,8 +187,7 @@ gm20b_pmu_recv(struct nvkm_pmu *pmu) if (!pmu->initmsg_received) { int ret = pmu->func->initmsg(pmu); if (ret) { - nvkm_error(&pmu->subdev, - "error parsing init message: %d\n", ret); + nvkm_error(&pmu->subdev, "error parsing init message: %d\n", ret); return; } @@ -209,9 +197,44 @@ gm20b_pmu_recv(struct nvkm_pmu *pmu) nvkm_falcon_msgq_recv(pmu->msgq); } +static void +gm20b_pmu_fini(struct nvkm_pmu *pmu) +{ + /*TODO: shutdown RTOS. */ + + flush_work(&pmu->recv.work); + nvkm_falcon_cmdq_fini(pmu->lpq); + nvkm_falcon_cmdq_fini(pmu->hpq); + + reinit_completion(&pmu->wpr_ready); + + nvkm_falcon_put(&pmu->falcon, &pmu->subdev); +} + +static int +gm20b_pmu_init(struct nvkm_pmu *pmu) +{ + struct nvkm_falcon *falcon = &pmu->falcon; + struct nv_pmu_args args = { .secure_mode = true }; + u32 addr_args = falcon->data.limit - sizeof(args); + int ret; + + ret = nvkm_falcon_get(&pmu->falcon, &pmu->subdev); + if (ret) + return ret; + + pmu->initmsg_received = false; + + nvkm_falcon_load_dmem(falcon, &args, addr_args, sizeof(args), 0); + nvkm_falcon_start(falcon); + return 0; +} + const struct nvkm_pmu_func gm20b_pmu = { .flcn = &gm200_pmu_flcn, + .init = gm20b_pmu_init, + .fini = gm20b_pmu_fini, .intr = gt215_pmu_intr, .recv = gm20b_pmu_recv, .initmsg = gm20b_pmu_initmsg, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c index 9fba37ec05e09..a6f410ba60bc9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp10b.c @@ -68,7 +68,6 @@ gp10b_pmu_acr = { .bld_size = sizeof(struct loader_config), .bld_write = gm20b_pmu_acr_bld_write, .bld_patch = gm20b_pmu_acr_bld_patch, - .boot = gm20b_pmu_acr_boot, .bootstrap_falcons = BIT_ULL(NVKM_ACR_LSF_PMU) | BIT_ULL(NVKM_ACR_LSF_FECS) | BIT_ULL(NVKM_ACR_LSF_GPCCS), diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c index 6d1a966be081c..32cee21ed8585 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gt215.c @@ -178,6 +178,7 @@ void gt215_pmu_fini(struct nvkm_pmu *pmu) { nvkm_wr32(pmu->subdev.device, 0x10a014, 0x00000060); + flush_work(&pmu->recv.work); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h index 1e7815b1a6499..75ede265a52c1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h @@ -50,7 +50,6 @@ extern const struct nvkm_falcon_func gm200_pmu_flcn; extern const struct nvkm_pmu_func gm20b_pmu; void gm20b_pmu_acr_bld_patch(struct nvkm_acr *, u32, s64); void gm20b_pmu_acr_bld_write(struct nvkm_acr *, u32, struct nvkm_acr_lsfw *); -int gm20b_pmu_acr_boot(struct nvkm_falcon *); int gm20b_pmu_acr_bootstrap_falcon(struct nvkm_falcon *, enum nvkm_acr_lsf_id); struct nvkm_pmu_fwif { -- GitLab From 3b330f0875014b475ca3dc37e4397aee13f00986 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:49 +1000 Subject: [PATCH 1342/1927] drm/nouveau/sec2: unload RTOS before tearing down WPR Reset regs won't be available on Ampere while SEC2 RTOS is running, and we're apparently supposed to be doing this on earlier GPUs too. v2: - fixed some excessive indentation Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvfw/sec2.h | 5 ++ .../drm/nouveau/include/nvkm/core/falcon.h | 1 + .../drm/nouveau/include/nvkm/engine/sec2.h | 4 +- .../gpu/drm/nouveau/include/nvkm/subdev/acr.h | 1 - .../gpu/drm/nouveau/nvkm/engine/sec2/base.c | 73 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/sec2/gp102.c | 33 +++++---- .../gpu/drm/nouveau/nvkm/engine/sec2/priv.h | 1 + .../gpu/drm/nouveau/nvkm/engine/sec2/tu102.c | 5 +- drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c | 8 +- drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h | 2 +- .../gpu/drm/nouveau/nvkm/subdev/acr/base.c | 9 --- 11 files changed, 91 insertions(+), 51 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvfw/sec2.h b/drivers/gpu/drm/nouveau/include/nvfw/sec2.h index 9a37ad4179cbc..07d7094d8ed85 100644 --- a/drivers/gpu/drm/nouveau/include/nvfw/sec2.h +++ b/drivers/gpu/drm/nouveau/include/nvfw/sec2.h @@ -10,6 +10,7 @@ struct nv_sec2_args { }; #define NV_SEC2_UNIT_INIT 0x01 +#define NV_SEC2_UNIT_UNLOAD 0x06 #define NV_SEC2_UNIT_ACR 0x08 struct nv_sec2_init_msg { @@ -57,4 +58,8 @@ struct nv_sec2_acr_bootstrap_falcon_msg { u32 error_code; u32 falcon_id; }; + +#define NV_SEC2_UNIT_V2_INIT 0x01 +#define NV_SEC2_UNIT_V2_UNLOAD 0x05 +#define NV_SEC2_UNIT_V2_ACR 0x07 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index d647d8acb15b6..2db6b5d2ebc6a 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -72,6 +72,7 @@ int nvkm_falcon_msgq_new(struct nvkm_falcon_qmgr *, const char *name, void nvkm_falcon_msgq_del(struct nvkm_falcon_msgq **); void nvkm_falcon_msgq_init(struct nvkm_falcon_msgq *, u32 index, u32 offset, u32 size); +bool nvkm_falcon_msgq_empty(struct nvkm_falcon_msgq *); int nvkm_falcon_msgq_recv_initmsg(struct nvkm_falcon_msgq *, void *, u32 size); void nvkm_falcon_msgq_recv(struct nvkm_falcon_msgq *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h index 06264c840eae6..a41b86426f80b 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h @@ -10,12 +10,14 @@ struct nvkm_sec2 { struct nvkm_engine engine; struct nvkm_falcon falcon; + atomic_t running; + atomic_t initmsg; + struct nvkm_falcon_qmgr *qmgr; struct nvkm_falcon_cmdq *cmdq; struct nvkm_falcon_msgq *msgq; struct work_struct work; - bool initmsg_received; }; int gp102_sec2_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_sec2 **); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h index 3a269bb1d41b4..fe5024b6d0f37 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h @@ -109,7 +109,6 @@ struct nvkm_acr_lsf_func { u32 bld_size; void (*bld_write)(struct nvkm_acr *, u32 bld, struct nvkm_acr_lsfw *); void (*bld_patch)(struct nvkm_acr *, u32 bld, s64 adjust); - int (*boot)(struct nvkm_falcon *); u64 bootstrap_falcons; int (*bootstrap_falcon)(struct nvkm_falcon *, enum nvkm_acr_lsf_id); int (*bootstrap_multiple_falcons)(struct nvkm_falcon *, u32 mask); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c index 092c6d0b8e01e..428bbda30a6f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c @@ -22,25 +22,17 @@ #include "priv.h" #include -#include +#include -static void -nvkm_sec2_recv(struct work_struct *work) -{ - struct nvkm_sec2 *sec2 = container_of(work, typeof(*sec2), work); - - if (!sec2->initmsg_received) { - int ret = sec2->func->initmsg(sec2); - if (ret) { - nvkm_error(&sec2->engine.subdev, - "error parsing init message: %d\n", ret); - return; - } +#include - sec2->initmsg_received = true; - } +static int +nvkm_sec2_finimsg(void *priv, struct nvfw_falcon_msg *hdr) +{ + struct nvkm_sec2 *sec2 = priv; - nvkm_falcon_msgq_recv(sec2->msgq); + atomic_set(&sec2->running, 0); + return 0; } static void @@ -54,14 +46,52 @@ static int nvkm_sec2_fini(struct nvkm_engine *engine, bool suspend) { struct nvkm_sec2 *sec2 = nvkm_sec2(engine); + struct nvkm_subdev *subdev = &sec2->engine.subdev; + struct nvkm_falcon *falcon = &sec2->falcon; + struct nvkm_falcon_cmdq *cmdq = sec2->cmdq; + struct nvfw_falcon_cmd cmd = { + .unit_id = sec2->func->unit_unload, + .size = sizeof(cmd), + }; + int ret; + + if (!subdev->use.enabled) + return 0; - flush_work(&sec2->work); + if (atomic_read(&sec2->initmsg) == 1) { + ret = nvkm_falcon_cmdq_send(cmdq, &cmd, nvkm_sec2_finimsg, sec2, + msecs_to_jiffies(1000)); + WARN_ON(ret); - if (suspend) { - nvkm_falcon_cmdq_fini(sec2->cmdq); - sec2->initmsg_received = false; + nvkm_msec(subdev->device, 2000, + if (nvkm_falcon_rd32(falcon, 0x100) & 0x00000010) + break; + ); } + nvkm_falcon_cmdq_fini(cmdq); + falcon->func->disable(falcon); + nvkm_falcon_put(falcon, subdev); + return 0; +} + +static int +nvkm_sec2_init(struct nvkm_engine *engine) +{ + struct nvkm_sec2 *sec2 = nvkm_sec2(engine); + struct nvkm_subdev *subdev = &sec2->engine.subdev; + struct nvkm_falcon *falcon = &sec2->falcon; + int ret; + + ret = nvkm_falcon_get(falcon, subdev); + if (ret) + return ret; + + nvkm_falcon_wr32(falcon, 0x014, 0xffffffff); + atomic_set(&sec2->initmsg, 0); + atomic_set(&sec2->running, 1); + + nvkm_falcon_start(falcon); return 0; } @@ -69,6 +99,7 @@ static void * nvkm_sec2_dtor(struct nvkm_engine *engine) { struct nvkm_sec2 *sec2 = nvkm_sec2(engine); + nvkm_falcon_msgq_del(&sec2->msgq); nvkm_falcon_cmdq_del(&sec2->cmdq); nvkm_falcon_qmgr_del(&sec2->qmgr); @@ -79,6 +110,7 @@ nvkm_sec2_dtor(struct nvkm_engine *engine) static const struct nvkm_engine_func nvkm_sec2 = { .dtor = nvkm_sec2_dtor, + .init = nvkm_sec2_init, .fini = nvkm_sec2_fini, .intr = nvkm_sec2_intr, }; @@ -113,6 +145,5 @@ nvkm_sec2_new_(const struct nvkm_sec2_fwif *fwif, struct nvkm_device *device, (ret = nvkm_falcon_msgq_new(sec2->qmgr, "msgq", &sec2->msgq))) return ret; - INIT_WORK(&sec2->work, nvkm_sec2_recv); return 0; }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index 44e39f5743d59..fb6d5417b2909 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -74,16 +74,6 @@ gp102_sec2_acr_bootstrap_falcon(struct nvkm_falcon *falcon, msecs_to_jiffies(1000)); } -static int -gp102_sec2_acr_boot(struct nvkm_falcon *falcon) -{ - struct nv_sec2_args args = {}; - nvkm_falcon_load_dmem(falcon, &args, - falcon->func->emem_addr, sizeof(args), 0); - nvkm_falcon_start(falcon); - return 0; -} - static void gp102_sec2_acr_bld_patch(struct nvkm_acr *acr, u32 bld, s64 adjust) { @@ -122,7 +112,6 @@ gp102_sec2_acr_0 = { .bld_size = sizeof(struct loader_config_v1), .bld_write = gp102_sec2_acr_bld_write, .bld_patch = gp102_sec2_acr_bld_patch, - .boot = gp102_sec2_acr_boot, .bootstrap_falcons = BIT_ULL(NVKM_ACR_LSF_FECS) | BIT_ULL(NVKM_ACR_LSF_GPCCS) | BIT_ULL(NVKM_ACR_LSF_SEC2), @@ -169,11 +158,29 @@ gp102_sec2_intr(struct nvkm_sec2 *sec2) u32 intr = nvkm_falcon_rd32(falcon, 0x008) & disp & ~(disp >> 16); if (intr & 0x00000040) { - schedule_work(&sec2->work); + if (unlikely(atomic_read(&sec2->initmsg) == 0)) { + int ret = sec2->func->initmsg(sec2); + + if (ret) + nvkm_error(subdev, "error parsing init message: %d\n", ret); + + atomic_set(&sec2->initmsg, ret ?: 1); + } + + if (atomic_read(&sec2->initmsg) > 0) { + if (!nvkm_falcon_msgq_empty(sec2->msgq)) + nvkm_falcon_msgq_recv(sec2->msgq); + } + nvkm_falcon_wr32(falcon, 0x004, 0x00000040); intr &= ~0x00000040; } + if (intr & 0x00000010) { + nvkm_falcon_wr32(falcon, 0x004, 0x00000010); + intr &= ~0x00000010; + } + if (intr) { nvkm_error(subdev, "unhandled intr %08x\n", intr); nvkm_falcon_wr32(falcon, 0x004, intr); @@ -250,6 +257,7 @@ gp102_sec2_flcn = { const struct nvkm_sec2_func gp102_sec2 = { .flcn = &gp102_sec2_flcn, + .unit_unload = NV_SEC2_UNIT_UNLOAD, .unit_acr = NV_SEC2_UNIT_ACR, .intr = gp102_sec2_intr, .initmsg = gp102_sec2_initmsg, @@ -304,7 +312,6 @@ gp102_sec2_acr_1 = { .bld_size = sizeof(struct flcn_bl_dmem_desc_v2), .bld_write = gp102_sec2_acr_bld_write_1, .bld_patch = gp102_sec2_acr_bld_patch_1, - .boot = gp102_sec2_acr_boot, .bootstrap_falcons = BIT_ULL(NVKM_ACR_LSF_FECS) | BIT_ULL(NVKM_ACR_LSF_GPCCS) | BIT_ULL(NVKM_ACR_LSF_SEC2), diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h index af19229e885dd..814a5f11def02 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h @@ -5,6 +5,7 @@ struct nvkm_sec2_func { const struct nvkm_falcon_func *flcn; + u8 unit_unload; u8 unit_acr; void (*intr)(struct nvkm_sec2 *); int (*initmsg)(struct nvkm_sec2 *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c index f3faeb7055755..39e42be039741 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c @@ -22,6 +22,8 @@ #include "priv.h" #include +#include + static const struct nvkm_falcon_func tu102_sec2_flcn = { .debug = 0x408, @@ -44,7 +46,8 @@ tu102_sec2_flcn = { static const struct nvkm_sec2_func tu102_sec2 = { .flcn = &tu102_sec2_flcn, - .unit_acr = 0x07, + .unit_unload = NV_SEC2_UNIT_V2_UNLOAD, + .unit_acr = NV_SEC2_UNIT_V2_ACR, .intr = gp102_sec2_intr, .initmsg = gp102_sec2_initmsg, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c index e74371dffc76c..04f8531516481 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c @@ -25,7 +25,7 @@ static void nvkm_falcon_msgq_open(struct nvkm_falcon_msgq *msgq) { - mutex_lock(&msgq->mutex); + spin_lock(&msgq->lock); msgq->position = nvkm_falcon_rd32(msgq->qmgr->falcon, msgq->tail_reg); } @@ -37,10 +37,10 @@ nvkm_falcon_msgq_close(struct nvkm_falcon_msgq *msgq, bool commit) if (commit) nvkm_falcon_wr32(falcon, msgq->tail_reg, msgq->position); - mutex_unlock(&msgq->mutex); + spin_unlock(&msgq->lock); } -static bool +bool nvkm_falcon_msgq_empty(struct nvkm_falcon_msgq *msgq) { u32 head = nvkm_falcon_rd32(msgq->qmgr->falcon, msgq->head_reg); @@ -208,6 +208,6 @@ nvkm_falcon_msgq_new(struct nvkm_falcon_qmgr *qmgr, const char *name, msgq->qmgr = qmgr; msgq->name = name; - mutex_init(&msgq->mutex); + spin_lock_init(&msgq->lock); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h b/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h index 33564ac906a8b..79f0da9e749f8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/qmgr.h @@ -73,7 +73,7 @@ struct nvkm_falcon_cmdq { struct nvkm_falcon_msgq { struct nvkm_falcon_qmgr *qmgr; const char *name; - struct mutex mutex; + spinlock_t lock; u32 head_reg; u32 tail_reg; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c index 777b6cb33b478..882d8d30bd64b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c @@ -97,7 +97,6 @@ nvkm_acr_load(struct nvkm_acr *acr) { struct nvkm_subdev *subdev = &acr->subdev; struct nvkm_acr_lsf *rtos = nvkm_acr_rtos(acr); - struct nvkm_acr_lsf *lsf; u64 start, limit; int ret; @@ -129,14 +128,6 @@ nvkm_acr_load(struct nvkm_acr *acr) acr->rtos = rtos; } - list_for_each_entry(lsf, &acr->lsf, head) { - if (lsf->func->boot) { - ret = lsf->func->boot(lsf->falcon); - if (ret) - break; - } - } - return ret; } -- GitLab From c7c0aac7421331baffdeb8f9c3e9702bdb1c0389 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:50 +1000 Subject: [PATCH 1343/1927] drm/nouveau/sec2: switch to newer style interrupt handler Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/sec2/base.c | 25 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/sec2/gp102.c | 7 ++++-- .../gpu/drm/nouveau/nvkm/engine/sec2/priv.h | 4 +-- .../gpu/drm/nouveau/nvkm/subdev/mc/gp100.c | 3 ++- 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c index 428bbda30a6f1..b1e5f543c99d1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c @@ -22,6 +22,7 @@ #include "priv.h" #include +#include #include #include @@ -35,13 +36,6 @@ nvkm_sec2_finimsg(void *priv, struct nvfw_falcon_msg *hdr) return 0; } -static void -nvkm_sec2_intr(struct nvkm_engine *engine) -{ - struct nvkm_sec2 *sec2 = nvkm_sec2(engine); - sec2->func->intr(sec2); -} - static int nvkm_sec2_fini(struct nvkm_engine *engine, bool suspend) { @@ -69,6 +63,8 @@ nvkm_sec2_fini(struct nvkm_engine *engine, bool suspend) ); } + nvkm_inth_block(&subdev->inth); + nvkm_falcon_cmdq_fini(cmdq); falcon->func->disable(falcon); nvkm_falcon_put(falcon, subdev); @@ -90,11 +86,24 @@ nvkm_sec2_init(struct nvkm_engine *engine) nvkm_falcon_wr32(falcon, 0x014, 0xffffffff); atomic_set(&sec2->initmsg, 0); atomic_set(&sec2->running, 1); + nvkm_inth_allow(&subdev->inth); nvkm_falcon_start(falcon); return 0; } +static int +nvkm_sec2_oneinit(struct nvkm_engine *engine) +{ + struct nvkm_sec2 *sec2 = nvkm_sec2(engine); + struct nvkm_subdev *subdev = &sec2->engine.subdev; + struct nvkm_intr *intr = &sec2->engine.subdev.device->mc->intr; + enum nvkm_intr_type type = NVKM_INTR_SUBDEV; + + return nvkm_inth_add(intr, type, NVKM_INTR_PRIO_NORMAL, subdev, sec2->func->intr, + &subdev->inth); +} + static void * nvkm_sec2_dtor(struct nvkm_engine *engine) { @@ -110,9 +119,9 @@ nvkm_sec2_dtor(struct nvkm_engine *engine) static const struct nvkm_engine_func nvkm_sec2 = { .dtor = nvkm_sec2_dtor, + .oneinit = nvkm_sec2_oneinit, .init = nvkm_sec2_init, .fini = nvkm_sec2_fini, - .intr = nvkm_sec2_intr, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index fb6d5417b2909..639ab9dfa4525 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -149,9 +149,10 @@ gp102_sec2_initmsg(struct nvkm_sec2 *sec2) return 0; } -void -gp102_sec2_intr(struct nvkm_sec2 *sec2) +irqreturn_t +gp102_sec2_intr(struct nvkm_inth *inth) { + struct nvkm_sec2 *sec2 = container_of(inth, typeof(*sec2), engine.subdev.inth); struct nvkm_subdev *subdev = &sec2->engine.subdev; struct nvkm_falcon *falcon = &sec2->falcon; u32 disp = nvkm_falcon_rd32(falcon, 0x01c); @@ -185,6 +186,8 @@ gp102_sec2_intr(struct nvkm_sec2 *sec2) nvkm_error(subdev, "unhandled intr %08x\n", intr); nvkm_falcon_wr32(falcon, 0x004, intr); } + + return IRQ_HANDLED; } int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h index 814a5f11def02..4997b8903a784 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h @@ -7,11 +7,11 @@ struct nvkm_sec2_func { const struct nvkm_falcon_func *flcn; u8 unit_unload; u8 unit_acr; - void (*intr)(struct nvkm_sec2 *); + irqreturn_t (*intr)(struct nvkm_inth *); int (*initmsg)(struct nvkm_sec2 *); }; -void gp102_sec2_intr(struct nvkm_sec2 *); +irqreturn_t gp102_sec2_intr(struct nvkm_inth *); int gp102_sec2_initmsg(struct nvkm_sec2 *); struct nvkm_sec2_fwif { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c index 7606bed2ff6fb..3a99a450b6b06 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c @@ -37,7 +37,8 @@ gp100_mc_intrs[] = { { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, { NVKM_SUBDEV_THERM , 0, 0, 0x00040000, true }, - { NVKM_SUBDEV_TOP , 0, 0, 0xffffffff, true }, + { NVKM_SUBDEV_TOP , 0, 0, 0x00008000 }, + { NVKM_SUBDEV_TOP , 0, 0, 0xffff7fff, true }, {}, }; -- GitLab From f15cde64b66161bfa74fb58f4e5697d8265b802e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:51 +1000 Subject: [PATCH 1344/1927] drm/nouveau/flcn: rework falcon reset Mostly preparation to fit in Ampere changes, but should result in reset sequences a lot closer to RM's, and perhaps help out with the issues we sometimes see reported in this area. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/falcon.h | 10 ++- .../drm/nouveau/include/nvkm/engine/falcon.h | 12 +-- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 8 -- .../gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c | 6 +- .../gpu/drm/nouveau/nvkm/engine/nvenc/gm107.c | 11 --- .../gpu/drm/nouveau/nvkm/engine/sec2/gp102.c | 16 ++-- .../gpu/drm/nouveau/nvkm/engine/sec2/tu102.c | 7 +- drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild | 3 + drivers/gpu/drm/nouveau/nvkm/falcon/base.c | 37 +-------- drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c | 83 +++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c | 32 +++++++ drivers/gpu/drm/nouveau/nvkm/falcon/priv.h | 8 ++ drivers/gpu/drm/nouveau/nvkm/falcon/v1.c | 43 ---------- .../gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c | 6 +- .../gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c | 17 +--- .../gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c | 29 +++++-- 16 files changed, 185 insertions(+), 143 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index 2db6b5d2ebc6a..625ffe31eaaf6 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -5,6 +5,13 @@ int nvkm_falcon_ctor(const struct nvkm_falcon_func *, struct nvkm_subdev *owner, const char *name, u32 addr, struct nvkm_falcon *); void nvkm_falcon_dtor(struct nvkm_falcon *); +int nvkm_falcon_reset(struct nvkm_falcon *); + +int gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *); +int gm200_flcn_disable(struct nvkm_falcon *); +int gm200_flcn_enable(struct nvkm_falcon *); + +int gp102_flcn_reset_eng(struct nvkm_falcon *); void nvkm_falcon_v1_load_imem(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); @@ -15,11 +22,8 @@ int nvkm_falcon_v1_wait_for_halt(struct nvkm_falcon *, u32); int nvkm_falcon_v1_clear_interrupt(struct nvkm_falcon *, u32); void nvkm_falcon_v1_set_start_addr(struct nvkm_falcon *, u32 start_addr); void nvkm_falcon_v1_start(struct nvkm_falcon *); -int nvkm_falcon_v1_enable(struct nvkm_falcon *); -void nvkm_falcon_v1_disable(struct nvkm_falcon *); void gp102_sec2_flcn_bind_context(struct nvkm_falcon *, struct nvkm_memory *); -int gp102_sec2_flcn_enable(struct nvkm_falcon *); #define FLCN_PRINTK(f,l,p,fmt,a...) ({ \ if ((f)->owner->name != (f)->name) \ diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index 780e25fdd1199..45c73893f1009 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -57,6 +57,12 @@ int nvkm_falcon_new_(const struct nvkm_falcon_func *, struct nvkm_device *, enum nvkm_subdev_type, int inst, bool enable, u32 addr, struct nvkm_engine **); struct nvkm_falcon_func { + int (*disable)(struct nvkm_falcon *); + int (*enable)(struct nvkm_falcon *); + bool reset_pmc; + int (*reset_eng)(struct nvkm_falcon *); + int (*reset_wait_mem_scrubbing)(struct nvkm_falcon *); + struct { u32 *data; u32 size; @@ -80,9 +86,6 @@ struct nvkm_falcon_func { int (*clear_interrupt)(struct nvkm_falcon *, u32); void (*set_start_addr)(struct nvkm_falcon *, u32 start_addr); void (*start)(struct nvkm_falcon *); - int (*enable)(struct nvkm_falcon *falcon); - void (*disable)(struct nvkm_falcon *falcon); - int (*reset)(struct nvkm_falcon *); struct { u32 head; @@ -122,7 +125,4 @@ void nvkm_falcon_set_start_addr(struct nvkm_falcon *, u32); void nvkm_falcon_start(struct nvkm_falcon *); int nvkm_falcon_wait_for_halt(struct nvkm_falcon *, u32); int nvkm_falcon_clear_interrupt(struct nvkm_falcon *, u32); -int nvkm_falcon_enable(struct nvkm_falcon *); -void nvkm_falcon_disable(struct nvkm_falcon *); -int nvkm_falcon_reset(struct nvkm_falcon *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 68da77df856ae..ee14115d669ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2073,17 +2073,9 @@ gf100_gr_ = { static const struct nvkm_falcon_func gf100_gr_flcn = { - .fbif = 0x600, .load_imem = nvkm_falcon_v1_load_imem, .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = nvkm_falcon_v1_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, .start = nvkm_falcon_v1_start, - .enable = nvkm_falcon_v1_enable, - .disable = nvkm_falcon_v1_disable, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c index 8c44ce44a6d7b..d9425e9195ce1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c @@ -23,6 +23,10 @@ static const struct nvkm_falcon_func gm107_nvdec_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .reset_pmc = true, + .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0xd00, .fbif = 0x600, .load_imem = nvkm_falcon_v1_load_imem, @@ -33,8 +37,6 @@ gm107_nvdec_flcn = { .clear_interrupt = nvkm_falcon_v1_clear_interrupt, .set_start_addr = nvkm_falcon_v1_set_start_addr, .start = nvkm_falcon_v1_start, - .enable = nvkm_falcon_v1_enable, - .disable = nvkm_falcon_v1_disable, }; static const struct nvkm_nvdec_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvenc/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvenc/gm107.c index f44d41bf20340..ad27d8b975696 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvenc/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvenc/gm107.c @@ -24,17 +24,6 @@ static const struct nvkm_falcon_func gm107_nvenc_flcn = { - .fbif = 0x800, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = nvkm_falcon_v1_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, - .start = nvkm_falcon_v1_start, - .enable = nvkm_falcon_v1_enable, - .disable = nvkm_falcon_v1_disable, }; static const struct nvkm_nvenc_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index 639ab9dfa4525..ae910c9bdc0a7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -190,15 +190,6 @@ gp102_sec2_intr(struct nvkm_inth *inth) return IRQ_HANDLED; } -int -gp102_sec2_flcn_enable(struct nvkm_falcon *falcon) -{ - nvkm_falcon_mask(falcon, 0x3c0, 0x00000001, 0x00000001); - udelay(10); - nvkm_falcon_mask(falcon, 0x3c0, 0x00000001, 0x00000000); - return nvkm_falcon_v1_enable(falcon); -} - void gp102_sec2_flcn_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *ctx) @@ -240,6 +231,11 @@ gp102_sec2_flcn_bind_context(struct nvkm_falcon *falcon, static const struct nvkm_falcon_func gp102_sec2_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .reset_pmc = true, + .reset_eng = gp102_flcn_reset_eng, + .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0x408, .fbif = 0x600, .load_imem = nvkm_falcon_v1_load_imem, @@ -251,8 +247,6 @@ gp102_sec2_flcn = { .clear_interrupt = nvkm_falcon_v1_clear_interrupt, .set_start_addr = nvkm_falcon_v1_set_start_addr, .start = nvkm_falcon_v1_start, - .enable = gp102_sec2_flcn_enable, - .disable = nvkm_falcon_v1_disable, .cmdq = { 0xa00, 0xa04, 8 }, .msgq = { 0xa30, 0xa34, 8 }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c index 39e42be039741..0f859f3235047 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c @@ -26,6 +26,11 @@ static const struct nvkm_falcon_func tu102_sec2_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .reset_pmc = true, + .reset_eng = gp102_flcn_reset_eng, + .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0x408, .fbif = 0x600, .load_imem = nvkm_falcon_v1_load_imem, @@ -37,8 +42,6 @@ tu102_sec2_flcn = { .clear_interrupt = nvkm_falcon_v1_clear_interrupt, .set_start_addr = nvkm_falcon_v1_set_start_addr, .start = nvkm_falcon_v1_start, - .enable = nvkm_falcon_v1_enable, - .disable = nvkm_falcon_v1_disable, .cmdq = { 0xc00, 0xc04, 8 }, .msgq = { 0xc80, 0xc84, 8 }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild b/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild index d79d783904eec..f2ffca4afbe3d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild @@ -4,3 +4,6 @@ nvkm-y += nvkm/falcon/cmdq.o nvkm-y += nvkm/falcon/msgq.o nvkm-y += nvkm/falcon/qmgr.o nvkm-y += nvkm/falcon/v1.o + +nvkm-y += nvkm/falcon/gm200.o +nvkm-y += nvkm/falcon/gp102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index 5a5b96dad6403..ed88cfb17f128 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -85,44 +85,15 @@ nvkm_falcon_start(struct nvkm_falcon *falcon) } int -nvkm_falcon_enable(struct nvkm_falcon *falcon) +nvkm_falcon_reset(struct nvkm_falcon *falcon) { - struct nvkm_device *device = falcon->owner->device; int ret; - nvkm_mc_enable(device, falcon->owner->type, falcon->owner->inst); - ret = falcon->func->enable(falcon); - if (ret) { - nvkm_mc_disable(device, falcon->owner->type, falcon->owner->inst); + ret = falcon->func->disable(falcon); + if (WARN_ON(ret)) return ret; - } - - return 0; -} - -void -nvkm_falcon_disable(struct nvkm_falcon *falcon) -{ - struct nvkm_device *device = falcon->owner->device; - - /* already disabled, return or wait_idle will timeout */ - if (!nvkm_mc_enabled(device, falcon->owner->type, falcon->owner->inst)) - return; - - falcon->func->disable(falcon); - - nvkm_mc_disable(device, falcon->owner->type, falcon->owner->inst); -} - -int -nvkm_falcon_reset(struct nvkm_falcon *falcon) -{ - if (!falcon->func->reset) { - nvkm_falcon_disable(falcon); - return nvkm_falcon_enable(falcon); - } - return falcon->func->reset(falcon); + return nvkm_falcon_enable(falcon); } int diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c new file mode 100644 index 0000000000000..9144bcbc7f451 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -0,0 +1,83 @@ +/* + * Copyright 2022 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include +#include + +int +gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *falcon) +{ + nvkm_falcon_mask(falcon, 0x040, 0x00000000, 0x00000000); + + if (nvkm_msec(falcon->owner->device, 10, + if (!(nvkm_falcon_rd32(falcon, 0x10c) & 0x00000006)) + break; + ) < 0) + return -ETIMEDOUT; + + return 0; +} + +int +gm200_flcn_enable(struct nvkm_falcon *falcon) +{ + struct nvkm_device *device = falcon->owner->device; + int ret; + + if (falcon->func->reset_eng) { + ret = falcon->func->reset_eng(falcon); + if (ret) + return ret; + } + + if (falcon->func->reset_pmc) + nvkm_mc_enable(device, falcon->owner->type, falcon->owner->inst); + + ret = falcon->func->reset_wait_mem_scrubbing(falcon); + if (ret) + return ret; + + nvkm_falcon_wr32(falcon, 0x084, nvkm_rd32(device, 0x000000)); + return 0; +} + +int +gm200_flcn_disable(struct nvkm_falcon *falcon) +{ + struct nvkm_device *device = falcon->owner->device; + int ret; + + nvkm_falcon_mask(falcon, 0x048, 0x00000003, 0x00000000); + nvkm_falcon_wr32(falcon, 0x014, 0xffffffff); + + if (falcon->func->reset_pmc) + nvkm_mc_disable(device, falcon->owner->type, falcon->owner->inst); + + if (falcon->func->reset_eng) { + ret = falcon->func->reset_eng(falcon); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c new file mode 100644 index 0000000000000..f49918530d0b7 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c @@ -0,0 +1,32 @@ +/* + * Copyright 2022 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +int +gp102_flcn_reset_eng(struct nvkm_falcon *falcon) +{ + nvkm_falcon_mask(falcon, 0x3c0, 0x00000001, 0x00000001); + udelay(10); + nvkm_falcon_mask(falcon, 0x3c0, 0x00000001, 0x00000000); + + return falcon->func->reset_wait_mem_scrubbing(falcon); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/priv.h b/drivers/gpu/drm/nouveau/nvkm/falcon/priv.h index 466188752eb08..11a24b9c85698 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/priv.h @@ -2,4 +2,12 @@ #ifndef __NVKM_FALCON_PRIV_H__ #define __NVKM_FALCON_PRIV_H__ #include + +static inline int +nvkm_falcon_enable(struct nvkm_falcon *falcon) +{ + if (falcon->func->enable) + return falcon->func->enable(falcon); + return 0; +} #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c index b0ee4c31414c8..9a9e1e6f70a67 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c @@ -266,46 +266,3 @@ nvkm_falcon_v1_clear_interrupt(struct nvkm_falcon *falcon, u32 mask) return 0; } - -static int -falcon_v1_wait_idle(struct nvkm_falcon *falcon) -{ - struct nvkm_device *device = falcon->owner->device; - int ret; - - ret = nvkm_wait_msec(device, 10, falcon->addr + 0x04c, 0xffff, 0x0); - if (ret < 0) - return ret; - - return 0; -} - -int -nvkm_falcon_v1_enable(struct nvkm_falcon *falcon) -{ - struct nvkm_device *device = falcon->owner->device; - int ret; - - ret = nvkm_wait_msec(device, 10, falcon->addr + 0x10c, 0x6, 0x0); - if (ret < 0) { - nvkm_error(falcon->user, "Falcon mem scrubbing timeout\n"); - return ret; - } - - ret = falcon_v1_wait_idle(falcon); - if (ret) - return ret; - - /* enable IRQs */ - nvkm_falcon_wr32(falcon, 0x010, 0xff); - - return 0; -} - -void -nvkm_falcon_v1_disable(struct nvkm_falcon *falcon) -{ - /* disable IRQs and wait for any previous code to complete */ - nvkm_falcon_wr32(falcon, 0x014, 0xff); - falcon_v1_wait_idle(falcon); -} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c index bc1138799ce50..a247e57f7d9fa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c @@ -23,6 +23,10 @@ static const struct nvkm_falcon_func gv100_gsp_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .reset_eng = gp102_flcn_reset_eng, + .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .fbif = 0x600, .load_imem = nvkm_falcon_v1_load_imem, .load_dmem = nvkm_falcon_v1_load_dmem, @@ -32,8 +36,6 @@ gv100_gsp_flcn = { .clear_interrupt = nvkm_falcon_v1_clear_interrupt, .set_start_addr = nvkm_falcon_v1_set_start_addr, .start = nvkm_falcon_v1_start, - .enable = gp102_sec2_flcn_enable, - .disable = nvkm_falcon_v1_disable, }; static const struct nvkm_gsp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c index 34e8320421f5a..0bd854092da96 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c @@ -23,18 +23,12 @@ */ #include "priv.h" -static int -gm200_pmu_flcn_reset(struct nvkm_falcon *falcon) -{ - struct nvkm_pmu *pmu = container_of(falcon, typeof(*pmu), falcon); - - nvkm_falcon_wr32(falcon, 0x014, 0x0000ffff); - pmu->func->reset(pmu); - return nvkm_falcon_enable(falcon); -} - const struct nvkm_falcon_func gm200_pmu_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .reset_pmc = true, + .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0xc08, .fbif = 0xe00, .load_imem = nvkm_falcon_v1_load_imem, @@ -45,9 +39,6 @@ gm200_pmu_flcn = { .clear_interrupt = nvkm_falcon_v1_clear_interrupt, .set_start_addr = nvkm_falcon_v1_set_start_addr, .start = nvkm_falcon_v1_start, - .enable = nvkm_falcon_v1_enable, - .disable = nvkm_falcon_v1_disable, - .reset = gm200_pmu_flcn_reset, .cmdq = { 0x4a0, 0x4b0, 4 }, .msgq = { 0x4c8, 0x4cc, 0 }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c index 9fd1116ebe27b..47c7412f86e86 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c @@ -23,18 +23,29 @@ */ #include "priv.h" -void -gp102_pmu_reset(struct nvkm_pmu *pmu) -{ - struct nvkm_device *device = pmu->subdev.device; - nvkm_mask(device, 0x10a3c0, 0x00000001, 0x00000001); - nvkm_mask(device, 0x10a3c0, 0x00000001, 0x00000000); -} +static const struct nvkm_falcon_func +gp102_pmu_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .reset_eng = gp102_flcn_reset_eng, + .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, + .debug = 0xc08, + .fbif = 0xe00, + .load_imem = nvkm_falcon_v1_load_imem, + .load_dmem = nvkm_falcon_v1_load_dmem, + .read_dmem = nvkm_falcon_v1_read_dmem, + .bind_context = nvkm_falcon_v1_bind_context, + .wait_for_halt = nvkm_falcon_v1_wait_for_halt, + .clear_interrupt = nvkm_falcon_v1_clear_interrupt, + .set_start_addr = nvkm_falcon_v1_set_start_addr, + .start = nvkm_falcon_v1_start, + .cmdq = { 0x4a0, 0x4b0, 4 }, + .msgq = { 0x4c8, 0x4cc, 0 }, +}; static const struct nvkm_pmu_func gp102_pmu = { - .flcn = &gm200_pmu_flcn, - .reset = gp102_pmu_reset, + .flcn = &gp102_pmu_flcn, }; static const struct nvkm_pmu_fwif -- GitLab From 0e44c21708761977dcbea9b846b51a6fb684907a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:51 +1000 Subject: [PATCH 1345/1927] drm/nouveau/flcn: new code to load+boot simple HS FWs (VPR scrubber) Adds the start of common interfaces to load and boot the HS binaries provided by NVIDIA that enable the usage of GR. ACR already handles most of this, but it's very much tied into ACR's init process, and there's other code that could benefit from reusing a lot of this stuff too (ie. VBIOS DEVINIT/PreOS, VPR scrubber). The VPR scrubber code is fairly independent, and a good first target. - adds better debug output to fw loading process, to ease bring-up/debug v2: - whitespace, 0->false Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/falcon.h | 94 +++++++ .../drm/nouveau/include/nvkm/core/firmware.h | 17 ++ .../drm/nouveau/include/nvkm/engine/falcon.h | 5 +- drivers/gpu/drm/nouveau/nvkm/core/firmware.c | 42 ++++ .../gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c | 11 +- drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild | 1 + drivers/gpu/drm/nouveau/nvkm/falcon/base.c | 54 +++++ drivers/gpu/drm/nouveau/nvkm/falcon/fw.c | 229 ++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c | 144 +++++++++++ .../gpu/drm/nouveau/nvkm/subdev/fb/gp102.c | 63 +---- 10 files changed, 594 insertions(+), 66 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/falcon/fw.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index 625ffe31eaaf6..d8fd2cbd2872c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -1,15 +1,44 @@ #ifndef __NVKM_FALCON_H__ #define __NVKM_FALCON_H__ +#include #include +enum nvkm_falcon_mem { + IMEM, + DMEM, +}; + +static inline const char * +nvkm_falcon_mem(enum nvkm_falcon_mem mem) +{ + switch (mem) { + case IMEM: return "imem"; + case DMEM: return "dmem"; + default: + WARN_ON(1); + return "?mem"; + } +} + +struct nvkm_falcon_func_pio { + int min; + int max; + void (*wr_init)(struct nvkm_falcon *, u8 port, bool sec, u32 mem_base); + void (*wr)(struct nvkm_falcon *, u8 port, const u8 *img, int len, u16 tag); +}; + int nvkm_falcon_ctor(const struct nvkm_falcon_func *, struct nvkm_subdev *owner, const char *name, u32 addr, struct nvkm_falcon *); void nvkm_falcon_dtor(struct nvkm_falcon *); int nvkm_falcon_reset(struct nvkm_falcon *); +int nvkm_falcon_pio_wr(struct nvkm_falcon *, const u8 *img, u32 img_base, u8 port, + enum nvkm_falcon_mem mem_type, u32 mem_base, int len, u16 tag, bool sec); int gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *); int gm200_flcn_disable(struct nvkm_falcon *); int gm200_flcn_enable(struct nvkm_falcon *); +extern const struct nvkm_falcon_func_pio gm200_flcn_imem_pio; +extern const struct nvkm_falcon_func_pio gm200_flcn_dmem_pio; int gp102_flcn_reset_eng(struct nvkm_falcon *); @@ -33,6 +62,71 @@ void gp102_sec2_flcn_bind_context(struct nvkm_falcon *, struct nvkm_memory *); }) #define FLCN_DBG(f,fmt,a...) FLCN_PRINTK((f), DEBUG, info, " "fmt"\n", ##a) #define FLCN_ERR(f,fmt,a...) FLCN_PRINTK((f), ERROR, err, " "fmt"\n", ##a) +#define FLCN_ERRON(f,c,fmt,a...) \ + ({ bool _cond = (c); _cond ? FLCN_ERR(f, fmt, ##a) : FLCN_DBG(f, fmt, ##a); _cond; }) + + +struct nvkm_falcon_fw { + const struct nvkm_falcon_fw_func { + int (*signature)(struct nvkm_falcon_fw *, u32 *sig_base_src); + int (*reset)(struct nvkm_falcon_fw *); + int (*load)(struct nvkm_falcon_fw *); + int (*boot)(struct nvkm_falcon_fw *, + u32 *mbox0, u32 *mbox1, u32 mbox0_ok, u32 irqsclr); + } *func; + struct nvkm_firmware fw; + + u32 sig_base_prd; + u32 sig_base_dbg; + u32 sig_base_img; + u32 sig_size; + int sig_nr; + u8 *sigs; + + u32 nmem_base_img; + u32 nmem_base; + u32 nmem_size; + + u32 imem_base_img; + u32 imem_base; + u32 imem_size; + + u32 dmem_base_img; + u32 dmem_base; + u32 dmem_size; + u32 dmem_sign; + + u32 boot_addr; + + struct nvkm_falcon *falcon; + struct nvkm_memory *inst; + struct nvkm_vmm *vmm; +}; + +int nvkm_falcon_fw_ctor(const struct nvkm_falcon_fw_func *, const char *name, struct nvkm_device *, + bool bl, const void *src, u32 len, struct nvkm_falcon *, + struct nvkm_falcon_fw *); +int nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *, const char *name, + struct nvkm_subdev *, const char *bl, const char *img, int ver, + struct nvkm_falcon *falcon, struct nvkm_falcon_fw *fw); +int nvkm_falcon_fw_sign(struct nvkm_falcon_fw *, u32 sig_base_img, u32 sig_size, const u8 *sigs, + int sig_nr_prd, u32 sig_base_prd, int sig_nr_dbg, u32 sig_base_dbg); +int nvkm_falcon_fw_patch(struct nvkm_falcon_fw *); +void nvkm_falcon_fw_dtor(struct nvkm_falcon_fw *); +int nvkm_falcon_fw_oneinit(struct nvkm_falcon_fw *, struct nvkm_falcon *, struct nvkm_vmm *, + struct nvkm_memory *inst); +int nvkm_falcon_fw_boot(struct nvkm_falcon_fw *, struct nvkm_subdev *user, + bool release, u32 *pmbox0, u32 *pmbox1, u32 mbox0_ok, u32 irqsclr); + +extern const struct nvkm_falcon_fw_func gm200_flcn_fw; +int gm200_flcn_fw_signature(struct nvkm_falcon_fw *, u32 *); +int gm200_flcn_fw_reset(struct nvkm_falcon_fw *); +int gm200_flcn_fw_load(struct nvkm_falcon_fw *); +int gm200_flcn_fw_boot(struct nvkm_falcon_fw *, u32 *, u32 *, u32, u32); + +#define FLCNFW_PRINTK(f,l,p,fmt,a...) FLCN_PRINTK((f)->falcon, l, p, "%s: "fmt, (f)->fw.name, ##a) +#define FLCNFW_DBG(f,fmt,a...) FLCNFW_PRINTK((f), DEBUG, info, fmt"\n", ##a) +#define FLCNFW_ERR(f,fmt,a...) FLCNFW_PRINTK((f), ERROR, err, fmt"\n", ##a) /** * struct nvfw_falcon_msg - header for all messages diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h b/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h index 85bcb80f6873f..8453025891a5e 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h @@ -4,6 +4,23 @@ #include #include +struct nvkm_firmware { + const struct nvkm_firmware_func { + enum nvkm_firmware_type { + NVKM_FIRMWARE_IMG_RAM, + } type; + } *func; + const char *name; + struct nvkm_device *device; + + int len; + u8 *img; +}; + +int nvkm_firmware_ctor(const struct nvkm_firmware_func *, const char *name, struct nvkm_device *, + const void *ptr, int len, struct nvkm_firmware *); +void nvkm_firmware_dtor(struct nvkm_firmware *); + int nvkm_firmware_get(const struct nvkm_subdev *, const char *fwname, int ver, const struct firmware **); void nvkm_firmware_put(const struct firmware *); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index 45c73893f1009..ca751199379e3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -63,6 +63,10 @@ struct nvkm_falcon_func { int (*reset_eng)(struct nvkm_falcon *); int (*reset_wait_mem_scrubbing)(struct nvkm_falcon *); + u32 debug; + const struct nvkm_falcon_func_pio *imem_pio; + const struct nvkm_falcon_func_pio *dmem_pio; + struct { u32 *data; u32 size; @@ -74,7 +78,6 @@ struct nvkm_falcon_func { void (*init)(struct nvkm_falcon *); void (*intr)(struct nvkm_falcon *, struct nvkm_chan *); - u32 debug; u32 fbif; void (*load_imem)(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c index ca1f8463cff51..aa1bf6b5a8cdc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c @@ -107,3 +107,45 @@ nvkm_firmware_put(const struct firmware *fw) { release_firmware(fw); } + +void +nvkm_firmware_dtor(struct nvkm_firmware *fw) +{ + if (!fw->img) + return; + + switch (fw->func->type) { + case NVKM_FIRMWARE_IMG_RAM: + kfree(fw->img); + break; + default: + WARN_ON(1); + break; + } + + fw->img = NULL; +} + +int +nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, + struct nvkm_device *device, const void *src, int len, struct nvkm_firmware *fw) +{ + fw->func = func; + fw->name = name; + fw->device = device; + + switch (fw->func->type) { + case NVKM_FIRMWARE_IMG_RAM: + fw->len = len; + fw->img = kmemdup(src, fw->len, GFP_KERNEL); + break; + default: + WARN_ON(1); + return -EINVAL; + } + + if (!fw->img) + return -ENOMEM; + + return 0; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c index d9425e9195ce1..5d04ded35cc34 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c @@ -28,15 +28,8 @@ gm107_nvdec_flcn = { .reset_pmc = true, .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0xd00, - .fbif = 0x600, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = nvkm_falcon_v1_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, - .start = nvkm_falcon_v1_start, + .imem_pio = &gm200_flcn_imem_pio, + .dmem_pio = &gm200_flcn_dmem_pio, }; static const struct nvkm_nvdec_func diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild b/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild index f2ffca4afbe3d..6ffde5290b876 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/falcon/base.o nvkm-y += nvkm/falcon/cmdq.o +nvkm-y += nvkm/falcon/fw.o nvkm-y += nvkm/falcon/msgq.o nvkm-y += nvkm/falcon/qmgr.o nvkm-y += nvkm/falcon/v1.o diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index ed88cfb17f128..c15dda020c775 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -24,6 +24,60 @@ #include #include +static const struct nvkm_falcon_func_pio * +nvkm_falcon_pio(struct nvkm_falcon *falcon, enum nvkm_falcon_mem *mem_type, u32 *mem_base) +{ + switch (*mem_type) { + case IMEM: + return falcon->func->imem_pio; + case DMEM: + if (!falcon->func->emem_addr || *mem_base < falcon->func->emem_addr) + return falcon->func->dmem_pio; + + *mem_base -= falcon->func->emem_addr; + fallthrough; + default: + return NULL; + } +} + +int +nvkm_falcon_pio_wr(struct nvkm_falcon *falcon, const u8 *img, u32 img_base, u8 port, + enum nvkm_falcon_mem mem_type, u32 mem_base, int len, u16 tag, bool sec) +{ + const struct nvkm_falcon_func_pio *pio = nvkm_falcon_pio(falcon, &mem_type, &mem_base); + const char *type = nvkm_falcon_mem(mem_type); + int xfer_len; + + if (WARN_ON(!pio || !pio->wr)) + return -EINVAL; + + FLCN_DBG(falcon, "%s %08x <- %08x bytes at %08x", type, mem_base, len, img_base); + if (WARN_ON(!len || (len & (pio->min - 1)))) + return -EINVAL; + + pio->wr_init(falcon, port, sec, mem_base); + do { + xfer_len = min(len, pio->max); + pio->wr(falcon, port, img, xfer_len, tag++); + + if (nvkm_printk_ok(falcon->owner, falcon->user, NV_DBG_TRACE)) { + for (img_base = 0; img_base < xfer_len; img_base += 4, mem_base += 4) { + if (((img_base / 4) % 8) == 0) + printk(KERN_INFO "%s %08x <-", type, mem_base); + printk(KERN_CONT " %08x", *(u32 *)(img + img_base)); + if ((img_base / 4) == 7 && mem_type == IMEM) + printk(KERN_CONT " %04x", tag - 1); + } + } + + img += xfer_len; + len -= xfer_len; + } while (len); + + return 0; +} + void nvkm_falcon_load_imem(struct nvkm_falcon *falcon, void *data, u32 start, u32 size, u16 tag, u8 port, bool secure) diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c new file mode 100644 index 0000000000000..12a899dbec35b --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -0,0 +1,229 @@ +/* + * Copyright 2022 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include +#include + +#include +#include + +int +nvkm_falcon_fw_patch(struct nvkm_falcon_fw *fw) +{ + struct nvkm_falcon *falcon = fw->falcon; + u32 sig_base_src = fw->sig_base_prd; + u32 src, dst, len, i; + int idx = 0; + + FLCNFW_DBG(fw, "patching sigs:%d size:%d", fw->sig_nr, fw->sig_size); + if (fw->func->signature) { + idx = fw->func->signature(fw, &sig_base_src); + if (idx < 0) + return idx; + } + + src = idx * fw->sig_size; + dst = fw->sig_base_img; + len = fw->sig_size / 4; + FLCNFW_DBG(fw, "patch idx:%d src:%08x dst:%08x", idx, sig_base_src + src, dst); + for (i = 0; i < len; i++) { + u32 sig = *(u32 *)(fw->sigs + src); + + if (nvkm_printk_ok(falcon->owner, falcon->user, NV_DBG_TRACE)) { + if (i % 8 == 0) + printk(KERN_INFO "sig -> %08x:", dst); + printk(KERN_CONT " %08x", sig); + } + + *(u32 *)(fw->fw.img + dst) = sig; + src += 4; + dst += 4; + } + + return 0; +} + +static void +nvkm_falcon_fw_dtor_sigs(struct nvkm_falcon_fw *fw) +{ + kfree(fw->sigs); + fw->sigs = NULL; +} + +int +nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, + bool release, u32 *pmbox0, u32 *pmbox1, u32 mbox0_ok, u32 irqsclr) +{ + struct nvkm_falcon *falcon = fw->falcon; + int ret; + + ret = nvkm_falcon_get(falcon, user); + if (ret) + return ret; + + if (fw->sigs) { + ret = nvkm_falcon_fw_patch(fw); + if (ret) + goto done; + + nvkm_falcon_fw_dtor_sigs(fw); + } + + FLCNFW_DBG(fw, "resetting"); + fw->func->reset(fw); + + FLCNFW_DBG(fw, "loading"); + ret = fw->func->load(fw); + if (ret) + goto done; + + FLCNFW_DBG(fw, "booting"); + ret = fw->func->boot(fw, pmbox0, pmbox1, mbox0_ok, irqsclr); + if (ret) + FLCNFW_ERR(fw, "boot failed: %d", ret); + else + FLCNFW_DBG(fw, "booted"); + +done: + if (ret || release) + nvkm_falcon_put(falcon, user); + return ret; +} + +int +nvkm_falcon_fw_oneinit(struct nvkm_falcon_fw *fw, struct nvkm_falcon *falcon, + struct nvkm_vmm *vmm, struct nvkm_memory *inst) +{ + fw->falcon = falcon; + fw->vmm = nvkm_vmm_ref(vmm); + fw->inst = nvkm_memory_ref(inst); + return 0; +} + +void +nvkm_falcon_fw_dtor(struct nvkm_falcon_fw *fw) +{ + nvkm_vmm_unref(&fw->vmm); + nvkm_memory_unref(&fw->inst); + nvkm_falcon_fw_dtor_sigs(fw); + nvkm_firmware_dtor(&fw->fw); +} + +static const struct nvkm_firmware_func +nvkm_falcon_fw = { + .type = NVKM_FIRMWARE_IMG_RAM, +}; + +int +nvkm_falcon_fw_sign(struct nvkm_falcon_fw *fw, u32 sig_base_img, u32 sig_size, const u8 *sigs, + int sig_nr_prd, u32 sig_base_prd, int sig_nr_dbg, u32 sig_base_dbg) +{ + fw->sig_base_prd = sig_base_prd; + fw->sig_base_dbg = sig_base_dbg; + fw->sig_base_img = sig_base_img; + fw->sig_size = sig_size; + fw->sig_nr = sig_nr_prd + sig_nr_dbg; + + fw->sigs = kmalloc_array(fw->sig_nr, fw->sig_size, GFP_KERNEL); + if (!fw->sigs) + return -ENOMEM; + + memcpy(fw->sigs, sigs + sig_base_prd, sig_nr_prd * fw->sig_size); + if (sig_nr_dbg) + memcpy(fw->sigs + sig_size, sigs + sig_base_dbg, sig_nr_dbg * fw->sig_size); + + return 0; +} + +int +nvkm_falcon_fw_ctor(const struct nvkm_falcon_fw_func *func, const char *name, + struct nvkm_device *device, bool dma, const void *src, u32 len, + struct nvkm_falcon *falcon, struct nvkm_falcon_fw *fw) +{ + const struct nvkm_firmware_func *type = &nvkm_falcon_fw; + int ret; + + fw->func = func; + + ret = nvkm_firmware_ctor(type, name, device, src, len, &fw->fw); + if (ret) + return ret; + + return falcon ? nvkm_falcon_fw_oneinit(fw, falcon, NULL, NULL) : 0; +} + +int +nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *func, const char *name, + struct nvkm_subdev *subdev, const char *bl, const char *img, int ver, + struct nvkm_falcon *falcon, struct nvkm_falcon_fw *fw) +{ + const struct firmware *blob; + const struct nvfw_bin_hdr *hdr; + const struct nvfw_hs_header *hshdr; + const struct nvfw_hs_load_header *lhdr; + u32 loc, sig; + int ret; + + ret = nvkm_firmware_load_name(subdev, img, "", ver, &blob); + if (ret) + return ret; + + hdr = nvfw_bin_hdr(subdev, blob->data); + hshdr = nvfw_hs_header(subdev, blob->data + hdr->header_offset); + loc = *(u32 *)(blob->data + hshdr->patch_loc); + sig = *(u32 *)(blob->data + hshdr->patch_sig); + + ret = nvkm_falcon_fw_ctor(func, name, subdev->device, bl != NULL, + blob->data + hdr->data_offset, hdr->data_size, falcon, fw); + if (ret) + goto done; + + ret = nvkm_falcon_fw_sign(fw, loc, hshdr->sig_prod_size, blob->data, + 1, hshdr->sig_prod_offset + sig, + 1, hshdr->sig_dbg_offset + sig); + if (ret) + goto done; + + lhdr = nvfw_hs_load_header(subdev, blob->data + hshdr->hdr_offset); + + fw->nmem_base_img = 0; + fw->nmem_base = lhdr->non_sec_code_off; + fw->nmem_size = lhdr->non_sec_code_size; + + fw->imem_base_img = lhdr->apps[0]; + fw->imem_base = ALIGN(lhdr->apps[0], 0x100); + fw->imem_size = lhdr->apps[lhdr->num_apps + 0]; + + fw->dmem_base_img = lhdr->data_dma_base; + fw->dmem_base = 0; + fw->dmem_size = lhdr->data_size; + fw->dmem_sign = loc - lhdr->data_dma_base; + + fw->boot_addr = fw->nmem_base; +done: + if (ret) + nvkm_falcon_fw_dtor(fw); + + nvkm_firmware_put(blob); + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c index 9144bcbc7f451..014ca38b8ff3e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -24,6 +24,55 @@ #include #include +static void +gm200_flcn_pio_dmem_wr(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len, u16 tag) +{ + while (len >= 4) { + nvkm_falcon_wr32(falcon, 0x1c4 + (port * 8), *(u32 *)img); + img += 4; + len -= 4; + } +} + +static void +gm200_flcn_pio_dmem_wr_init(struct nvkm_falcon *falcon, u8 port, bool sec, u32 dmem_base) +{ + nvkm_falcon_wr32(falcon, 0x1c0 + (port * 8), BIT(24) | dmem_base); +} + +const struct nvkm_falcon_func_pio +gm200_flcn_dmem_pio = { + .min = 4, + .max = 0x100, + .wr_init = gm200_flcn_pio_dmem_wr_init, + .wr = gm200_flcn_pio_dmem_wr, +}; + +static void +gm200_flcn_pio_imem_wr_init(struct nvkm_falcon *falcon, u8 port, bool sec, u32 imem_base) +{ + nvkm_falcon_wr32(falcon, 0x180 + (port * 0x10), (sec ? BIT(28) : 0) | BIT(24) | imem_base); +} + +static void +gm200_flcn_pio_imem_wr(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len, u16 tag) +{ + nvkm_falcon_wr32(falcon, 0x188 + (port * 0x10), tag++); + while (len >= 4) { + nvkm_falcon_wr32(falcon, 0x184 + (port * 0x10), *(u32 *)img); + img += 4; + len -= 4; + } +} + +const struct nvkm_falcon_func_pio +gm200_flcn_imem_pio = { + .min = 0x100, + .max = 0x100, + .wr_init = gm200_flcn_pio_imem_wr_init, + .wr = gm200_flcn_pio_imem_wr, +}; + int gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *falcon) { @@ -81,3 +130,98 @@ gm200_flcn_disable(struct nvkm_falcon *falcon) return 0; } + +int +gm200_flcn_fw_boot(struct nvkm_falcon_fw *fw, u32 *pmbox0, u32 *pmbox1, u32 mbox0_ok, u32 irqsclr) +{ + struct nvkm_falcon *falcon = fw->falcon; + u32 mbox0, mbox1; + int ret = 0; + + nvkm_falcon_wr32(falcon, 0x040, pmbox0 ? *pmbox0 : 0xcafebeef); + if (pmbox1) + nvkm_falcon_wr32(falcon, 0x044, *pmbox1); + + nvkm_falcon_wr32(falcon, 0x104, fw->boot_addr); + nvkm_falcon_wr32(falcon, 0x100, 0x00000002); + + if (nvkm_msec(falcon->owner->device, 2000, + if (nvkm_falcon_rd32(falcon, 0x100) & 0x00000010) + break; + ) < 0) + ret = -ETIMEDOUT; + + mbox0 = nvkm_falcon_rd32(falcon, 0x040); + mbox1 = nvkm_falcon_rd32(falcon, 0x044); + if (FLCN_ERRON(falcon, ret || mbox0 != mbox0_ok, "mbox %08x %08x", mbox0, mbox1)) + ret = ret ?: -EIO; + + if (irqsclr) + nvkm_falcon_mask(falcon, 0x004, 0xffffffff, irqsclr); + + return ret; +} + +int +gm200_flcn_fw_load(struct nvkm_falcon_fw *fw) +{ + struct nvkm_falcon *falcon = fw->falcon; + int ret; + + if (1) { + nvkm_falcon_mask(falcon, 0x624, 0x00000080, 0x00000080); + nvkm_falcon_wr32(falcon, 0x10c, 0x00000000); + } + + ret = nvkm_falcon_pio_wr(falcon, fw->fw.img + fw->nmem_base_img, fw->nmem_base_img, 0, + IMEM, fw->nmem_base, fw->nmem_size, fw->nmem_base >> 8, false); + if (ret) + return ret; + + ret = nvkm_falcon_pio_wr(falcon, fw->fw.img + fw->imem_base_img, fw->imem_base_img, 0, + IMEM, fw->imem_base, fw->imem_size, fw->imem_base >> 8, true); + if (ret) + return ret; + + ret = nvkm_falcon_pio_wr(falcon, fw->fw.img + fw->dmem_base_img, fw->dmem_base_img, 0, + DMEM, fw->dmem_base, fw->dmem_size, 0, false); + if (ret) + return ret; + + return 0; +} + +int +gm200_flcn_fw_reset(struct nvkm_falcon_fw *fw) +{ + return nvkm_falcon_reset(fw->falcon); +} + +int +gm200_flcn_fw_signature(struct nvkm_falcon_fw *fw, u32 *sig_base_src) +{ + struct nvkm_falcon *falcon = fw->falcon; + u32 addr = falcon->func->debug; + int ret = 0; + + if (addr) { + ret = nvkm_falcon_enable(falcon); + if (ret) + return ret; + + if (nvkm_falcon_rd32(falcon, addr) & 0x00100000) { + *sig_base_src = fw->sig_base_dbg; + return 1; + } + } + + return ret; +} + +const struct nvkm_falcon_fw_func +gm200_flcn_fw = { + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .load = gm200_flcn_fw_load, + .boot = gm200_flcn_fw_boot, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c index 0e78b3d734a0f..e15ac4b7598e6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c @@ -24,71 +24,22 @@ #include "gf100.h" #include "ram.h" -#include -#include -#include -#include #include int gp102_fb_vpr_scrub(struct nvkm_fb *fb) { struct nvkm_subdev *subdev = &fb->subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_falcon *falcon = &device->nvdec[0]->falcon; - struct nvkm_blob *blob = &fb->vpr_scrubber; - const struct nvfw_bin_hdr *hsbin_hdr; - const struct nvfw_hs_header *fw_hdr; - const struct nvfw_hs_load_header *lhdr; - void *scrub_data; - u32 patch_loc, patch_sig; + struct nvkm_falcon_fw fw = {}; int ret; - nvkm_falcon_get(falcon, subdev); - - hsbin_hdr = nvfw_bin_hdr(subdev, blob->data); - fw_hdr = nvfw_hs_header(subdev, blob->data + hsbin_hdr->header_offset); - lhdr = nvfw_hs_load_header(subdev, blob->data + fw_hdr->hdr_offset); - scrub_data = blob->data + hsbin_hdr->data_offset; - - patch_loc = *(u32 *)(blob->data + fw_hdr->patch_loc); - patch_sig = *(u32 *)(blob->data + fw_hdr->patch_sig); - if (falcon->debug) { - memcpy(scrub_data + patch_loc, - blob->data + fw_hdr->sig_dbg_offset + patch_sig, - fw_hdr->sig_dbg_size); - } else { - memcpy(scrub_data + patch_loc, - blob->data + fw_hdr->sig_prod_offset + patch_sig, - fw_hdr->sig_prod_size); - } - - nvkm_falcon_reset(falcon); - nvkm_falcon_bind_context(falcon, NULL); - - nvkm_falcon_load_imem(falcon, scrub_data, lhdr->non_sec_code_off, - lhdr->non_sec_code_size, - lhdr->non_sec_code_off >> 8, 0, false); - nvkm_falcon_load_imem(falcon, scrub_data + lhdr->apps[0], - ALIGN(lhdr->apps[0], 0x100), - lhdr->apps[1], - lhdr->apps[0] >> 8, 0, true); - nvkm_falcon_load_dmem(falcon, scrub_data + lhdr->data_dma_base, 0, - lhdr->data_size, 0); - - nvkm_falcon_set_start_addr(falcon, 0x0); - nvkm_falcon_start(falcon); - - ret = nvkm_falcon_wait_for_halt(falcon, 500); - if (ret < 0) { - ret = -ETIMEDOUT; - goto end; - } + ret = nvkm_falcon_fw_ctor_hs(&gm200_flcn_fw, "mem-unlock", subdev, NULL, + "nvdec/scrubber", 0, &subdev->device->nvdec[0]->falcon, &fw); + if (ret) + return ret; - /* put nvdec in clean state - without reset it will remain in HS mode */ - nvkm_falcon_reset(falcon); -end: - nvkm_falcon_put(falcon, subdev); + ret = nvkm_falcon_fw_boot(&fw, subdev, true, NULL, NULL, 0, 0x00000000); + nvkm_falcon_fw_dtor(&fw); return ret; } -- GitLab From 5728d064190e169f1a42381bd7e5fc4d411f3188 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:51 +1000 Subject: [PATCH 1346/1927] drm/nouveau/fb: handle sysmem flush page from common code - also executes pre-DEVINIT, so early boot is able to DMA sysmem Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 5 +++ drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 44 +++++++++++++++++-- .../gpu/drm/nouveau/nvkm/subdev/fb/ga100.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/fb/ga102.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/fb/gf100.c | 25 +++-------- .../gpu/drm/nouveau/nvkm/subdev/fb/gf100.h | 4 -- .../gpu/drm/nouveau/nvkm/subdev/fb/gm200.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/fb/gp100.c | 18 +------- .../gpu/drm/nouveau/nvkm/subdev/fb/gp102.c | 3 +- .../gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c | 1 + .../gpu/drm/nouveau/nvkm/subdev/fb/gv100.c | 3 +- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c | 38 +++------------- drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h | 2 - drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h | 9 +++- 15 files changed, 80 insertions(+), 83 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index ef6a6297148c1..f2bd9b11c84a2 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -35,6 +35,11 @@ struct nvkm_fb { struct nvkm_blob vpr_scrubber; + struct { + struct page *flush_page; + dma_addr_t flush_page_addr; + } sysmem; + struct nvkm_ram *ram; struct { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 6faaea948fc44..2ebdec44236d8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -57,6 +57,15 @@ nvkm_fb_tile_prog(struct nvkm_fb *fb, int region, struct nvkm_fb_tile *tile) } } +static void +nvkm_fb_sysmem_flush_page_init(struct nvkm_device *device) +{ + struct nvkm_fb *fb = device->fb; + + if (fb->func->sysmem.flush_page_init) + fb->func->sysmem.flush_page_init(fb); +} + int nvkm_fb_bios_memtype(struct nvkm_bios *bios) { @@ -168,6 +177,8 @@ nvkm_fb_init(struct nvkm_subdev *subdev) for (i = 0; i < fb->tile.regions; i++) fb->func->tile.prog(fb, i, &fb->tile.region[i]); + nvkm_fb_sysmem_flush_page_init(subdev->device); + if (fb->func->init) fb->func->init(fb); @@ -193,6 +204,13 @@ nvkm_fb_init(struct nvkm_subdev *subdev) return 0; } +static int +nvkm_fb_preinit(struct nvkm_subdev *subdev) +{ + nvkm_fb_sysmem_flush_page_init(subdev->device); + return 0; +} + static void * nvkm_fb_dtor(struct nvkm_subdev *subdev) { @@ -212,20 +230,28 @@ nvkm_fb_dtor(struct nvkm_subdev *subdev) nvkm_blob_dtor(&fb->vpr_scrubber); + if (fb->sysmem.flush_page) { + dma_unmap_page(subdev->device->dev, fb->sysmem.flush_page_addr, + PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(fb->sysmem.flush_page); + } + if (fb->func->dtor) return fb->func->dtor(fb); + return fb; } static const struct nvkm_subdev_func nvkm_fb = { .dtor = nvkm_fb_dtor, + .preinit = nvkm_fb_preinit, .oneinit = nvkm_fb_oneinit, .init = nvkm_fb_init, .intr = nvkm_fb_intr, }; -void +int nvkm_fb_ctor(const struct nvkm_fb_func *func, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fb *fb) { @@ -234,6 +260,19 @@ nvkm_fb_ctor(const struct nvkm_fb_func *func, struct nvkm_device *device, fb->tile.regions = fb->func->tile.regions; fb->page = nvkm_longopt(device->cfgopt, "NvFbBigPage", fb->func->default_bigpage); mutex_init(&fb->tags.mutex); + + if (func->sysmem.flush_page_init) { + fb->sysmem.flush_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!fb->sysmem.flush_page) + return -ENOMEM; + + fb->sysmem.flush_page_addr = dma_map_page(device->dev, fb->sysmem.flush_page, + 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(device->dev, fb->sysmem.flush_page_addr)) + return -EFAULT; + } + + return 0; } int @@ -242,6 +281,5 @@ nvkm_fb_new_(const struct nvkm_fb_func *func, struct nvkm_device *device, { if (!(*pfb = kzalloc(sizeof(**pfb), GFP_KERNEL))) return -ENOMEM; - nvkm_fb_ctor(func, device, type, inst, *pfb); - return 0; + return nvkm_fb_ctor(func, device, type, inst, *pfb); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga100.c index b47bebfbc26f9..5098f219e3e6f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga100.c @@ -26,9 +26,10 @@ static const struct nvkm_fb_func ga100_fb = { .dtor = gf100_fb_dtor, .oneinit = gf100_fb_oneinit, - .init = gp100_fb_init, + .init = gm200_fb_init, .init_page = gv100_fb_init_page, .init_unkn = gp100_fb_init_unkn, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .ram_new = gp100_ram_new, .default_bigpage = 16, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c index 6ea7908f05638..52435c0a485c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c @@ -26,9 +26,10 @@ static const struct nvkm_fb_func ga102_fb = { .dtor = gf100_fb_dtor, .oneinit = gf100_fb_oneinit, - .init = gp100_fb_init, + .init = gm200_fb_init, .init_page = gv100_fb_init_page, .init_unkn = gp100_fb_init_unkn, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .ram_new = ga102_ram_new, .default_bigpage = 16, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c index 9dcc40f9ef797..07db9b397ac1f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c @@ -61,14 +61,6 @@ gf100_fb_oneinit(struct nvkm_fb *base) if (ret) return ret; - fb->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (fb->r100c10_page) { - fb->r100c10 = dma_map_page(device->dev, fb->r100c10_page, 0, - PAGE_SIZE, DMA_BIDIRECTIONAL); - if (dma_mapping_error(device->dev, fb->r100c10)) - return -EFAULT; - } - return 0; } @@ -85,15 +77,18 @@ gf100_fb_init_page(struct nvkm_fb *fb) return 0; } +void +gf100_fb_sysmem_flush_page_init(struct nvkm_fb *fb) +{ + nvkm_wr32(fb->subdev.device, 0x100c10, fb->sysmem.flush_page_addr >> 8); +} + void gf100_fb_init(struct nvkm_fb *base) { struct gf100_fb *fb = gf100_fb(base); struct nvkm_device *device = fb->base.subdev.device; - if (fb->r100c10_page) - nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8); - if (base->func->clkgate_pack) { nvkm_therm_clkgate_init(device->therm, base->func->clkgate_pack); @@ -104,13 +99,6 @@ void * gf100_fb_dtor(struct nvkm_fb *base) { struct gf100_fb *fb = gf100_fb(base); - struct nvkm_device *device = fb->base.subdev.device; - - if (fb->r100c10_page) { - dma_unmap_page(device->dev, fb->r100c10, PAGE_SIZE, - DMA_BIDIRECTIONAL); - __free_page(fb->r100c10_page); - } return fb; } @@ -136,6 +124,7 @@ gf100_fb = { .init = gf100_fb_init, .init_page = gf100_fb_init_page, .intr = gf100_fb_intr, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .ram_new = gf100_ram_new, .default_bigpage = 17, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h index 0cac7b06acc81..77472b558591f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h @@ -6,8 +6,6 @@ struct gf100_fb { struct nvkm_fb base; - struct page *r100c10_page; - dma_addr_t r100c10; }; int gf100_fb_new_(const struct nvkm_fb_func *, struct nvkm_device *, enum nvkm_subdev_type, int, @@ -16,7 +14,5 @@ void *gf100_fb_dtor(struct nvkm_fb *); void gf100_fb_init(struct nvkm_fb *); void gf100_fb_intr(struct nvkm_fb *); -void gp100_fb_init(struct nvkm_fb *); - void gm200_fb_init(struct nvkm_fb *base); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c index 5acf8d15d06fb..fb02092a65eb7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c @@ -46,9 +46,6 @@ gm200_fb_init(struct nvkm_fb *base) struct gf100_fb *fb = gf100_fb(base); struct nvkm_device *device = fb->base.subdev.device; - if (fb->r100c10_page) - nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8); - nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->base.mmu_wr) >> 8); nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->base.mmu_rd) >> 8); nvkm_mask(device, 0x100cc4, 0x00060000, @@ -62,6 +59,7 @@ gm200_fb = { .init = gm200_fb_init, .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .ram_new = gm200_ram_new, .default_bigpage = 0 /* per-instance. */, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c index 86f61a3f2feae..50875af94c18e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm20b.c @@ -30,6 +30,7 @@ gm20b_fb = { .init = gm200_fb_init, .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .default_bigpage = 0 /* per-instance. */, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c index 09e943edc3628..110c08c948497 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c @@ -44,29 +44,15 @@ gp100_fb_init_remapper(struct nvkm_fb *fb) nvkm_mask(device, 0x100c14, 0x00040000, 0x00000000); } -void -gp100_fb_init(struct nvkm_fb *base) -{ - struct gf100_fb *fb = gf100_fb(base); - struct nvkm_device *device = fb->base.subdev.device; - - if (fb->r100c10_page) - nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8); - - nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->base.mmu_wr) >> 8); - nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->base.mmu_rd) >> 8); - nvkm_mask(device, 0x100cc4, 0x00060000, - min(nvkm_memory_size(fb->base.mmu_rd) >> 16, (u64)2) << 17); -} - static const struct nvkm_fb_func gp100_fb = { .dtor = gf100_fb_dtor, .oneinit = gf100_fb_oneinit, - .init = gp100_fb_init, + .init = gm200_fb_init, .init_remapper = gp100_fb_init_remapper, .init_page = gm200_fb_init_page, .init_unkn = gp100_fb_init_unkn, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .ram_new = gp100_ram_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c index e15ac4b7598e6..2658481d575b6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp102.c @@ -55,9 +55,10 @@ static const struct nvkm_fb_func gp102_fb = { .dtor = gf100_fb_dtor, .oneinit = gf100_fb_oneinit, - .init = gp100_fb_init, + .init = gm200_fb_init, .init_remapper = gp100_fb_init_remapper, .init_page = gm200_fb_init_page, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .vpr.scrub_required = gp102_fb_vpr_scrub_required, .vpr.scrub = gp102_fb_vpr_scrub, .ram_new = gp100_ram_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c index 84c9815a6d48a..a04a5f7120194 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp10b.c @@ -28,6 +28,7 @@ gp10b_fb = { .init = gm200_fb_init, .init_page = gm200_fb_init_page, .intr = gf100_fb_intr, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, }; int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c index 63daa83ae12d1..1f0126437c1aa 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gv100.c @@ -32,9 +32,10 @@ static const struct nvkm_fb_func gv100_fb = { .dtor = gf100_fb_dtor, .oneinit = gf100_fb_oneinit, - .init = gp100_fb_init, + .init = gm200_fb_init, .init_page = gv100_fb_init_page, .init_unkn = gp100_fb_init_unkn, + .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .vpr.scrub_required = gp102_fb_vpr_scrub_required, .vpr.scrub = gp102_fb_vpr_scrub, .ram_new = gp100_ram_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c index 48641735168e7..a6efbd913c138 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.c @@ -191,35 +191,12 @@ nv50_fb_intr(struct nvkm_fb *base) nvkm_chan_put(&chan, flags); } -static int -nv50_fb_oneinit(struct nvkm_fb *base) -{ - struct nv50_fb *fb = nv50_fb(base); - struct nvkm_device *device = fb->base.subdev.device; - - fb->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO); - if (fb->r100c08_page) { - fb->r100c08 = dma_map_page(device->dev, fb->r100c08_page, 0, - PAGE_SIZE, DMA_BIDIRECTIONAL); - if (dma_mapping_error(device->dev, fb->r100c08)) - return -EFAULT; - } - - return 0; -} - static void nv50_fb_init(struct nvkm_fb *base) { struct nv50_fb *fb = nv50_fb(base); struct nvkm_device *device = fb->base.subdev.device; - /* Not a clue what this is exactly. Without pointing it at a - * scratch page, VRAM->GART blits with M2MF (as in DDX DFS) - * cause IOMMU "read from address 0" errors (rh#561267) - */ - nvkm_wr32(device, 0x100c08, fb->r100c08 >> 8); - /* This is needed to get meaningful information from 100c90 * on traps. No idea what these values mean exactly. */ nvkm_wr32(device, 0x100c90, fb->func->trap); @@ -234,17 +211,16 @@ nv50_fb_tags(struct nvkm_fb *base) return 0; } +static void +nv50_fb_sysmem_flush_page_init(struct nvkm_fb *fb) +{ + nvkm_wr32(fb->subdev.device, 0x100c08, fb->sysmem.flush_page_addr >> 8); +} + static void * nv50_fb_dtor(struct nvkm_fb *base) { struct nv50_fb *fb = nv50_fb(base); - struct nvkm_device *device = fb->base.subdev.device; - - if (fb->r100c08_page) { - dma_unmap_page(device->dev, fb->r100c08, PAGE_SIZE, - DMA_BIDIRECTIONAL); - __free_page(fb->r100c08_page); - } return fb; } @@ -253,9 +229,9 @@ static const struct nvkm_fb_func nv50_fb_ = { .dtor = nv50_fb_dtor, .tags = nv50_fb_tags, - .oneinit = nv50_fb_oneinit, .init = nv50_fb_init, .intr = nv50_fb_intr, + .sysmem.flush_page_init = nv50_fb_sysmem_flush_page_init, .ram_new = nv50_fb_ram_new, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h index a5e673859a901..4f68bc4513a7e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/nv50.h @@ -7,8 +7,6 @@ struct nv50_fb { const struct nv50_fb_func *func; struct nvkm_fb base; - struct page *r100c08_page; - dma_addr_t r100c08; }; struct nv50_fb_func { diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h index 3f1be9780c65f..ac03eac0f2616 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h @@ -16,6 +16,10 @@ struct nvkm_fb_func { void (*init_unkn)(struct nvkm_fb *); void (*intr)(struct nvkm_fb *); + struct nvkm_fb_func_sysmem { + void (*flush_page_init)(struct nvkm_fb *); + } sysmem; + struct { bool (*scrub_required)(struct nvkm_fb *); int (*scrub)(struct nvkm_fb *); @@ -37,8 +41,8 @@ struct nvkm_fb_func { const struct nvkm_therm_clkgate_pack *clkgate_pack; }; -void nvkm_fb_ctor(const struct nvkm_fb_func *, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_fb *); +int nvkm_fb_ctor(const struct nvkm_fb_func *, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_fb *); int nvkm_fb_new_(const struct nvkm_fb_func *, struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_fb **); int nvkm_fb_bios_memtype(struct nvkm_bios *); @@ -72,6 +76,7 @@ void nv46_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size, int gf100_fb_oneinit(struct nvkm_fb *); int gf100_fb_init_page(struct nvkm_fb *); +void gf100_fb_sysmem_flush_page_init(struct nvkm_fb *); int gm200_fb_init_page(struct nvkm_fb *); -- GitLab From e3f324956a32d08a9361ee1e3beca383f1b01eba Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:52 +1000 Subject: [PATCH 1347/1927] drm/nouveau/fb/gp102-: unlock VPR right after devinit Under memory load, instmem allocations could end up in the regions of VRAM that are inaccessible right after boot, and be corrupted after a suspend/resume cycle as a result of being restored before booting the mem unlock firmware. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/subdev/fb.h | 2 ++ .../gpu/drm/nouveau/nvkm/engine/device/base.c | 4 ++++ drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h index f2bd9b11c84a2..40768373cdd90 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h @@ -58,6 +58,8 @@ struct nvkm_fb { struct nvkm_memory *mmu_wr; }; +int nvkm_fb_mem_unlock(struct nvkm_fb *); + void nvkm_fb_tile_init(struct nvkm_fb *, int region, u32 addr, u32 size, u32 pitch, u32 flags, struct nvkm_fb_tile *); void nvkm_fb_tile_fini(struct nvkm_fb *, int region, struct nvkm_fb_tile *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 852c18aec4cdb..c94de7b455567 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2808,6 +2808,10 @@ nvkm_device_preinit(struct nvkm_device *device) if (ret) goto fail; + ret = nvkm_fb_mem_unlock(device->fb); + if (ret) + goto fail; + time = ktime_to_us(ktime_get()) - time; nvdev_trace(device, "preinit completed in %lldus\n", time); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c index 2ebdec44236d8..bac7dcc4c2c13 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c @@ -134,12 +134,20 @@ nvkm_fb_oneinit(struct nvkm_subdev *subdev) return nvkm_mm_init(&fb->tags.mm, 0, 0, tags, 1); } -static int -nvkm_fb_init_scrub_vpr(struct nvkm_fb *fb) +int +nvkm_fb_mem_unlock(struct nvkm_fb *fb) { struct nvkm_subdev *subdev = &fb->subdev; int ret; + if (!fb->func->vpr.scrub_required) + return 0; + + if (!fb->func->vpr.scrub_required(fb)) { + nvkm_debug(subdev, "VPR not locked\n"); + return 0; + } + nvkm_debug(subdev, "VPR locked, running scrubber binary\n"); if (!fb->vpr_scrubber.size) { @@ -194,13 +202,6 @@ nvkm_fb_init(struct nvkm_subdev *subdev) if (fb->func->init_unkn) fb->func->init_unkn(fb); - if (fb->func->vpr.scrub_required && - fb->func->vpr.scrub_required(fb)) { - ret = nvkm_fb_init_scrub_vpr(fb); - if (ret) - return ret; - } - return 0; } -- GitLab From 2541626cfb794e57ba0575a6920826f591f7ced0 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:52 +1000 Subject: [PATCH 1348/1927] drm/nouveau/acr: use common falcon HS FW code for ACR FWs Adds context binding and support for FWs with a bootloader to the code that was added to load VPR scrubber HS binaries, and ports ACR over to using all of it. - gv100 split from gp108 to handle FW exit status differences Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../drm/nouveau/include/nvkm/core/falcon.h | 21 +- .../drm/nouveau/include/nvkm/core/firmware.h | 8 + .../drm/nouveau/include/nvkm/engine/falcon.h | 32 ++- .../gpu/drm/nouveau/include/nvkm/subdev/acr.h | 3 +- drivers/gpu/drm/nouveau/nvkm/core/firmware.c | 87 +++++++- .../gpu/drm/nouveau/nvkm/engine/device/base.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/sec2/gp102.c | 53 +---- .../gpu/drm/nouveau/nvkm/engine/sec2/tu102.c | 14 +- drivers/gpu/drm/nouveau/nvkm/falcon/base.c | 79 ++++--- drivers/gpu/drm/nouveau/nvkm/falcon/cmdq.c | 2 +- drivers/gpu/drm/nouveau/nvkm/falcon/fw.c | 75 ++++++- drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c | 88 +++++++- drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c | 42 ++++ drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c | 2 +- drivers/gpu/drm/nouveau/nvkm/falcon/v1.c | 167 --------------- .../gpu/drm/nouveau/nvkm/subdev/acr/Kbuild | 2 +- .../gpu/drm/nouveau/nvkm/subdev/acr/base.c | 73 ++++--- .../gpu/drm/nouveau/nvkm/subdev/acr/gm200.c | 195 ++++-------------- .../gpu/drm/nouveau/nvkm/subdev/acr/gm20b.c | 44 ++-- .../gpu/drm/nouveau/nvkm/subdev/acr/gp102.c | 27 +-- .../gpu/drm/nouveau/nvkm/subdev/acr/gp108.c | 55 +++-- .../gpu/drm/nouveau/nvkm/subdev/acr/gp10b.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/acr/gv100.c | 67 ++++++ .../gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c | 177 ---------------- .../gpu/drm/nouveau/nvkm/subdev/acr/priv.h | 93 +++------ .../gpu/drm/nouveau/nvkm/subdev/acr/tu102.c | 36 +--- .../gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c | 14 +- .../gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c | 31 ++- .../gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c | 12 +- .../gpu/drm/nouveau/nvkm/subdev/pmu/priv.h | 1 + 30 files changed, 648 insertions(+), 856 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/acr/gv100.c delete mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index d8fd2cbd2872c..4c8b668bacb45 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -6,6 +6,7 @@ enum nvkm_falcon_mem { IMEM, DMEM, + EMEM, }; static inline const char * @@ -14,6 +15,7 @@ nvkm_falcon_mem(enum nvkm_falcon_mem mem) switch (mem) { case IMEM: return "imem"; case DMEM: return "dmem"; + case EMEM: return "emem"; default: WARN_ON(1); return "?mem"; @@ -25,6 +27,8 @@ struct nvkm_falcon_func_pio { int max; void (*wr_init)(struct nvkm_falcon *, u8 port, bool sec, u32 mem_base); void (*wr)(struct nvkm_falcon *, u8 port, const u8 *img, int len, u16 tag); + void (*rd_init)(struct nvkm_falcon *, u8 port, u32 mem_base); + void (*rd)(struct nvkm_falcon *, u8 port, const u8 *img, int len); }; int nvkm_falcon_ctor(const struct nvkm_falcon_func *, struct nvkm_subdev *owner, @@ -33,27 +37,25 @@ void nvkm_falcon_dtor(struct nvkm_falcon *); int nvkm_falcon_reset(struct nvkm_falcon *); int nvkm_falcon_pio_wr(struct nvkm_falcon *, const u8 *img, u32 img_base, u8 port, enum nvkm_falcon_mem mem_type, u32 mem_base, int len, u16 tag, bool sec); +int nvkm_falcon_pio_rd(struct nvkm_falcon *, u8 port, enum nvkm_falcon_mem type, u32 mem_base, + const u8 *img, u32 img_base, int len); int gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *); int gm200_flcn_disable(struct nvkm_falcon *); int gm200_flcn_enable(struct nvkm_falcon *); +void gm200_flcn_bind_inst(struct nvkm_falcon *, int, u64); +int gm200_flcn_bind_stat(struct nvkm_falcon *, bool); extern const struct nvkm_falcon_func_pio gm200_flcn_imem_pio; extern const struct nvkm_falcon_func_pio gm200_flcn_dmem_pio; int gp102_flcn_reset_eng(struct nvkm_falcon *); +extern const struct nvkm_falcon_func_pio gp102_flcn_emem_pio; void nvkm_falcon_v1_load_imem(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); void nvkm_falcon_v1_load_dmem(struct nvkm_falcon *, void *, u32, u32, u8); -void nvkm_falcon_v1_read_dmem(struct nvkm_falcon *, u32, u32, u8, void *); -void nvkm_falcon_v1_bind_context(struct nvkm_falcon *, struct nvkm_memory *); -int nvkm_falcon_v1_wait_for_halt(struct nvkm_falcon *, u32); -int nvkm_falcon_v1_clear_interrupt(struct nvkm_falcon *, u32); -void nvkm_falcon_v1_set_start_addr(struct nvkm_falcon *, u32 start_addr); void nvkm_falcon_v1_start(struct nvkm_falcon *); -void gp102_sec2_flcn_bind_context(struct nvkm_falcon *, struct nvkm_memory *); - #define FLCN_PRINTK(f,l,p,fmt,a...) ({ \ if ((f)->owner->name != (f)->name) \ nvkm_printk___((f)->owner, (f)->user, NV_DBG_##l, p, "%s:"fmt, (f)->name, ##a); \ @@ -70,7 +72,9 @@ struct nvkm_falcon_fw { const struct nvkm_falcon_fw_func { int (*signature)(struct nvkm_falcon_fw *, u32 *sig_base_src); int (*reset)(struct nvkm_falcon_fw *); + int (*setup)(struct nvkm_falcon_fw *); int (*load)(struct nvkm_falcon_fw *); + int (*load_bld)(struct nvkm_falcon_fw *); int (*boot)(struct nvkm_falcon_fw *, u32 *mbox0, u32 *mbox1, u32 mbox0_ok, u32 irqsclr); } *func; @@ -96,11 +100,14 @@ struct nvkm_falcon_fw { u32 dmem_size; u32 dmem_sign; + u8 *boot; + u32 boot_size; u32 boot_addr; struct nvkm_falcon *falcon; struct nvkm_memory *inst; struct nvkm_vmm *vmm; + struct nvkm_vma *vma; }; int nvkm_falcon_fw_ctor(const struct nvkm_falcon_fw_func *, const char *name, struct nvkm_device *, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h b/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h index 8453025891a5e..d4e507e252b13 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/firmware.h @@ -1,6 +1,7 @@ /* SPDX-License-Identifier: MIT */ #ifndef __NVKM_FIRMWARE_H__ #define __NVKM_FIRMWARE_H__ +#include #include #include @@ -8,6 +9,7 @@ struct nvkm_firmware { const struct nvkm_firmware_func { enum nvkm_firmware_type { NVKM_FIRMWARE_IMG_RAM, + NVKM_FIRMWARE_IMG_DMA, } type; } *func; const char *name; @@ -15,6 +17,12 @@ struct nvkm_firmware { int len; u8 *img; + u64 phys; + + struct nvkm_firmware_mem { + struct nvkm_memory memory; + struct scatterlist sgl; + } mem; }; int nvkm_firmware_ctor(const struct nvkm_firmware_func *, const char *name, struct nvkm_device *, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index ca751199379e3..f576ca246d100 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -64,9 +64,22 @@ struct nvkm_falcon_func { int (*reset_wait_mem_scrubbing)(struct nvkm_falcon *); u32 debug; + void (*bind_inst)(struct nvkm_falcon *, int target, u64 addr); + int (*bind_stat)(struct nvkm_falcon *, bool intr); + bool bind_intr; + const struct nvkm_falcon_func_pio *imem_pio; const struct nvkm_falcon_func_pio *dmem_pio; + u32 emem_addr; + const struct nvkm_falcon_func_pio *emem_pio; + + struct { + u32 head; + u32 tail; + u32 stride; + } cmdq, msgq; + struct { u32 *data; u32 size; @@ -78,24 +91,10 @@ struct nvkm_falcon_func { void (*init)(struct nvkm_falcon *); void (*intr)(struct nvkm_falcon *, struct nvkm_chan *); - u32 fbif; - void (*load_imem)(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); void (*load_dmem)(struct nvkm_falcon *, void *, u32, u32, u8); - void (*read_dmem)(struct nvkm_falcon *, u32, u32, u8, void *); - u32 emem_addr; - void (*bind_context)(struct nvkm_falcon *, struct nvkm_memory *); - int (*wait_for_halt)(struct nvkm_falcon *, u32); - int (*clear_interrupt)(struct nvkm_falcon *, u32); - void (*set_start_addr)(struct nvkm_falcon *, u32 start_addr); void (*start)(struct nvkm_falcon *); - struct { - u32 head; - u32 tail; - u32 stride; - } cmdq, msgq; - struct nvkm_sclass sclass[]; }; @@ -122,10 +121,5 @@ nvkm_falcon_mask(struct nvkm_falcon *falcon, u32 addr, u32 mask, u32 val) void nvkm_falcon_load_imem(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); void nvkm_falcon_load_dmem(struct nvkm_falcon *, void *, u32, u32, u8); -void nvkm_falcon_read_dmem(struct nvkm_falcon *, u32, u32, u8, void *); -void nvkm_falcon_bind_context(struct nvkm_falcon *, struct nvkm_memory *); -void nvkm_falcon_set_start_addr(struct nvkm_falcon *, u32); void nvkm_falcon_start(struct nvkm_falcon *); -int nvkm_falcon_wait_for_halt(struct nvkm_falcon *, u32); -int nvkm_falcon_clear_interrupt(struct nvkm_falcon *, u32); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h index fe5024b6d0f37..12c0b8395262d 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h @@ -36,7 +36,7 @@ struct nvkm_acr { const struct nvkm_acr_func *func; struct nvkm_subdev subdev; - struct list_head hsfw, hsf; + struct list_head hsfw; struct list_head lsfw, lsf; u64 managed_falcons; @@ -65,6 +65,7 @@ int gm20b_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gp102_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); int gp108_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); int gp10b_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); +int gv100_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); int tu102_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); struct nvkm_acr_lsfw { diff --git a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c index aa1bf6b5a8cdc..fcf2a002f6cb2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/firmware.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/firmware.c @@ -22,6 +22,9 @@ #include #include +#include +#include + int nvkm_firmware_load_name(const struct nvkm_subdev *subdev, const char *base, const char *name, int ver, const struct firmware **pfw) @@ -108,9 +111,70 @@ nvkm_firmware_put(const struct firmware *fw) release_firmware(fw); } +#define nvkm_firmware_mem(p) container_of((p), struct nvkm_firmware, mem.memory) + +static int +nvkm_firmware_mem_map(struct nvkm_memory *memory, u64 offset, struct nvkm_vmm *vmm, + struct nvkm_vma *vma, void *argv, u32 argc) +{ + struct nvkm_firmware *fw = nvkm_firmware_mem(memory); + struct nvkm_vmm_map map = { + .memory = &fw->mem.memory, + .offset = offset, + .sgl = &fw->mem.sgl, + }; + + if (WARN_ON(fw->func->type != NVKM_FIRMWARE_IMG_DMA)) + return -ENOSYS; + + return nvkm_vmm_map(vmm, vma, argv, argc, &map); +} + +static u64 +nvkm_firmware_mem_size(struct nvkm_memory *memory) +{ + return sg_dma_len(&nvkm_firmware_mem(memory)->mem.sgl); +} + +static u64 +nvkm_firmware_mem_addr(struct nvkm_memory *memory) +{ + return nvkm_firmware_mem(memory)->phys; +} + +static u8 +nvkm_firmware_mem_page(struct nvkm_memory *memory) +{ + return PAGE_SHIFT; +} + +static enum nvkm_memory_target +nvkm_firmware_mem_target(struct nvkm_memory *memory) +{ + return NVKM_MEM_TARGET_HOST; +} + +static void * +nvkm_firmware_mem_dtor(struct nvkm_memory *memory) +{ + return NULL; +} + +static const struct nvkm_memory_func +nvkm_firmware_mem = { + .dtor = nvkm_firmware_mem_dtor, + .target = nvkm_firmware_mem_target, + .page = nvkm_firmware_mem_page, + .addr = nvkm_firmware_mem_addr, + .size = nvkm_firmware_mem_size, + .map = nvkm_firmware_mem_map, +}; + void nvkm_firmware_dtor(struct nvkm_firmware *fw) { + struct nvkm_memory *memory = &fw->mem.memory; + if (!fw->img) return; @@ -118,6 +182,10 @@ nvkm_firmware_dtor(struct nvkm_firmware *fw) case NVKM_FIRMWARE_IMG_RAM: kfree(fw->img); break; + case NVKM_FIRMWARE_IMG_DMA: + nvkm_memory_unref(&memory); + dma_free_coherent(fw->device->dev, sg_dma_len(&fw->mem.sgl), fw->img, fw->phys); + break; default: WARN_ON(1); break; @@ -133,12 +201,28 @@ nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, fw->func = func; fw->name = name; fw->device = device; + fw->len = len; switch (fw->func->type) { case NVKM_FIRMWARE_IMG_RAM: - fw->len = len; fw->img = kmemdup(src, fw->len, GFP_KERNEL); break; + case NVKM_FIRMWARE_IMG_DMA: { + dma_addr_t addr; + + len = ALIGN(fw->len, PAGE_SIZE); + + fw->img = dma_alloc_coherent(fw->device->dev, len, &addr, GFP_KERNEL); + if (fw->img) { + memcpy(fw->img, src, fw->len); + fw->phys = addr; + } + + sg_init_one(&fw->mem.sgl, fw->img, len); + sg_dma_address(&fw->mem.sgl) = fw->phys; + sg_dma_len(&fw->mem.sgl) = len; + } + break; default: WARN_ON(1); return -EINVAL; @@ -147,5 +231,6 @@ nvkm_firmware_ctor(const struct nvkm_firmware_func *func, const char *name, if (!fw->img) return -ENOMEM; + nvkm_memory_ctor(&nvkm_firmware_mem, &fw->mem.memory); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index c94de7b455567..931a595818159 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2364,7 +2364,7 @@ nv13b_chipset = { static const struct nvkm_device_chip nv140_chipset = { .name = "GV100", - .acr = { 0x00000001, gp108_acr_new }, + .acr = { 0x00000001, gv100_acr_new }, .bar = { 0x00000001, gm107_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .bus = { 0x00000001, gf100_bus_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index ae910c9bdc0a7..a5c987ce58ed5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -190,45 +190,6 @@ gp102_sec2_intr(struct nvkm_inth *inth) return IRQ_HANDLED; } -void -gp102_sec2_flcn_bind_context(struct nvkm_falcon *falcon, - struct nvkm_memory *ctx) -{ - struct nvkm_device *device = falcon->owner->device; - - nvkm_falcon_v1_bind_context(falcon, ctx); - if (!ctx) - return; - - /* Not sure if this is a WAR for a HW issue, or some additional - * programming sequence that's needed to properly complete the - * context switch we trigger above. - * - * Fixes unreliability of booting the SEC2 RTOS on Quadro P620, - * particularly when resuming from suspend. - * - * Also removes the need for an odd workaround where we needed - * to program SEC2's FALCON_CPUCTL_ALIAS_STARTCPU twice before - * the SEC2 RTOS would begin executing. - */ - nvkm_msec(device, 10, - u32 irqstat = nvkm_falcon_rd32(falcon, 0x008); - u32 flcn0dc = nvkm_falcon_rd32(falcon, 0x0dc); - if ((irqstat & 0x00000008) && - (flcn0dc & 0x00007000) == 0x00005000) - break; - ); - - nvkm_falcon_mask(falcon, 0x004, 0x00000008, 0x00000008); - nvkm_falcon_mask(falcon, 0x058, 0x00000002, 0x00000002); - - nvkm_msec(device, 10, - u32 flcn0dc = nvkm_falcon_rd32(falcon, 0x0dc); - if ((flcn0dc & 0x00007000) == 0x00000000) - break; - ); -} - static const struct nvkm_falcon_func gp102_sec2_flcn = { .disable = gm200_flcn_disable, @@ -237,15 +198,13 @@ gp102_sec2_flcn = { .reset_eng = gp102_flcn_reset_eng, .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0x408, - .fbif = 0x600, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, + .bind_inst = gm200_flcn_bind_inst, + .bind_stat = gm200_flcn_bind_stat, + .bind_intr = true, + .imem_pio = &gm200_flcn_imem_pio, + .dmem_pio = &gm200_flcn_dmem_pio, .emem_addr = 0x01000000, - .bind_context = gp102_sec2_flcn_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, + .emem_pio = &gp102_flcn_emem_pio, .start = nvkm_falcon_v1_start, .cmdq = { 0xa00, 0xa04, 8 }, .msgq = { 0xa30, 0xa34, 8 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c index 0f859f3235047..0afc4b2fa5294 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/tu102.c @@ -32,15 +32,13 @@ tu102_sec2_flcn = { .reset_eng = gp102_flcn_reset_eng, .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0x408, - .fbif = 0x600, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, + .bind_inst = gm200_flcn_bind_inst, + .bind_stat = gm200_flcn_bind_stat, + .bind_intr = true, + .imem_pio = &gm200_flcn_imem_pio, + .dmem_pio = &gm200_flcn_dmem_pio, .emem_addr = 0x01000000, - .bind_context = gp102_sec2_flcn_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, + .emem_pio = &gp102_flcn_emem_pio, .start = nvkm_falcon_v1_start, .cmdq = { 0xc00, 0xc04, 8 }, .msgq = { 0xc80, 0xc84, 8 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index c15dda020c775..e4075aa441f37 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -22,6 +22,7 @@ #include "priv.h" #include +#include #include static const struct nvkm_falcon_func_pio * @@ -36,11 +37,48 @@ nvkm_falcon_pio(struct nvkm_falcon *falcon, enum nvkm_falcon_mem *mem_type, u32 *mem_base -= falcon->func->emem_addr; fallthrough; + case EMEM: + return falcon->func->emem_pio; default: return NULL; } } +int +nvkm_falcon_pio_rd(struct nvkm_falcon *falcon, u8 port, enum nvkm_falcon_mem mem_type, u32 mem_base, + const u8 *img, u32 img_base, int len) +{ + const struct nvkm_falcon_func_pio *pio = nvkm_falcon_pio(falcon, &mem_type, &mem_base); + const char *type = nvkm_falcon_mem(mem_type); + int xfer_len; + + if (WARN_ON(!pio || !pio->rd)) + return -EINVAL; + + FLCN_DBG(falcon, "%s %08x -> %08x bytes at %08x", type, mem_base, len, img_base); + if (WARN_ON(!len || (len & (pio->min - 1)))) + return -EINVAL; + + pio->rd_init(falcon, port, mem_base); + do { + xfer_len = min(len, pio->max); + pio->rd(falcon, port, img, xfer_len); + + if (nvkm_printk_ok(falcon->owner, falcon->user, NV_DBG_TRACE)) { + for (img_base = 0; img_base < xfer_len; img_base += 4, mem_base += 4) { + if (((img_base / 4) % 8) == 0) + printk(KERN_INFO "%s %08x ->", type, mem_base); + printk(KERN_CONT " %08x", *(u32 *)(img + img_base)); + } + } + + img += xfer_len; + len -= xfer_len; + } while (len); + + return 0; +} + int nvkm_falcon_pio_wr(struct nvkm_falcon *falcon, const u8 *img, u32 img_base, u8 port, enum nvkm_falcon_mem mem_type, u32 mem_base, int len, u16 tag, bool sec) @@ -103,35 +141,6 @@ nvkm_falcon_load_dmem(struct nvkm_falcon *falcon, void *data, u32 start, mutex_unlock(&falcon->dmem_mutex); } -void -nvkm_falcon_read_dmem(struct nvkm_falcon *falcon, u32 start, u32 size, u8 port, - void *data) -{ - mutex_lock(&falcon->dmem_mutex); - - falcon->func->read_dmem(falcon, start, size, port, data); - - mutex_unlock(&falcon->dmem_mutex); -} - -void -nvkm_falcon_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *inst) -{ - if (!falcon->func->bind_context) { - nvkm_error(falcon->user, - "Context binding not supported on this falcon!\n"); - return; - } - - falcon->func->bind_context(falcon, inst); -} - -void -nvkm_falcon_set_start_addr(struct nvkm_falcon *falcon, u32 start_addr) -{ - falcon->func->set_start_addr(falcon, start_addr); -} - void nvkm_falcon_start(struct nvkm_falcon *falcon) { @@ -150,18 +159,6 @@ nvkm_falcon_reset(struct nvkm_falcon *falcon) return nvkm_falcon_enable(falcon); } -int -nvkm_falcon_wait_for_halt(struct nvkm_falcon *falcon, u32 ms) -{ - return falcon->func->wait_for_halt(falcon, ms); -} - -int -nvkm_falcon_clear_interrupt(struct nvkm_falcon *falcon, u32 mask) -{ - return falcon->func->clear_interrupt(falcon, mask); -} - static int nvkm_falcon_oneinit(struct nvkm_falcon *falcon) { diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/cmdq.c b/drivers/gpu/drm/nouveau/nvkm/falcon/cmdq.c index 44cf6a8862e1e..211ebe7afac69 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/cmdq.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/cmdq.c @@ -51,7 +51,7 @@ static void nvkm_falcon_cmdq_push(struct nvkm_falcon_cmdq *cmdq, void *data, u32 size) { struct nvkm_falcon *falcon = cmdq->qmgr->falcon; - nvkm_falcon_load_dmem(falcon, data, cmdq->position, size, 0); + nvkm_falcon_pio_wr(falcon, data, 0, 0, DMEM, cmdq->position, size, 0, false); cmdq->position += ALIGN(size, QUEUE_ALIGNMENT); } diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c index 12a899dbec35b..13d52d7e4f60b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -93,6 +93,12 @@ nvkm_falcon_fw_boot(struct nvkm_falcon_fw *fw, struct nvkm_subdev *user, fw->func->reset(fw); FLCNFW_DBG(fw, "loading"); + if (fw->func->setup) { + ret = fw->func->setup(fw); + if (ret) + goto done; + } + ret = fw->func->load(fw); if (ret) goto done; @@ -114,21 +120,45 @@ int nvkm_falcon_fw_oneinit(struct nvkm_falcon_fw *fw, struct nvkm_falcon *falcon, struct nvkm_vmm *vmm, struct nvkm_memory *inst) { + int ret; + fw->falcon = falcon; fw->vmm = nvkm_vmm_ref(vmm); fw->inst = nvkm_memory_ref(inst); + + if (fw->boot) { + FLCN_DBG(falcon, "mapping %s fw", fw->fw.name); + ret = nvkm_vmm_get(fw->vmm, 12, nvkm_memory_size(&fw->fw.mem.memory), &fw->vma); + if (ret) { + FLCN_ERR(falcon, "get %d", ret); + return ret; + } + + ret = nvkm_memory_map(&fw->fw.mem.memory, 0, fw->vmm, fw->vma, NULL, 0); + if (ret) { + FLCN_ERR(falcon, "map %d", ret); + return ret; + } + } + return 0; } void nvkm_falcon_fw_dtor(struct nvkm_falcon_fw *fw) { + nvkm_vmm_put(fw->vmm, &fw->vma); nvkm_vmm_unref(&fw->vmm); nvkm_memory_unref(&fw->inst); nvkm_falcon_fw_dtor_sigs(fw); nvkm_firmware_dtor(&fw->fw); } +static const struct nvkm_firmware_func +nvkm_falcon_fw_dma = { + .type = NVKM_FIRMWARE_IMG_DMA, +}; + static const struct nvkm_firmware_func nvkm_falcon_fw = { .type = NVKM_FIRMWARE_IMG_RAM, @@ -160,7 +190,7 @@ nvkm_falcon_fw_ctor(const struct nvkm_falcon_fw_func *func, const char *name, struct nvkm_device *device, bool dma, const void *src, u32 len, struct nvkm_falcon *falcon, struct nvkm_falcon_fw *fw) { - const struct nvkm_firmware_func *type = &nvkm_falcon_fw; + const struct nvkm_firmware_func *type = dma ? &nvkm_falcon_fw_dma : &nvkm_falcon_fw; int ret; fw->func = func; @@ -181,6 +211,7 @@ nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *func, const char *name, const struct nvfw_bin_hdr *hdr; const struct nvfw_hs_header *hshdr; const struct nvfw_hs_load_header *lhdr; + const struct nvfw_bl_desc *desc; u32 loc, sig; int ret; @@ -190,14 +221,31 @@ nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *func, const char *name, hdr = nvfw_bin_hdr(subdev, blob->data); hshdr = nvfw_hs_header(subdev, blob->data + hdr->header_offset); - loc = *(u32 *)(blob->data + hshdr->patch_loc); - sig = *(u32 *)(blob->data + hshdr->patch_sig); ret = nvkm_falcon_fw_ctor(func, name, subdev->device, bl != NULL, blob->data + hdr->data_offset, hdr->data_size, falcon, fw); if (ret) goto done; + /* Earlier FW releases by NVIDIA for Nouveau's use aren't in NVIDIA's + * standard format, and don't have the indirection seen in the 0x10de + * case. + */ + switch (hdr->bin_magic) { + case 0x000010de: + loc = *(u32 *)(blob->data + hshdr->patch_loc); + sig = *(u32 *)(blob->data + hshdr->patch_sig); + break; + case 0x3b1d14f0: + loc = hshdr->patch_loc; + sig = hshdr->patch_sig; + break; + default: + WARN_ON(1); + ret = -EINVAL; + goto done; + } + ret = nvkm_falcon_fw_sign(fw, loc, hshdr->sig_prod_size, blob->data, 1, hshdr->sig_prod_offset + sig, 1, hshdr->sig_dbg_offset + sig); @@ -219,7 +267,26 @@ nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *func, const char *name, fw->dmem_size = lhdr->data_size; fw->dmem_sign = loc - lhdr->data_dma_base; - fw->boot_addr = fw->nmem_base; + if (bl) { + nvkm_firmware_put(blob); + + ret = nvkm_firmware_load_name(subdev, bl, "", ver, &blob); + if (ret) + return ret; + + hdr = nvfw_bin_hdr(subdev, blob->data); + desc = nvfw_bl_desc(subdev, blob->data + hdr->header_offset); + + fw->boot_addr = desc->start_tag << 8; + fw->boot_size = desc->code_size; + fw->boot = kmemdup(blob->data + hdr->data_offset + desc->code_off, + fw->boot_size, GFP_KERNEL); + if (!fw->boot) + ret = -ENOMEM; + } else { + fw->boot_addr = fw->nmem_base; + } + done: if (ret) nvkm_falcon_fw_dtor(fw); diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c index 014ca38b8ff3e..b615067761058 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -21,9 +21,26 @@ */ #include "priv.h" +#include #include #include +static void +gm200_flcn_pio_dmem_rd(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len) +{ + while (len >= 4) { + *(u32 *)img = nvkm_falcon_rd32(falcon, 0x1c4 + (port * 8)); + img += 4; + len -= 4; + } +} + +static void +gm200_flcn_pio_dmem_rd_init(struct nvkm_falcon *falcon, u8 port, u32 dmem_base) +{ + nvkm_falcon_wr32(falcon, 0x1c0 + (port * 8), BIT(25) | dmem_base); +} + static void gm200_flcn_pio_dmem_wr(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len, u16 tag) { @@ -46,6 +63,8 @@ gm200_flcn_dmem_pio = { .max = 0x100, .wr_init = gm200_flcn_pio_dmem_wr_init, .wr = gm200_flcn_pio_dmem_wr, + .rd_init = gm200_flcn_pio_dmem_rd_init, + .rd = gm200_flcn_pio_dmem_rd, }; static void @@ -73,6 +92,24 @@ gm200_flcn_imem_pio = { .wr = gm200_flcn_pio_imem_wr, }; +int +gm200_flcn_bind_stat(struct nvkm_falcon *falcon, bool intr) +{ + if (intr && !(nvkm_falcon_rd32(falcon, 0x008) & 0x00000008)) + return -1; + + return (nvkm_falcon_rd32(falcon, 0x0dc) & 0x00007000) >> 12; +} + +void +gm200_flcn_bind_inst(struct nvkm_falcon *falcon, int target, u64 addr) +{ + nvkm_falcon_mask(falcon, 0x604, 0x00000007, 0x00000000); /* DMAIDX_VIRT */ + nvkm_falcon_wr32(falcon, 0x054, (1 << 30) | (target << 28) | (addr >> 12)); + nvkm_falcon_mask(falcon, 0x090, 0x00010000, 0x00010000); + nvkm_falcon_mask(falcon, 0x0a4, 0x00000008, 0x00000008); +} + int gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *falcon) { @@ -166,13 +203,60 @@ int gm200_flcn_fw_load(struct nvkm_falcon_fw *fw) { struct nvkm_falcon *falcon = fw->falcon; - int ret; + int target, ret; + + if (fw->inst) { + nvkm_falcon_mask(falcon, 0x048, 0x00000001, 0x00000001); + + switch (nvkm_memory_target(fw->inst)) { + case NVKM_MEM_TARGET_VRAM: target = 0; break; + case NVKM_MEM_TARGET_HOST: target = 2; break; + case NVKM_MEM_TARGET_NCOH: target = 3; break; + default: + WARN_ON(1); + return -EINVAL; + } + + falcon->func->bind_inst(falcon, target, nvkm_memory_addr(fw->inst)); + + if (nvkm_msec(falcon->owner->device, 10, + if (falcon->func->bind_stat(falcon, falcon->func->bind_intr) == 5) + break; + ) < 0) + return -ETIMEDOUT; - if (1) { + nvkm_falcon_mask(falcon, 0x004, 0x00000008, 0x00000008); + nvkm_falcon_mask(falcon, 0x058, 0x00000002, 0x00000002); + + if (nvkm_msec(falcon->owner->device, 10, + if (falcon->func->bind_stat(falcon, false) == 0) + break; + ) < 0) + return -ETIMEDOUT; + } else { nvkm_falcon_mask(falcon, 0x624, 0x00000080, 0x00000080); nvkm_falcon_wr32(falcon, 0x10c, 0x00000000); } + if (fw->boot) { + switch (nvkm_memory_target(&fw->fw.mem.memory)) { + case NVKM_MEM_TARGET_VRAM: target = 4; break; + case NVKM_MEM_TARGET_HOST: target = 5; break; + case NVKM_MEM_TARGET_NCOH: target = 6; break; + default: + WARN_ON(1); + return -EINVAL; + } + + ret = nvkm_falcon_pio_wr(falcon, fw->boot, 0, 0, + IMEM, falcon->code.limit - fw->boot_size, fw->boot_size, + fw->boot_addr >> 8, false); + if (ret) + return ret; + + return fw->func->load_bld(fw); + } + ret = nvkm_falcon_pio_wr(falcon, fw->fw.img + fw->nmem_base_img, fw->nmem_base_img, 0, IMEM, fw->nmem_base, fw->nmem_size, fw->nmem_base >> 8, false); if (ret) diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c index f49918530d0b7..c70beacb8d30b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c @@ -21,6 +21,48 @@ */ #include "priv.h" +static void +gp102_flcn_pio_emem_rd(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len) +{ + while (len >= 4) { + *(u32 *)img = nvkm_falcon_rd32(falcon, 0xac4 + (port * 8)); + img += 4; + len -= 4; + } +} + +static void +gp102_flcn_pio_emem_rd_init(struct nvkm_falcon *falcon, u8 port, u32 dmem_base) +{ + nvkm_falcon_wr32(falcon, 0xac0 + (port * 8), BIT(25) | dmem_base); +} + +static void +gp102_flcn_pio_emem_wr(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len, u16 tag) +{ + while (len >= 4) { + nvkm_falcon_wr32(falcon, 0xac4 + (port * 8), *(u32 *)img); + img += 4; + len -= 4; + } +} + +static void +gp102_flcn_pio_emem_wr_init(struct nvkm_falcon *falcon, u8 port, bool sec, u32 emem_base) +{ + nvkm_falcon_wr32(falcon, 0xac0 + (port * 8), BIT(24) | emem_base); +} + +const struct nvkm_falcon_func_pio +gp102_flcn_emem_pio = { + .min = 4, + .max = 0x100, + .wr_init = gp102_flcn_pio_emem_wr_init, + .wr = gp102_flcn_pio_emem_wr, + .rd_init = gp102_flcn_pio_emem_rd_init, + .rd = gp102_flcn_pio_emem_rd, +}; + int gp102_flcn_reset_eng(struct nvkm_falcon *falcon) { diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c b/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c index 04f8531516481..16b246fda6662 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/msgq.c @@ -68,7 +68,7 @@ nvkm_falcon_msgq_pop(struct nvkm_falcon_msgq *msgq, void *data, u32 size) return -EINVAL; } - nvkm_falcon_read_dmem(falcon, tail, size, 0, data); + nvkm_falcon_pio_rd(falcon, 0, DMEM, tail, data, 0, size); msgq->position += ALIGN(size, QUEUE_ALIGNMENT); return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c index 9a9e1e6f70a67..dd2ddc54ac60a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/v1.c @@ -64,44 +64,13 @@ nvkm_falcon_v1_load_imem(struct nvkm_falcon *falcon, void *data, u32 start, nvkm_falcon_wr32(falcon, 0x184 + (port * 16), 0); } -static void -nvkm_falcon_v1_load_emem(struct nvkm_falcon *falcon, void *data, u32 start, - u32 size, u8 port) -{ - u8 rem = size % 4; - int i; - - size -= rem; - - nvkm_falcon_wr32(falcon, 0xac0 + (port * 8), start | (0x1 << 24)); - for (i = 0; i < size / 4; i++) - nvkm_falcon_wr32(falcon, 0xac4 + (port * 8), ((u32 *)data)[i]); - - /* - * If size is not a multiple of 4, mask the last word to ensure garbage - * does not get written - */ - if (rem) { - u32 extra = ((u32 *)data)[i]; - - nvkm_falcon_wr32(falcon, 0xac4 + (port * 8), - extra & (BIT(rem * 8) - 1)); - } -} - void nvkm_falcon_v1_load_dmem(struct nvkm_falcon *falcon, void *data, u32 start, u32 size, u8 port) { - const struct nvkm_falcon_func *func = falcon->func; u8 rem = size % 4; int i; - if (func->emem_addr && start >= func->emem_addr) - return nvkm_falcon_v1_load_emem(falcon, data, - start - func->emem_addr, size, - port); - size -= rem; nvkm_falcon_wr32(falcon, 0x1c0 + (port * 8), start | (0x1 << 24)); @@ -120,113 +89,6 @@ nvkm_falcon_v1_load_dmem(struct nvkm_falcon *falcon, void *data, u32 start, } } -static void -nvkm_falcon_v1_read_emem(struct nvkm_falcon *falcon, u32 start, u32 size, - u8 port, void *data) -{ - u8 rem = size % 4; - int i; - - size -= rem; - - nvkm_falcon_wr32(falcon, 0xac0 + (port * 8), start | (0x1 << 25)); - for (i = 0; i < size / 4; i++) - ((u32 *)data)[i] = nvkm_falcon_rd32(falcon, 0xac4 + (port * 8)); - - /* - * If size is not a multiple of 4, mask the last word to ensure garbage - * does not get read - */ - if (rem) { - u32 extra = nvkm_falcon_rd32(falcon, 0xac4 + (port * 8)); - - for (i = size; i < size + rem; i++) { - ((u8 *)data)[i] = (u8)(extra & 0xff); - extra >>= 8; - } - } -} - -void -nvkm_falcon_v1_read_dmem(struct nvkm_falcon *falcon, u32 start, u32 size, - u8 port, void *data) -{ - const struct nvkm_falcon_func *func = falcon->func; - u8 rem = size % 4; - int i; - - if (func->emem_addr && start >= func->emem_addr) - return nvkm_falcon_v1_read_emem(falcon, start - func->emem_addr, - size, port, data); - - size -= rem; - - nvkm_falcon_wr32(falcon, 0x1c0 + (port * 8), start | (0x1 << 25)); - for (i = 0; i < size / 4; i++) - ((u32 *)data)[i] = nvkm_falcon_rd32(falcon, 0x1c4 + (port * 8)); - - /* - * If size is not a multiple of 4, mask the last word to ensure garbage - * does not get read - */ - if (rem) { - u32 extra = nvkm_falcon_rd32(falcon, 0x1c4 + (port * 8)); - - for (i = size; i < size + rem; i++) { - ((u8 *)data)[i] = (u8)(extra & 0xff); - extra >>= 8; - } - } -} - -void -nvkm_falcon_v1_bind_context(struct nvkm_falcon *falcon, struct nvkm_memory *ctx) -{ - const u32 fbif = falcon->func->fbif; - u32 inst_loc; - - /* disable instance block binding */ - if (ctx == NULL) { - nvkm_falcon_wr32(falcon, 0x10c, 0x0); - return; - } - - nvkm_falcon_wr32(falcon, 0x10c, 0x1); - - /* setup apertures - virtual */ - nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_UCODE, 0x4); - nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_VIRT, 0x0); - /* setup apertures - physical */ - nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_PHYS_VID, 0x4); - nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_PHYS_SYS_COH, 0x5); - nvkm_falcon_wr32(falcon, fbif + 4 * FALCON_DMAIDX_PHYS_SYS_NCOH, 0x6); - - /* Set context */ - switch (nvkm_memory_target(ctx)) { - case NVKM_MEM_TARGET_VRAM: inst_loc = 0; break; - case NVKM_MEM_TARGET_HOST: inst_loc = 2; break; - case NVKM_MEM_TARGET_NCOH: inst_loc = 3; break; - default: - WARN_ON(1); - return; - } - - /* Enable context */ - nvkm_falcon_mask(falcon, 0x048, 0x1, 0x1); - nvkm_falcon_wr32(falcon, 0x054, - ((nvkm_memory_addr(ctx) >> 12) & 0xfffffff) | - (inst_loc << 28) | (1 << 30)); - - nvkm_falcon_mask(falcon, 0x090, 0x10000, 0x10000); - nvkm_falcon_mask(falcon, 0x0a4, 0x8, 0x8); -} - -void -nvkm_falcon_v1_set_start_addr(struct nvkm_falcon *falcon, u32 start_addr) -{ - nvkm_falcon_wr32(falcon, 0x104, start_addr); -} - void nvkm_falcon_v1_start(struct nvkm_falcon *falcon) { @@ -237,32 +99,3 @@ nvkm_falcon_v1_start(struct nvkm_falcon *falcon) else nvkm_falcon_wr32(falcon, 0x100, 0x2); } - -int -nvkm_falcon_v1_wait_for_halt(struct nvkm_falcon *falcon, u32 ms) -{ - struct nvkm_device *device = falcon->owner->device; - int ret; - - ret = nvkm_wait_msec(device, ms, falcon->addr + 0x100, 0x10, 0x10); - if (ret < 0) - return ret; - - return 0; -} - -int -nvkm_falcon_v1_clear_interrupt(struct nvkm_falcon *falcon, u32 mask) -{ - struct nvkm_device *device = falcon->owner->device; - int ret; - - /* clear interrupt(s) */ - nvkm_falcon_mask(falcon, 0x004, mask, mask); - /* wait until interrupts are cleared */ - ret = nvkm_wait_msec(device, 10, falcon->addr + 0x008, mask, 0x0); - if (ret < 0) - return ret; - - return 0; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild index 5b9f64a8957f8..ce6ece75f07ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild @@ -1,10 +1,10 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/subdev/acr/base.o -nvkm-y += nvkm/subdev/acr/hsfw.o nvkm-y += nvkm/subdev/acr/lsfw.o nvkm-y += nvkm/subdev/acr/gm200.o nvkm-y += nvkm/subdev/acr/gm20b.o nvkm-y += nvkm/subdev/acr/gp102.o nvkm-y += nvkm/subdev/acr/gp108.o +nvkm-y += nvkm/subdev/acr/gv100.o nvkm-y += nvkm/subdev/acr/gp10b.o nvkm-y += nvkm/subdev/acr/tu102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c index 882d8d30bd64b..795f3a649b122 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c @@ -24,43 +24,36 @@ #include #include #include +#include +#include +#include +#include -static struct nvkm_acr_hsf * -nvkm_acr_hsf_find(struct nvkm_acr *acr, const char *name) +static struct nvkm_acr_hsfw * +nvkm_acr_hsfw_find(struct nvkm_acr *acr, const char *name) { - struct nvkm_acr_hsf *hsf; - list_for_each_entry(hsf, &acr->hsf, head) { - if (!strcmp(hsf->name, name)) - return hsf; + struct nvkm_acr_hsfw *hsfw; + + list_for_each_entry(hsfw, &acr->hsfw, head) { + if (!strcmp(hsfw->fw.fw.name, name)) + return hsfw; } + return NULL; } int -nvkm_acr_hsf_boot(struct nvkm_acr *acr, const char *name) +nvkm_acr_hsfw_boot(struct nvkm_acr *acr, const char *name) { struct nvkm_subdev *subdev = &acr->subdev; - struct nvkm_acr_hsf *hsf; - int ret; + struct nvkm_acr_hsfw *hsfw; - hsf = nvkm_acr_hsf_find(acr, name); - if (!hsf) + hsfw = nvkm_acr_hsfw_find(acr, name); + if (!hsfw) return -EINVAL; - nvkm_debug(subdev, "executing %s binary\n", hsf->name); - ret = nvkm_falcon_get(hsf->falcon, subdev); - if (ret) - return ret; - - ret = hsf->func->boot(acr, hsf); - nvkm_falcon_put(hsf->falcon, subdev); - if (ret) { - nvkm_error(subdev, "%s binary failed\n", hsf->name); - return ret; - } - - nvkm_debug(subdev, "%s binary completed successfully\n", hsf->name); - return 0; + return nvkm_falcon_fw_boot(&hsfw->fw, subdev, true, NULL, NULL, + hsfw->boot_mbox0, hsfw->intr_clear); } static struct nvkm_acr_lsf * @@ -87,7 +80,7 @@ nvkm_acr_unload(struct nvkm_acr *acr) acr->rtos = NULL; } - nvkm_acr_hsf_boot(acr, "unload"); + nvkm_acr_hsfw_boot(acr, "unload"); acr->done = false; } } @@ -213,7 +206,7 @@ static void nvkm_acr_cleanup(struct nvkm_acr *acr) { nvkm_acr_lsfw_del_all(acr); - nvkm_acr_hsfw_del_all(acr); + nvkm_firmware_put(acr->wpr_fw); acr->wpr_fw = NULL; } @@ -226,6 +219,7 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev) struct nvkm_acr_hsfw *hsfw; struct nvkm_acr_lsfw *lsfw, *lsft; struct nvkm_acr_lsf *lsf, *rtos; + struct nvkm_falcon *falcon; u32 wpr_size = 0; u64 falcons; int ret, i; @@ -343,8 +337,16 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev) /* Load HS firmware blobs into ACR VMM. */ list_for_each_entry(hsfw, &acr->hsfw, head) { - nvkm_debug(subdev, "loading %s fw\n", hsfw->name); - ret = hsfw->func->load(acr, hsfw); + switch (hsfw->falcon_id) { + case NVKM_ACR_HSF_PMU : falcon = &device->pmu->falcon; break; + case NVKM_ACR_HSF_SEC2: falcon = &device->sec2->falcon; break; + case NVKM_ACR_HSF_GSP : falcon = &device->gsp->falcon; break; + default: + WARN_ON(1); + return -EINVAL; + } + + ret = nvkm_falcon_fw_oneinit(&hsfw->fw, falcon, acr->vmm, acr->inst); if (ret) return ret; } @@ -358,15 +360,13 @@ static void * nvkm_acr_dtor(struct nvkm_subdev *subdev) { struct nvkm_acr *acr = nvkm_acr(subdev); - struct nvkm_acr_hsf *hsf, *hst; + struct nvkm_acr_hsfw *hsfw, *hsft; struct nvkm_acr_lsf *lsf, *lst; - list_for_each_entry_safe(hsf, hst, &acr->hsf, head) { - nvkm_vmm_put(acr->vmm, &hsf->vma); - nvkm_memory_unref(&hsf->ucode); - kfree(hsf->imem); - list_del(&hsf->head); - kfree(hsf); + list_for_each_entry_safe(hsfw, hsft, &acr->hsfw, head) { + nvkm_falcon_fw_dtor(&hsfw->fw); + list_del(&hsfw->head); + kfree(hsfw); } nvkm_vmm_part(acr->vmm, acr->inst); @@ -427,7 +427,6 @@ nvkm_acr_new_(const struct nvkm_acr_fwif *fwif, struct nvkm_device *device, nvkm_subdev_ctor(&nvkm_acr, device, type, inst, &acr->subdev); INIT_LIST_HEAD(&acr->hsfw); INIT_LIST_HEAD(&acr->lsfw); - INIT_LIST_HEAD(&acr->hsf); INIT_LIST_HEAD(&acr->lsf); fwif = nvkm_firmware_load(&acr->subdev, fwif, "Acr", acr); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c index 82b4c8e1457c2..7a11151af3bdf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c @@ -46,7 +46,7 @@ gm200_acr_nofw(struct nvkm_acr *acr, int ver, const struct nvkm_acr_fwif *fwif) int gm200_acr_init(struct nvkm_acr *acr) { - return nvkm_acr_hsf_boot(acr, "load"); + return nvkm_acr_hsfw_boot(acr, "load"); } void @@ -219,162 +219,50 @@ gm200_acr_wpr_parse(struct nvkm_acr *acr) return 0; } -void -gm200_acr_hsfw_bld(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf) +int +gm200_acr_hsfw_load_bld(struct nvkm_falcon_fw *fw) { struct flcn_bl_dmem_desc_v1 hsdesc = { .ctx_dma = FALCON_DMAIDX_VIRT, - .code_dma_base = hsf->vma->addr, - .non_sec_code_off = hsf->non_sec_addr, - .non_sec_code_size = hsf->non_sec_size, - .sec_code_off = hsf->sec_addr, - .sec_code_size = hsf->sec_size, + .code_dma_base = fw->vma->addr, + .non_sec_code_off = fw->nmem_base, + .non_sec_code_size = fw->nmem_size, + .sec_code_off = fw->imem_base, + .sec_code_size = fw->imem_size, .code_entry_point = 0, - .data_dma_base = hsf->vma->addr + hsf->data_addr, - .data_size = hsf->data_size, + .data_dma_base = fw->vma->addr + fw->dmem_base_img, + .data_size = fw->dmem_size, }; - flcn_bl_dmem_desc_v1_dump(&acr->subdev, &hsdesc); - - nvkm_falcon_load_dmem(hsf->falcon, &hsdesc, 0, sizeof(hsdesc), 0); -} - -int -gm200_acr_hsfw_boot(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf, - u32 intr_clear, u32 mbox0_ok) -{ - struct nvkm_subdev *subdev = &acr->subdev; - struct nvkm_device *device = subdev->device; - struct nvkm_falcon *falcon = hsf->falcon; - u32 mbox0, mbox1; - int ret; - - /* Reset falcon. */ - nvkm_falcon_reset(falcon); - nvkm_falcon_bind_context(falcon, acr->inst); - - /* Load bootloader into IMEM. */ - nvkm_falcon_load_imem(falcon, hsf->imem, - falcon->code.limit - hsf->imem_size, - hsf->imem_size, - hsf->imem_tag, - 0, false); - - /* Load bootloader data into DMEM. */ - hsf->func->bld(acr, hsf); - - /* Boot the falcon. */ - nvkm_mc_intr_mask(device, falcon->owner->type, falcon->owner->inst, false); + flcn_bl_dmem_desc_v1_dump(fw->falcon->user, &hsdesc); - nvkm_falcon_wr32(falcon, 0x040, 0xdeada5a5); - nvkm_falcon_set_start_addr(falcon, hsf->imem_tag << 8); - nvkm_falcon_start(falcon); - ret = nvkm_falcon_wait_for_halt(falcon, 100); - if (ret) - return ret; - - /* Check for successful completion. */ - mbox0 = nvkm_falcon_rd32(falcon, 0x040); - mbox1 = nvkm_falcon_rd32(falcon, 0x044); - nvkm_debug(subdev, "mailbox %08x %08x\n", mbox0, mbox1); - if (mbox0 && mbox0 != mbox0_ok) - return -EIO; - - nvkm_falcon_clear_interrupt(falcon, intr_clear); - nvkm_mc_intr_mask(device, falcon->owner->type, falcon->owner->inst, true); - return ret; + return nvkm_falcon_pio_wr(fw->falcon, (u8 *)&hsdesc, 0, 0, DMEM, 0, sizeof(hsdesc), 0, 0); } int -gm200_acr_hsfw_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw, - struct nvkm_falcon *falcon) +gm200_acr_hsfw_ctor(struct nvkm_acr *acr, const char *bl, const char *fw, const char *name, int ver, + const struct nvkm_acr_hsf_fwif *fwif) { - struct nvkm_subdev *subdev = &acr->subdev; - struct nvkm_acr_hsf *hsf; - int ret; - - /* Patch the appropriate signature (production/debug) into the FW - * image, as determined by the mode the falcon is in. - */ - ret = nvkm_falcon_get(falcon, subdev); - if (ret) - return ret; - - if (hsfw->sig.patch_loc) { - if (!falcon->debug) { - nvkm_debug(subdev, "patching production signature\n"); - memcpy(hsfw->image + hsfw->sig.patch_loc, - hsfw->sig.prod.data, - hsfw->sig.prod.size); - } else { - nvkm_debug(subdev, "patching debug signature\n"); - memcpy(hsfw->image + hsfw->sig.patch_loc, - hsfw->sig.dbg.data, - hsfw->sig.dbg.size); - } - } - - nvkm_falcon_put(falcon, subdev); + struct nvkm_acr_hsfw *hsfw; - if (!(hsf = kzalloc(sizeof(*hsf), GFP_KERNEL))) + if (!(hsfw = kzalloc(sizeof(*hsfw), GFP_KERNEL))) return -ENOMEM; - hsf->func = hsfw->func; - hsf->name = hsfw->name; - list_add_tail(&hsf->head, &acr->hsf); - - hsf->imem_size = hsfw->imem_size; - hsf->imem_tag = hsfw->imem_tag; - hsf->imem = kmemdup(hsfw->imem, hsfw->imem_size, GFP_KERNEL); - if (!hsf->imem) - return -ENOMEM; - - hsf->non_sec_addr = hsfw->non_sec_addr; - hsf->non_sec_size = hsfw->non_sec_size; - hsf->sec_addr = hsfw->sec_addr; - hsf->sec_size = hsfw->sec_size; - hsf->data_addr = hsfw->data_addr; - hsf->data_size = hsfw->data_size; - - /* Make the FW image accessible to the HS bootloader. */ - ret = nvkm_memory_new(subdev->device, NVKM_MEM_TARGET_INST, - hsfw->image_size, 0x1000, false, &hsf->ucode); - if (ret) - return ret; - - nvkm_kmap(hsf->ucode); - nvkm_wobj(hsf->ucode, 0, hsfw->image, hsfw->image_size); - nvkm_done(hsf->ucode); - - ret = nvkm_vmm_get(acr->vmm, 12, nvkm_memory_size(hsf->ucode), - &hsf->vma); - if (ret) - return ret; - - ret = nvkm_memory_map(hsf->ucode, 0, acr->vmm, hsf->vma, NULL, 0); - if (ret) - return ret; - - hsf->falcon = falcon; - return 0; -} -int -gm200_acr_unload_boot(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf) -{ - return gm200_acr_hsfw_boot(acr, hsf, 0, 0x1d); -} + hsfw->falcon_id = fwif->falcon_id; + hsfw->boot_mbox0 = fwif->boot_mbox0; + hsfw->intr_clear = fwif->intr_clear; + list_add_tail(&hsfw->head, &acr->hsfw); -int -gm200_acr_unload_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) -{ - return gm200_acr_hsfw_load(acr, hsfw, &acr->subdev.device->pmu->falcon); + return nvkm_falcon_fw_ctor_hs(fwif->func, name, &acr->subdev, bl, fw, ver, NULL, &hsfw->fw); } -const struct nvkm_acr_hsf_func +const struct nvkm_falcon_fw_func gm200_acr_unload_0 = { - .load = gm200_acr_unload_load, - .boot = gm200_acr_unload_boot, - .bld = gm200_acr_hsfw_bld, + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .load = gm200_flcn_fw_load, + .load_bld = gm200_acr_hsfw_load_bld, + .boot = gm200_flcn_fw_boot, }; MODULE_FIRMWARE("nvidia/gm200/acr/ucode_unload.bin"); @@ -384,20 +272,15 @@ MODULE_FIRMWARE("nvidia/gp100/acr/ucode_unload.bin"); static const struct nvkm_acr_hsf_fwif gm200_acr_unload_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gm200_acr_unload_0 }, + { 0, gm200_acr_hsfw_ctor, &gm200_acr_unload_0, NVKM_ACR_HSF_PMU, 0, 0x00000010 }, {} }; -int -gm200_acr_load_boot(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf) -{ - return gm200_acr_hsfw_boot(acr, hsf, 0x10, 0); -} - static int -gm200_acr_load_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) +gm200_acr_load_setup(struct nvkm_falcon_fw *fw) { - struct flcn_acr_desc *desc = (void *)&hsfw->image[hsfw->data_addr]; + struct flcn_acr_desc *desc = (void *)&fw->fw.img[fw->dmem_base_img]; + struct nvkm_acr *acr = fw->falcon->owner->device->acr; desc->wpr_region_id = 1; desc->regions.no_regions = 2; @@ -408,15 +291,17 @@ gm200_acr_load_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) desc->regions.region_props[0].write_mask = 0xc; desc->regions.region_props[0].client_mask = 0x2; flcn_acr_desc_dump(&acr->subdev, desc); - - return gm200_acr_hsfw_load(acr, hsfw, &acr->subdev.device->pmu->falcon); + return 0; } -static const struct nvkm_acr_hsf_func +static const struct nvkm_falcon_fw_func gm200_acr_load_0 = { - .load = gm200_acr_load_load, - .boot = gm200_acr_load_boot, - .bld = gm200_acr_hsfw_bld, + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .setup = gm200_acr_load_setup, + .load = gm200_flcn_fw_load, + .load_bld = gm200_acr_hsfw_load_bld, + .boot = gm200_flcn_fw_boot, }; MODULE_FIRMWARE("nvidia/gm200/acr/bl.bin"); @@ -433,7 +318,7 @@ MODULE_FIRMWARE("nvidia/gp100/acr/ucode_load.bin"); static const struct nvkm_acr_hsf_fwif gm200_acr_load_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gm200_acr_load_0 }, + { 0, gm200_acr_hsfw_ctor, &gm200_acr_load_0, NVKM_ACR_HSF_PMU, 0, 0x00000010 }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm20b.c index 54e996f2f630b..ef5fb79128b10 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm20b.c @@ -45,43 +45,47 @@ gm20b_acr_wpr_alloc(struct nvkm_acr *acr, u32 wpr_size) wpr_size, 0, true, &acr->wpr); } -static void -gm20b_acr_load_bld(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf) +static int +gm20b_acr_hsfw_load_bld(struct nvkm_falcon_fw *fw) { struct flcn_bl_dmem_desc hsdesc = { .ctx_dma = FALCON_DMAIDX_VIRT, - .code_dma_base = hsf->vma->addr >> 8, - .non_sec_code_off = hsf->non_sec_addr, - .non_sec_code_size = hsf->non_sec_size, - .sec_code_off = hsf->sec_addr, - .sec_code_size = hsf->sec_size, + .code_dma_base = fw->vma->addr >> 8, + .non_sec_code_off = fw->nmem_base, + .non_sec_code_size = fw->nmem_size, + .sec_code_off = fw->imem_base, + .sec_code_size = fw->imem_size, .code_entry_point = 0, - .data_dma_base = (hsf->vma->addr + hsf->data_addr) >> 8, - .data_size = hsf->data_size, + .data_dma_base = (fw->vma->addr + fw->dmem_base_img) >> 8, + .data_size = fw->dmem_size, }; - flcn_bl_dmem_desc_dump(&acr->subdev, &hsdesc); + flcn_bl_dmem_desc_dump(fw->falcon->user, &hsdesc); - nvkm_falcon_load_dmem(hsf->falcon, &hsdesc, 0, sizeof(hsdesc), 0); + return nvkm_falcon_pio_wr(fw->falcon, (u8 *)&hsdesc, 0, 0, DMEM, 0, sizeof(hsdesc), 0, 0); } + static int -gm20b_acr_load_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) +gm20b_acr_load_setup(struct nvkm_falcon_fw *fw) { - struct flcn_acr_desc *desc = (void *)&hsfw->image[hsfw->data_addr]; + struct flcn_acr_desc *desc = (void *)&fw->fw.img[fw->dmem_base_img]; + struct nvkm_acr *acr = fw->falcon->owner->device->acr; desc->ucode_blob_base = nvkm_memory_addr(acr->wpr); desc->ucode_blob_size = nvkm_memory_size(acr->wpr); flcn_acr_desc_dump(&acr->subdev, desc); - - return gm200_acr_hsfw_load(acr, hsfw, &acr->subdev.device->pmu->falcon); + return 0; } -const struct nvkm_acr_hsf_func +const struct nvkm_falcon_fw_func gm20b_acr_load_0 = { - .load = gm20b_acr_load_load, - .boot = gm200_acr_load_boot, - .bld = gm20b_acr_load_bld, + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .setup = gm20b_acr_load_setup, + .load = gm200_flcn_fw_load, + .load_bld = gm20b_acr_hsfw_load_bld, + .boot = gm200_flcn_fw_boot, }; #if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC) @@ -91,7 +95,7 @@ MODULE_FIRMWARE("nvidia/gm20b/acr/ucode_load.bin"); static const struct nvkm_acr_hsf_fwif gm20b_acr_load_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gm20b_acr_load_0 }, + { 0, gm200_acr_hsfw_ctor, &gm20b_acr_load_0, NVKM_ACR_HSF_PMU, 0, 0x10 }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c index fd97a935a380e..f4c2d3729feb7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c @@ -187,14 +187,15 @@ MODULE_FIRMWARE("nvidia/gp107/acr/ucode_unload.bin"); static const struct nvkm_acr_hsf_fwif gp102_acr_unload_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gm200_acr_unload_0 }, + { 0, gm200_acr_hsfw_ctor, &gm200_acr_unload_0, NVKM_ACR_HSF_PMU, 0x1d, 0x00000010 }, {} }; int -gp102_acr_load_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) +gp102_acr_load_setup(struct nvkm_falcon_fw *fw) { - struct flcn_acr_desc_v1 *desc = (void *)&hsfw->image[hsfw->data_addr]; + struct flcn_acr_desc_v1 *desc = (void *)&fw->fw.img[fw->dmem_base_img]; + struct nvkm_acr *acr = fw->falcon->owner->device->acr; desc->wpr_region_id = 1; desc->regions.no_regions = 2; @@ -204,19 +205,19 @@ gp102_acr_load_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) desc->regions.region_props[0].read_mask = 0xf; desc->regions.region_props[0].write_mask = 0xc; desc->regions.region_props[0].client_mask = 0x2; - desc->regions.region_props[0].shadow_mem_start_addr = - acr->shadow_start >> 8; + desc->regions.region_props[0].shadow_mem_start_addr = acr->shadow_start >> 8; flcn_acr_desc_v1_dump(&acr->subdev, desc); - - return gm200_acr_hsfw_load(acr, hsfw, - &acr->subdev.device->sec2->falcon); + return 0; } -static const struct nvkm_acr_hsf_func +static const struct nvkm_falcon_fw_func gp102_acr_load_0 = { - .load = gp102_acr_load_load, - .boot = gm200_acr_load_boot, - .bld = gm200_acr_hsfw_bld, + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .setup = gp102_acr_load_setup, + .load = gm200_flcn_fw_load, + .load_bld = gm200_acr_hsfw_load_bld, + .boot = gm200_flcn_fw_boot, }; MODULE_FIRMWARE("nvidia/gp102/acr/bl.bin"); @@ -233,7 +234,7 @@ MODULE_FIRMWARE("nvidia/gp107/acr/ucode_load.bin"); static const struct nvkm_acr_hsf_fwif gp102_acr_load_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gp102_acr_load_0 }, + { 0, gm200_acr_hsfw_ctor, &gp102_acr_load_0, NVKM_ACR_HSF_SEC2, 0, 0x00000010 }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp108.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp108.c index 373d638a21777..6ab9d4959c175 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp108.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp108.c @@ -25,63 +25,62 @@ #include -void -gp108_acr_hsfw_bld(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf) +int +gp108_acr_hsfw_load_bld(struct nvkm_falcon_fw *fw) { struct flcn_bl_dmem_desc_v2 hsdesc = { .ctx_dma = FALCON_DMAIDX_VIRT, - .code_dma_base = hsf->vma->addr, - .non_sec_code_off = hsf->non_sec_addr, - .non_sec_code_size = hsf->non_sec_size, - .sec_code_off = hsf->sec_addr, - .sec_code_size = hsf->sec_size, + .code_dma_base = fw->vma->addr, + .non_sec_code_off = fw->nmem_base, + .non_sec_code_size = fw->nmem_size, + .sec_code_off = fw->imem_base, + .sec_code_size = fw->imem_size, .code_entry_point = 0, - .data_dma_base = hsf->vma->addr + hsf->data_addr, - .data_size = hsf->data_size, + .data_dma_base = fw->vma->addr + fw->dmem_base_img, + .data_size = fw->dmem_size, .argc = 0, .argv = 0, }; - flcn_bl_dmem_desc_v2_dump(&acr->subdev, &hsdesc); + flcn_bl_dmem_desc_v2_dump(fw->falcon->user, &hsdesc); - nvkm_falcon_load_dmem(hsf->falcon, &hsdesc, 0, sizeof(hsdesc), 0); + return nvkm_falcon_pio_wr(fw->falcon, (u8 *)&hsdesc, 0, 0, DMEM, 0, sizeof(hsdesc), 0, 0); } -const struct nvkm_acr_hsf_func -gp108_acr_unload_0 = { - .load = gm200_acr_unload_load, - .boot = gm200_acr_unload_boot, - .bld = gp108_acr_hsfw_bld, +const struct nvkm_falcon_fw_func +gp108_acr_hsfw_0 = { + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .load = gm200_flcn_fw_load, + .load_bld = gp108_acr_hsfw_load_bld, + .boot = gm200_flcn_fw_boot, }; MODULE_FIRMWARE("nvidia/gp108/acr/unload_bl.bin"); MODULE_FIRMWARE("nvidia/gp108/acr/ucode_unload.bin"); -MODULE_FIRMWARE("nvidia/gv100/acr/unload_bl.bin"); -MODULE_FIRMWARE("nvidia/gv100/acr/ucode_unload.bin"); - static const struct nvkm_acr_hsf_fwif gp108_acr_unload_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gp108_acr_unload_0 }, + { 0, gm200_acr_hsfw_ctor, &gp108_acr_hsfw_0, NVKM_ACR_HSF_PMU, 0x1d, 0x00000010 }, {} }; -static const struct nvkm_acr_hsf_func +const struct nvkm_falcon_fw_func gp108_acr_load_0 = { - .load = gp102_acr_load_load, - .boot = gm200_acr_load_boot, - .bld = gp108_acr_hsfw_bld, + .signature = gm200_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .setup = gp102_acr_load_setup, + .load = gm200_flcn_fw_load, + .load_bld = gp108_acr_hsfw_load_bld, + .boot = gm200_flcn_fw_boot, }; MODULE_FIRMWARE("nvidia/gp108/acr/bl.bin"); MODULE_FIRMWARE("nvidia/gp108/acr/ucode_load.bin"); -MODULE_FIRMWARE("nvidia/gv100/acr/bl.bin"); -MODULE_FIRMWARE("nvidia/gv100/acr/ucode_load.bin"); - static const struct nvkm_acr_hsf_fwif gp108_acr_load_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gp108_acr_load_0 }, + { 0, gm200_acr_hsfw_ctor, &gp108_acr_load_0, NVKM_ACR_HSF_SEC2, 0, 0x00000010 }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp10b.c index f03ba028867b6..a3422ab6deab7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp10b.c @@ -28,7 +28,7 @@ MODULE_FIRMWARE("nvidia/gp10b/acr/ucode_load.bin"); static const struct nvkm_acr_hsf_fwif gp10b_acr_load_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gm20b_acr_load_0 }, + { 0, gm200_acr_hsfw_ctor, &gm20b_acr_load_0, NVKM_ACR_HSF_PMU, 0, 0x00000010 }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gv100.c new file mode 100644 index 0000000000000..4c5ca6b400271 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gv100.c @@ -0,0 +1,67 @@ +/* + * Copyright 2022 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +MODULE_FIRMWARE("nvidia/gv100/acr/unload_bl.bin"); +MODULE_FIRMWARE("nvidia/gv100/acr/ucode_unload.bin"); + +static const struct nvkm_acr_hsf_fwif +gv100_acr_unload_fwif[] = { + { 0, gm200_acr_hsfw_ctor, &gp108_acr_hsfw_0, NVKM_ACR_HSF_PMU, 0, 0x00000000 }, + {} +}; + +MODULE_FIRMWARE("nvidia/gv100/acr/bl.bin"); +MODULE_FIRMWARE("nvidia/gv100/acr/ucode_load.bin"); + +static const struct nvkm_acr_hsf_fwif +gv100_acr_load_fwif[] = { + { 0, gm200_acr_hsfw_ctor, &gp108_acr_load_0, NVKM_ACR_HSF_SEC2, 0, 0x00000010 }, + {} +}; + +static const struct nvkm_acr_func +gv100_acr = { + .load = gv100_acr_load_fwif, + .unload = gv100_acr_unload_fwif, + .wpr_parse = gp102_acr_wpr_parse, + .wpr_layout = gp102_acr_wpr_layout, + .wpr_alloc = gp102_acr_wpr_alloc, + .wpr_build = gp102_acr_wpr_build, + .wpr_patch = gp102_acr_wpr_patch, + .wpr_check = gm200_acr_wpr_check, + .init = gm200_acr_init, +}; + +static const struct nvkm_acr_fwif +gv100_acr_fwif[] = { + { 0, gp102_acr_load, &gv100_acr }, + { -1, gm200_acr_nofw, &gm200_acr }, + {} +}; + +int +gv100_acr_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_acr **pacr) +{ + return nvkm_acr_new_(gv100_acr_fwif, device, type, inst, pacr); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c deleted file mode 100644 index a6ea89a5d51ab..0000000000000 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/hsfw.c +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2019 Red Hat Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR - * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ -#include "priv.h" - -#include - -#include -#include - -static void -nvkm_acr_hsfw_del(struct nvkm_acr_hsfw *hsfw) -{ - list_del(&hsfw->head); - kfree(hsfw->imem); - kfree(hsfw->image); - kfree(hsfw->sig.prod.data); - kfree(hsfw->sig.dbg.data); - kfree(hsfw); -} - -void -nvkm_acr_hsfw_del_all(struct nvkm_acr *acr) -{ - struct nvkm_acr_hsfw *hsfw, *hsft; - list_for_each_entry_safe(hsfw, hsft, &acr->hsfw, head) { - nvkm_acr_hsfw_del(hsfw); - } -} - -static int -nvkm_acr_hsfw_load_image(struct nvkm_acr *acr, const char *name, int ver, - struct nvkm_acr_hsfw *hsfw) -{ - struct nvkm_subdev *subdev = &acr->subdev; - const struct firmware *fw; - const struct nvfw_bin_hdr *hdr; - const struct nvfw_hs_header *fwhdr; - const struct nvfw_hs_load_header *lhdr; - u32 loc, sig; - int ret; - - ret = nvkm_firmware_get(subdev, name, ver, &fw); - if (ret < 0) - return ret; - - hdr = nvfw_bin_hdr(subdev, fw->data); - fwhdr = nvfw_hs_header(subdev, fw->data + hdr->header_offset); - - /* Earlier FW releases by NVIDIA for Nouveau's use aren't in NVIDIA's - * standard format, and don't have the indirection seen in the 0x10de - * case. - */ - switch (hdr->bin_magic) { - case 0x000010de: - loc = *(u32 *)(fw->data + fwhdr->patch_loc); - sig = *(u32 *)(fw->data + fwhdr->patch_sig); - break; - case 0x3b1d14f0: - loc = fwhdr->patch_loc; - sig = fwhdr->patch_sig; - break; - default: - ret = -EINVAL; - goto done; - } - - lhdr = nvfw_hs_load_header(subdev, fw->data + fwhdr->hdr_offset); - - if (!(hsfw->image = kmalloc(hdr->data_size, GFP_KERNEL))) { - ret = -ENOMEM; - goto done; - } - - memcpy(hsfw->image, fw->data + hdr->data_offset, hdr->data_size); - hsfw->image_size = hdr->data_size; - hsfw->non_sec_addr = lhdr->non_sec_code_off; - hsfw->non_sec_size = lhdr->non_sec_code_size; - hsfw->sec_addr = lhdr->apps[0]; - hsfw->sec_size = lhdr->apps[lhdr->num_apps]; - hsfw->data_addr = lhdr->data_dma_base; - hsfw->data_size = lhdr->data_size; - - hsfw->sig.prod.size = fwhdr->sig_prod_size; - hsfw->sig.prod.data = kmemdup(fw->data + fwhdr->sig_prod_offset + sig, - hsfw->sig.prod.size, GFP_KERNEL); - if (!hsfw->sig.prod.data) { - ret = -ENOMEM; - goto done; - } - - hsfw->sig.dbg.size = fwhdr->sig_dbg_size; - hsfw->sig.dbg.data = kmemdup(fw->data + fwhdr->sig_dbg_offset + sig, - hsfw->sig.dbg.size, GFP_KERNEL); - if (!hsfw->sig.dbg.data) { - ret = -ENOMEM; - goto done; - } - - hsfw->sig.patch_loc = loc; -done: - nvkm_firmware_put(fw); - return ret; -} - -static int -nvkm_acr_hsfw_load_bl(struct nvkm_acr *acr, const char *name, int ver, - struct nvkm_acr_hsfw *hsfw) -{ - struct nvkm_subdev *subdev = &acr->subdev; - const struct nvfw_bin_hdr *hdr; - const struct nvfw_bl_desc *desc; - const struct firmware *fw; - u8 *data; - int ret; - - ret = nvkm_firmware_get(subdev, name, ver, &fw); - if (ret) - return ret; - - hdr = nvfw_bin_hdr(subdev, fw->data); - desc = nvfw_bl_desc(subdev, fw->data + hdr->header_offset); - data = (void *)fw->data + hdr->data_offset; - - hsfw->imem_size = desc->code_size; - hsfw->imem_tag = desc->start_tag; - hsfw->imem = kmemdup(data + desc->code_off, desc->code_size, GFP_KERNEL); - nvkm_firmware_put(fw); - if (!hsfw->imem) - return -ENOMEM; - else - return 0; -} - -int -nvkm_acr_hsfw_load(struct nvkm_acr *acr, const char *bl, const char *fw, - const char *name, int version, - const struct nvkm_acr_hsf_fwif *fwif) -{ - struct nvkm_acr_hsfw *hsfw; - int ret; - - if (!(hsfw = kzalloc(sizeof(*hsfw), GFP_KERNEL))) - return -ENOMEM; - - hsfw->func = fwif->func; - hsfw->name = name; - list_add_tail(&hsfw->head, &acr->hsfw); - - ret = nvkm_acr_hsfw_load_bl(acr, bl, version, hsfw); - if (ret) - goto done; - - ret = nvkm_acr_hsfw_load_image(acr, fw, version, hsfw); -done: - if (ret) - nvkm_acr_hsfw_del(hsfw); - return ret; -} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h index c30b841c9d352..66a764e24bc6c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h @@ -51,93 +51,50 @@ int gp102_acr_wpr_build_lsb(struct nvkm_acr *, struct nvkm_acr_lsfw *); void gp102_acr_wpr_patch(struct nvkm_acr *, s64); struct nvkm_acr_hsfw { - const struct nvkm_acr_hsf_func *func; - const char *name; - struct list_head head; + struct nvkm_falcon_fw fw; + + enum nvkm_acr_hsf_id { + NVKM_ACR_HSF_PMU, + NVKM_ACR_HSF_SEC2, + NVKM_ACR_HSF_GSP, + } falcon_id; + u32 boot_mbox0; + u32 intr_clear; - u32 imem_size; - u32 imem_tag; - u32 *imem; - - u8 *image; - u32 image_size; - u32 non_sec_addr; - u32 non_sec_size; - u32 sec_addr; - u32 sec_size; - u32 data_addr; - u32 data_size; - - struct { - struct { - void *data; - u32 size; - } prod, dbg; - u32 patch_loc; - } sig; + struct list_head head; }; +int nvkm_acr_hsfw_boot(struct nvkm_acr *, const char *name); + struct nvkm_acr_hsf_fwif { int version; int (*load)(struct nvkm_acr *, const char *bl, const char *fw, const char *name, int version, const struct nvkm_acr_hsf_fwif *); - const struct nvkm_acr_hsf_func *func; -}; - -int nvkm_acr_hsfw_load(struct nvkm_acr *, const char *, const char *, - const char *, int, const struct nvkm_acr_hsf_fwif *); -void nvkm_acr_hsfw_del_all(struct nvkm_acr *); - -struct nvkm_acr_hsf { - const struct nvkm_acr_hsf_func *func; - const char *name; - struct list_head head; - - u32 imem_size; - u32 imem_tag; - u32 *imem; - - u32 non_sec_addr; - u32 non_sec_size; - u32 sec_addr; - u32 sec_size; - u32 data_addr; - u32 data_size; - - struct nvkm_memory *ucode; - struct nvkm_vma *vma; - struct nvkm_falcon *falcon; -}; + const struct nvkm_falcon_fw_func *func; -struct nvkm_acr_hsf_func { - int (*load)(struct nvkm_acr *, struct nvkm_acr_hsfw *); - int (*boot)(struct nvkm_acr *, struct nvkm_acr_hsf *); - void (*bld)(struct nvkm_acr *, struct nvkm_acr_hsf *); + enum nvkm_acr_hsf_id falcon_id; + u32 boot_mbox0; + u32 intr_clear; }; -int gm200_acr_hsfw_load(struct nvkm_acr *, struct nvkm_acr_hsfw *, - struct nvkm_falcon *); -int gm200_acr_hsfw_boot(struct nvkm_acr *, struct nvkm_acr_hsf *, - u32 clear_intr, u32 mbox0_ok); -int gm200_acr_load_boot(struct nvkm_acr *, struct nvkm_acr_hsf *); +int gm200_acr_hsfw_ctor(struct nvkm_acr *, const char *, const char *, const char *, int, + const struct nvkm_acr_hsf_fwif *); +int gm200_acr_hsfw_load_bld(struct nvkm_falcon_fw *); +extern const struct nvkm_falcon_fw_func gm200_acr_unload_0; -extern const struct nvkm_acr_hsf_func gm200_acr_unload_0; -int gm200_acr_unload_load(struct nvkm_acr *, struct nvkm_acr_hsfw *); -int gm200_acr_unload_boot(struct nvkm_acr *, struct nvkm_acr_hsf *); -void gm200_acr_hsfw_bld(struct nvkm_acr *, struct nvkm_acr_hsf *); +extern const struct nvkm_falcon_fw_func gm20b_acr_load_0; -extern const struct nvkm_acr_hsf_func gm20b_acr_load_0; +int gp102_acr_load_setup(struct nvkm_falcon_fw *); -int gp102_acr_load_load(struct nvkm_acr *, struct nvkm_acr_hsfw *); +extern const struct nvkm_falcon_fw_func gp108_acr_load_0; -extern const struct nvkm_acr_hsf_func gp108_acr_unload_0; -void gp108_acr_hsfw_bld(struct nvkm_acr *, struct nvkm_acr_hsf *); +extern const struct nvkm_falcon_fw_func gp108_acr_hsfw_0; +int gp108_acr_hsfw_load_bld(struct nvkm_falcon_fw *); int nvkm_acr_new_(const struct nvkm_acr_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); -int nvkm_acr_hsf_boot(struct nvkm_acr *, const char *name); struct nvkm_acr_lsf { const struct nvkm_acr_lsf_func *func; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c index 05a87e77525f6..ad45f5cb452b0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c @@ -32,11 +32,11 @@ static int tu102_acr_init(struct nvkm_acr *acr) { - int ret = nvkm_acr_hsf_boot(acr, "AHESASC"); + int ret = nvkm_acr_hsfw_boot(acr, "AHESASC"); if (ret) return ret; - return nvkm_acr_hsf_boot(acr, "ASB"); + return nvkm_acr_hsfw_boot(acr, "ASB"); } static int @@ -84,12 +84,6 @@ tu102_acr_wpr_build(struct nvkm_acr *acr, struct nvkm_acr_lsf *rtos) return 0; } -static int -tu102_acr_hsfw_boot(struct nvkm_acr *acr, struct nvkm_acr_hsf *hsf) -{ - return gm200_acr_hsfw_boot(acr, hsf, 0, 0); -} - static int tu102_acr_hsfw_nofw(struct nvkm_acr *acr, const char *bl, const char *fw, const char *name, int version, @@ -115,24 +109,11 @@ MODULE_FIRMWARE("nvidia/tu117/acr/ucode_unload.bin"); static const struct nvkm_acr_hsf_fwif tu102_acr_unload_fwif[] = { - { 0, nvkm_acr_hsfw_load, &gp108_acr_unload_0 }, + { 0, gm200_acr_hsfw_ctor, &gp108_acr_hsfw_0, NVKM_ACR_HSF_PMU, 0, 0x00000000 }, { -1, tu102_acr_hsfw_nofw }, {} }; -static int -tu102_acr_asb_load(struct nvkm_acr *acr, struct nvkm_acr_hsfw *hsfw) -{ - return gm200_acr_hsfw_load(acr, hsfw, &acr->subdev.device->gsp->falcon); -} - -static const struct nvkm_acr_hsf_func -tu102_acr_asb_0 = { - .load = tu102_acr_asb_load, - .boot = tu102_acr_hsfw_boot, - .bld = gp108_acr_hsfw_bld, -}; - MODULE_FIRMWARE("nvidia/tu102/acr/ucode_asb.bin"); MODULE_FIRMWARE("nvidia/tu104/acr/ucode_asb.bin"); MODULE_FIRMWARE("nvidia/tu106/acr/ucode_asb.bin"); @@ -141,18 +122,11 @@ MODULE_FIRMWARE("nvidia/tu117/acr/ucode_asb.bin"); static const struct nvkm_acr_hsf_fwif tu102_acr_asb_fwif[] = { - { 0, nvkm_acr_hsfw_load, &tu102_acr_asb_0 }, + { 0, gm200_acr_hsfw_ctor, &gp108_acr_hsfw_0, NVKM_ACR_HSF_GSP, 0, 0x00000000 }, { -1, tu102_acr_hsfw_nofw }, {} }; -static const struct nvkm_acr_hsf_func -tu102_acr_ahesasc_0 = { - .load = gp102_acr_load_load, - .boot = tu102_acr_hsfw_boot, - .bld = gp108_acr_hsfw_bld, -}; - MODULE_FIRMWARE("nvidia/tu102/acr/bl.bin"); MODULE_FIRMWARE("nvidia/tu102/acr/ucode_ahesasc.bin"); @@ -170,7 +144,7 @@ MODULE_FIRMWARE("nvidia/tu117/acr/ucode_ahesasc.bin"); static const struct nvkm_acr_hsf_fwif tu102_acr_ahesasc_fwif[] = { - { 0, nvkm_acr_hsfw_load, &tu102_acr_ahesasc_0 }, + { 0, gm200_acr_hsfw_ctor, &gp108_acr_load_0, NVKM_ACR_HSF_SEC2, 0, 0x00000000 }, { -1, tu102_acr_hsfw_nofw }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c index a247e57f7d9fa..da6a809cd3173 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/gv100.c @@ -27,15 +27,11 @@ gv100_gsp_flcn = { .enable = gm200_flcn_enable, .reset_eng = gp102_flcn_reset_eng, .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, - .fbif = 0x600, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = gp102_sec2_flcn_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, - .start = nvkm_falcon_v1_start, + .bind_inst = gm200_flcn_bind_inst, + .bind_stat = gm200_flcn_bind_stat, + .bind_intr = true, + .imem_pio = &gm200_flcn_imem_pio, + .dmem_pio = &gm200_flcn_dmem_pio, }; static const struct nvkm_gsp_func diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c index 0bd854092da96..7359991f94c25 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gm200.c @@ -23,6 +23,25 @@ */ #include "priv.h" +static int +gm200_pmu_flcn_bind_stat(struct nvkm_falcon *falcon, bool intr) +{ + nvkm_falcon_wr32(falcon, 0x200, 0x0000030e); + return (nvkm_falcon_rd32(falcon, 0x20c) & 0x00007000) >> 12; +} + +void +gm200_pmu_flcn_bind_inst(struct nvkm_falcon *falcon, int target, u64 addr) +{ + nvkm_falcon_wr32(falcon, 0xe00, 4); /* DMAIDX_UCODE */ + nvkm_falcon_wr32(falcon, 0xe04, 0); /* DMAIDX_VIRT */ + nvkm_falcon_wr32(falcon, 0xe08, 4); /* DMAIDX_PHYS_VID */ + nvkm_falcon_wr32(falcon, 0xe0c, 5); /* DMAIDX_PHYS_SYS_COH */ + nvkm_falcon_wr32(falcon, 0xe10, 6); /* DMAIDX_PHYS_SYS_NCOH */ + nvkm_falcon_mask(falcon, 0x090, 0x00010000, 0x00010000); + nvkm_falcon_wr32(falcon, 0x480, (1 << 30) | (target << 28) | (addr >> 12)); +} + const struct nvkm_falcon_func gm200_pmu_flcn = { .disable = gm200_flcn_disable, @@ -30,14 +49,10 @@ gm200_pmu_flcn = { .reset_pmc = true, .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0xc08, - .fbif = 0xe00, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = nvkm_falcon_v1_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, + .bind_inst = gm200_pmu_flcn_bind_inst, + .bind_stat = gm200_pmu_flcn_bind_stat, + .imem_pio = &gm200_flcn_imem_pio, + .dmem_pio = &gm200_flcn_dmem_pio, .start = nvkm_falcon_v1_start, .cmdq = { 0x4a0, 0x4b0, 4 }, .msgq = { 0x4c8, 0x4cc, 0 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c index 47c7412f86e86..cd3148360996f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/gp102.c @@ -30,14 +30,10 @@ gp102_pmu_flcn = { .reset_eng = gp102_flcn_reset_eng, .reset_wait_mem_scrubbing = gm200_flcn_reset_wait_mem_scrubbing, .debug = 0xc08, - .fbif = 0xe00, - .load_imem = nvkm_falcon_v1_load_imem, - .load_dmem = nvkm_falcon_v1_load_dmem, - .read_dmem = nvkm_falcon_v1_read_dmem, - .bind_context = nvkm_falcon_v1_bind_context, - .wait_for_halt = nvkm_falcon_v1_wait_for_halt, - .clear_interrupt = nvkm_falcon_v1_clear_interrupt, - .set_start_addr = nvkm_falcon_v1_set_start_addr, + .bind_inst = gm200_pmu_flcn_bind_inst, + .bind_stat = gm200_flcn_bind_stat, + .imem_pio = &gm200_flcn_imem_pio, + .dmem_pio = &gm200_flcn_dmem_pio, .start = nvkm_falcon_v1_start, .cmdq = { 0x4a0, 0x4b0, 4 }, .msgq = { 0x4c8, 0x4cc, 0 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h index 75ede265a52c1..2d0a8fa6f1967 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pmu/priv.h @@ -46,6 +46,7 @@ void gp102_pmu_reset(struct nvkm_pmu *pmu); void gk110_pmu_pgob(struct nvkm_pmu *, bool); extern const struct nvkm_falcon_func gm200_pmu_flcn; +void gm200_pmu_flcn_bind_inst(struct nvkm_falcon *, int, u64); extern const struct nvkm_pmu_func gm20b_pmu; void gm20b_pmu_acr_bld_patch(struct nvkm_acr *, u32, s64); -- GitLab From d2922879116ca753e454fecde531d509eb0af69f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:47:53 +1000 Subject: [PATCH 1349/1927] drm/nouveau/sec2: dump tracepc info on halt - useful to distinguish between different issues. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/core/falcon.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c | 5 +++++ drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c | 15 +++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index 4c8b668bacb45..4868d2cb796f8 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -47,6 +47,7 @@ void gm200_flcn_bind_inst(struct nvkm_falcon *, int, u64); int gm200_flcn_bind_stat(struct nvkm_falcon *, bool); extern const struct nvkm_falcon_func_pio gm200_flcn_imem_pio; extern const struct nvkm_falcon_func_pio gm200_flcn_dmem_pio; +void gm200_flcn_tracepc(struct nvkm_falcon *); int gp102_flcn_reset_eng(struct nvkm_falcon *); extern const struct nvkm_falcon_func_pio gp102_flcn_emem_pio; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index a5c987ce58ed5..b1e74bc49d728 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -178,6 +178,11 @@ gp102_sec2_intr(struct nvkm_inth *inth) } if (intr & 0x00000010) { + if (atomic_read(&sec2->running)) { + FLCN_ERR(falcon, "halted"); + gm200_flcn_tracepc(falcon); + } + nvkm_falcon_wr32(falcon, 0x004, 0x00000010); intr &= ~0x00000010; } diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c index b615067761058..af53cbbc632ca 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -25,6 +25,21 @@ #include #include +void +gm200_flcn_tracepc(struct nvkm_falcon *falcon) +{ + u32 sctl = nvkm_falcon_rd32(falcon, 0x240); + u32 tidx = nvkm_falcon_rd32(falcon, 0x148); + int nr = (tidx & 0x00ff0000) >> 16, sp, ip; + + FLCN_ERR(falcon, "TRACEPC SCTL %08x TIDX %08x", sctl, tidx); + for (sp = 0; sp < nr; sp++) { + nvkm_falcon_wr32(falcon, 0x148, sp); + ip = nvkm_falcon_rd32(falcon, 0x14c); + FLCN_ERR(falcon, "TRACEPC: %08x", ip); + } +} + static void gm200_flcn_pio_dmem_rd(struct nvkm_falcon *falcon, u8 port, const u8 *img, int len) { -- GitLab From 1ed02c3f2db00f3c29e88a3d880d64a7e1b7a047 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:02 +1000 Subject: [PATCH 1350/1927] drm/nouveau/engine: add HAL for engine-specific rc reset procedure Will be used to improve gr reset on GF100 and newer. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvkm/core/engine.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/core/engine.c | 10 ++++++++++ drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h index 6d15c13509bfb..b67b9c1a6b4e5 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/engine.h @@ -21,6 +21,7 @@ struct nvkm_engine_func { int (*info)(struct nvkm_engine *, u64 mthd, u64 *data); int (*init)(struct nvkm_engine *); int (*fini)(struct nvkm_engine *, bool suspend); + int (*reset)(struct nvkm_engine *); void (*intr)(struct nvkm_engine *); void (*tile)(struct nvkm_engine *, int region, struct nvkm_fb_tile *); bool (*chsw_load)(struct nvkm_engine *); @@ -48,6 +49,7 @@ int nvkm_engine_new_(const struct nvkm_engine_func *, struct nvkm_device *, struct nvkm_engine *nvkm_engine_ref(struct nvkm_engine *); void nvkm_engine_unref(struct nvkm_engine **); +int nvkm_engine_reset(struct nvkm_engine *); void nvkm_engine_tile(struct nvkm_engine *, int region); bool nvkm_engine_chsw_load(struct nvkm_engine *); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/core/engine.c b/drivers/gpu/drm/nouveau/nvkm/core/engine.c index 964615a609971..36a31e9eea228 100644 --- a/drivers/gpu/drm/nouveau/nvkm/core/engine.c +++ b/drivers/gpu/drm/nouveau/nvkm/core/engine.c @@ -35,6 +35,16 @@ nvkm_engine_chsw_load(struct nvkm_engine *engine) return false; } +int +nvkm_engine_reset(struct nvkm_engine *engine) +{ + if (engine->func->reset) + return engine->func->reset(engine); + + nvkm_subdev_fini(&engine->subdev, false); + return nvkm_subdev_init(&engine->subdev); +} + void nvkm_engine_unref(struct nvkm_engine **pengine) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c index fe51c075d6c41..b5836cbc29aa3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c @@ -113,8 +113,8 @@ nvkm_runl_rc(struct nvkm_runl *runl) } ENGN_DEBUG(engn, "resetting..."); - nvkm_subdev_fini(&engn->engine->subdev, false); - WARN_ON(nvkm_subdev_init(&engn->engine->subdev)); + /*TODO: can we do something less of a potential catastrophe on failure? */ + WARN_ON(nvkm_engine_reset(engn->engine)); } /* Submit runlist update, and clear any remaining exception state. */ -- GitLab From 4500031f86691a44ecbbebfc77872c60c5a1b8e6 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:03 +1000 Subject: [PATCH 1351/1927] drm/nouveau/ltc: split color vs depth/stencil zbc counts These differ on Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 15 +++++++++------ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 12 ++++++------ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 6 +++--- drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c | 2 +- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c | 10 +++++++--- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gk104.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm200.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp10b.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h | 3 ++- 13 files changed, 42 insertions(+), 27 deletions(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h index d32a326a92907..213d5ff33cae3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h @@ -4,7 +4,8 @@ #include #include -#define NVKM_LTC_MAX_ZBC_CNT 16 +#define NVKM_LTC_MAX_ZBC_COLOR_CNT 16 +#define NVKM_LTC_MAX_ZBC_DEPTH_CNT 16 struct nvkm_ltc { const struct nvkm_ltc_func *func; @@ -18,11 +19,13 @@ struct nvkm_ltc { u32 tag_base; struct nvkm_memory *tag_ram; - int zbc_min; - int zbc_max; - u32 zbc_color[NVKM_LTC_MAX_ZBC_CNT][4]; - u32 zbc_depth[NVKM_LTC_MAX_ZBC_CNT]; - u32 zbc_stencil[NVKM_LTC_MAX_ZBC_CNT]; + int zbc_color_min; + int zbc_color_max; + u32 zbc_color[NVKM_LTC_MAX_ZBC_COLOR_CNT][4]; + int zbc_depth_min; + int zbc_depth_max; + u32 zbc_depth[NVKM_LTC_MAX_ZBC_DEPTH_CNT]; + u32 zbc_stencil[NVKM_LTC_MAX_ZBC_DEPTH_CNT]; }; void nvkm_ltc_tags_clear(struct nvkm_device *, u32 first, u32 count); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index ee14115d669ca..1a5ec366b6722 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -67,7 +67,7 @@ gf100_gr_zbc_color_get(struct gf100_gr *gr, int format, struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc; int zbc = -ENOSPC, i; - for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) { + for (i = ltc->zbc_color_min; i <= ltc->zbc_color_max; i++) { if (gr->zbc_color[i].format) { if (gr->zbc_color[i].format != format) continue; @@ -114,7 +114,7 @@ gf100_gr_zbc_depth_get(struct gf100_gr *gr, int format, struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc; int zbc = -ENOSPC, i; - for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) { + for (i = ltc->zbc_depth_min; i <= ltc->zbc_depth_max; i++) { if (gr->zbc_depth[i].format) { if (gr->zbc_depth[i].format != format) continue; @@ -955,7 +955,7 @@ gf100_gr_zbc_init(struct gf100_gr *gr) const u32 f32_1[] = { 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000, 0x3f800000 }; struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc; - int index, c = ltc->zbc_min, d = ltc->zbc_min, s = ltc->zbc_min; + int index, c = ltc->zbc_color_min, d = ltc->zbc_depth_min, s = ltc->zbc_depth_min; if (!gr->zbc_color[0].format) { gf100_gr_zbc_color_get(gr, 1, & zero[0], &zero[4]); c++; @@ -971,13 +971,13 @@ gf100_gr_zbc_init(struct gf100_gr *gr) } } - for (index = c; index <= ltc->zbc_max; index++) + for (index = c; index <= ltc->zbc_color_max; index++) gr->func->zbc->clear_color(gr, index); - for (index = d; index <= ltc->zbc_max; index++) + for (index = d; index <= ltc->zbc_depth_max; index++) gr->func->zbc->clear_depth(gr, index); if (gr->func->zbc->clear_stencil) { - for (index = s; index <= ltc->zbc_max; index++) + for (index = s; index <= ltc->zbc_depth_max; index++) gr->func->zbc->clear_stencil(gr, index); } } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index c0038f9061355..e90eb134b561f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -105,9 +105,9 @@ struct gf100_gr { struct gf100_gr_pack *bundle; struct gf100_gr_pack *method; - struct gf100_gr_zbc_color zbc_color[NVKM_LTC_MAX_ZBC_CNT]; - struct gf100_gr_zbc_depth zbc_depth[NVKM_LTC_MAX_ZBC_CNT]; - struct gf100_gr_zbc_stencil zbc_stencil[NVKM_LTC_MAX_ZBC_CNT]; + struct gf100_gr_zbc_color zbc_color[NVKM_LTC_MAX_ZBC_COLOR_CNT]; + struct gf100_gr_zbc_depth zbc_depth[NVKM_LTC_MAX_ZBC_DEPTH_CNT]; + struct gf100_gr_zbc_stencil zbc_stencil[NVKM_LTC_MAX_ZBC_DEPTH_CNT]; u8 rop_nr; u8 gpc_nr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c index 5b001f374be0b..1d1f3c0a90145 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c @@ -47,7 +47,7 @@ gp102_gr_zbc_stencil_get(struct gf100_gr *gr, int format, struct nvkm_ltc *ltc = gr->base.engine.subdev.device->ltc; int zbc = -ENOSPC, i; - for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) { + for (i = ltc->zbc_depth_min; i <= ltc->zbc_depth_max; i++) { if (gr->zbc_stencil[i].format) { if (gr->zbc_stencil[i].format != format) continue; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c index fa683c1907955..f742a7b7b1753 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/base.c @@ -97,8 +97,10 @@ nvkm_ltc_init(struct nvkm_subdev *subdev) struct nvkm_ltc *ltc = nvkm_ltc(subdev); int i; - for (i = ltc->zbc_min; i <= ltc->zbc_max; i++) { + for (i = ltc->zbc_color_min; i <= ltc->zbc_color_max; i++) ltc->func->zbc_clear_color(ltc, i, ltc->zbc_color[i]); + + for (i = ltc->zbc_depth_min; i <= ltc->zbc_depth_max; i++) { ltc->func->zbc_clear_depth(ltc, i, ltc->zbc_depth[i]); if (ltc->func->zbc_clear_stencil) ltc->func->zbc_clear_stencil(ltc, i, ltc->zbc_stencil[i]); @@ -137,7 +139,9 @@ nvkm_ltc_new_(const struct nvkm_ltc_func *func, struct nvkm_device *device, nvkm_subdev_ctor(&nvkm_ltc, device, type, inst, <c->subdev); ltc->func = func; mutex_init(<c->mutex); - ltc->zbc_min = 1; /* reserve 0 for disabled */ - ltc->zbc_max = min(func->zbc, NVKM_LTC_MAX_ZBC_CNT) - 1; + ltc->zbc_color_min = 1; /* reserve 0 for disabled */ + ltc->zbc_color_max = min(func->zbc_color, NVKM_LTC_MAX_ZBC_COLOR_CNT) - 1; + ltc->zbc_depth_min = 1; /* reserve 0 for disabled */ + ltc->zbc_depth_max = min(func->zbc_depth, NVKM_LTC_MAX_ZBC_DEPTH_CNT) - 1; return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c index fd8aeafc812d4..de71ba3c92921 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c @@ -241,7 +241,8 @@ gf100_ltc = { .intr = gf100_ltc_intr, .cbc_clear = gf100_ltc_cbc_clear, .cbc_wait = gf100_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gf100_ltc_zbc_clear_color, .zbc_clear_depth = gf100_ltc_zbc_clear_depth, .invalidate = gf100_ltc_invalidate, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gk104.c index 94aa09244d673..5d61e3c6ff590 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gk104.c @@ -42,7 +42,8 @@ gk104_ltc = { .intr = gf100_ltc_intr, .cbc_clear = gf100_ltc_cbc_clear, .cbc_wait = gf100_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gf100_ltc_zbc_clear_color, .zbc_clear_depth = gf100_ltc_zbc_clear_depth, .invalidate = gf100_ltc_invalidate, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c index 54d1d65d5a858..18685d8496570 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c @@ -137,7 +137,8 @@ gm107_ltc = { .intr = gm107_ltc_intr, .cbc_clear = gm107_ltc_cbc_clear, .cbc_wait = gm107_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gm107_ltc_zbc_clear_color, .zbc_clear_depth = gm107_ltc_zbc_clear_depth, .invalidate = gf100_ltc_invalidate, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm200.c index 8cfdbbdd8e8d8..7a9464b9def5f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm200.c @@ -49,7 +49,8 @@ gm200_ltc = { .intr = gm107_ltc_intr, .cbc_clear = gm107_ltc_cbc_clear, .cbc_wait = gm107_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gm107_ltc_zbc_clear_color, .zbc_clear_depth = gm107_ltc_zbc_clear_depth, .invalidate = gf100_ltc_invalidate, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c index a4a6cd9b435ad..1a17a451754c3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c @@ -61,7 +61,8 @@ gp100_ltc = { .intr = gp100_ltc_intr, .cbc_clear = gm107_ltc_cbc_clear, .cbc_wait = gm107_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gm107_ltc_zbc_clear_color, .zbc_clear_depth = gm107_ltc_zbc_clear_depth, .invalidate = gf100_ltc_invalidate, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c index ff05d617e7f45..265a05fd5f6bc 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp102.c @@ -36,7 +36,8 @@ gp102_ltc = { .intr = gp100_ltc_intr, .cbc_clear = gm107_ltc_cbc_clear, .cbc_wait = gm107_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gm107_ltc_zbc_clear_color, .zbc_clear_depth = gm107_ltc_zbc_clear_depth, .zbc_clear_stencil = gp102_ltc_zbc_clear_stencil, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp10b.c index dfebd796cb4ba..e7e8fdf3adab7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp10b.c @@ -50,7 +50,8 @@ gp10b_ltc = { .intr = gp100_ltc_intr, .cbc_clear = gm107_ltc_cbc_clear, .cbc_wait = gm107_ltc_cbc_wait, - .zbc = 16, + .zbc_color = 16, + .zbc_depth = 16, .zbc_clear_color = gm107_ltc_zbc_clear_color, .zbc_clear_depth = gm107_ltc_zbc_clear_depth, .zbc_clear_stencil = gp102_ltc_zbc_clear_stencil, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h index 2bebe139005d9..134e90c9e8613 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h @@ -16,7 +16,8 @@ struct nvkm_ltc_func { void (*cbc_clear)(struct nvkm_ltc *, u32 start, u32 limit); void (*cbc_wait)(struct nvkm_ltc *); - int zbc; + int zbc_color; + int zbc_depth; void (*zbc_clear_color)(struct nvkm_ltc *, int, const u32[4]); void (*zbc_clear_depth)(struct nvkm_ltc *, int, const u32); void (*zbc_clear_stencil)(struct nvkm_ltc *, int, const u32); -- GitLab From 0ceceaa9ae8294b0da91d731027b5c4b1bb4187e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:04 +1000 Subject: [PATCH 1352/1927] drm/nouveau/fifo: expose function to read engine ctxsw status Needed to support Ampere differences in gr/gf100-: Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/engine/fifo.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c | 16 ++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h index 4825403a26885..221abd6c43103 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h @@ -75,6 +75,7 @@ struct nvkm_fifo { void nvkm_fifo_fault(struct nvkm_fifo *, struct nvkm_fault_data *); void nvkm_fifo_pause(struct nvkm_fifo *, unsigned long *); void nvkm_fifo_start(struct nvkm_fifo *, unsigned long *); +bool nvkm_fifo_ctxsw_in_progress(struct nvkm_engine *); int nv04_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); int nv10_fifo_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_fifo **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c index abce1ecfd4d44..5ea9a2ff0663b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c @@ -35,6 +35,22 @@ #include #include +bool +nvkm_fifo_ctxsw_in_progress(struct nvkm_engine *engine) +{ + struct nvkm_runl *runl; + struct nvkm_engn *engn; + + nvkm_runl_foreach(runl, engine->subdev.device->fifo) { + nvkm_runl_foreach_engn(engn, runl) { + if (engn->engine == engine) + return engn->func->chsw ? engn->func->chsw(engn) : false; + } + } + + return false; +} + void nvkm_fifo_pause(struct nvkm_fifo *fifo, unsigned long *flags) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 1a5ec366b6722..acc7a74f9cc36 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1003,7 +1003,7 @@ gf100_gr_wait_idle(struct gf100_gr *gr) nvkm_rd32(device, 0x400700); gr_enabled = nvkm_rd32(device, 0x200) & 0x1000; - ctxsw_active = nvkm_rd32(device, 0x2640) & 0x8000; + ctxsw_active = nvkm_fifo_ctxsw_in_progress(&gr->base.engine); gr_busy = nvkm_rd32(device, 0x40060c) & 0x1; if (!gr_enabled || (!gr_busy && !ctxsw_active)) -- GitLab From 113750215058a19330858dc41fbaac8f9540d45f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:04 +1000 Subject: [PATCH 1353/1927] drm/nouveau/gr/gf100-: move some code around to make next commits nicer Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 104 +++++++++--------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index acc7a74f9cc36..662d953eaddb5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2055,22 +2055,6 @@ gf100_gr_dtor(struct nvkm_gr *base) return gr; } -static const struct nvkm_gr_func -gf100_gr_ = { - .dtor = gf100_gr_dtor, - .oneinit = gf100_gr_oneinit, - .init = gf100_gr_init_, - .fini = gf100_gr_fini, - .intr = gf100_gr_intr, - .units = gf100_gr_units, - .chan_new = gf100_gr_chan_new, - .object_get = gf100_gr_object_get, - .chsw_load = gf100_gr_chsw_load, - .ctxsw.pause = gf100_gr_fecs_stop_ctxsw, - .ctxsw.resume = gf100_gr_fecs_start_ctxsw, - .ctxsw.inst = gf100_gr_ctxsw_inst, -}; - static const struct nvkm_falcon_func gf100_gr_flcn = { .load_imem = nvkm_falcon_v1_load_imem, @@ -2078,42 +2062,6 @@ gf100_gr_flcn = { .start = nvkm_falcon_v1_start, }; -int -gf100_gr_new_(const struct gf100_gr_fwif *fwif, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr) -{ - struct gf100_gr *gr; - int ret; - - if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL))) - return -ENOMEM; - *pgr = &gr->base; - - ret = nvkm_gr_ctor(&gf100_gr_, device, type, inst, true, &gr->base); - if (ret) - return ret; - - fwif = nvkm_firmware_load(&gr->base.engine.subdev, fwif, "Gr", gr); - if (IS_ERR(fwif)) - return PTR_ERR(fwif); - - gr->func = fwif->func; - - ret = nvkm_falcon_ctor(&gf100_gr_flcn, &gr->base.engine.subdev, - "fecs", 0x409000, &gr->fecs.falcon); - if (ret) - return ret; - - mutex_init(&gr->fecs.mutex); - - ret = nvkm_falcon_ctor(&gf100_gr_flcn, &gr->base.engine.subdev, - "gpccs", 0x41a000, &gr->gpccs.falcon); - if (ret) - return ret; - - return 0; -} - void gf100_gr_init_num_tpc_per_gpc(struct gf100_gr *gr, bool pd, bool ds) { @@ -2383,6 +2331,22 @@ gf100_gr_gpccs_ucode = { .data.size = sizeof(gf100_grgpc_data), }; +static const struct nvkm_gr_func +gf100_gr_ = { + .dtor = gf100_gr_dtor, + .oneinit = gf100_gr_oneinit, + .init = gf100_gr_init_, + .fini = gf100_gr_fini, + .intr = gf100_gr_intr, + .units = gf100_gr_units, + .chan_new = gf100_gr_chan_new, + .object_get = gf100_gr_object_get, + .chsw_load = gf100_gr_chsw_load, + .ctxsw.pause = gf100_gr_fecs_stop_ctxsw, + .ctxsw.resume = gf100_gr_fecs_start_ctxsw, + .ctxsw.inst = gf100_gr_ctxsw_inst, +}; + static const struct gf100_gr_func gf100_gr = { .oneinit_tiles = gf100_gr_oneinit_tiles, @@ -2474,6 +2438,42 @@ gf100_gr_fwif[] = { {} }; +int +gf100_gr_new_(const struct gf100_gr_fwif *fwif, struct nvkm_device *device, + enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr) +{ + struct gf100_gr *gr; + int ret; + + if (!(gr = kzalloc(sizeof(*gr), GFP_KERNEL))) + return -ENOMEM; + *pgr = &gr->base; + + ret = nvkm_gr_ctor(&gf100_gr_, device, type, inst, true, &gr->base); + if (ret) + return ret; + + fwif = nvkm_firmware_load(&gr->base.engine.subdev, fwif, "Gr", gr); + if (IS_ERR(fwif)) + return PTR_ERR(fwif); + + gr->func = fwif->func; + + ret = nvkm_falcon_ctor(&gf100_gr_flcn, &gr->base.engine.subdev, + "fecs", 0x409000, &gr->fecs.falcon); + if (ret) + return ret; + + mutex_init(&gr->fecs.mutex); + + ret = nvkm_falcon_ctor(&gf100_gr_flcn, &gr->base.engine.subdev, + "gpccs", 0x41a000, &gr->gpccs.falcon); + if (ret) + return ret; + + return 0; +} + int gf100_gr_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr) { -- GitLab From ca081fff6ecc63c86a99918230cc9b947bebae8a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:05 +1000 Subject: [PATCH 1354/1927] drm/nouveau/gr/gf100-: generate golden context during first object alloc Needed for GV100 (and only GV100 for some reason) for WFI_GOLDEN_SAVE. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 45 ++++++------------- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 2 +- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 28 +++++------- 3 files changed, 26 insertions(+), 49 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 297915719bf26..45db67d551697 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -26,6 +26,7 @@ #include #include #include +#include /******************************************************************************* * PGRAPH context register lists @@ -1433,14 +1434,12 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) #define CB_RESERVED 0x80000 int -gf100_grctx_generate(struct gf100_gr *gr) +gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvkm_gpuobj *inst) { const struct gf100_grctx_func *grctx = gr->func->grctx; struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; - struct nvkm_memory *inst = NULL; struct nvkm_memory *data = NULL; - struct nvkm_vmm *vmm = NULL; struct nvkm_vma *ctx = NULL; struct gf100_grctx info; int ret, i; @@ -1472,38 +1471,20 @@ gf100_grctx_generate(struct gf100_gr *gr) /* Init SCC RAM. */ nvkm_wr32(device, 0x40802c, 0x00000001); - /* Allocate memory to for a "channel", which we'll use to generate - * the default context values. - */ - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - 0x1000, 0x1000, true, &inst); - if (ret) - goto done; - - ret = nvkm_vmm_new(device, 0, 0, NULL, 0, NULL, "grctx", &vmm); - if (ret) - goto done; - - vmm->debug = subdev->debug; - - ret = nvkm_vmm_join(vmm, inst); - if (ret) - goto done; - + /* Allocate memory to store context, and dummy global context buffers. */ ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, CB_RESERVED + gr->size, 0, true, &data); if (ret) goto done; - ret = nvkm_vmm_get(vmm, 0, nvkm_memory_size(data), &ctx); + ret = nvkm_vmm_get(chan->vmm, 0, nvkm_memory_size(data), &ctx); if (ret) goto done; - ret = nvkm_memory_map(data, 0, vmm, ctx, NULL, 0); + ret = nvkm_memory_map(data, 0, chan->vmm, ctx, NULL, 0); if (ret) goto done; - /* Setup context pointer. */ nvkm_kmap(inst); nvkm_wo32(inst, 0x0210, lower_32_bits(ctx->addr + CB_RESERVED) | 4); @@ -1518,11 +1499,11 @@ gf100_grctx_generate(struct gf100_gr *gr) info.buffer_nr = 0; /* Make channel current. */ - addr = nvkm_memory_addr(inst) >> 12; + addr = inst->addr >> 12; if (gr->firmware) { ret = gf100_gr_fecs_bind_pointer(gr, 0x80000000 | addr); if (ret) - goto done; + goto done_inst; nvkm_kmap(data); nvkm_wo32(data, 0x1c, 1); @@ -1552,7 +1533,7 @@ gf100_grctx_generate(struct gf100_gr *gr) break; ) < 0) { ret = -EBUSY; - goto done; + goto done_inst; } gr->data = kmalloc(gr->size, GFP_KERNEL); @@ -1566,12 +1547,14 @@ gf100_grctx_generate(struct gf100_gr *gr) ret = -ENOMEM; } +done_inst: + nvkm_kmap(inst); + nvkm_wo32(inst, 0x0210, 0); + nvkm_wo32(inst, 0x0214, 0); + nvkm_done(inst); done: - nvkm_vmm_put(vmm, &ctx); + nvkm_vmm_put(chan->vmm, &ctx); nvkm_memory_unref(&data); - nvkm_vmm_part(vmm, inst); - nvkm_vmm_unref(&vmm); - nvkm_memory_unref(&inst); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 32bbddc0993e8..7b968ef681911 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -81,7 +81,7 @@ struct gf100_grctx_func { }; extern const struct gf100_grctx_func gf100_grctx; -int gf100_grctx_generate(struct gf100_gr *); +int gf100_grctx_generate(struct gf100_gr *, struct gf100_gr_chan *, struct nvkm_gpuobj *inst); void gf100_grctx_generate_main(struct gf100_gr *, struct gf100_grctx *); void gf100_grctx_generate_bundle(struct gf100_grctx *); void gf100_grctx_generate_pagepool(struct gf100_grctx *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 662d953eaddb5..310949c641ce3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -394,6 +394,17 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, chan->vmm = nvkm_vmm_ref(fifoch->vmm); *pobject = &chan->object; + /* Generate golden context image. */ + mutex_lock(&gr->fecs.mutex); + if (gr->data == NULL) { + ret = gf100_grctx_generate(gr, chan, fifoch->inst); + if (ret) { + nvkm_error(&base->engine.subdev, "failed to construct context\n"); + return ret; + } + } + mutex_unlock(&gr->fecs.mutex); + /* allocate memory for a "mmio list" buffer that's used by the HUB * fuc to modify some per-context register settings on first load * of the context. @@ -1763,15 +1774,6 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) return ret; } - /* Generate golden context image. */ - if (gr->data == NULL) { - int ret = gf100_grctx_generate(gr); - if (ret) { - nvkm_error(subdev, "failed to construct context\n"); - return ret; - } - } - return 0; } @@ -1823,14 +1825,6 @@ gf100_gr_init_ctxctl_int(struct gf100_gr *gr) } gr->size = nvkm_rd32(device, 0x409804); - if (gr->data == NULL) { - int ret = gf100_grctx_generate(gr); - if (ret) { - nvkm_error(subdev, "failed to construct context\n"); - return ret; - } - } - return 0; } -- GitLab From d05095b53cd41c7e1db8f680610386f73c0f7290 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:05 +1000 Subject: [PATCH 1355/1927] drm/nouveau/gr/gf100-: make global pagepool actually global This was thought to be per-channel initially - it's not. The backing pages for the VMM mappings are shared for all channels. - switches to more straight-forward patch interfaces - prepares for sub-context support Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 32 ++++++++++++------- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 15 +++++---- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c | 12 ++----- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c | 5 +-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c | 13 ++------ .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c | 5 +-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c | 13 +++----- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 22 ++++++++++++- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 4 +++ 9 files changed, 71 insertions(+), 50 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 45db67d551697..291d98fece327 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -991,6 +991,18 @@ gf100_grctx_pack_tpc[] = { * PGRAPH context implementation ******************************************************************************/ +void +gf100_grctx_patch_wr32(struct gf100_gr_chan *chan, u32 addr, u32 data) +{ + if (unlikely(!chan->mmio)) { + nvkm_wr32(chan->gr->base.engine.subdev.device, addr, data); + return; + } + + nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, addr); + nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, data); +} + int gf100_grctx_mmio_data(struct gf100_grctx *info, u32 size, u32 align, bool priv) { @@ -1050,15 +1062,12 @@ gf100_grctx_generate_bundle(struct gf100_grctx *info) } void -gf100_grctx_generate_pagepool(struct gf100_grctx *info) +gf100_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); - mmio_refn(info, 0x40800c, 0x00000000, s, b); - mmio_wr32(info, 0x408010, 0x80000000); - mmio_refn(info, 0x419004, 0x00000000, s, b); - mmio_wr32(info, 0x419008, 0x00000000); + gf100_grctx_patch_wr32(chan, 0x40800c, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x408010, 0x80000000); + gf100_grctx_patch_wr32(chan, 0x419004, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x419008, 0x00000000); } void @@ -1362,8 +1371,9 @@ gf100_grctx_generate_floorsweep(struct gf100_gr *gr) } void -gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) +gf100_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) { + struct gf100_gr *gr = chan->gr; struct nvkm_device *device = gr->base.engine.subdev.device; const struct gf100_grctx_func *grctx = gr->func->grctx; u32 idle_timeout; @@ -1385,7 +1395,7 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000); - grctx->pagepool(info); + grctx->pagepool(chan, chan->pagepool->addr); grctx->bundle(info); grctx->attrib(info); if (grctx->patch_ltc) @@ -1521,7 +1531,7 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk ); } - grctx->main(gr, &info); + grctx->main(chan, &info); /* Trigger a context unload by unsetting the "next channel valid" bit * and faking a context switch interrupt. diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 7b968ef681911..1c68a008c3779 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -14,6 +14,7 @@ struct gf100_grctx { int gf100_grctx_mmio_data(struct gf100_grctx *, u32 size, u32 align, bool priv); void gf100_grctx_mmio_item(struct gf100_grctx *, u32 addr, u32 data, int s, int); +void gf100_grctx_patch_wr32(struct gf100_gr_chan *, u32 addr, u32 data); #define mmio_vram(a,b,c,d) gf100_grctx_mmio_data((a), (b), (c), (d)) #define mmio_refn(a,b,c,d,e) gf100_grctx_mmio_item((a), (b), (c), (d), (e)) @@ -23,7 +24,7 @@ void gf100_grctx_mmio_item(struct gf100_grctx *, u32 addr, u32 data, int s, int) struct gf100_grctx_func { void (*unkn88c)(struct gf100_gr *, bool on); /* main context generation function */ - void (*main)(struct gf100_gr *, struct gf100_grctx *); + void (*main)(struct gf100_gr_chan *, struct gf100_grctx *); /* context-specific modify-on-first-load list generation function */ void (*unkn)(struct gf100_gr *); /* mmio context data */ @@ -43,7 +44,7 @@ struct gf100_grctx_func { u32 bundle_min_gpm_fifo_depth; u32 bundle_token_limit; /* pagepool */ - void (*pagepool)(struct gf100_grctx *); + void (*pagepool)(struct gf100_gr_chan *, u64 addr); u32 pagepool_size; /* attribute(/alpha) circular buffer */ void (*attrib)(struct gf100_grctx *); @@ -82,9 +83,9 @@ struct gf100_grctx_func { extern const struct gf100_grctx_func gf100_grctx; int gf100_grctx_generate(struct gf100_gr *, struct gf100_gr_chan *, struct nvkm_gpuobj *inst); -void gf100_grctx_generate_main(struct gf100_gr *, struct gf100_grctx *); +void gf100_grctx_generate_main(struct gf100_gr_chan *, struct gf100_grctx *); void gf100_grctx_generate_bundle(struct gf100_grctx *); -void gf100_grctx_generate_pagepool(struct gf100_grctx *); +void gf100_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gf100_grctx_generate_attrib(struct gf100_grctx *); void gf100_grctx_generate_unkn(struct gf100_gr *); void gf100_grctx_generate_floorsweep(struct gf100_gr *); @@ -116,7 +117,7 @@ void gk104_grctx_generate_gpc_tpc_nr(struct gf100_gr *); extern const struct gf100_grctx_func gk20a_grctx; void gk104_grctx_generate_bundle(struct gf100_grctx *); -void gk104_grctx_generate_pagepool(struct gf100_grctx *); +void gk104_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gk104_grctx_generate_patch_ltc(struct gf100_grctx *); void gk104_grctx_generate_unkn(struct gf100_gr *); void gk104_grctx_generate_r418800(struct gf100_gr *); @@ -129,7 +130,7 @@ extern const struct gf100_grctx_func gk208_grctx; extern const struct gf100_grctx_func gm107_grctx; void gm107_grctx_generate_bundle(struct gf100_grctx *); -void gm107_grctx_generate_pagepool(struct gf100_grctx *); +void gm107_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gm107_grctx_generate_attrib(struct gf100_grctx *); void gm107_grctx_generate_sm_id(struct gf100_gr *, int, int, int); @@ -143,7 +144,7 @@ void gm200_grctx_generate_r419a3c(struct gf100_gr *); extern const struct gf100_grctx_func gm20b_grctx; extern const struct gf100_grctx_func gp100_grctx; -void gp100_grctx_generate_pagepool(struct gf100_grctx *); +void gp100_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gp100_grctx_generate_smid_config(struct gf100_gr *); extern const struct gf100_grctx_func gp102_grctx; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c index 304e9d268bad4..10e78639a8162 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c @@ -888,16 +888,10 @@ gk104_grctx_generate_bundle(struct gf100_grctx *info) } void -gk104_grctx_generate_pagepool(struct gf100_grctx *info) +gk104_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); - mmio_refn(info, 0x40800c, 0x00000000, s, b); - mmio_wr32(info, 0x408010, 0x80000000); - mmio_refn(info, 0x419004, 0x00000000, s, b); - mmio_wr32(info, 0x419008, 0x00000000); - mmio_wr32(info, 0x4064cc, 0x80000000); + gf100_grctx_generate_pagepool(chan, addr); + gf100_grctx_patch_wr32(chan, 0x4064cc, 0x80000000); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c index c0d36bc601f9b..46d64c2087548 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c @@ -25,8 +25,9 @@ #include static void -gk20a_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) +gk20a_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) { + struct gf100_gr *gr = chan->gr; struct nvkm_device *device = gr->base.engine.subdev.device; const struct gf100_grctx_func *grctx = gr->func->grctx; u32 idle_timeout; @@ -60,7 +61,7 @@ gk20a_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) gf100_gr_wait_idle(gr); gf100_gr_icmd(gr, gr->bundle); - grctx->pagepool(info); + grctx->pagepool(chan, chan->pagepool->addr); grctx->bundle(info); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c index 0b3964e6b36e2..11ccee875ed7b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c @@ -892,17 +892,10 @@ gm107_grctx_generate_bundle(struct gf100_grctx *info) } void -gm107_grctx_generate_pagepool(struct gf100_grctx *info) +gm107_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); - mmio_refn(info, 0x40800c, 0x00000000, s, b); - mmio_wr32(info, 0x408010, 0x80000000); - mmio_refn(info, 0x419004, 0x00000000, s, b); - mmio_wr32(info, 0x419008, 0x00000000); - mmio_wr32(info, 0x4064cc, 0x80000000); - mmio_wr32(info, 0x418e30, 0x80000000); /* guess at it being related */ + gk104_grctx_generate_pagepool(chan, addr); + gf100_grctx_patch_wr32(chan, 0x418e30, 0x80000000); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c index 6b92f8aa18a35..d0b45d1074a29 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c @@ -22,8 +22,9 @@ #include "ctxgf100.h" static void -gm20b_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) +gm20b_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) { + struct gf100_gr *gr = chan->gr; struct nvkm_device *device = gr->base.engine.subdev.device; const struct gf100_grctx_func *grctx = gr->func->grctx; u32 idle_timeout; @@ -63,7 +64,7 @@ gm20b_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info) gf100_gr_wait_idle(gr); gf100_gr_icmd(gr, gr->bundle); - grctx->pagepool(info); + grctx->pagepool(chan, chan->pagepool->addr); grctx->bundle(info); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c index 0b3326262e123..b2fa7c943c494 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c @@ -30,15 +30,12 @@ ******************************************************************************/ void -gp100_grctx_generate_pagepool(struct gf100_grctx *info) +gp100_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const int s = 8; - const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), true); - mmio_refn(info, 0x40800c, 0x00000000, s, b); - mmio_wr32(info, 0x408010, 0x8007d800); - mmio_refn(info, 0x419004, 0x00000000, s, b); - mmio_wr32(info, 0x419008, 0x00000000); + gf100_grctx_patch_wr32(chan, 0x40800c, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x408010, 0x8007d800); + gf100_grctx_patch_wr32(chan, 0x419004, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x419008, 0x00000000); } static void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 310949c641ce3..56f662fa37d3e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -364,6 +364,8 @@ gf100_gr_chan_dtor(struct nvkm_object *object) nvkm_vmm_put(chan->vmm, &chan->mmio_vma); nvkm_memory_unref(&chan->mmio); + + nvkm_vmm_put(chan->vmm, &chan->pagepool); nvkm_vmm_unref(&chan->vmm); return chan; } @@ -394,6 +396,15 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, chan->vmm = nvkm_vmm_ref(fifoch->vmm); *pobject = &chan->object; + /* Map pagepool. */ + ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->pagepool), &chan->pagepool); + if (ret) + return ret; + + ret = nvkm_memory_map(gr->pagepool, 0, chan->vmm, chan->pagepool, &args, sizeof(args)); + if (ret) + return ret; + /* Generate golden context image. */ mutex_lock(&gr->fecs.mutex); if (gr->data == NULL) { @@ -449,6 +460,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, /* finally, fill in the mmio list and point the context at it */ nvkm_kmap(chan->mmio); + gr->func->grctx->pagepool(chan, chan->pagepool->addr); for (i = 0; mmio->addr && i < ARRAY_SIZE(gr->mmio_list); i++) { u32 addr = mmio->addr; u32 data = mmio->data; @@ -1938,7 +1950,7 @@ gf100_gr_oneinit(struct nvkm_gr *base) struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; - int i, j; + int ret, i, j; nvkm_pmu_pgob(device->pmu, false); @@ -1964,6 +1976,12 @@ gf100_gr_oneinit(struct nvkm_gr *base) } } + /* Allocate global context buffers. */ + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, gr->func->grctx->pagepool_size, + 0x100, false, &gr->pagepool); + if (ret) + return ret; + memset(gr->tile, 0xff, sizeof(gr->tile)); gr->func->oneinit_tiles(gr); gr->func->oneinit_sm_id(gr); @@ -2033,6 +2051,8 @@ gf100_gr_dtor(struct nvkm_gr *base) kfree(gr->data); + nvkm_memory_unref(&gr->pagepool); + nvkm_falcon_dtor(&gr->gpccs.falcon); nvkm_falcon_dtor(&gr->fecs.falcon); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index e90eb134b561f..c74cc23859a50 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -121,6 +121,8 @@ struct gf100_gr { u8 ppc_tpc_min; u8 ppc_tpc_max; + struct nvkm_memory *pagepool; + u8 screen_tile_row_offset; u8 tile[TPC_MAX]; @@ -258,6 +260,8 @@ struct gf100_gr_chan { struct gf100_gr *gr; struct nvkm_vmm *vmm; + struct nvkm_vma *pagepool; + struct nvkm_memory *mmio; struct nvkm_vma *mmio_vma; int mmio_nr; -- GitLab From 95f78acd88594101f658a4587f51ccb6a6556cc3 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:06 +1000 Subject: [PATCH 1356/1927] drm/nouveau/gr/gf100-: make global bundle_cb actually global This was thought to be per-channel initially - it's not. The backing pages for the VMM mappings are shared for all channels. - switches to more straight-forward patch interfaces - prepares for sub-context support Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 15 ++++++-------- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 8 ++++---- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c | 17 ++++++---------- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c | 20 +++++++++---------- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 17 ++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 2 ++ 8 files changed, 46 insertions(+), 37 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 291d98fece327..3d06ca63e532c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1050,15 +1050,12 @@ gf100_grctx_generate_r419cb8(struct gf100_gr *gr) } void -gf100_grctx_generate_bundle(struct gf100_grctx *info) +gf100_grctx_generate_bundle(struct gf100_gr_chan *chan, u64 addr, u32 size) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const int s = 8; - const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true); - mmio_refn(info, 0x408004, 0x00000000, s, b); - mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s)); - mmio_refn(info, 0x418808, 0x00000000, s, b); - mmio_wr32(info, 0x41880c, 0x80000000 | (grctx->bundle_size >> s)); + gf100_grctx_patch_wr32(chan, 0x408004, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x408008, 0x80000000 | (size >> 8)); + gf100_grctx_patch_wr32(chan, 0x418808, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x41880c, 0x80000000 | (size >> 8)); } void @@ -1396,7 +1393,7 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000); grctx->pagepool(chan, chan->pagepool->addr); - grctx->bundle(info); + grctx->bundle(chan, chan->bundle_cb->addr, grctx->bundle_size); grctx->attrib(info); if (grctx->patch_ltc) grctx->patch_ltc(info); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 1c68a008c3779..6e5fabf460159 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -39,7 +39,7 @@ struct gf100_grctx_func { const struct gf100_gr_pack *mthd; const struct gf100_gr_pack *sw_veid_bundle_init; /* bundle circular buffer */ - void (*bundle)(struct gf100_grctx *); + void (*bundle)(struct gf100_gr_chan *, u64 addr, u32 size); u32 bundle_size; u32 bundle_min_gpm_fifo_depth; u32 bundle_token_limit; @@ -84,8 +84,8 @@ struct gf100_grctx_func { extern const struct gf100_grctx_func gf100_grctx; int gf100_grctx_generate(struct gf100_gr *, struct gf100_gr_chan *, struct nvkm_gpuobj *inst); void gf100_grctx_generate_main(struct gf100_gr_chan *, struct gf100_grctx *); -void gf100_grctx_generate_bundle(struct gf100_grctx *); void gf100_grctx_generate_pagepool(struct gf100_gr_chan *, u64); +void gf100_grctx_generate_bundle(struct gf100_gr_chan *, u64, u32); void gf100_grctx_generate_attrib(struct gf100_grctx *); void gf100_grctx_generate_unkn(struct gf100_gr *); void gf100_grctx_generate_floorsweep(struct gf100_gr *); @@ -116,8 +116,8 @@ void gk104_grctx_generate_alpha_beta_tables(struct gf100_gr *); void gk104_grctx_generate_gpc_tpc_nr(struct gf100_gr *); extern const struct gf100_grctx_func gk20a_grctx; -void gk104_grctx_generate_bundle(struct gf100_grctx *); void gk104_grctx_generate_pagepool(struct gf100_gr_chan *, u64); +void gk104_grctx_generate_bundle(struct gf100_gr_chan *, u64, u32); void gk104_grctx_generate_patch_ltc(struct gf100_grctx *); void gk104_grctx_generate_unkn(struct gf100_gr *); void gk104_grctx_generate_r418800(struct gf100_gr *); @@ -129,8 +129,8 @@ extern const struct gf100_grctx_func gk110b_grctx; extern const struct gf100_grctx_func gk208_grctx; extern const struct gf100_grctx_func gm107_grctx; -void gm107_grctx_generate_bundle(struct gf100_grctx *); void gm107_grctx_generate_pagepool(struct gf100_gr_chan *, u64); +void gm107_grctx_generate_bundle(struct gf100_gr_chan *, u64, u32); void gm107_grctx_generate_attrib(struct gf100_grctx *); void gm107_grctx_generate_sm_id(struct gf100_gr *, int, int, int); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c index 10e78639a8162..d897090e829cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c @@ -872,19 +872,14 @@ gk104_grctx_generate_patch_ltc(struct gf100_grctx *info) } void -gk104_grctx_generate_bundle(struct gf100_grctx *info) +gk104_grctx_generate_bundle(struct gf100_gr_chan *chan, u64 addr, u32 size) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth, - grctx->bundle_size / 0x20); + const struct gf100_grctx_func *grctx = chan->gr->func->grctx; + const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth, size / 0x20); const u32 token_limit = grctx->bundle_token_limit; - const int s = 8; - const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true); - mmio_refn(info, 0x408004, 0x00000000, s, b); - mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s)); - mmio_refn(info, 0x418808, 0x00000000, s, b); - mmio_wr32(info, 0x41880c, 0x80000000 | (grctx->bundle_size >> s)); - mmio_wr32(info, 0x4064c8, (state_limit << 16) | token_limit); + + gf100_grctx_generate_bundle(chan, addr, size); + gf100_grctx_patch_wr32(chan, 0x4064c8, (state_limit << 16) | token_limit); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c index 46d64c2087548..2380ecce8cd69 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c @@ -62,7 +62,7 @@ gk20a_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) gf100_gr_icmd(gr, gr->bundle); grctx->pagepool(chan, chan->pagepool->addr); - grctx->bundle(info); + grctx->bundle(chan, chan->bundle_cb->addr, grctx->bundle_size); } const struct gf100_grctx_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c index 11ccee875ed7b..d968bcdfa90df 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c @@ -876,19 +876,17 @@ gm107_grctx_generate_r419e00(struct gf100_gr *gr) } void -gm107_grctx_generate_bundle(struct gf100_grctx *info) +gm107_grctx_generate_bundle(struct gf100_gr_chan *chan, u64 addr, u32 size) { - const struct gf100_grctx_func *grctx = info->gr->func->grctx; - const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth, - grctx->bundle_size / 0x20); + const struct gf100_grctx_func *grctx = chan->gr->func->grctx; + const u32 state_limit = min(grctx->bundle_min_gpm_fifo_depth, size / 0x20); const u32 token_limit = grctx->bundle_token_limit; - const int s = 8; - const int b = mmio_vram(info, grctx->bundle_size, (1 << s), true); - mmio_refn(info, 0x408004, 0x00000000, s, b); - mmio_wr32(info, 0x408008, 0x80000000 | (grctx->bundle_size >> s)); - mmio_refn(info, 0x418e24, 0x00000000, s, b); - mmio_wr32(info, 0x418e28, 0x80000000 | (grctx->bundle_size >> s)); - mmio_wr32(info, 0x4064c8, (state_limit << 16) | token_limit); + + gf100_grctx_patch_wr32(chan, 0x408004, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x408008, 0x80000000 | (size >> 8)); + gf100_grctx_patch_wr32(chan, 0x418e24, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x418e28, 0x80000000 | (size >> 8)); + gf100_grctx_patch_wr32(chan, 0x4064c8, (state_limit << 16) | token_limit); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c index d0b45d1074a29..cb02b02a4c867 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c @@ -65,7 +65,7 @@ gm20b_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) gf100_gr_icmd(gr, gr->bundle); grctx->pagepool(chan, chan->pagepool->addr); - grctx->bundle(info); + grctx->bundle(chan, chan->bundle_cb->addr, grctx->bundle_size); } const struct gf100_grctx_func diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 56f662fa37d3e..0a45e78a2f830 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -365,6 +365,7 @@ gf100_gr_chan_dtor(struct nvkm_object *object) nvkm_vmm_put(chan->vmm, &chan->mmio_vma); nvkm_memory_unref(&chan->mmio); + nvkm_vmm_put(chan->vmm, &chan->bundle_cb); nvkm_vmm_put(chan->vmm, &chan->pagepool); nvkm_vmm_unref(&chan->vmm); return chan; @@ -405,6 +406,15 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; + /* Map bundle circular buffer. */ + ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->bundle_cb), &chan->bundle_cb); + if (ret) + return ret; + + ret = nvkm_memory_map(gr->bundle_cb, 0, chan->vmm, chan->bundle_cb, &args, sizeof(args)); + if (ret) + return ret; + /* Generate golden context image. */ mutex_lock(&gr->fecs.mutex); if (gr->data == NULL) { @@ -461,6 +471,7 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, /* finally, fill in the mmio list and point the context at it */ nvkm_kmap(chan->mmio); gr->func->grctx->pagepool(chan, chan->pagepool->addr); + gr->func->grctx->bundle(chan, chan->bundle_cb->addr, gr->func->grctx->bundle_size); for (i = 0; mmio->addr && i < ARRAY_SIZE(gr->mmio_list); i++) { u32 addr = mmio->addr; u32 data = mmio->data; @@ -1982,6 +1993,11 @@ gf100_gr_oneinit(struct nvkm_gr *base) if (ret) return ret; + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, gr->func->grctx->bundle_size, + 0x100, false, &gr->bundle_cb); + if (ret) + return ret; + memset(gr->tile, 0xff, sizeof(gr->tile)); gr->func->oneinit_tiles(gr); gr->func->oneinit_sm_id(gr); @@ -2051,6 +2067,7 @@ gf100_gr_dtor(struct nvkm_gr *base) kfree(gr->data); + nvkm_memory_unref(&gr->bundle_cb); nvkm_memory_unref(&gr->pagepool); nvkm_falcon_dtor(&gr->gpccs.falcon); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index c74cc23859a50..7ac5e740aece7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -122,6 +122,7 @@ struct gf100_gr { u8 ppc_tpc_max; struct nvkm_memory *pagepool; + struct nvkm_memory *bundle_cb; u8 screen_tile_row_offset; u8 tile[TPC_MAX]; @@ -261,6 +262,7 @@ struct gf100_gr_chan { struct nvkm_vmm *vmm; struct nvkm_vma *pagepool; + struct nvkm_vma *bundle_cb; struct nvkm_memory *mmio; struct nvkm_vma *mmio_vma; -- GitLab From 5eee9fdd69d6e65d2eff9826a638f8da10066e25 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:06 +1000 Subject: [PATCH 1357/1927] drm/nouveau/gr/gf100-: move misc context patching out of attrib_cb funcs Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 4 ++- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 9 +++++-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c | 9 ++++--- .../gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c | 22 +++++++--------- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 25 +++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 2 ++ 6 files changed, 51 insertions(+), 20 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 3d06ca63e532c..076861eebc29f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1396,7 +1396,9 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) grctx->bundle(chan, chan->bundle_cb->addr, grctx->bundle_size); grctx->attrib(info); if (grctx->patch_ltc) - grctx->patch_ltc(info); + grctx->patch_ltc(chan); + if (grctx->unknown_size) + grctx->unknown(chan, chan->unknown->addr, grctx->unknown_size); grctx->unkn(gr); gf100_grctx_generate_floorsweep(gr); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 6e5fabf460159..57b778e6b86ff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -53,8 +53,11 @@ struct gf100_grctx_func { u32 alpha_nr_max; u32 alpha_nr; u32 gfxp_nr; + /* some other context buffer */ + void (*unknown)(struct gf100_gr_chan *, u64 addr, u32 size); + u32 unknown_size; /* other patch buffer stuff */ - void (*patch_ltc)(struct gf100_grctx *); + void (*patch_ltc)(struct gf100_gr_chan *); /* floorsweeping */ void (*sm_id)(struct gf100_gr *, int gpc, int tpc, int sm); void (*tpc_nr)(struct gf100_gr *, int gpc); @@ -118,7 +121,7 @@ void gk104_grctx_generate_gpc_tpc_nr(struct gf100_gr *); extern const struct gf100_grctx_func gk20a_grctx; void gk104_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gk104_grctx_generate_bundle(struct gf100_gr_chan *, u64, u32); -void gk104_grctx_generate_patch_ltc(struct gf100_grctx *); +void gk104_grctx_generate_patch_ltc(struct gf100_gr_chan *); void gk104_grctx_generate_unkn(struct gf100_gr *); void gk104_grctx_generate_r418800(struct gf100_gr *); @@ -164,6 +167,8 @@ void gv100_grctx_generate_attrib(struct gf100_grctx *); void gv100_grctx_generate_rop_mapping(struct gf100_gr *); void gv100_grctx_generate_r400088(struct gf100_gr *, bool); +void tu102_grctx_generate_unknown(struct gf100_gr_chan *, u64, u32); + /* context init value lists */ extern const struct gf100_gr_pack gf100_grctx_pack_icmd[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c index d897090e829cb..ec347405c3d8d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c @@ -861,14 +861,15 @@ gk104_grctx_generate_r418800(struct gf100_gr *gr) } void -gk104_grctx_generate_patch_ltc(struct gf100_grctx *info) +gk104_grctx_generate_patch_ltc(struct gf100_gr_chan *chan) { - struct nvkm_device *device = info->gr->base.engine.subdev.device; + struct nvkm_device *device = chan->gr->base.engine.subdev.device; u32 data0 = nvkm_rd32(device, 0x17e91c); u32 data1 = nvkm_rd32(device, 0x17e920); + /*XXX: Figure out how to modify this correctly! */ - mmio_wr32(info, 0x17e91c, data0); - mmio_wr32(info, 0x17e920, data1); + gf100_grctx_patch_wr32(chan, 0x17e91c, data0); + gf100_grctx_patch_wr32(chan, 0x17e920, data1); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c index 2299ca07d04ad..1abad38c67679 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c @@ -53,19 +53,13 @@ tu102_grctx_pack_sw_veid_bundle_init[] = { {} }; -static void -tu102_grctx_generate_attrib(struct gf100_grctx *info) +void +tu102_grctx_generate_unknown(struct gf100_gr_chan *chan, u64 addr, u32 size) { - const u64 size = 0x80000; /*XXX: educated guess */ - const int s = 8; - const int b = mmio_vram(info, size, (1 << s), true); - - gv100_grctx_generate_attrib(info); - - mmio_refn(info, 0x408070, 0x00000000, s, b); - mmio_wr32(info, 0x408074, size >> s); /*XXX: guess */ - mmio_refn(info, 0x419034, 0x00000000, s, b); - mmio_wr32(info, 0x408078, 0x00000000); + gf100_grctx_patch_wr32(chan, 0x408070, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x408074, size >> 8); /*XXX: guess */ + gf100_grctx_patch_wr32(chan, 0x419034, addr >> 8); + gf100_grctx_patch_wr32(chan, 0x408078, 0x00000000); } const struct gf100_grctx_func @@ -80,9 +74,11 @@ tu102_grctx = { .bundle_token_limit = 0xa80, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, - .attrib = tu102_grctx_generate_attrib, + .attrib = gv100_grctx_generate_attrib, .attrib_nr_max = 0x800, .attrib_nr = 0x700, + .unknown_size = 0x80000, + .unknown = tu102_grctx_generate_unknown, .alpha_nr_max = 0xc00, .alpha_nr = 0x800, .gfxp_nr = 0xfa8, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 0a45e78a2f830..5d1e45b635b98 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -365,6 +365,7 @@ gf100_gr_chan_dtor(struct nvkm_object *object) nvkm_vmm_put(chan->vmm, &chan->mmio_vma); nvkm_memory_unref(&chan->mmio); + nvkm_vmm_put(chan->vmm, &chan->unknown); nvkm_vmm_put(chan->vmm, &chan->bundle_cb); nvkm_vmm_put(chan->vmm, &chan->pagepool); nvkm_vmm_unref(&chan->vmm); @@ -415,6 +416,18 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; + /* Map some context buffer of unknown purpose. */ + if (gr->func->grctx->unknown_size) { + ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->unknown), &chan->unknown); + if (ret) + return ret; + + ret = nvkm_memory_map(gr->unknown, 0, chan->vmm, chan->unknown, + &args, sizeof(args)); + if (ret) + return ret; + } + /* Generate golden context image. */ mutex_lock(&gr->fecs.mutex); if (gr->data == NULL) { @@ -485,6 +498,10 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, data); mmio++; } + if (gr->func->grctx->patch_ltc) + gr->func->grctx->patch_ltc(chan); + if (gr->func->grctx->unknown_size) + gr->func->grctx->unknown(chan, chan->unknown->addr, gr->func->grctx->unknown_size); nvkm_done(chan->mmio); return 0; } @@ -1998,6 +2015,13 @@ gf100_gr_oneinit(struct nvkm_gr *base) if (ret) return ret; + if (gr->func->grctx->unknown_size) { + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, gr->func->grctx->unknown_size, + 0x100, false, &gr->unknown); + if (ret) + return ret; + } + memset(gr->tile, 0xff, sizeof(gr->tile)); gr->func->oneinit_tiles(gr); gr->func->oneinit_sm_id(gr); @@ -2067,6 +2091,7 @@ gf100_gr_dtor(struct nvkm_gr *base) kfree(gr->data); + nvkm_memory_unref(&gr->unknown); nvkm_memory_unref(&gr->bundle_cb); nvkm_memory_unref(&gr->pagepool); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 7ac5e740aece7..ae3eee650d8ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -123,6 +123,7 @@ struct gf100_gr { struct nvkm_memory *pagepool; struct nvkm_memory *bundle_cb; + struct nvkm_memory *unknown; u8 screen_tile_row_offset; u8 tile[TPC_MAX]; @@ -263,6 +264,7 @@ struct gf100_gr_chan { struct nvkm_vma *pagepool; struct nvkm_vma *bundle_cb; + struct nvkm_vma *unknown; struct nvkm_memory *mmio; struct nvkm_vma *mmio_vma; -- GitLab From 78a43c7e3b2ff5aed1809f93b4f87a418355789e Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:07 +1000 Subject: [PATCH 1358/1927] drm/nouveau/gr/gf100-: make global attrib_cb actually global This was thought to be per-channel initially - it's not. The backing pages for the VMM mappings are shared for all channels. - switches to more straight-forward patch interfaces - prepares for sub-context support - this is saving a *sizeable* amount of vram v2: - whitespace Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 86 ++++++------------- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 42 ++++----- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c | 21 ++--- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c | 22 +++-- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c | 2 + .../drm/nouveau/nvkm/engine/gr/ctxgk110b.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c | 7 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c | 36 ++++---- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c | 7 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c | 65 ++++++++------ .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c | 58 +++++++------ .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c | 50 +++++------ .../gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 72 ++++++---------- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 22 +---- 23 files changed, 241 insertions(+), 269 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 076861eebc29f..332590f861fb5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1003,45 +1003,6 @@ gf100_grctx_patch_wr32(struct gf100_gr_chan *chan, u32 addr, u32 data) nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, data); } -int -gf100_grctx_mmio_data(struct gf100_grctx *info, u32 size, u32 align, bool priv) -{ - if (info->data) { - info->buffer[info->buffer_nr] = round_up(info->addr, align); - info->addr = info->buffer[info->buffer_nr] + size; - info->data->size = size; - info->data->align = align; - info->data->priv = priv; - info->data++; - return info->buffer_nr++; - } - return -1; -} - -void -gf100_grctx_mmio_item(struct gf100_grctx *info, u32 addr, u32 data, - int shift, int buffer) -{ - struct nvkm_device *device = info->gr->base.engine.subdev.device; - if (info->data) { - if (shift >= 0) { - info->mmio->addr = addr; - info->mmio->data = data; - info->mmio->shift = shift; - info->mmio->buffer = buffer; - if (buffer >= 0) - data |= info->buffer[buffer] >> shift; - info->mmio++; - } else - return; - } else { - if (buffer >= 0) - return; - } - - nvkm_wr32(device, addr, data); -} - void gf100_grctx_generate_r419cb8(struct gf100_gr *gr) { @@ -1068,31 +1029,41 @@ gf100_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) } void -gf100_grctx_generate_attrib(struct gf100_grctx *info) +gf100_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 attrib = grctx->attrib_nr; - const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); int gpc, tpc; u32 bo = 0; - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_wr32(info, 0x405830, (attrib << 16)); + gf100_grctx_patch_wr32(chan, 0x405830, (attrib << 16)); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) { const u32 o = TPC_UNIT(gpc, tpc, 0x0520); - mmio_skip(info, o, (attrib << 16) | ++bo); - mmio_wr32(info, o, (attrib << 16) | --bo); + + gf100_grctx_patch_wr32(chan, o, (attrib << 16) | bo); bo += grctx->attrib_nr_max; } } } +void +gf100_grctx_generate_attrib_cb(struct gf100_gr_chan *chan, u64 addr, u32 size) +{ + gf100_grctx_patch_wr32(chan, 0x418810, 0x80000000 | addr >> 12); + gf100_grctx_patch_wr32(chan, 0x419848, 0x10000000 | addr >> 12); +} + +u32 +gf100_grctx_generate_attrib_cb_size(struct gf100_gr *gr) +{ + const struct gf100_grctx_func *grctx = gr->func->grctx; + + return 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max) * gr->tpc_total; +} + void gf100_grctx_generate_unkn(struct gf100_gr *gr) { @@ -1368,7 +1339,7 @@ gf100_grctx_generate_floorsweep(struct gf100_gr *gr) } void -gf100_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) +gf100_grctx_generate_main(struct gf100_gr_chan *chan) { struct gf100_gr *gr = chan->gr; struct nvkm_device *device = gr->base.engine.subdev.device; @@ -1394,7 +1365,8 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) grctx->pagepool(chan, chan->pagepool->addr); grctx->bundle(chan, chan->bundle_cb->addr, grctx->bundle_size); - grctx->attrib(info); + grctx->attrib_cb(chan, chan->attrib_cb->addr, grctx->attrib_cb_size(gr)); + grctx->attrib(chan); if (grctx->patch_ltc) grctx->patch_ltc(chan); if (grctx->unknown_size) @@ -1450,7 +1422,6 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk struct nvkm_device *device = subdev->device; struct nvkm_memory *data = NULL; struct nvkm_vma *ctx = NULL; - struct gf100_grctx info; int ret, i; u64 addr; @@ -1500,13 +1471,6 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk nvkm_wo32(inst, 0x0214, upper_32_bits(ctx->addr + CB_RESERVED)); nvkm_done(inst); - /* Setup default state for mmio list construction. */ - info.gr = gr; - info.data = gr->mmio_data; - info.mmio = gr->mmio_list; - info.addr = ctx->addr; - info.buffer_nr = 0; - /* Make channel current. */ addr = inst->addr >> 12; if (gr->firmware) { @@ -1530,7 +1494,7 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk ); } - grctx->main(chan, &info); + grctx->main(chan); /* Trigger a context unload by unsetting the "next channel valid" bit * and faking a context switch interrupt. @@ -1582,6 +1546,8 @@ gf100_grctx = { .bundle_size = 0x1800, .pagepool = gf100_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf100_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 57b778e6b86ff..99bb9af4955a7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -3,28 +3,12 @@ #define __NVKM_GRCTX_NVC0_H__ #include "gf100.h" -struct gf100_grctx { - struct gf100_gr *gr; - struct gf100_gr_data *data; - struct gf100_gr_mmio *mmio; - int buffer_nr; - u64 buffer[4]; - u64 addr; -}; - -int gf100_grctx_mmio_data(struct gf100_grctx *, u32 size, u32 align, bool priv); -void gf100_grctx_mmio_item(struct gf100_grctx *, u32 addr, u32 data, int s, int); void gf100_grctx_patch_wr32(struct gf100_gr_chan *, u32 addr, u32 data); -#define mmio_vram(a,b,c,d) gf100_grctx_mmio_data((a), (b), (c), (d)) -#define mmio_refn(a,b,c,d,e) gf100_grctx_mmio_item((a), (b), (c), (d), (e)) -#define mmio_skip(a,b,c) mmio_refn((a), (b), (c), -1, -1) -#define mmio_wr32(a,b,c) mmio_refn((a), (b), (c), 0, -1) - struct gf100_grctx_func { void (*unkn88c)(struct gf100_gr *, bool on); /* main context generation function */ - void (*main)(struct gf100_gr_chan *, struct gf100_grctx *); + void (*main)(struct gf100_gr_chan *); /* context-specific modify-on-first-load list generation function */ void (*unkn)(struct gf100_gr *); /* mmio context data */ @@ -47,7 +31,9 @@ struct gf100_grctx_func { void (*pagepool)(struct gf100_gr_chan *, u64 addr); u32 pagepool_size; /* attribute(/alpha) circular buffer */ - void (*attrib)(struct gf100_grctx *); + u32 (*attrib_cb_size)(struct gf100_gr *); + void (*attrib_cb)(struct gf100_gr_chan *, u64 addr, u32 size); + void (*attrib)(struct gf100_gr_chan *); u32 attrib_nr_max; u32 attrib_nr; u32 alpha_nr_max; @@ -86,10 +72,12 @@ struct gf100_grctx_func { extern const struct gf100_grctx_func gf100_grctx; int gf100_grctx_generate(struct gf100_gr *, struct gf100_gr_chan *, struct nvkm_gpuobj *inst); -void gf100_grctx_generate_main(struct gf100_gr_chan *, struct gf100_grctx *); +void gf100_grctx_generate_main(struct gf100_gr_chan *); void gf100_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gf100_grctx_generate_bundle(struct gf100_gr_chan *, u64, u32); -void gf100_grctx_generate_attrib(struct gf100_grctx *); +u32 gf100_grctx_generate_attrib_cb_size(struct gf100_gr *); +void gf100_grctx_generate_attrib_cb(struct gf100_gr_chan *, u64, u32); +void gf100_grctx_generate_attrib(struct gf100_gr_chan *); void gf100_grctx_generate_unkn(struct gf100_gr *); void gf100_grctx_generate_floorsweep(struct gf100_gr *); void gf100_grctx_generate_sm_id(struct gf100_gr *, int, int, int); @@ -101,14 +89,14 @@ void gf100_grctx_generate_max_ways_evict(struct gf100_gr *); void gf100_grctx_generate_r419cb8(struct gf100_gr *); extern const struct gf100_grctx_func gf108_grctx; -void gf108_grctx_generate_attrib(struct gf100_grctx *); +void gf108_grctx_generate_attrib(struct gf100_gr_chan *); void gf108_grctx_generate_unkn(struct gf100_gr *); extern const struct gf100_grctx_func gf104_grctx; extern const struct gf100_grctx_func gf110_grctx; extern const struct gf100_grctx_func gf117_grctx; -void gf117_grctx_generate_attrib(struct gf100_grctx *); +void gf117_grctx_generate_attrib(struct gf100_gr_chan *); void gf117_grctx_generate_rop_mapping(struct gf100_gr *); void gf117_grctx_generate_dist_skip_table(struct gf100_gr *); @@ -134,7 +122,8 @@ extern const struct gf100_grctx_func gk208_grctx; extern const struct gf100_grctx_func gm107_grctx; void gm107_grctx_generate_pagepool(struct gf100_gr_chan *, u64); void gm107_grctx_generate_bundle(struct gf100_gr_chan *, u64, u32); -void gm107_grctx_generate_attrib(struct gf100_grctx *); +void gm107_grctx_generate_attrib_cb(struct gf100_gr_chan *, u64, u32); +void gm107_grctx_generate_attrib(struct gf100_gr_chan *); void gm107_grctx_generate_sm_id(struct gf100_gr *, int, int, int); extern const struct gf100_grctx_func gm200_grctx; @@ -148,10 +137,12 @@ extern const struct gf100_grctx_func gm20b_grctx; extern const struct gf100_grctx_func gp100_grctx; void gp100_grctx_generate_pagepool(struct gf100_gr_chan *, u64); +void gp100_grctx_generate_attrib_cb(struct gf100_gr_chan *, u64, u32); void gp100_grctx_generate_smid_config(struct gf100_gr *); extern const struct gf100_grctx_func gp102_grctx; -void gp102_grctx_generate_attrib(struct gf100_grctx *); +u32 gp102_grctx_generate_attrib_cb_size(struct gf100_gr *); +void gp102_grctx_generate_attrib(struct gf100_gr_chan *); extern const struct gf100_grctx_func gp104_grctx; @@ -163,7 +154,8 @@ extern const struct gf100_grctx_func tu102_grctx; void gv100_grctx_unkn88c(struct gf100_gr *, bool); void gv100_grctx_generate_unkn(struct gf100_gr *); extern const struct gf100_gr_init gv100_grctx_init_sw_veid_bundle_init_0[]; -void gv100_grctx_generate_attrib(struct gf100_grctx *); +void gv100_grctx_generate_attrib_cb(struct gf100_gr_chan *, u64, u32); +void gv100_grctx_generate_attrib(struct gf100_gr_chan *); void gv100_grctx_generate_rop_mapping(struct gf100_gr *); void gv100_grctx_generate_r400088(struct gf100_gr *, bool); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c index 7a0564b6e3c71..ba63a3b46518e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf104.c @@ -94,6 +94,8 @@ gf104_grctx = { .bundle_size = 0x1800, .pagepool = gf100_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf100_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c index dda2c32e62324..0bc2eab6ad986 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf108.c @@ -733,25 +733,20 @@ gf108_grctx_pack_tpc[] = { ******************************************************************************/ void -gf108_grctx_generate_attrib(struct gf100_grctx *info) +gf108_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 alpha = grctx->alpha_nr; const u32 beta = grctx->attrib_nr; - const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); const int timeslice_mode = 1; const int max_batches = 0xffff; u32 bo = 0; u32 ao = bo + grctx->attrib_nr_max * gr->tpc_total; int gpc, tpc; - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_wr32(info, 0x405830, (beta << 16) | alpha); - mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches); + gf100_grctx_patch_wr32(chan, 0x405830, (beta << 16) | alpha); + gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) { @@ -759,10 +754,10 @@ gf108_grctx_generate_attrib(struct gf100_grctx *info) const u32 b = beta; const u32 t = timeslice_mode; const u32 o = TPC_UNIT(gpc, tpc, 0x500); - mmio_skip(info, o + 0x20, (t << 28) | (b << 16) | ++bo); - mmio_wr32(info, o + 0x20, (t << 28) | (b << 16) | --bo); + + gf100_grctx_patch_wr32(chan, o + 0x20, (t << 28) | (b << 16) | bo); bo += grctx->attrib_nr_max; - mmio_wr32(info, o + 0x44, (a << 16) | ao); + gf100_grctx_patch_wr32(chan, o + 0x44, (a << 16) | ao); ao += grctx->alpha_nr_max; } } @@ -795,6 +790,8 @@ gf108_grctx = { .bundle_size = 0x1800, .pagepool = gf100_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf108_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c index f5cca5e6a4f24..64b723b0afb53 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf110.c @@ -342,6 +342,8 @@ gf110_grctx = { .bundle_size = 0x1800, .pagepool = gf100_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf100_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c index 276c282d19aa6..6c1f631eebd5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c @@ -241,25 +241,20 @@ gf117_grctx_generate_rop_mapping(struct gf100_gr *gr) } void -gf117_grctx_generate_attrib(struct gf100_grctx *info) +gf117_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 alpha = grctx->alpha_nr; const u32 beta = grctx->attrib_nr; - const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); const int timeslice_mode = 1; const int max_batches = 0xffff; u32 bo = 0; u32 ao = bo + grctx->attrib_nr_max * gr->tpc_total; int gpc, ppc; - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_wr32(info, 0x405830, (beta << 16) | alpha); - mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches); + gf100_grctx_patch_wr32(chan, 0x405830, (beta << 16) | alpha); + gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++) { @@ -267,12 +262,13 @@ gf117_grctx_generate_attrib(struct gf100_grctx *info) const u32 b = beta * gr->ppc_tpc_nr[gpc][ppc]; const u32 t = timeslice_mode; const u32 o = PPC_UNIT(gpc, ppc, 0); + if (!(gr->ppc_mask[gpc] & (1 << ppc))) continue; - mmio_skip(info, o + 0xc0, (t << 28) | (b << 16) | ++bo); - mmio_wr32(info, o + 0xc0, (t << 28) | (b << 16) | --bo); + + gf100_grctx_patch_wr32(chan, o + 0xc0, (t << 28) | (b << 16) | bo); bo += grctx->attrib_nr_max * gr->ppc_tpc_nr[gpc][ppc]; - mmio_wr32(info, o + 0xe4, (a << 16) | ao); + gf100_grctx_patch_wr32(chan, o + 0xe4, (a << 16) | ao); ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc]; } } @@ -294,6 +290,8 @@ gf117_grctx = { .bundle_size = 0x1800, .pagepool = gf100_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf117_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c index 0cfe46366af60..426ad1b8d426b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf119.c @@ -510,6 +510,8 @@ gf119_grctx = { .bundle_size = 0x1800, .pagepool = gf100_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf108_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c index ec347405c3d8d..94233d0119dff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c @@ -981,6 +981,8 @@ gk104_grctx = { .bundle_token_limit = 0x600, .pagepool = gk104_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf117_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c index 86547cfc38dce..4391458e1fb2f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110.c @@ -838,6 +838,8 @@ gk110_grctx = { .bundle_token_limit = 0x7c0, .pagepool = gk104_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf117_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c index ebb947bd1446b..7b9a34f9ec3c7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk110b.c @@ -87,6 +87,8 @@ gk110b_grctx = { .bundle_token_limit = 0x600, .pagepool = gk104_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf117_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c index 4d40512b5c998..c78d07a8bb7df 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk208.c @@ -553,6 +553,8 @@ gk208_grctx = { .bundle_token_limit = 0x200, .pagepool = gk104_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf117_grctx_generate_attrib, .attrib_nr_max = 0x324, .attrib_nr = 0x218, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c index 2380ecce8cd69..ac5fdcb5cd3fd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk20a.c @@ -25,7 +25,7 @@ #include static void -gk20a_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) +gk20a_grctx_generate_main(struct gf100_gr_chan *chan) { struct gf100_gr *gr = chan->gr; struct nvkm_device *device = gr->base.engine.subdev.device; @@ -39,7 +39,8 @@ gk20a_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000); - grctx->attrib(info); + grctx->attrib_cb(chan, chan->attrib_cb->addr, grctx->attrib_cb_size(gr)); + grctx->attrib(chan); grctx->unkn(gr); @@ -75,6 +76,8 @@ gk20a_grctx = { .bundle_token_limit = 0x100, .pagepool = gk104_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gf100_grctx_generate_attrib_cb, .attrib = gf117_grctx_generate_attrib, .attrib_nr_max = 0x240, .attrib_nr = 0x240, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c index d968bcdfa90df..019c698eb5a55 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c @@ -897,25 +897,19 @@ gm107_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) } void -gm107_grctx_generate_attrib(struct gf100_grctx *info) +gm107_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 alpha = grctx->alpha_nr; const u32 attrib = grctx->attrib_nr; - const u32 size = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max); - const int s = 12; - const int b = mmio_vram(info, size * gr->tpc_total, (1 << s), false); const int max_batches = 0xffff; u32 bo = 0; u32 ao = bo + grctx->attrib_nr_max * gr->tpc_total; int gpc, ppc, n = 0; - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_refn(info, 0x419c2c, 0x10000000, s, b); - mmio_wr32(info, 0x405830, (attrib << 16) | alpha); - mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches); + gf100_grctx_patch_wr32(chan, 0x405830, (attrib << 16) | alpha); + gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { @@ -923,19 +917,29 @@ gm107_grctx_generate_attrib(struct gf100_grctx *info) const u32 bs = attrib * gr->ppc_tpc_nr[gpc][ppc]; const u32 u = 0x418ea0 + (n * 0x04); const u32 o = PPC_UNIT(gpc, ppc, 0); + if (!(gr->ppc_mask[gpc] & (1 << ppc))) continue; - mmio_wr32(info, o + 0xc0, bs); - mmio_wr32(info, o + 0xf4, bo); + + gf100_grctx_patch_wr32(chan, o + 0xc0, bs); + gf100_grctx_patch_wr32(chan, o + 0xf4, bo); bo += grctx->attrib_nr_max * gr->ppc_tpc_nr[gpc][ppc]; - mmio_wr32(info, o + 0xe4, as); - mmio_wr32(info, o + 0xf8, ao); + gf100_grctx_patch_wr32(chan, o + 0xe4, as); + gf100_grctx_patch_wr32(chan, o + 0xf8, ao); ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc]; - mmio_wr32(info, u, ((bs / 3) << 16) | bs); + gf100_grctx_patch_wr32(chan, u, ((bs / 3) << 16) | bs); } } } +void +gm107_grctx_generate_attrib_cb(struct gf100_gr_chan *chan, u64 addr, u32 size) +{ + gf100_grctx_generate_attrib_cb(chan, addr, size); + + gf100_grctx_patch_wr32(chan, 0x419c2c, 0x10000000 | addr >> 12); +} + static void gm107_grctx_generate_r406500(struct gf100_gr *gr) { @@ -969,6 +973,8 @@ gm107_grctx = { .bundle_token_limit = 0x2c0, .pagepool = gm107_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gm107_grctx_generate_attrib_cb, .attrib = gm107_grctx_generate_attrib, .attrib_nr_max = 0xff0, .attrib_nr = 0xaa0, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c index 013d05a0f0f64..6b7034b2ff766 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c @@ -111,6 +111,8 @@ gm200_grctx = { .bundle_token_limit = 0x780, .pagepool = gm107_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gm107_grctx_generate_attrib_cb, .attrib = gm107_grctx_generate_attrib, .attrib_nr_max = 0x600, .attrib_nr = 0x400, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c index cb02b02a4c867..b8edccfada586 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm20b.c @@ -22,7 +22,7 @@ #include "ctxgf100.h" static void -gm20b_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) +gm20b_grctx_generate_main(struct gf100_gr_chan *chan) { struct gf100_gr *gr = chan->gr; struct nvkm_device *device = gr->base.engine.subdev.device; @@ -36,7 +36,8 @@ gm20b_grctx_generate_main(struct gf100_gr_chan *chan, struct gf100_grctx *info) idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000); - grctx->attrib(info); + grctx->attrib_cb(chan, chan->attrib_cb->addr, grctx->attrib_cb_size(gr)); + grctx->attrib(chan); grctx->unkn(gr); @@ -78,6 +79,8 @@ gm20b_grctx = { .bundle_token_limit = 0x1c0, .pagepool = gm107_grctx_generate_pagepool, .pagepool_size = 0x8000, + .attrib_cb_size = gf100_grctx_generate_attrib_cb_size, + .attrib_cb = gm107_grctx_generate_attrib_cb, .attrib = gm107_grctx_generate_attrib, .attrib_nr_max = 0x600, .attrib_nr = 0x400, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c index b2fa7c943c494..d8977caa0d040 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c @@ -39,32 +39,21 @@ gp100_grctx_generate_pagepool(struct gf100_gr_chan *chan, u64 addr) } static void -gp100_grctx_generate_attrib(struct gf100_grctx *info) +gp100_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 alpha = grctx->alpha_nr; const u32 attrib = grctx->attrib_nr; - const int s = 12; const int max_batches = 0xffff; u32 size = grctx->alpha_nr_max * gr->tpc_total; u32 ao = 0; u32 bo = ao + size; - int gpc, ppc, b, n = 0; + int gpc, ppc, n = 0; - for (gpc = 0; gpc < gr->gpc_nr; gpc++) - size += grctx->attrib_nr_max * gr->ppc_nr[gpc] * gr->ppc_tpc_max; - size = ((size * 0x20) + 128) & ~127; - b = mmio_vram(info, size, (1 << s), false); - - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_refn(info, 0x419c2c, 0x10000000, s, b); - mmio_refn(info, 0x419b00, 0x00000000, s, b); - mmio_wr32(info, 0x419b04, 0x80000000 | size >> 7); - mmio_wr32(info, 0x405830, attrib); - mmio_wr32(info, 0x40585c, alpha); - mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches); + gf100_grctx_patch_wr32(chan, 0x405830, attrib); + gf100_grctx_patch_wr32(chan, 0x40585c, alpha); + gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { @@ -72,21 +61,45 @@ gp100_grctx_generate_attrib(struct gf100_grctx *info) const u32 bs = attrib * gr->ppc_tpc_max; const u32 u = 0x418ea0 + (n * 0x04); const u32 o = PPC_UNIT(gpc, ppc, 0); + if (!(gr->ppc_mask[gpc] & (1 << ppc))) continue; - mmio_wr32(info, o + 0xc0, bs); - mmio_wr32(info, o + 0xf4, bo); - mmio_wr32(info, o + 0xf0, bs); + + gf100_grctx_patch_wr32(chan, o + 0xc0, bs); + gf100_grctx_patch_wr32(chan, o + 0xf4, bo); + gf100_grctx_patch_wr32(chan, o + 0xf0, bs); bo += grctx->attrib_nr_max * gr->ppc_tpc_max; - mmio_wr32(info, o + 0xe4, as); - mmio_wr32(info, o + 0xf8, ao); + gf100_grctx_patch_wr32(chan, o + 0xe4, as); + gf100_grctx_patch_wr32(chan, o + 0xf8, ao); ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc]; - mmio_wr32(info, u, bs); + gf100_grctx_patch_wr32(chan, u, bs); } } - mmio_wr32(info, 0x418eec, 0x00000000); - mmio_wr32(info, 0x41befc, 0x00000000); + gf100_grctx_patch_wr32(chan, 0x418eec, 0x00000000); + gf100_grctx_patch_wr32(chan, 0x41befc, 0x00000000); +} + +void +gp100_grctx_generate_attrib_cb(struct gf100_gr_chan *chan, u64 addr, u32 size) +{ + gm107_grctx_generate_attrib_cb(chan, addr, size); + + gf100_grctx_patch_wr32(chan, 0x419b00, 0x00000000 | addr >> 12); + gf100_grctx_patch_wr32(chan, 0x419b04, 0x80000000 | size >> 7); +} + +static u32 +gp100_grctx_generate_attrib_cb_size(struct gf100_gr *gr) +{ + const struct gf100_grctx_func *grctx = gr->func->grctx; + u32 size = grctx->alpha_nr_max * gr->tpc_total; + int gpc; + + for (gpc = 0; gpc < gr->gpc_nr; gpc++) + size += grctx->attrib_nr_max * gr->func->ppc_nr * gr->ppc_tpc_max; + + return ((size * 0x20) + 128) & ~127; } void @@ -120,6 +133,8 @@ gp100_grctx = { .bundle_token_limit = 0x1080, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gp100_grctx_generate_attrib_cb_size, + .attrib_cb = gp100_grctx_generate_attrib_cb, .attrib = gp100_grctx_generate_attrib, .attrib_nr_max = 0x660, .attrib_nr = 0x440, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c index daee17bf7d0df..7d372d786b8bd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c @@ -37,33 +37,22 @@ gp102_grctx_generate_r408840(struct gf100_gr *gr) } void -gp102_grctx_generate_attrib(struct gf100_grctx *info) +gp102_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 alpha = grctx->alpha_nr; const u32 attrib = grctx->attrib_nr; const u32 gfxp = grctx->gfxp_nr; - const int s = 12; const int max_batches = 0xffff; u32 size = grctx->alpha_nr_max * gr->tpc_total; u32 ao = 0; u32 bo = ao + size; - int gpc, ppc, b, n = 0; + int gpc, ppc, n = 0; - for (gpc = 0; gpc < gr->gpc_nr; gpc++) - size += grctx->gfxp_nr * gr->ppc_nr[gpc] * gr->ppc_tpc_max; - size = ((size * 0x20) + 128) & ~127; - b = mmio_vram(info, size, (1 << s), false); - - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_refn(info, 0x419c2c, 0x10000000, s, b); - mmio_refn(info, 0x419b00, 0x00000000, s, b); - mmio_wr32(info, 0x419b04, 0x80000000 | size >> 7); - mmio_wr32(info, 0x405830, attrib); - mmio_wr32(info, 0x40585c, alpha); - mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches); + gf100_grctx_patch_wr32(chan, 0x405830, attrib); + gf100_grctx_patch_wr32(chan, 0x40585c, alpha); + gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { @@ -73,22 +62,37 @@ gp102_grctx_generate_attrib(struct gf100_grctx *info) const u32 u = 0x418ea0 + (n * 0x04); const u32 o = PPC_UNIT(gpc, ppc, 0); const u32 p = GPC_UNIT(gpc, 0xc44 + (ppc * 4)); + if (!(gr->ppc_mask[gpc] & (1 << ppc))) continue; - mmio_wr32(info, o + 0xc0, gs); - mmio_wr32(info, p, bs); - mmio_wr32(info, o + 0xf4, bo); - mmio_wr32(info, o + 0xf0, bs); + + gf100_grctx_patch_wr32(chan, o + 0xc0, gs); + gf100_grctx_patch_wr32(chan, p, bs); + gf100_grctx_patch_wr32(chan, o + 0xf4, bo); + gf100_grctx_patch_wr32(chan, o + 0xf0, bs); bo += gs; - mmio_wr32(info, o + 0xe4, as); - mmio_wr32(info, o + 0xf8, ao); + gf100_grctx_patch_wr32(chan, o + 0xe4, as); + gf100_grctx_patch_wr32(chan, o + 0xf8, ao); ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc]; - mmio_wr32(info, u, bs); + gf100_grctx_patch_wr32(chan, u, bs); } } - mmio_wr32(info, 0x4181e4, 0x00000100); - mmio_wr32(info, 0x41befc, 0x00000100); + gf100_grctx_patch_wr32(chan, 0x4181e4, 0x00000100); + gf100_grctx_patch_wr32(chan, 0x41befc, 0x00000100); +} + +u32 +gp102_grctx_generate_attrib_cb_size(struct gf100_gr *gr) +{ + const struct gf100_grctx_func *grctx = gr->func->grctx; + u32 size = grctx->alpha_nr_max * gr->tpc_total; + int gpc; + + for (gpc = 0; gpc < gr->gpc_nr; gpc++) + size += grctx->gfxp_nr * gr->func->ppc_nr * gr->ppc_tpc_max; + + return ((size * 0x20) + 127) & ~127; } const struct gf100_grctx_func @@ -101,6 +105,8 @@ gp102_grctx = { .bundle_token_limit = 0x900, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gp102_grctx_generate_attrib_cb_size, + .attrib_cb = gp100_grctx_generate_attrib_cb, .attrib = gp102_grctx_generate_attrib, .attrib_nr_max = 0x4b0, .attrib_nr = 0x320, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c index 3b85e3d326b25..90b5f793e5676 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp104.c @@ -31,6 +31,8 @@ gp104_grctx = { .bundle_token_limit = 0x900, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gp102_grctx_generate_attrib_cb_size, + .attrib_cb = gp100_grctx_generate_attrib_cb, .attrib = gp102_grctx_generate_attrib, .attrib_nr_max = 0x4b0, .attrib_nr = 0x320, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c index 5060c5ee5ce04..d191761a04711 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp107.c @@ -39,6 +39,8 @@ gp107_grctx = { .bundle_token_limit = 0x300, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gp102_grctx_generate_attrib_cb_size, + .attrib_cb = gp100_grctx_generate_attrib_cb, .attrib = gp102_grctx_generate_attrib, .attrib_nr_max = 0x15de, .attrib_nr = 0x540, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c index 39553d55d3f3a..c2e5d43b1c37b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c @@ -59,31 +59,20 @@ gv100_grctx_pack_sw_veid_bundle_init[] = { }; void -gv100_grctx_generate_attrib(struct gf100_grctx *info) +gv100_grctx_generate_attrib(struct gf100_gr_chan *chan) { - struct gf100_gr *gr = info->gr; + struct gf100_gr *gr = chan->gr; const struct gf100_grctx_func *grctx = gr->func->grctx; const u32 alpha = grctx->alpha_nr; const u32 attrib = grctx->attrib_nr; const u32 gfxp = grctx->gfxp_nr; - const int s = 12; u32 size = grctx->alpha_nr_max * gr->tpc_total; u32 ao = 0; u32 bo = ao + size; - int gpc, ppc, b, n = 0; + int gpc, ppc, n = 0; - for (gpc = 0; gpc < gr->gpc_nr; gpc++) - size += grctx->gfxp_nr * gr->ppc_nr[gpc] * gr->ppc_tpc_max; - size = ((size * 0x20) + 127) & ~127; - b = mmio_vram(info, size, (1 << s), false); - - mmio_refn(info, 0x418810, 0x80000000, s, b); - mmio_refn(info, 0x419848, 0x10000000, s, b); - mmio_refn(info, 0x419c2c, 0x10000000, s, b); - mmio_refn(info, 0x419e00, 0x00000000, s, b); - mmio_wr32(info, 0x419e04, 0x80000000 | size >> 7); - mmio_wr32(info, 0x405830, attrib); - mmio_wr32(info, 0x40585c, alpha); + gf100_grctx_patch_wr32(chan, 0x405830, attrib); + gf100_grctx_patch_wr32(chan, 0x40585c, alpha); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { @@ -92,21 +81,32 @@ gv100_grctx_generate_attrib(struct gf100_grctx *info) const u32 gs = gfxp * gr->ppc_tpc_max; const u32 u = 0x418ea0 + (n * 0x04); const u32 o = PPC_UNIT(gpc, ppc, 0); + if (!(gr->ppc_mask[gpc] & (1 << ppc))) continue; - mmio_wr32(info, o + 0xc0, gs); - mmio_wr32(info, o + 0xf4, bo); - mmio_wr32(info, o + 0xf0, bs); + + gf100_grctx_patch_wr32(chan, o + 0xc0, gs); + gf100_grctx_patch_wr32(chan, o + 0xf4, bo); + gf100_grctx_patch_wr32(chan, o + 0xf0, bs); bo += gs; - mmio_wr32(info, o + 0xe4, as); - mmio_wr32(info, o + 0xf8, ao); + gf100_grctx_patch_wr32(chan, o + 0xe4, as); + gf100_grctx_patch_wr32(chan, o + 0xf8, ao); ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc]; - mmio_wr32(info, u, bs); + gf100_grctx_patch_wr32(chan, u, bs); } } - mmio_wr32(info, 0x4181e4, 0x00000100); - mmio_wr32(info, 0x41befc, 0x00000100); + gf100_grctx_patch_wr32(chan, 0x4181e4, 0x00000100); + gf100_grctx_patch_wr32(chan, 0x41befc, 0x00000100); +} + +void +gv100_grctx_generate_attrib_cb(struct gf100_gr_chan *chan, u64 addr, u32 size) +{ + gm107_grctx_generate_attrib_cb(chan, addr, size); + + gf100_grctx_patch_wr32(chan, 0x419e00, 0x00000000 | addr >> 12); + gf100_grctx_patch_wr32(chan, 0x419e04, 0x80000000 | size >> 7); } void @@ -198,6 +198,8 @@ gv100_grctx = { .bundle_token_limit = 0x1680, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gp102_grctx_generate_attrib_cb_size, + .attrib_cb = gv100_grctx_generate_attrib_cb, .attrib = gv100_grctx_generate_attrib, .attrib_nr_max = 0x6c0, .attrib_nr = 0x480, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c index 1abad38c67679..1a151e89ea05c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c @@ -74,6 +74,8 @@ tu102_grctx = { .bundle_token_limit = 0xa80, .pagepool = gp100_grctx_generate_pagepool, .pagepool_size = 0x20000, + .attrib_cb_size = gp102_grctx_generate_attrib_cb_size, + .attrib_cb = gv100_grctx_generate_attrib_cb, .attrib = gv100_grctx_generate_attrib, .attrib_nr_max = 0x800, .attrib_nr = 0x700, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 5d1e45b635b98..09141b1b42201 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -355,16 +355,11 @@ static void * gf100_gr_chan_dtor(struct nvkm_object *object) { struct gf100_gr_chan *chan = gf100_gr_chan(object); - int i; - - for (i = 0; i < ARRAY_SIZE(chan->data); i++) { - nvkm_vmm_put(chan->vmm, &chan->data[i].vma); - nvkm_memory_unref(&chan->data[i].mem); - } nvkm_vmm_put(chan->vmm, &chan->mmio_vma); nvkm_memory_unref(&chan->mmio); + nvkm_vmm_put(chan->vmm, &chan->attrib_cb); nvkm_vmm_put(chan->vmm, &chan->unknown); nvkm_vmm_put(chan->vmm, &chan->bundle_cb); nvkm_vmm_put(chan->vmm, &chan->pagepool); @@ -384,12 +379,10 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, struct nvkm_object **pobject) { struct gf100_gr *gr = gf100_gr(base); - struct gf100_gr_data *data = gr->mmio_data; - struct gf100_gr_mmio *mmio = gr->mmio_list; struct gf100_gr_chan *chan; struct gf100_vmm_map_v0 args = { .priv = 1 }; struct nvkm_device *device = gr->base.engine.subdev.device; - int ret, i; + int ret; if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) return -ENOMEM; @@ -416,6 +409,22 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; + /* Map attribute circular buffer. */ + ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->attrib_cb), &chan->attrib_cb); + if (ret) + return ret; + + if (device->card_type < GP100) { + ret = nvkm_memory_map(gr->attrib_cb, 0, chan->vmm, chan->attrib_cb, NULL, 0); + if (ret) + return ret; + } else { + ret = nvkm_memory_map(gr->attrib_cb, 0, chan->vmm, chan->attrib_cb, + &args, sizeof(args));; + if (ret) + return ret; + } + /* Map some context buffer of unknown purpose. */ if (gr->func->grctx->unknown_size) { ret = nvkm_vmm_get(chan->vmm, 12, nvkm_memory_size(gr->unknown), &chan->unknown); @@ -457,47 +466,12 @@ gf100_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch, if (ret) return ret; - /* allocate buffers referenced by mmio list */ - for (i = 0; data->size && i < ARRAY_SIZE(gr->mmio_data); i++) { - ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, - data->size, data->align, false, - &chan->data[i].mem); - if (ret) - return ret; - - ret = nvkm_vmm_get(fifoch->vmm, 12, - nvkm_memory_size(chan->data[i].mem), - &chan->data[i].vma); - if (ret) - return ret; - - args.priv = data->priv; - - ret = nvkm_memory_map(chan->data[i].mem, 0, chan->vmm, - chan->data[i].vma, &args, sizeof(args)); - if (ret) - return ret; - - data++; - } - /* finally, fill in the mmio list and point the context at it */ nvkm_kmap(chan->mmio); gr->func->grctx->pagepool(chan, chan->pagepool->addr); gr->func->grctx->bundle(chan, chan->bundle_cb->addr, gr->func->grctx->bundle_size); - for (i = 0; mmio->addr && i < ARRAY_SIZE(gr->mmio_list); i++) { - u32 addr = mmio->addr; - u32 data = mmio->data; - - if (mmio->buffer >= 0) { - u64 info = chan->data[mmio->buffer].vma->addr; - data |= info >> mmio->shift; - } - - nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, addr); - nvkm_wo32(chan->mmio, chan->mmio_nr++ * 4, data); - mmio++; - } + gr->func->grctx->attrib_cb(chan, chan->attrib_cb->addr, gr->func->grctx->attrib_cb_size(gr)); + gr->func->grctx->attrib(chan); if (gr->func->grctx->patch_ltc) gr->func->grctx->patch_ltc(chan); if (gr->func->grctx->unknown_size) @@ -2015,6 +1989,11 @@ gf100_gr_oneinit(struct nvkm_gr *base) if (ret) return ret; + ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, gr->func->grctx->attrib_cb_size(gr), + 0x1000, false, &gr->attrib_cb); + if (ret) + return ret; + if (gr->func->grctx->unknown_size) { ret = nvkm_memory_new(device, NVKM_MEM_TARGET_INST, gr->func->grctx->unknown_size, 0x100, false, &gr->unknown); @@ -2092,6 +2071,7 @@ gf100_gr_dtor(struct nvkm_gr *base) kfree(gr->data); nvkm_memory_unref(&gr->unknown); + nvkm_memory_unref(&gr->attrib_cb); nvkm_memory_unref(&gr->bundle_cb); nvkm_memory_unref(&gr->pagepool); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index ae3eee650d8ef..7f467f6d0462c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -44,19 +44,6 @@ struct nvkm_acr_lsfw; #define PPC_UNIT(t, m, r) (0x503000 + (t) * 0x8000 + (m) * 0x200 + (r)) #define TPC_UNIT(t, m, r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r)) -struct gf100_gr_data { - u32 size; - u32 align; - bool priv; -}; - -struct gf100_gr_mmio { - u32 addr; - u32 data; - u32 shift; - int buffer; -}; - struct gf100_gr_zbc_color { u32 format; u32 ds[4]; @@ -123,6 +110,7 @@ struct gf100_gr { struct nvkm_memory *pagepool; struct nvkm_memory *bundle_cb; + struct nvkm_memory *attrib_cb; struct nvkm_memory *unknown; u8 screen_tile_row_offset; @@ -134,8 +122,6 @@ struct gf100_gr { } sm[TPC_MAX]; u8 sm_nr; - struct gf100_gr_data mmio_data[4]; - struct gf100_gr_mmio mmio_list[4096/8]; u32 size; u32 *data; u32 size_zcull; @@ -264,16 +250,12 @@ struct gf100_gr_chan { struct nvkm_vma *pagepool; struct nvkm_vma *bundle_cb; + struct nvkm_vma *attrib_cb; struct nvkm_vma *unknown; struct nvkm_memory *mmio; struct nvkm_vma *mmio_vma; int mmio_nr; - - struct { - struct nvkm_memory *mem; - struct nvkm_vma *vma; - } data[4]; }; void gf100_gr_ctxctl_debug(struct gf100_gr *); -- GitLab From 6c55b5947b174cce16fcc56dc79f50df2b48e737 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:07 +1000 Subject: [PATCH 1359/1927] drm/nouveau/gr/gf100-: stop using NV_PGRAPH_FECS_CTXSW_MAILBOX_CLEAR This doesn't work on Ampere for some reason, switch to directly modifying NV_PGRAPH_FECS_CTXSW_MAILBOX instead. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 09141b1b42201..30dce77ffa11d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -752,7 +752,7 @@ gf100_gr_fecs_ctrl_ctxsw(struct gf100_gr *gr, u32 mthd) struct nvkm_device *device = gr->base.engine.subdev.device; nvkm_wr32(device, 0x409804, 0xffffffff); - nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409800, 0x00000000); nvkm_wr32(device, 0x409500, 0xffffffff); nvkm_wr32(device, 0x409504, mthd); nvkm_msec(device, 2000, @@ -801,7 +801,7 @@ gf100_gr_fecs_bind_pointer(struct gf100_gr *gr, u32 inst) { struct nvkm_device *device = gr->base.engine.subdev.device; - nvkm_wr32(device, 0x409840, 0x00000030); + nvkm_mask(device, 0x409800, 0x00000030, 0x00000000); nvkm_wr32(device, 0x409500, inst); nvkm_wr32(device, 0x409504, 0x00000003); nvkm_msec(device, 2000, @@ -892,7 +892,7 @@ gf100_gr_fecs_discover_pm_image_size(struct gf100_gr *gr, u32 *psize) { struct nvkm_device *device = gr->base.engine.subdev.device; - nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409800, 0x00000000); nvkm_wr32(device, 0x409500, 0x00000000); nvkm_wr32(device, 0x409504, 0x00000025); nvkm_msec(device, 2000, @@ -908,7 +908,7 @@ gf100_gr_fecs_discover_zcull_image_size(struct gf100_gr *gr, u32 *psize) { struct nvkm_device *device = gr->base.engine.subdev.device; - nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409800, 0x00000000); nvkm_wr32(device, 0x409500, 0x00000000); nvkm_wr32(device, 0x409504, 0x00000016); nvkm_msec(device, 2000, @@ -924,7 +924,7 @@ gf100_gr_fecs_discover_image_size(struct gf100_gr *gr, u32 *psize) { struct nvkm_device *device = gr->base.engine.subdev.device; - nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409800, 0x00000000); nvkm_wr32(device, 0x409500, 0x00000000); nvkm_wr32(device, 0x409504, 0x00000010); nvkm_msec(device, 2000, @@ -940,7 +940,7 @@ gf100_gr_fecs_set_watchdog_timeout(struct gf100_gr *gr, u32 timeout) { struct nvkm_device *device = gr->base.engine.subdev.device; - nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409800, 0x00000000); nvkm_wr32(device, 0x409500, timeout); nvkm_wr32(device, 0x409504, 0x00000021); } @@ -1746,7 +1746,7 @@ gf100_gr_init_ctxctl_ext(struct gf100_gr *gr) nvkm_mc_unk260(device, 1); /* start both of them running */ - nvkm_wr32(device, 0x409840, 0xffffffff); + nvkm_wr32(device, 0x409800, 0x00000000); nvkm_wr32(device, 0x41a10c, 0x00000000); nvkm_wr32(device, 0x40910c, 0x00000000); -- GitLab From 6762510bc8447dc4eb4c3d99541de6b31843e649 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:08 +1000 Subject: [PATCH 1360/1927] drm/nouveau/gr/gf100-: call FECS WFI_GOLDEN_SAVE method This won't work on Ampere, and, it's questionable whether we should have been using our FW's method of storing the golden context image with NV's firmware to begin with. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 30 ++++++++++++------- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 19 ++++++++++++ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 1 + 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 332590f861fb5..eef9b3f065fd9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1496,17 +1496,25 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk grctx->main(chan); - /* Trigger a context unload by unsetting the "next channel valid" bit - * and faking a context switch interrupt. - */ - nvkm_mask(device, 0x409b04, 0x80000000, 0x00000000); - nvkm_wr32(device, 0x409000, 0x00000100); - if (nvkm_msec(device, 2000, - if (!(nvkm_rd32(device, 0x409b00) & 0x80000000)) - break; - ) < 0) { - ret = -EBUSY; - goto done_inst; + if (!gr->firmware) { + /* Trigger a context unload by unsetting the "next channel valid" bit + * and faking a context switch interrupt. + */ + nvkm_mask(device, 0x409b04, 0x80000000, 0x00000000); + nvkm_wr32(device, 0x409000, 0x00000100); + if (nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x409b00) & 0x80000000)) + break; + ) < 0) { + ret = -EBUSY; + goto done_inst; + } + } else { + ret = gf100_gr_fecs_wfi_golden_save(gr, 0x80000000 | addr); + if (ret) + goto done_inst; + + nvkm_mask(device, 0x409b00, 0x80000000, 0x00000000); } gr->data = kmalloc(gr->size, GFP_KERNEL); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 30dce77ffa11d..22f360df1b3a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -796,6 +796,25 @@ gf100_gr_fecs_stop_ctxsw(struct nvkm_gr *base) return ret; } +int +gf100_gr_fecs_wfi_golden_save(struct gf100_gr *gr, u32 inst) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_mask(device, 0x409800, 0x00000003, 0x00000000); + nvkm_wr32(device, 0x409500, inst); + nvkm_wr32(device, 0x409504, 0x00000009); + nvkm_msec(device, 2000, + u32 stat = nvkm_rd32(device, 0x409800); + if (stat & 0x00000002) + return -EIO; + if (stat & 0x00000001) + return 0; + ); + + return -ETIMEDOUT; +} + int gf100_gr_fecs_bind_pointer(struct gf100_gr *gr, u32 inst) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 7f467f6d0462c..66aefc972a183 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -129,6 +129,7 @@ struct gf100_gr { }; int gf100_gr_fecs_bind_pointer(struct gf100_gr *, u32 inst); +int gf100_gr_fecs_wfi_golden_save(struct gf100_gr *, u32 inst); struct gf100_gr_func_zbc { void (*clear_color)(struct gf100_gr *, int zbc); -- GitLab From f1f4d9181484a80928aaf5dfed96897ee3257e13 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:08 +1000 Subject: [PATCH 1361/1927] drm/nouveau/gr/gf100-: call FECS HALT_PIPE method before RC reset Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c | 12 +++++++ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 33 +++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h | 1 + 3 files changed, 46 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c index 61759f54406e4..71b824e6da9d6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/base.c @@ -135,6 +135,17 @@ nvkm_gr_oneinit(struct nvkm_engine *engine) return 0; } +static int +nvkm_gr_reset(struct nvkm_engine *engine) +{ + struct nvkm_gr *gr = nvkm_gr(engine); + + if (gr->func->reset) + return gr->func->reset(gr); + + return -ENOSYS; +} + static int nvkm_gr_init(struct nvkm_engine *engine) { @@ -166,6 +177,7 @@ nvkm_gr = { .oneinit = nvkm_gr_oneinit, .init = nvkm_gr_init, .fini = nvkm_gr_fini, + .reset = nvkm_gr_reset, .intr = nvkm_gr_intr, .tile = nvkm_gr_tile, .chsw_load = nvkm_gr_chsw_load, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 22f360df1b3a2..ffdb5c38afee4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -796,6 +796,20 @@ gf100_gr_fecs_stop_ctxsw(struct nvkm_gr *base) return ret; } +static int +gf100_gr_fecs_halt_pipeline(struct gf100_gr *gr) +{ + int ret = 0; + + if (gr->firmware) { + mutex_lock(&gr->fecs.mutex); + ret = gf100_gr_fecs_ctrl_ctxsw(gr, 0x04); + mutex_unlock(&gr->fecs.mutex); + } + + return ret; +} + int gf100_gr_fecs_wfi_golden_save(struct gf100_gr *gr, u32 inst) { @@ -2247,6 +2261,24 @@ gf100_gr_init_vsc_stream_master(struct gf100_gr *gr) nvkm_mask(device, TPC_UNIT(0, 0, 0x05c), 0x00000001, 0x00000001); } +static int +gf100_gr_reset(struct nvkm_gr *base) +{ + struct nvkm_subdev *subdev = &base->engine.subdev; + struct nvkm_device *device = subdev->device; + struct gf100_gr *gr = gf100_gr(base); + + nvkm_mask(device, 0x400500, 0x00000001, 0x00000000); + + WARN_ON(gf100_gr_fecs_halt_pipeline(gr)); + + subdev->func->fini(subdev, false); + nvkm_mc_disable(device, subdev->type, subdev->inst); + + nvkm_mc_enable(device, subdev->type, subdev->inst); + return subdev->func->init(subdev); +} + int gf100_gr_init(struct gf100_gr *gr) { @@ -2392,6 +2424,7 @@ gf100_gr_ = { .oneinit = gf100_gr_oneinit, .init = gf100_gr_init_, .fini = gf100_gr_fini, + .reset = gf100_gr_reset, .intr = gf100_gr_intr, .units = gf100_gr_units, .chan_new = gf100_gr_chan_new, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h index 9b2c66e8be909..08d5c96e64583 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/priv.h @@ -17,6 +17,7 @@ struct nvkm_gr_func { int (*oneinit)(struct nvkm_gr *); int (*init)(struct nvkm_gr *); int (*fini)(struct nvkm_gr *, bool); + int (*reset)(struct nvkm_gr *); void (*intr)(struct nvkm_gr *); void (*tile)(struct nvkm_gr *, int region, struct nvkm_fb_tile *); int (*tlb_flush)(struct nvkm_gr *); -- GitLab From 366e01092a050579b1f28129bdb72bc7c65ba47a Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:09 +1000 Subject: [PATCH 1362/1927] drm/nouveau/gr/gf100-: wait for FE_PWR_MODE_AUTO This doesn't fix any known issue, but RM started doing it at some point, so presumably it's needed for something. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index eef9b3f065fd9..9795896763610 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1447,6 +1447,10 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk /* NV_PGRAPH_FE_PWR_MODE_AUTO. */ nvkm_wr32(device, 0x404170, 0x00000010); + nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, 0x404170) & 0x00000010)) + break; + ); /* Init SCC RAM. */ nvkm_wr32(device, 0x40802c, 0x00000001); -- GitLab From 0d7557072414af191cefbaa7c908e1c09f5b7d7b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:10 +1000 Subject: [PATCH 1363/1927] drm/nouveau/gr/gf100-: gpfifo_ctl zero before init Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index ffdb5c38afee4..17e1c0dfda478 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2285,6 +2285,8 @@ gf100_gr_init(struct gf100_gr *gr) struct nvkm_device *device = gr->base.engine.subdev.device; int gpc, tpc, rop; + nvkm_mask(device, 0x400500, 0x00010001, 0x00000000); + if (gr->func->init_419bd8) gr->func->init_419bd8(gr); -- GitLab From 40bbee55c114750d2995028603902d7d2ed35ebd Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:10 +1000 Subject: [PATCH 1364/1927] drm/nouveau/gr/gf100-: wfi after register-bashing golden init Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 9795896763610..edd4b867a1e22 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1410,6 +1410,8 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan) grctx->r408840(gr); if (grctx->r419c0c) grctx->r419c0c(gr); + + gf100_gr_wait_idle(gr); } #define CB_RESERVED 0x80000 -- GitLab From be99d041b6b5be6be004366de8c4fc6b9832f195 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:11 +1000 Subject: [PATCH 1365/1927] drm/nouveau/gr/gf100-: move reset during golden ctx init to fecs_reset() Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 6 +----- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 13 +++++++++++++ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 1 + 23 files changed, 36 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index edd4b867a1e22..a56f4f3f6c1e9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1438,11 +1438,7 @@ gf100_grctx_generate(struct gf100_gr *gr, struct gf100_gr_chan *chan, struct nvk grctx->unkn88c(gr, true); /* Reset FECS. */ - nvkm_wr32(device, 0x409614, 0x00000070); - nvkm_usec(device, 10, NVKM_DELAY); - nvkm_mask(device, 0x409614, 0x00000700, 0x00000700); - nvkm_usec(device, 10, NVKM_DELAY); - nvkm_rd32(device, 0x409614); + gr->func->fecs.reset(gr); if (grctx->unkn88c) grctx->unkn88c(gr, false); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 17e1c0dfda478..8f935b9064b38 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2400,6 +2400,18 @@ gf100_gr_init(struct gf100_gr *gr) return gf100_gr_init_ctxctl(gr); } +void +gf100_gr_fecs_reset(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409614, 0x00000070); + nvkm_usec(device, 10, NVKM_DELAY); + nvkm_mask(device, 0x409614, 0x00000700, 0x00000700); + nvkm_usec(device, 10, NVKM_DELAY); + nvkm_rd32(device, 0x409614); +} + #include "fuc/hubgf100.fuc3.h" struct gf100_gr_ucode @@ -2456,6 +2468,7 @@ gf100_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gf100_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gf100_gr_gpccs_ucode, .rops = gf100_gr_rops, .grctx = &gf100_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 66aefc972a183..f98c739fb0b96 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -172,6 +172,7 @@ struct gf100_gr_func { const struct gf100_gr_pack *mmio; struct { struct gf100_gr_ucode *ucode; + void (*reset)(struct gf100_gr *); } fecs; struct { struct gf100_gr_ucode *ucode; @@ -202,6 +203,7 @@ void gf100_gr_init_shader_exceptions(struct gf100_gr *, int, int); void gf100_gr_init_400054(struct gf100_gr *); void gf100_gr_init_num_tpc_per_gpc(struct gf100_gr *, bool, bool); extern const struct gf100_gr_func_zbc gf100_gr_zbc; +void gf100_gr_fecs_reset(struct gf100_gr *); void gf117_gr_init_zcull(struct gf100_gr *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c index 3acd99c306f20..bb82c0a25ef87 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c @@ -131,6 +131,7 @@ gf104_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gf104_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gf100_gr_gpccs_ucode, .rops = gf100_gr_rops, .grctx = &gf104_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c index ab3760e804b8a..e62520653e40f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c @@ -129,6 +129,7 @@ gf108_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gf108_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gf100_gr_gpccs_ucode, .rops = gf100_gr_rops, .grctx = &gf108_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c index 616e2def1865f..d489602be1d34 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c @@ -103,6 +103,7 @@ gf110_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gf110_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gf100_gr_gpccs_ucode, .rops = gf100_gr_rops, .grctx = &gf110_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c index 669e7536970e6..e3e69ab34fa3c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c @@ -167,6 +167,7 @@ gf117_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gf117_gr_pack_mmio, .fecs.ucode = &gf117_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gf117_gr_gpccs_ucode, .rops = gf100_gr_rops, .ppc_nr = 1, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c index 5b09bda8110c3..4dccf0bf25b7c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c @@ -194,6 +194,7 @@ gf119_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gf119_gr_pack_mmio, .fecs.ucode = &gf100_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gf100_gr_gpccs_ucode, .rops = gf100_gr_rops, .grctx = &gf119_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c index b680eaa0f3502..cff24a01ddc01 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c @@ -474,6 +474,7 @@ gk104_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gk104_gr_pack_mmio, .fecs.ucode = &gk104_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gk104_gr_gpccs_ucode, .rops = gf100_gr_rops, .ppc_nr = 1, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c index 103e06a77e658..ae430b351d4f7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c @@ -370,6 +370,7 @@ gk110_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gk110_gr_pack_mmio, .fecs.ucode = &gk110_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gk110_gr_gpccs_ucode, .rops = gf100_gr_rops, .ppc_nr = 2, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c index 034d0b11a17d4..c22eadaee1619 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c @@ -122,6 +122,7 @@ gk110b_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gk110b_gr_pack_mmio, .fecs.ucode = &gk110_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gk110_gr_gpccs_ucode, .rops = gf100_gr_rops, .ppc_nr = 2, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c index 116d682f9f962..35a70a48ed791 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c @@ -180,6 +180,7 @@ gk208_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gk208_gr_pack_mmio, .fecs.ucode = &gk208_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gk208_gr_gpccs_ucode, .rops = gf100_gr_rops, .ppc_nr = 1, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c index be0b2cefd8e8a..d5c59ff33d225 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c @@ -294,6 +294,7 @@ gk20a_gr = { .init_rop_active_fbps = gk104_gr_init_rop_active_fbps, .trap_mp = gf100_gr_trap_mp, .set_hww_esr_report_mask = gk20a_gr_set_hww_esr_report_mask, + .fecs.reset = gf100_gr_fecs_reset, .rops = gf100_gr_rops, .ppc_nr = 1, .grctx = &gk20a_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c index 310987174cb56..17c36f30c86c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c @@ -415,6 +415,7 @@ gm107_gr = { .trap_mp = gf100_gr_trap_mp, .mmio = gm107_gr_pack_mmio, .fecs.ucode = &gm107_gr_fecs_ucode, + .fecs.reset = gf100_gr_fecs_reset, .gpccs.ucode = &gm107_gr_gpccs_ucode, .rops = gf100_gr_rops, .ppc_nr = 2, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c index 385cfd91b2669..31e753f0aaf4e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c @@ -201,6 +201,7 @@ gm200_gr = { .init_shader_exceptions = gm107_gr_init_shader_exceptions, .init_400054 = gm107_gr_init_400054, .trap_mp = gf100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .tpc_nr = 4, .ppc_nr = 2, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c index ec1c46e47e006..458cd1a00d3f8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm20b.c @@ -123,6 +123,7 @@ gm20b_gr = { .init_rop_active_fbps = gk104_gr_init_rop_active_fbps, .trap_mp = gf100_gr_trap_mp, .set_hww_esr_report_mask = gm20b_gr_set_hww_esr_report_mask, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .ppc_nr = 1, .grctx = &gm20b_grctx, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c index 0550dd6f46f13..7a6657bcc7c35 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c @@ -120,6 +120,7 @@ gp100_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .trap_mp = gf100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, .tpc_nr = 5, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c index 1d1f3c0a90145..ebde815ac9d9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c @@ -116,6 +116,7 @@ gp102_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .trap_mp = gf100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, .tpc_nr = 5, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c index 2655574ec63b0..3d824bd7180c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c @@ -44,6 +44,7 @@ gp104_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .trap_mp = gf100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, .tpc_nr = 5, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c index adabc04d4f3a9..d4f87d9237871 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c @@ -46,6 +46,7 @@ gp107_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .trap_mp = gf100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 2, .tpc_nr = 3, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c index e13683b6e7b10..a20db4d93b28f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c @@ -56,6 +56,7 @@ gp10b_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .trap_mp = gf100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 1, .tpc_nr = 2, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index 4d043c1173eaf..c57bb3970eb6d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -105,6 +105,7 @@ gv100_gr = { .init_shader_exceptions = gv100_gr_init_shader_exceptions, .init_4188a4 = gv100_gr_init_4188a4, .trap_mp = gv100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, .tpc_nr = 5, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 1a8a21844e12e..88f768ca8e47e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -110,6 +110,7 @@ tu102_gr = { .init_504430 = gv100_gr_init_504430, .init_shader_exceptions = gv100_gr_init_shader_exceptions, .trap_mp = gv100_gr_trap_mp, + .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, .tpc_nr = 5, -- GitLab From b6d93fa74adac565129e6f6df6722acf87a11d49 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:12 +1000 Subject: [PATCH 1366/1927] drm/nouveau/gr/gf100-: move some init to init_rop_exceptions() Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 24 +++++++++++++------ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 2 ++ .../gpu/drm/nouveau/nvkm/engine/gr/gf104.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf108.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf117.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf119.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk110.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk110b.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gk208.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gm107.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gm200.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gp100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gp102.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gp104.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gp107.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gp10b.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 1 + 20 files changed, 37 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 8f935b9064b38..5dcebda6ddc00 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2154,6 +2154,20 @@ gf100_gr_init_400054(struct gf100_gr *gr) nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x34ce3464); } +void +gf100_gr_init_rop_exceptions(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + int rop; + + for (rop = 0; rop < gr->rop_nr; rop++) { + nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000); + nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000); + nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff); + nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff); + } +} + void gf100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc) { @@ -2283,7 +2297,7 @@ int gf100_gr_init(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; - int gpc, tpc, rop; + int gpc, tpc; nvkm_mask(device, 0x400500, 0x00010001, 0x00000000); @@ -2375,12 +2389,7 @@ gf100_gr_init(struct gf100_gr *gr) nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff); } - for (rop = 0; rop < gr->rop_nr; rop++) { - nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000); - nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000); - nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff); - nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff); - } + gr->func->init_rop_exceptions(gr); nvkm_wr32(device, 0x400108, 0xffffffff); nvkm_wr32(device, 0x400138, 0xffffffff); @@ -2464,6 +2473,7 @@ gf100_gr = { .init_419eb4 = gf100_gr_init_419eb4, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf100_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index f98c739fb0b96..6f1f4fc1fd885 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -165,6 +165,7 @@ struct gf100_gr_func { void (*init_tex_hww_esr)(struct gf100_gr *, int gpc, int tpc); void (*init_504430)(struct gf100_gr *, int gpc, int tpc); void (*init_shader_exceptions)(struct gf100_gr *, int gpc, int tpc); + void (*init_rop_exceptions)(struct gf100_gr *); void (*init_400054)(struct gf100_gr *); void (*init_4188a4)(struct gf100_gr *); void (*trap_mp)(struct gf100_gr *, int gpc, int tpc); @@ -200,6 +201,7 @@ void gf100_gr_init_419cc0(struct gf100_gr *); void gf100_gr_init_419eb4(struct gf100_gr *); void gf100_gr_init_tex_hww_esr(struct gf100_gr *, int, int); void gf100_gr_init_shader_exceptions(struct gf100_gr *, int, int); +void gf100_gr_init_rop_exceptions(struct gf100_gr *); void gf100_gr_init_400054(struct gf100_gr *); void gf100_gr_init_num_tpc_per_gpc(struct gf100_gr *, bool, bool); extern const struct gf100_gr_func_zbc gf100_gr_zbc; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c index bb82c0a25ef87..a1a338c1626a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c @@ -127,6 +127,7 @@ gf104_gr = { .init_419eb4 = gf100_gr_init_419eb4, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf104_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c index e62520653e40f..5687835f5fec9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c @@ -125,6 +125,7 @@ gf108_gr = { .init_419eb4 = gf100_gr_init_419eb4, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf108_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c index d489602be1d34..f6a77094fb5a2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c @@ -99,6 +99,7 @@ gf110_gr = { .init_419eb4 = gf100_gr_init_419eb4, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf110_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c index e3e69ab34fa3c..065d1831f1b14 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c @@ -163,6 +163,7 @@ gf117_gr = { .init_419eb4 = gf100_gr_init_419eb4, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf117_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c index 4dccf0bf25b7c..e4add185d2663 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c @@ -190,6 +190,7 @@ gf119_gr = { .init_419eb4 = gf100_gr_init_419eb4, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf119_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c index cff24a01ddc01..85db57b50c2c6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c @@ -470,6 +470,7 @@ gk104_gr = { .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk104_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c index ae430b351d4f7..157de0003c183 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c @@ -366,6 +366,7 @@ gk110_gr = { .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk110_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c index c22eadaee1619..7f590d36b6f8e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c @@ -118,6 +118,7 @@ gk110b_gr = { .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk110b_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c index 35a70a48ed791..68c74d425ddfd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c @@ -176,6 +176,7 @@ gk208_gr = { .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk208_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c index 17c36f30c86c4..8ac7131877fa1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c @@ -411,6 +411,7 @@ gm107_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gm107_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gm107_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gm107_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c index 31e753f0aaf4e..0145d0cc32eee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c @@ -199,6 +199,7 @@ gm200_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gm107_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_400054 = gm107_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c index 7a6657bcc7c35..47cb5a20fec66 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c @@ -119,6 +119,7 @@ gp100_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c index ebde815ac9d9e..e8a5dc8d14a62 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c @@ -115,6 +115,7 @@ gp102_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c index 3d824bd7180c4..4c9549e5c9017 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c @@ -43,6 +43,7 @@ gp104_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c index d4f87d9237871..e624162453a2f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c @@ -45,6 +45,7 @@ gp107_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c index a20db4d93b28f..10348223e9a73 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c @@ -55,6 +55,7 @@ gp10b_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index c57bb3970eb6d..07507a0ee3182 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -103,6 +103,7 @@ gv100_gr = { .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .init_504430 = gv100_gr_init_504430, .init_shader_exceptions = gv100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_4188a4 = gv100_gr_init_4188a4, .trap_mp = gv100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 88f768ca8e47e..bc5d77ded240c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -109,6 +109,7 @@ tu102_gr = { .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, .init_504430 = gv100_gr_init_504430, .init_shader_exceptions = gv100_gr_init_shader_exceptions, + .init_rop_exceptions = gf100_gr_init_rop_exceptions, .trap_mp = gv100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, -- GitLab From 1a34468855bdd0a8caef2fd6e8e23ecc25407bb2 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:12 +1000 Subject: [PATCH 1367/1927] drm/nouveau/gr/gf100-: move some init to init_exception2() Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 14 ++++++++++++-- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 2 ++ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 1 + 20 files changed, 32 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 5dcebda6ddc00..b9961d254749f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2154,6 +2154,15 @@ gf100_gr_init_400054(struct gf100_gr *gr) nvkm_wr32(gr->base.engine.subdev.device, 0x400054, 0x34ce3464); } +void +gf100_gr_init_exception2(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x40011c, 0xffffffff); + nvkm_wr32(device, 0x400134, 0xffffffff); +} + void gf100_gr_init_rop_exceptions(struct gf100_gr *gr) { @@ -2395,8 +2404,8 @@ gf100_gr_init(struct gf100_gr *gr) nvkm_wr32(device, 0x400138, 0xffffffff); nvkm_wr32(device, 0x400118, 0xffffffff); nvkm_wr32(device, 0x400130, 0xffffffff); - nvkm_wr32(device, 0x40011c, 0xffffffff); - nvkm_wr32(device, 0x400134, 0xffffffff); + if (gr->func->init_exception2) + gr->func->init_exception2(gr); if (gr->func->init_400054) gr->func->init_400054(gr); @@ -2474,6 +2483,7 @@ gf100_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf100_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 6f1f4fc1fd885..cb1d7b2c05587 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -166,6 +166,7 @@ struct gf100_gr_func { void (*init_504430)(struct gf100_gr *, int gpc, int tpc); void (*init_shader_exceptions)(struct gf100_gr *, int gpc, int tpc); void (*init_rop_exceptions)(struct gf100_gr *); + void (*init_exception2)(struct gf100_gr *); void (*init_400054)(struct gf100_gr *); void (*init_4188a4)(struct gf100_gr *); void (*trap_mp)(struct gf100_gr *, int gpc, int tpc); @@ -202,6 +203,7 @@ void gf100_gr_init_419eb4(struct gf100_gr *); void gf100_gr_init_tex_hww_esr(struct gf100_gr *, int, int); void gf100_gr_init_shader_exceptions(struct gf100_gr *, int, int); void gf100_gr_init_rop_exceptions(struct gf100_gr *); +void gf100_gr_init_exception2(struct gf100_gr *); void gf100_gr_init_400054(struct gf100_gr *); void gf100_gr_init_num_tpc_per_gpc(struct gf100_gr *, bool, bool); extern const struct gf100_gr_func_zbc gf100_gr_zbc; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c index a1a338c1626a8..63bd29c22fe15 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf104.c @@ -128,6 +128,7 @@ gf104_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf104_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c index 5687835f5fec9..495a844f925f1 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf108.c @@ -126,6 +126,7 @@ gf108_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf108_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c index f6a77094fb5a2..70fad235d161f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf110.c @@ -100,6 +100,7 @@ gf110_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf110_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c index 065d1831f1b14..68fd4384b5e5d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c @@ -164,6 +164,7 @@ gf117_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf117_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c index e4add185d2663..75ceb514c06ed 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf119.c @@ -191,6 +191,7 @@ gf119_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gf119_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c index 85db57b50c2c6..3f7b4ef3529ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c @@ -471,6 +471,7 @@ gk104_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk104_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c index 157de0003c183..c7e1c5dbc6a9e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110.c @@ -367,6 +367,7 @@ gk110_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk110_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c index 7f590d36b6f8e..458abae571bf7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk110b.c @@ -119,6 +119,7 @@ gk110b_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk110b_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c index 68c74d425ddfd..d3f6b65c21d22 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk208.c @@ -177,6 +177,7 @@ gk208_gr = { .init_tex_hww_esr = gf100_gr_init_tex_hww_esr, .init_shader_exceptions = gf100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gf100_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gk208_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c index 8ac7131877fa1..797b828a943bd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm107.c @@ -412,6 +412,7 @@ gm107_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gm107_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gm107_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .mmio = gm107_gr_pack_mmio, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c index 0145d0cc32eee..b9d74d65038d4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c @@ -200,6 +200,7 @@ gm200_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gm107_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_400054 = gm107_gr_init_400054, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c index 47cb5a20fec66..576cabd80a4a8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c @@ -120,6 +120,7 @@ gp100_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c index e8a5dc8d14a62..f265ba08c5ef4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c @@ -116,6 +116,7 @@ gp102_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c index 4c9549e5c9017..6802cb9b199f0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp104.c @@ -44,6 +44,7 @@ gp104_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c index e624162453a2f..cc2bb0d0a9870 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp107.c @@ -46,6 +46,7 @@ gp107_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c index 10348223e9a73..5008881ca079f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp10b.c @@ -56,6 +56,7 @@ gp10b_gr = { .init_504430 = gm107_gr_init_504430, .init_shader_exceptions = gp100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .trap_mp = gf100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index 07507a0ee3182..25228d09a30b0 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -104,6 +104,7 @@ gv100_gr = { .init_504430 = gv100_gr_init_504430, .init_shader_exceptions = gv100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .init_4188a4 = gv100_gr_init_4188a4, .trap_mp = gv100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index bc5d77ded240c..ce968be6f290c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -110,6 +110,7 @@ tu102_gr = { .init_504430 = gv100_gr_init_504430, .init_shader_exceptions = gv100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, + .init_exception2 = gf100_gr_init_exception2, .trap_mp = gv100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, -- GitLab From 9aa3faced0a3dea04294afe781e07e1ef1118e30 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:13 +1000 Subject: [PATCH 1368/1927] drm/nouveau/gr/gf100-: switch to newer style interrupt handler Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 27 ++++++++++++++----- .../gpu/drm/nouveau/nvkm/subdev/mc/gf100.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/mc/gk104.c | 3 ++- .../gpu/drm/nouveau/nvkm/subdev/mc/gp100.c | 4 +-- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index b9961d254749f..6e31ba69ed4a3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1600,10 +1600,10 @@ gf100_gr_ctxctl_isr(struct gf100_gr *gr) } } -static void -gf100_gr_intr(struct nvkm_gr *base) +static irqreturn_t +gf100_gr_intr(struct nvkm_inth *inth) { - struct gf100_gr *gr = gf100_gr(base); + struct gf100_gr *gr = container_of(inth, typeof(*gr), base.engine.subdev.inth); struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; struct nvkm_chan *chan; @@ -1690,6 +1690,7 @@ gf100_gr_intr(struct nvkm_gr *base) nvkm_wr32(device, 0x400500, 0x00010001); nvkm_chan_put(&chan, flags); + return IRQ_HANDLED; } static void @@ -1985,8 +1986,15 @@ gf100_gr_oneinit(struct nvkm_gr *base) struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; struct nvkm_device *device = subdev->device; + struct nvkm_intr *intr = &device->mc->intr; + enum nvkm_intr_type intr_type = NVKM_INTR_SUBDEV; int ret, i, j; + ret = nvkm_inth_add(intr, intr_type, NVKM_INTR_PRIO_NORMAL, &gr->base.engine.subdev, + gf100_gr_intr, &gr->base.engine.subdev.inth); + if (ret) + return ret; + nvkm_pmu_pgob(device->pmu, false); gr->rop_nr = gr->func->rops(gr); @@ -2047,7 +2055,7 @@ gf100_gr_init_(struct nvkm_gr *base) struct nvkm_subdev *subdev = &base->engine.subdev; struct nvkm_device *device = subdev->device; bool reset = device->chipset == 0x137 || device->chipset == 0x138; - u32 ret; + int ret; /* On certain GP107/GP108 boards, we trigger a weird issue where * GR will stop responding to PRI accesses after we've asked the @@ -2083,7 +2091,12 @@ gf100_gr_init_(struct nvkm_gr *base) if (ret) return ret; - return gr->func->init(gr); + ret = gr->func->init(gr); + if (ret) + return ret; + + nvkm_inth_allow(&subdev->inth); + return 0; } static int @@ -2091,6 +2104,9 @@ gf100_gr_fini(struct nvkm_gr *base, bool suspend) { struct gf100_gr *gr = gf100_gr(base); struct nvkm_subdev *subdev = &gr->base.engine.subdev; + + nvkm_inth_block(&subdev->inth); + nvkm_falcon_put(&gr->gpccs.falcon, subdev); nvkm_falcon_put(&gr->fecs.falcon, subdev); return 0; @@ -2457,7 +2473,6 @@ gf100_gr_ = { .init = gf100_gr_init_, .fini = gf100_gr_fini, .reset = gf100_gr_reset, - .intr = gf100_gr_intr, .units = gf100_gr_units, .chan_new = gf100_gr_chan_new, .object_get = gf100_gr_object_get, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c index 5118e413d053b..ab1eaa37123a9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c @@ -41,7 +41,7 @@ gf100_mc_intrs[] = { { NVKM_ENGINE_DISP , 0, 0, 0x04000000, true }, { NVKM_ENGINE_MSPDEC , 0, 0, 0x00020000, true }, { NVKM_ENGINE_MSVLD , 0, 0, 0x00008000, true }, - { NVKM_ENGINE_GR , 0, 0, 0x00001000, true }, + { NVKM_ENGINE_GR , 0, 0, 0x00001000 }, { NVKM_ENGINE_FIFO , 0, 0, 0x00000100 }, { NVKM_ENGINE_CE , 1, 0, 0x00000040, true }, { NVKM_ENGINE_CE , 0, 0, 0x00000020, true }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c index 9d482f3581386..66829586a124a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c @@ -43,7 +43,8 @@ gk104_mc_intrs[] = { { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, { NVKM_SUBDEV_THERM , 0, 0, 0x00040000, true }, - { NVKM_SUBDEV_TOP , 0, 0, 0xffffffff, true }, + { NVKM_SUBDEV_TOP , 0, 0, 0x00001000 }, + { NVKM_SUBDEV_TOP , 0, 0, 0xffffefff, true }, {}, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c index 3a99a450b6b06..eb2ab03f43607 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c @@ -37,8 +37,8 @@ gp100_mc_intrs[] = { { NVKM_SUBDEV_I2C , 0, 0, 0x00200000, true }, { NVKM_SUBDEV_TIMER , 0, 0, 0x00100000, true }, { NVKM_SUBDEV_THERM , 0, 0, 0x00040000, true }, - { NVKM_SUBDEV_TOP , 0, 0, 0x00008000 }, - { NVKM_SUBDEV_TOP , 0, 0, 0xffff7fff, true }, + { NVKM_SUBDEV_TOP , 0, 0, 0x00009000 }, + { NVKM_SUBDEV_TOP , 0, 0, 0xffff6fff, true }, {}, }; -- GitLab From 0c520ad45a65313c310fe96e17bda51e2c019ec4 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:14 +1000 Subject: [PATCH 1369/1927] drm/nouveau/gr/gf117-: make ppc_nr[gpc] accurate We're going to be pulling in a chunk of code from NVGPU to fixup our SMID mappings on Volta and above, which depends on ppc_nr[gpc] reflecting the actual number of PPCs present, not the maximum number. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 6 ++++-- drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c | 2 +- 8 files changed, 11 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c index 6c1f631eebd5e..e34c5da2a9ff6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c @@ -257,7 +257,7 @@ gf117_grctx_generate_attrib(struct gf100_gr_chan *chan) gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++) { const u32 a = alpha * gr->ppc_tpc_nr[gpc][ppc]; const u32 b = beta * gr->ppc_tpc_nr[gpc][ppc]; const u32 t = timeslice_mode; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c index 019c698eb5a55..beac66eb2a803 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm107.c @@ -912,7 +912,7 @@ gm107_grctx_generate_attrib(struct gf100_gr_chan *chan) gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++, n++) { const u32 as = alpha * gr->ppc_tpc_nr[gpc][ppc]; const u32 bs = attrib * gr->ppc_tpc_nr[gpc][ppc]; const u32 u = 0x418ea0 + (n * 0x04); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c index 6b7034b2ff766..175da8ac656ce 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgm200.c @@ -87,7 +87,7 @@ gm200_grctx_generate_dist_skip_table(struct gf100_gr *gr) int gpc, ppc, i; for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++) { u8 ppc_tpcs = gr->ppc_tpc_nr[gpc][ppc]; u8 ppc_tpcm = gr->ppc_tpc_mask[gpc][ppc]; while (ppc_tpcs-- > gr->ppc_tpc_min) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c index d8977caa0d040..8485aaeae7a92 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c @@ -56,7 +56,7 @@ gp100_grctx_generate_attrib(struct gf100_gr_chan *chan) gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++, n++) { const u32 as = alpha * gr->ppc_tpc_nr[gpc][ppc]; const u32 bs = attrib * gr->ppc_tpc_max; const u32 u = 0x418ea0 + (n * 0x04); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c index 7d372d786b8bd..7537979a54927 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp102.c @@ -55,7 +55,7 @@ gp102_grctx_generate_attrib(struct gf100_gr_chan *chan) gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++, n++) { const u32 as = alpha * gr->ppc_tpc_nr[gpc][ppc]; const u32 bs = attrib * gr->ppc_tpc_max; const u32 gs = gfxp * gr->ppc_tpc_max; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c index c2e5d43b1c37b..9975a4d5c058b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c @@ -75,7 +75,7 @@ gv100_grctx_generate_attrib(struct gf100_gr_chan *chan) gf100_grctx_patch_wr32(chan, 0x40585c, alpha); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++, n++) { const u32 as = alpha * gr->ppc_tpc_nr[gpc][ppc]; const u32 bs = attrib * gr->ppc_tpc_max; const u32 gs = gfxp * gr->ppc_tpc_max; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 6e31ba69ed4a3..ebbd7476e0fee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2003,12 +2003,14 @@ gf100_gr_oneinit(struct nvkm_gr *base) gr->tpc_nr[i] = nvkm_rd32(device, GPC_UNIT(i, 0x2608)); gr->tpc_max = max(gr->tpc_max, gr->tpc_nr[i]); gr->tpc_total += gr->tpc_nr[i]; - gr->ppc_nr[i] = gr->func->ppc_nr; - for (j = 0; j < gr->ppc_nr[i]; j++) { + for (j = 0; j < gr->func->ppc_nr; j++) { gr->ppc_tpc_mask[i][j] = nvkm_rd32(device, GPC_UNIT(i, 0x0c30 + (j * 4))); if (gr->ppc_tpc_mask[i][j] == 0) continue; + + gr->ppc_nr[i]++; + gr->ppc_mask[i] |= (1 << j); gr->ppc_tpc_nr[i][j] = hweight8(gr->ppc_tpc_mask[i][j]); if (gr->ppc_tpc_min == 0 || diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c index 3f7b4ef3529ef..e53ade24ad23b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk104.c @@ -418,7 +418,7 @@ gk104_gr_init_ppc_exceptions(struct gf100_gr *gr) int gpc, ppc; for (gpc = 0; gpc < gr->gpc_nr; gpc++) { - for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++) { + for (ppc = 0; ppc < gr->func->ppc_nr; ppc++) { if (!(gr->ppc_mask[gpc] & (1 << ppc))) continue; nvkm_wr32(device, PPC_UNIT(gpc, ppc, 0x038), 0xc0000000); -- GitLab From e2eeec758c367963c74d56eb89a10920e418bfdf Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:14 +1000 Subject: [PATCH 1370/1927] drm/nouveau/gr/gp100-: fix number of zcull tile regs Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c | 4 +++- drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c index 68fd4384b5e5d..f127282480485 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf117.c @@ -125,7 +125,9 @@ gf117_gr_init_zcull(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total); - const u8 tile_nr = ALIGN(gr->tpc_total, 32); + /*TODO: fill in litter vals for gf117-gm2xx */ + const u8 tile_nr = !gr->func->gpc_nr ? ALIGN(gr->tpc_total, 32) : + (gr->func->gpc_nr * gr->func->tpc_nr); u8 bank[GPC_MAX] = {}, gpc, i, j; u32 data; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index ce968be6f290c..10b6e78bcf8ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -53,7 +53,7 @@ tu102_gr_init_zcull(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total); - const u8 tile_nr = ALIGN(gr->tpc_total, 64); + const u8 tile_nr = gr->func->gpc_nr * gr->func->tpc_nr; u8 bank[GPC_MAX] = {}, gpc, i, j; u32 data; @@ -115,7 +115,7 @@ tu102_gr = { .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, - .tpc_nr = 5, + .tpc_nr = 6, .ppc_nr = 3, .grctx = &tu102_grctx, .zbc = &gp102_gr_zbc, -- GitLab From de8be616b6ea83f468da3265aff58d25f57494b5 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:15 +1000 Subject: [PATCH 1371/1927] drm/nouveau/gr/gk20a,gm20b,gp10b: split out netlist parsing from fw loading We'll want to reuse the former for loading from proper netlist images. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 6 + .../gpu/drm/nouveau/nvkm/engine/gr/gk20a.c | 110 +++++++----------- 2 files changed, 49 insertions(+), 67 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index cb1d7b2c05587..9c6823672d2ef 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -223,6 +223,9 @@ void gm107_gr_init_shader_exceptions(struct gf100_gr *, int, int); void gm107_gr_init_400054(struct gf100_gr *); int gk20a_gr_init(struct gf100_gr *); +int gk20a_gr_av_to_init(struct nvkm_blob *, struct gf100_gr_pack **); +int gk20a_gr_aiv_to_init(struct nvkm_blob *, struct gf100_gr_pack **); +int gk20a_gr_av_to_method(struct nvkm_blob *, struct gf100_gr_pack **); void gm200_gr_oneinit_tiles(struct gf100_gr *); void gm200_gr_oneinit_sm_id(struct gf100_gr *); @@ -400,6 +403,9 @@ int gf100_gr_load(struct gf100_gr *, int, const struct gf100_gr_fwif *); int gf100_gr_nofw(struct gf100_gr *, int, const struct gf100_gr_fwif *); int gk20a_gr_load_sw(struct gf100_gr *, const char *path, int ver); +int gk20a_gr_load_net(struct gf100_gr *, const char *, const char *, int, + int (*)(struct nvkm_blob *, struct gf100_gr_pack **), + struct gf100_gr_pack **); int gm200_gr_nofw(struct gf100_gr *, int, const struct gf100_gr_fwif *); int gm200_gr_load(struct gf100_gr *, int, const struct gf100_gr_fwif *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c index d5c59ff33d225..2aee0559efca3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c @@ -33,35 +33,25 @@ struct gk20a_fw_av u32 data; }; -static int -gk20a_gr_av_to_init(struct gf100_gr *gr, const char *path, const char *name, - int ver, struct gf100_gr_pack **ppack) +int +gk20a_gr_av_to_init(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) { - struct nvkm_subdev *subdev = &gr->base.engine.subdev; - struct nvkm_blob blob; struct gf100_gr_init *init; struct gf100_gr_pack *pack; int nent; - int ret; int i; - ret = nvkm_firmware_load_blob(subdev, path, name, ver, &blob); - if (ret) - return ret; - - nent = (blob.size / sizeof(struct gk20a_fw_av)); + nent = (blob->size / sizeof(struct gk20a_fw_av)); pack = vzalloc((sizeof(*pack) * 2) + (sizeof(*init) * (nent + 1))); - if (!pack) { - ret = -ENOMEM; - goto end; - } + if (!pack) + return -ENOMEM; init = (void *)(pack + 2); pack[0].init = init; for (i = 0; i < nent; i++) { struct gf100_gr_init *ent = &init[i]; - struct gk20a_fw_av *av = &((struct gk20a_fw_av *)blob.data)[i]; + struct gk20a_fw_av *av = &((struct gk20a_fw_av *)blob->data)[i]; ent->addr = av->addr; ent->data = av->data; @@ -70,10 +60,7 @@ gk20a_gr_av_to_init(struct gf100_gr *gr, const char *path, const char *name, } *ppack = pack; - -end: - nvkm_blob_dtor(&blob); - return ret; + return 0; } struct gk20a_fw_aiv @@ -83,35 +70,25 @@ struct gk20a_fw_aiv u32 data; }; -static int -gk20a_gr_aiv_to_init(struct gf100_gr *gr, const char *path, const char *name, - int ver, struct gf100_gr_pack **ppack) +int +gk20a_gr_aiv_to_init(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) { - struct nvkm_subdev *subdev = &gr->base.engine.subdev; - struct nvkm_blob blob; struct gf100_gr_init *init; struct gf100_gr_pack *pack; int nent; - int ret; int i; - ret = nvkm_firmware_load_blob(subdev, path, name, ver, &blob); - if (ret) - return ret; - - nent = (blob.size / sizeof(struct gk20a_fw_aiv)); + nent = (blob->size / sizeof(struct gk20a_fw_aiv)); pack = vzalloc((sizeof(*pack) * 2) + (sizeof(*init) * (nent + 1))); - if (!pack) { - ret = -ENOMEM; - goto end; - } + if (!pack) + return -ENOMEM; init = (void *)(pack + 2); pack[0].init = init; for (i = 0; i < nent; i++) { struct gf100_gr_init *ent = &init[i]; - struct gk20a_fw_aiv *av = &((struct gk20a_fw_aiv *)blob.data)[i]; + struct gk20a_fw_aiv *av = &((struct gk20a_fw_aiv *)blob->data)[i]; ent->addr = av->addr; ent->data = av->data; @@ -120,44 +97,30 @@ gk20a_gr_aiv_to_init(struct gf100_gr *gr, const char *path, const char *name, } *ppack = pack; - -end: - nvkm_blob_dtor(&blob); - return ret; + return 0; } -static int -gk20a_gr_av_to_method(struct gf100_gr *gr, const char *path, const char *name, - int ver, struct gf100_gr_pack **ppack) +int +gk20a_gr_av_to_method(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) { - struct nvkm_subdev *subdev = &gr->base.engine.subdev; - struct nvkm_blob blob; struct gf100_gr_init *init; struct gf100_gr_pack *pack; /* We don't suppose we will initialize more than 16 classes here... */ static const unsigned int max_classes = 16; u32 classidx = 0, prevclass = 0; int nent; - int ret; int i; - ret = nvkm_firmware_load_blob(subdev, path, name, ver, &blob); - if (ret) - return ret; - - nent = (blob.size / sizeof(struct gk20a_fw_av)); - + nent = (blob->size / sizeof(struct gk20a_fw_av)); pack = vzalloc((sizeof(*pack) * (max_classes + 1)) + (sizeof(*init) * (nent + max_classes + 1))); - if (!pack) { - ret = -ENOMEM; - goto end; - } + if (!pack) + return -ENOMEM; init = (void *)(pack + max_classes + 1); for (i = 0; i < nent; i++, init++) { - struct gk20a_fw_av *av = &((struct gk20a_fw_av *)blob.data)[i]; + struct gk20a_fw_av *av = &((struct gk20a_fw_av *)blob->data)[i]; u32 class = av->addr & 0xffff; u32 addr = (av->addr & 0xffff0000) >> 14; @@ -169,8 +132,7 @@ gk20a_gr_av_to_method(struct gf100_gr *gr, const char *path, const char *name, prevclass = class; if (++classidx >= max_classes) { vfree(pack); - ret = -ENOSPC; - goto end; + return -ENOSPC; } } @@ -181,10 +143,7 @@ gk20a_gr_av_to_method(struct gf100_gr *gr, const char *path, const char *name, } *ppack = pack; - -end: - nvkm_blob_dtor(&blob); - return ret; + return 0; } static int @@ -308,13 +267,30 @@ gk20a_gr = { } }; +int +gk20a_gr_load_net(struct gf100_gr *gr, const char *path, const char *name, int ver, + int (*load)(struct nvkm_blob *, struct gf100_gr_pack **), + struct gf100_gr_pack **ppack) +{ + struct nvkm_blob blob; + int ret; + + ret = nvkm_firmware_load_blob(&gr->base.engine.subdev, path, name, ver, &blob); + if (ret) + return ret; + + ret = load(&blob, ppack); + nvkm_blob_dtor(&blob); + return 0; +} + int gk20a_gr_load_sw(struct gf100_gr *gr, const char *path, int ver) { - if (gk20a_gr_av_to_init(gr, path, "sw_nonctx", ver, &gr->sw_nonctx) || - gk20a_gr_aiv_to_init(gr, path, "sw_ctx", ver, &gr->sw_ctx) || - gk20a_gr_av_to_init(gr, path, "sw_bundle_init", ver, &gr->bundle) || - gk20a_gr_av_to_method(gr, path, "sw_method_init", ver, &gr->method)) + if (gk20a_gr_load_net(gr, path, "sw_nonctx", ver, gk20a_gr_av_to_init, &gr->sw_nonctx) || + gk20a_gr_load_net(gr, path, "sw_ctx", ver, gk20a_gr_aiv_to_init, &gr->sw_ctx) || + gk20a_gr_load_net(gr, path, "sw_bundle_init", ver, gk20a_gr_av_to_init, &gr->bundle) || + gk20a_gr_load_net(gr, path, "sw_method_init", ver, gk20a_gr_av_to_method, &gr->method)) return -ENOENT; return 0; -- GitLab From d94ac9dddef792be7741cbb558d3be2b796567d7 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:16 +1000 Subject: [PATCH 1372/1927] drm/nouveau/gr/gp100-: modify init_fecs_exceptions Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c | 2 +- drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c index 576cabd80a4a8..851e743d2cab8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c @@ -87,7 +87,7 @@ gp100_gr_init_419c9c(struct gf100_gr *gr) void gp100_gr_init_fecs_exceptions(struct gf100_gr *gr) { - nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, 0x000f0002); + nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, 0x000e0002); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 10b6e78bcf8ee..3491005cf5d7c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -27,7 +27,7 @@ static void tu102_gr_init_fecs_exceptions(struct gf100_gr *gr) { - nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, 0x006f0002); + nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, 0x006e0003); } static void -- GitLab From 3ffa6f329b610029b44ebd7bc2320a92468a0e42 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:17 +1000 Subject: [PATCH 1373/1927] drm/nouveau/gr/gv100-: port smid mapping code from nvgpu Essentially ripped verbatim from NVGPU, comments and all, and adapted to nvkm's structs and style. - maybe fixes an nvgpu bug though, a small tweak was needed to match RM v2: - remove unnecessary WARN_ON Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c | 3 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c | 3 + .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 11 +- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 9 +- .../gpu/drm/nouveau/nvkm/engine/gr/gm200.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 194 +++++++++++++++++- .../gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 7 +- 7 files changed, 219 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c index 9975a4d5c058b..d68741a81a581 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c @@ -157,6 +157,9 @@ static void gv100_grctx_generate_sm_id(struct gf100_gr *gr, int gpc, int tpc, int sm) { struct nvkm_device *device = gr->base.engine.subdev.device; + + tpc = gv100_gr_nonpes_aware_tpc(gr, gpc, tpc); + nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x608), sm); nvkm_wr32(device, GPC_UNIT(gpc, 0x0c10 + tpc * 4), sm); nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), sm); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c index 1a151e89ea05c..da1bac3963a95 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c @@ -34,6 +34,9 @@ static void tu102_grctx_generate_sm_id(struct gf100_gr *gr, int gpc, int tpc, int sm) { struct nvkm_device *device = gr->base.engine.subdev.device; + + tpc = gv100_gr_nonpes_aware_tpc(gr, gpc, tpc); + nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x608), sm); nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x088), sm); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index ebbd7476e0fee..5ce83b915ebb9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1889,10 +1889,11 @@ gf100_gr_init_ctxctl(struct gf100_gr *gr) return ret; } -void +int gf100_gr_oneinit_sm_id(struct gf100_gr *gr) { int tpc, gpc; + for (tpc = 0; tpc < gr->tpc_max; tpc++) { for (gpc = 0; gpc < gr->gpc_nr; gpc++) { if (tpc < gr->tpc_nr[gpc]) { @@ -1902,6 +1903,8 @@ gf100_gr_oneinit_sm_id(struct gf100_gr *gr) } } } + + return 0; } void @@ -2019,6 +2022,8 @@ gf100_gr_oneinit(struct nvkm_gr *base) if (gr->ppc_tpc_max < gr->ppc_tpc_nr[i][j]) gr->ppc_tpc_max = gr->ppc_tpc_nr[i][j]; } + + gr->ppc_total += gr->ppc_nr[i]; } /* Allocate global context buffers. */ @@ -2046,8 +2051,8 @@ gf100_gr_oneinit(struct nvkm_gr *base) memset(gr->tile, 0xff, sizeof(gr->tile)); gr->func->oneinit_tiles(gr); - gr->func->oneinit_sm_id(gr); - return 0; + + return gr->func->oneinit_sm_id(gr); } static int diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 9c6823672d2ef..a9ba9af25728a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -107,6 +107,7 @@ struct gf100_gr { u8 ppc_tpc_nr[GPC_MAX][4]; u8 ppc_tpc_min; u8 ppc_tpc_max; + u8 ppc_total; struct nvkm_memory *pagepool; struct nvkm_memory *bundle_cb; @@ -141,7 +142,7 @@ struct gf100_gr_func_zbc { struct gf100_gr_func { void (*oneinit_tiles)(struct gf100_gr *); - void (*oneinit_sm_id)(struct gf100_gr *); + int (*oneinit_sm_id)(struct gf100_gr *); int (*init)(struct gf100_gr *); void (*init_419bd8)(struct gf100_gr *); void (*init_gpc_mmu)(struct gf100_gr *); @@ -191,7 +192,7 @@ struct gf100_gr_func { int gf100_gr_rops(struct gf100_gr *); void gf100_gr_oneinit_tiles(struct gf100_gr *); -void gf100_gr_oneinit_sm_id(struct gf100_gr *); +int gf100_gr_oneinit_sm_id(struct gf100_gr *); int gf100_gr_init(struct gf100_gr *); void gf100_gr_init_vsc_stream_master(struct gf100_gr *); void gf100_gr_init_zcull(struct gf100_gr *); @@ -228,7 +229,7 @@ int gk20a_gr_aiv_to_init(struct nvkm_blob *, struct gf100_gr_pack **); int gk20a_gr_av_to_method(struct nvkm_blob *, struct gf100_gr_pack **); void gm200_gr_oneinit_tiles(struct gf100_gr *); -void gm200_gr_oneinit_sm_id(struct gf100_gr *); +int gm200_gr_oneinit_sm_id(struct gf100_gr *); int gm200_gr_rops(struct gf100_gr *); void gm200_gr_init_num_active_ltcs(struct gf100_gr *); void gm200_gr_init_ds_hww_esr_2(struct gf100_gr *); @@ -245,6 +246,8 @@ extern const struct gf100_gr_func_zbc gp102_gr_zbc; extern const struct gf100_gr_func gp107_gr; +int gv100_gr_oneinit_sm_id(struct gf100_gr *); +u32 gv100_gr_nonpes_aware_tpc(struct gf100_gr *gr, u32 gpc, u32 tpc); void gv100_gr_init_419bd8(struct gf100_gr *); void gv100_gr_init_504430(struct gf100_gr *, int, int); void gv100_gr_init_shader_exceptions(struct gf100_gr *, int, int); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c index b9d74d65038d4..b5210b31c1b2c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c @@ -148,11 +148,11 @@ gm200_gr_tile_map_2_8[] = { 0, 1, 1, 0, 0, 1, 1, 0, }; -void +int gm200_gr_oneinit_sm_id(struct gf100_gr *gr) { /*XXX: There's a different algorithm here I've not yet figured out. */ - gf100_gr_oneinit_sm_id(gr); + return gf100_gr_oneinit_sm_id(gr); } void diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index 25228d09a30b0..aeb767e582c8f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -85,10 +85,202 @@ gv100_gr_init_419bd8(struct gf100_gr *gr) nvkm_mask(device, 0x419bd8, 0x00000700, 0x00000000); } +u32 +gv100_gr_nonpes_aware_tpc(struct gf100_gr *gr, u32 gpc, u32 tpc) +{ + u32 pes, temp, tpc_new = 0; + + for (pes = 0; pes < gr->ppc_nr[gpc]; pes++) { + if (gr->ppc_tpc_mask[gpc][pes] & BIT(tpc)) + break; + + tpc_new += gr->ppc_tpc_nr[gpc][pes]; + } + + temp = (BIT(tpc) - 1) & gr->ppc_tpc_mask[gpc][pes]; + temp = hweight32(temp); + return tpc_new + temp; +} + +static int +gv100_gr_scg_estimate_perf(struct gf100_gr *gr, unsigned long *gpc_tpc_mask, + u32 disable_gpc, u32 disable_tpc, int *perf) +{ + const u32 scale_factor = 512UL; /* Use fx23.9 */ + const u32 pix_scale = 1024*1024UL; /* Pix perf in [29:20] */ + const u32 world_scale = 1024UL; /* World performance in [19:10] */ + const u32 tpc_scale = 1; /* TPC balancing in [9:0] */ + u32 scg_num_pes = 0; + u32 min_scg_gpc_pix_perf = scale_factor; /* Init perf as maximum */ + u32 average_tpcs = 0; /* Average of # of TPCs per GPC */ + u32 deviation; /* absolute diff between TPC# and average_tpcs, averaged across GPCs */ + u32 norm_tpc_deviation; /* deviation/max_tpc_per_gpc */ + u32 tpc_balance; + u32 scg_gpc_pix_perf; + u32 scg_world_perf; + u32 gpc; + u32 pes; + int diff; + bool tpc_removed_gpc = false; + bool tpc_removed_pes = false; + u32 max_tpc_gpc = 0; + u32 num_tpc_mask; + u32 *num_tpc_gpc; + int ret = -EINVAL; + + if (!(num_tpc_gpc = kcalloc(gr->gpc_nr, sizeof(*num_tpc_gpc), GFP_KERNEL))) + return -ENOMEM; + + /* Calculate pix-perf-reduction-rate per GPC and find bottleneck TPC */ + for (gpc = 0; gpc < gr->gpc_nr; gpc++) { + num_tpc_mask = gpc_tpc_mask[gpc]; + + if ((gpc == disable_gpc) && num_tpc_mask & BIT(disable_tpc)) { + /* Safety check if a TPC is removed twice */ + if (WARN_ON(tpc_removed_gpc)) + goto done; + + /* Remove logical TPC from set */ + num_tpc_mask &= ~BIT(disable_tpc); + tpc_removed_gpc = true; + } + + /* track balancing of tpcs across gpcs */ + num_tpc_gpc[gpc] = hweight32(num_tpc_mask); + average_tpcs += num_tpc_gpc[gpc]; + + /* save the maximum numer of gpcs */ + max_tpc_gpc = num_tpc_gpc[gpc] > max_tpc_gpc ? num_tpc_gpc[gpc] : max_tpc_gpc; + + /* + * Calculate ratio between TPC count and post-FS and post-SCG + * + * ratio represents relative throughput of the GPC + */ + scg_gpc_pix_perf = scale_factor * num_tpc_gpc[gpc] / gr->tpc_nr[gpc]; + if (min_scg_gpc_pix_perf > scg_gpc_pix_perf) + min_scg_gpc_pix_perf = scg_gpc_pix_perf; + + /* Calculate # of surviving PES */ + for (pes = 0; pes < gr->ppc_nr[gpc]; pes++) { + /* Count the number of TPC on the set */ + num_tpc_mask = gr->ppc_tpc_mask[gpc][pes] & gpc_tpc_mask[gpc]; + + if ((gpc == disable_gpc) && (num_tpc_mask & BIT(disable_tpc))) { + if (WARN_ON(tpc_removed_pes)) + goto done; + + num_tpc_mask &= ~BIT(disable_tpc); + tpc_removed_pes = true; + } + + if (hweight32(num_tpc_mask)) + scg_num_pes++; + } + } + + if (WARN_ON(!tpc_removed_gpc || !tpc_removed_pes)) + goto done; + + if (max_tpc_gpc == 0) { + *perf = 0; + goto done_ok; + } + + /* Now calculate perf */ + scg_world_perf = (scale_factor * scg_num_pes) / gr->ppc_total; + deviation = 0; + average_tpcs = scale_factor * average_tpcs / gr->gpc_nr; + for (gpc = 0; gpc < gr->gpc_nr; gpc++) { + diff = average_tpcs - scale_factor * num_tpc_gpc[gpc]; + if (diff < 0) + diff = -diff; + + deviation += diff; + } + + deviation /= gr->gpc_nr; + + norm_tpc_deviation = deviation / max_tpc_gpc; + + tpc_balance = scale_factor - norm_tpc_deviation; + + if ((tpc_balance > scale_factor) || + (scg_world_perf > scale_factor) || + (min_scg_gpc_pix_perf > scale_factor) || + (norm_tpc_deviation > scale_factor)) { + WARN_ON(1); + goto done; + } + + *perf = (pix_scale * min_scg_gpc_pix_perf) + + (world_scale * scg_world_perf) + + (tpc_scale * tpc_balance); +done_ok: + ret = 0; +done: + kfree(num_tpc_gpc); + return ret; +} + +int +gv100_gr_oneinit_sm_id(struct gf100_gr *gr) +{ + unsigned long *gpc_tpc_mask; + u32 *tpc_table, *gpc_table; + u32 gpc, tpc, pes, gtpc; + int perf, maxperf, ret = 0; + + gpc_tpc_mask = kcalloc(gr->gpc_nr, sizeof(*gpc_tpc_mask), GFP_KERNEL); + gpc_table = kcalloc(gr->tpc_total, sizeof(*gpc_table), GFP_KERNEL); + tpc_table = kcalloc(gr->tpc_total, sizeof(*tpc_table), GFP_KERNEL); + if (!gpc_table || !tpc_table || !gpc_tpc_mask) { + ret = -ENOMEM; + goto done; + } + + for (gpc = 0; gpc < gr->gpc_nr; gpc++) { + for (pes = 0; pes < gr->ppc_nr[gpc]; pes++) + gpc_tpc_mask[gpc] |= gr->ppc_tpc_mask[gpc][pes]; + } + + for (gtpc = 0; gtpc < gr->tpc_total; gtpc++) { + for (maxperf = -1, gpc = 0; gpc < gr->gpc_nr; gpc++) { + for_each_set_bit(tpc, &gpc_tpc_mask[gpc], gr->tpc_nr[gpc]) { + ret = gv100_gr_scg_estimate_perf(gr, gpc_tpc_mask, gpc, tpc, &perf); + if (ret) + goto done; + + /* nvgpu does ">=" here, but this gets us RM's numbers. */ + if (perf > maxperf) { + maxperf = perf; + gpc_table[gtpc] = gpc; + tpc_table[gtpc] = tpc; + } + } + } + + gpc_tpc_mask[gpc_table[gtpc]] &= ~BIT(tpc_table[gtpc]); + } + + /*TODO: build table for sm_per_tpc != 1, don't use yet, but might need later? */ + for (gtpc = 0; gtpc < gr->tpc_total; gtpc++) { + gr->sm[gtpc].gpc = gpc_table[gtpc]; + gr->sm[gtpc].tpc = tpc_table[gtpc]; + gr->sm_nr++; + } + +done: + kfree(gpc_table); + kfree(tpc_table); + kfree(gpc_tpc_mask); + return ret; +} + static const struct gf100_gr_func gv100_gr = { .oneinit_tiles = gm200_gr_oneinit_tiles, - .oneinit_sm_id = gm200_gr_oneinit_sm_id, + .oneinit_sm_id = gv100_gr_oneinit_sm_id, .init = gf100_gr_init, .init_419bd8 = gv100_gr_init_419bd8, .init_gpc_mmu = gm200_gr_init_gpc_mmu, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 3491005cf5d7c..1b1c6b219fc23 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -40,8 +40,9 @@ tu102_gr_init_fs(struct gf100_gr *gr) gk104_grctx_generate_gpc_tpc_nr(gr); for (sm = 0; sm < gr->sm_nr; sm++) { - nvkm_wr32(device, GPC_UNIT(gr->sm[sm].gpc, 0x0c10 + - gr->sm[sm].tpc * 4), sm); + int tpc = gv100_gr_nonpes_aware_tpc(gr, gr->sm[sm].gpc, gr->sm[sm].tpc); + + nvkm_wr32(device, GPC_UNIT(gr->sm[sm].gpc, 0x0c10 + tpc * 4), sm); } gm200_grctx_generate_dist_skip_table(gr); @@ -93,7 +94,7 @@ tu102_gr_init_gpc_mmu(struct gf100_gr *gr) static const struct gf100_gr_func tu102_gr = { .oneinit_tiles = gm200_gr_oneinit_tiles, - .oneinit_sm_id = gm200_gr_oneinit_sm_id, + .oneinit_sm_id = gv100_gr_oneinit_sm_id, .init = gf100_gr_init, .init_419bd8 = gv100_gr_init_419bd8, .init_gpc_mmu = tu102_gr_init_gpc_mmu, -- GitLab From abf8d63113078341c232446d3c2b2bd42c9ca6ef Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:17 +1000 Subject: [PATCH 1374/1927] drm/nouveau/gr/gv100-: fix number of tile map registers Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c index d68741a81a581..5b6ab9e088c5e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c @@ -113,13 +113,14 @@ void gv100_grctx_generate_rop_mapping(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; + const u32 mapregs = DIV_ROUND_UP(gr->func->gpc_nr * gr->func->tpc_nr, 6); u32 data; int i, j; /* Pack tile map into register format. */ nvkm_wr32(device, 0x418bb8, (gr->tpc_total << 8) | gr->screen_tile_row_offset); - for (i = 0; i < 11; i++) { + for (i = 0; i < mapregs; i++) { for (data = 0, j = 0; j < 6; j++) data |= (gr->tile[i * 6 + j] & 0x1f) << (j * 5); nvkm_wr32(device, 0x418b08 + (i * 4), data); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index aeb767e582c8f..4951d482f4de3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -302,7 +302,7 @@ gv100_gr = { .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, .gpc_nr = 6, - .tpc_nr = 5, + .tpc_nr = 7, .ppc_nr = 3, .grctx = &gv100_grctx, .zbc = &gp102_gr_zbc, -- GitLab From 81d4201e7ffc0d86e3cf3659f8341545d410b023 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:18 +1000 Subject: [PATCH 1375/1927] drm/nouveau/gr/gv100-: add NV_PGRAPH_PRI_PD_AB_DIST_CONFIG_1 to patch list Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c index 5b6ab9e088c5e..65812452f149f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c @@ -66,6 +66,7 @@ gv100_grctx_generate_attrib(struct gf100_gr_chan *chan) const u32 alpha = grctx->alpha_nr; const u32 attrib = grctx->attrib_nr; const u32 gfxp = grctx->gfxp_nr; + const int max_batches = 0xffff; u32 size = grctx->alpha_nr_max * gr->tpc_total; u32 ao = 0; u32 bo = ao + size; @@ -73,6 +74,7 @@ gv100_grctx_generate_attrib(struct gf100_gr_chan *chan) gf100_grctx_patch_wr32(chan, 0x405830, attrib); gf100_grctx_patch_wr32(chan, 0x40585c, alpha); + gf100_grctx_patch_wr32(chan, 0x4064c4, ((alpha / 4) << 16) | max_batches); for (gpc = 0; gpc < gr->gpc_nr; gpc++) { for (ppc = 0; ppc < gr->func->ppc_nr; ppc++, n++) { -- GitLab From 75e637db601acbd2b59ca9f97474203635da5b52 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:19 +1000 Subject: [PATCH 1376/1927] drm/nouveau/gr/gv100-: move init_419bd8() after sw_ctx load Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 3 +++ drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index a56f4f3f6c1e9..7c318cdcb428b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1359,6 +1359,9 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan) gf100_gr_mmio(gr, gr->sw_ctx); } + if (gr->func->init_419bd8) + gr->func->init_419bd8(gr); + gf100_gr_wait_idle(gr); idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 5ce83b915ebb9..ddf06f5e75350 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2333,9 +2333,6 @@ gf100_gr_init(struct gf100_gr *gr) nvkm_mask(device, 0x400500, 0x00010001, 0x00000000); - if (gr->func->init_419bd8) - gr->func->init_419bd8(gr); - gr->func->init_gpc_mmu(gr); if (gr->sw_nonctx) -- GitLab From edc6938f7bc6b954137b073c928dc71af34cc158 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:19 +1000 Subject: [PATCH 1377/1927] drm/nouveau/gr/gv100-: drop a write from init_shader_exceptions() Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index 4951d482f4de3..ebf79ea985b3c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -65,7 +65,6 @@ gv100_gr_init_shader_exceptions(struct gf100_gr *gr, int gpc, int tpc) struct nvkm_device *device = gr->base.engine.subdev.device; int sm; for (sm = 0; sm < 0x100; sm += 0x80) { - nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x728 + sm), 0x0085eb64); nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x610), 0x00000001); nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x72c + sm), 0x00000004); } -- GitLab From 1cd97b5490c860409338eda1d9c16df20621024c Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:20 +1000 Subject: [PATCH 1378/1927] drm/nouveau/gr/tu102-: use sw_veid_bundle_init from firmware NVIDIA provided this on Turing, but we kept using the hardcoded version from Volta (where they didn't). Switch to the firmware version prior to Ampere. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 5 ++++ .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 1 - .../gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c | 1 - .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 1 + .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 4 ++++ .../gpu/drm/nouveau/nvkm/engine/gr/gk20a.c | 12 +++++++--- .../gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 24 +++++++++++++++++++ 8 files changed, 44 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 7c318cdcb428b..38502dfd984b2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1381,12 +1381,17 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan) gf100_gr_wait_idle(gr); if (grctx->r400088) grctx->r400088(gr, false); + if (gr->bundle) gf100_gr_icmd(gr, gr->bundle); else gf100_gr_icmd(gr, grctx->icmd); + + if (gr->bundle_veid) + gf100_gr_icmd(gr, gr->bundle_veid); if (grctx->sw_veid_bundle_init) gf100_gr_icmd(gr, grctx->sw_veid_bundle_init); + if (grctx->r400088) grctx->r400088(gr, true); nvkm_wr32(device, 0x404154, idle_timeout); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 99bb9af4955a7..7a4ce97b033d2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -153,7 +153,6 @@ extern const struct gf100_grctx_func gv100_grctx; extern const struct gf100_grctx_func tu102_grctx; void gv100_grctx_unkn88c(struct gf100_gr *, bool); void gv100_grctx_generate_unkn(struct gf100_gr *); -extern const struct gf100_gr_init gv100_grctx_init_sw_veid_bundle_init_0[]; void gv100_grctx_generate_attrib_cb(struct gf100_gr_chan *, u64, u32); void gv100_grctx_generate_attrib(struct gf100_gr_chan *); void gv100_grctx_generate_rop_mapping(struct gf100_gr *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c index 65812452f149f..957ea9d6bad4b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgv100.c @@ -25,7 +25,7 @@ * PGRAPH context implementation ******************************************************************************/ -const struct gf100_gr_init +static const struct gf100_gr_init gv100_grctx_init_sw_veid_bundle_init_0[] = { { 0x00001000, 64, 0x00100000, 0x00000008 }, { 0x00000941, 64, 0x00100000, 0x00000000 }, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c index da1bac3963a95..54bdfe304ac7e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c @@ -51,7 +51,6 @@ tu102_grctx_init_unknown_bundle_init_0[] = { static const struct gf100_gr_pack tu102_grctx_pack_sw_veid_bundle_init[] = { - { gv100_grctx_init_sw_veid_bundle_init_0 }, { tu102_grctx_init_unknown_bundle_init_0 }, {} }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index ddf06f5e75350..6b559b2bd8f8f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -2139,6 +2139,7 @@ gf100_gr_dtor(struct nvkm_gr *base) nvkm_blob_dtor(&gr->gpccs.inst); nvkm_blob_dtor(&gr->gpccs.data); + vfree(gr->bundle_veid); vfree(gr->bundle); vfree(gr->method); vfree(gr->sw_ctx); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index a9ba9af25728a..6e1c730932b48 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -90,6 +90,7 @@ struct gf100_gr { struct gf100_gr_pack *sw_nonctx; struct gf100_gr_pack *sw_ctx; struct gf100_gr_pack *bundle; + struct gf100_gr_pack *bundle_veid; struct gf100_gr_pack *method; struct gf100_gr_zbc_color zbc_color[NVKM_LTC_MAX_ZBC_COLOR_CNT]; @@ -224,6 +225,7 @@ void gm107_gr_init_shader_exceptions(struct gf100_gr *, int, int); void gm107_gr_init_400054(struct gf100_gr *); int gk20a_gr_init(struct gf100_gr *); +int gk20a_gr_av_to_init_(struct nvkm_blob *, u8 count, u32 pitch, struct gf100_gr_pack **); int gk20a_gr_av_to_init(struct nvkm_blob *, struct gf100_gr_pack **); int gk20a_gr_aiv_to_init(struct nvkm_blob *, struct gf100_gr_pack **); int gk20a_gr_av_to_method(struct nvkm_blob *, struct gf100_gr_pack **); @@ -253,6 +255,8 @@ void gv100_gr_init_504430(struct gf100_gr *, int, int); void gv100_gr_init_shader_exceptions(struct gf100_gr *, int, int); void gv100_gr_trap_mp(struct gf100_gr *, int, int); +int tu102_gr_av_to_init_veid(struct nvkm_blob *, struct gf100_gr_pack **); + #define gf100_gr_chan(p) container_of((p), struct gf100_gr_chan, object) #include diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c index 2aee0559efca3..035ea213f5433 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c @@ -34,7 +34,7 @@ struct gk20a_fw_av }; int -gk20a_gr_av_to_init(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) +gk20a_gr_av_to_init_(struct nvkm_blob *blob, u8 count, u32 pitch, struct gf100_gr_pack **ppack) { struct gf100_gr_init *init; struct gf100_gr_pack *pack; @@ -55,14 +55,20 @@ gk20a_gr_av_to_init(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) ent->addr = av->addr; ent->data = av->data; - ent->count = 1; - ent->pitch = 1; + ent->count = ((ent->addr & 0xffff) != 0xe100) ? count : 1; + ent->pitch = pitch; } *ppack = pack; return 0; } +int +gk20a_gr_av_to_init(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) +{ + return gk20a_gr_av_to_init_(blob, 1, 1, ppack); +} + struct gk20a_fw_aiv { u32 addr; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 1b1c6b219fc23..96ba82a263854 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -141,6 +141,7 @@ MODULE_FIRMWARE("nvidia/tu102/gr/sw_ctx.bin"); MODULE_FIRMWARE("nvidia/tu102/gr/sw_nonctx.bin"); MODULE_FIRMWARE("nvidia/tu102/gr/sw_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu102/gr/sw_method_init.bin"); +MODULE_FIRMWARE("nvidia/tu102/gr/sw_veid_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu104/gr/fecs_bl.bin"); MODULE_FIRMWARE("nvidia/tu104/gr/fecs_inst.bin"); @@ -154,6 +155,7 @@ MODULE_FIRMWARE("nvidia/tu104/gr/sw_ctx.bin"); MODULE_FIRMWARE("nvidia/tu104/gr/sw_nonctx.bin"); MODULE_FIRMWARE("nvidia/tu104/gr/sw_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu104/gr/sw_method_init.bin"); +MODULE_FIRMWARE("nvidia/tu104/gr/sw_veid_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu106/gr/fecs_bl.bin"); MODULE_FIRMWARE("nvidia/tu106/gr/fecs_inst.bin"); @@ -167,6 +169,7 @@ MODULE_FIRMWARE("nvidia/tu106/gr/sw_ctx.bin"); MODULE_FIRMWARE("nvidia/tu106/gr/sw_nonctx.bin"); MODULE_FIRMWARE("nvidia/tu106/gr/sw_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu106/gr/sw_method_init.bin"); +MODULE_FIRMWARE("nvidia/tu106/gr/sw_veid_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu117/gr/fecs_bl.bin"); MODULE_FIRMWARE("nvidia/tu117/gr/fecs_inst.bin"); @@ -180,6 +183,7 @@ MODULE_FIRMWARE("nvidia/tu117/gr/sw_ctx.bin"); MODULE_FIRMWARE("nvidia/tu117/gr/sw_nonctx.bin"); MODULE_FIRMWARE("nvidia/tu117/gr/sw_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu117/gr/sw_method_init.bin"); +MODULE_FIRMWARE("nvidia/tu117/gr/sw_veid_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu116/gr/fecs_bl.bin"); MODULE_FIRMWARE("nvidia/tu116/gr/fecs_inst.bin"); @@ -193,6 +197,26 @@ MODULE_FIRMWARE("nvidia/tu116/gr/sw_ctx.bin"); MODULE_FIRMWARE("nvidia/tu116/gr/sw_nonctx.bin"); MODULE_FIRMWARE("nvidia/tu116/gr/sw_bundle_init.bin"); MODULE_FIRMWARE("nvidia/tu116/gr/sw_method_init.bin"); +MODULE_FIRMWARE("nvidia/tu116/gr/sw_veid_bundle_init.bin"); + +int +tu102_gr_av_to_init_veid(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) +{ + return gk20a_gr_av_to_init_(blob, 64, 0x00100000, ppack); +} + +int +tu102_gr_load(struct gf100_gr *gr, int ver, const struct gf100_gr_fwif *fwif) +{ + int ret; + + ret = gm200_gr_load(gr, ver, fwif); + if (ret) + return ret; + + return gk20a_gr_load_net(gr, "gr/", "sw_veid_bundle_init", ver, tu102_gr_av_to_init_veid, + &gr->bundle_veid); +} static const struct gf100_gr_fwif tu102_gr_fwif[] = { -- GitLab From c4d66f7db67248b110b4183f1b155c28c752121f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:21 +1000 Subject: [PATCH 1379/1927] drm/nouveau/gr/tu102-: fix support for sw_bundle64_init We weren't sending the high bits, though they're zero currently anyway. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 8 +++++++- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c | 6 +++--- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 5 ++++- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 3 ++- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 38502dfd984b2..2a31d4464a326 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1389,9 +1389,15 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan) if (gr->bundle_veid) gf100_gr_icmd(gr, gr->bundle_veid); - if (grctx->sw_veid_bundle_init) + else gf100_gr_icmd(gr, grctx->sw_veid_bundle_init); + if (gr->bundle64) + gf100_gr_icmd(gr, gr->bundle64); + else + if (grctx->sw_bundle64_init) + gf100_gr_icmd(gr, grctx->sw_bundle64_init); + if (grctx->r400088) grctx->r400088(gr, true); nvkm_wr32(device, 0x404154, idle_timeout); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 7a4ce97b033d2..46f94c1439361 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -22,6 +22,7 @@ struct gf100_grctx_func { const struct gf100_gr_pack *icmd; const struct gf100_gr_pack *mthd; const struct gf100_gr_pack *sw_veid_bundle_init; + const struct gf100_gr_pack *sw_bundle64_init; /* bundle circular buffer */ void (*bundle)(struct gf100_gr_chan *, u64 addr, u32 size); u32 bundle_size; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c index 54bdfe304ac7e..36c38d0040878 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c @@ -50,8 +50,8 @@ tu102_grctx_init_unknown_bundle_init_0[] = { }; static const struct gf100_gr_pack -tu102_grctx_pack_sw_veid_bundle_init[] = { - { tu102_grctx_init_unknown_bundle_init_0 }, +tu102_grctx_pack_sw_bundle64_init[] = { + { tu102_grctx_init_unknown_bundle_init_0, .type = 64 }, {} }; @@ -69,7 +69,7 @@ tu102_grctx = { .unkn88c = gv100_grctx_unkn88c, .main = gf100_grctx_generate_main, .unkn = gv100_grctx_generate_unkn, - .sw_veid_bundle_init = tu102_grctx_pack_sw_veid_bundle_init, + .sw_bundle64_init = tu102_grctx_pack_sw_bundle64_init, .bundle = gm107_grctx_generate_bundle, .bundle_size = 0x3000, .bundle_min_gpm_fifo_depth = 0x180, diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index 6b559b2bd8f8f..b82989ef7f3b3 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1097,7 +1097,7 @@ gf100_gr_icmd(struct gf100_gr *gr, const struct gf100_gr_pack *p) struct nvkm_device *device = gr->base.engine.subdev.device; const struct gf100_gr_pack *pack; const struct gf100_gr_init *init; - u32 data = 0; + u64 data = 0; nvkm_wr32(device, 0x400208, 0x80000000); @@ -1107,6 +1107,8 @@ gf100_gr_icmd(struct gf100_gr *gr, const struct gf100_gr_pack *p) if ((pack == p && init == p->init) || data != init->data) { nvkm_wr32(device, 0x400204, init->data); + if (pack->type == 64) + nvkm_wr32(device, 0x40020c, upper_32_bits(init->data)); data = init->data; } @@ -2139,6 +2141,7 @@ gf100_gr_dtor(struct nvkm_gr *base) nvkm_blob_dtor(&gr->gpccs.inst); nvkm_blob_dtor(&gr->gpccs.data); + vfree(gr->bundle64); vfree(gr->bundle_veid); vfree(gr->bundle); vfree(gr->method); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 6e1c730932b48..baf21aaf00217 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -91,6 +91,7 @@ struct gf100_gr { struct gf100_gr_pack *sw_ctx; struct gf100_gr_pack *bundle; struct gf100_gr_pack *bundle_veid; + struct gf100_gr_pack *bundle64; struct gf100_gr_pack *method; struct gf100_gr_zbc_color zbc_color[NVKM_LTC_MAX_ZBC_COLOR_CNT]; @@ -286,7 +287,7 @@ struct gf100_gr_init { u32 addr; u8 count; u32 pitch; - u32 data; + u64 data; }; struct gf100_gr_pack { -- GitLab From 6a2b09e7bf3fe9bcdd38b84c12e34c7b4cbb263d Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:21 +1000 Subject: [PATCH 1380/1927] drm/nouveau/gr/tu102: add gv100_gr_init_4188a4 Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 1 + drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c | 3 ++- drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index baf21aaf00217..88fccea248e55 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -254,6 +254,7 @@ u32 gv100_gr_nonpes_aware_tpc(struct gf100_gr *gr, u32 gpc, u32 tpc); void gv100_gr_init_419bd8(struct gf100_gr *); void gv100_gr_init_504430(struct gf100_gr *, int, int); void gv100_gr_init_shader_exceptions(struct gf100_gr *, int, int); +void gv100_gr_init_4188a4(struct gf100_gr *); void gv100_gr_trap_mp(struct gf100_gr *, int, int); int tu102_gr_av_to_init_veid(struct nvkm_blob *, struct gf100_gr_pack **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c index ebf79ea985b3c..7f7404a761409 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gv100.c @@ -52,10 +52,11 @@ gv100_gr_trap_mp(struct gf100_gr *gr, int gpc, int tpc) gv100_gr_trap_sm(gr, gpc, tpc, 1); } -static void +void gv100_gr_init_4188a4(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; + nvkm_mask(device, 0x4188a4, 0x03000000, 0x03000000); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 96ba82a263854..255d50c4c65ff 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -112,6 +112,7 @@ tu102_gr = { .init_shader_exceptions = gv100_gr_init_shader_exceptions, .init_rop_exceptions = gf100_gr_init_rop_exceptions, .init_exception2 = gf100_gr_init_exception2, + .init_4188a4 = gv100_gr_init_4188a4, .trap_mp = gv100_gr_trap_mp, .fecs.reset = gf100_gr_fecs_reset, .rops = gm200_gr_rops, -- GitLab From 21876b0e4284169ddbc834d02f60940a3dd27471 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:22 +1000 Subject: [PATCH 1381/1927] drm/nouveau/gr/tu102: remove gv100_grctx_unkn88c Match RM. Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c index 36c38d0040878..542ab0c78be65 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxtu102.c @@ -66,7 +66,6 @@ tu102_grctx_generate_unknown(struct gf100_gr_chan *chan, u64 addr, u32 size) const struct gf100_grctx_func tu102_grctx = { - .unkn88c = gv100_grctx_unkn88c, .main = gf100_grctx_generate_main, .unkn = gv100_grctx_generate_unkn, .sw_bundle64_init = tu102_grctx_pack_sw_bundle64_init, -- GitLab From a51c69ee74d195e6d84fe9d48ca55f51d39e1b5b Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:32 +1000 Subject: [PATCH 1382/1927] drm/nouveau/fb/ga102: load and boot VPR scrubber FW v2. fixup for ga103 early merge Signed-off-by: Ben Skeggs Signed-off-by: Gourav Samaiya --- drivers/gpu/drm/nouveau/include/nvfw/hs.h | 28 ++++ .../drm/nouveau/include/nvkm/core/falcon.h | 25 ++++ .../drm/nouveau/include/nvkm/engine/falcon.h | 6 + .../drm/nouveau/include/nvkm/engine/nvdec.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 5 + .../gpu/drm/nouveau/nvkm/engine/nvdec/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/engine/nvdec/base.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/nvdec/ga102.c | 61 ++++++++ .../gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c | 2 +- .../gpu/drm/nouveau/nvkm/engine/nvdec/priv.h | 2 +- drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild | 2 + drivers/gpu/drm/nouveau/nvkm/falcon/base.c | 77 ++++++++++ drivers/gpu/drm/nouveau/nvkm/falcon/fw.c | 58 ++++++++ drivers/gpu/drm/nouveau/nvkm/falcon/ga100.c | 62 ++++++++ drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c | 134 ++++++++++++++++++ drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c | 9 +- drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c | 8 ++ drivers/gpu/drm/nouveau/nvkm/nvfw/hs.c | 39 +++++ .../gpu/drm/nouveau/nvkm/subdev/fb/ga102.c | 32 +++++ 19 files changed, 551 insertions(+), 5 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/nvdec/ga102.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/falcon/ga100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c diff --git a/drivers/gpu/drm/nouveau/include/nvfw/hs.h b/drivers/gpu/drm/nouveau/include/nvfw/hs.h index b53bbc4cd130c..8c4cd08a7b5f8 100644 --- a/drivers/gpu/drm/nouveau/include/nvfw/hs.h +++ b/drivers/gpu/drm/nouveau/include/nvfw/hs.h @@ -17,6 +17,20 @@ struct nvfw_hs_header { const struct nvfw_hs_header *nvfw_hs_header(struct nvkm_subdev *, const void *); +struct nvfw_hs_header_v2 { + u32 sig_prod_offset; + u32 sig_prod_size; + u32 patch_loc; + u32 patch_sig; + u32 meta_data_offset; + u32 meta_data_size; + u32 num_sig; + u32 header_offset; + u32 header_size; +}; + +const struct nvfw_hs_header_v2 *nvfw_hs_header_v2(struct nvkm_subdev *, const void *); + struct nvfw_hs_load_header { u32 non_sec_code_off; u32 non_sec_code_size; @@ -28,4 +42,18 @@ struct nvfw_hs_load_header { const struct nvfw_hs_load_header * nvfw_hs_load_header(struct nvkm_subdev *, const void *); + +struct nvfw_hs_load_header_v2 { + u32 os_code_offset; + u32 os_code_size; + u32 os_data_offset; + u32 os_data_size; + u32 num_apps; + struct { + u32 offset; + u32 size; + } app[0]; +}; + +const struct nvfw_hs_load_header_v2 *nvfw_hs_load_header_v2(struct nvkm_subdev *, const void *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index 4868d2cb796f8..45d70aa4fd6d3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -31,6 +31,13 @@ struct nvkm_falcon_func_pio { void (*rd)(struct nvkm_falcon *, u8 port, const u8 *img, int len); }; +struct nvkm_falcon_func_dma { + int (*init)(struct nvkm_falcon *, u64 dma_addr, int xfer_len, + enum nvkm_falcon_mem, bool sec, u32 *cmd); + void (*xfer)(struct nvkm_falcon *, u32 mem_base, u32 dma_base, u32 cmd); + bool (*done)(struct nvkm_falcon *); +}; + int nvkm_falcon_ctor(const struct nvkm_falcon_func *, struct nvkm_subdev *owner, const char *name, u32 addr, struct nvkm_falcon *); void nvkm_falcon_dtor(struct nvkm_falcon *); @@ -39,6 +46,8 @@ int nvkm_falcon_pio_wr(struct nvkm_falcon *, const u8 *img, u32 img_base, u8 por enum nvkm_falcon_mem mem_type, u32 mem_base, int len, u16 tag, bool sec); int nvkm_falcon_pio_rd(struct nvkm_falcon *, u8 port, enum nvkm_falcon_mem type, u32 mem_base, const u8 *img, u32 img_base, int len); +int nvkm_falcon_dma_wr(struct nvkm_falcon *, const u8 *img, u64 dma_addr, u32 dma_base, + enum nvkm_falcon_mem mem_type, u32 mem_base, int len, bool sec); int gm200_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *); int gm200_flcn_disable(struct nvkm_falcon *); @@ -52,6 +61,10 @@ void gm200_flcn_tracepc(struct nvkm_falcon *); int gp102_flcn_reset_eng(struct nvkm_falcon *); extern const struct nvkm_falcon_func_pio gp102_flcn_emem_pio; +int ga102_flcn_reset_prep(struct nvkm_falcon *); +int ga102_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *); +extern const struct nvkm_falcon_func_dma ga102_flcn_dma; + void nvkm_falcon_v1_load_imem(struct nvkm_falcon *, void *, u32, u32, u16, u8, bool); void nvkm_falcon_v1_load_dmem(struct nvkm_falcon *, void *, u32, u32, u8); @@ -87,6 +100,9 @@ struct nvkm_falcon_fw { u32 sig_size; int sig_nr; u8 *sigs; + u32 fuse_ver; + u32 engine_id; + u32 ucode_id; u32 nmem_base_img; u32 nmem_base; @@ -117,6 +133,9 @@ int nvkm_falcon_fw_ctor(const struct nvkm_falcon_fw_func *, const char *name, st int nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *, const char *name, struct nvkm_subdev *, const char *bl, const char *img, int ver, struct nvkm_falcon *falcon, struct nvkm_falcon_fw *fw); +int nvkm_falcon_fw_ctor_hs_v2(const struct nvkm_falcon_fw_func *, const char *name, + struct nvkm_subdev *, const char *img, int ver, struct nvkm_falcon *, + struct nvkm_falcon_fw *); int nvkm_falcon_fw_sign(struct nvkm_falcon_fw *, u32 sig_base_img, u32 sig_size, const u8 *sigs, int sig_nr_prd, u32 sig_base_prd, int sig_nr_dbg, u32 sig_base_dbg); int nvkm_falcon_fw_patch(struct nvkm_falcon_fw *); @@ -132,6 +151,12 @@ int gm200_flcn_fw_reset(struct nvkm_falcon_fw *); int gm200_flcn_fw_load(struct nvkm_falcon_fw *); int gm200_flcn_fw_boot(struct nvkm_falcon_fw *, u32 *, u32 *, u32, u32); +int ga100_flcn_fw_signature(struct nvkm_falcon_fw *, u32 *); + +extern const struct nvkm_falcon_fw_func ga102_flcn_fw; +int ga102_flcn_fw_load(struct nvkm_falcon_fw *); +int ga102_flcn_fw_boot(struct nvkm_falcon_fw *, u32 *, u32 *, u32, u32); + #define FLCNFW_PRINTK(f,l,p,fmt,a...) FLCN_PRINTK((f)->falcon, l, p, "%s: "fmt, (f)->fw.name, ##a) #define FLCNFW_DBG(f,fmt,a...) FLCNFW_PRINTK((f), DEBUG, info, fmt"\n", ##a) #define FLCNFW_ERR(f,fmt,a...) FLCNFW_PRINTK((f), ERROR, err, fmt"\n", ##a) diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index f576ca246d100..dacbd92edcd56 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -19,6 +19,7 @@ struct nvkm_falcon { struct nvkm_subdev *owner; const char *name; u32 addr; + u32 addr2; struct mutex mutex; struct mutex dmem_mutex; @@ -59,8 +60,10 @@ int nvkm_falcon_new_(const struct nvkm_falcon_func *, struct nvkm_device *, struct nvkm_falcon_func { int (*disable)(struct nvkm_falcon *); int (*enable)(struct nvkm_falcon *); + u32 addr2; bool reset_pmc; int (*reset_eng)(struct nvkm_falcon *); + int (*reset_prep)(struct nvkm_falcon *); int (*reset_wait_mem_scrubbing)(struct nvkm_falcon *); u32 debug; @@ -69,7 +72,10 @@ struct nvkm_falcon_func { bool bind_intr; const struct nvkm_falcon_func_pio *imem_pio; + const struct nvkm_falcon_func_dma *imem_dma; + const struct nvkm_falcon_func_pio *dmem_pio; + const struct nvkm_falcon_func_dma *dmem_dma; u32 emem_addr; const struct nvkm_falcon_func_pio *emem_pio; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h index 97bd3092f68a6..9baf197ac8336 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/nvdec.h @@ -12,4 +12,5 @@ struct nvkm_nvdec { }; int gm107_nvdec_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_nvdec **); +int ga102_nvdec_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_nvdec **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 931a595818159..8162efcf2dd64 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2615,6 +2615,7 @@ nv172_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .nvdec = { 0x00000001, ga102_nvdec_new }, }; static const struct nvkm_device_chip @@ -2639,6 +2640,7 @@ nv173_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .nvdec = { 0x00000001, ga102_nvdec_new }, }; static const struct nvkm_device_chip @@ -2663,6 +2665,7 @@ nv174_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .nvdec = { 0x00000001, ga102_nvdec_new }, }; static const struct nvkm_device_chip @@ -2687,6 +2690,7 @@ nv176_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .nvdec = { 0x00000001, ga102_nvdec_new }, }; static const struct nvkm_device_chip @@ -2711,6 +2715,7 @@ nv177_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .nvdec = { 0x00000001, ga102_nvdec_new }, }; struct nvkm_subdev * diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/Kbuild index 9a0fd9812750d..f05e79670d227 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/Kbuild @@ -1,3 +1,4 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/engine/nvdec/base.o nvkm-y += nvkm/engine/nvdec/gm107.o +nvkm-y += nvkm/engine/nvdec/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c index b0181cc5953bc..1f6e3b32ba163 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/base.c @@ -37,7 +37,7 @@ nvkm_nvdec = { int nvkm_nvdec_new_(const struct nvkm_nvdec_fwif *fwif, struct nvkm_device *device, - enum nvkm_subdev_type type, int inst, struct nvkm_nvdec **pnvdec) + enum nvkm_subdev_type type, int inst, u32 addr, struct nvkm_nvdec **pnvdec) { struct nvkm_nvdec *nvdec; int ret; @@ -57,5 +57,5 @@ nvkm_nvdec_new_(const struct nvkm_nvdec_fwif *fwif, struct nvkm_device *device, nvdec->func = fwif->func; return nvkm_falcon_ctor(nvdec->func->flcn, &nvdec->engine.subdev, - nvdec->engine.subdev.name, 0, &nvdec->falcon); + nvdec->engine.subdev.name, addr, &nvdec->falcon); }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/ga102.c new file mode 100644 index 0000000000000..37d8c3c0f3abc --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/ga102.c @@ -0,0 +1,61 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include +#include + +static const struct nvkm_falcon_func +ga102_nvdec_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .addr2 = 0x1c00, + .reset_pmc = true, + .reset_prep = ga102_flcn_reset_prep, + .reset_wait_mem_scrubbing = ga102_flcn_reset_wait_mem_scrubbing, + .imem_dma = &ga102_flcn_dma, + .dmem_dma = &ga102_flcn_dma, +}; + +static const struct nvkm_nvdec_func +ga102_nvdec = { + .flcn = &ga102_nvdec_flcn, +}; + +static int +ga102_nvdec_nofw(struct nvkm_nvdec *nvdec, int ver, const struct nvkm_nvdec_fwif *fwif) +{ + return 0; +} + +static const struct nvkm_nvdec_fwif +ga102_nvdec_fwif[] = { + { -1, ga102_nvdec_nofw, &ga102_nvdec }, + {} +}; + +int +ga102_nvdec_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_nvdec **pnvdec) +{ + return nvkm_nvdec_new_(ga102_nvdec_fwif, device, type, inst, 0x848000, pnvdec); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c index 5d04ded35cc34..564f7e8960a20 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/gm107.c @@ -54,5 +54,5 @@ int gm107_nvdec_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_nvdec **pnvdec) { - return nvkm_nvdec_new_(gm107_nvdec_fwif, device, type, inst, pnvdec); + return nvkm_nvdec_new_(gm107_nvdec_fwif, device, type, inst, 0, pnvdec); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/priv.h index 0920f6a887e2a..61e1f7aaa5093 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/nvdec/priv.h @@ -15,5 +15,5 @@ struct nvkm_nvdec_fwif { }; int nvkm_nvdec_new_(const struct nvkm_nvdec_fwif *fwif, struct nvkm_device *, - enum nvkm_subdev_type, int, struct nvkm_nvdec **); + enum nvkm_subdev_type, int, u32 addr, struct nvkm_nvdec **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild b/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild index 6ffde5290b876..9ffe7b921ccb5 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/Kbuild @@ -8,3 +8,5 @@ nvkm-y += nvkm/falcon/v1.o nvkm-y += nvkm/falcon/gm200.o nvkm-y += nvkm/falcon/gp102.o +nvkm-y += nvkm/falcon/ga100.o +nvkm-y += nvkm/falcon/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c index e4075aa441f37..235149f73a690 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/base.c @@ -25,6 +25,82 @@ #include #include +static const struct nvkm_falcon_func_dma * +nvkm_falcon_dma(struct nvkm_falcon *falcon, enum nvkm_falcon_mem *mem_type, u32 *mem_base) +{ + switch (*mem_type) { + case IMEM: return falcon->func->imem_dma; + case DMEM: return falcon->func->dmem_dma; + default: + return NULL; + } +} + +int +nvkm_falcon_dma_wr(struct nvkm_falcon *falcon, const u8 *img, u64 dma_addr, u32 dma_base, + enum nvkm_falcon_mem mem_type, u32 mem_base, int len, bool sec) +{ + const struct nvkm_falcon_func_dma *dma = nvkm_falcon_dma(falcon, &mem_type, &mem_base); + const char *type = nvkm_falcon_mem(mem_type); + const int dmalen = 256; + u32 dma_start = 0; + u32 dst, src, cmd; + int ret, i; + + if (WARN_ON(!dma->xfer)) + return -EINVAL; + + if (mem_type == DMEM) { + dma_start = dma_base; + dma_addr += dma_base; + } + + FLCN_DBG(falcon, "%s %08x <- %08x bytes at %08x (%010llx %08x)", + type, mem_base, len, dma_base, dma_addr - dma_base, dma_start); + if (WARN_ON(!len || (len & (dmalen - 1)))) + return -EINVAL; + + ret = dma->init(falcon, dma_addr, dmalen, mem_type, sec, &cmd); + if (ret) + return ret; + + dst = mem_base; + src = dma_base; + if (len) { + while (len >= dmalen) { + dma->xfer(falcon, dst, src - dma_start, cmd); + + if (img && nvkm_printk_ok(falcon->owner, falcon->user, NV_DBG_TRACE)) { + for (i = 0; i < dmalen; i += 4, mem_base += 4) { + const int w = 8, x = (i / 4) % w; + + if (x == 0) + printk(KERN_INFO "%s %08x <-", type, mem_base); + printk(KERN_CONT " %08x", *(u32 *)(img + src + i)); + if (x == (w - 1) || ((i + 4) == dmalen)) + printk(KERN_CONT " <- %08x+%08x", dma_base, + src + i - dma_base - (x * 4)); + if (i == (7 * 4)) + printk(KERN_CONT " *"); + } + } + + if (nvkm_msec(falcon->owner->device, 2000, + if (dma->done(falcon)) + break; + ) < 0) + return -ETIMEDOUT; + + src += dmalen; + dst += dmalen; + len -= dmalen; + } + WARN_ON(len); + } + + return 0; +} + static const struct nvkm_falcon_func_pio * nvkm_falcon_pio(struct nvkm_falcon *falcon, enum nvkm_falcon_mem *mem_type, u32 *mem_base) { @@ -239,6 +315,7 @@ nvkm_falcon_ctor(const struct nvkm_falcon_func *func, falcon->owner = subdev; falcon->name = name; falcon->addr = addr; + falcon->addr2 = func->addr2; mutex_init(&falcon->mutex); mutex_init(&falcon->dmem_mutex); return 0; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c index 13d52d7e4f60b..80a480b121746 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/fw.c @@ -294,3 +294,61 @@ nvkm_falcon_fw_ctor_hs(const struct nvkm_falcon_fw_func *func, const char *name, nvkm_firmware_put(blob); return ret; } + +int +nvkm_falcon_fw_ctor_hs_v2(const struct nvkm_falcon_fw_func *func, const char *name, + struct nvkm_subdev *subdev, const char *img, int ver, + struct nvkm_falcon *falcon, struct nvkm_falcon_fw *fw) +{ + const struct nvfw_bin_hdr *hdr; + const struct nvfw_hs_header_v2 *hshdr; + const struct nvfw_hs_load_header_v2 *lhdr; + const struct firmware *blob; + u32 loc, sig, cnt, *meta; + int ret; + + ret = nvkm_firmware_load_name(subdev, img, "", ver, &blob); + if (ret) + return ret; + + hdr = nvfw_bin_hdr(subdev, blob->data); + hshdr = nvfw_hs_header_v2(subdev, blob->data + hdr->header_offset); + meta = (u32 *)(blob->data + hshdr->meta_data_offset); + loc = *(u32 *)(blob->data + hshdr->patch_loc); + sig = *(u32 *)(blob->data + hshdr->patch_sig); + cnt = *(u32 *)(blob->data + hshdr->num_sig); + + ret = nvkm_falcon_fw_ctor(func, name, subdev->device, true, + blob->data + hdr->data_offset, hdr->data_size, falcon, fw); + if (ret) + goto done; + + ret = nvkm_falcon_fw_sign(fw, loc, hshdr->sig_prod_size / cnt, blob->data, + cnt, hshdr->sig_prod_offset + sig, 0, 0); + if (ret) + goto done; + + lhdr = nvfw_hs_load_header_v2(subdev, blob->data + hshdr->header_offset); + + fw->imem_base_img = lhdr->app[0].offset; + fw->imem_base = 0; + fw->imem_size = lhdr->app[0].size; + + fw->dmem_base_img = lhdr->os_data_offset; + fw->dmem_base = 0; + fw->dmem_size = lhdr->os_data_size; + fw->dmem_sign = loc - lhdr->os_data_offset; + + fw->boot_addr = lhdr->app[0].offset; + + fw->fuse_ver = meta[0]; + fw->engine_id = meta[1]; + fw->ucode_id = meta[2]; + +done: + if (ret) + nvkm_falcon_fw_dtor(fw); + + nvkm_firmware_put(blob); + return ret; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/ga100.c b/drivers/gpu/drm/nouveau/nvkm/falcon/ga100.c new file mode 100644 index 0000000000000..49fd32943916f --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/ga100.c @@ -0,0 +1,62 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +int +ga100_flcn_fw_signature(struct nvkm_falcon_fw *fw, u32 *src_base_src) +{ + struct nvkm_falcon *falcon = fw->falcon; + struct nvkm_device *device = falcon->owner->device; + u32 reg_fuse_version; + int idx; + + FLCN_DBG(falcon, "brom: %08x %08x", fw->engine_id, fw->ucode_id); + FLCN_DBG(falcon, "fuse_version: %d", fw->fuse_ver); + + if (fw->engine_id & 0x00000001) { + reg_fuse_version = nvkm_rd32(device, 0x824140 + (fw->ucode_id - 1) * 4); + } else + if (fw->engine_id & 0x00000004) { + reg_fuse_version = nvkm_rd32(device, 0x824100 + (fw->ucode_id - 1) * 4); + } else + if (fw->engine_id & 0x00000400) { + reg_fuse_version = nvkm_rd32(device, 0x8241c0 + (fw->ucode_id - 1) * 4); + } else { + WARN_ON(1); + return -ENOSYS; + } + + FLCN_DBG(falcon, "reg_fuse_version: %08x", reg_fuse_version); + if (reg_fuse_version) { + reg_fuse_version = fls(reg_fuse_version); + FLCN_DBG(falcon, "reg_fuse_version: %d", reg_fuse_version); + + if (WARN_ON(fw->fuse_ver < reg_fuse_version)) + return -EINVAL; + + idx = fw->fuse_ver - reg_fuse_version; + } else { + idx = fw->sig_nr - 1; + } + + return idx; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c b/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c new file mode 100644 index 0000000000000..38306f9920b43 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c @@ -0,0 +1,134 @@ +/* + * Copyright 2022 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include +#include + +static bool +ga102_flcn_dma_done(struct nvkm_falcon *falcon) +{ + return !!(nvkm_falcon_rd32(falcon, 0x118) & 0x00000002); +} + +static void +ga102_flcn_dma_xfer(struct nvkm_falcon *falcon, u32 mem_base, u32 dma_base, u32 cmd) +{ + nvkm_falcon_wr32(falcon, 0x114, mem_base); + nvkm_falcon_wr32(falcon, 0x11c, dma_base); + nvkm_falcon_wr32(falcon, 0x118, cmd); +} + +static int +ga102_flcn_dma_init(struct nvkm_falcon *falcon, u64 dma_addr, int xfer_len, + enum nvkm_falcon_mem mem_type, bool sec, u32 *cmd) +{ + *cmd = (ilog2(xfer_len) - 2) << 8; + if (mem_type == IMEM) + *cmd |= 0x00000010; + if (sec) + *cmd |= 0x00000004; + + nvkm_falcon_wr32(falcon, 0x110, dma_addr >> 8); + nvkm_falcon_wr32(falcon, 0x128, 0x00000000); + return 0; +} + +const struct nvkm_falcon_func_dma +ga102_flcn_dma = { + .init = ga102_flcn_dma_init, + .xfer = ga102_flcn_dma_xfer, + .done = ga102_flcn_dma_done, +}; + +int +ga102_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *falcon) +{ + nvkm_falcon_mask(falcon, 0x040, 0x00000000, 0x00000000); + + if (nvkm_msec(falcon->owner->device, 20, + if (!(nvkm_falcon_rd32(falcon, 0x0f4) & 0x00001000)) + break; + ) < 0) + return -ETIMEDOUT; + + return 0; +} + +int +ga102_flcn_reset_prep(struct nvkm_falcon *falcon) +{ + const u32 addr2 = (falcon->owner->type != NVKM_ENGINE_NVDEC) ? 0x530 : 0x930; + + if (nvkm_msec(falcon->owner->device, 10, + if ((nvkm_falcon_rd32(falcon, falcon->addr2 + 0x1ec) & 0x00000003) == 0x00000001 && + (nvkm_falcon_rd32(falcon, addr2) & 0x00000008) == 0x00000008) + break; + ) < 0) + return -ETIMEDOUT; + + return 0; +} + +int +ga102_flcn_fw_boot(struct nvkm_falcon_fw *fw, u32 *mbox0, u32 *mbox1, u32 mbox0_ok, u32 irqsclr) +{ + struct nvkm_falcon *falcon = fw->falcon; + + nvkm_falcon_wr32(falcon, falcon->addr2 + 0x210, fw->dmem_sign); + nvkm_falcon_wr32(falcon, falcon->addr2 + 0x19c, fw->engine_id); + nvkm_falcon_wr32(falcon, falcon->addr2 + 0x198, fw->ucode_id); + nvkm_falcon_wr32(falcon, falcon->addr2 + 0x180, 0x00000001); + + return gm200_flcn_fw_boot(fw, mbox0, mbox1, mbox0_ok, irqsclr); +} + +int +ga102_flcn_fw_load(struct nvkm_falcon_fw *fw) +{ + struct nvkm_falcon *falcon = fw->falcon; + int ret = 0; + + nvkm_falcon_mask(falcon, 0x624, 0x00000080, 0x00000080); + nvkm_falcon_wr32(falcon, 0x10c, 0x00000000); + nvkm_falcon_mask(falcon, 0x600, 0x00010007, (0 << 16) | (1 << 2) | 1); + + ret = nvkm_falcon_dma_wr(falcon, fw->fw.img, fw->fw.phys, fw->imem_base_img, + IMEM, fw->imem_base, fw->imem_size, true); + if (ret) + return ret; + + ret = nvkm_falcon_dma_wr(falcon, fw->fw.img, fw->fw.phys, fw->dmem_base_img, + DMEM, fw->dmem_base, fw->dmem_size, false); + if (ret) + return ret; + + return 0; +} + +const struct nvkm_falcon_fw_func +ga102_flcn_fw = { + .signature = ga100_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .load = ga102_flcn_fw_load, + .boot = ga102_flcn_fw_boot, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c index af53cbbc632ca..6990890a760e6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -171,8 +171,15 @@ gm200_flcn_disable(struct nvkm_falcon *falcon) nvkm_falcon_mask(falcon, 0x048, 0x00000003, 0x00000000); nvkm_falcon_wr32(falcon, 0x014, 0xffffffff); - if (falcon->func->reset_pmc) + if (falcon->func->reset_pmc) { + if (falcon->func->reset_prep) { + ret = falcon->func->reset_prep(falcon); + if (ret) + return ret; + } + nvkm_mc_disable(device, falcon->owner->type, falcon->owner->inst); + } if (falcon->func->reset_eng) { ret = falcon->func->reset_eng(falcon); diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c index c70beacb8d30b..c774935f3077b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gp102.c @@ -66,6 +66,14 @@ gp102_flcn_emem_pio = { int gp102_flcn_reset_eng(struct nvkm_falcon *falcon) { + int ret; + + if (falcon->func->reset_prep) { + ret = falcon->func->reset_prep(falcon); + if (ret) + return ret; + } + nvkm_falcon_mask(falcon, 0x3c0, 0x00000001, 0x00000001); udelay(10); nvkm_falcon_mask(falcon, 0x3c0, 0x00000001, 0x00000000); diff --git a/drivers/gpu/drm/nouveau/nvkm/nvfw/hs.c b/drivers/gpu/drm/nouveau/nvkm/nvfw/hs.c index 04ed77cb2ebab..a7e0583401d0e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/nvfw/hs.c +++ b/drivers/gpu/drm/nouveau/nvkm/nvfw/hs.c @@ -38,6 +38,24 @@ nvfw_hs_header(struct nvkm_subdev *subdev, const void *data) return hdr; } +const struct nvfw_hs_header_v2 * +nvfw_hs_header_v2(struct nvkm_subdev *subdev, const void *data) +{ + const struct nvfw_hs_header_v2 *hdr = data; + + nvkm_debug(subdev, "hsHeader:\n"); + nvkm_debug(subdev, "\tsigProdOffset : 0x%x\n", hdr->sig_prod_offset); + nvkm_debug(subdev, "\tsigProdSize : 0x%x\n", hdr->sig_prod_size); + nvkm_debug(subdev, "\tpatchLoc : 0x%x\n", hdr->patch_loc); + nvkm_debug(subdev, "\tpatchSig : 0x%x\n", hdr->patch_sig); + nvkm_debug(subdev, "\tmetadataOffset : 0x%x\n", hdr->meta_data_offset); + nvkm_debug(subdev, "\tmetadataSize : 0x%x\n", hdr->meta_data_size); + nvkm_debug(subdev, "\tnumSig : 0x%x\n", hdr->num_sig); + nvkm_debug(subdev, "\theaderOffset : 0x%x\n", hdr->header_offset); + nvkm_debug(subdev, "\theaderSize : 0x%x\n", hdr->header_size); + return hdr; +} + const struct nvfw_hs_load_header * nvfw_hs_load_header(struct nvkm_subdev *subdev, const void *data) { @@ -60,3 +78,24 @@ nvfw_hs_load_header(struct nvkm_subdev *subdev, const void *data) return hdr; } + +const struct nvfw_hs_load_header_v2 * +nvfw_hs_load_header_v2(struct nvkm_subdev *subdev, const void *data) +{ + const struct nvfw_hs_load_header_v2 *hdr = data; + int i; + + nvkm_debug(subdev, "hsLoadHeader:\n"); + nvkm_debug(subdev, "\tosCodeOffset : 0x%x\n", hdr->os_code_offset); + nvkm_debug(subdev, "\tosCodeSize : 0x%x\n", hdr->os_code_size); + nvkm_debug(subdev, "\tosDataOffset : 0x%x\n", hdr->os_data_offset); + nvkm_debug(subdev, "\tosDataSize : 0x%x\n", hdr->os_data_size); + nvkm_debug(subdev, "\tnumApps : 0x%x\n", hdr->num_apps); + for (i = 0; i < hdr->num_apps; i++) { + nvkm_debug(subdev, + "\tApp[%d] : offset 0x%x size 0x%x\n", i, + hdr->app[i].offset, hdr->app[i].size); + } + + return hdr; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c index 52435c0a485c7..8b7c8ea5e8a5f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ga102.c @@ -22,6 +22,30 @@ #include "gf100.h" #include "ram.h" +#include + +static int +ga102_fb_vpr_scrub(struct nvkm_fb *fb) +{ + struct nvkm_falcon_fw fw = {}; + int ret; + + ret = nvkm_falcon_fw_ctor_hs_v2(&ga102_flcn_fw, "mem-unlock", &fb->subdev, "nvdec/scrubber", + 0, &fb->subdev.device->nvdec[0]->falcon, &fw); + if (ret) + return ret; + + ret = nvkm_falcon_fw_boot(&fw, &fb->subdev, true, NULL, NULL, 0, 0); + nvkm_falcon_fw_dtor(&fw); + return ret; +} + +static bool +ga102_fb_vpr_scrub_required(struct nvkm_fb *fb) +{ + return (nvkm_rd32(fb->subdev.device, 0x1fa80c) & 0x00000010) != 0; +} + static const struct nvkm_fb_func ga102_fb = { .dtor = gf100_fb_dtor, @@ -32,6 +56,8 @@ ga102_fb = { .sysmem.flush_page_init = gf100_fb_sysmem_flush_page_init, .ram_new = ga102_ram_new, .default_bigpage = 16, + .vpr.scrub_required = ga102_fb_vpr_scrub_required, + .vpr.scrub = ga102_fb_vpr_scrub, }; int @@ -39,3 +65,9 @@ ga102_fb_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, s { return gp102_fb_new_(&ga102_fb, device, type, inst, pfb); } + +MODULE_FIRMWARE("nvidia/ga102/nvdec/scrubber.bin"); +MODULE_FIRMWARE("nvidia/ga103/nvdec/scrubber.bin"); +MODULE_FIRMWARE("nvidia/ga104/nvdec/scrubber.bin"); +MODULE_FIRMWARE("nvidia/ga106/nvdec/scrubber.bin"); +MODULE_FIRMWARE("nvidia/ga107/nvdec/scrubber.bin"); -- GitLab From 4b569ded09fdadb0c14f797c8dae4e8bc4bbad9f Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:33 +1000 Subject: [PATCH 1383/1927] drm/nouveau/acr/ga102: initial support v2. fixup for ga103 early merge Signed-off-by: Ben Skeggs Signed-off-by: Gourav Samaiya --- drivers/gpu/drm/nouveau/include/nvfw/acr.h | 85 +++++ drivers/gpu/drm/nouveau/include/nvfw/ls.h | 51 +++ drivers/gpu/drm/nouveau/include/nvfw/sec2.h | 40 +++ .../drm/nouveau/include/nvkm/core/falcon.h | 1 + .../drm/nouveau/include/nvkm/engine/falcon.h | 1 + .../drm/nouveau/include/nvkm/engine/sec2.h | 1 + .../gpu/drm/nouveau/include/nvkm/subdev/acr.h | 17 + .../gpu/drm/nouveau/include/nvkm/subdev/gsp.h | 1 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 15 + .../gpu/drm/nouveau/nvkm/engine/sec2/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/engine/sec2/base.c | 6 + .../gpu/drm/nouveau/nvkm/engine/sec2/ga102.c | 197 +++++++++++ .../gpu/drm/nouveau/nvkm/engine/sec2/gp102.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/sec2/priv.h | 4 + drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c | 26 +- drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c | 12 + drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c | 41 +++ drivers/gpu/drm/nouveau/nvkm/nvfw/ls.c | 72 ++++ .../gpu/drm/nouveau/nvkm/subdev/acr/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/subdev/acr/ga100.c | 49 +++ .../gpu/drm/nouveau/nvkm/subdev/acr/ga102.c | 326 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/subdev/acr/gm200.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/acr/gp102.c | 4 +- .../gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c | 70 ++++ .../gpu/drm/nouveau/nvkm/subdev/acr/priv.h | 13 +- .../gpu/drm/nouveau/nvkm/subdev/acr/tu102.c | 2 +- .../gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c | 59 ++++ 28 files changed, 1091 insertions(+), 14 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/sec2/ga102.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga100.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga102.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c diff --git a/drivers/gpu/drm/nouveau/include/nvfw/acr.h b/drivers/gpu/drm/nouveau/include/nvfw/acr.h index e65d6a8db104b..6f19560bc54b1 100644 --- a/drivers/gpu/drm/nouveau/include/nvfw/acr.h +++ b/drivers/gpu/drm/nouveau/include/nvfw/acr.h @@ -39,6 +39,23 @@ struct wpr_header_v1 { void wpr_header_v1_dump(struct nvkm_subdev *, const struct wpr_header_v1 *); +struct wpr_generic_header { +#define WPR_GENERIC_HEADER_ID_LSF_UCODE_DESC 1 +#define WPR_GENERIC_HEADER_ID_LSF_WPR_HEADER 2 +#define WPR_GENERIC_HEADER_ID_LSF_SHARED_SUB_WPR 3 +#define WPR_GENERIC_HEADER_ID_LSF_LSB_HEADER 4 + u16 identifier; + u16 version; + u32 size; +}; + +struct wpr_header_v2 { + struct wpr_generic_header hdr; + struct wpr_header_v1 wpr; +}; + +void wpr_header_v2_dump(struct nvkm_subdev *, const struct wpr_header_v2 *); + struct lsf_signature { u8 prd_keys[2][16]; u8 dbg_keys[2][16]; @@ -89,6 +106,74 @@ struct lsb_header_v1 { void lsb_header_v1_dump(struct nvkm_subdev *, struct lsb_header_v1 *); +struct lsb_header_v2 { + struct wpr_generic_header hdr; + struct lsf_signature_v2 { + struct wpr_generic_header hdr; + u32 falcon_id; + u8 prd_present; + u8 dbg_present; + u16 reserved; + u32 sig_size; + u8 prod_sig[2][384 + 128]; + u8 debug_sig[2][384 + 128]; + u16 sig_algo_ver; + u16 sig_algo; + u16 hash_algo_ver; + u16 hash_algo; + u32 sig_algo_padding_type; + u8 depmap[11 * 2 * 4]; + u32 depmap_count; + u8 supports_versioning; + u8 pad[3]; + u32 ls_ucode_version; + u32 ls_ucode_id; + u32 ucode_ls_encrypted; + u32 ls_eng_algo_type; + u32 ls_eng_algo_ver; + u8 ls_enc_iv[16]; + u8 rsvd[36]; + } signature; + u32 ucode_off; + u32 ucode_size; + u32 data_size; + u32 bl_code_size; + u32 bl_imem_off; + u32 bl_data_off; + u32 bl_data_size; + u32 rsvd0; + u32 app_code_off; + u32 app_code_size; + u32 app_data_off; + u32 app_data_size; + u32 app_imem_offset; + u32 app_dmem_offset; + u32 flags; + u32 monitor_code_offset; + u32 monitor_data_offset; + u32 manifest_offset; + struct hs_fmc_params { + u8 hs_fmc; + u8 padding[3]; + u16 pkc_algo; + u16 pkc_algo_version; + u32 engid_mask; + u32 ucode_id; + u32 fuse_ver; + u8 pkc_signature[384 + 128]; + u8 pkc_key[2048]; + u8 rsvd[4]; + } hs_fmc_params; + struct hs_ovl_sig_blob_params { + u8 hs_ovl_sig_blob_present; + u32 hs_ovl_sig_blob_offset; + u32 hs_ovl_sig_blob_size; + } hs_ovl_sig_blob_params; + u8 rsvd[20]; +}; + +void lsb_header_v2_dump(struct nvkm_subdev *, struct lsb_header_v2 *); + struct flcn_acr_desc { union { u8 reserved_dmem[0x200]; diff --git a/drivers/gpu/drm/nouveau/include/nvfw/ls.h b/drivers/gpu/drm/nouveau/include/nvfw/ls.h index f63692a2a16cf..d531121bfa35f 100644 --- a/drivers/gpu/drm/nouveau/include/nvfw/ls.h +++ b/drivers/gpu/drm/nouveau/include/nvfw/ls.h @@ -50,4 +50,55 @@ struct nvfw_ls_desc_v1 { const struct nvfw_ls_desc_v1 * nvfw_ls_desc_v1(struct nvkm_subdev *, const void *); + +struct nvfw_ls_desc_v2 { + u32 descriptor_size; + u32 image_size; + u32 tools_version; + u32 app_version; + char date[64]; + u32 secure_bootloader; + u32 bootloader_start_offset; + u32 bootloader_size; + u32 bootloader_imem_offset; + u32 bootloader_entry_point; + u32 app_start_offset; + u32 app_size; + u32 app_imem_offset; + u32 app_imem_entry; + u32 app_dmem_offset; + u32 app_resident_code_offset; + u32 app_resident_code_size; + u32 app_resident_data_offset; + u32 app_resident_data_size; + u32 nb_imem_overlays; + u32 nb_dmem_overlays; + struct { + u32 start; + u32 size; + } load_ovl[64]; +}; + +const struct nvfw_ls_desc_v2 *nvfw_ls_desc_v2(struct nvkm_subdev *, const void *); + +struct nvfw_ls_hsbl_bin_hdr { + u32 bin_magic; + u32 bin_ver; + u32 bin_size; + u32 header_offset; +}; + +const struct nvfw_ls_hsbl_bin_hdr *nvfw_ls_hsbl_bin_hdr(struct nvkm_subdev *, const void *); + +struct nvfw_ls_hsbl_hdr { + u32 sig_prod_offset; + u32 sig_prod_size; + u32 patch_loc; + u32 patch_sig; + u32 meta_data_offset; + u32 meta_data_size; + u32 num_sig; +}; + +const struct nvfw_ls_hsbl_hdr *nvfw_ls_hsbl_hdr(struct nvkm_subdev *, const void *); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvfw/sec2.h b/drivers/gpu/drm/nouveau/include/nvfw/sec2.h index 07d7094d8ed85..b3331d679c4ef 100644 --- a/drivers/gpu/drm/nouveau/include/nvfw/sec2.h +++ b/drivers/gpu/drm/nouveau/include/nvfw/sec2.h @@ -34,6 +34,29 @@ struct nv_sec2_init_msg { u16 sw_managed_area_size; }; +struct nv_sec2_init_msg_v1 { + struct nvfw_falcon_msg hdr; +#define NV_SEC2_INIT_MSG_INIT 0x00 + u8 msg_type; + + u8 num_queues; + u16 os_debug_entry_point; + + struct { + u32 offset; + u16 size; + u8 index; +#define NV_SEC2_INIT_MSG_QUEUE_ID_CMDQ 0x00 +#define NV_SEC2_INIT_MSG_QUEUE_ID_MSGQ 0x01 + u8 id; + } queue_info[2]; + + u32 sw_managed_area_offset; + u16 sw_managed_area_size; + + u32 unkn[8]; +}; + struct nv_sec2_acr_cmd { struct nvfw_falcon_cmd hdr; #define NV_SEC2_ACR_CMD_BOOTSTRAP_FALCON 0x00 @@ -62,4 +85,21 @@ struct nv_sec2_acr_bootstrap_falcon_msg { #define NV_SEC2_UNIT_V2_INIT 0x01 #define NV_SEC2_UNIT_V2_UNLOAD 0x05 #define NV_SEC2_UNIT_V2_ACR 0x07 + +struct nv_sec2_acr_bootstrap_falcon_cmd_v1 { + struct nv_sec2_acr_cmd cmd; +#define NV_SEC2_ACR_BOOTSTRAP_FALCON_FLAGS_RESET_YES 0x00000000 +#define NV_SEC2_ACR_BOOTSTRAP_FALCON_FLAGS_RESET_NO 0x00000001 + u32 flags; + u32 falcon_id; + u32 unkn08; + u32 unkn0c; +}; + +struct nv_sec2_acr_bootstrap_falcon_msg_v1 { + struct nv_sec2_acr_msg msg; + u32 error_code; + u32 falcon_id; + u32 unkn08; +}; #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h index 45d70aa4fd6d3..b857cf142c4a3 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/core/falcon.h @@ -61,6 +61,7 @@ void gm200_flcn_tracepc(struct nvkm_falcon *); int gp102_flcn_reset_eng(struct nvkm_falcon *); extern const struct nvkm_falcon_func_pio gp102_flcn_emem_pio; +int ga102_flcn_select(struct nvkm_falcon *); int ga102_flcn_reset_prep(struct nvkm_falcon *); int ga102_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *); extern const struct nvkm_falcon_func_dma ga102_flcn_dma; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h index dacbd92edcd56..cd86d9198e4ac 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/falcon.h @@ -60,6 +60,7 @@ int nvkm_falcon_new_(const struct nvkm_falcon_func *, struct nvkm_device *, struct nvkm_falcon_func { int (*disable)(struct nvkm_falcon *); int (*enable)(struct nvkm_falcon *); + int (*select)(struct nvkm_falcon *); u32 addr2; bool reset_pmc; int (*reset_eng)(struct nvkm_falcon *); diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h index a41b86426f80b..8d48fb20fa549 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/sec2.h @@ -23,4 +23,5 @@ struct nvkm_sec2 { int gp102_sec2_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_sec2 **); int gp108_sec2_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_sec2 **); int tu102_sec2_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_sec2 **); +int ga102_sec2_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_sec2 **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h index 12c0b8395262d..4c1f81ee1cef9 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h @@ -67,6 +67,7 @@ int gp108_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gp10b_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); int gv100_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); int tu102_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); +int ga102_acr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); struct nvkm_acr_lsfw { const struct nvkm_acr_lsf_func *func; @@ -79,6 +80,7 @@ struct nvkm_acr_lsfw { const struct firmware *sig; + bool secure_bootloader; u32 bootloader_size; u32 bootloader_imem_offset; @@ -89,10 +91,19 @@ struct nvkm_acr_lsfw { u32 app_resident_code_size; u32 app_resident_data_offset; u32 app_resident_data_size; + u32 app_imem_offset; + u32 app_dmem_offset; u32 ucode_size; u32 data_size; + u32 fuse_ver; + u32 engine_id; + u32 ucode_id; + u32 sig_size; + u32 sig_nr; + u8 *sigs; + struct { u32 lsb; u32 img; @@ -123,6 +134,12 @@ int nvkm_acr_lsfw_load_sig_image_desc_v1(struct nvkm_subdev *, struct nvkm_falcon *, enum nvkm_acr_lsf_id, const char *path, int ver, const struct nvkm_acr_lsf_func *); + +int +nvkm_acr_lsfw_load_sig_image_desc_v2(struct nvkm_subdev *, struct nvkm_falcon *, + enum nvkm_acr_lsf_id, const char *path, + int ver, const struct nvkm_acr_lsf_func *); + int nvkm_acr_lsfw_load_bl_inst_data_sig(struct nvkm_subdev *, struct nvkm_falcon *, enum nvkm_acr_lsf_id, const char *path, diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h index ccf886ec1f84a..72619d7df73e7 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/gsp.h @@ -12,4 +12,5 @@ struct nvkm_gsp { }; int gv100_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); +int ga102_gsp_new(struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gsp **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 8162efcf2dd64..666300dc0bf04 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2596,12 +2596,14 @@ nv170_chipset = { static const struct nvkm_device_chip nv172_chipset = { .name = "GA102", + .acr = { 0x00000001, ga102_acr_new }, .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, + .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .mc = { 0x00000001, ga100_mc_new }, @@ -2616,17 +2618,20 @@ nv172_chipset = { .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, + .sec2 = { 0x00000001, ga102_sec2_new }, }; static const struct nvkm_device_chip nv173_chipset = { .name = "GA103", + .acr = { 0x00000001, ga102_acr_new }, .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, + .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .mc = { 0x00000001, ga100_mc_new }, @@ -2641,17 +2646,20 @@ nv173_chipset = { .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, + .sec2 = { 0x00000001, ga102_sec2_new }, }; static const struct nvkm_device_chip nv174_chipset = { .name = "GA104", + .acr = { 0x00000001, ga102_acr_new }, .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, + .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .mc = { 0x00000001, ga100_mc_new }, @@ -2666,17 +2674,20 @@ nv174_chipset = { .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, + .sec2 = { 0x00000001, ga102_sec2_new }, }; static const struct nvkm_device_chip nv176_chipset = { .name = "GA106", + .acr = { 0x00000001, ga102_acr_new }, .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, + .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .mc = { 0x00000001, ga100_mc_new }, @@ -2691,17 +2702,20 @@ nv176_chipset = { .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, + .sec2 = { 0x00000001, ga102_sec2_new }, }; static const struct nvkm_device_chip nv177_chipset = { .name = "GA107", + .acr = { 0x00000001, ga102_acr_new }, .bar = { 0x00000001, tu102_bar_new }, .bios = { 0x00000001, nvkm_bios_new }, .devinit = { 0x00000001, ga100_devinit_new }, .fault = { 0x00000001, tu102_fault_new }, .fb = { 0x00000001, ga102_fb_new }, .gpio = { 0x00000001, ga102_gpio_new }, + .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, .mc = { 0x00000001, ga100_mc_new }, @@ -2716,6 +2730,7 @@ nv177_chipset = { .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, + .sec2 = { 0x00000001, ga102_sec2_new }, }; struct nvkm_subdev * diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild index 63cd2be3de08f..19feadb1f67b4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/Kbuild @@ -3,3 +3,4 @@ nvkm-y += nvkm/engine/sec2/base.o nvkm-y += nvkm/engine/sec2/gp102.o nvkm-y += nvkm/engine/sec2/gp108.o nvkm-y += nvkm/engine/sec2/tu102.o +nvkm-y += nvkm/engine/sec2/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c index b1e5f543c99d1..f2c60da5d1e86 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/base.c @@ -100,6 +100,12 @@ nvkm_sec2_oneinit(struct nvkm_engine *engine) struct nvkm_intr *intr = &sec2->engine.subdev.device->mc->intr; enum nvkm_intr_type type = NVKM_INTR_SUBDEV; + if (sec2->func->intr_vector) { + intr = sec2->func->intr_vector(sec2, &type); + if (IS_ERR(intr)) + return PTR_ERR(intr); + } + return nvkm_inth_add(intr, type, NVKM_INTR_PRIO_NORMAL, subdev, sec2->func->intr, &subdev->inth); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/ga102.c new file mode 100644 index 0000000000000..945abb8156d72 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/ga102.c @@ -0,0 +1,197 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" +#include +#include + +#include +#include + +static int +ga102_sec2_initmsg(struct nvkm_sec2 *sec2) +{ + struct nv_sec2_init_msg_v1 msg; + int ret, i; + + ret = nvkm_falcon_msgq_recv_initmsg(sec2->msgq, &msg, sizeof(msg)); + if (ret) + return ret; + + if (msg.hdr.unit_id != NV_SEC2_UNIT_INIT || + msg.msg_type != NV_SEC2_INIT_MSG_INIT) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(msg.queue_info); i++) { + if (msg.queue_info[i].id == NV_SEC2_INIT_MSG_QUEUE_ID_MSGQ) { + nvkm_falcon_msgq_init(sec2->msgq, msg.queue_info[i].index, + msg.queue_info[i].offset, + msg.queue_info[i].size); + } else { + nvkm_falcon_cmdq_init(sec2->cmdq, msg.queue_info[i].index, + msg.queue_info[i].offset, + msg.queue_info[i].size); + } + } + + return 0; +} + +static struct nvkm_intr * +ga102_sec2_intr_vector(struct nvkm_sec2 *sec2, enum nvkm_intr_type *pvector) +{ + struct nvkm_device *device = sec2->engine.subdev.device; + struct nvkm_falcon *falcon = &sec2->falcon; + int ret; + + ret = ga102_flcn_select(falcon); + if (ret) + return ERR_PTR(ret); + + *pvector = nvkm_rd32(device, 0x8403e0) & 0x000000ff; + return &device->vfn->intr; +} + +static int +ga102_sec2_acr_bootstrap_falcon_callback(void *priv, struct nvfw_falcon_msg *hdr) +{ + struct nv_sec2_acr_bootstrap_falcon_msg_v1 *msg = + container_of(hdr, typeof(*msg), msg.hdr); + struct nvkm_subdev *subdev = priv; + const char *name = nvkm_acr_lsf_id(msg->falcon_id); + + if (msg->error_code) { + nvkm_error(subdev, "ACR_BOOTSTRAP_FALCON failed for falcon %d [%s]: %08x %08x\n", + msg->falcon_id, name, msg->error_code, msg->unkn08); + return -EINVAL; + } + + nvkm_debug(subdev, "%s booted\n", name); + return 0; +} + +static int +ga102_sec2_acr_bootstrap_falcon(struct nvkm_falcon *falcon, enum nvkm_acr_lsf_id id) +{ + struct nvkm_sec2 *sec2 = container_of(falcon, typeof(*sec2), falcon); + struct nv_sec2_acr_bootstrap_falcon_cmd_v1 cmd = { + .cmd.hdr.unit_id = sec2->func->unit_acr, + .cmd.hdr.size = sizeof(cmd), + .cmd.cmd_type = NV_SEC2_ACR_CMD_BOOTSTRAP_FALCON, + .flags = NV_SEC2_ACR_BOOTSTRAP_FALCON_FLAGS_RESET_YES, + .falcon_id = id, + }; + + return nvkm_falcon_cmdq_send(sec2->cmdq, &cmd.cmd.hdr, + ga102_sec2_acr_bootstrap_falcon_callback, + &sec2->engine.subdev, + msecs_to_jiffies(1000)); +} + +static const struct nvkm_acr_lsf_func +ga102_sec2_acr_0 = { + .bld_size = sizeof(struct flcn_bl_dmem_desc_v2), + .bld_write = gp102_sec2_acr_bld_write_1, + .bld_patch = gp102_sec2_acr_bld_patch_1, + .bootstrap_falcons = BIT_ULL(NVKM_ACR_LSF_FECS) | + BIT_ULL(NVKM_ACR_LSF_GPCCS) | + BIT_ULL(NVKM_ACR_LSF_SEC2), + .bootstrap_falcon = ga102_sec2_acr_bootstrap_falcon, +}; + +static const struct nvkm_falcon_func +ga102_sec2_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .select = ga102_flcn_select, + .addr2 = 0x1000, + .reset_pmc = true, + .reset_eng = gp102_flcn_reset_eng, + .reset_prep = ga102_flcn_reset_prep, + .reset_wait_mem_scrubbing = ga102_flcn_reset_wait_mem_scrubbing, + .imem_dma = &ga102_flcn_dma, + .dmem_pio = &gm200_flcn_dmem_pio, + .dmem_dma = &ga102_flcn_dma, + .emem_addr = 0x01000000, + .emem_pio = &gp102_flcn_emem_pio, + .start = nvkm_falcon_v1_start, + .cmdq = { 0xc00, 0xc04, 8 }, + .msgq = { 0xc80, 0xc84, 8 }, +}; + +static const struct nvkm_sec2_func +ga102_sec2 = { + .flcn = &ga102_sec2_flcn, + .intr_vector = ga102_sec2_intr_vector, + .intr = gp102_sec2_intr, + .initmsg = ga102_sec2_initmsg, + .unit_acr = NV_SEC2_UNIT_V2_ACR, + .unit_unload = NV_SEC2_UNIT_V2_UNLOAD, +}; + +MODULE_FIRMWARE("nvidia/ga102/sec2/desc.bin"); +MODULE_FIRMWARE("nvidia/ga102/sec2/image.bin"); +MODULE_FIRMWARE("nvidia/ga102/sec2/sig.bin"); +MODULE_FIRMWARE("nvidia/ga102/sec2/hs_bl_sig.bin"); + +MODULE_FIRMWARE("nvidia/ga103/sec2/desc.bin"); +MODULE_FIRMWARE("nvidia/ga103/sec2/image.bin"); +MODULE_FIRMWARE("nvidia/ga103/sec2/sig.bin"); +MODULE_FIRMWARE("nvidia/ga103/sec2/hs_bl_sig.bin"); + +MODULE_FIRMWARE("nvidia/ga104/sec2/desc.bin"); +MODULE_FIRMWARE("nvidia/ga104/sec2/image.bin"); +MODULE_FIRMWARE("nvidia/ga104/sec2/sig.bin"); +MODULE_FIRMWARE("nvidia/ga104/sec2/hs_bl_sig.bin"); + +MODULE_FIRMWARE("nvidia/ga106/sec2/desc.bin"); +MODULE_FIRMWARE("nvidia/ga106/sec2/image.bin"); +MODULE_FIRMWARE("nvidia/ga106/sec2/sig.bin"); +MODULE_FIRMWARE("nvidia/ga106/sec2/hs_bl_sig.bin"); + +MODULE_FIRMWARE("nvidia/ga107/sec2/desc.bin"); +MODULE_FIRMWARE("nvidia/ga107/sec2/image.bin"); +MODULE_FIRMWARE("nvidia/ga107/sec2/sig.bin"); +MODULE_FIRMWARE("nvidia/ga107/sec2/hs_bl_sig.bin"); + +static int +ga102_sec2_load(struct nvkm_sec2 *sec2, int ver, + const struct nvkm_sec2_fwif *fwif) +{ + return nvkm_acr_lsfw_load_sig_image_desc_v2(&sec2->engine.subdev, &sec2->falcon, + NVKM_ACR_LSF_SEC2, "sec2/", ver, fwif->acr); +} + +static const struct nvkm_sec2_fwif +ga102_sec2_fwif[] = { + { 0, ga102_sec2_load, &ga102_sec2, &ga102_sec2_acr_0 }, + { -1, gp102_sec2_nofw, &ga102_sec2 } +}; + +int +ga102_sec2_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_sec2 **psec2) +{ + /* TOP info wasn't updated on Turing to reflect the PRI + * address change for some reason. We override it here. + */ + return nvkm_sec2_new_(ga102_sec2_fwif, device, type, inst, 0x840000, psec2); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c index b1e74bc49d728..c64013d105006 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/gp102.c @@ -237,7 +237,7 @@ MODULE_FIRMWARE("nvidia/gp107/sec2/desc.bin"); MODULE_FIRMWARE("nvidia/gp107/sec2/image.bin"); MODULE_FIRMWARE("nvidia/gp107/sec2/sig.bin"); -static void +void gp102_sec2_acr_bld_patch_1(struct nvkm_acr *acr, u32 bld, s64 adjust) { struct flcn_bl_dmem_desc_v2 hdr; @@ -248,7 +248,7 @@ gp102_sec2_acr_bld_patch_1(struct nvkm_acr *acr, u32 bld, s64 adjust) flcn_bl_dmem_desc_v2_dump(&acr->subdev, &hdr); } -static void +void gp102_sec2_acr_bld_write_1(struct nvkm_acr *acr, u32 bld, struct nvkm_acr_lsfw *lsfw) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h index 4997b8903a784..172d2705c1992 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/sec2/priv.h @@ -2,11 +2,13 @@ #ifndef __NVKM_SEC2_PRIV_H__ #define __NVKM_SEC2_PRIV_H__ #include +struct nvkm_acr_lsfw; struct nvkm_sec2_func { const struct nvkm_falcon_func *flcn; u8 unit_unload; u8 unit_acr; + struct nvkm_intr *(*intr_vector)(struct nvkm_sec2 *, enum nvkm_intr_type *); irqreturn_t (*intr)(struct nvkm_inth *); int (*initmsg)(struct nvkm_sec2 *); }; @@ -25,6 +27,8 @@ int gp102_sec2_nofw(struct nvkm_sec2 *, int, const struct nvkm_sec2_fwif *); int gp102_sec2_load(struct nvkm_sec2 *, int, const struct nvkm_sec2_fwif *); extern const struct nvkm_sec2_func gp102_sec2; extern const struct nvkm_acr_lsf_func gp102_sec2_acr_1; +void gp102_sec2_acr_bld_write_1(struct nvkm_acr *, u32, struct nvkm_acr_lsfw *); +void gp102_sec2_acr_bld_patch_1(struct nvkm_acr *, u32, s64); int nvkm_sec2_new_(const struct nvkm_sec2_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int, u32 addr, struct nvkm_sec2 **); diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c b/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c index 38306f9920b43..0ff450fe35908 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/ga102.c @@ -77,14 +77,28 @@ ga102_flcn_reset_wait_mem_scrubbing(struct nvkm_falcon *falcon) int ga102_flcn_reset_prep(struct nvkm_falcon *falcon) { - const u32 addr2 = (falcon->owner->type != NVKM_ENGINE_NVDEC) ? 0x530 : 0x930; + nvkm_falcon_rd32(falcon, 0x0f4); - if (nvkm_msec(falcon->owner->device, 10, - if ((nvkm_falcon_rd32(falcon, falcon->addr2 + 0x1ec) & 0x00000003) == 0x00000001 && - (nvkm_falcon_rd32(falcon, addr2) & 0x00000008) == 0x00000008) + nvkm_usec(falcon->owner->device, 150, + if (nvkm_falcon_rd32(falcon, 0x0f4) & 0x80000000) break; - ) < 0) - return -ETIMEDOUT; + _warn = false; + ); + + return 0; +} + +int +ga102_flcn_select(struct nvkm_falcon *falcon) +{ + if ((nvkm_falcon_rd32(falcon, falcon->addr2 + 0x668) & 0x00000010) != 0x00000000) { + nvkm_falcon_wr32(falcon, falcon->addr2 + 0x668, 0x00000000); + if (nvkm_msec(falcon->owner->device, 10, + if (nvkm_falcon_rd32(falcon, falcon->addr2 + 0x668) & 0x00000001) + break; + ) < 0) + return -ETIMEDOUT; + } return 0; } diff --git a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c index 6990890a760e6..393ade9f7e6c4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/falcon/gm200.c @@ -151,6 +151,12 @@ gm200_flcn_enable(struct nvkm_falcon *falcon) return ret; } + if (falcon->func->select) { + ret = falcon->func->select(falcon); + if (ret) + return ret; + } + if (falcon->func->reset_pmc) nvkm_mc_enable(device, falcon->owner->type, falcon->owner->inst); @@ -168,6 +174,12 @@ gm200_flcn_disable(struct nvkm_falcon *falcon) struct nvkm_device *device = falcon->owner->device; int ret; + if (falcon->func->select) { + ret = falcon->func->select(falcon); + if (ret) + return ret; + } + nvkm_falcon_mask(falcon, 0x048, 0x00000003, 0x00000000); nvkm_falcon_wr32(falcon, 0x014, 0xffffffff); diff --git a/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c b/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c index bef790ad8f2f3..83a9c48bc58cb 100644 --- a/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c +++ b/drivers/gpu/drm/nouveau/nvkm/nvfw/acr.c @@ -45,6 +45,47 @@ wpr_header_v1_dump(struct nvkm_subdev *subdev, const struct wpr_header_v1 *hdr) nvkm_debug(subdev, "\tstatus : %d\n", hdr->status); } +void +wpr_generic_header_dump(struct nvkm_subdev *subdev, const struct wpr_generic_header *hdr) +{ + nvkm_debug(subdev, "wprGenericHeader\n"); + nvkm_debug(subdev, "\tidentifier : %04x\n", hdr->identifier); + nvkm_debug(subdev, "\tversion : %04x\n", hdr->version); + nvkm_debug(subdev, "\tsize : %08x\n", hdr->size); +} + +void +wpr_header_v2_dump(struct nvkm_subdev *subdev, const struct wpr_header_v2 *hdr) +{ + wpr_generic_header_dump(subdev, &hdr->hdr); + wpr_header_v1_dump(subdev, &hdr->wpr); +} + +void +lsb_header_v2_dump(struct nvkm_subdev *subdev, struct lsb_header_v2 *hdr) +{ + wpr_generic_header_dump(subdev, &hdr->hdr); + nvkm_debug(subdev, "lsbHeader\n"); + nvkm_debug(subdev, "\tucodeOff : 0x%x\n", hdr->ucode_off); + nvkm_debug(subdev, "\tucodeSize : 0x%x\n", hdr->ucode_size); + nvkm_debug(subdev, "\tdataSize : 0x%x\n", hdr->data_size); + nvkm_debug(subdev, "\tblCodeSize : 0x%x\n", hdr->bl_code_size); + nvkm_debug(subdev, "\tblImemOff : 0x%x\n", hdr->bl_imem_off); + nvkm_debug(subdev, "\tblDataOff : 0x%x\n", hdr->bl_data_off); + nvkm_debug(subdev, "\tblDataSize : 0x%x\n", hdr->bl_data_size); + nvkm_debug(subdev, "\treserved0 : %08x\n", hdr->rsvd0); + nvkm_debug(subdev, "\tappCodeOff : 0x%x\n", hdr->app_code_off); + nvkm_debug(subdev, "\tappCodeSize : 0x%x\n", hdr->app_code_size); + nvkm_debug(subdev, "\tappDataOff : 0x%x\n", hdr->app_data_off); + nvkm_debug(subdev, "\tappDataSize : 0x%x\n", hdr->app_data_size); + nvkm_debug(subdev, "\tappImemOffset : 0x%x\n", hdr->app_imem_offset); + nvkm_debug(subdev, "\tappDmemOffset : 0x%x\n", hdr->app_dmem_offset); + nvkm_debug(subdev, "\tflags : 0x%x\n", hdr->flags); + nvkm_debug(subdev, "\tmonitorCodeOff: 0x%x\n", hdr->monitor_code_offset); + nvkm_debug(subdev, "\tmonitorDataOff: 0x%x\n", hdr->monitor_data_offset); + nvkm_debug(subdev, "\tmanifestOffset: 0x%x\n", hdr->manifest_offset); +} + static void lsb_header_tail_dump(struct nvkm_subdev *subdev, struct lsb_header_tail *hdr) { diff --git a/drivers/gpu/drm/nouveau/nvkm/nvfw/ls.c b/drivers/gpu/drm/nouveau/nvkm/nvfw/ls.c index b847f281ce975..45c3a6c5e0885 100644 --- a/drivers/gpu/drm/nouveau/nvkm/nvfw/ls.c +++ b/drivers/gpu/drm/nouveau/nvkm/nvfw/ls.c @@ -106,3 +106,75 @@ nvfw_ls_desc_v1(struct nvkm_subdev *subdev, const void *data) return hdr; } + +const struct nvfw_ls_desc_v2 * +nvfw_ls_desc_v2(struct nvkm_subdev *subdev, const void *data) +{ + const struct nvfw_ls_desc_v2 *hdr = data; + char *date; + int i; + + nvkm_debug(subdev, "lsUcodeImgDesc:\n"); + nvkm_debug(subdev, "\tdescriptorSize : %d\n", hdr->descriptor_size); + nvkm_debug(subdev, "\timageSize : %d\n", hdr->image_size); + nvkm_debug(subdev, "\ttoolsVersion : 0x%x\n", hdr->tools_version); + nvkm_debug(subdev, "\tappVersion : 0x%x\n", hdr->app_version); + + date = kstrndup(hdr->date, sizeof(hdr->date), GFP_KERNEL); + nvkm_debug(subdev, "\tdate : %s\n", date); + kfree(date); + + nvkm_debug(subdev, "\tsecureBootloader : 0x%x\n", hdr->secure_bootloader); + nvkm_debug(subdev, "\tbootloaderStartOffset: 0x%x\n", hdr->bootloader_start_offset); + nvkm_debug(subdev, "\tbootloaderSize : 0x%x\n", hdr->bootloader_size); + nvkm_debug(subdev, "\tbootloaderImemOffset : 0x%x\n", hdr->bootloader_imem_offset); + nvkm_debug(subdev, "\tbootloaderEntryPoint : 0x%x\n", hdr->bootloader_entry_point); + + nvkm_debug(subdev, "\tappStartOffset : 0x%x\n", hdr->app_start_offset); + nvkm_debug(subdev, "\tappSize : 0x%x\n", hdr->app_size); + nvkm_debug(subdev, "\tappImemOffset : 0x%x\n", hdr->app_imem_offset); + nvkm_debug(subdev, "\tappImemEntry : 0x%x\n", hdr->app_imem_entry); + nvkm_debug(subdev, "\tappDmemOffset : 0x%x\n", hdr->app_dmem_offset); + nvkm_debug(subdev, "\tappResidentCodeOffset: 0x%x\n", hdr->app_resident_code_offset); + nvkm_debug(subdev, "\tappResidentCodeSize : 0x%x\n", hdr->app_resident_code_size); + nvkm_debug(subdev, "\tappResidentDataOffset: 0x%x\n", hdr->app_resident_data_offset); + nvkm_debug(subdev, "\tappResidentDataSize : 0x%x\n", hdr->app_resident_data_size); + + nvkm_debug(subdev, "\tnbImemOverlays : %d\n", hdr->nb_imem_overlays); + nvkm_debug(subdev, "\tnbDmemOverlays : %d\n", hdr->nb_dmem_overlays); + for (i = 0; i < ARRAY_SIZE(hdr->load_ovl); i++) { + nvkm_debug(subdev, "\tloadOvl[%d] : 0x%x %d\n", i, + hdr->load_ovl[i].start, hdr->load_ovl[i].size); + } + + return hdr; +} + +const struct nvfw_ls_hsbl_bin_hdr * +nvfw_ls_hsbl_bin_hdr(struct nvkm_subdev *subdev, const void *data) +{ + const struct nvfw_ls_hsbl_bin_hdr *hdr = data; + + nvkm_debug(subdev, "lsHsblBinHdr:\n"); + nvkm_debug(subdev, "\tbinMagic : 0x%08x\n", hdr->bin_magic); + nvkm_debug(subdev, "\tbinVer : %d\n", hdr->bin_ver); + nvkm_debug(subdev, "\tbinSize : %d\n", hdr->bin_size); + nvkm_debug(subdev, "\theaderOffset : 0x%x\n", hdr->header_offset); + return hdr; +} + +const struct nvfw_ls_hsbl_hdr * +nvfw_ls_hsbl_hdr(struct nvkm_subdev *subdev, const void *data) +{ + const struct nvfw_ls_hsbl_hdr *hdr = data; + + nvkm_debug(subdev, "lsHsblHdr:\n"); + nvkm_debug(subdev, "\tsigProdOffset : 0x%x\n", hdr->sig_prod_offset); + nvkm_debug(subdev, "\tsigProdSize : 0x%x\n", hdr->sig_prod_size); + nvkm_debug(subdev, "\tpatchLoc : 0x%x\n", hdr->patch_loc); + nvkm_debug(subdev, "\tpatchSig : 0x%x\n", hdr->patch_sig); + nvkm_debug(subdev, "\tmetadataOffset : 0x%x\n", hdr->meta_data_offset); + nvkm_debug(subdev, "\tmetadataSize : 0x%x\n", hdr->meta_data_size); + nvkm_debug(subdev, "\tnumSig : 0x%x\n", hdr->num_sig); + return hdr; +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild index ce6ece75f07ef..5731f35b11e13 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/Kbuild @@ -8,3 +8,5 @@ nvkm-y += nvkm/subdev/acr/gp108.o nvkm-y += nvkm/subdev/acr/gv100.o nvkm-y += nvkm/subdev/acr/gp10b.o nvkm-y += nvkm/subdev/acr/tu102.o +nvkm-y += nvkm/subdev/acr/ga100.o +nvkm-y += nvkm/subdev/acr/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga100.c new file mode 100644 index 0000000000000..e3370c1551c0a --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga100.c @@ -0,0 +1,49 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +void +ga100_acr_wpr_check(struct nvkm_acr *acr, u64 *start, u64 *limit) +{ + struct nvkm_device *device = acr->subdev.device; + + *start = (u64)(nvkm_rd32(device, 0x1fa81c) & 0xffffff00) << 8; + *limit = (u64)(nvkm_rd32(device, 0x1fa820) & 0xffffff00) << 8; + *limit = *limit + 0x20000; +} + +int +ga100_acr_hsfw_ctor(struct nvkm_acr *acr, const char *bl, const char *fw, + const char *name, int ver, const struct nvkm_acr_hsf_fwif *fwif) +{ + struct nvkm_acr_hsfw *hsfw; + + if (!(hsfw = kzalloc(sizeof(*hsfw), GFP_KERNEL))) + return -ENOMEM; + + hsfw->falcon_id = fwif->falcon_id; + hsfw->boot_mbox0 = fwif->boot_mbox0; + hsfw->intr_clear = fwif->intr_clear; + list_add_tail(&hsfw->head, &acr->hsfw); + + return nvkm_falcon_fw_ctor_hs_v2(fwif->func, name, &acr->subdev, fw, ver, NULL, &hsfw->fw); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga102.c new file mode 100644 index 0000000000000..45dcf493e9729 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/ga102.c @@ -0,0 +1,326 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +#include + +static int +ga102_acr_wpr_patch(struct nvkm_acr *acr, s64 adjust) +{ + struct wpr_header_v2 hdr; + struct lsb_header_v2 *lsb; + struct nvkm_acr_lsfw *lsfw; + u32 offset = 0; + + lsb = kvmalloc(sizeof(*lsb), GFP_KERNEL); + if (!lsb) + return -ENOMEM; + + do { + nvkm_robj(acr->wpr, offset, &hdr, sizeof(hdr)); + wpr_header_v2_dump(&acr->subdev, &hdr); + + list_for_each_entry(lsfw, &acr->lsfw, head) { + if (lsfw->id != hdr.wpr.falcon_id) + continue; + + nvkm_robj(acr->wpr, hdr.wpr.lsb_offset, lsb, sizeof(*lsb)); + lsb_header_v2_dump(&acr->subdev, lsb); + + lsfw->func->bld_patch(acr, lsb->bl_data_off, adjust); + break; + } + + offset += sizeof(hdr); + } while (hdr.wpr.falcon_id != WPR_HEADER_V1_FALCON_ID_INVALID); + + kvfree(lsb); + return 0; +} + +static int +ga102_acr_wpr_build_lsb(struct nvkm_acr *acr, struct nvkm_acr_lsfw *lsfw) +{ + struct lsb_header_v2 *hdr; + int ret = 0; + + if (WARN_ON(lsfw->sig->size != sizeof(hdr->signature))) + return -EINVAL; + + hdr = kvzalloc(sizeof(*hdr), GFP_KERNEL); + if (!hdr) + return -ENOMEM; + + hdr->hdr.identifier = WPR_GENERIC_HEADER_ID_LSF_LSB_HEADER; + hdr->hdr.version = 2; + hdr->hdr.size = sizeof(*hdr); + + memcpy(&hdr->signature, lsfw->sig->data, lsfw->sig->size); + hdr->ucode_off = lsfw->offset.img; + hdr->ucode_size = lsfw->ucode_size; + hdr->data_size = lsfw->data_size; + hdr->bl_code_size = lsfw->bootloader_size; + hdr->bl_imem_off = lsfw->bootloader_imem_offset; + hdr->bl_data_off = lsfw->offset.bld; + hdr->bl_data_size = lsfw->bl_data_size; + hdr->app_code_off = lsfw->app_start_offset + lsfw->app_resident_code_offset; + hdr->app_code_size = ALIGN(lsfw->app_resident_code_size, 0x100); + hdr->app_data_off = lsfw->app_start_offset + lsfw->app_resident_data_offset; + hdr->app_data_size = ALIGN(lsfw->app_resident_data_size, 0x100); + hdr->app_imem_offset = lsfw->app_imem_offset; + hdr->app_dmem_offset = lsfw->app_dmem_offset; + hdr->flags = lsfw->func->flags; + hdr->monitor_code_offset = 0; + hdr->monitor_data_offset = 0; + hdr->manifest_offset = 0; + + if (lsfw->secure_bootloader) { + struct nvkm_falcon_fw fw = { + .fw.img = hdr->hs_fmc_params.pkc_signature, + .fw.name = "LSFW", + .func = &(const struct nvkm_falcon_fw_func) { + .signature = ga100_flcn_fw_signature, + }, + .sig_size = lsfw->sig_size, + .sig_nr = lsfw->sig_nr, + .sigs = lsfw->sigs, + .fuse_ver = lsfw->fuse_ver, + .engine_id = lsfw->engine_id, + .ucode_id = lsfw->ucode_id, + .falcon = lsfw->falcon, + + }; + + ret = nvkm_falcon_get(fw.falcon, &acr->subdev); + if (ret == 0) { + hdr->hs_fmc_params.hs_fmc = 1; + hdr->hs_fmc_params.pkc_algo = 0; + hdr->hs_fmc_params.pkc_algo_version = 1; + hdr->hs_fmc_params.engid_mask = lsfw->engine_id; + hdr->hs_fmc_params.ucode_id = lsfw->ucode_id; + hdr->hs_fmc_params.fuse_ver = lsfw->fuse_ver; + ret = nvkm_falcon_fw_patch(&fw); + nvkm_falcon_put(fw.falcon, &acr->subdev); + } + } + + nvkm_wobj(acr->wpr, lsfw->offset.lsb, hdr, sizeof(*hdr)); + kvfree(hdr); + return ret; +} + +static int +ga102_acr_wpr_build(struct nvkm_acr *acr, struct nvkm_acr_lsf *rtos) +{ + struct nvkm_acr_lsfw *lsfw; + struct wpr_header_v2 hdr; + u32 offset = 0; + int ret; + + /*XXX: shared sub-WPR headers, fill terminator for now. */ + nvkm_wo32(acr->wpr, 0x300, (2 << 16) | WPR_GENERIC_HEADER_ID_LSF_SHARED_SUB_WPR); + nvkm_wo32(acr->wpr, 0x304, 0x14); + nvkm_wo32(acr->wpr, 0x308, 0xffffffff); + nvkm_wo32(acr->wpr, 0x30c, 0); + nvkm_wo32(acr->wpr, 0x310, 0); + + /* Fill per-LSF structures. */ + list_for_each_entry(lsfw, &acr->lsfw, head) { + struct lsf_signature_v2 *sig = (void *)lsfw->sig->data; + + hdr.hdr.identifier = WPR_GENERIC_HEADER_ID_LSF_WPR_HEADER; + hdr.hdr.version = 2; + hdr.hdr.size = sizeof(hdr); + hdr.wpr.falcon_id = lsfw->id; + hdr.wpr.lsb_offset = lsfw->offset.lsb; + hdr.wpr.bootstrap_owner = NVKM_ACR_LSF_GSPLITE; + hdr.wpr.lazy_bootstrap = 1; + hdr.wpr.bin_version = sig->ls_ucode_version; + hdr.wpr.status = WPR_HEADER_V1_STATUS_COPY; + + /* Write WPR header. */ + nvkm_wobj(acr->wpr, offset, &hdr, sizeof(hdr)); + offset += sizeof(hdr); + + /* Write LSB header. */ + ret = ga102_acr_wpr_build_lsb(acr, lsfw); + if (ret) + return ret; + + /* Write ucode image. */ + nvkm_wobj(acr->wpr, lsfw->offset.img, + lsfw->img.data, + lsfw->img.size); + + /* Write bootloader data. */ + lsfw->func->bld_write(acr, lsfw->offset.bld, lsfw); + } + + /* Finalise WPR. */ + hdr.hdr.identifier = WPR_GENERIC_HEADER_ID_LSF_WPR_HEADER; + hdr.hdr.version = 2; + hdr.hdr.size = sizeof(hdr); + hdr.wpr.falcon_id = WPR_HEADER_V1_FALCON_ID_INVALID; + nvkm_wobj(acr->wpr, offset, &hdr, sizeof(hdr)); + return 0; +} + +static u32 +ga102_acr_wpr_layout(struct nvkm_acr *acr) +{ + struct nvkm_acr_lsfw *lsfw; + u32 wpr = 0; + + wpr += 21 /* MAX_LSF */ * sizeof(struct wpr_header_v2); + wpr = ALIGN(wpr, 256); + + wpr += 0x100; /* Shared sub-WPR headers. */ + + list_for_each_entry(lsfw, &acr->lsfw, head) { + wpr = ALIGN(wpr, 256); + lsfw->offset.lsb = wpr; + wpr += sizeof(struct lsb_header_v2); + + wpr = ALIGN(wpr, 4096); + lsfw->offset.img = wpr; + wpr += lsfw->img.size; + + wpr = ALIGN(wpr, 256); + lsfw->offset.bld = wpr; + lsfw->bl_data_size = ALIGN(lsfw->func->bld_size, 256); + wpr += lsfw->bl_data_size; + } + + return wpr; +} + +static int +ga102_acr_wpr_parse(struct nvkm_acr *acr) +{ + const struct wpr_header_v2 *hdr = (void *)acr->wpr_fw->data; + + while (hdr->wpr.falcon_id != WPR_HEADER_V1_FALCON_ID_INVALID) { + wpr_header_v2_dump(&acr->subdev, hdr); + if (!nvkm_acr_lsfw_add(NULL, acr, NULL, (hdr++)->wpr.falcon_id)) + return -ENOMEM; + } + + return 0; +} + +MODULE_FIRMWARE("nvidia/ga102/acr/ucode_unload.bin"); +MODULE_FIRMWARE("nvidia/ga103/acr/ucode_unload.bin"); +MODULE_FIRMWARE("nvidia/ga104/acr/ucode_unload.bin"); +MODULE_FIRMWARE("nvidia/ga106/acr/ucode_unload.bin"); +MODULE_FIRMWARE("nvidia/ga107/acr/ucode_unload.bin"); + +static const struct nvkm_acr_hsf_fwif +ga102_acr_unload_fwif[] = { + { 0, ga100_acr_hsfw_ctor, &ga102_flcn_fw, NVKM_ACR_HSF_SEC2 }, + {} +}; + +MODULE_FIRMWARE("nvidia/ga102/acr/ucode_asb.bin"); +MODULE_FIRMWARE("nvidia/ga103/acr/ucode_asb.bin"); +MODULE_FIRMWARE("nvidia/ga104/acr/ucode_asb.bin"); +MODULE_FIRMWARE("nvidia/ga106/acr/ucode_asb.bin"); +MODULE_FIRMWARE("nvidia/ga107/acr/ucode_asb.bin"); + +static const struct nvkm_acr_hsf_fwif +ga102_acr_asb_fwif[] = { + { 0, ga100_acr_hsfw_ctor, &ga102_flcn_fw, NVKM_ACR_HSF_GSP }, + {} +}; + +static const struct nvkm_falcon_fw_func +ga102_acr_ahesasc_0 = { + .signature = ga100_flcn_fw_signature, + .reset = gm200_flcn_fw_reset, + .setup = gp102_acr_load_setup, + .load = ga102_flcn_fw_load, + .boot = ga102_flcn_fw_boot, +}; + +MODULE_FIRMWARE("nvidia/ga102/acr/ucode_ahesasc.bin"); +MODULE_FIRMWARE("nvidia/ga103/acr/ucode_ahesasc.bin"); +MODULE_FIRMWARE("nvidia/ga104/acr/ucode_ahesasc.bin"); +MODULE_FIRMWARE("nvidia/ga106/acr/ucode_ahesasc.bin"); +MODULE_FIRMWARE("nvidia/ga107/acr/ucode_ahesasc.bin"); + +static const struct nvkm_acr_hsf_fwif +ga102_acr_ahesasc_fwif[] = { + { 0, ga100_acr_hsfw_ctor, &ga102_acr_ahesasc_0, NVKM_ACR_HSF_SEC2 }, + {} +}; + +static const struct nvkm_acr_func +ga102_acr = { + .ahesasc = ga102_acr_ahesasc_fwif, + .asb = ga102_acr_asb_fwif, + .unload = ga102_acr_unload_fwif, + .wpr_parse = ga102_acr_wpr_parse, + .wpr_layout = ga102_acr_wpr_layout, + .wpr_alloc = gp102_acr_wpr_alloc, + .wpr_patch = ga102_acr_wpr_patch, + .wpr_build = ga102_acr_wpr_build, + .wpr_check = ga100_acr_wpr_check, + .init = tu102_acr_init, +}; + +static int +ga102_acr_load(struct nvkm_acr *acr, int version, + const struct nvkm_acr_fwif *fwif) +{ + struct nvkm_subdev *subdev = &acr->subdev; + const struct nvkm_acr_hsf_fwif *hsfwif; + + hsfwif = nvkm_firmware_load(subdev, fwif->func->ahesasc, "AcrAHESASC", + acr, NULL, "acr/ucode_ahesasc", "AHESASC"); + if (IS_ERR(hsfwif)) + return PTR_ERR(hsfwif); + + hsfwif = nvkm_firmware_load(subdev, fwif->func->asb, "AcrASB", + acr, NULL, "acr/ucode_asb", "ASB"); + if (IS_ERR(hsfwif)) + return PTR_ERR(hsfwif); + + hsfwif = nvkm_firmware_load(subdev, fwif->func->unload, "AcrUnload", + acr, NULL, "acr/ucode_unload", "unload"); + if (IS_ERR(hsfwif)) + return PTR_ERR(hsfwif); + + return 0; +} + +static const struct nvkm_acr_fwif +ga102_acr_fwif[] = { + { 0, ga102_acr_load, &ga102_acr }, + { -1, gm200_acr_nofw, &gm200_acr }, + {} +}; + +int +ga102_acr_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_acr **pacr) +{ + return nvkm_acr_new_(ga102_acr_fwif, device, type, inst, pacr); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c index 7a11151af3bdf..31079c947758e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gm200.c @@ -61,7 +61,7 @@ gm200_acr_wpr_check(struct nvkm_acr *acr, u64 *start, u64 *limit) *limit = *limit + 0x20000; } -void +int gm200_acr_wpr_patch(struct nvkm_acr *acr, s64 adjust) { struct nvkm_subdev *subdev = &acr->subdev; @@ -86,6 +86,8 @@ gm200_acr_wpr_patch(struct nvkm_acr *acr, s64 adjust) } offset += sizeof(hdr); } while (hdr.falcon_id != WPR_HEADER_V0_FALCON_ID_INVALID); + + return 0; } void diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c index f4c2d3729feb7..084f28449e527 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/gp102.c @@ -29,7 +29,7 @@ #include #include -void +int gp102_acr_wpr_patch(struct nvkm_acr *acr, s64 adjust) { struct wpr_header_v1 hdr; @@ -54,6 +54,8 @@ gp102_acr_wpr_patch(struct nvkm_acr *acr, s64 adjust) offset += sizeof(hdr); } while (hdr.falcon_id != WPR_HEADER_V1_FALCON_ID_INVALID); + + return 0; } int diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c index 9b1cf6711ae9c..69a059c648c68 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c @@ -29,6 +29,7 @@ void nvkm_acr_lsfw_del(struct nvkm_acr_lsfw *lsfw) { nvkm_blob_dtor(&lsfw->img); + kfree(lsfw->sigs); nvkm_firmware_put(lsfw->sig); list_del(&lsfw->head); kfree(lsfw); @@ -176,6 +177,75 @@ nvkm_acr_lsfw_load_sig_image_desc_v1(struct nvkm_subdev *subdev, return 0; } +int +nvkm_acr_lsfw_load_sig_image_desc_v2(struct nvkm_subdev *subdev, + struct nvkm_falcon *falcon, + enum nvkm_acr_lsf_id id, + const char *path, int ver, + const struct nvkm_acr_lsf_func *func) +{ + const struct firmware *fw; + struct nvkm_acr_lsfw *lsfw; + const struct nvfw_ls_desc_v2 *desc; + int ret = 0; + + lsfw = nvkm_acr_lsfw_load_sig_image_desc_(subdev, falcon, id, path, ver, func, &fw); + if (IS_ERR(lsfw)) + return PTR_ERR(lsfw); + + desc = nvfw_ls_desc_v2(subdev, fw->data); + + lsfw->secure_bootloader = desc->secure_bootloader; + lsfw->bootloader_size = ALIGN(desc->bootloader_size, 256); + lsfw->bootloader_imem_offset = desc->bootloader_imem_offset; + + lsfw->app_size = ALIGN(desc->app_size, 256); + lsfw->app_start_offset = desc->app_start_offset; + lsfw->app_imem_entry = desc->app_imem_entry; + lsfw->app_resident_code_offset = desc->app_resident_code_offset; + lsfw->app_resident_code_size = desc->app_resident_code_size; + lsfw->app_resident_data_offset = desc->app_resident_data_offset; + lsfw->app_resident_data_size = desc->app_resident_data_size; + lsfw->app_imem_offset = desc->app_imem_offset; + lsfw->app_dmem_offset = desc->app_dmem_offset; + + lsfw->ucode_size = ALIGN(lsfw->app_resident_data_offset, 256) + lsfw->bootloader_size; + lsfw->data_size = lsfw->app_size + lsfw->bootloader_size - lsfw->ucode_size; + + nvkm_firmware_put(fw); + + if (lsfw->secure_bootloader) { + const struct firmware *hsbl; + const struct nvfw_ls_hsbl_bin_hdr *hdr; + const struct nvfw_ls_hsbl_hdr *hshdr; + u32 loc, sig, cnt, *meta; + + ret = nvkm_firmware_load_name(subdev, path, "hs_bl_sig", ver, &hsbl); + if (ret) + return ret; + + hdr = nvfw_ls_hsbl_bin_hdr(subdev, hsbl->data); + hshdr = nvfw_ls_hsbl_hdr(subdev, hsbl->data + hdr->header_offset); + meta = (u32 *)(hsbl->data + hshdr->meta_data_offset); + loc = *(u32 *)(hsbl->data + hshdr->patch_loc); + sig = *(u32 *)(hsbl->data + hshdr->patch_sig); + cnt = *(u32 *)(hsbl->data + hshdr->num_sig); + + lsfw->fuse_ver = meta[0]; + lsfw->engine_id = meta[1]; + lsfw->ucode_id = meta[2]; + lsfw->sig_size = hshdr->sig_prod_size / cnt; + lsfw->sig_nr = cnt; + lsfw->sigs = kmemdup(hsbl->data + hshdr->sig_prod_offset + sig, + lsfw->sig_nr * lsfw->sig_size, GFP_KERNEL); + nvkm_firmware_put(hsbl); + if (!lsfw->sigs) + ret = -ENOMEM; + } + + return ret; +} + int nvkm_acr_lsfw_load_bl_inst_data_sig(struct nvkm_subdev *subdev, struct nvkm_falcon *falcon, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h index 66a764e24bc6c..4881c8ba3880e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/priv.h @@ -24,7 +24,7 @@ struct nvkm_acr_func { u32 (*wpr_layout)(struct nvkm_acr *); int (*wpr_alloc)(struct nvkm_acr *, u32 wpr_size); int (*wpr_build)(struct nvkm_acr *, struct nvkm_acr_lsf *rtos); - void (*wpr_patch)(struct nvkm_acr *, s64 adjust); + int (*wpr_patch)(struct nvkm_acr *, s64 adjust); void (*wpr_check)(struct nvkm_acr *, u64 *start, u64 *limit); int (*init)(struct nvkm_acr *); void (*fini)(struct nvkm_acr *); @@ -35,7 +35,7 @@ extern const struct nvkm_acr_func gm200_acr; int gm200_acr_wpr_parse(struct nvkm_acr *); u32 gm200_acr_wpr_layout(struct nvkm_acr *); int gm200_acr_wpr_build(struct nvkm_acr *, struct nvkm_acr_lsf *); -void gm200_acr_wpr_patch(struct nvkm_acr *, s64); +int gm200_acr_wpr_patch(struct nvkm_acr *, s64); void gm200_acr_wpr_check(struct nvkm_acr *, u64 *, u64 *); void gm200_acr_wpr_build_lsb_tail(struct nvkm_acr_lsfw *, struct lsb_header_tail *); @@ -48,7 +48,11 @@ u32 gp102_acr_wpr_layout(struct nvkm_acr *); int gp102_acr_wpr_alloc(struct nvkm_acr *, u32 wpr_size); int gp102_acr_wpr_build(struct nvkm_acr *, struct nvkm_acr_lsf *); int gp102_acr_wpr_build_lsb(struct nvkm_acr *, struct nvkm_acr_lsfw *); -void gp102_acr_wpr_patch(struct nvkm_acr *, s64); +int gp102_acr_wpr_patch(struct nvkm_acr *, s64); + +int tu102_acr_init(struct nvkm_acr *); + +void ga100_acr_wpr_check(struct nvkm_acr *, u64 *, u64 *); struct nvkm_acr_hsfw { struct nvkm_falcon_fw fw; @@ -93,6 +97,9 @@ extern const struct nvkm_falcon_fw_func gp108_acr_load_0; extern const struct nvkm_falcon_fw_func gp108_acr_hsfw_0; int gp108_acr_hsfw_load_bld(struct nvkm_falcon_fw *); +int ga100_acr_hsfw_ctor(struct nvkm_acr *, const char *, const char *, const char *, int, + const struct nvkm_acr_hsf_fwif *); + int nvkm_acr_new_(const struct nvkm_acr_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_acr **); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c index ad45f5cb452b0..c22d551c00781 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/tu102.c @@ -29,7 +29,7 @@ #include -static int +int tu102_acr_init(struct nvkm_acr *acr) { int ret = nvkm_acr_hsfw_boot(acr, "AHESASC"); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild index 67cc3b320169a..7f61a1ed158b9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/Kbuild @@ -1,3 +1,4 @@ # SPDX-License-Identifier: MIT nvkm-y += nvkm/subdev/gsp/base.o nvkm-y += nvkm/subdev/gsp/gv100.o +nvkm-y += nvkm/subdev/gsp/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c new file mode 100644 index 0000000000000..525267412c3e1 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/ga102.c @@ -0,0 +1,59 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static const struct nvkm_falcon_func +ga102_gsp_flcn = { + .disable = gm200_flcn_disable, + .enable = gm200_flcn_enable, + .select = ga102_flcn_select, + .addr2 = 0x1000, + .reset_eng = gp102_flcn_reset_eng, + .reset_prep = ga102_flcn_reset_prep, + .reset_wait_mem_scrubbing = ga102_flcn_reset_wait_mem_scrubbing, + .imem_dma = &ga102_flcn_dma, + .dmem_dma = &ga102_flcn_dma, +}; + +static const struct nvkm_gsp_func +ga102_gsp = { + .flcn = &ga102_gsp_flcn, +}; + +static int +ga102_gsp_nofw(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif) +{ + return 0; +} + +struct nvkm_gsp_fwif +ga102_gsps[] = { + { -1, ga102_gsp_nofw, &ga102_gsp }, + {} +}; + +int +ga102_gsp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_gsp **pgsp) +{ + return nvkm_gsp_new_(ga102_gsps, device, type, inst, pgsp); +} -- GitLab From 21e938d0017d3523ef5114a8feebc32e95007983 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:34 +1000 Subject: [PATCH 1384/1927] drm/nouveau/ltc/ga102: initial support v2. fixup for ga103 early merge Signed-off-by: Ben Skeggs Reviewed-by: Lyude Paul --- .../gpu/drm/nouveau/include/nvkm/subdev/ltc.h | 3 +- .../gpu/drm/nouveau/nvkm/engine/device/base.c | 5 ++ .../gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild | 1 + .../gpu/drm/nouveau/nvkm/subdev/ltc/ga102.c | 57 +++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/subdev/ltc/ga102.c diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h index 213d5ff33cae3..64294042ec07c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h @@ -4,7 +4,7 @@ #include #include -#define NVKM_LTC_MAX_ZBC_COLOR_CNT 16 +#define NVKM_LTC_MAX_ZBC_COLOR_CNT 32 #define NVKM_LTC_MAX_ZBC_DEPTH_CNT 16 struct nvkm_ltc { @@ -44,4 +44,5 @@ int gm200_ltc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gp100_ltc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_ltc **); int gp102_ltc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_ltc **); int gp10b_ltc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_ltc **); +int ga102_ltc_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_ltc **); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 666300dc0bf04..9e4b194f24bf8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2606,6 +2606,7 @@ nv172_chipset = { .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, + .ltc = { 0x00000001, ga102_ltc_new }, .mc = { 0x00000001, ga100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, @@ -2634,6 +2635,7 @@ nv173_chipset = { .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, + .ltc = { 0x00000001, ga102_ltc_new }, .mc = { 0x00000001, ga100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, @@ -2662,6 +2664,7 @@ nv174_chipset = { .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, + .ltc = { 0x00000001, ga102_ltc_new }, .mc = { 0x00000001, ga100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, @@ -2690,6 +2693,7 @@ nv176_chipset = { .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, + .ltc = { 0x00000001, ga102_ltc_new }, .mc = { 0x00000001, ga100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, @@ -2718,6 +2722,7 @@ nv177_chipset = { .gsp = { 0x00000001, ga102_gsp_new }, .i2c = { 0x00000001, gm200_i2c_new }, .imem = { 0x00000001, nv50_instmem_new }, + .ltc = { 0x00000001, ga102_ltc_new }, .mc = { 0x00000001, ga100_mc_new }, .mmu = { 0x00000001, tu102_mmu_new }, .pci = { 0x00000001, gp100_pci_new }, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild index 728d75010847a..0d8a915d727ea 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild @@ -7,3 +7,4 @@ nvkm-y += nvkm/subdev/ltc/gm200.o nvkm-y += nvkm/subdev/ltc/gp100.o nvkm-y += nvkm/subdev/ltc/gp102.o nvkm-y += nvkm/subdev/ltc/gp10b.o +nvkm-y += nvkm/subdev/ltc/ga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/ga102.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/ga102.c new file mode 100644 index 0000000000000..159d9f8c95f34 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/ga102.c @@ -0,0 +1,57 @@ +/* + * Copyright 2021 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "priv.h" + +static void +ga102_ltc_zbc_clear_color(struct nvkm_ltc *ltc, int i, const u32 color[4]) +{ + struct nvkm_device *device = ltc->subdev.device; + + nvkm_mask(device, 0x17e338, 0x0000001f, i); + nvkm_wr32(device, 0x17e33c, color[0]); + nvkm_wr32(device, 0x17e340, color[1]); + nvkm_wr32(device, 0x17e344, color[2]); + nvkm_wr32(device, 0x17e348, color[3]); +} + +static const struct nvkm_ltc_func +ga102_ltc = { + .oneinit = gp100_ltc_oneinit, + .init = gp100_ltc_init, + .intr = gp100_ltc_intr, + .cbc_clear = gm107_ltc_cbc_clear, + .cbc_wait = gm107_ltc_cbc_wait, + .zbc_color = 31, + .zbc_depth = 16, + .zbc_clear_color = ga102_ltc_zbc_clear_color, + .zbc_clear_depth = gm107_ltc_zbc_clear_depth, + .zbc_clear_stencil = gp102_ltc_zbc_clear_stencil, + .invalidate = gf100_ltc_invalidate, + .flush = gf100_ltc_flush, +}; + +int +ga102_ltc_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_ltc **pltc) +{ + return nvkm_ltc_new_(&ga102_ltc, device, type, inst, pltc); +} -- GitLab From c4bdac754ca0bb21bad598b47dd8c2c86eee1cc1 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Wed, 1 Jun 2022 20:48:35 +1000 Subject: [PATCH 1385/1927] drm/nouveau/gr/ga102: initial support v2: - whitespace Signed-off-by: Ben Skeggs Signed-off-by: Gourav Samaiya --- drivers/gpu/drm/nouveau/include/nvif/class.h | 3 + .../gpu/drm/nouveau/include/nvkm/engine/gr.h | 1 + .../gpu/drm/nouveau/include/nvkm/subdev/acr.h | 7 + .../gpu/drm/nouveau/nvkm/engine/device/base.c | 5 + drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxga102.c | 77 ++++ .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c | 2 + .../gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h | 3 + .../gpu/drm/nouveau/nvkm/engine/gr/ga102.c | 347 ++++++++++++++++++ .../gpu/drm/nouveau/nvkm/engine/gr/gf100.c | 24 +- .../gpu/drm/nouveau/nvkm/engine/gr/gf100.h | 14 + .../gpu/drm/nouveau/nvkm/engine/gr/gp102.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/gr/gp108.c | 4 +- .../gpu/drm/nouveau/nvkm/engine/gr/tu102.c | 6 +- .../gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c | 75 ++++ 15 files changed, 565 insertions(+), 9 deletions(-) create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxga102.c create mode 100644 drivers/gpu/drm/nouveau/nvkm/engine/gr/ga102.c diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h index 15529d2c85f96..ad1e5de84e80f 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/class.h +++ b/drivers/gpu/drm/nouveau/include/nvif/class.h @@ -190,6 +190,8 @@ #define TURING_A /* cl9097.h */ 0x0000c597 +#define AMPERE_B /* cl9097.h */ 0x0000c797 + #define NV74_BSP 0x000074b0 #define GT212_MSVLD 0x000085b1 @@ -234,6 +236,7 @@ #define PASCAL_COMPUTE_B 0x0000c1c0 #define VOLTA_COMPUTE_A 0x0000c3c0 #define TURING_COMPUTE_A 0x0000c5c0 +#define AMPERE_COMPUTE_B 0x0000c7c0 #define NV74_CIPHER 0x000074c1 #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h index b28b752ffaa25..a2333cfe6955c 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h @@ -54,4 +54,5 @@ int gp108_gr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct n int gp10b_gr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_gr **); int gv100_gr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_gr **); int tu102_gr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_gr **); +int ga102_gr_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_gr **); #endif diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h index 4c1f81ee1cef9..73d2a6ae9ab22 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/acr.h @@ -118,6 +118,7 @@ struct nvkm_acr_lsf_func { #define NVKM_ACR_LSF_DMACTL_REQ_CTX 0x00000004 #define NVKM_ACR_LSF_FORCE_PRIV_LOAD 0x00000008 u32 flags; + u32 bl_entry; u32 bld_size; void (*bld_write)(struct nvkm_acr *, u32 bld, struct nvkm_acr_lsfw *); void (*bld_patch)(struct nvkm_acr *, u32 bld, s64 adjust); @@ -144,4 +145,10 @@ int nvkm_acr_lsfw_load_bl_inst_data_sig(struct nvkm_subdev *, struct nvkm_falcon *, enum nvkm_acr_lsf_id, const char *path, int ver, const struct nvkm_acr_lsf_func *); + +int +nvkm_acr_lsfw_load_bl_sig_net(struct nvkm_subdev *, struct nvkm_falcon *, + enum nvkm_acr_lsf_id, const char *path, + int ver, const struct nvkm_acr_lsf_func *, + const void *, u32, const void *, u32); #endif diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index 9e4b194f24bf8..364fea320cb37 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2618,6 +2618,7 @@ nv172_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .gr = { 0x00000001, ga102_gr_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, .sec2 = { 0x00000001, ga102_sec2_new }, }; @@ -2647,6 +2648,7 @@ nv173_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .gr = { 0x00000001, ga102_gr_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, .sec2 = { 0x00000001, ga102_sec2_new }, }; @@ -2676,6 +2678,7 @@ nv174_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .gr = { 0x00000001, ga102_gr_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, .sec2 = { 0x00000001, ga102_sec2_new }, }; @@ -2705,6 +2708,7 @@ nv176_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .gr = { 0x00000001, ga102_gr_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, .sec2 = { 0x00000001, ga102_sec2_new }, }; @@ -2734,6 +2738,7 @@ nv177_chipset = { .disp = { 0x00000001, ga102_disp_new }, .dma = { 0x00000001, gv100_dma_new }, .fifo = { 0x00000001, ga102_fifo_new }, + .gr = { 0x00000001, ga102_gr_new }, .nvdec = { 0x00000001, ga102_nvdec_new }, .sec2 = { 0x00000001, ga102_sec2_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild index 558c86fd8e82e..b5418f05ccd8b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild @@ -40,6 +40,7 @@ nvkm-y += nvkm/engine/gr/gp108.o nvkm-y += nvkm/engine/gr/gp10b.o nvkm-y += nvkm/engine/gr/gv100.o nvkm-y += nvkm/engine/gr/tu102.o +nvkm-y += nvkm/engine/gr/ga102.o nvkm-y += nvkm/engine/gr/ctxnv40.o nvkm-y += nvkm/engine/gr/ctxnv50.o @@ -63,3 +64,4 @@ nvkm-y += nvkm/engine/gr/ctxgp104.o nvkm-y += nvkm/engine/gr/ctxgp107.o nvkm-y += nvkm/engine/gr/ctxgv100.o nvkm-y += nvkm/engine/gr/ctxtu102.o +nvkm-y += nvkm/engine/gr/ctxga102.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxga102.c new file mode 100644 index 0000000000000..11461adf50360 --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxga102.c @@ -0,0 +1,77 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "ctxgf100.h" + +static void +ga102_grctx_generate_sm_id(struct gf100_gr *gr, int gpc, int tpc, int sm) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + tpc = gv100_gr_nonpes_aware_tpc(gr, gpc, tpc); + + nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x608), sm); +} + +static void +ga102_grctx_generate_unkn(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_mask(device, 0x41980c, 0x00000010, 0x00000010); + nvkm_mask(device, 0x41be08, 0x00000004, 0x00000004); +} + +static void +ga102_grctx_generate_r419ea8(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x419ea8, nvkm_rd32(device, 0x504728) | 0x08000000); +} + +const struct gf100_grctx_func +ga102_grctx = { + .main = gf100_grctx_generate_main, + .unkn = ga102_grctx_generate_unkn, + .bundle = gm107_grctx_generate_bundle, + .bundle_size = 0x3000, + .bundle_min_gpm_fifo_depth = 0x180, + .bundle_token_limit = 0x1140, + .pagepool = gp100_grctx_generate_pagepool, + .pagepool_size = 0x20000, + .attrib_cb_size = gp102_grctx_generate_attrib_cb_size, + .attrib_cb = gv100_grctx_generate_attrib_cb, + .attrib = gv100_grctx_generate_attrib, + .attrib_nr_max = 0x800, + .attrib_nr = 0x4a1, + .alpha_nr_max = 0xc00, + .alpha_nr = 0x800, + .unknown_size = 0x80000, + .unknown = tu102_grctx_generate_unknown, + .gfxp_nr = 0xd28, + .sm_id = ga102_grctx_generate_sm_id, + .skip_pd_num_tpc_per_gpc = true, + .rop_mapping = gv100_grctx_generate_rop_mapping, + .r406500 = gm200_grctx_generate_r406500, + .r400088 = gv100_grctx_generate_r400088, + .r419ea8 = ga102_grctx_generate_r419ea8, +}; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c index 2a31d4464a326..cb390e0134a23 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c @@ -1361,6 +1361,8 @@ gf100_grctx_generate_main(struct gf100_gr_chan *chan) if (gr->func->init_419bd8) gr->func->init_419bd8(gr); + if (grctx->r419ea8) + grctx->r419ea8(gr); gf100_gr_wait_idle(gr); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h index 46f94c1439361..00dbeda7e3464 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h @@ -69,6 +69,7 @@ struct gf100_grctx_func { void (*r419a3c)(struct gf100_gr *); void (*r408840)(struct gf100_gr *); void (*r419c0c)(struct gf100_gr *); + void (*r419ea8)(struct gf100_gr *); }; extern const struct gf100_grctx_func gf100_grctx; @@ -161,6 +162,8 @@ void gv100_grctx_generate_r400088(struct gf100_gr *, bool); void tu102_grctx_generate_unknown(struct gf100_gr_chan *, u64, u32); +extern const struct gf100_grctx_func ga102_grctx; + /* context init value lists */ extern const struct gf100_gr_pack gf100_grctx_pack_icmd[]; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ga102.c new file mode 100644 index 0000000000000..a5b5ac2755a2a --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ga102.c @@ -0,0 +1,347 @@ +/* + * Copyright 2019 Red Hat Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ +#include "gf100.h" +#include "ctxgf100.h" + +#include +#include +#include +#include + +#include + +#include + +static void +ga102_gr_zbc_clear_color(struct gf100_gr *gr, int zbc) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + u32 invalid[] = { 0, 0, 0, 0 }, *color; + + if (gr->zbc_color[zbc].format) + color = gr->zbc_color[zbc].l2; + else + color = invalid; + + nvkm_mask(device, 0x41bcb4, 0x0000001f, zbc); + nvkm_wr32(device, 0x41bcec, color[0]); + nvkm_wr32(device, 0x41bcf0, color[1]); + nvkm_wr32(device, 0x41bcf4, color[2]); + nvkm_wr32(device, 0x41bcf8, color[3]); +} + +static const struct gf100_gr_func_zbc +ga102_gr_zbc = { + .clear_color = ga102_gr_zbc_clear_color, + .clear_depth = gp100_gr_zbc_clear_depth, + .stencil_get = gp102_gr_zbc_stencil_get, + .clear_stencil = gp102_gr_zbc_clear_stencil, +}; + +static void +ga102_gr_gpccs_reset(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x41a610, 0x00000000); + nvkm_msec(device, 1, NVKM_DELAY); + nvkm_wr32(device, 0x41a610, 0x00000001); +} + +static const struct nvkm_acr_lsf_func +ga102_gr_gpccs_acr = { + .flags = NVKM_ACR_LSF_FORCE_PRIV_LOAD, + .bl_entry = 0x3400, + .bld_size = sizeof(struct flcn_bl_dmem_desc_v2), + .bld_write = gp108_gr_acr_bld_write, + .bld_patch = gp108_gr_acr_bld_patch, +}; + +static void +ga102_gr_fecs_reset(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x409614, 0x00000010); + nvkm_wr32(device, 0x41a614, 0x00000020); + nvkm_usec(device, 10, NVKM_DELAY); + nvkm_wr32(device, 0x409614, 0x00000110); + nvkm_wr32(device, 0x41a614, 0x00000a20); + nvkm_usec(device, 10, NVKM_DELAY); + nvkm_rd32(device, 0x409614); + nvkm_rd32(device, 0x41a614); +} + +static const struct nvkm_acr_lsf_func +ga102_gr_fecs_acr = { + .bl_entry = 0x7e00, + .bld_size = sizeof(struct flcn_bl_dmem_desc_v2), + .bld_write = gp108_gr_acr_bld_write, + .bld_patch = gp108_gr_acr_bld_patch, +}; + +static void +ga102_gr_init_rop_exceptions(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x41bcbc, 0x40000000); + nvkm_wr32(device, 0x41bc38, 0x40000000); + nvkm_wr32(device, 0x41ac94, nvkm_rd32(device, 0x502c94)); +} + +static void +ga102_gr_init_40a790(struct gf100_gr *gr) +{ + nvkm_wr32(gr->base.engine.subdev.device, 0x40a790, 0xc0000000); +} + +static void +ga102_gr_init_gpc_mmu(struct gf100_gr *gr) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + nvkm_wr32(device, 0x418880, nvkm_rd32(device, 0x100c80) & 0xf8001fff); + nvkm_wr32(device, 0x418894, 0x00000000); + + nvkm_wr32(device, 0x4188b4, nvkm_rd32(device, 0x100cc8)); + nvkm_wr32(device, 0x4188b8, nvkm_rd32(device, 0x100ccc)); + nvkm_wr32(device, 0x4188b0, nvkm_rd32(device, 0x100cc4)); +} + +static struct nvkm_intr * +ga102_gr_oneinit_intr(struct gf100_gr *gr, enum nvkm_intr_type *pvector) +{ + struct nvkm_device *device = gr->base.engine.subdev.device; + + *pvector = nvkm_rd32(device, 0x400154) & 0x00000fff; + return &device->vfn->intr; +} + +static const struct gf100_gr_func +ga102_gr = { + .oneinit_intr = ga102_gr_oneinit_intr, + .oneinit_tiles = gm200_gr_oneinit_tiles, + .oneinit_sm_id = gv100_gr_oneinit_sm_id, + .init = gf100_gr_init, + .init_419bd8 = gv100_gr_init_419bd8, + .init_gpc_mmu = ga102_gr_init_gpc_mmu, + .init_vsc_stream_master = gk104_gr_init_vsc_stream_master, + .init_zcull = tu102_gr_init_zcull, + .init_num_active_ltcs = gf100_gr_init_num_active_ltcs, + .init_swdx_pes_mask = gp102_gr_init_swdx_pes_mask, + .init_fs = tu102_gr_init_fs, + .init_fecs_exceptions = tu102_gr_init_fecs_exceptions, + .init_40a790 = ga102_gr_init_40a790, + .init_ds_hww_esr_2 = gm200_gr_init_ds_hww_esr_2, + .init_sked_hww_esr = gk104_gr_init_sked_hww_esr, + .init_ppc_exceptions = gk104_gr_init_ppc_exceptions, + .init_504430 = gv100_gr_init_504430, + .init_shader_exceptions = gv100_gr_init_shader_exceptions, + .init_rop_exceptions = ga102_gr_init_rop_exceptions, + .init_4188a4 = gv100_gr_init_4188a4, + .trap_mp = gv100_gr_trap_mp, + .fecs.reset = ga102_gr_fecs_reset, + .gpccs.reset = ga102_gr_gpccs_reset, + .rops = gm200_gr_rops, + .gpc_nr = 7, + .tpc_nr = 6, + .ppc_nr = 3, + .grctx = &ga102_grctx, + .zbc = &ga102_gr_zbc, + .sclass = { + { -1, -1, FERMI_TWOD_A }, + { -1, -1, KEPLER_INLINE_TO_MEMORY_B }, + { -1, -1, AMPERE_B, &gf100_fermi }, + { -1, -1, AMPERE_COMPUTE_B }, + {} + } +}; + +MODULE_FIRMWARE("nvidia/ga102/gr/fecs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga102/gr/fecs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga102/gr/gpccs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga102/gr/gpccs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga102/gr/NET_img.bin"); + +MODULE_FIRMWARE("nvidia/ga103/gr/fecs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga103/gr/fecs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga103/gr/gpccs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga103/gr/gpccs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga103/gr/NET_img.bin"); + +MODULE_FIRMWARE("nvidia/ga104/gr/fecs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga104/gr/fecs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga104/gr/gpccs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga104/gr/gpccs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga104/gr/NET_img.bin"); + +MODULE_FIRMWARE("nvidia/ga106/gr/fecs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga106/gr/fecs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga106/gr/gpccs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga106/gr/gpccs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga106/gr/NET_img.bin"); + +MODULE_FIRMWARE("nvidia/ga107/gr/fecs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga107/gr/fecs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga107/gr/gpccs_bl.bin"); +MODULE_FIRMWARE("nvidia/ga107/gr/gpccs_sig.bin"); +MODULE_FIRMWARE("nvidia/ga107/gr/NET_img.bin"); + +struct netlist_region { + u32 region_id; + u32 data_size; + u32 data_offset; +}; + +struct netlist_image_header { + u32 version; + u32 regions; +}; + +struct netlist_image { + struct netlist_image_header header; + struct netlist_region regions[]; +}; + +struct netlist_av64 { + u32 addr; + u32 data_hi; + u32 data_lo; +}; + +static int +ga102_gr_av64_to_init(struct nvkm_blob *blob, struct gf100_gr_pack **ppack) +{ + struct gf100_gr_init *init; + struct gf100_gr_pack *pack; + int nent; + int i; + + nent = (blob->size / sizeof(struct netlist_av64)); + pack = vzalloc((sizeof(*pack) * 2) + (sizeof(*init) * (nent + 1))); + if (!pack) + return -ENOMEM; + + init = (void *)(pack + 2); + pack[0].init = init; + pack[0].type = 64; + + for (i = 0; i < nent; i++) { + struct gf100_gr_init *ent = &init[i]; + struct netlist_av64 *av = &((struct netlist_av64 *)blob->data)[i]; + + ent->addr = av->addr; + ent->data = ((u64)av->data_hi << 32) | av->data_lo; + ent->count = 1; + ent->pitch = 1; + } + + *ppack = pack; + return 0; +} + +static int +ga102_gr_load(struct gf100_gr *gr, int ver, const struct gf100_gr_fwif *fwif) +{ + struct nvkm_subdev *subdev = &gr->base.engine.subdev; + const struct firmware *fw; + const struct netlist_image *net; + const struct netlist_region *fecs_inst = NULL; + const struct netlist_region *fecs_data = NULL; + const struct netlist_region *gpccs_inst = NULL; + const struct netlist_region *gpccs_data = NULL; + int ret, i; + + ret = nvkm_firmware_get(subdev, "gr/NET_img", 0, &fw); + if (ret) + return ret; + + net = (const void *)fw->data; + nvkm_debug(subdev, "netlist version %d, %d regions\n", + net->header.version, net->header.regions); + + for (i = 0; i < net->header.regions; i++) { + const struct netlist_region *reg = &net->regions[i]; + struct nvkm_blob blob = { + .data = (void *)fw->data + reg->data_offset, + .size = reg->data_size, + }; + + nvkm_debug(subdev, "\t%2d: %08x %08x\n", + reg->region_id, reg->data_offset, reg->data_size); + + switch (reg->region_id) { + case 0: fecs_data = reg; break; + case 1: fecs_inst = reg; break; + case 2: gpccs_data = reg; break; + case 3: gpccs_inst = reg; break; + case 4: gk20a_gr_av_to_init(&blob, &gr->bundle); break; + case 5: gk20a_gr_aiv_to_init(&blob, &gr->sw_ctx); break; + case 7: gk20a_gr_av_to_method(&blob, &gr->method); break; + case 28: tu102_gr_av_to_init_veid(&blob, &gr->bundle_veid); break; + case 34: ga102_gr_av64_to_init(&blob, &gr->bundle64); break; + case 48: gk20a_gr_av_to_init(&blob, &gr->sw_nonctx1); break; + case 49: gk20a_gr_av_to_init(&blob, &gr->sw_nonctx2); break; + case 50: gk20a_gr_av_to_init(&blob, &gr->sw_nonctx3); break; + case 51: gk20a_gr_av_to_init(&blob, &gr->sw_nonctx4); break; + default: + break; + } + } + + ret = nvkm_acr_lsfw_load_bl_sig_net(subdev, &gr->fecs.falcon, NVKM_ACR_LSF_FECS, + "gr/fecs_", ver, fwif->fecs, + fw->data + fecs_inst->data_offset, + fecs_inst->data_size, + fw->data + fecs_data->data_offset, + fecs_data->data_size); + if (ret) + return ret; + + ret = nvkm_acr_lsfw_load_bl_sig_net(subdev, &gr->gpccs.falcon, NVKM_ACR_LSF_GPCCS, + "gr/gpccs_", ver, fwif->gpccs, + fw->data + gpccs_inst->data_offset, + gpccs_inst->data_size, + fw->data + gpccs_data->data_offset, + gpccs_data->data_size); + if (ret) + return ret; + + gr->firmware = true; + + nvkm_firmware_put(fw); + return 0; +} + +static const struct gf100_gr_fwif +ga102_gr_fwif[] = { + { 0, ga102_gr_load, &ga102_gr, &ga102_gr_fecs_acr, &ga102_gr_gpccs_acr }, + { -1, gm200_gr_nofw }, + {} +}; + +int +ga102_gr_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_gr **pgr) +{ + return gf100_gr_new_(ga102_gr_fwif, device, type, inst, pgr); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c index b82989ef7f3b3..5f20079c3660f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c @@ -1995,6 +1995,9 @@ gf100_gr_oneinit(struct nvkm_gr *base) enum nvkm_intr_type intr_type = NVKM_INTR_SUBDEV; int ret, i, j; + if (gr->func->oneinit_intr) + intr = gr->func->oneinit_intr(gr, &intr_type); + ret = nvkm_inth_add(intr, intr_type, NVKM_INTR_PRIO_NORMAL, &gr->base.engine.subdev, gf100_gr_intr, &gr->base.engine.subdev.inth); if (ret) @@ -2147,6 +2150,10 @@ gf100_gr_dtor(struct nvkm_gr *base) vfree(gr->method); vfree(gr->sw_ctx); vfree(gr->sw_nonctx); + vfree(gr->sw_nonctx1); + vfree(gr->sw_nonctx2); + vfree(gr->sw_nonctx3); + vfree(gr->sw_nonctx4); return gr; } @@ -2324,6 +2331,8 @@ gf100_gr_reset(struct nvkm_gr *base) subdev->func->fini(subdev, false); nvkm_mc_disable(device, subdev->type, subdev->inst); + if (gr->func->gpccs.reset) + gr->func->gpccs.reset(gr); nvkm_mc_enable(device, subdev->type, subdev->inst); return subdev->func->init(subdev); @@ -2339,10 +2348,17 @@ gf100_gr_init(struct gf100_gr *gr) gr->func->init_gpc_mmu(gr); - if (gr->sw_nonctx) + if (gr->sw_nonctx1) { + gf100_gr_mmio(gr, gr->sw_nonctx1); + gf100_gr_mmio(gr, gr->sw_nonctx2); + gf100_gr_mmio(gr, gr->sw_nonctx3); + gf100_gr_mmio(gr, gr->sw_nonctx4); + } else + if (gr->sw_nonctx) { gf100_gr_mmio(gr, gr->sw_nonctx); - else + } else { gf100_gr_mmio(gr, gr->func->mmio); + } gf100_gr_wait_idle(gr); @@ -2374,6 +2390,10 @@ gf100_gr_init(struct gf100_gr *gr) nvkm_wr32(device, 0x400124, 0x00000002); gr->func->init_fecs_exceptions(gr); + + if (gr->func->init_40a790) + gr->func->init_40a790(gr); + if (gr->func->init_ds_hww_esr_2) gr->func->init_ds_hww_esr_2(gr); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h index 88fccea248e55..94ca7ac16acfd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h @@ -88,6 +88,10 @@ struct gf100_gr { * using hardcoded arrays. To be allocated with vzalloc(). */ struct gf100_gr_pack *sw_nonctx; + struct gf100_gr_pack *sw_nonctx1; + struct gf100_gr_pack *sw_nonctx2; + struct gf100_gr_pack *sw_nonctx3; + struct gf100_gr_pack *sw_nonctx4; struct gf100_gr_pack *sw_ctx; struct gf100_gr_pack *bundle; struct gf100_gr_pack *bundle_veid; @@ -143,6 +147,7 @@ struct gf100_gr_func_zbc { }; struct gf100_gr_func { + struct nvkm_intr *(*oneinit_intr)(struct gf100_gr *, enum nvkm_intr_type *); void (*oneinit_tiles)(struct gf100_gr *); int (*oneinit_sm_id)(struct gf100_gr *); int (*init)(struct gf100_gr *); @@ -158,6 +163,7 @@ struct gf100_gr_func { void (*init_swdx_pes_mask)(struct gf100_gr *); void (*init_fs)(struct gf100_gr *); void (*init_fecs_exceptions)(struct gf100_gr *); + void (*init_40a790)(struct gf100_gr *); void (*init_ds_hww_esr_2)(struct gf100_gr *); void (*init_40601c)(struct gf100_gr *); void (*init_sked_hww_esr)(struct gf100_gr *); @@ -181,6 +187,7 @@ struct gf100_gr_func { } fecs; struct { struct gf100_gr_ucode *ucode; + void (*reset)(struct gf100_gr *); } gpccs; int (*rops)(struct gf100_gr *); int gpc_nr; @@ -246,6 +253,8 @@ extern const struct gf100_gr_func_zbc gp100_gr_zbc; void gp102_gr_init_swdx_pes_mask(struct gf100_gr *); extern const struct gf100_gr_func_zbc gp102_gr_zbc; +int gp102_gr_zbc_stencil_get(struct gf100_gr *, int, const u32, const u32); +void gp102_gr_zbc_clear_stencil(struct gf100_gr *, int); extern const struct gf100_gr_func gp107_gr; @@ -258,6 +267,9 @@ void gv100_gr_init_4188a4(struct gf100_gr *); void gv100_gr_trap_mp(struct gf100_gr *, int, int); int tu102_gr_av_to_init_veid(struct nvkm_blob *, struct gf100_gr_pack **); +void tu102_gr_init_zcull(struct gf100_gr *); +void tu102_gr_init_fs(struct gf100_gr *); +void tu102_gr_init_fecs_exceptions(struct gf100_gr *); #define gf100_gr_chan(p) container_of((p), struct gf100_gr_chan, object) #include @@ -427,6 +439,8 @@ void gm20b_gr_acr_bld_patch(struct nvkm_acr *, u32, s64); extern const struct nvkm_acr_lsf_func gp108_gr_gpccs_acr; extern const struct nvkm_acr_lsf_func gp108_gr_fecs_acr; +void gp108_gr_acr_bld_write(struct nvkm_acr *, u32, struct nvkm_acr_lsfw *); +void gp108_gr_acr_bld_patch(struct nvkm_acr *, u32, s64); int gf100_gr_new_(const struct gf100_gr_fwif *, struct nvkm_device *, enum nvkm_subdev_type, int, struct nvkm_gr **); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c index f265ba08c5ef4..0e223b7b5f0e8 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp102.c @@ -26,7 +26,7 @@ #include -static void +void gp102_gr_zbc_clear_stencil(struct gf100_gr *gr, int zbc) { struct nvkm_device *device = gr->base.engine.subdev.device; @@ -40,7 +40,7 @@ gp102_gr_zbc_clear_stencil(struct gf100_gr *gr, int zbc) gr->zbc_stencil[zbc].format << ((znum % 4) * 7)); } -static int +int gp102_gr_zbc_stencil_get(struct gf100_gr *gr, int format, const u32 ds, const u32 l2) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp108.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp108.c index 7310f0466bb70..311f703439e42 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp108.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp108.c @@ -25,7 +25,7 @@ #include -static void +void gp108_gr_acr_bld_patch(struct nvkm_acr *acr, u32 bld, s64 adjust) { struct flcn_bl_dmem_desc_v2 hdr; @@ -36,7 +36,7 @@ gp108_gr_acr_bld_patch(struct nvkm_acr *acr, u32 bld, s64 adjust) flcn_bl_dmem_desc_v2_dump(&acr->subdev, &hdr); } -static void +void gp108_gr_acr_bld_write(struct nvkm_acr *acr, u32 bld, struct nvkm_acr_lsfw *lsfw) { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c index 255d50c4c65ff..3b6c8100a2428 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/tu102.c @@ -24,13 +24,13 @@ #include -static void +void tu102_gr_init_fecs_exceptions(struct gf100_gr *gr) { nvkm_wr32(gr->base.engine.subdev.device, 0x409c24, 0x006e0003); } -static void +void tu102_gr_init_fs(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; @@ -49,7 +49,7 @@ tu102_gr_init_fs(struct gf100_gr *gr) gf100_gr_init_num_tpc_per_gpc(gr, true, true); } -static void +void tu102_gr_init_zcull(struct gf100_gr *gr) { struct nvkm_device *device = gr->base.engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c index 69a059c648c68..f36a359d4531c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/lsfw.c @@ -321,3 +321,78 @@ nvkm_acr_lsfw_load_bl_inst_data_sig(struct nvkm_subdev *subdev, nvkm_firmware_put(bl); return ret; } + +int +nvkm_acr_lsfw_load_bl_sig_net(struct nvkm_subdev *subdev, + struct nvkm_falcon *falcon, + enum nvkm_acr_lsf_id id, + const char *path, int ver, + const struct nvkm_acr_lsf_func *func, + const void *inst_data, u32 inst_size, + const void *data_data, u32 data_size) +{ + struct nvkm_acr *acr = subdev->device->acr; + struct nvkm_acr_lsfw *lsfw; + const struct firmware _inst = { .data = inst_data, .size = inst_size }; + const struct firmware _data = { .data = data_data, .size = data_size }; + const struct firmware *bl = NULL, *inst = &_inst, *data = &_data; + const struct { + int bin_magic; + int bin_version; + int bin_size; + int header_offset; + int header_size; + } *hdr; + u32 *bldata; + int ret; + + if (IS_ERR((lsfw = nvkm_acr_lsfw_add(func, acr, falcon, id)))) + return PTR_ERR(lsfw); + + ret = nvkm_firmware_load_name(subdev, path, "bl", ver, &bl); + if (ret) + goto done; + + hdr = (const void *)bl->data; + bldata = (void *)(bl->data + hdr->header_offset); + + ret = nvkm_firmware_load_name(subdev, path, "sig", ver, &lsfw->sig); + if (ret) + goto done; + + lsfw->bootloader_size = ALIGN(hdr->header_size, 256); + lsfw->bootloader_imem_offset = func->bl_entry; + + lsfw->app_start_offset = lsfw->bootloader_size; + lsfw->app_imem_entry = 0; + lsfw->app_resident_code_offset = 0; + lsfw->app_resident_code_size = ALIGN(inst->size, 256); + lsfw->app_resident_data_offset = lsfw->app_resident_code_size; + lsfw->app_resident_data_size = ALIGN(data->size, 256); + lsfw->app_imem_offset = 0; + lsfw->app_dmem_offset = 0; + lsfw->app_size = lsfw->app_resident_code_size + lsfw->app_resident_data_size; + + lsfw->img.size = lsfw->bootloader_size + lsfw->app_size; + if (!(lsfw->img.data = kzalloc(lsfw->img.size, GFP_KERNEL))) { + ret = -ENOMEM; + goto done; + } + + memcpy(lsfw->img.data, bldata, lsfw->bootloader_size); + memcpy(lsfw->img.data + lsfw->app_start_offset + + lsfw->app_resident_code_offset, inst->data, inst->size); + memcpy(lsfw->img.data + lsfw->app_start_offset + + lsfw->app_resident_data_offset, data->data, data->size); + + lsfw->ucode_size = ALIGN(lsfw->app_resident_data_offset, 256) + + lsfw->bootloader_size; + lsfw->data_size = lsfw->app_size + lsfw->bootloader_size - + lsfw->ucode_size; + +done: + if (ret) + nvkm_acr_lsfw_del(lsfw); + nvkm_firmware_put(bl); + return ret; +} -- GitLab From 519b58bbfa825f042fcf80261cc18e1e35f85ffd Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Tue, 8 Nov 2022 10:56:07 +0800 Subject: [PATCH 1386/1927] net: marvell: prestera: fix memory leak in prestera_rxtx_switch_init() When prestera_sdma_switch_init() failed, the memory pointed to by sw->rxtx isn't released. Fix it. Only be compiled, not be tested. Fixes: 501ef3066c89 ("net: marvell: prestera: Add driver for Prestera family ASIC devices") Signed-off-by: Zhengchao Shao Reviewed-by: Vadym Kochan Link: https://lore.kernel.org/r/20221108025607.338450-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/marvell/prestera/prestera_rxtx.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c index 42ee963e9f759..9277a8fd1339d 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c @@ -776,6 +776,7 @@ static netdev_tx_t prestera_sdma_xmit(struct prestera_sdma *sdma, int prestera_rxtx_switch_init(struct prestera_switch *sw) { struct prestera_rxtx *rxtx; + int err; rxtx = kzalloc(sizeof(*rxtx), GFP_KERNEL); if (!rxtx) @@ -783,7 +784,11 @@ int prestera_rxtx_switch_init(struct prestera_switch *sw) sw->rxtx = rxtx; - return prestera_sdma_switch_init(sw); + err = prestera_sdma_switch_init(sw); + if (err) + kfree(rxtx); + + return err; } void prestera_rxtx_switch_fini(struct prestera_switch *sw) -- GitLab From 07d120aa33cc9d9115753d159f64d20c94458781 Mon Sep 17 00:00:00 2001 From: Eric Dumazet Date: Mon, 7 Nov 2022 18:00:11 +0000 Subject: [PATCH 1387/1927] net: tun: call napi_schedule_prep() to ensure we own a napi A recent patch exposed another issue in napi_get_frags() caught by syzbot [1] Before feeding packets to GRO, and calling napi_complete() we must first grab NAPI_STATE_SCHED. [1] WARNING: CPU: 0 PID: 3612 at net/core/dev.c:6076 napi_complete_done+0x45b/0x880 net/core/dev.c:6076 Modules linked in: CPU: 0 PID: 3612 Comm: syz-executor408 Not tainted 6.1.0-rc3-syzkaller-00175-g1118b2049d77 #0 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 10/26/2022 RIP: 0010:napi_complete_done+0x45b/0x880 net/core/dev.c:6076 Code: c1 ea 03 0f b6 14 02 4c 89 f0 83 e0 07 83 c0 03 38 d0 7c 08 84 d2 0f 85 24 04 00 00 41 89 5d 1c e9 73 fc ff ff e8 b5 53 22 fa <0f> 0b e9 82 fe ff ff e8 a9 53 22 fa 48 8b 5c 24 08 31 ff 48 89 de RSP: 0018:ffffc90003c4f920 EFLAGS: 00010293 RAX: 0000000000000000 RBX: 0000000000000030 RCX: 0000000000000000 RDX: ffff8880251c0000 RSI: ffffffff875a58db RDI: 0000000000000007 RBP: 0000000000000001 R08: 0000000000000007 R09: 0000000000000000 R10: 0000000000000001 R11: 0000000000000001 R12: ffff888072d02628 R13: ffff888072d02618 R14: ffff888072d02634 R15: 0000000000000000 FS: 0000555555f13300(0000) GS:ffff8880b9a00000(0000) knlGS:0000000000000000 CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 CR2: 000055c44d3892b8 CR3: 00000000172d2000 CR4: 00000000003506f0 DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 Call Trace: napi_complete include/linux/netdevice.h:510 [inline] tun_get_user+0x206d/0x3a60 drivers/net/tun.c:1980 tun_chr_write_iter+0xdb/0x200 drivers/net/tun.c:2027 call_write_iter include/linux/fs.h:2191 [inline] do_iter_readv_writev+0x20b/0x3b0 fs/read_write.c:735 do_iter_write+0x182/0x700 fs/read_write.c:861 vfs_writev+0x1aa/0x630 fs/read_write.c:934 do_writev+0x133/0x2f0 fs/read_write.c:977 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x35/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7f37021a3c19 Fixes: 1118b2049d77 ("net: tun: Fix memory leaks of napi_get_frags") Reported-by: syzbot Signed-off-by: Eric Dumazet Cc: Wang Yufen Link: https://lore.kernel.org/r/20221107180011.188437-1-edumazet@google.com Signed-off-by: Jakub Kicinski --- drivers/net/tun.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/net/tun.c b/drivers/net/tun.c index eb12f3136a549..7a3ab3427369a 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1967,18 +1967,25 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, skb_headlen(skb)); if (unlikely(headlen > skb_headlen(skb))) { + WARN_ON_ONCE(1); + err = -ENOMEM; dev_core_stats_rx_dropped_inc(tun->dev); +napi_busy: napi_free_frags(&tfile->napi); rcu_read_unlock(); mutex_unlock(&tfile->napi_mutex); - WARN_ON(1); - return -ENOMEM; + return err; } - local_bh_disable(); - napi_gro_frags(&tfile->napi); - napi_complete(&tfile->napi); - local_bh_enable(); + if (likely(napi_schedule_prep(&tfile->napi))) { + local_bh_disable(); + napi_gro_frags(&tfile->napi); + napi_complete(&tfile->napi); + local_bh_enable(); + } else { + err = -EBUSY; + goto napi_busy; + } mutex_unlock(&tfile->napi_mutex); } else if (tfile->napi_enabled) { struct sk_buff_head *queue = &tfile->sk.sk_write_queue; -- GitLab From b06334919c7a068d54ba5b219c05e919d89943f7 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Mon, 7 Nov 2022 18:14:43 +0800 Subject: [PATCH 1388/1927] net: nixge: disable napi when enable interrupts failed in nixge_open() When failed to enable interrupts in nixge_open() for opening device, napi isn't disabled. When open nixge device next time, it will reports a invalid opcode issue. Fix it. Only be compiled, not be tested. Fixes: 492caffa8a1a ("net: ethernet: nixge: Add support for National Instruments XGE netdev") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221107101443.120205-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ni/nixge.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c index 3db4a24317419..19d043b593ccd 100644 --- a/drivers/net/ethernet/ni/nixge.c +++ b/drivers/net/ethernet/ni/nixge.c @@ -900,6 +900,7 @@ static int nixge_open(struct net_device *ndev) err_rx_irq: free_irq(priv->tx_irq, ndev); err_tx_irq: + napi_disable(&priv->napi); phy_stop(phy); phy_disconnect(phy); tasklet_kill(&priv->dma_err_tasklet); -- GitLab From 742c60e1285ca40642e988f7e3db92232171b27d Mon Sep 17 00:00:00 2001 From: Nick Child Date: Mon, 7 Nov 2022 14:32:15 -0600 Subject: [PATCH 1389/1927] ibmveth: Reduce default tx queues to 8 Previously, the default number of transmit queues was 16. Due to resource concerns, set to 8 queues instead. Still allow the user to set more queues (max 16) if they like. Since the driver is virtualized away from the physical NIC, the purpose of multiple queues is purely to allow for parallel calls to the hypervisor. Therefore, there is no noticeable effect on performance by reducing queue count to 8. Fixes: d926793c1de9 ("ibmveth: Implement multi queue on xmit") Reported-by: Dave Taht Signed-off-by: Nick Child Link: https://lore.kernel.org/r/20221107203215.58206-1-nnac123@linux.ibm.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ibm/ibmveth.c | 3 ++- drivers/net/ethernet/ibm/ibmveth.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 3b14dc93f59dd..5b96cd94dcd24 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -1757,7 +1757,8 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id) kobject_uevent(kobj, KOBJ_ADD); } - rc = netif_set_real_num_tx_queues(netdev, ibmveth_real_max_tx_queues()); + rc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(), + IBMVETH_DEFAULT_QUEUES)); if (rc) { netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n", rc); diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h index daf6f615c03f8..115d4c45aa77d 100644 --- a/drivers/net/ethernet/ibm/ibmveth.h +++ b/drivers/net/ethernet/ibm/ibmveth.h @@ -100,6 +100,7 @@ static inline long h_illan_attributes(unsigned long unit_address, #define IBMVETH_MAX_BUF_SIZE (1024 * 128) #define IBMVETH_MAX_TX_BUF_SIZE (1024 * 64) #define IBMVETH_MAX_QUEUES 16U +#define IBMVETH_DEFAULT_QUEUES 8U static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 }; static int pool_count[] = { 256, 512, 256, 256, 256 }; -- GitLab From 8cbd4725ffff3eface1f5f3397af02acad5b2831 Mon Sep 17 00:00:00 2001 From: Jussi Laako Date: Wed, 9 Nov 2022 00:12:41 +0200 Subject: [PATCH 1390/1927] ALSA: usb-audio: Add DSD support for Accuphase DAC-60 Accuphase DAC-60 option card supports native DSD up to DSD256, but doesn't have support for auto-detection. Explicitly enable DSD support for the correct altsetting. Signed-off-by: Jussi Laako Cc: Link: https://lore.kernel.org/r/20221108221241.1220878-1-jussi@sonarnerd.net Signed-off-by: Takashi Iwai --- sound/usb/quirks.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index 2dd19a5c3026e..aedc3987a4cf2 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1875,6 +1875,7 @@ u64 snd_usb_interface_dsd_format_quirks(struct snd_usb_audio *chip, /* XMOS based USB DACs */ switch (chip->usb_id) { case USB_ID(0x1511, 0x0037): /* AURALiC VEGA */ + case USB_ID(0x21ed, 0xd75a): /* Accuphase DAC-60 option card */ case USB_ID(0x2522, 0x0012): /* LH Labs VI DAC Infinity */ case USB_ID(0x2772, 0x0230): /* Pro-Ject Pre Box S2 Digital */ if (fp->altsetting == 2) -- GitLab From fed74d75277da865da9ba334d3f5d5e3e327971d Mon Sep 17 00:00:00 2001 From: AngeloGioacchino Del Regno Date: Fri, 4 Nov 2022 11:56:05 +0100 Subject: [PATCH 1391/1927] pinctrl: mediatek: common-v2: Fix bias-disable for PULL_PU_PD_RSEL_TYPE In pinctrl-paris we're calling the .bias_set_combo() callback when we are asked to set the pin bias to either pull up/down or pull disable. On newer platforms, this callback is mtk_pinconf_bias_set_combo(), located in pinctrl-mtk-common-v2.c: this will check the "pull type" assigned to the requested pin and in case said pin's pull type is MTK_PULL_PU_PD_RSEL_TYPE, this function will set RSEL first, PUPD last, which is fine. The issue comes when we're requesting PIN_CONFIG_BIAS_DISABLE, as this does *not* require setting RSEL but only PU_PD: in this case, the arg is MTK_DISABLE (zero), which is not a supported RSEL, due to which function mtk_pinconf_bias_set_rsel() returns a failure; because of that, mtk_pinconf_bias_set_pu_pd() is never called, hence the pin bias is never set to DISABLE. To fix this issue, add a check to mtk_pinconf_bias_set_rsel(): if we are entering that function with no pullup requested and at the same time the arg is MTK_DISABLE, this means that we're trying to disable pin bias, hence it's safe to return cleanly without ever setting any RSEL register. This makes mtk_pinconf_bias_set_combo() happy, going on with setting the PU_PD registers, which is the only action to actually take to disable bias on a pin/pingroup. Fixes: fb34a9ae383a ("pinctrl: mediatek: support rsel feature") Signed-off-by: AngeloGioacchino Del Regno Link: https://lore.kernel.org/r/20221104105605.33720-1-angelogioacchino.delregno@collabora.com Signed-off-by: Linus Walleij --- drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c index e1ae3beb9f72b..b7921b59eb7b1 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common-v2.c @@ -709,6 +709,9 @@ static int mtk_pinconf_bias_set_rsel(struct mtk_pinctrl *hw, { int err, rsel_val; + if (!pullup && arg == MTK_DISABLE) + return 0; + if (hw->rsel_si_unit) { /* find pin rsel_index from pin_rsel array*/ err = mtk_hw_pin_rsel_lookup(hw, desc, pullup, arg, &rsel_val); -- GitLab From 58bb78ce02269c0cf5b1f2bd2e4a605500b44c6b Mon Sep 17 00:00:00 2001 From: Phil Sutter Date: Wed, 9 Nov 2022 10:06:04 +0100 Subject: [PATCH 1392/1927] selftests: netfilter: Fix and review rpath.sh Address a few problems with the initial test script version: * On systems with ip6tables but no ip6tables-legacy, testing for ip6tables was disabled by accident. * Firewall setup phase did not respect possibly unavailable tools. * Consistently call nft via '$nft'. Fixes: 6e31ce831c63b ("selftests: netfilter: Test reverse path filtering") Signed-off-by: Phil Sutter Signed-off-by: Pablo Neira Ayuso --- tools/testing/selftests/netfilter/rpath.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/netfilter/rpath.sh b/tools/testing/selftests/netfilter/rpath.sh index 2d8da7bd8ab74..f7311e66d2193 100755 --- a/tools/testing/selftests/netfilter/rpath.sh +++ b/tools/testing/selftests/netfilter/rpath.sh @@ -15,7 +15,7 @@ fi if ip6tables-legacy --version >/dev/null 2>&1; then ip6tables='ip6tables-legacy' -elif ! ip6tables --version >/dev/null 2>&1; then +elif ip6tables --version >/dev/null 2>&1; then ip6tables='ip6tables' else ip6tables='' @@ -62,9 +62,11 @@ ip -net "$ns1" a a fec0:42::2/64 dev v0 nodad ip -net "$ns2" a a fec0:42::1/64 dev d0 nodad # firewall matches to test -ip netns exec "$ns2" "$iptables" -t raw -A PREROUTING -s 192.168.0.0/16 -m rpfilter -ip netns exec "$ns2" "$ip6tables" -t raw -A PREROUTING -s fec0::/16 -m rpfilter -ip netns exec "$ns2" nft -f - < Date: Wed, 2 Nov 2022 09:01:06 -0700 Subject: [PATCH 1393/1927] arm64: Fix bit-shifting UB in the MIDR_CPU_MODEL() macro CONFIG_UBSAN_SHIFT with gcc-5 complains that the shifting of ARM_CPU_IMP_AMPERE (0xC0) into bits [31:24] by MIDR_CPU_MODEL() is undefined behavior. Well, sort of, it actually spells the error as: arch/arm64/kernel/proton-pack.c: In function 'spectre_bhb_loop_affected': arch/arm64/include/asm/cputype.h:44:2: error: initializer element is not constant (((imp) << MIDR_IMPLEMENTOR_SHIFT) | \ ^ This isn't an issue for other Implementor codes, as all the other codes have zero in the top bit and so are representable as a signed int. Cast the implementor code to unsigned in MIDR_CPU_MODEL to remove the undefined behavior. Fixes: 0e5d5ae837c8 ("arm64: Add AMPERE1 to the Spectre-BHB affected list") Reported-by: Geert Uytterhoeven Signed-off-by: D Scott Phillips Link: https://lore.kernel.org/r/20221102160106.1096948-1-scott@os.amperecomputing.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/cputype.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index abc418650fec0..65e53ef5a3960 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -41,7 +41,7 @@ (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) #define MIDR_CPU_MODEL(imp, partnum) \ - (((imp) << MIDR_IMPLEMENTOR_SHIFT) | \ + ((_AT(u32, imp) << MIDR_IMPLEMENTOR_SHIFT) | \ (0xf << MIDR_ARCHITECTURE_SHIFT) | \ ((partnum) << MIDR_PARTNUM_SHIFT)) -- GitLab From acfc35cfcee5df419391671ef1a631f43feee4e3 Mon Sep 17 00:00:00 2001 From: Kuniyuki Iwashima Date: Mon, 31 Oct 2022 14:57:28 -0700 Subject: [PATCH 1394/1927] arm64/syscall: Include asm/ptrace.h in syscall_wrapper header. Add the same change for ARM64 as done in the commit 9440c4294160 ("x86/syscall: Include asm/ptrace.h in syscall_wrapper header") to make sure all syscalls see 'struct pt_regs' definition and resulted BTF for '__arm64_sys_*(struct pt_regs *regs)' functions point to actual struct. Without this patch, the BPF verifier refuses to load a tracing prog which accesses pt_regs. bpf(BPF_PROG_LOAD, {prog_type=0x1a, ...}, 128) = -1 EACCES With this patch, we can see the correct error, which saves us time in debugging the prog. bpf(BPF_PROG_LOAD, {prog_type=0x1a, ...}, 128) = 4 bpf(BPF_RAW_TRACEPOINT_OPEN, {raw_tracepoint={name=NULL, prog_fd=4}}, 128) = -1 ENOTSUPP Signed-off-by: Kuniyuki Iwashima Acked-by: Andrii Nakryiko Link: https://lore.kernel.org/r/20221031215728.50389-1-kuniyu@amazon.com Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/syscall_wrapper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/syscall_wrapper.h b/arch/arm64/include/asm/syscall_wrapper.h index b383b4802a7bd..d30217c21eff7 100644 --- a/arch/arm64/include/asm/syscall_wrapper.h +++ b/arch/arm64/include/asm/syscall_wrapper.h @@ -8,7 +8,7 @@ #ifndef __ASM_SYSCALL_WRAPPER_H #define __ASM_SYSCALL_WRAPPER_H -struct pt_regs; +#include #define SC_ARM64_REGS_TO_ARGS(x, ...) \ __MAP(x,__SC_ARGS \ -- GitLab From 8c94951560ef29c455043bf2dfa2275b011edc66 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Tue, 8 Nov 2022 10:32:38 +0000 Subject: [PATCH 1395/1927] drm/i915: use i915_sg_dma_sizes() for all backends We rely on page_sizes.sg in setup_scratch_page() reporting the correct value if the underlying sgl is not contiguous, however in get_pages_internal() we are only looking at the layout of the created pages when calculating the sg_page_sizes, and not the final sgl, which could in theory be completely different. In such a situation we might incorrectly think we have a 64K scratch page, when it is actually only 4K or similar split over multiple non-contiguous entries, which could lead to broken behaviour when touching the scratch space within the padding of a 64K GTT page-table. For most of the other backends we already just call i915_sg_dma_sizes() on the final mapping, so rather just move that into __i915_gem_object_set_pages() to avoid such issues coming back to bite us later. v2: Update missing conversion in gvt Suggested-by: Tvrtko Ursulin Signed-off-by: Matthew Auld Cc: Stuart Summers Cc: Andrzej Hajda Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20221108103238.165447-1-matthew.auld@intel.com --- drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c | 4 +--- drivers/gpu/drm/i915/gem/i915_gem_internal.c | 5 +---- drivers/gpu/drm/i915/gem/i915_gem_object.h | 3 +-- drivers/gpu/drm/i915/gem/i915_gem_pages.c | 7 +++---- drivers/gpu/drm/i915/gem/i915_gem_phys.c | 9 +++------ drivers/gpu/drm/i915/gem/i915_gem_shmem.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_stolen.c | 2 +- drivers/gpu/drm/i915/gem/i915_gem_ttm.c | 3 +-- drivers/gpu/drm/i915/gem/i915_gem_userptr.c | 4 +--- drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c | 2 +- drivers/gpu/drm/i915/gem/selftests/huge_pages.c | 10 +++------- drivers/gpu/drm/i915/gvt/dmabuf.c | 2 +- drivers/gpu/drm/i915/selftests/i915_gem_gtt.c | 5 +---- drivers/gpu/drm/i915/selftests/mock_region.c | 2 +- 14 files changed, 20 insertions(+), 40 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c index ec6f7ae47783a..1df74f7aa3dcb 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c @@ -238,7 +238,6 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *sgt; - unsigned int sg_page_sizes; assert_object_held(obj); @@ -262,8 +261,7 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj) (!HAS_LLC(i915) && !IS_DG1(i915))) wbinvd_on_all_cpus(); - sg_page_sizes = i915_sg_dma_sizes(sgt->sgl); - __i915_gem_object_set_pages(obj, sgt, sg_page_sizes); + __i915_gem_object_set_pages(obj, sgt); return 0; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index 629acb403a2c9..f66bcefc09ec0 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -35,7 +35,6 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *st; struct scatterlist *sg; - unsigned int sg_page_sizes; unsigned int npages; int max_order = MAX_ORDER; unsigned int max_segment; @@ -64,7 +63,6 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) sg = st->sgl; st->nents = 0; - sg_page_sizes = 0; do { int order = min(fls(npages) - 1, max_order); @@ -83,7 +81,6 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) } while (1); sg_set_page(sg, page, PAGE_SIZE << order, 0); - sg_page_sizes |= PAGE_SIZE << order; st->nents++; npages -= 1 << order; @@ -105,7 +102,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) goto err; } - __i915_gem_object_set_pages(obj, st, sg_page_sizes); + __i915_gem_object_set_pages(obj, st); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h index 6b9ecff42bb5c..3db53769864c2 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_object.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h @@ -403,8 +403,7 @@ i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, unsigned long n); void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages, - unsigned int sg_page_sizes); + struct sg_table *pages); int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj); int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index 16f845663ff2c..05a27723ebb8c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -16,8 +16,7 @@ #include "i915_gem_mman.h" void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, - struct sg_table *pages, - unsigned int sg_page_sizes) + struct sg_table *pages) { struct drm_i915_private *i915 = to_i915(obj->base.dev); unsigned long supported = RUNTIME_INFO(i915)->page_sizes; @@ -45,8 +44,8 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, obj->mm.pages = pages; - GEM_BUG_ON(!sg_page_sizes); - obj->mm.page_sizes.phys = sg_page_sizes; + obj->mm.page_sizes.phys = i915_sg_dma_sizes(pages->sgl); + GEM_BUG_ON(!obj->mm.page_sizes.phys); /* * Calculate the supported page-sizes which fit into the given diff --git a/drivers/gpu/drm/i915/gem/i915_gem_phys.c b/drivers/gpu/drm/i915/gem/i915_gem_phys.c index 0d0e46dae559c..68453572275ba 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_phys.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_phys.c @@ -79,7 +79,7 @@ static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) /* We're no longer struct page backed */ obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE; - __i915_gem_object_set_pages(obj, st, sg->length); + __i915_gem_object_set_pages(obj, st); return 0; @@ -209,11 +209,8 @@ static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj) return 0; err_xfer: - if (!IS_ERR_OR_NULL(pages)) { - unsigned int sg_page_sizes = i915_sg_dma_sizes(pages->sgl); - - __i915_gem_object_set_pages(obj, pages, sg_page_sizes); - } + if (!IS_ERR_OR_NULL(pages)) + __i915_gem_object_set_pages(obj, pages); return err; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c index 2f7804492cd5c..9c759df700caa 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c @@ -247,7 +247,7 @@ static int shmem_get_pages(struct drm_i915_gem_object *obj) if (i915_gem_object_can_bypass_llc(obj)) obj->cache_dirty = true; - __i915_gem_object_set_pages(obj, st, i915_sg_dma_sizes(st->sgl)); + __i915_gem_object_set_pages(obj, st); return 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 0c70711818edd..bc95210788073 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -628,7 +628,7 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) sg_dma_len(pages->sgl), POISON_INUSE); - __i915_gem_object_set_pages(obj, pages, obj->stolen->size); + __i915_gem_object_set_pages(obj, pages); return 0; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 2c8b2d5ae9035..e4e55e3f4e411 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -815,8 +815,7 @@ static int __i915_ttm_get_pages(struct drm_i915_gem_object *obj, GEM_BUG_ON(obj->mm.rsgt); obj->mm.rsgt = rsgt; - __i915_gem_object_set_pages(obj, &rsgt->table, - i915_sg_dma_sizes(rsgt->table.sgl)); + __i915_gem_object_set_pages(obj, &rsgt->table); } GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages)); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c index ca7a388ba2bf5..9348b1804d53c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_userptr.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_userptr.c @@ -131,7 +131,6 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) const unsigned long num_pages = obj->base.size >> PAGE_SHIFT; unsigned int max_segment = i915_sg_segment_size(obj->base.dev->dev); struct sg_table *st; - unsigned int sg_page_sizes; struct page **pvec; int ret; @@ -170,8 +169,7 @@ static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) if (i915_gem_object_can_bypass_llc(obj)) obj->cache_dirty = true; - sg_page_sizes = i915_sg_dma_sizes(st->sgl); - __i915_gem_object_set_pages(obj, st, sg_page_sizes); + __i915_gem_object_set_pages(obj, st); return 0; diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c index f963b8e1e37b5..cbd9b624a7888 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c @@ -68,7 +68,7 @@ static int huge_get_pages(struct drm_i915_gem_object *obj) if (i915_gem_gtt_prepare_pages(obj, pages)) goto err; - __i915_gem_object_set_pages(obj, pages, PAGE_SIZE); + __i915_gem_object_set_pages(obj, pages); return 0; diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c index 0cb99e75b0bc2..beaf27e09e8a9 100644 --- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c +++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c @@ -136,7 +136,7 @@ static int get_huge_pages(struct drm_i915_gem_object *obj) goto err; GEM_BUG_ON(sg_page_sizes != obj->mm.page_mask); - __i915_gem_object_set_pages(obj, st, sg_page_sizes); + __i915_gem_object_set_pages(obj, st); return 0; @@ -210,7 +210,6 @@ static int fake_get_huge_pages(struct drm_i915_gem_object *obj) const u64 max_len = rounddown_pow_of_two(UINT_MAX); struct sg_table *st; struct scatterlist *sg; - unsigned int sg_page_sizes; u64 rem; st = kmalloc(sizeof(*st), GFP); @@ -226,7 +225,6 @@ static int fake_get_huge_pages(struct drm_i915_gem_object *obj) rem = obj->base.size; sg = st->sgl; st->nents = 0; - sg_page_sizes = 0; do { unsigned int page_size = get_largest_page_size(i915, rem); unsigned int len = min(page_size * div_u64(rem, page_size), @@ -239,8 +237,6 @@ static int fake_get_huge_pages(struct drm_i915_gem_object *obj) sg_dma_len(sg) = len; sg_dma_address(sg) = page_size; - sg_page_sizes |= len; - st->nents++; rem -= len; @@ -254,7 +250,7 @@ static int fake_get_huge_pages(struct drm_i915_gem_object *obj) i915_sg_trim(st); - __i915_gem_object_set_pages(obj, st, sg_page_sizes); + __i915_gem_object_set_pages(obj, st); return 0; } @@ -286,7 +282,7 @@ static int fake_get_huge_pages_single(struct drm_i915_gem_object *obj) sg_dma_len(sg) = obj->base.size; sg_dma_address(sg) = page_size; - __i915_gem_object_set_pages(obj, st, sg->length); + __i915_gem_object_set_pages(obj, st); return 0; #undef GFP diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c index 01e54b45c5c1b..355f1c0e86641 100644 --- a/drivers/gpu/drm/i915/gvt/dmabuf.c +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -88,7 +88,7 @@ static int vgpu_gem_get_pages( sg_dma_address(sg) = dma_addr; } - __i915_gem_object_set_pages(obj, st, PAGE_SIZE); + __i915_gem_object_set_pages(obj, st); out: if (ret) { dma_addr_t dma_addr; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 27c733b009762..eae7d947d7de2 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -61,7 +61,6 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) #define PFN_BIAS 0x1000 struct sg_table *pages; struct scatterlist *sg; - unsigned int sg_page_sizes; typeof(obj->base.size) rem; pages = kmalloc(sizeof(*pages), GFP); @@ -74,7 +73,6 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) return -ENOMEM; } - sg_page_sizes = 0; rem = obj->base.size; for (sg = pages->sgl; sg; sg = sg_next(sg)) { unsigned long len = min_t(typeof(rem), rem, BIT(31)); @@ -83,13 +81,12 @@ static int fake_get_pages(struct drm_i915_gem_object *obj) sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); sg_dma_address(sg) = page_to_phys(sg_page(sg)); sg_dma_len(sg) = len; - sg_page_sizes |= len; rem -= len; } GEM_BUG_ON(rem); - __i915_gem_object_set_pages(obj, pages, sg_page_sizes); + __i915_gem_object_set_pages(obj, pages); return 0; #undef GFP diff --git a/drivers/gpu/drm/i915/selftests/mock_region.c b/drivers/gpu/drm/i915/selftests/mock_region.c index bac21fe84ca56..6324eb32d4ddc 100644 --- a/drivers/gpu/drm/i915/selftests/mock_region.c +++ b/drivers/gpu/drm/i915/selftests/mock_region.c @@ -41,7 +41,7 @@ static int mock_region_get_pages(struct drm_i915_gem_object *obj) } pages = &obj->mm.rsgt->table; - __i915_gem_object_set_pages(obj, pages, i915_sg_dma_sizes(pages->sgl)); + __i915_gem_object_set_pages(obj, pages); return 0; -- GitLab From c8af247de385ce49afabc3bf1cf4fd455c94bfe8 Mon Sep 17 00:00:00 2001 From: ZhangPeng Date: Wed, 9 Nov 2022 01:35:42 +0000 Subject: [PATCH 1396/1927] udf: Fix a slab-out-of-bounds write bug in udf_find_entry() Syzbot reported a slab-out-of-bounds Write bug: loop0: detected capacity change from 0 to 2048 ================================================================== BUG: KASAN: slab-out-of-bounds in udf_find_entry+0x8a5/0x14f0 fs/udf/namei.c:253 Write of size 105 at addr ffff8880123ff896 by task syz-executor323/3610 CPU: 0 PID: 3610 Comm: syz-executor323 Not tainted 6.1.0-rc2-syzkaller-00105-gb229b6ca5abb #0 Hardware name: Google Compute Engine/Google Compute Engine, BIOS Google 10/11/2022 Call Trace: __dump_stack lib/dump_stack.c:88 [inline] dump_stack_lvl+0x1b1/0x28e lib/dump_stack.c:106 print_address_description+0x74/0x340 mm/kasan/report.c:284 print_report+0x107/0x1f0 mm/kasan/report.c:395 kasan_report+0xcd/0x100 mm/kasan/report.c:495 kasan_check_range+0x2a7/0x2e0 mm/kasan/generic.c:189 memcpy+0x3c/0x60 mm/kasan/shadow.c:66 udf_find_entry+0x8a5/0x14f0 fs/udf/namei.c:253 udf_lookup+0xef/0x340 fs/udf/namei.c:309 lookup_open fs/namei.c:3391 [inline] open_last_lookups fs/namei.c:3481 [inline] path_openat+0x10e6/0x2df0 fs/namei.c:3710 do_filp_open+0x264/0x4f0 fs/namei.c:3740 do_sys_openat2+0x124/0x4e0 fs/open.c:1310 do_sys_open fs/open.c:1326 [inline] __do_sys_creat fs/open.c:1402 [inline] __se_sys_creat fs/open.c:1396 [inline] __x64_sys_creat+0x11f/0x160 fs/open.c:1396 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x3d/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd RIP: 0033:0x7ffab0d164d9 Code: ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 40 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 c0 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007ffe1a7e6bb8 EFLAGS: 00000246 ORIG_RAX: 0000000000000055 RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 00007ffab0d164d9 RDX: 00007ffab0d164d9 RSI: 0000000000000000 RDI: 0000000020000180 RBP: 00007ffab0cd5a10 R08: 0000000000000000 R09: 0000000000000000 R10: 00005555573552c0 R11: 0000000000000246 R12: 00007ffab0cd5aa0 R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 Allocated by task 3610: kasan_save_stack mm/kasan/common.c:45 [inline] kasan_set_track+0x3d/0x60 mm/kasan/common.c:52 ____kasan_kmalloc mm/kasan/common.c:371 [inline] __kasan_kmalloc+0x97/0xb0 mm/kasan/common.c:380 kmalloc include/linux/slab.h:576 [inline] udf_find_entry+0x7b6/0x14f0 fs/udf/namei.c:243 udf_lookup+0xef/0x340 fs/udf/namei.c:309 lookup_open fs/namei.c:3391 [inline] open_last_lookups fs/namei.c:3481 [inline] path_openat+0x10e6/0x2df0 fs/namei.c:3710 do_filp_open+0x264/0x4f0 fs/namei.c:3740 do_sys_openat2+0x124/0x4e0 fs/open.c:1310 do_sys_open fs/open.c:1326 [inline] __do_sys_creat fs/open.c:1402 [inline] __se_sys_creat fs/open.c:1396 [inline] __x64_sys_creat+0x11f/0x160 fs/open.c:1396 do_syscall_x64 arch/x86/entry/common.c:50 [inline] do_syscall_64+0x3d/0xb0 arch/x86/entry/common.c:80 entry_SYSCALL_64_after_hwframe+0x63/0xcd The buggy address belongs to the object at ffff8880123ff800 which belongs to the cache kmalloc-256 of size 256 The buggy address is located 150 bytes inside of 256-byte region [ffff8880123ff800, ffff8880123ff900) The buggy address belongs to the physical page: page:ffffea000048ff80 refcount:1 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x123fe head:ffffea000048ff80 order:1 compound_mapcount:0 compound_pincount:0 flags: 0xfff00000010200(slab|head|node=0|zone=1|lastcpupid=0x7ff) raw: 00fff00000010200 ffffea00004b8500 dead000000000003 ffff888012041b40 raw: 0000000000000000 0000000080100010 00000001ffffffff 0000000000000000 page dumped because: kasan: bad access detected page_owner tracks the page as allocated page last allocated via order 0, migratetype Unmovable, gfp_mask 0x0(), pid 1, tgid 1 (swapper/0), ts 1841222404, free_ts 0 create_dummy_stack mm/page_owner.c:67 [inline] register_early_stack+0x77/0xd0 mm/page_owner.c:83 init_page_owner+0x3a/0x731 mm/page_owner.c:93 kernel_init_freeable+0x41c/0x5d5 init/main.c:1629 kernel_init+0x19/0x2b0 init/main.c:1519 page_owner free stack trace missing Memory state around the buggy address: ffff8880123ff780: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff8880123ff800: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 >ffff8880123ff880: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 06 ^ ffff8880123ff900: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ffff8880123ff980: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc ================================================================== Fix this by changing the memory size allocated for copy_name from UDF_NAME_LEN(254) to UDF_NAME_LEN_CS0(255), because the total length (lfi) of subsequent memcpy can be up to 255. CC: stable@vger.kernel.org Reported-by: syzbot+69c9fdccc6dd08961d34@syzkaller.appspotmail.com Fixes: 066b9cded00b ("udf: Use separate buffer for copying split names") Signed-off-by: ZhangPeng Signed-off-by: Jan Kara Link: https://lore.kernel.org/r/20221109013542.442790-1-zhangpeng362@huawei.com --- fs/udf/namei.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fs/udf/namei.c b/fs/udf/namei.c index fb4c30e052453..ae7bc13a5298a 100644 --- a/fs/udf/namei.c +++ b/fs/udf/namei.c @@ -240,7 +240,7 @@ static struct fileIdentDesc *udf_find_entry(struct inode *dir, poffset - lfi); else { if (!copy_name) { - copy_name = kmalloc(UDF_NAME_LEN, + copy_name = kmalloc(UDF_NAME_LEN_CS0, GFP_NOFS); if (!copy_name) { fi = ERR_PTR(-ENOMEM); -- GitLab From 7a58b8d6021426b796eebfae80983374d9a80a75 Mon Sep 17 00:00:00 2001 From: Duoming Zhou Date: Sun, 18 Sep 2022 11:33:12 +0800 Subject: [PATCH 1397/1927] usb: chipidea: fix deadlock in ci_otg_del_timer There is a deadlock in ci_otg_del_timer(), the process is shown below: (thread 1) | (thread 2) ci_otg_del_timer() | ci_otg_hrtimer_func() ... | spin_lock_irqsave() //(1) | ... ... | hrtimer_cancel() | spin_lock_irqsave() //(2) (block forever) We hold ci->lock in position (1) and use hrtimer_cancel() to wait ci_otg_hrtimer_func() to stop, but ci_otg_hrtimer_func() also need ci->lock in position (2). As a result, the hrtimer_cancel() in ci_otg_del_timer() will be blocked forever. This patch extracts hrtimer_cancel() from the protection of spin_lock_irqsave() in order that the ci_otg_hrtimer_func() could obtain the ci->lock. What`s more, there will be no race happen. Because the "next_timer" is always under the protection of spin_lock_irqsave() and we only check whether "next_timer" equals to NUM_OTG_FSM_TIMERS in the following code. Fixes: 3a316ec4c91c ("usb: chipidea: use hrtimer for otg fsm timers") Cc: stable Signed-off-by: Duoming Zhou Link: https://lore.kernel.org/r/20220918033312.94348-1-duoming@zju.edu.cn Signed-off-by: Greg Kroah-Hartman --- drivers/usb/chipidea/otg_fsm.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/usb/chipidea/otg_fsm.c b/drivers/usb/chipidea/otg_fsm.c index ada78daba6df9..c17516c29b63b 100644 --- a/drivers/usb/chipidea/otg_fsm.c +++ b/drivers/usb/chipidea/otg_fsm.c @@ -256,8 +256,10 @@ static void ci_otg_del_timer(struct ci_hdrc *ci, enum otg_fsm_timer t) ci->enabled_otg_timer_bits &= ~(1 << t); if (ci->next_otg_timer == t) { if (ci->enabled_otg_timer_bits == 0) { + spin_unlock_irqrestore(&ci->lock, flags); /* No enabled timers after delete it */ hrtimer_cancel(&ci->otg_fsm_hrtimer); + spin_lock_irqsave(&ci->lock, flags); ci->next_otg_timer = NUM_OTG_FSM_TIMERS; } else { /* Find the next timer */ -- GitLab From 9d5333c931347005352d5b8beaa43528c94cfc9c Mon Sep 17 00:00:00 2001 From: Li Jun Date: Wed, 26 Oct 2022 15:07:49 -0400 Subject: [PATCH 1398/1927] usb: cdns3: host: fix endless superspeed hub port reset When usb 3.0 hub connect with one USB 2.0 device and NO USB 3.0 device, some usb hub reports endless port reset message. [ 190.324169] usb 2-1: new SuperSpeed USB device number 88 using xhci-hcd [ 190.352834] hub 2-1:1.0: USB hub found [ 190.356995] hub 2-1:1.0: 4 ports detected [ 190.700056] usb 2-1: USB disconnect, device number 88 [ 192.472139] usb 2-1: new SuperSpeed USB device number 89 using xhci-hcd [ 192.500820] hub 2-1:1.0: USB hub found [ 192.504977] hub 2-1:1.0: 4 ports detected [ 192.852066] usb 2-1: USB disconnect, device number 89 The reason is the runtime pm state of USB2.0 port is active and USB 3.0 port is suspend, so parent device is active state. cat /sys/bus/platform/devices/5b110000.usb/5b130000.usb/xhci-hcd.1.auto/usb2/power/runtime_status suspended cat /sys/bus/platform/devices/5b110000.usb/5b130000.usb/xhci-hcd.1.auto/usb1/power/runtime_status active cat /sys/bus/platform/devices/5b110000.usb/5b130000.usb/xhci-hcd.1.auto/power/runtime_status active cat /sys/bus/platform/devices/5b110000.usb/5b130000.usb/power/runtime_status active So xhci_cdns3_suspend_quirk() have not called. U3 configure is not applied. move U3 configure into host start. Reinit again in resume function in case controller power lost during suspend. Cc: stable@vger.kernel.org 5.10 Signed-off-by: Li Jun Signed-off-by: Frank Li Reviewed-by: Peter Chen Acked-by: Alexander Stein Link: https://lore.kernel.org/r/20221026190749.2280367-1-Frank.Li@nxp.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/cdns3/host.c | 56 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/drivers/usb/cdns3/host.c b/drivers/usb/cdns3/host.c index 9643b905e2d8b..6164fc4c96a49 100644 --- a/drivers/usb/cdns3/host.c +++ b/drivers/usb/cdns3/host.c @@ -24,11 +24,37 @@ #define CFG_RXDET_P3_EN BIT(15) #define LPM_2_STB_SWITCH_EN BIT(25) -static int xhci_cdns3_suspend_quirk(struct usb_hcd *hcd); +static void xhci_cdns3_plat_start(struct usb_hcd *hcd) +{ + struct xhci_hcd *xhci = hcd_to_xhci(hcd); + u32 value; + + /* set usbcmd.EU3S */ + value = readl(&xhci->op_regs->command); + value |= CMD_PM_INDEX; + writel(value, &xhci->op_regs->command); + + if (hcd->regs) { + value = readl(hcd->regs + XECP_AUX_CTRL_REG1); + value |= CFG_RXDET_P3_EN; + writel(value, hcd->regs + XECP_AUX_CTRL_REG1); + + value = readl(hcd->regs + XECP_PORT_CAP_REG); + value |= LPM_2_STB_SWITCH_EN; + writel(value, hcd->regs + XECP_PORT_CAP_REG); + } +} + +static int xhci_cdns3_resume_quirk(struct usb_hcd *hcd) +{ + xhci_cdns3_plat_start(hcd); + return 0; +} static const struct xhci_plat_priv xhci_plat_cdns3_xhci = { .quirks = XHCI_SKIP_PHY_INIT | XHCI_AVOID_BEI, - .suspend_quirk = xhci_cdns3_suspend_quirk, + .plat_start = xhci_cdns3_plat_start, + .resume_quirk = xhci_cdns3_resume_quirk, }; static int __cdns_host_init(struct cdns *cdns) @@ -90,32 +116,6 @@ static int __cdns_host_init(struct cdns *cdns) return ret; } -static int xhci_cdns3_suspend_quirk(struct usb_hcd *hcd) -{ - struct xhci_hcd *xhci = hcd_to_xhci(hcd); - u32 value; - - if (pm_runtime_status_suspended(hcd->self.controller)) - return 0; - - /* set usbcmd.EU3S */ - value = readl(&xhci->op_regs->command); - value |= CMD_PM_INDEX; - writel(value, &xhci->op_regs->command); - - if (hcd->regs) { - value = readl(hcd->regs + XECP_AUX_CTRL_REG1); - value |= CFG_RXDET_P3_EN; - writel(value, hcd->regs + XECP_AUX_CTRL_REG1); - - value = readl(hcd->regs + XECP_PORT_CAP_REG); - value |= LPM_2_STB_SWITCH_EN; - writel(value, hcd->regs + XECP_PORT_CAP_REG); - } - - return 0; -} - static void cdns_host_exit(struct cdns *cdns) { kfree(cdns->xhci_plat_data); -- GitLab From bdfe34597139cfcecd47a2eb97fea44d77157491 Mon Sep 17 00:00:00 2001 From: Sandipan Das Date: Thu, 8 Sep 2022 10:33:15 +0530 Subject: [PATCH 1399/1927] perf/x86/amd/uncore: Fix memory leak for events array When a CPU comes online, the per-CPU NB and LLC uncore contexts are freed but not the events array within the context structure. This causes a memory leak as identified by the kmemleak detector. [...] unreferenced object 0xffff8c5944b8e320 (size 32): comm "swapper/0", pid 1, jiffies 4294670387 (age 151.072s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<000000000759fb79>] amd_uncore_cpu_up_prepare+0xaf/0x230 [<00000000ddc9e126>] cpuhp_invoke_callback+0x2cf/0x470 [<0000000093e727d4>] cpuhp_issue_call+0x14d/0x170 [<0000000045464d54>] __cpuhp_setup_state_cpuslocked+0x11e/0x330 [<0000000069f67cbd>] __cpuhp_setup_state+0x6b/0x110 [<0000000015365e0f>] amd_uncore_init+0x260/0x321 [<00000000089152d2>] do_one_initcall+0x3f/0x1f0 [<000000002d0bd18d>] kernel_init_freeable+0x1ca/0x212 [<0000000030be8dde>] kernel_init+0x11/0x120 [<0000000059709e59>] ret_from_fork+0x22/0x30 unreferenced object 0xffff8c5944b8dd40 (size 64): comm "swapper/0", pid 1, jiffies 4294670387 (age 151.072s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................ backtrace: [<00000000306efe8b>] amd_uncore_cpu_up_prepare+0x183/0x230 [<00000000ddc9e126>] cpuhp_invoke_callback+0x2cf/0x470 [<0000000093e727d4>] cpuhp_issue_call+0x14d/0x170 [<0000000045464d54>] __cpuhp_setup_state_cpuslocked+0x11e/0x330 [<0000000069f67cbd>] __cpuhp_setup_state+0x6b/0x110 [<0000000015365e0f>] amd_uncore_init+0x260/0x321 [<00000000089152d2>] do_one_initcall+0x3f/0x1f0 [<000000002d0bd18d>] kernel_init_freeable+0x1ca/0x212 [<0000000030be8dde>] kernel_init+0x11/0x120 [<0000000059709e59>] ret_from_fork+0x22/0x30 [...] Fix the problem by freeing the events array before freeing the uncore context. Fixes: 39621c5808f5 ("perf/x86/amd/uncore: Use dynamic events array") Reported-by: Ravi Bangoria Signed-off-by: Sandipan Das Signed-off-by: Borislav Petkov Tested-by: Ravi Bangoria Cc: Link: https://lore.kernel.org/r/4fa9e5ac6d6e41fa889101e7af7e6ba372cfea52.1662613255.git.sandipan.das@amd.com --- arch/x86/events/amd/uncore.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index d568afc705d2e..83f15fe411b3f 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -553,6 +553,7 @@ static void uncore_clean_online(void) hlist_for_each_entry_safe(uncore, n, &uncore_unused_list, node) { hlist_del(&uncore->node); + kfree(uncore->events); kfree(uncore); } } -- GitLab From f002f45a00ee14214d96b18b9a555fe2c56afb20 Mon Sep 17 00:00:00 2001 From: Haibo Chen Date: Tue, 8 Nov 2022 15:45:03 +0800 Subject: [PATCH 1400/1927] mmc: sdhci-esdhc-imx: use the correct host caps for MMC_CAP_8_BIT_DATA MMC_CAP_8_BIT_DATA belongs to struct mmc_host, not struct sdhci_host. So correct it here. Fixes: 1ed5c3b22fc7 ("mmc: sdhci-esdhc-imx: Propagate ESDHC_FLAG_HS400* only on 8bit bus") Signed-off-by: Haibo Chen Cc: stable@vger.kernel.org Acked-by: Adrian Hunter Link: https://lore.kernel.org/r/1667893503-20583-1-git-send-email-haibo.chen@nxp.com Signed-off-by: Ulf Hansson --- drivers/mmc/host/sdhci-esdhc-imx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/mmc/host/sdhci-esdhc-imx.c b/drivers/mmc/host/sdhci-esdhc-imx.c index 89225faa242ab..31ea0a2fce358 100644 --- a/drivers/mmc/host/sdhci-esdhc-imx.c +++ b/drivers/mmc/host/sdhci-esdhc-imx.c @@ -1672,14 +1672,14 @@ static int sdhci_esdhc_imx_probe(struct platform_device *pdev) if (imx_data->socdata->flags & ESDHC_FLAG_ERR004536) host->quirks |= SDHCI_QUIRK_BROKEN_ADMA; - if (host->caps & MMC_CAP_8_BIT_DATA && + if (host->mmc->caps & MMC_CAP_8_BIT_DATA && imx_data->socdata->flags & ESDHC_FLAG_HS400) host->mmc->caps2 |= MMC_CAP2_HS400; if (imx_data->socdata->flags & ESDHC_FLAG_BROKEN_AUTO_CMD23) host->quirks2 |= SDHCI_QUIRK2_ACMD23_BROKEN; - if (host->caps & MMC_CAP_8_BIT_DATA && + if (host->mmc->caps & MMC_CAP_8_BIT_DATA && imx_data->socdata->flags & ESDHC_FLAG_HS400_ES) { host->mmc->caps2 |= MMC_CAP2_HS400_ES; host->mmc_host_ops.hs400_enhanced_strobe = -- GitLab From a931237cbea256aff13bb403da13a97b2d1605d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Tue, 8 Nov 2022 14:19:49 +0200 Subject: [PATCH 1401/1927] serial: 8250: Fall back to non-DMA Rx if IIR_RDI occurs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DW UART sometimes triggers IIR_RDI during DMA Rx when IIR_RX_TIMEOUT should have been triggered instead. Since IIR_RDI has higher priority than IIR_RX_TIMEOUT, this causes the Rx to hang into interrupt loop. The problem seems to occur at least with some combinations of small-sized transfers (I've reproduced the problem on Elkhart Lake PSE UARTs). If there's already an on-going Rx DMA and IIR_RDI triggers, fall graciously back to non-DMA Rx. That is, behave as if IIR_RX_TIMEOUT had occurred. 8250_omap already considers IIR_RDI similar to this change so its nothing unheard of. Fixes: 75df022b5f89 ("serial: 8250_dma: Fix RX handling") Cc: Co-developed-by: Srikanth Thokala Signed-off-by: Srikanth Thokala Co-developed-by: Aman Kumar Signed-off-by: Aman Kumar Signed-off-by: Ilpo Järvinen Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20221108121952.5497-2-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_port.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index fe8662cd94024..92dd18716169e 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -1897,6 +1897,10 @@ EXPORT_SYMBOL_GPL(serial8250_modem_status); static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir) { switch (iir & 0x3f) { + case UART_IIR_RDI: + if (!up->dma->rx_running) + break; + fallthrough; case UART_IIR_RX_TIMEOUT: serial8250_rx_dma_flush(up); fallthrough; -- GitLab From 1bfcbe5805d0cfc83c3544dcd01e0a282c1f6790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Tue, 8 Nov 2022 14:19:50 +0200 Subject: [PATCH 1402/1927] serial: 8250_lpss: Configure DMA also w/o DMA filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the platform doesn't use DMA device filter (as is the case with Elkhart Lake), whole lpss8250_dma_setup() setup is skipped. This results in skipping also *_maxburst setup which is undesirable. Refactor lpss8250_dma_setup() to configure DMA even if filter is not setup. Cc: stable Signed-off-by: Ilpo Järvinen Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20221108121952.5497-3-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_lpss.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c index 44cc755b1a29a..7d9cddbfef408 100644 --- a/drivers/tty/serial/8250/8250_lpss.c +++ b/drivers/tty/serial/8250/8250_lpss.c @@ -277,8 +277,13 @@ static int lpss8250_dma_setup(struct lpss8250 *lpss, struct uart_8250_port *port struct dw_dma_slave *rx_param, *tx_param; struct device *dev = port->port.dev; - if (!lpss->dma_param.dma_dev) + if (!lpss->dma_param.dma_dev) { + dma = port->dma; + if (dma) + goto out_configuration_only; + return 0; + } rx_param = devm_kzalloc(dev, sizeof(*rx_param), GFP_KERNEL); if (!rx_param) @@ -289,16 +294,18 @@ static int lpss8250_dma_setup(struct lpss8250 *lpss, struct uart_8250_port *port return -ENOMEM; *rx_param = lpss->dma_param; - dma->rxconf.src_maxburst = lpss->dma_maxburst; - *tx_param = lpss->dma_param; - dma->txconf.dst_maxburst = lpss->dma_maxburst; dma->fn = lpss8250_dma_filter; dma->rx_param = rx_param; dma->tx_param = tx_param; port->dma = dma; + +out_configuration_only: + dma->rxconf.src_maxburst = lpss->dma_maxburst; + dma->txconf.dst_maxburst = lpss->dma_maxburst; + return 0; } -- GitLab From 7090abd6ad0610a144523ce4ffcb8560909bf2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Tue, 8 Nov 2022 14:19:51 +0200 Subject: [PATCH 1403/1927] serial: 8250_lpss: Use 16B DMA burst with Elkhart Lake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Configure DMA to use 16B burst size with Elkhart Lake. This makes the bus use more efficient and works around an issue which occurs with the previously used 1B. The fix was initially developed by Srikanth Thokala and Aman Kumar. This together with the previous config change is the cleaned up version of the original fix. Fixes: 0a9410b981e9 ("serial: 8250_lpss: Enable DMA on Intel Elkhart Lake") Cc: # serial: 8250_lpss: Configure DMA also w/o DMA filter Reported-by: Wentong Wu Signed-off-by: Ilpo Järvinen Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20221108121952.5497-4-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_lpss.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/tty/serial/8250/8250_lpss.c b/drivers/tty/serial/8250/8250_lpss.c index 7d9cddbfef408..0e43bdfb74598 100644 --- a/drivers/tty/serial/8250/8250_lpss.c +++ b/drivers/tty/serial/8250/8250_lpss.c @@ -174,6 +174,8 @@ static int ehl_serial_setup(struct lpss8250 *lpss, struct uart_port *port) */ up->dma = dma; + lpss->dma_maxburst = 16; + port->set_termios = dw8250_do_set_termios; return 0; -- GitLab From 1980860e0c8299316cddaf0992dd9e1258ec9d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ilpo=20J=C3=A4rvinen?= Date: Tue, 8 Nov 2022 14:19:52 +0200 Subject: [PATCH 1404/1927] serial: 8250: Flush DMA Rx on RLSI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returning true from handle_rx_dma() without flushing DMA first creates a data ordering hazard. If DMA Rx has handled any character at the point when RLSI occurs, the non-DMA path handles any pending characters jumping them ahead of those characters that are pending under DMA. Fixes: 75df022b5f89 ("serial: 8250_dma: Fix RX handling") Cc: Signed-off-by: Ilpo Järvinen Reviewed-by: Andy Shevchenko Link: https://lore.kernel.org/r/20221108121952.5497-5-ilpo.jarvinen@linux.intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/tty/serial/8250/8250_port.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index 92dd18716169e..388172289627a 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -1901,10 +1901,9 @@ static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir) if (!up->dma->rx_running) break; fallthrough; + case UART_IIR_RLSI: case UART_IIR_RX_TIMEOUT: serial8250_rx_dma_flush(up); - fallthrough; - case UART_IIR_RLSI: return true; } return up->dma->rx_dma(up); -- GitLab From 181135bb20dcb184edd89817831b888eb8132741 Mon Sep 17 00:00:00 2001 From: Nicolas Dumazet Date: Wed, 9 Nov 2022 13:29:46 +0100 Subject: [PATCH 1405/1927] usb: add NO_LPM quirk for Realforce 87U Keyboard Before adding this quirk, this (mechanical keyboard) device would not be recognized, logging: new full-speed USB device number 56 using xhci_hcd unable to read config index 0 descriptor/start: -32 chopping to 0 config(s) It would take dozens of plugging/unpuggling cycles for the keyboard to be recognized. Keyboard seems to simply work after applying this quirk. This issue had been reported by users in two places already ([1], [2]) but nobody tried upstreaming a patch yet. After testing I believe their suggested fix (DELAY_INIT + NO_LPM + DEVICE_QUALIFIER) was probably a little overkill. I assume this particular combination was tested because it had been previously suggested in [3], but only NO_LPM seems sufficient for this device. [1]: https://qiita.com/float168/items/fed43d540c8e2201b543 [2]: https://blog.kostic.dev/posts/making-the-realforce-87ub-work-with-usb30-on-Ubuntu/ [3]: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678477 Cc: stable@vger.kernel.org Signed-off-by: Nicolas Dumazet Link: https://lore.kernel.org/r/20221109122946.706036-1-ndumazet@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/usb/core/quirks.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index 0722d21313055..079e183cf3bff 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -362,6 +362,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x0781, 0x5583), .driver_info = USB_QUIRK_NO_LPM }, { USB_DEVICE(0x0781, 0x5591), .driver_info = USB_QUIRK_NO_LPM }, + /* Realforce 87U Keyboard */ + { USB_DEVICE(0x0853, 0x011b), .driver_info = USB_QUIRK_NO_LPM }, + /* M-Systems Flash Disk Pioneers */ { USB_DEVICE(0x08ec, 0x1000), .driver_info = USB_QUIRK_RESET_RESUME }, -- GitLab From d7ac8dca938cd60cf7bd9a89a229a173c6bcba87 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 28 Oct 2022 13:14:15 -0700 Subject: [PATCH 1406/1927] nvme: quiet user passthrough command errors The driver is spamming the kernel logs for entirely harmless errors from user space submitting unsupported commands. Just silence the errors. The application has direct access to command status, so there's no need to log these. And since every passthrough command now uses the quiet flag, move the setting to the common initializer. Signed-off-by: Keith Busch Reviewed-by: Alan Adamson Reviewed-by: Jens Axboe Reviewed-by: Kanchan Joshi Reviewed-by: Chaitanya Kulkarni Reviewed-by: Daniel Wagner Tested-by: Alan Adamson Signed-off-by: Christoph Hellwig --- drivers/nvme/host/core.c | 3 +-- drivers/nvme/host/pci.c | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index dc42206005855..da55ce45ac701 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -675,6 +675,7 @@ void nvme_init_request(struct request *req, struct nvme_command *cmd) if (req->mq_hctx->type == HCTX_TYPE_POLL) req->cmd_flags |= REQ_POLLED; nvme_clear_nvme_request(req); + req->rq_flags |= RQF_QUIET; memcpy(nvme_req(req)->cmd, cmd, sizeof(*cmd)); } EXPORT_SYMBOL_GPL(nvme_init_request); @@ -1037,7 +1038,6 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, goto out; } - req->rq_flags |= RQF_QUIET; ret = nvme_execute_rq(req, at_head); if (result && ret >= 0) *result = nvme_req(req)->result; @@ -1227,7 +1227,6 @@ static void nvme_keep_alive_work(struct work_struct *work) rq->timeout = ctrl->kato * HZ; rq->end_io = nvme_keep_alive_end_io; rq->end_io_data = ctrl; - rq->rq_flags |= RQF_QUIET; blk_execute_rq_nowait(rq, false); } diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 31e577b01257d..02b5578773a17 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1436,7 +1436,6 @@ static enum blk_eh_timer_return nvme_timeout(struct request *req) abort_req->end_io = abort_endio; abort_req->end_io_data = NULL; - abort_req->rq_flags |= RQF_QUIET; blk_execute_rq_nowait(abort_req, false); /* @@ -2490,7 +2489,6 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode) req->end_io_data = nvmeq; init_completion(&nvmeq->delete_done); - req->rq_flags |= RQF_QUIET; blk_execute_rq_nowait(req, false); return 0; } -- GitLab From becc4cac309dc867571f0080fde4426a6c2222e0 Mon Sep 17 00:00:00 2001 From: Aleksandr Miloserdov Date: Wed, 26 Oct 2022 12:31:33 +0400 Subject: [PATCH 1407/1927] nvmet: fix memory leak in nvmet_subsys_attr_model_store_locked Since model_number is allocated before it needs to be freed before kmemdump_nul. Reviewed-by: Konstantin Shelekhin Reviewed-by: Dmitriy Bogdanov Signed-off-by: Aleksandr Miloserdov Reviewed-by: Sagi Grimberg Signed-off-by: Christoph Hellwig --- drivers/nvme/target/configfs.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 9443ee1d4ae3d..d0b9eea15ff89 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1215,6 +1215,7 @@ static ssize_t nvmet_subsys_attr_model_store_locked(struct nvmet_subsys *subsys, const char *page, size_t count) { int pos = 0, len; + char *val; if (subsys->subsys_discovered) { pr_err("Can't set model number. %s is already assigned\n", @@ -1237,9 +1238,11 @@ static ssize_t nvmet_subsys_attr_model_store_locked(struct nvmet_subsys *subsys, return -EINVAL; } - subsys->model_number = kmemdup_nul(page, len, GFP_KERNEL); - if (!subsys->model_number) + val = kmemdup_nul(page, len, GFP_KERNEL); + if (!val) return -ENOMEM; + kfree(subsys->model_number); + subsys->model_number = val; return count; } -- GitLab From e65fdf530f55c5e387db14470a59a399faa29613 Mon Sep 17 00:00:00 2001 From: Sagi Grimberg Date: Wed, 9 Nov 2022 05:29:57 +0200 Subject: [PATCH 1408/1927] nvmet: fix a memory leak We need to also free the dhchap_ctrl_secret when releasing nvmet_host. kmemleak complaint: -- unreferenced object 0xffff99b1cbca5140 (size 64): comm "check", pid 4864, jiffies 4305092436 (age 2913.583s) hex dump (first 32 bytes): 44 48 48 43 2d 31 3a 30 30 3a 65 36 2b 41 63 44 DHHC-1:00:e6+AcD 39 76 47 4d 52 57 59 78 67 54 47 44 51 59 47 78 9vGMRWYxgTGDQYGx backtrace: [<00000000c07d369d>] kstrdup+0x2e/0x60 [<000000001372171c>] 0xffffffffc0cceec6 [<0000000010dbf50b>] 0xffffffffc0cc6783 [<000000007465e93c>] configfs_write_iter+0xb1/0x120 [<0000000039c23f62>] vfs_write+0x2be/0x3c0 [<000000002da4351c>] ksys_write+0x5f/0xe0 [<00000000d5011e32>] do_syscall_64+0x38/0x90 [<00000000503870cf>] entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: db1312dd9548 ("nvmet: implement basic In-Band Authentication") Signed-off-by: Sagi Grimberg Reviewed-by: Chaitanya Kulkarni Signed-off-by: Christoph Hellwig --- drivers/nvme/target/configfs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index d0b9eea15ff89..6a2816f3b4e80 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -1839,6 +1839,7 @@ static void nvmet_host_release(struct config_item *item) #ifdef CONFIG_NVME_TARGET_AUTH kfree(host->dhchap_secret); + kfree(host->dhchap_ctrl_secret); #endif kfree(host); } -- GitLab From 3ec17cb325ac731c2211e13f7eaa4b812694e218 Mon Sep 17 00:00:00 2001 From: Akira Yokosawa Date: Wed, 2 Nov 2022 20:48:01 +0900 Subject: [PATCH 1409/1927] docs/driver-api/miscellaneous: Remove kernel-doc of serial_core.c Since merge of tty-6.0-rc1, "make htmldocs" with Sphinx >=3.1 emits a bunch of warnings indicating duplicate kernel-doc comments from drivers/tty/serial/serial_core.c. This is due to the kernel-doc directive for serial_core.c in serial/drivers.rst added in the merge. It conflicts with an existing kernel-doc directive in miscellaneous.rst. Remove the latter directive and resolve the duplicates. Signed-off-by: Akira Yokosawa Fixes: 607ca0f742b7 ("Merge tag 'tty-6.0-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty") Cc: stable@vger.kernel.org # 6.0 Cc: Jiri Slaby Cc: Greg Kroah-Hartman Reviewed-by: Jiri Slaby Link: https://lore.kernel.org/r/4e54c76a-138a-07e0-985a-dd83cb622208@gmail.com Signed-off-by: Greg Kroah-Hartman --- Documentation/driver-api/miscellaneous.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Documentation/driver-api/miscellaneous.rst b/Documentation/driver-api/miscellaneous.rst index 304ffb146cf9c..4a5104a368ac6 100644 --- a/Documentation/driver-api/miscellaneous.rst +++ b/Documentation/driver-api/miscellaneous.rst @@ -16,12 +16,11 @@ Parallel Port Devices 16x50 UART Driver ================= -.. kernel-doc:: drivers/tty/serial/serial_core.c - :export: - .. kernel-doc:: drivers/tty/serial/8250/8250_core.c :export: +See serial/driver.rst for related APIs. + Pulse-Width Modulation (PWM) ============================ -- GitLab From d38a648d2d6cc7bee11c6f533ff9426a00c2a74c Mon Sep 17 00:00:00 2001 From: M Chetan Kumar Date: Mon, 7 Nov 2022 13:04:49 +0530 Subject: [PATCH 1410/1927] net: wwan: iosm: fix memory leak in ipc_pcie_read_bios_cfg ipc_pcie_read_bios_cfg() is using the acpi_evaluate_dsm() to obtain the wwan power state configuration from BIOS but is not freeing the acpi_object. The acpi_evaluate_dsm() returned acpi_object to be freed. Free the acpi_object after use. Fixes: 7e98d785ae61 ("net: iosm: entry point") Signed-off-by: M Chetan Kumar Signed-off-by: David S. Miller --- drivers/net/wwan/iosm/iosm_ipc_pcie.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/net/wwan/iosm/iosm_ipc_pcie.c b/drivers/net/wwan/iosm/iosm_ipc_pcie.c index 31f57b986df28..97cb6846c6ae2 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_pcie.c +++ b/drivers/net/wwan/iosm/iosm_ipc_pcie.c @@ -232,6 +232,7 @@ static void ipc_pcie_config_init(struct iosm_pcie *ipc_pcie) */ static enum ipc_pcie_sleep_state ipc_pcie_read_bios_cfg(struct device *dev) { + enum ipc_pcie_sleep_state sleep_state = IPC_PCIE_D0L12; union acpi_object *object; acpi_handle handle_acpi; @@ -242,12 +243,16 @@ static enum ipc_pcie_sleep_state ipc_pcie_read_bios_cfg(struct device *dev) } object = acpi_evaluate_dsm(handle_acpi, &wwan_acpi_guid, 0, 3, NULL); + if (!object) + goto default_ret; + + if (object->integer.value == 3) + sleep_state = IPC_PCIE_D3L2; - if (object && object->integer.value == 3) - return IPC_PCIE_D3L2; + kfree(object); default_ret: - return IPC_PCIE_D0L12; + return sleep_state; } static int ipc_pcie_probe(struct pci_dev *pci, -- GitLab From 035e3befc191347331dd2530c3686e05a8acfbb2 Mon Sep 17 00:00:00 2001 From: M Chetan Kumar Date: Mon, 7 Nov 2022 13:05:02 +0530 Subject: [PATCH 1411/1927] net: wwan: iosm: fix driver not working with INTEL_IOMMU disabled With INTEL_IOMMU disable config or by forcing intel_iommu=off from grub some of the features of IOSM driver like browsing, flashing & coredump collection is not working. When driver calls DMA API - dma_map_single() for tx transfers. It is resulting in dma mapping error. Set the device DMA addressing capabilities using dma_set_mask() and remove the INTEL_IOMMU dependency in kconfig so that driver follows the platform config either INTEL_IOMMU enable or disable. Fixes: f7af616c632e ("net: iosm: infrastructure") Signed-off-by: M Chetan Kumar Signed-off-by: David S. Miller --- drivers/net/wwan/Kconfig | 2 +- drivers/net/wwan/iosm/iosm_ipc_pcie.c | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/net/wwan/Kconfig b/drivers/net/wwan/Kconfig index 3486ffe94ac46..ac4d73b5626f4 100644 --- a/drivers/net/wwan/Kconfig +++ b/drivers/net/wwan/Kconfig @@ -94,7 +94,7 @@ config RPMSG_WWAN_CTRL config IOSM tristate "IOSM Driver for Intel M.2 WWAN Device" - depends on INTEL_IOMMU + depends on PCI select NET_DEVLINK select RELAY if WWAN_DEBUGFS help diff --git a/drivers/net/wwan/iosm/iosm_ipc_pcie.c b/drivers/net/wwan/iosm/iosm_ipc_pcie.c index 97cb6846c6ae2..d3d34d1c47044 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_pcie.c +++ b/drivers/net/wwan/iosm/iosm_ipc_pcie.c @@ -259,6 +259,7 @@ static int ipc_pcie_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) { struct iosm_pcie *ipc_pcie = kzalloc(sizeof(*ipc_pcie), GFP_KERNEL); + int ret; pr_debug("Probing device 0x%X from the vendor 0x%X", pci_id->device, pci_id->vendor); @@ -291,6 +292,12 @@ static int ipc_pcie_probe(struct pci_dev *pci, goto pci_enable_fail; } + ret = dma_set_mask(ipc_pcie->dev, DMA_BIT_MASK(64)); + if (ret) { + dev_err(ipc_pcie->dev, "Could not set PCI DMA mask: %d", ret); + return ret; + } + ipc_pcie_config_aspm(ipc_pcie); dev_dbg(ipc_pcie->dev, "PCIe device enabled."); -- GitLab From 02d2d2ea4a3bc2391f6ac31f6854da83e8a63829 Mon Sep 17 00:00:00 2001 From: M Chetan Kumar Date: Mon, 7 Nov 2022 13:05:13 +0530 Subject: [PATCH 1412/1927] net: wwan: iosm: fix invalid mux header type Data stall seen during peak DL throughput test & packets are dropped by mux layer due to invalid header type in datagram. During initlization Mux aggregration protocol is set to default UL/DL size and TD count of Mux lite protocol. This configuration mismatch between device and driver is resulting in data stall/packet drops. Override the UL/DL size and TD count for Mux aggregation protocol. Fixes: 1f52d7b62285 ("net: wwan: iosm: Enable M.2 7360 WWAN card support") Signed-off-by: M Chetan Kumar Signed-off-by: David S. Miller --- drivers/net/wwan/iosm/iosm_ipc_imem_ops.c | 8 ++++++++ drivers/net/wwan/iosm/iosm_ipc_mux.h | 1 + 2 files changed, 9 insertions(+) diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c index b7f9237dedf76..66b90cc4c3460 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c +++ b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c @@ -91,6 +91,14 @@ void ipc_imem_wwan_channel_init(struct iosm_imem *ipc_imem, } ipc_chnl_cfg_get(&chnl_cfg, ipc_imem->nr_of_channels); + + if (ipc_imem->mmio->mux_protocol == MUX_AGGREGATION && + ipc_imem->nr_of_channels == IPC_MEM_IP_CHL_ID_0) { + chnl_cfg.ul_nr_of_entries = IPC_MEM_MAX_TDS_MUX_AGGR_UL; + chnl_cfg.dl_nr_of_entries = IPC_MEM_MAX_TDS_MUX_AGGR_DL; + chnl_cfg.dl_buf_size = IPC_MEM_MAX_ADB_BUF_SIZE; + } + ipc_imem_channel_init(ipc_imem, IPC_CTYPE_WWAN, chnl_cfg, IRQ_MOD_OFF); diff --git a/drivers/net/wwan/iosm/iosm_ipc_mux.h b/drivers/net/wwan/iosm/iosm_ipc_mux.h index cd9d74cc097f1..9968bb885c1f3 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_mux.h +++ b/drivers/net/wwan/iosm/iosm_ipc_mux.h @@ -10,6 +10,7 @@ #define IPC_MEM_MAX_UL_DG_ENTRIES 100 #define IPC_MEM_MAX_TDS_MUX_AGGR_UL 60 +#define IPC_MEM_MAX_TDS_MUX_AGGR_DL 60 #define IPC_MEM_MAX_ADB_BUF_SIZE (16 * 1024) #define IPC_MEM_MAX_UL_ADB_BUF_SIZE IPC_MEM_MAX_ADB_BUF_SIZE -- GitLab From 980ec04a88c9f0046c1da65833fb77b2ffa34b04 Mon Sep 17 00:00:00 2001 From: M Chetan Kumar Date: Mon, 7 Nov 2022 13:05:24 +0530 Subject: [PATCH 1413/1927] net: wwan: iosm: fix kernel test robot reported errors Include linux/vmalloc.h in iosm_ipc_coredump.c & iosm_ipc_devlink.c to resolve kernel test robot errors. Reported-by: kernel test robot Signed-off-by: M Chetan Kumar Signed-off-by: David S. Miller --- drivers/net/wwan/iosm/iosm_ipc_coredump.c | 1 + drivers/net/wwan/iosm/iosm_ipc_devlink.c | 1 + 2 files changed, 2 insertions(+) diff --git a/drivers/net/wwan/iosm/iosm_ipc_coredump.c b/drivers/net/wwan/iosm/iosm_ipc_coredump.c index 9acd87724c9de..26ca30476f409 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_coredump.c +++ b/drivers/net/wwan/iosm/iosm_ipc_coredump.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2020-2021 Intel Corporation. */ +#include #include "iosm_ipc_coredump.h" diff --git a/drivers/net/wwan/iosm/iosm_ipc_devlink.c b/drivers/net/wwan/iosm/iosm_ipc_devlink.c index 17da85a8f3371..2fe724d623c06 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_devlink.c +++ b/drivers/net/wwan/iosm/iosm_ipc_devlink.c @@ -2,6 +2,7 @@ /* * Copyright (C) 2020-2021 Intel Corporation. */ +#include #include "iosm_ipc_chnl_cfg.h" #include "iosm_ipc_coredump.h" -- GitLab From f352262f727215553879705bacbcb208979f3eff Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 8 Nov 2022 17:06:19 +0000 Subject: [PATCH 1414/1927] drm/panfrost: Split io-pgtable requests properly Although we don't use 1GB block mappings, we still need to split map/unmap requests at 1GB boundaries to match what io-pgtable expects. Fix that, and add some explanation to make sense of it all. Fixes: 3740b081795a ("drm/panfrost: Update io-pgtable API") Reported-by: Dmitry Osipenko Signed-off-by: Robin Murphy Tested-by: Dmitry Osipenko Reviewed-by: Steven Price Signed-off-by: Steven Price Link: https://patchwork.freedesktop.org/patch/msgid/49e54bb4019cd06e01549b106d7ac37c3d182cd3.1667927179.git.robin.murphy@arm.com --- drivers/gpu/drm/panfrost/panfrost_mmu.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index e246d914e7f6b..4e83a1891f3ed 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -250,13 +250,22 @@ void panfrost_mmu_reset(struct panfrost_device *pfdev) static size_t get_pgsize(u64 addr, size_t size, size_t *count) { + /* + * io-pgtable only operates on multiple pages within a single table + * entry, so we need to split at boundaries of the table size, i.e. + * the next block size up. The distance from address A to the next + * boundary of block size B is logically B - A % B, but in unsigned + * two's complement where B is a power of two we get the equivalence + * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :) + */ size_t blk_offset = -addr % SZ_2M; if (blk_offset || size < SZ_2M) { *count = min_not_zero(blk_offset, size) / SZ_4K; return SZ_4K; } - *count = size / SZ_2M; + blk_offset = -addr % SZ_1G ?: SZ_1G; + *count = min(blk_offset, size) / SZ_2M; return SZ_2M; } -- GitLab From 0fc801f8018000c8e64a275a20cb1da7c54e46df Mon Sep 17 00:00:00 2001 From: Mushahid Hussain Date: Mon, 10 Oct 2022 21:57:20 +0500 Subject: [PATCH 1415/1927] speakup: fix a segfault caused by switching consoles This patch fixes a segfault by adding a null check on synth in speakup_con_update(). The segfault can be reproduced as follows: - Login into a text console - Load speakup and speakup_soft modules - Remove speakup_soft - Switch to a graphics console This is caused by lack of a null check on `synth` in speakup_con_update(). Here's the sequence that causes the segfault: - When we remove the speakup_soft, synth_release() sets the synth to null. - After that, when we change the virtual console to graphics console, vt_notifier_call() is fired, which then calls speakup_con_update(). - Inside speakup_con_update() there's no null check on synth, so it calls synth_printf(). - Inside synth_printf(), synth_buffer_add() and synth_start(), both access synth, when it is null and causing a segfault. Therefore adding a null check on synth solves the issue. Fixes: 2610df41489f ("staging: speakup: Add pause command used on switching to graphical mode") Cc: stable Signed-off-by: Mushahid Hussain Signed-off-by: Samuel Thibault Link: https://lore.kernel.org/r/20221010165720.397042-1-mushi.shar@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/speakup/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/accessibility/speakup/main.c b/drivers/accessibility/speakup/main.c index f52265293482e..73db0cb44fc7b 100644 --- a/drivers/accessibility/speakup/main.c +++ b/drivers/accessibility/speakup/main.c @@ -1778,7 +1778,7 @@ static void speakup_con_update(struct vc_data *vc) { unsigned long flags; - if (!speakup_console[vc->vc_num] || spk_parked) + if (!speakup_console[vc->vc_num] || spk_parked || !synth) return; if (!spin_trylock_irqsave(&speakup_info.spinlock, flags)) /* Speakup output, discard */ -- GitLab From 92ca969ff8815f3feef2645199bd39bf594e5eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Mon, 17 Oct 2022 15:09:36 +0700 Subject: [PATCH 1416/1927] speakup: replace utils' u_char with unsigned char MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drivers/accessibility/speakup/utils.h will be used to compile host tool to generate metadata. "u_char" is a non-standard type, which is defined to "unsigned char" on glibc but not defined by some libc, e.g. musl. Let's replace "u_char" with "unsigned char" Signed-off-by: Đoàn Trần Công Danh Reviewed-by: Samuel Thibault Cc: stable Link: https://lore.kernel.org/r/b75743026aaee2d81efe3d7f2e8fa47f7d0b8ea7.1665736571.git.congdanhqx@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/accessibility/speakup/utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/accessibility/speakup/utils.h b/drivers/accessibility/speakup/utils.h index 4bf2ee8ac246a..4ce9a12f7664d 100644 --- a/drivers/accessibility/speakup/utils.h +++ b/drivers/accessibility/speakup/utils.h @@ -54,7 +54,7 @@ static inline int oops(const char *msg, const char *info) static inline struct st_key *hash_name(char *name) { - u_char *pn = (u_char *)name; + unsigned char *pn = (unsigned char *)name; int hash = 0; while (*pn) { -- GitLab From a830a15678593948f3271a5a398c9b67d8beedb9 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sun, 2 Oct 2022 00:33:27 +0300 Subject: [PATCH 1417/1927] drm: rcar-du: Fix Kconfig dependency between RCAR_DU and RCAR_MIPI_DSI When the R-Car MIPI DSI driver was added, it was a standalone encoder driver without any dependency to or from the R-Car DU driver. Commit 957fe62d7d15 ("drm: rcar-du: Fix DSI enable & disable sequence") then added a direct call from the DU driver to the MIPI DSI driver, without updating Kconfig to take the new dependency into account. Fix it the same way that the LVDS encoder is handled. Fixes: 957fe62d7d15 ("drm: rcar-du: Fix DSI enable & disable sequence") Reported-by: kernel test robot Reviewed-by: Tomi Valkeinen Signed-off-by: Laurent Pinchart --- drivers/gpu/drm/rcar-du/Kconfig | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig index c959e8c6be7d4..fd2c2eaee26ba 100644 --- a/drivers/gpu/drm/rcar-du/Kconfig +++ b/drivers/gpu/drm/rcar-du/Kconfig @@ -44,13 +44,18 @@ config DRM_RCAR_LVDS select OF_FLATTREE select OF_OVERLAY -config DRM_RCAR_MIPI_DSI - tristate "R-Car DU MIPI DSI Encoder Support" - depends on DRM && DRM_BRIDGE && OF - select DRM_MIPI_DSI +config DRM_RCAR_USE_MIPI_DSI + bool "R-Car DU MIPI DSI Encoder Support" + depends on DRM_BRIDGE && OF + default DRM_RCAR_DU help Enable support for the R-Car Display Unit embedded MIPI DSI encoders. +config DRM_RCAR_MIPI_DSI + def_tristate DRM_RCAR_DU + depends on DRM_RCAR_USE_MIPI_DSI + select DRM_MIPI_DSI + config DRM_RCAR_VSP bool "R-Car DU VSP Compositor Support" if ARM default y if ARM64 -- GitLab From e5b0d06d9b10f5f43101bd6598b076c347f9295f Mon Sep 17 00:00:00 2001 From: Alexander Potapenko Date: Fri, 4 Nov 2022 18:58:49 +0100 Subject: [PATCH 1418/1927] misc/vmw_vmci: fix an infoleak in vmci_host_do_receive_datagram() `struct vmci_event_qp` allocated by qp_notify_peer() contains padding, which may carry uninitialized data to the userspace, as observed by KMSAN: BUG: KMSAN: kernel-infoleak in instrument_copy_to_user ./include/linux/instrumented.h:121 instrument_copy_to_user ./include/linux/instrumented.h:121 _copy_to_user+0x5f/0xb0 lib/usercopy.c:33 copy_to_user ./include/linux/uaccess.h:169 vmci_host_do_receive_datagram drivers/misc/vmw_vmci/vmci_host.c:431 vmci_host_unlocked_ioctl+0x33d/0x43d0 drivers/misc/vmw_vmci/vmci_host.c:925 vfs_ioctl fs/ioctl.c:51 ... Uninit was stored to memory at: kmemdup+0x74/0xb0 mm/util.c:131 dg_dispatch_as_host drivers/misc/vmw_vmci/vmci_datagram.c:271 vmci_datagram_dispatch+0x4f8/0xfc0 drivers/misc/vmw_vmci/vmci_datagram.c:339 qp_notify_peer+0x19a/0x290 drivers/misc/vmw_vmci/vmci_queue_pair.c:1479 qp_broker_attach drivers/misc/vmw_vmci/vmci_queue_pair.c:1662 qp_broker_alloc+0x2977/0x2f30 drivers/misc/vmw_vmci/vmci_queue_pair.c:1750 vmci_qp_broker_alloc+0x96/0xd0 drivers/misc/vmw_vmci/vmci_queue_pair.c:1940 vmci_host_do_alloc_queuepair drivers/misc/vmw_vmci/vmci_host.c:488 vmci_host_unlocked_ioctl+0x24fd/0x43d0 drivers/misc/vmw_vmci/vmci_host.c:927 ... Local variable ev created at: qp_notify_peer+0x54/0x290 drivers/misc/vmw_vmci/vmci_queue_pair.c:1456 qp_broker_attach drivers/misc/vmw_vmci/vmci_queue_pair.c:1662 qp_broker_alloc+0x2977/0x2f30 drivers/misc/vmw_vmci/vmci_queue_pair.c:1750 Bytes 28-31 of 48 are uninitialized Memory access of size 48 starts at ffff888035155e00 Data copied to user address 0000000020000100 Use memset() to prevent the infoleaks. Also speculatively fix qp_notify_peer_local(), which may suffer from the same problem. Reported-by: syzbot+39be4da489ed2493ba25@syzkaller.appspotmail.com Cc: stable Fixes: 06164d2b72aa ("VMCI: queue pairs implementation.") Signed-off-by: Alexander Potapenko Reviewed-by: Vishnu Dasa Link: https://lore.kernel.org/r/20221104175849.2782567-1-glider@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/vmw_vmci/vmci_queue_pair.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/misc/vmw_vmci/vmci_queue_pair.c b/drivers/misc/vmw_vmci/vmci_queue_pair.c index e71068f7759b3..844264e1b88cc 100644 --- a/drivers/misc/vmw_vmci/vmci_queue_pair.c +++ b/drivers/misc/vmw_vmci/vmci_queue_pair.c @@ -854,6 +854,7 @@ static int qp_notify_peer_local(bool attach, struct vmci_handle handle) u32 context_id = vmci_get_context_id(); struct vmci_event_qp ev; + memset(&ev, 0, sizeof(ev)); ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER); ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, VMCI_CONTEXT_RESOURCE_ID); @@ -1467,6 +1468,7 @@ static int qp_notify_peer(bool attach, * kernel. */ + memset(&ev, 0, sizeof(ev)); ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER); ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, VMCI_CONTEXT_RESOURCE_ID); -- GitLab From 6e63153db50059fb78b8a8447b132664887d24e3 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Fri, 4 Nov 2022 10:13:34 +0800 Subject: [PATCH 1419/1927] siox: fix possible memory leak in siox_device_add() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If device_register() returns error in siox_device_add(), the name allocated by dev_set_name() need be freed. As comment of device_register() says, it should use put_device() to give up the reference in the error path. So fix this by calling put_device(), then the name can be freed in kobject_cleanup(), and sdevice is freed in siox_device_release(), set it to null in error path. Fixes: bbecb07fa0af ("siox: new driver framework for eckelmann SIOX") Signed-off-by: Yang Yingliang Reviewed-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20221104021334.618189-1-yangyingliang@huawei.com Signed-off-by: Greg Kroah-Hartman --- drivers/siox/siox-core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/siox/siox-core.c b/drivers/siox/siox-core.c index 7c4f32d769666..561408583b2bf 100644 --- a/drivers/siox/siox-core.c +++ b/drivers/siox/siox-core.c @@ -839,6 +839,8 @@ static struct siox_device *siox_device_add(struct siox_master *smaster, err_device_register: /* don't care to make the buffer smaller again */ + put_device(&sdevice->dev); + sdevice = NULL; err_buf_alloc: siox_master_unlock(smaster); -- GitLab From ab126f51c93a15093df604f661c9480854c005a3 Mon Sep 17 00:00:00 2001 From: "Maciej W. Rozycki" Date: Fri, 23 Sep 2022 19:52:08 +0100 Subject: [PATCH 1420/1927] parport_pc: Avoid FIFO port location truncation Match the data type of a temporary holding a reference to the FIFO port with the type of the original reference coming from `struct parport', avoiding data truncation with LP64 ports such as SPARC64 that refer to PCI port I/O locations via their corresponding MMIO addresses and will therefore have non-zero bits in the high 32-bit part of the reference. And in any case it is cleaner to have the data types matching here. Signed-off-by: Maciej W. Rozycki Fixes: 1da177e4c3f4 ("Linux-2.6.12-rc2") Link: https://lore.kernel.org/linux-pci/20220419033752.GA1101844@bhelgaas/ Acked-by: Sudip Mukherjee Link: https://lore.kernel.org/r/alpine.DEB.2.21.2209231912550.29493@angie.orcam.me.uk Signed-off-by: Greg Kroah-Hartman --- drivers/parport/parport_pc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/parport/parport_pc.c b/drivers/parport/parport_pc.c index 7c45927e2131c..5784dc20fb382 100644 --- a/drivers/parport/parport_pc.c +++ b/drivers/parport/parport_pc.c @@ -468,7 +468,7 @@ static size_t parport_pc_fifo_write_block_pio(struct parport *port, const unsigned char *bufp = buf; size_t left = length; unsigned long expire = jiffies + port->physport->cad->timeout; - const int fifo = FIFO(port); + const unsigned long fifo = FIFO(port); int poll_for = 8; /* 80 usecs */ const struct parport_pc_private *priv = port->physport->private_data; const int fifo_depth = priv->fifo_depth; -- GitLab From 3ce00bb7e91cf57d723905371507af57182c37ef Mon Sep 17 00:00:00 2001 From: Carlos Llamas Date: Fri, 4 Nov 2022 23:12:35 +0000 Subject: [PATCH 1421/1927] binder: validate alloc->mm in ->mmap() handler Since commit 1da52815d5f1 ("binder: fix alloc->vma_vm_mm null-ptr dereference") binder caches a pointer to the current->mm during open(). This fixes a null-ptr dereference reported by syzkaller. Unfortunately, it also opens the door for a process to update its mm after the open(), (e.g. via execve) making the cached alloc->mm pointer invalid. Things get worse when the process continues to mmap() a vma. From this point forward, binder will attempt to find this vma using an obsolete alloc->mm reference. Such as in binder_update_page_range(), where the wrong vma is obtained via vma_lookup(), yet binder proceeds to happily insert new pages into it. To avoid this issue fail the ->mmap() callback if we detect a mismatch between the vma->vm_mm and the original alloc->mm pointer. This prevents alloc->vm_addr from getting set, so that any subsequent vma_lookup() calls fail as expected. Fixes: 1da52815d5f1 ("binder: fix alloc->vma_vm_mm null-ptr dereference") Reported-by: Jann Horn Cc: # 5.15+ Signed-off-by: Carlos Llamas Acked-by: Todd Kjos Link: https://lore.kernel.org/r/20221104231235.348958-1-cmllamas@google.com Signed-off-by: Greg Kroah-Hartman --- drivers/android/binder_alloc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 1c39cfce32fad..4ad42b0f75cd9 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -739,6 +739,12 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, const char *failure_string; struct binder_buffer *buffer; + if (unlikely(vma->vm_mm != alloc->mm)) { + ret = -EINVAL; + failure_string = "invalid vma->vm_mm"; + goto err_invalid_mm; + } + mutex_lock(&binder_alloc_mmap_lock); if (alloc->buffer_size) { ret = -EBUSY; @@ -785,6 +791,7 @@ int binder_alloc_mmap_handler(struct binder_alloc *alloc, alloc->buffer_size = 0; err_already_mapped: mutex_unlock(&binder_alloc_mmap_lock); +err_invalid_mm: binder_alloc_debug(BINDER_DEBUG_USER_ERROR, "%s: %d %lx-%lx %s failed %d\n", __func__, alloc->pid, vma->vm_start, vma->vm_end, -- GitLab From 1760eb547276299ab0c6a6cd3d29469e54ade615 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sun, 2 Oct 2022 00:31:10 +0300 Subject: [PATCH 1422/1927] drm: rcar-du: Drop leftovers dependencies from Kconfig Commit 841281fe52a7 ("drm: rcar-du: Drop LVDS device tree backward compatibility") has removed device tree overlay sources used for backward compatibility with old bindings, but forgot to remove related dependencies from Kconfig. Fix it. Fixes: 841281fe52a7 ("drm: rcar-du: Drop LVDS device tree backward compatibility") Reviewed-by: Geert Uytterhoeven Reviewed-by: Kieran Bingham Signed-off-by: Laurent Pinchart --- drivers/gpu/drm/rcar-du/Kconfig | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig index c959e8c6be7d4..13c4ad327771e 100644 --- a/drivers/gpu/drm/rcar-du/Kconfig +++ b/drivers/gpu/drm/rcar-du/Kconfig @@ -41,8 +41,6 @@ config DRM_RCAR_LVDS depends on DRM_RCAR_USE_LVDS select DRM_KMS_HELPER select DRM_PANEL - select OF_FLATTREE - select OF_OVERLAY config DRM_RCAR_MIPI_DSI tristate "R-Car DU MIPI DSI Encoder Support" -- GitLab From b1a90f510230afa9483e38fccbf9e4274c92aa8c Mon Sep 17 00:00:00 2001 From: Biju Das Date: Tue, 20 Sep 2022 11:54:59 +0100 Subject: [PATCH 1423/1927] dt-bindings: display: bridge: Document RZ/G2L MIPI DSI TX bindings The RZ/G2L MIPI DSI TX is embedded in the Renesas RZ/G2L family SoC's. It can operate in DSI mode, with up to four data lanes. Signed-off-by: Biju Das Reviewed-by: Rob Herring Reviewed-by: Laurent Pinchart Reviewed-by: Geert Uytterhoeven Signed-off-by: Laurent Pinchart --- .../bindings/display/bridge/renesas,dsi.yaml | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml new file mode 100644 index 0000000000000..131d5b63ec4f7 --- /dev/null +++ b/Documentation/devicetree/bindings/display/bridge/renesas,dsi.yaml @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/display/bridge/renesas,dsi.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas RZ/G2L MIPI DSI Encoder + +maintainers: + - Biju Das + +description: | + This binding describes the MIPI DSI encoder embedded in the Renesas + RZ/G2L alike family of SoC's. The encoder can operate in DSI mode, with + up to four data lanes. + +allOf: + - $ref: /schemas/display/dsi-controller.yaml# + +properties: + compatible: + items: + - enum: + - renesas,r9a07g044-mipi-dsi # RZ/G2{L,LC} + - const: renesas,rzg2l-mipi-dsi + + reg: + maxItems: 1 + + interrupts: + items: + - description: Sequence operation channel 0 interrupt + - description: Sequence operation channel 1 interrupt + - description: Video-Input operation channel 1 interrupt + - description: DSI Packet Receive interrupt + - description: DSI Fatal Error interrupt + - description: DSI D-PHY PPI interrupt + - description: Debug interrupt + + interrupt-names: + items: + - const: seq0 + - const: seq1 + - const: vin1 + - const: rcv + - const: ferr + - const: ppi + - const: debug + + clocks: + items: + - description: DSI D-PHY PLL multiplied clock + - description: DSI D-PHY system clock + - description: DSI AXI bus clock + - description: DSI Register access clock + - description: DSI Video clock + - description: DSI D-PHY Escape mode transmit clock + + clock-names: + items: + - const: pllclk + - const: sysclk + - const: aclk + - const: pclk + - const: vclk + - const: lpclk + + resets: + items: + - description: MIPI_DSI_CMN_RSTB + - description: MIPI_DSI_ARESET_N + - description: MIPI_DSI_PRESET_N + + reset-names: + items: + - const: rst + - const: arst + - const: prst + + power-domains: + maxItems: 1 + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port@0: + $ref: /schemas/graph.yaml#/properties/port + description: Parallel input port + + port@1: + $ref: /schemas/graph.yaml#/$defs/port-base + unevaluatedProperties: false + description: DSI output port + + properties: + endpoint: + $ref: /schemas/media/video-interfaces.yaml# + unevaluatedProperties: false + + properties: + data-lanes: + description: array of physical DSI data lane indexes. + minItems: 1 + items: + - const: 1 + - const: 2 + - const: 3 + - const: 4 + + required: + - data-lanes + + required: + - port@0 + - port@1 + +required: + - compatible + - reg + - interrupts + - interrupt-names + - clocks + - clock-names + - resets + - reset-names + - power-domains + - ports + +additionalProperties: false + +examples: + - | + #include + #include + + dsi0: dsi@10850000 { + compatible = "renesas,r9a07g044-mipi-dsi", "renesas,rzg2l-mipi-dsi"; + reg = <0x10850000 0x20000>; + interrupts = , + , + , + , + , + , + ; + interrupt-names = "seq0", "seq1", "vin1", "rcv", + "ferr", "ppi", "debug"; + clocks = <&cpg CPG_MOD R9A07G044_MIPI_DSI_PLLCLK>, + <&cpg CPG_MOD R9A07G044_MIPI_DSI_SYSCLK>, + <&cpg CPG_MOD R9A07G044_MIPI_DSI_ACLK>, + <&cpg CPG_MOD R9A07G044_MIPI_DSI_PCLK>, + <&cpg CPG_MOD R9A07G044_MIPI_DSI_VCLK>, + <&cpg CPG_MOD R9A07G044_MIPI_DSI_LPCLK>; + clock-names = "pllclk", "sysclk", "aclk", "pclk", "vclk", "lpclk"; + resets = <&cpg R9A07G044_MIPI_DSI_CMN_RSTB>, + <&cpg R9A07G044_MIPI_DSI_ARESET_N>, + <&cpg R9A07G044_MIPI_DSI_PRESET_N>; + reset-names = "rst", "arst", "prst"; + power-domains = <&cpg>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + dsi0_in: endpoint { + remote-endpoint = <&du_out_dsi0>; + }; + }; + + port@1 { + reg = <1>; + dsi0_out: endpoint { + data-lanes = <1 2 3 4>; + remote-endpoint = <&adv7535_in>; + }; + }; + }; + }; +... -- GitLab From 7a043f978ed1433bddb088a732e9bb91501ebd76 Mon Sep 17 00:00:00 2001 From: Biju Das Date: Tue, 20 Sep 2022 11:55:00 +0100 Subject: [PATCH 1424/1927] drm: rcar-du: Add RZ/G2L DSI driver This driver supports the MIPI DSI encoder found in the RZ/G2L SoC. It currently supports DSI video mode only. Signed-off-by: Biju Das Acked-by: Sam Ravnborg Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- drivers/gpu/drm/rcar-du/Kconfig | 8 + drivers/gpu/drm/rcar-du/Makefile | 2 + drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c | 762 ++++++++++++++++++ drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi_regs.h | 151 ++++ 4 files changed, 923 insertions(+) create mode 100644 drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c create mode 100644 drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi_regs.h diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig index 13c4ad327771e..f14686549cbe4 100644 --- a/drivers/gpu/drm/rcar-du/Kconfig +++ b/drivers/gpu/drm/rcar-du/Kconfig @@ -49,6 +49,14 @@ config DRM_RCAR_MIPI_DSI help Enable support for the R-Car Display Unit embedded MIPI DSI encoders. +config DRM_RZG2L_MIPI_DSI + tristate "RZ/G2L MIPI DSI Encoder Support" + depends on DRM_BRIDGE && OF + depends on ARCH_RENESAS || COMPILE_TEST + select DRM_MIPI_DSI + help + Enable support for the RZ/G2L Display Unit embedded MIPI DSI encoders. + config DRM_RCAR_VSP bool "R-Car DU VSP Compositor Support" if ARM default y if ARM64 diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile index 6f132325c8b73..b8f2c82651d9d 100644 --- a/drivers/gpu/drm/rcar-du/Makefile +++ b/drivers/gpu/drm/rcar-du/Makefile @@ -14,3 +14,5 @@ obj-$(CONFIG_DRM_RCAR_DU) += rcar-du-drm.o obj-$(CONFIG_DRM_RCAR_DW_HDMI) += rcar_dw_hdmi.o obj-$(CONFIG_DRM_RCAR_LVDS) += rcar_lvds.o obj-$(CONFIG_DRM_RCAR_MIPI_DSI) += rcar_mipi_dsi.o + +obj-$(CONFIG_DRM_RZG2L_MIPI_DSI) += rzg2l_mipi_dsi.o diff --git a/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c new file mode 100644 index 0000000000000..8579208db218b --- /dev/null +++ b/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * RZ/G2L MIPI DSI Encoder Driver + * + * Copyright (C) 2022 Renesas Electronics Corporation + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "rzg2l_mipi_dsi_regs.h" + +struct rzg2l_mipi_dsi { + struct device *dev; + void __iomem *mmio; + + struct reset_control *rstc; + struct reset_control *arstc; + struct reset_control *prstc; + + struct mipi_dsi_host host; + struct drm_bridge bridge; + struct drm_bridge *next_bridge; + + struct clk *vclk; + + enum mipi_dsi_pixel_format format; + unsigned int num_data_lanes; + unsigned int lanes; + unsigned long mode_flags; +}; + +static inline struct rzg2l_mipi_dsi * +bridge_to_rzg2l_mipi_dsi(struct drm_bridge *bridge) +{ + return container_of(bridge, struct rzg2l_mipi_dsi, bridge); +} + +static inline struct rzg2l_mipi_dsi * +host_to_rzg2l_mipi_dsi(struct mipi_dsi_host *host) +{ + return container_of(host, struct rzg2l_mipi_dsi, host); +} + +struct rzg2l_mipi_dsi_timings { + unsigned long hsfreq_max; + u32 t_init; + u32 tclk_prepare; + u32 ths_prepare; + u32 tclk_zero; + u32 tclk_pre; + u32 tclk_post; + u32 tclk_trail; + u32 ths_zero; + u32 ths_trail; + u32 ths_exit; + u32 tlpx; +}; + +static const struct rzg2l_mipi_dsi_timings rzg2l_mipi_dsi_global_timings[] = { + { + .hsfreq_max = 80000, + .t_init = 79801, + .tclk_prepare = 8, + .ths_prepare = 13, + .tclk_zero = 33, + .tclk_pre = 24, + .tclk_post = 94, + .tclk_trail = 10, + .ths_zero = 23, + .ths_trail = 17, + .ths_exit = 13, + .tlpx = 6, + }, + { + .hsfreq_max = 125000, + .t_init = 79801, + .tclk_prepare = 8, + .ths_prepare = 12, + .tclk_zero = 33, + .tclk_pre = 15, + .tclk_post = 94, + .tclk_trail = 10, + .ths_zero = 23, + .ths_trail = 17, + .ths_exit = 13, + .tlpx = 6, + }, + { + .hsfreq_max = 250000, + .t_init = 79801, + .tclk_prepare = 8, + .ths_prepare = 12, + .tclk_zero = 33, + .tclk_pre = 13, + .tclk_post = 94, + .tclk_trail = 10, + .ths_zero = 23, + .ths_trail = 16, + .ths_exit = 13, + .tlpx = 6, + }, + { + .hsfreq_max = 360000, + .t_init = 79801, + .tclk_prepare = 8, + .ths_prepare = 10, + .tclk_zero = 33, + .tclk_pre = 4, + .tclk_post = 35, + .tclk_trail = 7, + .ths_zero = 16, + .ths_trail = 9, + .ths_exit = 13, + .tlpx = 6, + }, + { + .hsfreq_max = 720000, + .t_init = 79801, + .tclk_prepare = 8, + .ths_prepare = 9, + .tclk_zero = 33, + .tclk_pre = 4, + .tclk_post = 35, + .tclk_trail = 7, + .ths_zero = 16, + .ths_trail = 9, + .ths_exit = 13, + .tlpx = 6, + }, + { + .hsfreq_max = 1500000, + .t_init = 79801, + .tclk_prepare = 8, + .ths_prepare = 9, + .tclk_zero = 33, + .tclk_pre = 4, + .tclk_post = 35, + .tclk_trail = 7, + .ths_zero = 16, + .ths_trail = 9, + .ths_exit = 13, + .tlpx = 6, + }, +}; + +static void rzg2l_mipi_dsi_phy_write(struct rzg2l_mipi_dsi *dsi, u32 reg, u32 data) +{ + iowrite32(data, dsi->mmio + reg); +} + +static void rzg2l_mipi_dsi_link_write(struct rzg2l_mipi_dsi *dsi, u32 reg, u32 data) +{ + iowrite32(data, dsi->mmio + LINK_REG_OFFSET + reg); +} + +static u32 rzg2l_mipi_dsi_link_read(struct rzg2l_mipi_dsi *dsi, u32 reg) +{ + return ioread32(dsi->mmio + LINK_REG_OFFSET + reg); +} + +/* ----------------------------------------------------------------------------- + * Hardware Setup + */ + +static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, + const struct drm_display_mode *mode) +{ + const struct rzg2l_mipi_dsi_timings *dphy_timings; + unsigned long hsfreq; + unsigned int i, bpp; + u32 txsetr; + u32 clstptsetr; + u32 lptrnstsetr; + u32 clkkpt; + u32 clkbfht; + u32 clkstpt; + u32 golpbkt; + u32 dphyctrl0; + u32 dphytim0; + u32 dphytim1; + u32 dphytim2; + u32 dphytim3; + int ret; + + /* + * Relationship between hsclk and vclk must follow + * vclk * bpp = hsclk * 8 * lanes + * where vclk: video clock (Hz) + * bpp: video pixel bit depth + * hsclk: DSI HS Byte clock frequency (Hz) + * lanes: number of data lanes + * + * hsclk(bit) = hsclk(byte) * 8 + */ + bpp = mipi_dsi_pixel_format_to_bpp(dsi->format); + hsfreq = (mode->clock * bpp * 8) / (8 * dsi->lanes); + + /* All DSI global operation timings are set with recommended setting */ + for (i = 0; i < ARRAY_SIZE(rzg2l_mipi_dsi_global_timings); ++i) { + dphy_timings = &rzg2l_mipi_dsi_global_timings[i]; + if (hsfreq <= dphy_timings->hsfreq_max) + break; + } + + ret = pm_runtime_resume_and_get(dsi->dev); + if (ret < 0) + return ret; + + clk_set_rate(dsi->vclk, mode->clock * 1000); + + /* Initializing DPHY before accessing LINK */ + dphyctrl0 = DSIDPHYCTRL0_CAL_EN_HSRX_OFS | DSIDPHYCTRL0_CMN_MASTER_EN | + DSIDPHYCTRL0_RE_VDD_DETVCCQLV18 | DSIDPHYCTRL0_EN_BGR; + + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYCTRL0, dphyctrl0); + usleep_range(20, 30); + + dphyctrl0 |= DSIDPHYCTRL0_EN_LDO1200; + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYCTRL0, dphyctrl0); + usleep_range(10, 20); + + dphytim0 = DSIDPHYTIM0_TCLK_MISS(0) | + DSIDPHYTIM0_T_INIT(dphy_timings->t_init); + dphytim1 = DSIDPHYTIM1_THS_PREPARE(dphy_timings->ths_prepare) | + DSIDPHYTIM1_TCLK_PREPARE(dphy_timings->tclk_prepare) | + DSIDPHYTIM1_THS_SETTLE(0) | + DSIDPHYTIM1_TCLK_SETTLE(0); + dphytim2 = DSIDPHYTIM2_TCLK_TRAIL(dphy_timings->tclk_trail) | + DSIDPHYTIM2_TCLK_POST(dphy_timings->tclk_post) | + DSIDPHYTIM2_TCLK_PRE(dphy_timings->tclk_pre) | + DSIDPHYTIM2_TCLK_ZERO(dphy_timings->tclk_zero); + dphytim3 = DSIDPHYTIM3_TLPX(dphy_timings->tlpx) | + DSIDPHYTIM3_THS_EXIT(dphy_timings->ths_exit) | + DSIDPHYTIM3_THS_TRAIL(dphy_timings->ths_trail) | + DSIDPHYTIM3_THS_ZERO(dphy_timings->ths_zero); + + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM0, dphytim0); + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM1, dphytim1); + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM2, dphytim2); + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM3, dphytim3); + + ret = reset_control_deassert(dsi->rstc); + if (ret < 0) + goto err_pm_put; + + udelay(1); + + /* Enable Data lanes and Clock lanes */ + txsetr = TXSETR_DLEN | TXSETR_NUMLANEUSE(dsi->lanes - 1) | TXSETR_CLEN; + rzg2l_mipi_dsi_link_write(dsi, TXSETR, txsetr); + + /* + * Global timings characteristic depends on high speed Clock Frequency + * Currently MIPI DSI-IF just supports maximum FHD@60 with: + * - videoclock = 148.5 (MHz) + * - bpp: maximum 24bpp + * - data lanes: maximum 4 lanes + * Therefore maximum hsclk will be 891 Mbps. + */ + if (hsfreq > 445500) { + clkkpt = 12; + clkbfht = 15; + clkstpt = 48; + golpbkt = 75; + } else if (hsfreq > 250000) { + clkkpt = 7; + clkbfht = 8; + clkstpt = 27; + golpbkt = 40; + } else { + clkkpt = 8; + clkbfht = 6; + clkstpt = 24; + golpbkt = 29; + } + + clstptsetr = CLSTPTSETR_CLKKPT(clkkpt) | CLSTPTSETR_CLKBFHT(clkbfht) | + CLSTPTSETR_CLKSTPT(clkstpt); + rzg2l_mipi_dsi_link_write(dsi, CLSTPTSETR, clstptsetr); + + lptrnstsetr = LPTRNSTSETR_GOLPBKT(golpbkt); + rzg2l_mipi_dsi_link_write(dsi, LPTRNSTSETR, lptrnstsetr); + + return 0; + +err_pm_put: + pm_runtime_put(dsi->dev); + + return ret; +} + +static void rzg2l_mipi_dsi_stop(struct rzg2l_mipi_dsi *dsi) +{ + reset_control_assert(dsi->rstc); + pm_runtime_put(dsi->dev); +} + +static void rzg2l_mipi_dsi_set_display_timing(struct rzg2l_mipi_dsi *dsi, + const struct drm_display_mode *mode) +{ + u32 vich1ppsetr; + u32 vich1vssetr; + u32 vich1vpsetr; + u32 vich1hssetr; + u32 vich1hpsetr; + int dsi_format; + u32 delay[2]; + u8 index; + + /* Configuration for Pixel Packet */ + dsi_format = mipi_dsi_pixel_format_to_bpp(dsi->format); + switch (dsi_format) { + case 24: + vich1ppsetr = VICH1PPSETR_DT_RGB24; + break; + case 18: + vich1ppsetr = VICH1PPSETR_DT_RGB18; + break; + } + + if ((dsi->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE) && + !(dsi->mode_flags & MIPI_DSI_MODE_VIDEO_BURST)) + vich1ppsetr |= VICH1PPSETR_TXESYNC_PULSE; + + rzg2l_mipi_dsi_link_write(dsi, VICH1PPSETR, vich1ppsetr); + + /* Configuration for Video Parameters */ + vich1vssetr = VICH1VSSETR_VACTIVE(mode->vdisplay) | + VICH1VSSETR_VSA(mode->vsync_end - mode->vsync_start); + vich1vssetr |= (mode->flags & DRM_MODE_FLAG_PVSYNC) ? + VICH1VSSETR_VSPOL_HIGH : VICH1VSSETR_VSPOL_LOW; + + vich1vpsetr = VICH1VPSETR_VFP(mode->vsync_start - mode->vdisplay) | + VICH1VPSETR_VBP(mode->vtotal - mode->vsync_end); + + vich1hssetr = VICH1HSSETR_HACTIVE(mode->hdisplay) | + VICH1HSSETR_HSA(mode->hsync_end - mode->hsync_start); + vich1hssetr |= (mode->flags & DRM_MODE_FLAG_PHSYNC) ? + VICH1HSSETR_HSPOL_HIGH : VICH1HSSETR_HSPOL_LOW; + + vich1hpsetr = VICH1HPSETR_HFP(mode->hsync_start - mode->hdisplay) | + VICH1HPSETR_HBP(mode->htotal - mode->hsync_end); + + rzg2l_mipi_dsi_link_write(dsi, VICH1VSSETR, vich1vssetr); + rzg2l_mipi_dsi_link_write(dsi, VICH1VPSETR, vich1vpsetr); + rzg2l_mipi_dsi_link_write(dsi, VICH1HSSETR, vich1hssetr); + rzg2l_mipi_dsi_link_write(dsi, VICH1HPSETR, vich1hpsetr); + + /* + * Configuration for Delay Value + * Delay value based on 2 ranges of video clock. + * 74.25MHz is videoclock of HD@60p or FHD@30p + */ + if (mode->clock > 74250) { + delay[0] = 231; + delay[1] = 216; + } else { + delay[0] = 220; + delay[1] = 212; + } + + if (dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS) + index = 0; + else + index = 1; + + rzg2l_mipi_dsi_link_write(dsi, VICH1SET1R, + VICH1SET1R_DLY(delay[index])); +} + +static int rzg2l_mipi_dsi_start_hs_clock(struct rzg2l_mipi_dsi *dsi) +{ + bool is_clk_cont; + u32 hsclksetr; + u32 status; + int ret; + + is_clk_cont = !(dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS); + + /* Start HS clock */ + hsclksetr = HSCLKSETR_HSCLKRUN_HS | (is_clk_cont ? + HSCLKSETR_HSCLKMODE_CONT : + HSCLKSETR_HSCLKMODE_NON_CONT); + rzg2l_mipi_dsi_link_write(dsi, HSCLKSETR, hsclksetr); + + if (is_clk_cont) { + ret = read_poll_timeout(rzg2l_mipi_dsi_link_read, status, + status & PLSR_CLLP2HS, + 2000, 20000, false, dsi, PLSR); + if (ret < 0) { + dev_err(dsi->dev, "failed to start HS clock\n"); + return ret; + } + } + + dev_dbg(dsi->dev, "Start High Speed Clock with %s clock mode", + is_clk_cont ? "continuous" : "non-continuous"); + + return 0; +} + +static int rzg2l_mipi_dsi_stop_hs_clock(struct rzg2l_mipi_dsi *dsi) +{ + bool is_clk_cont; + u32 status; + int ret; + + is_clk_cont = !(dsi->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS); + + /* Stop HS clock */ + rzg2l_mipi_dsi_link_write(dsi, HSCLKSETR, + is_clk_cont ? HSCLKSETR_HSCLKMODE_CONT : + HSCLKSETR_HSCLKMODE_NON_CONT); + + if (is_clk_cont) { + ret = read_poll_timeout(rzg2l_mipi_dsi_link_read, status, + status & PLSR_CLHS2LP, + 2000, 20000, false, dsi, PLSR); + if (ret < 0) { + dev_err(dsi->dev, "failed to stop HS clock\n"); + return ret; + } + } + + return 0; +} + +static int rzg2l_mipi_dsi_start_video(struct rzg2l_mipi_dsi *dsi) +{ + u32 vich1set0r; + u32 status; + int ret; + + /* Configuration for Blanking sequence and start video input*/ + vich1set0r = VICH1SET0R_HFPNOLP | VICH1SET0R_HBPNOLP | + VICH1SET0R_HSANOLP | VICH1SET0R_VSTART; + rzg2l_mipi_dsi_link_write(dsi, VICH1SET0R, vich1set0r); + + ret = read_poll_timeout(rzg2l_mipi_dsi_link_read, status, + status & VICH1SR_VIRDY, + 2000, 20000, false, dsi, VICH1SR); + if (ret < 0) + dev_err(dsi->dev, "Failed to start video signal input\n"); + + return ret; +} + +static int rzg2l_mipi_dsi_stop_video(struct rzg2l_mipi_dsi *dsi) +{ + u32 status; + int ret; + + rzg2l_mipi_dsi_link_write(dsi, VICH1SET0R, VICH1SET0R_VSTPAFT); + ret = read_poll_timeout(rzg2l_mipi_dsi_link_read, status, + (status & VICH1SR_STOP) && (!(status & VICH1SR_RUNNING)), + 2000, 20000, false, dsi, VICH1SR); + if (ret < 0) + goto err; + + ret = read_poll_timeout(rzg2l_mipi_dsi_link_read, status, + !(status & LINKSR_HSBUSY), + 2000, 20000, false, dsi, LINKSR); + if (ret < 0) + goto err; + + return 0; + +err: + dev_err(dsi->dev, "Failed to stop video signal input\n"); + return ret; +} + +/* ----------------------------------------------------------------------------- + * Bridge + */ + +static int rzg2l_mipi_dsi_attach(struct drm_bridge *bridge, + enum drm_bridge_attach_flags flags) +{ + struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + + return drm_bridge_attach(bridge->encoder, dsi->next_bridge, bridge, + flags); +} + +static void rzg2l_mipi_dsi_atomic_enable(struct drm_bridge *bridge, + struct drm_bridge_state *old_bridge_state) +{ + struct drm_atomic_state *state = old_bridge_state->base.state; + struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + const struct drm_display_mode *mode; + struct drm_connector *connector; + struct drm_crtc *crtc; + int ret; + + connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder); + crtc = drm_atomic_get_new_connector_state(state, connector)->crtc; + mode = &drm_atomic_get_new_crtc_state(state, crtc)->adjusted_mode; + + ret = rzg2l_mipi_dsi_startup(dsi, mode); + if (ret < 0) + return; + + rzg2l_mipi_dsi_set_display_timing(dsi, mode); + + ret = rzg2l_mipi_dsi_start_hs_clock(dsi); + if (ret < 0) + goto err_stop; + + ret = rzg2l_mipi_dsi_start_video(dsi); + if (ret < 0) + goto err_stop_clock; + + return; + +err_stop_clock: + rzg2l_mipi_dsi_stop_hs_clock(dsi); +err_stop: + rzg2l_mipi_dsi_stop(dsi); +} + +static void rzg2l_mipi_dsi_atomic_disable(struct drm_bridge *bridge, + struct drm_bridge_state *old_bridge_state) +{ + struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + + rzg2l_mipi_dsi_stop_video(dsi); + rzg2l_mipi_dsi_stop_hs_clock(dsi); + rzg2l_mipi_dsi_stop(dsi); +} + +static enum drm_mode_status +rzg2l_mipi_dsi_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) +{ + if (mode->clock > 148500) + return MODE_CLOCK_HIGH; + + return MODE_OK; +} + +static const struct drm_bridge_funcs rzg2l_mipi_dsi_bridge_ops = { + .attach = rzg2l_mipi_dsi_attach, + .atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state, + .atomic_destroy_state = drm_atomic_helper_bridge_destroy_state, + .atomic_reset = drm_atomic_helper_bridge_reset, + .atomic_enable = rzg2l_mipi_dsi_atomic_enable, + .atomic_disable = rzg2l_mipi_dsi_atomic_disable, + .mode_valid = rzg2l_mipi_dsi_bridge_mode_valid, +}; + +/* ----------------------------------------------------------------------------- + * Host setting + */ + +static int rzg2l_mipi_dsi_host_attach(struct mipi_dsi_host *host, + struct mipi_dsi_device *device) +{ + struct rzg2l_mipi_dsi *dsi = host_to_rzg2l_mipi_dsi(host); + int ret; + + if (device->lanes > dsi->num_data_lanes) { + dev_err(dsi->dev, + "Number of lines of device (%u) exceeds host (%u)\n", + device->lanes, dsi->num_data_lanes); + return -EINVAL; + } + + switch (mipi_dsi_pixel_format_to_bpp(device->format)) { + case 24: + case 18: + break; + default: + dev_err(dsi->dev, "Unsupported format 0x%04x\n", device->format); + return -EINVAL; + } + + dsi->lanes = device->lanes; + dsi->format = device->format; + dsi->mode_flags = device->mode_flags; + + dsi->next_bridge = devm_drm_of_get_bridge(dsi->dev, dsi->dev->of_node, + 1, 0); + if (IS_ERR(dsi->next_bridge)) { + ret = PTR_ERR(dsi->next_bridge); + dev_err(dsi->dev, "failed to get next bridge: %d\n", ret); + return ret; + } + + drm_bridge_add(&dsi->bridge); + + return 0; +} + +static int rzg2l_mipi_dsi_host_detach(struct mipi_dsi_host *host, + struct mipi_dsi_device *device) +{ + struct rzg2l_mipi_dsi *dsi = host_to_rzg2l_mipi_dsi(host); + + drm_bridge_remove(&dsi->bridge); + + return 0; +} + +static const struct mipi_dsi_host_ops rzg2l_mipi_dsi_host_ops = { + .attach = rzg2l_mipi_dsi_host_attach, + .detach = rzg2l_mipi_dsi_host_detach, +}; + +/* ----------------------------------------------------------------------------- + * Power Management + */ + +static int __maybe_unused rzg2l_mipi_pm_runtime_suspend(struct device *dev) +{ + struct rzg2l_mipi_dsi *dsi = dev_get_drvdata(dev); + + reset_control_assert(dsi->prstc); + reset_control_assert(dsi->arstc); + + return 0; +} + +static int __maybe_unused rzg2l_mipi_pm_runtime_resume(struct device *dev) +{ + struct rzg2l_mipi_dsi *dsi = dev_get_drvdata(dev); + int ret; + + ret = reset_control_deassert(dsi->arstc); + if (ret < 0) + return ret; + + ret = reset_control_deassert(dsi->prstc); + if (ret < 0) + reset_control_assert(dsi->arstc); + + return ret; +} + +static const struct dev_pm_ops rzg2l_mipi_pm_ops = { + SET_RUNTIME_PM_OPS(rzg2l_mipi_pm_runtime_suspend, rzg2l_mipi_pm_runtime_resume, NULL) +}; + +/* ----------------------------------------------------------------------------- + * Probe & Remove + */ + +static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) +{ + struct rzg2l_mipi_dsi *dsi; + int ret; + + dsi = devm_kzalloc(&pdev->dev, sizeof(*dsi), GFP_KERNEL); + if (!dsi) + return -ENOMEM; + + platform_set_drvdata(pdev, dsi); + dsi->dev = &pdev->dev; + + ret = drm_of_get_data_lanes_count_ep(dsi->dev->of_node, 1, 0, 1, 4); + if (ret < 0) + return dev_err_probe(dsi->dev, ret, + "missing or invalid data-lanes property\n"); + + dsi->num_data_lanes = ret; + + dsi->mmio = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dsi->mmio)) + return PTR_ERR(dsi->mmio); + + dsi->vclk = devm_clk_get(dsi->dev, "vclk"); + if (IS_ERR(dsi->vclk)) + return PTR_ERR(dsi->vclk); + + dsi->rstc = devm_reset_control_get_exclusive(dsi->dev, "rst"); + if (IS_ERR(dsi->rstc)) + return dev_err_probe(dsi->dev, PTR_ERR(dsi->rstc), + "failed to get rst\n"); + + dsi->arstc = devm_reset_control_get_exclusive(dsi->dev, "arst"); + if (IS_ERR(dsi->arstc)) + return dev_err_probe(&pdev->dev, PTR_ERR(dsi->arstc), + "failed to get arst\n"); + + dsi->prstc = devm_reset_control_get_exclusive(dsi->dev, "prst"); + if (IS_ERR(dsi->prstc)) + return dev_err_probe(dsi->dev, PTR_ERR(dsi->prstc), + "failed to get prst\n"); + + platform_set_drvdata(pdev, dsi); + + pm_runtime_enable(dsi->dev); + + /* Initialize the DRM bridge. */ + dsi->bridge.funcs = &rzg2l_mipi_dsi_bridge_ops; + dsi->bridge.of_node = dsi->dev->of_node; + + /* Init host device */ + dsi->host.dev = dsi->dev; + dsi->host.ops = &rzg2l_mipi_dsi_host_ops; + ret = mipi_dsi_host_register(&dsi->host); + if (ret < 0) + goto err_pm_disable; + + return 0; + +err_pm_disable: + pm_runtime_disable(dsi->dev); + return ret; +} + +static int rzg2l_mipi_dsi_remove(struct platform_device *pdev) +{ + struct rzg2l_mipi_dsi *dsi = platform_get_drvdata(pdev); + + mipi_dsi_host_unregister(&dsi->host); + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static const struct of_device_id rzg2l_mipi_dsi_of_table[] = { + { .compatible = "renesas,rzg2l-mipi-dsi" }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, rzg2l_mipi_dsi_of_table); + +static struct platform_driver rzg2l_mipi_dsi_platform_driver = { + .probe = rzg2l_mipi_dsi_probe, + .remove = rzg2l_mipi_dsi_remove, + .driver = { + .name = "rzg2l-mipi-dsi", + .pm = &rzg2l_mipi_pm_ops, + .of_match_table = rzg2l_mipi_dsi_of_table, + }, +}; + +module_platform_driver(rzg2l_mipi_dsi_platform_driver); + +MODULE_AUTHOR("Biju Das "); +MODULE_DESCRIPTION("Renesas RZ/G2L MIPI DSI Encoder Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi_regs.h b/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi_regs.h new file mode 100644 index 0000000000000..1dbc16ec64a4b --- /dev/null +++ b/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi_regs.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * RZ/G2L MIPI DSI Interface Registers Definitions + * + * Copyright (C) 2022 Renesas Electronics Corporation + */ + +#ifndef __RZG2L_MIPI_DSI_REGS_H__ +#define __RZG2L_MIPI_DSI_REGS_H__ + +#include + +/* DPHY Registers */ +#define DSIDPHYCTRL0 0x00 +#define DSIDPHYCTRL0_CAL_EN_HSRX_OFS BIT(16) +#define DSIDPHYCTRL0_CMN_MASTER_EN BIT(8) +#define DSIDPHYCTRL0_RE_VDD_DETVCCQLV18 BIT(2) +#define DSIDPHYCTRL0_EN_LDO1200 BIT(1) +#define DSIDPHYCTRL0_EN_BGR BIT(0) + +#define DSIDPHYTIM0 0x04 +#define DSIDPHYTIM0_TCLK_MISS(x) ((x) << 24) +#define DSIDPHYTIM0_T_INIT(x) ((x) << 0) + +#define DSIDPHYTIM1 0x08 +#define DSIDPHYTIM1_THS_PREPARE(x) ((x) << 24) +#define DSIDPHYTIM1_TCLK_PREPARE(x) ((x) << 16) +#define DSIDPHYTIM1_THS_SETTLE(x) ((x) << 8) +#define DSIDPHYTIM1_TCLK_SETTLE(x) ((x) << 0) + +#define DSIDPHYTIM2 0x0c +#define DSIDPHYTIM2_TCLK_TRAIL(x) ((x) << 24) +#define DSIDPHYTIM2_TCLK_POST(x) ((x) << 16) +#define DSIDPHYTIM2_TCLK_PRE(x) ((x) << 8) +#define DSIDPHYTIM2_TCLK_ZERO(x) ((x) << 0) + +#define DSIDPHYTIM3 0x10 +#define DSIDPHYTIM3_TLPX(x) ((x) << 24) +#define DSIDPHYTIM3_THS_EXIT(x) ((x) << 16) +#define DSIDPHYTIM3_THS_TRAIL(x) ((x) << 8) +#define DSIDPHYTIM3_THS_ZERO(x) ((x) << 0) + +/* --------------------------------------------------------*/ +/* Link Registers */ +#define LINK_REG_OFFSET 0x10000 + +/* Link Status Register */ +#define LINKSR 0x10 +#define LINKSR_LPBUSY BIT(13) +#define LINKSR_HSBUSY BIT(12) +#define LINKSR_VICHRUN1 BIT(8) +#define LINKSR_SQCHRUN1 BIT(4) +#define LINKSR_SQCHRUN0 BIT(0) + +/* Tx Set Register */ +#define TXSETR 0x100 +#define TXSETR_NUMLANECAP (0x3 << 16) +#define TXSETR_DLEN (1 << 9) +#define TXSETR_CLEN (1 << 8) +#define TXSETR_NUMLANEUSE(x) (((x) & 0x3) << 0) + +/* HS Clock Set Register */ +#define HSCLKSETR 0x104 +#define HSCLKSETR_HSCLKMODE_CONT (1 << 1) +#define HSCLKSETR_HSCLKMODE_NON_CONT (0 << 1) +#define HSCLKSETR_HSCLKRUN_HS (1 << 0) +#define HSCLKSETR_HSCLKRUN_LP (0 << 0) + +/* Reset Control Register */ +#define RSTCR 0x110 +#define RSTCR_SWRST BIT(0) +#define RSTCR_FCETXSTP BIT(16) + +/* Reset Status Register */ +#define RSTSR 0x114 +#define RSTSR_DL0DIR (1 << 15) +#define RSTSR_DLSTPST (0xf << 8) +#define RSTSR_SWRSTV1 (1 << 4) +#define RSTSR_SWRSTIB (1 << 3) +#define RSTSR_SWRSTAPB (1 << 2) +#define RSTSR_SWRSTLP (1 << 1) +#define RSTSR_SWRSTHS (1 << 0) + +/* Clock Lane Stop Time Set Register */ +#define CLSTPTSETR 0x314 +#define CLSTPTSETR_CLKKPT(x) ((x) << 24) +#define CLSTPTSETR_CLKBFHT(x) ((x) << 16) +#define CLSTPTSETR_CLKSTPT(x) ((x) << 2) + +/* LP Transition Time Set Register */ +#define LPTRNSTSETR 0x318 +#define LPTRNSTSETR_GOLPBKT(x) ((x) << 0) + +/* Physical Lane Status Register */ +#define PLSR 0x320 +#define PLSR_CLHS2LP BIT(27) +#define PLSR_CLLP2HS BIT(26) + +/* Video-Input Channel 1 Set 0 Register */ +#define VICH1SET0R 0x400 +#define VICH1SET0R_VSEN BIT(12) +#define VICH1SET0R_HFPNOLP BIT(10) +#define VICH1SET0R_HBPNOLP BIT(9) +#define VICH1SET0R_HSANOLP BIT(8) +#define VICH1SET0R_VSTPAFT BIT(1) +#define VICH1SET0R_VSTART BIT(0) + +/* Video-Input Channel 1 Set 1 Register */ +#define VICH1SET1R 0x404 +#define VICH1SET1R_DLY(x) (((x) & 0xfff) << 2) + +/* Video-Input Channel 1 Status Register */ +#define VICH1SR 0x410 +#define VICH1SR_VIRDY BIT(3) +#define VICH1SR_RUNNING BIT(2) +#define VICH1SR_STOP BIT(1) +#define VICH1SR_START BIT(0) + +/* Video-Input Channel 1 Pixel Packet Set Register */ +#define VICH1PPSETR 0x420 +#define VICH1PPSETR_DT_RGB18 (0x1e << 16) +#define VICH1PPSETR_DT_RGB18_LS (0x2e << 16) +#define VICH1PPSETR_DT_RGB24 (0x3e << 16) +#define VICH1PPSETR_TXESYNC_PULSE (1 << 15) +#define VICH1PPSETR_VC(x) ((x) << 22) + +/* Video-Input Channel 1 Vertical Size Set Register */ +#define VICH1VSSETR 0x428 +#define VICH1VSSETR_VACTIVE(x) (((x) & 0x7fff) << 16) +#define VICH1VSSETR_VSPOL_LOW (1 << 15) +#define VICH1VSSETR_VSPOL_HIGH (0 << 15) +#define VICH1VSSETR_VSA(x) (((x) & 0xfff) << 0) + +/* Video-Input Channel 1 Vertical Porch Set Register */ +#define VICH1VPSETR 0x42c +#define VICH1VPSETR_VFP(x) (((x) & 0x1fff) << 16) +#define VICH1VPSETR_VBP(x) (((x) & 0x1fff) << 0) + +/* Video-Input Channel 1 Horizontal Size Set Register */ +#define VICH1HSSETR 0x430 +#define VICH1HSSETR_HACTIVE(x) (((x) & 0x7fff) << 16) +#define VICH1HSSETR_HSPOL_LOW (1 << 15) +#define VICH1HSSETR_HSPOL_HIGH (0 << 15) +#define VICH1HSSETR_HSA(x) (((x) & 0xfff) << 0) + +/* Video-Input Channel 1 Horizontal Porch Set Register */ +#define VICH1HPSETR 0x434 +#define VICH1HPSETR_HFP(x) (((x) & 0x1fff) << 16) +#define VICH1HPSETR_HBP(x) (((x) & 0x1fff) << 0) + +#endif /* __RZG2L_MIPI_DSI_REGS_H__ */ -- GitLab From 418bb3a69e1355c8977c528f8f3896a0aaaecb80 Mon Sep 17 00:00:00 2001 From: Biju Das Date: Tue, 20 Sep 2022 11:55:01 +0100 Subject: [PATCH 1425/1927] drm: rcar-du: rzg2l_mipi_dsi: Enhance device lanes check Enhance device lanes check by reading TXSETR register at probe(), and enforced in rzg2l_mipi_dsi_host_attach(). As per HW manual, we can read TXSETR register only after DPHY initialization. Suggested-by: Laurent Pinchart Signed-off-by: Biju Das Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c | 122 ++++++++++++++++------- 1 file changed, 88 insertions(+), 34 deletions(-) diff --git a/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c index 8579208db218b..aa95b85a29643 100644 --- a/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/rcar-du/rzg2l_mipi_dsi.c @@ -171,6 +171,11 @@ static void rzg2l_mipi_dsi_link_write(struct rzg2l_mipi_dsi *dsi, u32 reg, u32 d iowrite32(data, dsi->mmio + LINK_REG_OFFSET + reg); } +static u32 rzg2l_mipi_dsi_phy_read(struct rzg2l_mipi_dsi *dsi, u32 reg) +{ + return ioread32(dsi->mmio + reg); +} + static u32 rzg2l_mipi_dsi_link_read(struct rzg2l_mipi_dsi *dsi, u32 reg) { return ioread32(dsi->mmio + LINK_REG_OFFSET + reg); @@ -180,19 +185,11 @@ static u32 rzg2l_mipi_dsi_link_read(struct rzg2l_mipi_dsi *dsi, u32 reg) * Hardware Setup */ -static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, - const struct drm_display_mode *mode) +static int rzg2l_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi, + unsigned long hsfreq) { const struct rzg2l_mipi_dsi_timings *dphy_timings; - unsigned long hsfreq; - unsigned int i, bpp; - u32 txsetr; - u32 clstptsetr; - u32 lptrnstsetr; - u32 clkkpt; - u32 clkbfht; - u32 clkstpt; - u32 golpbkt; + unsigned int i; u32 dphyctrl0; u32 dphytim0; u32 dphytim1; @@ -200,19 +197,6 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, u32 dphytim3; int ret; - /* - * Relationship between hsclk and vclk must follow - * vclk * bpp = hsclk * 8 * lanes - * where vclk: video clock (Hz) - * bpp: video pixel bit depth - * hsclk: DSI HS Byte clock frequency (Hz) - * lanes: number of data lanes - * - * hsclk(bit) = hsclk(byte) * 8 - */ - bpp = mipi_dsi_pixel_format_to_bpp(dsi->format); - hsfreq = (mode->clock * bpp * 8) / (8 * dsi->lanes); - /* All DSI global operation timings are set with recommended setting */ for (i = 0; i < ARRAY_SIZE(rzg2l_mipi_dsi_global_timings); ++i) { dphy_timings = &rzg2l_mipi_dsi_global_timings[i]; @@ -220,12 +204,6 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, break; } - ret = pm_runtime_resume_and_get(dsi->dev); - if (ret < 0) - return ret; - - clk_set_rate(dsi->vclk, mode->clock * 1000); - /* Initializing DPHY before accessing LINK */ dphyctrl0 = DSIDPHYCTRL0_CAL_EN_HSRX_OFS | DSIDPHYCTRL0_CMN_MASTER_EN | DSIDPHYCTRL0_RE_VDD_DETVCCQLV18 | DSIDPHYCTRL0_EN_BGR; @@ -259,10 +237,62 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, ret = reset_control_deassert(dsi->rstc); if (ret < 0) - goto err_pm_put; + return ret; udelay(1); + return 0; +} + +static void rzg2l_mipi_dsi_dphy_exit(struct rzg2l_mipi_dsi *dsi) +{ + u32 dphyctrl0; + + dphyctrl0 = rzg2l_mipi_dsi_phy_read(dsi, DSIDPHYCTRL0); + + dphyctrl0 &= ~(DSIDPHYCTRL0_EN_LDO1200 | DSIDPHYCTRL0_EN_BGR); + rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYCTRL0, dphyctrl0); + + reset_control_assert(dsi->rstc); +} + +static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, + const struct drm_display_mode *mode) +{ + unsigned long hsfreq; + unsigned int bpp; + u32 txsetr; + u32 clstptsetr; + u32 lptrnstsetr; + u32 clkkpt; + u32 clkbfht; + u32 clkstpt; + u32 golpbkt; + int ret; + + /* + * Relationship between hsclk and vclk must follow + * vclk * bpp = hsclk * 8 * lanes + * where vclk: video clock (Hz) + * bpp: video pixel bit depth + * hsclk: DSI HS Byte clock frequency (Hz) + * lanes: number of data lanes + * + * hsclk(bit) = hsclk(byte) * 8 + */ + bpp = mipi_dsi_pixel_format_to_bpp(dsi->format); + hsfreq = (mode->clock * bpp * 8) / (8 * dsi->lanes); + + ret = pm_runtime_resume_and_get(dsi->dev); + if (ret < 0) + return ret; + + clk_set_rate(dsi->vclk, mode->clock * 1000); + + ret = rzg2l_mipi_dsi_dphy_init(dsi, hsfreq); + if (ret < 0) + goto err_phy; + /* Enable Data lanes and Clock lanes */ txsetr = TXSETR_DLEN | TXSETR_NUMLANEUSE(dsi->lanes - 1) | TXSETR_CLEN; rzg2l_mipi_dsi_link_write(dsi, TXSETR, txsetr); @@ -301,7 +331,8 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, return 0; -err_pm_put: +err_phy: + rzg2l_mipi_dsi_dphy_exit(dsi); pm_runtime_put(dsi->dev); return ret; @@ -309,7 +340,7 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, static void rzg2l_mipi_dsi_stop(struct rzg2l_mipi_dsi *dsi) { - reset_control_assert(dsi->rstc); + rzg2l_mipi_dsi_dphy_exit(dsi); pm_runtime_put(dsi->dev); } @@ -666,7 +697,9 @@ static const struct dev_pm_ops rzg2l_mipi_pm_ops = { static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) { + unsigned int num_data_lanes; struct rzg2l_mipi_dsi *dsi; + u32 txsetr; int ret; dsi = devm_kzalloc(&pdev->dev, sizeof(*dsi), GFP_KERNEL); @@ -681,7 +714,7 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) return dev_err_probe(dsi->dev, ret, "missing or invalid data-lanes property\n"); - dsi->num_data_lanes = ret; + num_data_lanes = ret; dsi->mmio = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dsi->mmio)) @@ -710,6 +743,24 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) pm_runtime_enable(dsi->dev); + ret = pm_runtime_resume_and_get(dsi->dev); + if (ret < 0) + goto err_pm_disable; + + /* + * TXSETR register can be read only after DPHY init. But during probe + * mode->clock and format are not available. So initialize DPHY with + * timing parameters for 80Mbps. + */ + ret = rzg2l_mipi_dsi_dphy_init(dsi, 80000); + if (ret < 0) + goto err_phy; + + txsetr = rzg2l_mipi_dsi_link_read(dsi, TXSETR); + dsi->num_data_lanes = min(((txsetr >> 16) & 3) + 1, num_data_lanes); + rzg2l_mipi_dsi_dphy_exit(dsi); + pm_runtime_put(dsi->dev); + /* Initialize the DRM bridge. */ dsi->bridge.funcs = &rzg2l_mipi_dsi_bridge_ops; dsi->bridge.of_node = dsi->dev->of_node; @@ -723,6 +774,9 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev) return 0; +err_phy: + rzg2l_mipi_dsi_dphy_exit(dsi); + pm_runtime_put(dsi->dev); err_pm_disable: pm_runtime_disable(dsi->dev); return ret; -- GitLab From cec9e59cae6071e58140baf54e47c00aaa39851b Mon Sep 17 00:00:00 2001 From: Nathan Huckleberry Date: Tue, 8 Nov 2022 17:14:25 -0700 Subject: [PATCH 1426/1927] drm: xlnx: Fix return type of zynqmp_dp_bridge_mode_valid The mode_valid field in drm_bridge_helper_funcs is expected to be of type enum drm_mode_status (* mode_valid) (struct drm_bridge *bridge, struct drm_display_mode *mode); The mismatched return type breaks forward edge kCFI since the underlying function definition does not match the function hook definition. A new warning in clang will catch this at compile time: drivers/gpu/drm/xlnx/zynqmp_dp.c:1573:16: error: incompatible function pointer types initializing 'enum drm_mode_status (*)(struct drm_bridge *, const struct drm_display_info *, const struct drm_display_mode *)' with an expression of type 'int (struct drm_bridge *, const struct drm_display_info *, const struct drm_display_mode *)' [-Werror,-Wincompatible-function-pointer-types-strict] .mode_valid = zynqmp_dp_bridge_mode_valid, ^~~~~~~~~~~~~~~~~~~~~~~~~~~ 1 error generated. The return type of zynqmp_dp_bridge_mode_valid should be changed from int to enum drm_mode_status. Reported-by: Dan Carpenter Link: https://github.com/ClangBuiltLinux/linux/issues/1703 Link: https://github.com/ClangBuiltLinux/linux/issues/1750 Signed-off-by: Nathan Huckleberry Reviewed-by: Laurent Pinchart [nathan: Rebase on drm-misc-next and fix conflicts Add note about new clang warning] Signed-off-by: Nathan Chancellor Signed-off-by: Laurent Pinchart --- drivers/gpu/drm/xlnx/zynqmp_dp.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/xlnx/zynqmp_dp.c b/drivers/gpu/drm/xlnx/zynqmp_dp.c index 7c9ae167eac7b..0a7b466446fb6 100644 --- a/drivers/gpu/drm/xlnx/zynqmp_dp.c +++ b/drivers/gpu/drm/xlnx/zynqmp_dp.c @@ -1362,9 +1362,10 @@ static void zynqmp_dp_bridge_detach(struct drm_bridge *bridge) zynqmp_dp_aux_cleanup(dp); } -static int zynqmp_dp_bridge_mode_valid(struct drm_bridge *bridge, - const struct drm_display_info *info, - const struct drm_display_mode *mode) +static enum drm_mode_status +zynqmp_dp_bridge_mode_valid(struct drm_bridge *bridge, + const struct drm_display_info *info, + const struct drm_display_mode *mode) { struct zynqmp_dp *dp = bridge_to_dp(bridge); int rate; -- GitLab From 6398acf34819da99e5110e7eae47e81ef5e4bb77 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 2 Nov 2022 12:08:24 +0200 Subject: [PATCH 1427/1927] drm/i915/display: move struct intel_link_m_n to intel_display_types.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit struct intel_crtc_state in intel_display_types.h actually needs the struct intel_link_m_n definition, while intel_display.h only needs the forward declaration. Signed-off-by: Jani Nikula Reviewed-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/1ec10e4415cf84c51b7eb51092e81876da0bc902.1667383630.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/intel_display.h | 10 +--------- drivers/gpu/drm/i915/display/intel_display_types.h | 9 +++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h index d9c8b8447f9e8..b78009a20e00f 100644 --- a/drivers/gpu/drm/i915/display/intel_display.h +++ b/drivers/gpu/drm/i915/display/intel_display.h @@ -53,6 +53,7 @@ struct intel_digital_port; struct intel_dp; struct intel_encoder; struct intel_initial_plane_config; +struct intel_link_m_n; struct intel_load_detect_pipe; struct intel_plane; struct intel_plane_state; @@ -287,15 +288,6 @@ enum aux_ch { #define aux_ch_name(a) ((a) + 'A') -/* Used by dp and fdi links */ -struct intel_link_m_n { - u32 tu; - u32 data_m; - u32 data_n; - u32 link_m; - u32 link_n; -}; - enum phy { PHY_NONE = -1, diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 64314273995dc..9258d50a07a05 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -969,6 +969,15 @@ struct intel_mpllb_state { u32 mpllb_sscstep; }; +/* Used by dp and fdi links */ +struct intel_link_m_n { + u32 tu; + u32 data_m; + u32 data_n; + u32 link_m; + u32 link_n; +}; + struct intel_crtc_state { /* * uapi (drm) state. This is the software state shown to userspace. -- GitLab From debc5a1ec0d195ffea70d11efeffb713de9fdbc7 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 8 Nov 2022 09:44:53 +0100 Subject: [PATCH 1428/1927] KVM: x86: use a separate asm-offsets.c file This already removes an ugly #include "" from asm-offsets.c, but especially it avoids a future error when trying to define asm-offsets for KVM's svm/svm.h header. This would not work for kernel/asm-offsets.c, because svm/svm.h includes kvm_cache_regs.h which is not in the include path when compiling asm-offsets.c. The problem is not there if the .c file is in arch/x86/kvm. Suggested-by: Sean Christopherson Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kernel/asm-offsets.c | 6 ------ arch/x86/kvm/.gitignore | 2 ++ arch/x86/kvm/Makefile | 9 +++++++++ arch/x86/kvm/kvm-asm-offsets.c | 18 ++++++++++++++++++ arch/x86/kvm/vmx/vmenter.S | 2 +- 5 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 arch/x86/kvm/.gitignore create mode 100644 arch/x86/kvm/kvm-asm-offsets.c diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index cb50589a7102f..437308004ef2e 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -19,7 +19,6 @@ #include #include #include -#include "../kvm/vmx/vmx.h" #ifdef CONFIG_XEN #include @@ -108,9 +107,4 @@ static void __used common(void) OFFSET(TSS_sp0, tss_struct, x86_tss.sp0); OFFSET(TSS_sp1, tss_struct, x86_tss.sp1); OFFSET(TSS_sp2, tss_struct, x86_tss.sp2); - - if (IS_ENABLED(CONFIG_KVM_INTEL)) { - BLANK(); - OFFSET(VMX_spec_ctrl, vcpu_vmx, spec_ctrl); - } } diff --git a/arch/x86/kvm/.gitignore b/arch/x86/kvm/.gitignore new file mode 100644 index 0000000000000..615d6ff35c009 --- /dev/null +++ b/arch/x86/kvm/.gitignore @@ -0,0 +1,2 @@ +/kvm-asm-offsets.s +/kvm-asm-offsets.h diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index 30f244b645234..a02cf9baacc88 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -34,3 +34,12 @@ endif obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o obj-$(CONFIG_KVM_AMD) += kvm-amd.o + +AFLAGS_vmx/vmenter.o := -iquote $(obj) +$(obj)/vmx/vmenter.o: $(obj)/kvm-asm-offsets.h + +$(obj)/kvm-asm-offsets.h: $(obj)/kvm-asm-offsets.s FORCE + $(call filechk,offsets,__KVM_ASM_OFFSETS_H__) + +targets += kvm-asm-offsets.s +clean-files += kvm-asm-offsets.h diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c new file mode 100644 index 0000000000000..9d84f2b32d7fe --- /dev/null +++ b/arch/x86/kvm/kvm-asm-offsets.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generate definitions needed by assembly language modules. + * This code generates raw asm output which is post-processed to extract + * and format the required data. + */ +#define COMPILE_OFFSETS + +#include +#include "vmx/vmx.h" + +static void __used common(void) +{ + if (IS_ENABLED(CONFIG_KVM_INTEL)) { + BLANK(); + OFFSET(VMX_spec_ctrl, vcpu_vmx, spec_ctrl); + } +} diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S index 8477d8bdd69c2..0b5db4de4d09e 100644 --- a/arch/x86/kvm/vmx/vmenter.S +++ b/arch/x86/kvm/vmx/vmenter.S @@ -1,12 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include #include -#include #include #include #include #include #include +#include "kvm-asm-offsets.h" #include "run_flags.h" #define WORD_SIZE (BITS_PER_LONG / 8) -- GitLab From 16fdc1de169ee0a4e59a8c02244414ec7acd55c3 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 30 Sep 2022 14:14:44 -0400 Subject: [PATCH 1429/1927] KVM: SVM: replace regs argument of __svm_vcpu_run() with vcpu_svm Since registers are reachable through vcpu_svm, and we will need to access more fields of that struct, pass it instead of the regs[] array. No functional change intended. Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/Makefile | 3 +++ arch/x86/kvm/kvm-asm-offsets.c | 6 ++++++ arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/svm/vmenter.S | 37 +++++++++++++++++----------------- 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index a02cf9baacc88..f453a0f96e243 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -35,6 +35,9 @@ obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o obj-$(CONFIG_KVM_AMD) += kvm-amd.o +AFLAGS_svm/vmenter.o := -iquote $(obj) +$(obj)/svm/vmenter.o: $(obj)/kvm-asm-offsets.h + AFLAGS_vmx/vmenter.o := -iquote $(obj) $(obj)/vmx/vmenter.o: $(obj)/kvm-asm-offsets.h diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c index 9d84f2b32d7fe..30db96852e2d9 100644 --- a/arch/x86/kvm/kvm-asm-offsets.c +++ b/arch/x86/kvm/kvm-asm-offsets.c @@ -8,9 +8,15 @@ #include #include "vmx/vmx.h" +#include "svm/svm.h" static void __used common(void) { + if (IS_ENABLED(CONFIG_KVM_AMD)) { + BLANK(); + OFFSET(SVM_vcpu_arch_regs, vcpu_svm, vcpu.arch.regs); + } + if (IS_ENABLED(CONFIG_KVM_INTEL)) { BLANK(); OFFSET(VMX_spec_ctrl, vcpu_vmx, spec_ctrl); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 58f0077d93579..b412bc5773c55 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3930,7 +3930,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) * vmcb02 when switching vmcbs for nested virtualization. */ vmload(svm->vmcb01.pa); - __svm_vcpu_run(vmcb_pa, (unsigned long *)&vcpu->arch.regs); + __svm_vcpu_run(vmcb_pa, svm); vmsave(svm->vmcb01.pa); vmload(__sme_page_pa(sd->save_area)); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 6a7686bf69000..447e25c9101a4 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -684,6 +684,6 @@ void sev_es_unmap_ghcb(struct vcpu_svm *svm); /* vmenter.S */ void __svm_sev_es_vcpu_run(unsigned long vmcb_pa); -void __svm_vcpu_run(unsigned long vmcb_pa, unsigned long *regs); +void __svm_vcpu_run(unsigned long vmcb_pa, struct vcpu_svm *svm); #endif diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index 723f8534986c3..f0ff41103e4ca 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -4,27 +4,28 @@ #include #include #include +#include "kvm-asm-offsets.h" #define WORD_SIZE (BITS_PER_LONG / 8) /* Intentionally omit RAX as it's context switched by hardware */ -#define VCPU_RCX __VCPU_REGS_RCX * WORD_SIZE -#define VCPU_RDX __VCPU_REGS_RDX * WORD_SIZE -#define VCPU_RBX __VCPU_REGS_RBX * WORD_SIZE +#define VCPU_RCX (SVM_vcpu_arch_regs + __VCPU_REGS_RCX * WORD_SIZE) +#define VCPU_RDX (SVM_vcpu_arch_regs + __VCPU_REGS_RDX * WORD_SIZE) +#define VCPU_RBX (SVM_vcpu_arch_regs + __VCPU_REGS_RBX * WORD_SIZE) /* Intentionally omit RSP as it's context switched by hardware */ -#define VCPU_RBP __VCPU_REGS_RBP * WORD_SIZE -#define VCPU_RSI __VCPU_REGS_RSI * WORD_SIZE -#define VCPU_RDI __VCPU_REGS_RDI * WORD_SIZE +#define VCPU_RBP (SVM_vcpu_arch_regs + __VCPU_REGS_RBP * WORD_SIZE) +#define VCPU_RSI (SVM_vcpu_arch_regs + __VCPU_REGS_RSI * WORD_SIZE) +#define VCPU_RDI (SVM_vcpu_arch_regs + __VCPU_REGS_RDI * WORD_SIZE) #ifdef CONFIG_X86_64 -#define VCPU_R8 __VCPU_REGS_R8 * WORD_SIZE -#define VCPU_R9 __VCPU_REGS_R9 * WORD_SIZE -#define VCPU_R10 __VCPU_REGS_R10 * WORD_SIZE -#define VCPU_R11 __VCPU_REGS_R11 * WORD_SIZE -#define VCPU_R12 __VCPU_REGS_R12 * WORD_SIZE -#define VCPU_R13 __VCPU_REGS_R13 * WORD_SIZE -#define VCPU_R14 __VCPU_REGS_R14 * WORD_SIZE -#define VCPU_R15 __VCPU_REGS_R15 * WORD_SIZE +#define VCPU_R8 (SVM_vcpu_arch_regs + __VCPU_REGS_R8 * WORD_SIZE) +#define VCPU_R9 (SVM_vcpu_arch_regs + __VCPU_REGS_R9 * WORD_SIZE) +#define VCPU_R10 (SVM_vcpu_arch_regs + __VCPU_REGS_R10 * WORD_SIZE) +#define VCPU_R11 (SVM_vcpu_arch_regs + __VCPU_REGS_R11 * WORD_SIZE) +#define VCPU_R12 (SVM_vcpu_arch_regs + __VCPU_REGS_R12 * WORD_SIZE) +#define VCPU_R13 (SVM_vcpu_arch_regs + __VCPU_REGS_R13 * WORD_SIZE) +#define VCPU_R14 (SVM_vcpu_arch_regs + __VCPU_REGS_R14 * WORD_SIZE) +#define VCPU_R15 (SVM_vcpu_arch_regs + __VCPU_REGS_R15 * WORD_SIZE) #endif .section .noinstr.text, "ax" @@ -32,7 +33,7 @@ /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode * @vmcb_pa: unsigned long - * @regs: unsigned long * (to guest registers) + * @svm: struct vcpu_svm * */ SYM_FUNC_START(__svm_vcpu_run) push %_ASM_BP @@ -47,13 +48,13 @@ SYM_FUNC_START(__svm_vcpu_run) #endif push %_ASM_BX - /* Save @regs. */ + /* Save @svm. */ push %_ASM_ARG2 /* Save @vmcb. */ push %_ASM_ARG1 - /* Move @regs to RAX. */ + /* Move @svm to RAX. */ mov %_ASM_ARG2, %_ASM_AX /* Load guest registers. */ @@ -89,7 +90,7 @@ SYM_FUNC_START(__svm_vcpu_run) FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE #endif - /* "POP" @regs to RAX. */ + /* "POP" @svm to RAX. */ pop %_ASM_AX /* Save all guest registers. */ -- GitLab From f7ef280132f9bf6f82acf5aa5c3c837206eef501 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 28 Oct 2022 17:30:07 -0400 Subject: [PATCH 1430/1927] KVM: SVM: adjust register allocation for __svm_vcpu_run() 32-bit ABI uses RAX/RCX/RDX as its argument registers, so they are in the way of instructions that hardcode their operands such as RDMSR/WRMSR or VMLOAD/VMRUN/VMSAVE. In preparation for moving vmload/vmsave to __svm_vcpu_run(), keep the pointer to the struct vcpu_svm in %rdi. In particular, it is now possible to load svm->vmcb01.pa in %rax without clobbering the struct vcpu_svm pointer. No functional change intended. Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/vmenter.S | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index f0ff41103e4ca..531510ab60720 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -54,29 +54,29 @@ SYM_FUNC_START(__svm_vcpu_run) /* Save @vmcb. */ push %_ASM_ARG1 - /* Move @svm to RAX. */ - mov %_ASM_ARG2, %_ASM_AX + /* Move @svm to RDI. */ + mov %_ASM_ARG2, %_ASM_DI + + /* "POP" @vmcb to RAX. */ + pop %_ASM_AX /* Load guest registers. */ - mov VCPU_RCX(%_ASM_AX), %_ASM_CX - mov VCPU_RDX(%_ASM_AX), %_ASM_DX - mov VCPU_RBX(%_ASM_AX), %_ASM_BX - mov VCPU_RBP(%_ASM_AX), %_ASM_BP - mov VCPU_RSI(%_ASM_AX), %_ASM_SI - mov VCPU_RDI(%_ASM_AX), %_ASM_DI + mov VCPU_RCX(%_ASM_DI), %_ASM_CX + mov VCPU_RDX(%_ASM_DI), %_ASM_DX + mov VCPU_RBX(%_ASM_DI), %_ASM_BX + mov VCPU_RBP(%_ASM_DI), %_ASM_BP + mov VCPU_RSI(%_ASM_DI), %_ASM_SI #ifdef CONFIG_X86_64 - mov VCPU_R8 (%_ASM_AX), %r8 - mov VCPU_R9 (%_ASM_AX), %r9 - mov VCPU_R10(%_ASM_AX), %r10 - mov VCPU_R11(%_ASM_AX), %r11 - mov VCPU_R12(%_ASM_AX), %r12 - mov VCPU_R13(%_ASM_AX), %r13 - mov VCPU_R14(%_ASM_AX), %r14 - mov VCPU_R15(%_ASM_AX), %r15 + mov VCPU_R8 (%_ASM_DI), %r8 + mov VCPU_R9 (%_ASM_DI), %r9 + mov VCPU_R10(%_ASM_DI), %r10 + mov VCPU_R11(%_ASM_DI), %r11 + mov VCPU_R12(%_ASM_DI), %r12 + mov VCPU_R13(%_ASM_DI), %r13 + mov VCPU_R14(%_ASM_DI), %r14 + mov VCPU_R15(%_ASM_DI), %r15 #endif - - /* "POP" @vmcb to RAX. */ - pop %_ASM_AX + mov VCPU_RDI(%_ASM_DI), %_ASM_DI /* Enter guest mode */ sti -- GitLab From f6d58266d731fd7e63163790aad21e0dbb1d5264 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 7 Nov 2022 04:17:29 -0500 Subject: [PATCH 1431/1927] KVM: SVM: retrieve VMCB from assembly Continue moving accesses to struct vcpu_svm to vmenter.S. Reducing the number of arguments limits the chance of mistakes due to different registers used for argument passing in 32- and 64-bit ABIs; pushing the VMCB argument and almost immediately popping it into a different register looks pretty weird. 32-bit ABI is not a concern for __svm_sev_es_vcpu_run() which is 64-bit only; however, it will soon need @svm to save/restore SPEC_CTRL so stay consistent with __svm_vcpu_run() and let them share the same prototype. No functional change intended. Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/kvm-asm-offsets.c | 2 ++ arch/x86/kvm/svm/svm.c | 5 ++--- arch/x86/kvm/svm/svm.h | 4 ++-- arch/x86/kvm/svm/vmenter.S | 20 ++++++++++---------- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c index 30db96852e2d9..f1b694e431aea 100644 --- a/arch/x86/kvm/kvm-asm-offsets.c +++ b/arch/x86/kvm/kvm-asm-offsets.c @@ -15,6 +15,8 @@ static void __used common(void) if (IS_ENABLED(CONFIG_KVM_AMD)) { BLANK(); OFFSET(SVM_vcpu_arch_regs, vcpu_svm, vcpu.arch.regs); + OFFSET(SVM_current_vmcb, vcpu_svm, current_vmcb); + OFFSET(KVM_VMCB_pa, kvm_vmcb_info, pa); } if (IS_ENABLED(CONFIG_KVM_INTEL)) { diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index b412bc5773c55..0c86c435c51f5 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3914,12 +3914,11 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - unsigned long vmcb_pa = svm->current_vmcb->pa; guest_state_enter_irqoff(); if (sev_es_guest(vcpu->kvm)) { - __svm_sev_es_vcpu_run(vmcb_pa); + __svm_sev_es_vcpu_run(svm); } else { struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); @@ -3930,7 +3929,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) * vmcb02 when switching vmcbs for nested virtualization. */ vmload(svm->vmcb01.pa); - __svm_vcpu_run(vmcb_pa, svm); + __svm_vcpu_run(svm); vmsave(svm->vmcb01.pa); vmload(__sme_page_pa(sd->save_area)); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 447e25c9101a4..7ff1879e73c56 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -683,7 +683,7 @@ void sev_es_unmap_ghcb(struct vcpu_svm *svm); /* vmenter.S */ -void __svm_sev_es_vcpu_run(unsigned long vmcb_pa); -void __svm_vcpu_run(unsigned long vmcb_pa, struct vcpu_svm *svm); +void __svm_sev_es_vcpu_run(struct vcpu_svm *svm); +void __svm_vcpu_run(struct vcpu_svm *svm); #endif diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index 531510ab60720..d07bac1952c52 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -32,7 +32,6 @@ /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode - * @vmcb_pa: unsigned long * @svm: struct vcpu_svm * */ SYM_FUNC_START(__svm_vcpu_run) @@ -49,16 +48,16 @@ SYM_FUNC_START(__svm_vcpu_run) push %_ASM_BX /* Save @svm. */ - push %_ASM_ARG2 - - /* Save @vmcb. */ push %_ASM_ARG1 +.ifnc _ASM_ARG1, _ASM_DI /* Move @svm to RDI. */ - mov %_ASM_ARG2, %_ASM_DI + mov %_ASM_ARG1, %_ASM_DI +.endif - /* "POP" @vmcb to RAX. */ - pop %_ASM_AX + /* Get svm->current_vmcb->pa into RAX. */ + mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX + mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX /* Load guest registers. */ mov VCPU_RCX(%_ASM_DI), %_ASM_CX @@ -170,7 +169,7 @@ SYM_FUNC_END(__svm_vcpu_run) /** * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode - * @vmcb_pa: unsigned long + * @svm: struct vcpu_svm * */ SYM_FUNC_START(__svm_sev_es_vcpu_run) push %_ASM_BP @@ -185,8 +184,9 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) #endif push %_ASM_BX - /* Move @vmcb to RAX. */ - mov %_ASM_ARG1, %_ASM_AX + /* Get svm->current_vmcb->pa into RAX. */ + mov SVM_current_vmcb(%_ASM_ARG1), %_ASM_AX + mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX /* Enter guest mode */ sti -- GitLab From 001459787158280d84fce1bc94cec49f34f48b9f Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 9 Nov 2022 04:54:40 -0500 Subject: [PATCH 1432/1927] KVM: SVM: remove unused field from struct vcpu_svm The pointer to svm_cpu_data in struct vcpu_svm looks interesting from the point of view of accessing it after vmexit, when the GSBASE is still containing the guest value. However, despite existing since the very first commit of drivers/kvm/svm.c (commit 6aa8b732ca01, "[PATCH] kvm: userspace interface", 2006-12-10), it was never set to anything. Ignore the opportunity to fix a 16 year old "bug" and delete it; doing things the "harder" way makes it possible to remove more old cruft. Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 7ff1879e73c56..626240707ba97 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -209,7 +209,6 @@ struct vcpu_svm { struct vmcb *vmcb; struct kvm_vmcb_info vmcb01; struct kvm_vmcb_info *current_vmcb; - struct svm_cpu_data *svm_data; u32 asid; u32 sysenter_esp_hi; u32 sysenter_eip_hi; -- GitLab From 181d0fb0bb023e8996b1cf7970e3708d72442b0b Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 9 Nov 2022 08:54:20 -0500 Subject: [PATCH 1433/1927] KVM: SVM: remove dead field from struct svm_cpu_data The "cpu" field of struct svm_cpu_data has been write-only since commit 4b656b120249 ("KVM: SVM: force new asid on vcpu migration", 2009-08-05). Remove it. Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/svm.c | 1 - arch/x86/kvm/svm/svm.h | 2 -- 2 files changed, 3 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0c86c435c51f5..0f873b2989318 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -665,7 +665,6 @@ static int svm_cpu_init(int cpu) sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); if (!sd) return ret; - sd->cpu = cpu; sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!sd->save_area) goto free_cpu_data; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 626240707ba97..7540db9902a61 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -280,8 +280,6 @@ struct vcpu_svm { }; struct svm_cpu_data { - int cpu; - u64 asid_generation; u32 max_asid; u32 next_asid; -- GitLab From 73412dfeea724e6bd775ba64d21157ff322eac9a Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 9 Nov 2022 09:07:55 -0500 Subject: [PATCH 1434/1927] KVM: SVM: do not allocate struct svm_cpu_data dynamically The svm_data percpu variable is a pointer, but it is allocated via svm_hardware_setup() when KVM is loaded. Unlike hardware_enable() this means that it is never NULL for the whole lifetime of KVM, and static allocation does not waste any memory compared to the status quo. It is also more efficient and more easily handled from assembly code, so do it and don't look back. Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 4 ++-- arch/x86/kvm/svm/svm.c | 41 +++++++++++++++-------------------------- arch/x86/kvm/svm/svm.h | 2 +- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 28064060413ac..9b66ee34e2642 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -196,7 +196,7 @@ static void sev_asid_free(struct kvm_sev_info *sev) __set_bit(sev->asid, sev_reclaim_asid_bitmap); for_each_possible_cpu(cpu) { - sd = per_cpu(svm_data, cpu); + sd = per_cpu_ptr(&svm_data, cpu); sd->sev_vmcbs[sev->asid] = NULL; } @@ -2600,7 +2600,7 @@ void sev_es_unmap_ghcb(struct vcpu_svm *svm) void pre_sev_run(struct vcpu_svm *svm, int cpu) { - struct svm_cpu_data *sd = per_cpu(svm_data, cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); int asid = sev_get_asid(svm->vcpu.kvm); /* Assign the asid allocated with this SEV guest */ diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0f873b2989318..48274c93d78bc 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -245,7 +245,7 @@ struct kvm_ldttss_desc { u32 zero1; } __attribute__((packed)); -DEFINE_PER_CPU(struct svm_cpu_data *, svm_data); +DEFINE_PER_CPU(struct svm_cpu_data, svm_data); /* * Only MSR_TSC_AUX is switched via the user return hook. EFER is switched via @@ -581,12 +581,7 @@ static int svm_hardware_enable(void) pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me); return -EINVAL; } - sd = per_cpu(svm_data, me); - if (!sd) { - pr_err("%s: svm_data is NULL on %d\n", __func__, me); - return -EINVAL; - } - + sd = per_cpu_ptr(&svm_data, me); sd->asid_generation = 1; sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; sd->next_asid = sd->max_asid + 1; @@ -646,41 +641,35 @@ static int svm_hardware_enable(void) static void svm_cpu_uninit(int cpu) { - struct svm_cpu_data *sd = per_cpu(svm_data, cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); - if (!sd) + if (!sd->save_area) return; - per_cpu(svm_data, cpu) = NULL; kfree(sd->sev_vmcbs); __free_page(sd->save_area); - kfree(sd); + sd->save_area = NULL; } static int svm_cpu_init(int cpu) { - struct svm_cpu_data *sd; + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); int ret = -ENOMEM; - sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL); - if (!sd) - return ret; + memset(sd, 0, sizeof(struct svm_cpu_data)); sd->save_area = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!sd->save_area) - goto free_cpu_data; + return ret; ret = sev_cpu_init(sd); if (ret) goto free_save_area; - per_cpu(svm_data, cpu) = sd; - return 0; free_save_area: __free_page(sd->save_area); -free_cpu_data: - kfree(sd); + sd->save_area = NULL; return ret; } @@ -1424,7 +1413,7 @@ static void svm_clear_current_vmcb(struct vmcb *vmcb) int i; for_each_online_cpu(i) - cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL); + cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL); } static void svm_vcpu_free(struct kvm_vcpu *vcpu) @@ -1449,7 +1438,7 @@ static void svm_vcpu_free(struct kvm_vcpu *vcpu) static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); if (sev_es_guest(vcpu->kvm)) sev_es_unmap_ghcb(svm); @@ -1486,7 +1475,7 @@ static void svm_prepare_host_switch(struct kvm_vcpu *vcpu) static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { struct vcpu_svm *svm = to_svm(vcpu); - struct svm_cpu_data *sd = per_cpu(svm_data, cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu); if (sd->current_vmcb != svm->vmcb) { sd->current_vmcb = svm->vmcb; @@ -3442,7 +3431,7 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath) static void reload_tss(struct kvm_vcpu *vcpu) { - struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); sd->tss_desc->type = 9; /* available 32/64-bit TSS */ load_TR_desc(); @@ -3450,7 +3439,7 @@ static void reload_tss(struct kvm_vcpu *vcpu) static void pre_svm_run(struct kvm_vcpu *vcpu) { - struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); struct vcpu_svm *svm = to_svm(vcpu); /* @@ -3919,7 +3908,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) if (sev_es_guest(vcpu->kvm)) { __svm_sev_es_vcpu_run(svm); } else { - struct svm_cpu_data *sd = per_cpu(svm_data, vcpu->cpu); + struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); /* * Use a single vmcb (vmcb01 because it's always valid) for diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 7540db9902a61..2af6a71126c1e 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -293,7 +293,7 @@ struct svm_cpu_data { struct vmcb **sev_vmcbs; }; -DECLARE_PER_CPU(struct svm_cpu_data *, svm_data); +DECLARE_PER_CPU(struct svm_cpu_data, svm_data); void recalc_intercepts(struct vcpu_svm *svm); -- GitLab From e61ab42de874c5af8c5d98b327c77a374d9e7da1 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 7 Nov 2022 05:14:27 -0500 Subject: [PATCH 1435/1927] KVM: SVM: move guest vmsave/vmload back to assembly It is error-prone that code after vmexit cannot access percpu data because GSBASE has not been restored yet. It forces MSR_IA32_SPEC_CTRL save/restore to happen very late, after the predictor untraining sequence, and it gets in the way of return stack depth tracking (a retbleed mitigation that is in linux-next as of 2022-11-09). As a first step towards fixing that, move the VMCB VMSAVE/VMLOAD to assembly, essentially undoing commit fb0c4a4fee5a ("KVM: SVM: move VMLOAD/VMSAVE to C code", 2021-03-15). The reason for that commit was that it made it simpler to use a different VMCB for VMLOAD/VMSAVE versus VMRUN; but that is not a big hassle anymore thanks to the kvm-asm-offsets machinery and other related cleanups. The idea on how to number the exception tables is stolen from a prototype patch by Peter Zijlstra. Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Link: Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/kvm-asm-offsets.c | 1 + arch/x86/kvm/svm/svm.c | 9 ------- arch/x86/kvm/svm/vmenter.S | 49 ++++++++++++++++++++++++++-------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c index f1b694e431aea..f83e88b85bf21 100644 --- a/arch/x86/kvm/kvm-asm-offsets.c +++ b/arch/x86/kvm/kvm-asm-offsets.c @@ -16,6 +16,7 @@ static void __used common(void) BLANK(); OFFSET(SVM_vcpu_arch_regs, vcpu_svm, vcpu.arch.regs); OFFSET(SVM_current_vmcb, vcpu_svm, current_vmcb); + OFFSET(SVM_vmcb01, vcpu_svm, vmcb01); OFFSET(KVM_VMCB_pa, kvm_vmcb_info, pa); } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 48274c93d78bc..4e3a47eb50025 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3910,16 +3910,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) } else { struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); - /* - * Use a single vmcb (vmcb01 because it's always valid) for - * context switching guest state via VMLOAD/VMSAVE, that way - * the state doesn't need to be copied between vmcb01 and - * vmcb02 when switching vmcbs for nested virtualization. - */ - vmload(svm->vmcb01.pa); __svm_vcpu_run(svm); - vmsave(svm->vmcb01.pa); - vmload(__sme_page_pa(sd->save_area)); } diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index d07bac1952c52..5bc2ed7d79c07 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -28,6 +28,8 @@ #define VCPU_R15 (SVM_vcpu_arch_regs + __VCPU_REGS_R15 * WORD_SIZE) #endif +#define SVM_vmcb01_pa (SVM_vmcb01 + KVM_VMCB_pa) + .section .noinstr.text, "ax" /** @@ -55,6 +57,16 @@ SYM_FUNC_START(__svm_vcpu_run) mov %_ASM_ARG1, %_ASM_DI .endif + /* + * Use a single vmcb (vmcb01 because it's always valid) for + * context switching guest state via VMLOAD/VMSAVE, that way + * the state doesn't need to be copied between vmcb01 and + * vmcb02 when switching vmcbs for nested virtualization. + */ + mov SVM_vmcb01_pa(%_ASM_DI), %_ASM_AX +1: vmload %_ASM_AX +2: + /* Get svm->current_vmcb->pa into RAX. */ mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX @@ -80,16 +92,11 @@ SYM_FUNC_START(__svm_vcpu_run) /* Enter guest mode */ sti -1: vmrun %_ASM_AX - -2: cli - -#ifdef CONFIG_RETPOLINE - /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */ - FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE -#endif +3: vmrun %_ASM_AX +4: + cli - /* "POP" @svm to RAX. */ + /* Pop @svm to RAX while it's the only available register. */ pop %_ASM_AX /* Save all guest registers. */ @@ -110,6 +117,18 @@ SYM_FUNC_START(__svm_vcpu_run) mov %r15, VCPU_R15(%_ASM_AX) #endif + /* @svm can stay in RDI from now on. */ + mov %_ASM_AX, %_ASM_DI + + mov SVM_vmcb01_pa(%_ASM_DI), %_ASM_AX +5: vmsave %_ASM_AX +6: + +#ifdef CONFIG_RETPOLINE + /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */ + FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE +#endif + /* * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be * untrained as soon as we exit the VM and are back to the @@ -159,11 +178,19 @@ SYM_FUNC_START(__svm_vcpu_run) pop %_ASM_BP RET -3: cmpb $0, kvm_rebooting +10: cmpb $0, kvm_rebooting jne 2b ud2 +30: cmpb $0, kvm_rebooting + jne 4b + ud2 +50: cmpb $0, kvm_rebooting + jne 6b + ud2 - _ASM_EXTABLE(1b, 3b) + _ASM_EXTABLE(1b, 10b) + _ASM_EXTABLE(3b, 30b) + _ASM_EXTABLE(5b, 50b) SYM_FUNC_END(__svm_vcpu_run) -- GitLab From e287bd005ad9d85dd6271dd795d3ecfb6bca46ad Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 7 Nov 2022 03:49:59 -0500 Subject: [PATCH 1436/1927] KVM: SVM: restore host save area from assembly Allow access to the percpu area via the GS segment base, which is needed in order to access the saved host spec_ctrl value. In linux-next FILL_RETURN_BUFFER also needs to access percpu data. For simplicity, the physical address of the save area is added to struct svm_cpu_data. Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Reported-by: Nathan Chancellor Analyzed-by: Andrew Cooper Tested-by: Nathan Chancellor Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kvm/kvm-asm-offsets.c | 1 + arch/x86/kvm/svm/svm.c | 14 ++++++-------- arch/x86/kvm/svm/svm.h | 2 ++ arch/x86/kvm/svm/svm_ops.h | 5 ----- arch/x86/kvm/svm/vmenter.S | 17 +++++++++++++++++ 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c index f83e88b85bf21..1b805cd24d669 100644 --- a/arch/x86/kvm/kvm-asm-offsets.c +++ b/arch/x86/kvm/kvm-asm-offsets.c @@ -18,6 +18,7 @@ static void __used common(void) OFFSET(SVM_current_vmcb, vcpu_svm, current_vmcb); OFFSET(SVM_vmcb01, vcpu_svm, vmcb01); OFFSET(KVM_VMCB_pa, kvm_vmcb_info, pa); + OFFSET(SD_save_area_pa, svm_cpu_data, save_area_pa); } if (IS_ENABLED(CONFIG_KVM_INTEL)) { diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 4e3a47eb50025..469c1b5617af7 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -592,7 +592,7 @@ static int svm_hardware_enable(void) wrmsrl(MSR_EFER, efer | EFER_SVME); - wrmsrl(MSR_VM_HSAVE_PA, __sme_page_pa(sd->save_area)); + wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) { /* @@ -648,6 +648,7 @@ static void svm_cpu_uninit(int cpu) kfree(sd->sev_vmcbs); __free_page(sd->save_area); + sd->save_area_pa = 0; sd->save_area = NULL; } @@ -665,6 +666,7 @@ static int svm_cpu_init(int cpu) if (ret) goto free_save_area; + sd->save_area_pa = __sme_page_pa(sd->save_area); return 0; free_save_area: @@ -1450,7 +1452,7 @@ static void svm_prepare_switch_to_guest(struct kvm_vcpu *vcpu) * Save additional host state that will be restored on VMEXIT (sev-es) * or subsequent vmload of host save area. */ - vmsave(__sme_page_pa(sd->save_area)); + vmsave(sd->save_area_pa); if (sev_es_guest(vcpu->kvm)) { struct sev_es_save_area *hostsa; hostsa = (struct sev_es_save_area *)(page_address(sd->save_area) + 0x400); @@ -3905,14 +3907,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) guest_state_enter_irqoff(); - if (sev_es_guest(vcpu->kvm)) { + if (sev_es_guest(vcpu->kvm)) __svm_sev_es_vcpu_run(svm); - } else { - struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); - + else __svm_vcpu_run(svm); - vmload(__sme_page_pa(sd->save_area)); - } guest_state_exit_irqoff(); } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 2af6a71126c1e..83955a4e520eb 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -287,6 +287,8 @@ struct svm_cpu_data { struct kvm_ldttss_desc *tss_desc; struct page *save_area; + unsigned long save_area_pa; + struct vmcb *current_vmcb; /* index = sev_asid, value = vmcb pointer */ diff --git a/arch/x86/kvm/svm/svm_ops.h b/arch/x86/kvm/svm/svm_ops.h index 9430d6437c9f6..36c8af87a707a 100644 --- a/arch/x86/kvm/svm/svm_ops.h +++ b/arch/x86/kvm/svm/svm_ops.h @@ -61,9 +61,4 @@ static __always_inline void vmsave(unsigned long pa) svm_asm1(vmsave, "a" (pa), "memory"); } -static __always_inline void vmload(unsigned long pa) -{ - svm_asm1(vmload, "a" (pa), "memory"); -} - #endif /* __KVM_X86_SVM_OPS_H */ diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index 5bc2ed7d79c07..57440acfc73ef 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -49,6 +49,14 @@ SYM_FUNC_START(__svm_vcpu_run) #endif push %_ASM_BX + /* + * Save variables needed after vmexit on the stack, in inverse + * order compared to when they are needed. + */ + + /* Needed to restore access to percpu variables. */ + __ASM_SIZE(push) PER_CPU_VAR(svm_data + SD_save_area_pa) + /* Save @svm. */ push %_ASM_ARG1 @@ -124,6 +132,11 @@ SYM_FUNC_START(__svm_vcpu_run) 5: vmsave %_ASM_AX 6: + /* Restores GSBASE among other things, allowing access to percpu data. */ + pop %_ASM_AX +7: vmload %_ASM_AX +8: + #ifdef CONFIG_RETPOLINE /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */ FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE @@ -187,10 +200,14 @@ SYM_FUNC_START(__svm_vcpu_run) 50: cmpb $0, kvm_rebooting jne 6b ud2 +70: cmpb $0, kvm_rebooting + jne 8b + ud2 _ASM_EXTABLE(1b, 10b) _ASM_EXTABLE(3b, 30b) _ASM_EXTABLE(5b, 50b) + _ASM_EXTABLE(7b, 70b) SYM_FUNC_END(__svm_vcpu_run) -- GitLab From 9f2febf3f04daebdaaa5a43cfa20e3844905c0f9 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 30 Sep 2022 14:24:40 -0400 Subject: [PATCH 1437/1927] KVM: SVM: move MSR_IA32_SPEC_CTRL save/restore to assembly Restoration of the host IA32_SPEC_CTRL value is probably too late with respect to the return thunk training sequence. With respect to the user/kernel boundary, AMD says, "If software chooses to toggle STIBP (e.g., set STIBP on kernel entry, and clear it on kernel exit), software should set STIBP to 1 before executing the return thunk training sequence." I assume the same requirements apply to the guest/host boundary. The return thunk training sequence is in vmenter.S, quite close to the VM-exit. On hosts without V_SPEC_CTRL, however, the host's IA32_SPEC_CTRL value is not restored until much later. To avoid this, move the restoration of host SPEC_CTRL to assembly and, for consistency, move the restoration of the guest SPEC_CTRL as well. This is not particularly difficult, apart from some care to cover both 32- and 64-bit, and to share code between SEV-ES and normal vmentry. Cc: stable@vger.kernel.org Fixes: a149180fbcf3 ("x86: Add magic AMD return-thunk") Suggested-by: Jim Mattson Reviewed-by: Sean Christopherson Signed-off-by: Paolo Bonzini --- arch/x86/kernel/cpu/bugs.c | 13 +--- arch/x86/kvm/kvm-asm-offsets.c | 1 + arch/x86/kvm/svm/svm.c | 37 ++++------ arch/x86/kvm/svm/svm.h | 4 +- arch/x86/kvm/svm/vmenter.S | 119 ++++++++++++++++++++++++++++++++- 5 files changed, 136 insertions(+), 38 deletions(-) diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index da7c361f47e0d..6ec0b7ce74531 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -196,22 +196,15 @@ void __init check_bugs(void) } /* - * NOTE: This function is *only* called for SVM. VMX spec_ctrl handling is - * done in vmenter.S. + * NOTE: This function is *only* called for SVM, since Intel uses + * MSR_IA32_SPEC_CTRL for SSBD. */ void x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) { - u64 msrval, guestval = guest_spec_ctrl, hostval = spec_ctrl_current(); + u64 guestval, hostval; struct thread_info *ti = current_thread_info(); - if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) { - if (hostval != guestval) { - msrval = setguest ? guestval : hostval; - wrmsrl(MSR_IA32_SPEC_CTRL, msrval); - } - } - /* * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported. diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c index 1b805cd24d669..24a710d373238 100644 --- a/arch/x86/kvm/kvm-asm-offsets.c +++ b/arch/x86/kvm/kvm-asm-offsets.c @@ -16,6 +16,7 @@ static void __used common(void) BLANK(); OFFSET(SVM_vcpu_arch_regs, vcpu_svm, vcpu.arch.regs); OFFSET(SVM_current_vmcb, vcpu_svm, current_vmcb); + OFFSET(SVM_spec_ctrl, vcpu_svm, spec_ctrl); OFFSET(SVM_vmcb01, vcpu_svm, vmcb01); OFFSET(KVM_VMCB_pa, kvm_vmcb_info, pa); OFFSET(SD_save_area_pa, svm_cpu_data, save_area_pa); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 469c1b5617af7..cf1aed25f4aba 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -720,6 +720,15 @@ static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr) u32 offset; u32 *msrpm; + /* + * For non-nested case: + * If the L01 MSR bitmap does not intercept the MSR, then we need to + * save it. + * + * For nested case: + * If the L02 MSR bitmap does not intercept the MSR, then we need to + * save it. + */ msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: to_svm(vcpu)->msrpm; @@ -3901,16 +3910,16 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu) return EXIT_FASTPATH_NONE; } -static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) +static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted) { struct vcpu_svm *svm = to_svm(vcpu); guest_state_enter_irqoff(); if (sev_es_guest(vcpu->kvm)) - __svm_sev_es_vcpu_run(svm); + __svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted); else - __svm_vcpu_run(svm); + __svm_vcpu_run(svm, spec_ctrl_intercepted); guest_state_exit_irqoff(); } @@ -3918,6 +3927,7 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu) static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); + bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL); trace_kvm_entry(vcpu); @@ -3976,26 +3986,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl); - svm_vcpu_enter_exit(vcpu); - - /* - * We do not use IBRS in the kernel. If this vCPU has used the - * SPEC_CTRL MSR it may have left it on; save the value and - * turn it off. This is much more efficient than blindly adding - * it to the atomic save/restore list. Especially as the former - * (Saving guest MSRs on vmexit) doesn't even exist in KVM. - * - * For non-nested case: - * If the L01 MSR bitmap does not intercept the MSR, then we need to - * save it. - * - * For nested case: - * If the L02 MSR bitmap does not intercept the MSR, then we need to - * save it. - */ - if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL) && - unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL))) - svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL); + svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted); if (!sev_es_guest(vcpu->kvm)) reload_tss(vcpu); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 83955a4e520eb..199a2ecef1cec 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -682,7 +682,7 @@ void sev_es_unmap_ghcb(struct vcpu_svm *svm); /* vmenter.S */ -void __svm_sev_es_vcpu_run(struct vcpu_svm *svm); -void __svm_vcpu_run(struct vcpu_svm *svm); +void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted); +void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted); #endif diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S index 57440acfc73ef..34367dc203f21 100644 --- a/arch/x86/kvm/svm/vmenter.S +++ b/arch/x86/kvm/svm/vmenter.S @@ -32,9 +32,69 @@ .section .noinstr.text, "ax" +.macro RESTORE_GUEST_SPEC_CTRL + /* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */ + ALTERNATIVE_2 "", \ + "jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \ + "", X86_FEATURE_V_SPEC_CTRL +801: +.endm +.macro RESTORE_GUEST_SPEC_CTRL_BODY +800: + /* + * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the + * host's, write the MSR. This is kept out-of-line so that the common + * case does not have to jump. + * + * IMPORTANT: To avoid RSB underflow attacks and any other nastiness, + * there must not be any returns or indirect branches between this code + * and vmentry. + */ + movl SVM_spec_ctrl(%_ASM_DI), %eax + cmp PER_CPU_VAR(x86_spec_ctrl_current), %eax + je 801b + mov $MSR_IA32_SPEC_CTRL, %ecx + xor %edx, %edx + wrmsr + jmp 801b +.endm + +.macro RESTORE_HOST_SPEC_CTRL + /* No need to do anything if SPEC_CTRL is unset or V_SPEC_CTRL is set */ + ALTERNATIVE_2 "", \ + "jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \ + "", X86_FEATURE_V_SPEC_CTRL +901: +.endm +.macro RESTORE_HOST_SPEC_CTRL_BODY +900: + /* Same for after vmexit. */ + mov $MSR_IA32_SPEC_CTRL, %ecx + + /* + * Load the value that the guest had written into MSR_IA32_SPEC_CTRL, + * if it was not intercepted during guest execution. + */ + cmpb $0, (%_ASM_SP) + jnz 998f + rdmsr + movl %eax, SVM_spec_ctrl(%_ASM_DI) +998: + + /* Now restore the host value of the MSR if different from the guest's. */ + movl PER_CPU_VAR(x86_spec_ctrl_current), %eax + cmp SVM_spec_ctrl(%_ASM_DI), %eax + je 901b + xor %edx, %edx + wrmsr + jmp 901b +.endm + + /** * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode * @svm: struct vcpu_svm * + * @spec_ctrl_intercepted: bool */ SYM_FUNC_START(__svm_vcpu_run) push %_ASM_BP @@ -54,17 +114,26 @@ SYM_FUNC_START(__svm_vcpu_run) * order compared to when they are needed. */ + /* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL. */ + push %_ASM_ARG2 + /* Needed to restore access to percpu variables. */ __ASM_SIZE(push) PER_CPU_VAR(svm_data + SD_save_area_pa) - /* Save @svm. */ + /* Finally save @svm. */ push %_ASM_ARG1 .ifnc _ASM_ARG1, _ASM_DI - /* Move @svm to RDI. */ + /* + * Stash @svm in RDI early. On 32-bit, arguments are in RAX, RCX + * and RDX which are clobbered by RESTORE_GUEST_SPEC_CTRL. + */ mov %_ASM_ARG1, %_ASM_DI .endif + /* Clobbers RAX, RCX, RDX. */ + RESTORE_GUEST_SPEC_CTRL + /* * Use a single vmcb (vmcb01 because it's always valid) for * context switching guest state via VMLOAD/VMSAVE, that way @@ -142,6 +211,9 @@ SYM_FUNC_START(__svm_vcpu_run) FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE #endif + /* Clobbers RAX, RCX, RDX. */ + RESTORE_HOST_SPEC_CTRL + /* * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be * untrained as soon as we exit the VM and are back to the @@ -177,6 +249,9 @@ SYM_FUNC_START(__svm_vcpu_run) xor %r15d, %r15d #endif + /* "Pop" @spec_ctrl_intercepted. */ + pop %_ASM_BX + pop %_ASM_BX #ifdef CONFIG_X86_64 @@ -191,6 +266,9 @@ SYM_FUNC_START(__svm_vcpu_run) pop %_ASM_BP RET + RESTORE_GUEST_SPEC_CTRL_BODY + RESTORE_HOST_SPEC_CTRL_BODY + 10: cmpb $0, kvm_rebooting jne 2b ud2 @@ -214,6 +292,7 @@ SYM_FUNC_END(__svm_vcpu_run) /** * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode * @svm: struct vcpu_svm * + * @spec_ctrl_intercepted: bool */ SYM_FUNC_START(__svm_sev_es_vcpu_run) push %_ASM_BP @@ -228,8 +307,30 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) #endif push %_ASM_BX + /* + * Save variables needed after vmexit on the stack, in inverse + * order compared to when they are needed. + */ + + /* Accessed directly from the stack in RESTORE_HOST_SPEC_CTRL. */ + push %_ASM_ARG2 + + /* Save @svm. */ + push %_ASM_ARG1 + +.ifnc _ASM_ARG1, _ASM_DI + /* + * Stash @svm in RDI early. On 32-bit, arguments are in RAX, RCX + * and RDX which are clobbered by RESTORE_GUEST_SPEC_CTRL. + */ + mov %_ASM_ARG1, %_ASM_DI +.endif + + /* Clobbers RAX, RCX, RDX. */ + RESTORE_GUEST_SPEC_CTRL + /* Get svm->current_vmcb->pa into RAX. */ - mov SVM_current_vmcb(%_ASM_ARG1), %_ASM_AX + mov SVM_current_vmcb(%_ASM_DI), %_ASM_AX mov KVM_VMCB_pa(%_ASM_AX), %_ASM_AX /* Enter guest mode */ @@ -239,11 +340,17 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) 2: cli + /* Pop @svm to RDI, guest registers have been saved already. */ + pop %_ASM_DI + #ifdef CONFIG_RETPOLINE /* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */ FILL_RETURN_BUFFER %_ASM_AX, RSB_CLEAR_LOOPS, X86_FEATURE_RETPOLINE #endif + /* Clobbers RAX, RCX, RDX. */ + RESTORE_HOST_SPEC_CTRL + /* * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be * untrained as soon as we exit the VM and are back to the @@ -253,6 +360,9 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) */ UNTRAIN_RET + /* "Pop" @spec_ctrl_intercepted. */ + pop %_ASM_BX + pop %_ASM_BX #ifdef CONFIG_X86_64 @@ -267,6 +377,9 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run) pop %_ASM_BP RET + RESTORE_GUEST_SPEC_CTRL_BODY + RESTORE_HOST_SPEC_CTRL_BODY + 3: cmpb $0, kvm_rebooting jne 2b ud2 -- GitLab From bd3d394e367e66e773a6cb25a82c29b04464230b Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Fri, 30 Sep 2022 14:48:24 -0400 Subject: [PATCH 1438/1927] x86, KVM: remove unnecessary argument to x86_virt_spec_ctrl and callers x86_virt_spec_ctrl only deals with the paravirtualized MSR_IA32_VIRT_SPEC_CTRL now and does not handle MSR_IA32_SPEC_CTRL anymore; remove the corresponding, unused argument. Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/spec-ctrl.h | 10 +++++----- arch/x86/kernel/cpu/bugs.c | 2 +- arch/x86/kvm/svm/svm.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/x86/include/asm/spec-ctrl.h b/arch/x86/include/asm/spec-ctrl.h index 5393babc05989..cb0386fc4dc3b 100644 --- a/arch/x86/include/asm/spec-ctrl.h +++ b/arch/x86/include/asm/spec-ctrl.h @@ -13,7 +13,7 @@ * Takes the guest view of SPEC_CTRL MSR as a parameter and also * the guest's version of VIRT_SPEC_CTRL, if emulated. */ -extern void x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool guest); +extern void x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool guest); /** * x86_spec_ctrl_set_guest - Set speculation control registers for the guest @@ -24,9 +24,9 @@ extern void x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bo * Avoids writing to the MSR if the content/bits are the same */ static inline -void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl) +void x86_spec_ctrl_set_guest(u64 guest_virt_spec_ctrl) { - x86_virt_spec_ctrl(guest_spec_ctrl, guest_virt_spec_ctrl, true); + x86_virt_spec_ctrl(guest_virt_spec_ctrl, true); } /** @@ -38,9 +38,9 @@ void x86_spec_ctrl_set_guest(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl) * Avoids writing to the MSR if the content/bits are the same */ static inline -void x86_spec_ctrl_restore_host(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl) +void x86_spec_ctrl_restore_host(u64 guest_virt_spec_ctrl) { - x86_virt_spec_ctrl(guest_spec_ctrl, guest_virt_spec_ctrl, false); + x86_virt_spec_ctrl(guest_virt_spec_ctrl, false); } /* AMD specific Speculative Store Bypass MSR data */ diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index 6ec0b7ce74531..3e3230cccaa79 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -200,7 +200,7 @@ void __init check_bugs(void) * MSR_IA32_SPEC_CTRL for SSBD. */ void -x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest) +x86_virt_spec_ctrl(u64 guest_virt_spec_ctrl, bool setguest) { u64 guestval, hostval; struct thread_info *ti = current_thread_info(); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index cf1aed25f4aba..9f88c8e6766e4 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -3984,7 +3984,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) * being speculatively taken. */ if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) - x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl); + x86_spec_ctrl_set_guest(svm->virt_spec_ctrl); svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted); @@ -3992,7 +3992,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) reload_tss(vcpu); if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL)) - x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl); + x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); if (!sev_es_guest(vcpu->kvm)) { vcpu->arch.cr2 = svm->vmcb->save.cr2; -- GitLab From 8e1071d0ad300fcce2e2b4e46cb15e41d0166bdc Mon Sep 17 00:00:00 2001 From: Matthias Gerstner Date: Thu, 3 Nov 2022 14:59:27 +0100 Subject: [PATCH 1439/1927] tools/kvm_stat: fix incorrect detection of debugfs The first field in /proc/mounts can be influenced by unprivileged users through the widespread `fusermount` setuid-root program. Example: ``` user$ mkdir ~/mydebugfs user$ export _FUSE_COMMFD=0 user$ fusermount ~/mydebugfs -ononempty,fsname=debugfs user$ grep debugfs /proc/mounts debugfs /home/user/mydebugfs fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=100 0 0 ``` If there is no debugfs already mounted in the system then this can be used by unprivileged users to trick kvm_stat into using a user controlled file system location for obtaining KVM statistics. Even though the root user is not allowed to access non-root FUSE mounts for security reasons, the unprivileged user can unmount the FUSE mount before kvm_stat uses the mounted path. If it wins the race, kvm_stat will read from the location where the FUSE mount resided. Note that the files in debugfs are only opened for reading, so the attacker can cause very large data to be read in by kvm_stat, or fake data to be processed, but there should be no viable way to turn this into a privilege escalation. The fix is simply to use the file system type field instead. Whitespace in the mount path is escaped in /proc/mounts thus no further safety measures in the parsing should be necessary to make this correct. Message-Id: <20221103135927.13656-1-matthias.gerstner@suse.de> Signed-off-by: Matthias Gerstner Signed-off-by: Paolo Bonzini --- tools/kvm/kvm_stat/kvm_stat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/kvm/kvm_stat/kvm_stat b/tools/kvm/kvm_stat/kvm_stat index 9c366b3a676db..88a73999aa58a 100755 --- a/tools/kvm/kvm_stat/kvm_stat +++ b/tools/kvm/kvm_stat/kvm_stat @@ -1756,7 +1756,7 @@ def assign_globals(): debugfs = '' for line in open('/proc/mounts'): - if line.split(' ')[0] == 'debugfs': + if line.split(' ')[2] == 'debugfs': debugfs = line.split(' ')[1] break if debugfs == '': -- GitLab From 2c1b54348a63096d4d4677743fba7a6d5fda9476 Mon Sep 17 00:00:00 2001 From: Rong Tao Date: Mon, 7 Nov 2022 22:52:49 +0800 Subject: [PATCH 1440/1927] tools/kvm_stat: update exit reasons for vmx/svm/aarch64/userspace Update EXIT_REASONS from source, including VMX_EXIT_REASONS, SVM_EXIT_REASONS, AARCH64_EXIT_REASONS, USERSPACE_EXIT_REASONS. Signed-off-by: Rong Tao Message-Id: Signed-off-by: Paolo Bonzini --- tools/kvm/kvm_stat/kvm_stat | 96 +++++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 14 deletions(-) diff --git a/tools/kvm/kvm_stat/kvm_stat b/tools/kvm/kvm_stat/kvm_stat index 88a73999aa58a..6f28180ffeeab 100755 --- a/tools/kvm/kvm_stat/kvm_stat +++ b/tools/kvm/kvm_stat/kvm_stat @@ -41,11 +41,14 @@ VMX_EXIT_REASONS = { 'EXCEPTION_NMI': 0, 'EXTERNAL_INTERRUPT': 1, 'TRIPLE_FAULT': 2, - 'PENDING_INTERRUPT': 7, + 'INIT_SIGNAL': 3, + 'SIPI_SIGNAL': 4, + 'INTERRUPT_WINDOW': 7, 'NMI_WINDOW': 8, 'TASK_SWITCH': 9, 'CPUID': 10, 'HLT': 12, + 'INVD': 13, 'INVLPG': 14, 'RDPMC': 15, 'RDTSC': 16, @@ -65,26 +68,48 @@ VMX_EXIT_REASONS = { 'MSR_READ': 31, 'MSR_WRITE': 32, 'INVALID_STATE': 33, + 'MSR_LOAD_FAIL': 34, 'MWAIT_INSTRUCTION': 36, + 'MONITOR_TRAP_FLAG': 37, 'MONITOR_INSTRUCTION': 39, 'PAUSE_INSTRUCTION': 40, 'MCE_DURING_VMENTRY': 41, 'TPR_BELOW_THRESHOLD': 43, 'APIC_ACCESS': 44, + 'EOI_INDUCED': 45, + 'GDTR_IDTR': 46, + 'LDTR_TR': 47, 'EPT_VIOLATION': 48, 'EPT_MISCONFIG': 49, + 'INVEPT': 50, + 'RDTSCP': 51, + 'PREEMPTION_TIMER': 52, + 'INVVPID': 53, 'WBINVD': 54, 'XSETBV': 55, 'APIC_WRITE': 56, + 'RDRAND': 57, 'INVPCID': 58, + 'VMFUNC': 59, + 'ENCLS': 60, + 'RDSEED': 61, + 'PML_FULL': 62, + 'XSAVES': 63, + 'XRSTORS': 64, + 'UMWAIT': 67, + 'TPAUSE': 68, + 'BUS_LOCK': 74, + 'NOTIFY': 75, } SVM_EXIT_REASONS = { 'READ_CR0': 0x000, + 'READ_CR2': 0x002, 'READ_CR3': 0x003, 'READ_CR4': 0x004, 'READ_CR8': 0x008, 'WRITE_CR0': 0x010, + 'WRITE_CR2': 0x012, 'WRITE_CR3': 0x013, 'WRITE_CR4': 0x014, 'WRITE_CR8': 0x018, @@ -105,6 +130,7 @@ SVM_EXIT_REASONS = { 'WRITE_DR6': 0x036, 'WRITE_DR7': 0x037, 'EXCP_BASE': 0x040, + 'LAST_EXCP': 0x05f, 'INTR': 0x060, 'NMI': 0x061, 'SMI': 0x062, @@ -151,21 +177,45 @@ SVM_EXIT_REASONS = { 'MWAIT': 0x08b, 'MWAIT_COND': 0x08c, 'XSETBV': 0x08d, + 'RDPRU': 0x08e, + 'EFER_WRITE_TRAP': 0x08f, + 'CR0_WRITE_TRAP': 0x090, + 'CR1_WRITE_TRAP': 0x091, + 'CR2_WRITE_TRAP': 0x092, + 'CR3_WRITE_TRAP': 0x093, + 'CR4_WRITE_TRAP': 0x094, + 'CR5_WRITE_TRAP': 0x095, + 'CR6_WRITE_TRAP': 0x096, + 'CR7_WRITE_TRAP': 0x097, + 'CR8_WRITE_TRAP': 0x098, + 'CR9_WRITE_TRAP': 0x099, + 'CR10_WRITE_TRAP': 0x09a, + 'CR11_WRITE_TRAP': 0x09b, + 'CR12_WRITE_TRAP': 0x09c, + 'CR13_WRITE_TRAP': 0x09d, + 'CR14_WRITE_TRAP': 0x09e, + 'CR15_WRITE_TRAP': 0x09f, + 'INVPCID': 0x0a2, 'NPF': 0x400, + 'AVIC_INCOMPLETE_IPI': 0x401, + 'AVIC_UNACCELERATED_ACCESS': 0x402, + 'VMGEXIT': 0x403, } -# EC definition of HSR (from arch/arm64/include/asm/kvm_arm.h) +# EC definition of HSR (from arch/arm64/include/asm/esr.h) AARCH64_EXIT_REASONS = { 'UNKNOWN': 0x00, - 'WFI': 0x01, + 'WFx': 0x01, 'CP15_32': 0x03, 'CP15_64': 0x04, 'CP14_MR': 0x05, 'CP14_LS': 0x06, 'FP_ASIMD': 0x07, 'CP10_ID': 0x08, + 'PAC': 0x09, 'CP14_64': 0x0C, - 'ILL_ISS': 0x0E, + 'BTI': 0x0D, + 'ILL': 0x0E, 'SVC32': 0x11, 'HVC32': 0x12, 'SMC32': 0x13, @@ -173,21 +223,26 @@ AARCH64_EXIT_REASONS = { 'HVC64': 0x16, 'SMC64': 0x17, 'SYS64': 0x18, - 'IABT': 0x20, - 'IABT_HYP': 0x21, + 'SVE': 0x19, + 'ERET': 0x1A, + 'FPAC': 0x1C, + 'SME': 0x1D, + 'IMP_DEF': 0x1F, + 'IABT_LOW': 0x20, + 'IABT_CUR': 0x21, 'PC_ALIGN': 0x22, - 'DABT': 0x24, - 'DABT_HYP': 0x25, + 'DABT_LOW': 0x24, + 'DABT_CUR': 0x25, 'SP_ALIGN': 0x26, 'FP_EXC32': 0x28, 'FP_EXC64': 0x2C, 'SERROR': 0x2F, - 'BREAKPT': 0x30, - 'BREAKPT_HYP': 0x31, - 'SOFTSTP': 0x32, - 'SOFTSTP_HYP': 0x33, - 'WATCHPT': 0x34, - 'WATCHPT_HYP': 0x35, + 'BREAKPT_LOW': 0x30, + 'BREAKPT_CUR': 0x31, + 'SOFTSTP_LOW': 0x32, + 'SOFTSTP_CUR': 0x33, + 'WATCHPT_LOW': 0x34, + 'WATCHPT_CUR': 0x35, 'BKPT32': 0x38, 'VECTOR32': 0x3A, 'BRK64': 0x3C, @@ -220,6 +275,19 @@ USERSPACE_EXIT_REASONS = { 'S390_TSCH': 22, 'EPR': 23, 'SYSTEM_EVENT': 24, + 'S390_STSI': 25, + 'IOAPIC_EOI': 26, + 'HYPERV': 27, + 'ARM_NISV': 28, + 'X86_RDMSR': 29, + 'X86_WRMSR': 30, + 'DIRTY_RING_FULL': 31, + 'AP_RESET_HOLD': 32, + 'X86_BUS_LOCK': 33, + 'XEN': 34, + 'RISCV_SBI': 35, + 'RISCV_CSR': 36, + 'NOTIFY': 37, } IOCTL_NUMBERS = { -- GitLab From 0bd8bd2f7a789fe1dcb21ad148199d2f62d79873 Mon Sep 17 00:00:00 2001 From: Peter Gonda Date: Fri, 4 Nov 2022 07:22:20 -0700 Subject: [PATCH 1441/1927] KVM: SVM: Only dump VMSA to klog at KERN_DEBUG level Explicitly print the VMSA dump at KERN_DEBUG log level, KERN_CONT uses KERNEL_DEFAULT if the previous log line has a newline, i.e. if there's nothing to continuing, and as a result the VMSA gets dumped when it shouldn't. The KERN_CONT documentation says it defaults back to KERNL_DEFAULT if the previous log line has a newline. So switch from KERN_CONT to print_hex_dump_debug(). Jarkko pointed this out in reference to the original patch. See: https://lore.kernel.org/all/YuPMeWX4uuR1Tz3M@kernel.org/ print_hex_dump(KERN_DEBUG, ...) was pointed out there, but print_hex_dump_debug() should similar. Fixes: 6fac42f127b8 ("KVM: SVM: Dump Virtual Machine Save Area (VMSA) to klog") Signed-off-by: Peter Gonda Reviewed-by: Sean Christopherson Cc: Jarkko Sakkinen Cc: Harald Hoyer Cc: Paolo Bonzini Cc: Thomas Gleixner Cc: Ingo Molnar Cc: Borislav Petkov Cc: Dave Hansen Cc: x86@kernel.org Cc: "H. Peter Anvin" Cc: kvm@vger.kernel.org Cc: linux-kernel@vger.kernel.org Cc: stable@vger.kernel.org Message-Id: <20221104142220.469452-1-pgonda@google.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/svm/sev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 9b66ee34e2642..efaaef2b7ae11 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -605,7 +605,7 @@ static int sev_es_sync_vmsa(struct vcpu_svm *svm) save->dr6 = svm->vcpu.arch.dr6; pr_debug("Virtual Machine Save Area (VMSA):\n"); - print_hex_dump(KERN_CONT, "", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false); + print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, save, sizeof(*save), false); return 0; } -- GitLab From 8631ef59b62290c7d88e7209e35dfb47f33f4902 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Mon, 19 Sep 2022 17:10:06 +0800 Subject: [PATCH 1442/1927] KVM: x86/pmu: Do not speculatively query Intel GP PMCs that don't exist yet The SDM lists an architectural MSR IA32_CORE_CAPABILITIES (0xCF) that limits the theoretical maximum value of the Intel GP PMC MSRs allocated at 0xC1 to 14; likewise the Intel April 2022 SDM adds IA32_OVERCLOCKING_STATUS at 0x195 which limits the number of event selection MSRs to 15 (0x186-0x194). Limiting the maximum number of counters to 14 or 18 based on the currently allocated MSRs is clearly fragile, and it seems likely that Intel will even place PMCs 8-15 at a completely different range of MSR indices. So stop at the maximum number of GP PMCs supported today on Intel processors. There are some machines, like Intel P4 with non Architectural PMU, that may indeed have 18 counters, but those counters are in a completely different MSR address range and are not supported by KVM. Cc: Vitaly Kuznetsov Cc: stable@vger.kernel.org Fixes: cf05a67b68b8 ("KVM: x86: omit "impossible" pmu MSRs from MSR list") Suggested-by: Jim Mattson Signed-off-by: Like Xu Reviewed-by: Jim Mattson Message-Id: <20220919091008.60695-1-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/kvm/x86.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5f5eb577d583c..73716fab120f7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1442,20 +1442,10 @@ static const u32 msrs_to_save_all[] = { MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3, MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5, MSR_ARCH_PERFMON_PERFCTR0 + 6, MSR_ARCH_PERFMON_PERFCTR0 + 7, - MSR_ARCH_PERFMON_PERFCTR0 + 8, MSR_ARCH_PERFMON_PERFCTR0 + 9, - MSR_ARCH_PERFMON_PERFCTR0 + 10, MSR_ARCH_PERFMON_PERFCTR0 + 11, - MSR_ARCH_PERFMON_PERFCTR0 + 12, MSR_ARCH_PERFMON_PERFCTR0 + 13, - MSR_ARCH_PERFMON_PERFCTR0 + 14, MSR_ARCH_PERFMON_PERFCTR0 + 15, - MSR_ARCH_PERFMON_PERFCTR0 + 16, MSR_ARCH_PERFMON_PERFCTR0 + 17, MSR_ARCH_PERFMON_EVENTSEL0, MSR_ARCH_PERFMON_EVENTSEL1, MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3, MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5, MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7, - MSR_ARCH_PERFMON_EVENTSEL0 + 8, MSR_ARCH_PERFMON_EVENTSEL0 + 9, - MSR_ARCH_PERFMON_EVENTSEL0 + 10, MSR_ARCH_PERFMON_EVENTSEL0 + 11, - MSR_ARCH_PERFMON_EVENTSEL0 + 12, MSR_ARCH_PERFMON_EVENTSEL0 + 13, - MSR_ARCH_PERFMON_EVENTSEL0 + 14, MSR_ARCH_PERFMON_EVENTSEL0 + 15, - MSR_ARCH_PERFMON_EVENTSEL0 + 16, MSR_ARCH_PERFMON_EVENTSEL0 + 17, MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, @@ -7041,12 +7031,12 @@ static void kvm_init_msr_list(void) intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2) continue; break; - case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 17: + case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 7: if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >= min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp)) continue; break; - case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 17: + case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 7: if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >= min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp)) continue; -- GitLab From 4f1fa2a1bbeb2feca436d2c86bf6f78dc4e5e4c4 Mon Sep 17 00:00:00 2001 From: Like Xu Date: Mon, 19 Sep 2022 17:10:07 +0800 Subject: [PATCH 1443/1927] KVM: x86/pmu: Limit the maximum number of supported Intel GP counters The Intel Architectural IA32_PMCx MSRs addresses range allows for a maximum of 8 GP counters, and KVM cannot address any more. Introduce a local macro (named KVM_INTEL_PMC_MAX_GENERIC) and use it consistently to refer to the number of counters supported by KVM, thus avoiding possible out-of-bound accesses. Suggested-by: Jim Mattson Signed-off-by: Like Xu Reviewed-by: Jim Mattson Message-Id: <20220919091008.60695-2-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 6 +++++- arch/x86/kvm/pmu.c | 2 +- arch/x86/kvm/vmx/pmu_intel.c | 4 ++-- arch/x86/kvm/x86.c | 12 +++++++----- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 7551b6f9c31c5..286f6130dcb47 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -501,6 +501,10 @@ struct kvm_pmc { bool intr; }; +/* More counters may conflict with other existing Architectural MSRs */ +#define KVM_INTEL_PMC_MAX_GENERIC 8 +#define MSR_ARCH_PERFMON_PERFCTR_MAX (MSR_ARCH_PERFMON_PERFCTR0 + KVM_INTEL_PMC_MAX_GENERIC - 1) +#define MSR_ARCH_PERFMON_EVENTSEL_MAX (MSR_ARCH_PERFMON_EVENTSEL0 + KVM_INTEL_PMC_MAX_GENERIC - 1) #define KVM_PMC_MAX_FIXED 3 struct kvm_pmu { unsigned nr_arch_gp_counters; @@ -516,7 +520,7 @@ struct kvm_pmu { u64 reserved_bits; u64 raw_event_mask; u8 version; - struct kvm_pmc gp_counters[INTEL_PMC_MAX_GENERIC]; + struct kvm_pmc gp_counters[KVM_INTEL_PMC_MAX_GENERIC]; struct kvm_pmc fixed_counters[KVM_PMC_MAX_FIXED]; struct irq_work irq_work; DECLARE_BITMAP(reprogram_pmi, X86_PMC_IDX_MAX); diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c index d9b9a0f0db17c..de1fd73697365 100644 --- a/arch/x86/kvm/pmu.c +++ b/arch/x86/kvm/pmu.c @@ -56,7 +56,7 @@ static const struct x86_cpu_id vmx_icl_pebs_cpu[] = { * code. Each pmc, stored in kvm_pmc.idx field, is unique across * all perf counters (both gp and fixed). The mapping relationship * between pmc and perf counters is as the following: - * * Intel: [0 .. INTEL_PMC_MAX_GENERIC-1] <=> gp counters + * * Intel: [0 .. KVM_INTEL_PMC_MAX_GENERIC-1] <=> gp counters * [INTEL_PMC_IDX_FIXED .. INTEL_PMC_IDX_FIXED + 2] <=> fixed * * AMD: [0 .. AMD64_NUM_COUNTERS-1] and, for families 15H * and later, [0 .. AMD64_NUM_COUNTERS_CORE-1] <=> gp counters diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index 25b70a85bef54..10b33da9bd058 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -617,7 +617,7 @@ static void intel_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); struct lbr_desc *lbr_desc = vcpu_to_lbr_desc(vcpu); - for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { + for (i = 0; i < KVM_INTEL_PMC_MAX_GENERIC; i++) { pmu->gp_counters[i].type = KVM_PMC_GP; pmu->gp_counters[i].vcpu = vcpu; pmu->gp_counters[i].idx = i; @@ -643,7 +643,7 @@ static void intel_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmc *pmc = NULL; int i; - for (i = 0; i < INTEL_PMC_MAX_GENERIC; i++) { + for (i = 0; i < KVM_INTEL_PMC_MAX_GENERIC; i++) { pmc = &pmu->gp_counters[i]; pmc_stop_counter(pmc); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 73716fab120f7..1e25f410d1fea 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1438,6 +1438,9 @@ static const u32 msrs_to_save_all[] = { MSR_ARCH_PERFMON_FIXED_CTR0 + 2, MSR_CORE_PERF_FIXED_CTR_CTRL, MSR_CORE_PERF_GLOBAL_STATUS, MSR_CORE_PERF_GLOBAL_CTRL, MSR_CORE_PERF_GLOBAL_OVF_CTRL, + MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, + + /* This part of MSRs should match KVM_INTEL_PMC_MAX_GENERIC. */ MSR_ARCH_PERFMON_PERFCTR0, MSR_ARCH_PERFMON_PERFCTR1, MSR_ARCH_PERFMON_PERFCTR0 + 2, MSR_ARCH_PERFMON_PERFCTR0 + 3, MSR_ARCH_PERFMON_PERFCTR0 + 4, MSR_ARCH_PERFMON_PERFCTR0 + 5, @@ -1446,7 +1449,6 @@ static const u32 msrs_to_save_all[] = { MSR_ARCH_PERFMON_EVENTSEL0 + 2, MSR_ARCH_PERFMON_EVENTSEL0 + 3, MSR_ARCH_PERFMON_EVENTSEL0 + 4, MSR_ARCH_PERFMON_EVENTSEL0 + 5, MSR_ARCH_PERFMON_EVENTSEL0 + 6, MSR_ARCH_PERFMON_EVENTSEL0 + 7, - MSR_IA32_PEBS_ENABLE, MSR_IA32_DS_AREA, MSR_PEBS_DATA_CFG, MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3, @@ -7031,14 +7033,14 @@ static void kvm_init_msr_list(void) intel_pt_validate_hw_cap(PT_CAP_num_address_ranges) * 2) continue; break; - case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR0 + 7: + case MSR_ARCH_PERFMON_PERFCTR0 ... MSR_ARCH_PERFMON_PERFCTR_MAX: if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_PERFCTR0 >= - min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp)) + min(KVM_INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp)) continue; break; - case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL0 + 7: + case MSR_ARCH_PERFMON_EVENTSEL0 ... MSR_ARCH_PERFMON_EVENTSEL_MAX: if (msrs_to_save_all[i] - MSR_ARCH_PERFMON_EVENTSEL0 >= - min(INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp)) + min(KVM_INTEL_PMC_MAX_GENERIC, kvm_pmu_cap.num_counters_gp)) continue; break; case MSR_IA32_XFD: -- GitLab From 556f3c9ad7c101aa16a43ef4539f3aabc1d7b32e Mon Sep 17 00:00:00 2001 From: Like Xu Date: Mon, 19 Sep 2022 17:10:08 +0800 Subject: [PATCH 1444/1927] KVM: x86/pmu: Limit the maximum number of supported AMD GP counters The AMD PerfMonV2 specification allows for a maximum of 16 GP counters, but currently only 6 pairs of MSRs are accepted by KVM. While AMD64_NUM_COUNTERS_CORE is already equal to 6, increasing without adjusting msrs_to_save_all[] could result in out-of-bounds accesses. Therefore introduce a macro (named KVM_AMD_PMC_MAX_GENERIC) to refer to the number of counters supported by KVM. Signed-off-by: Like Xu Reviewed-by: Jim Mattson Message-Id: <20220919091008.60695-3-likexu@tencent.com> Signed-off-by: Paolo Bonzini --- arch/x86/include/asm/kvm_host.h | 1 + arch/x86/kvm/svm/pmu.c | 7 ++++--- arch/x86/kvm/x86.c | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 286f6130dcb47..f05ebaa26f0ff 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -506,6 +506,7 @@ struct kvm_pmc { #define MSR_ARCH_PERFMON_PERFCTR_MAX (MSR_ARCH_PERFMON_PERFCTR0 + KVM_INTEL_PMC_MAX_GENERIC - 1) #define MSR_ARCH_PERFMON_EVENTSEL_MAX (MSR_ARCH_PERFMON_EVENTSEL0 + KVM_INTEL_PMC_MAX_GENERIC - 1) #define KVM_PMC_MAX_FIXED 3 +#define KVM_AMD_PMC_MAX_GENERIC 6 struct kvm_pmu { unsigned nr_arch_gp_counters; unsigned nr_arch_fixed_counters; diff --git a/arch/x86/kvm/svm/pmu.c b/arch/x86/kvm/svm/pmu.c index b68956299fa8e..9d65cd095691b 100644 --- a/arch/x86/kvm/svm/pmu.c +++ b/arch/x86/kvm/svm/pmu.c @@ -192,9 +192,10 @@ static void amd_pmu_init(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); int i; - BUILD_BUG_ON(AMD64_NUM_COUNTERS_CORE > INTEL_PMC_MAX_GENERIC); + BUILD_BUG_ON(KVM_AMD_PMC_MAX_GENERIC > AMD64_NUM_COUNTERS_CORE); + BUILD_BUG_ON(KVM_AMD_PMC_MAX_GENERIC > INTEL_PMC_MAX_GENERIC); - for (i = 0; i < AMD64_NUM_COUNTERS_CORE ; i++) { + for (i = 0; i < KVM_AMD_PMC_MAX_GENERIC ; i++) { pmu->gp_counters[i].type = KVM_PMC_GP; pmu->gp_counters[i].vcpu = vcpu; pmu->gp_counters[i].idx = i; @@ -207,7 +208,7 @@ static void amd_pmu_reset(struct kvm_vcpu *vcpu) struct kvm_pmu *pmu = vcpu_to_pmu(vcpu); int i; - for (i = 0; i < AMD64_NUM_COUNTERS_CORE; i++) { + for (i = 0; i < KVM_AMD_PMC_MAX_GENERIC; i++) { struct kvm_pmc *pmc = &pmu->gp_counters[i]; pmc_stop_counter(pmc); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 1e25f410d1fea..ecea83f0da498 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -1452,10 +1452,13 @@ static const u32 msrs_to_save_all[] = { MSR_K7_EVNTSEL0, MSR_K7_EVNTSEL1, MSR_K7_EVNTSEL2, MSR_K7_EVNTSEL3, MSR_K7_PERFCTR0, MSR_K7_PERFCTR1, MSR_K7_PERFCTR2, MSR_K7_PERFCTR3, + + /* This part of MSRs should match KVM_AMD_PMC_MAX_GENERIC. */ MSR_F15H_PERF_CTL0, MSR_F15H_PERF_CTL1, MSR_F15H_PERF_CTL2, MSR_F15H_PERF_CTL3, MSR_F15H_PERF_CTL4, MSR_F15H_PERF_CTL5, MSR_F15H_PERF_CTR0, MSR_F15H_PERF_CTR1, MSR_F15H_PERF_CTR2, MSR_F15H_PERF_CTR3, MSR_F15H_PERF_CTR4, MSR_F15H_PERF_CTR5, + MSR_IA32_XFD, MSR_IA32_XFD_ERR, }; -- GitLab From 30d17fac6aaedb40d111bb159f4b35525637ea78 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 31 Oct 2022 13:34:56 +0100 Subject: [PATCH 1445/1927] scripts/min-tool-version.sh: raise minimum clang version to 15.0.0 for s390 Before version 15.0.0 llvm's integrated assembler may silently generate corrupted code on s390. See e.g. commit e9953b729b78 ("s390/boot: workaround llvm IAS bug") for further details. While there have been workarounds applied for all known existing locations, there is nothing that prevents that new code with problematic patterns will be added. Therefore raise the minimum clang version to 15.0.0. Note that llvm commit e547b04d5b2c ("[SystemZ] Bugfix for symbolic displacements."), which is included in 15.0.0, fixes the broken code generation. Signed-off-by: Heiko Carstens Acked-by: Nathan Chancellor Link: https://lore.kernel.org/r/20221031123456.3872220-1-hca@linux.ibm.com Signed-off-by: Alexander Gordeev --- scripts/min-tool-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh index b6593eac5003c..201bccfbc6788 100755 --- a/scripts/min-tool-version.sh +++ b/scripts/min-tool-version.sh @@ -25,7 +25,7 @@ icc) ;; llvm) if [ "$SRCARCH" = s390 ]; then - echo 14.0.0 + echo 15.0.0 else echo 11.0.0 fi -- GitLab From 7d945b046be3d2605dbb1806e73095aadd7ae129 Mon Sep 17 00:00:00 2001 From: Olivier Moysan Date: Wed, 9 Nov 2022 18:08:49 +0100 Subject: [PATCH 1446/1927] ASoC: stm32: dfsdm: manage cb buffers cleanup Ensure that resources allocated by iio_channel_get_all_cb() are released on driver unbind. Signed-off-by: Olivier Moysan Link: https://lore.kernel.org/r/20221109170849.273719-1-olivier.moysan@foss.st.com Signed-off-by: Mark Brown --- sound/soc/stm/stm32_adfsdm.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c index 643fc8a170184..837c1848d9bff 100644 --- a/sound/soc/stm/stm32_adfsdm.c +++ b/sound/soc/stm/stm32_adfsdm.c @@ -304,6 +304,11 @@ static int stm32_adfsdm_dummy_cb(const void *data, void *private) return 0; } +static void stm32_adfsdm_cleanup(void *data) +{ + iio_channel_release_all_cb(data); +} + static struct snd_soc_component_driver stm32_adfsdm_soc_platform = { .open = stm32_adfsdm_pcm_open, .close = stm32_adfsdm_pcm_close, @@ -350,6 +355,12 @@ static int stm32_adfsdm_probe(struct platform_device *pdev) if (IS_ERR(priv->iio_cb)) return PTR_ERR(priv->iio_cb); + ret = devm_add_action_or_reset(&pdev->dev, stm32_adfsdm_cleanup, priv->iio_cb); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to add action\n"); + return ret; + } + component = devm_kzalloc(&pdev->dev, sizeof(*component), GFP_KERNEL); if (!component) return -ENOMEM; -- GitLab From 15f8f168952f54d3c86d734dc764f20844e423ac Mon Sep 17 00:00:00 2001 From: Vlad Buslov Date: Wed, 2 Nov 2022 23:55:37 -0700 Subject: [PATCH 1447/1927] net/mlx5: Bridge, verify LAG state when adding bond to bridge Mlx5 LAG is initialized asynchronously on a workqueue which means that for a brief moment after setting mlx5 UL representors as lower devices of a bond netdevice the LAG itself is not fully initialized in the driver. When adding such bond device to a bridge mlx5 bridge code will not consider it as offload-capable, skip creating necessary bookkeeping and fail any further bridge offload-related commands with it (setting VLANs, offloading FDBs, etc.). In order to make the error explicit during bridge initialization stage implement the code that detects such condition during NETDEV_PRECHANGEUPPER event and returns an error. Fixes: ff9b7521468b ("net/mlx5: Bridge, support LAG") Signed-off-by: Vlad Buslov Reviewed-by: Roi Dayan Reviewed-by: Mark Bloch Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/en/rep/bridge.c | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index 39ef2a2561a30..8099a21e674c9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -164,6 +164,36 @@ static int mlx5_esw_bridge_port_changeupper(struct notifier_block *nb, void *ptr return err; } +static int +mlx5_esw_bridge_changeupper_validate_netdev(void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct netdev_notifier_changeupper_info *info = ptr; + struct net_device *upper = info->upper_dev; + struct net_device *lower; + struct list_head *iter; + + if (!netif_is_bridge_master(upper) || !netif_is_lag_master(dev)) + return 0; + + netdev_for_each_lower_dev(dev, lower, iter) { + struct mlx5_core_dev *mdev; + struct mlx5e_priv *priv; + + if (!mlx5e_eswitch_rep(lower)) + continue; + + priv = netdev_priv(lower); + mdev = priv->mdev; + if (!mlx5_lag_is_active(mdev)) + return -EAGAIN; + if (!mlx5_lag_is_shared_fdb(mdev)) + return -EOPNOTSUPP; + } + + return 0; +} + static int mlx5_esw_bridge_switchdev_port_event(struct notifier_block *nb, unsigned long event, void *ptr) { @@ -171,6 +201,7 @@ static int mlx5_esw_bridge_switchdev_port_event(struct notifier_block *nb, switch (event) { case NETDEV_PRECHANGEUPPER: + err = mlx5_esw_bridge_changeupper_validate_netdev(ptr); break; case NETDEV_CHANGEUPPER: -- GitLab From 2808b37b59288ad8f1897e3546c2296df3384b65 Mon Sep 17 00:00:00 2001 From: Roy Novich Date: Wed, 2 Nov 2022 23:55:38 -0700 Subject: [PATCH 1448/1927] net/mlx5: Allow async trigger completion execution on single CPU systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For a single CPU system, the kernel thread executing mlx5_cmd_flush() never releases the CPU but calls down_trylock(&cmd→sem) in a busy loop. On a single processor system, this leads to a deadlock as the kernel thread which executes mlx5_cmd_invoke() never gets scheduled. Fix this, by adding the cond_resched() call to the loop, allow the command completion kernel thread to execute. Fixes: 8e715cd613a1 ("net/mlx5: Set command entry semaphore up once got index free") Signed-off-by: Alexander Schmidt Signed-off-by: Roy Novich Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/cmd.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 46ba4c2faad21..2e0d59ca62b50 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -1770,12 +1770,17 @@ void mlx5_cmd_flush(struct mlx5_core_dev *dev) struct mlx5_cmd *cmd = &dev->cmd; int i; - for (i = 0; i < cmd->max_reg_cmds; i++) - while (down_trylock(&cmd->sem)) + for (i = 0; i < cmd->max_reg_cmds; i++) { + while (down_trylock(&cmd->sem)) { mlx5_cmd_trigger_completions(dev); + cond_resched(); + } + } - while (down_trylock(&cmd->pages_sem)) + while (down_trylock(&cmd->pages_sem)) { mlx5_cmd_trigger_completions(dev); + cond_resched(); + } /* Unlock cmdif */ up(&cmd->pages_sem); -- GitLab From e12de39c07a7872c1ac7250311bb60b74ff29f25 Mon Sep 17 00:00:00 2001 From: Chris Mi Date: Wed, 2 Nov 2022 23:55:39 -0700 Subject: [PATCH 1449/1927] net/mlx5: E-switch, Set to legacy mode if failed to change switchdev mode No need to rollback to the other mode because probably will fail again. Just set to legacy mode and clear fdb table created flag. So that fdb table will not be cleared again. Fixes: f019679ea5f2 ("net/mlx5: E-switch, Remove dependency between sriov and eswitch mode") Signed-off-by: Chris Mi Reviewed-by: Roi Dayan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/eswitch.c | 14 ++++++++------ .../mellanox/mlx5/core/eswitch_offloads.c | 18 +++--------------- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index c59107fa9e6d2..2169486c4bfb3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -1387,12 +1387,14 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw) esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); - esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED; - if (esw->mode == MLX5_ESWITCH_OFFLOADS) - esw_offloads_disable(esw); - else if (esw->mode == MLX5_ESWITCH_LEGACY) - esw_legacy_disable(esw); - mlx5_esw_acls_ns_cleanup(esw); + if (esw->fdb_table.flags & MLX5_ESW_FDB_CREATED) { + esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED; + if (esw->mode == MLX5_ESWITCH_OFFLOADS) + esw_offloads_disable(esw); + else if (esw->mode == MLX5_ESWITCH_LEGACY) + esw_legacy_disable(esw); + mlx5_esw_acls_ns_cleanup(esw); + } if (esw->mode == MLX5_ESWITCH_OFFLOADS) devl_rate_nodes_destroy(devlink); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 4e50df3139c68..728ca9f2bb9d6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -2310,7 +2310,7 @@ static int esw_create_restore_table(struct mlx5_eswitch *esw) static int esw_offloads_start(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) { - int err, err1; + int err; esw->mode = MLX5_ESWITCH_OFFLOADS; err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); @@ -2318,11 +2318,6 @@ static int esw_offloads_start(struct mlx5_eswitch *esw, NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to offloads"); esw->mode = MLX5_ESWITCH_LEGACY; - err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); - if (err1) { - NL_SET_ERR_MSG_MOD(extack, - "Failed setting eswitch back to legacy"); - } mlx5_rescan_drivers(esw->dev); } if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { @@ -3389,19 +3384,12 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) static int esw_offloads_stop(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) { - int err, err1; + int err; esw->mode = MLX5_ESWITCH_LEGACY; err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); - if (err) { + if (err) NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); - esw->mode = MLX5_ESWITCH_OFFLOADS; - err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); - if (err1) { - NL_SET_ERR_MSG_MOD(extack, - "Failed setting eswitch back to offloads"); - } - } return err; } -- GitLab From 7d167b4a4c810919ba1affd02fc6b7f40e7e6eeb Mon Sep 17 00:00:00 2001 From: Shay Drory Date: Wed, 2 Nov 2022 23:55:40 -0700 Subject: [PATCH 1450/1927] net/mlx5: fw_reset: Don't try to load device in case PCI isn't working In case PCI reads fail after unload, there is no use in trying to load the device. Fixes: 5ec697446f46 ("net/mlx5: Add support for devlink reload action fw activate") Signed-off-by: Shay Drory Reviewed-by: Moshe Shemesh Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 07c583996c297..9d908a0ccfef1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -152,7 +152,8 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) mlx5_unload_one(dev); if (mlx5_health_wait_pci_up(dev)) mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); - mlx5_load_one(dev, false); + else + mlx5_load_one(dev, false); devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0, BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE)); -- GitLab From f9c955b4fe5c8626d11b8a9b93ccc0ba77358fec Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Wed, 2 Nov 2022 23:55:42 -0700 Subject: [PATCH 1451/1927] net/mlx5e: Add missing sanity checks for max TX WQE size The commit cited below started using the firmware capability for the maximum TX WQE size. This commit adds an important check to verify that the driver doesn't attempt to exceed this capability, and also restores another check mistakenly removed in the cited commit (a WQE must not exceed the page size). Fixes: c27bd1718c06 ("net/mlx5e: Read max WQEBBs on the SQ from firmware") Signed-off-by: Maxim Mikityanskiy Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/en/txrx.h | 24 ++++++++++++++++++- .../net/ethernet/mellanox/mlx5/core/en_main.c | 7 ++++++ .../net/ethernet/mellanox/mlx5/core/en_tx.c | 5 ++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h index cb164b62f5436..853f312cd7572 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h @@ -11,6 +11,27 @@ #define INL_HDR_START_SZ (sizeof(((struct mlx5_wqe_eth_seg *)NULL)->inline_hdr.start)) +/* IPSEC inline data includes: + * 1. ESP trailer: up to 255 bytes of padding, 1 byte for pad length, 1 byte for + * next header. + * 2. ESP authentication data: 16 bytes for ICV. + */ +#define MLX5E_MAX_TX_IPSEC_DS DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + \ + 255 + 1 + 1 + 16, MLX5_SEND_WQE_DS) + +/* 366 should be big enough to cover all L2, L3 and L4 headers with possible + * encapsulations. + */ +#define MLX5E_MAX_TX_INLINE_DS DIV_ROUND_UP(366 - INL_HDR_START_SZ + VLAN_HLEN, \ + MLX5_SEND_WQE_DS) + +/* Sync the calculation with mlx5e_sq_calc_wqe_attr. */ +#define MLX5E_MAX_TX_WQEBBS DIV_ROUND_UP(MLX5E_TX_WQE_EMPTY_DS_COUNT + \ + MLX5E_MAX_TX_INLINE_DS + \ + MLX5E_MAX_TX_IPSEC_DS + \ + MAX_SKB_FRAGS + 1, \ + MLX5_SEND_WQEBB_NUM_DS) + #define MLX5E_RX_ERR_CQE(cqe) (get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND) static inline @@ -424,6 +445,8 @@ mlx5e_set_eseg_swp(struct sk_buff *skb, struct mlx5_wqe_eth_seg *eseg, static inline u16 mlx5e_stop_room_for_wqe(struct mlx5_core_dev *mdev, u16 wqe_size) { + WARN_ON_ONCE(PAGE_SIZE / MLX5_SEND_WQE_BB < mlx5e_get_max_sq_wqebbs(mdev)); + /* A WQE must not cross the page boundary, hence two conditions: * 1. Its size must not exceed the page size. * 2. If the WQE size is X, and the space remaining in a page is less @@ -436,7 +459,6 @@ static inline u16 mlx5e_stop_room_for_wqe(struct mlx5_core_dev *mdev, u16 wqe_si "wqe_size %u is greater than max SQ WQEBBs %u", wqe_size, mlx5e_get_max_sq_wqebbs(mdev)); - return MLX5E_STOP_ROOM(wqe_size); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 364f043091492..e3a4f01bcceb1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -5694,6 +5694,13 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv) mlx5e_fs_set_state_destroy(priv->fs, !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); + /* Validate the max_wqe_size_sq capability. */ + if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) { + mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %lu\n", + mlx5e_get_max_sq_wqebbs(priv->mdev), MLX5E_MAX_TX_WQEBBS); + return -EIO; + } + /* max number of channels may have changed */ max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); if (priv->channels.params.num_channels > max_nch) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 6adca01fbdc9d..f7897ddb29c52 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -305,6 +305,8 @@ static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_at u16 ds_cnt_inl = 0; u16 ds_cnt_ids = 0; + /* Sync the calculation with MLX5E_MAX_TX_WQEBBS. */ + if (attr->insz) ds_cnt_ids = DIV_ROUND_UP(sizeof(struct mlx5_wqe_inline_seg) + attr->insz, MLX5_SEND_WQE_DS); @@ -317,6 +319,9 @@ static void mlx5e_sq_calc_wqe_attr(struct sk_buff *skb, const struct mlx5e_tx_at inl += VLAN_HLEN; ds_cnt_inl = DIV_ROUND_UP(inl, MLX5_SEND_WQE_DS); + if (WARN_ON_ONCE(ds_cnt_inl > MLX5E_MAX_TX_INLINE_DS)) + netdev_warn(skb->dev, "ds_cnt_inl = %u > max %u\n", ds_cnt_inl, + (u16)MLX5E_MAX_TX_INLINE_DS); ds_cnt += ds_cnt_inl; } -- GitLab From 8d4b475e9d0f100c3920d8bf7d392a6dac88583e Mon Sep 17 00:00:00 2001 From: Maxim Mikityanskiy Date: Wed, 2 Nov 2022 23:55:43 -0700 Subject: [PATCH 1452/1927] net/mlx5e: Fix usage of DMA sync API DMA sync functions should use the same direction that was used by DMA mapping. Use DMA_BIDIRECTIONAL for XDP_TX from regular RQ, which reuses the same mapping that was used for RX, and DMA_TO_DEVICE for XDP_TX from XSK RQ and XDP_REDIRECT, which establish a new mapping in this direction. On the RX side, use the same direction that was used when setting up the mapping (DMA_BIDIRECTIONAL for XDP, DMA_FROM_DEVICE otherwise). Also don't skip sync for device when establishing a DMA_FROM_DEVICE mapping for RX, as some architectures (ARM) may require invalidating caches before the device can use the mapping. It doesn't break the bugfix made in commit 0b7cfa4082fb ("net/mlx5e: Fix page DMA map/unmap attributes"), since the bug happened on unmap. Fixes: 0b7cfa4082fb ("net/mlx5e: Fix page DMA map/unmap attributes") Fixes: b5503b994ed5 ("net/mlx5e: XDP TX forwarding support") Signed-off-by: Maxim Mikityanskiy Reviewed-by: Gal Pressman Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../net/ethernet/mellanox/mlx5/core/en/xdp.c | 4 +-- .../net/ethernet/mellanox/mlx5/core/en_rx.c | 27 ++++++++++--------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 4685c652c97e1..20507ef2f9569 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -117,7 +117,7 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, xdpi.page.rq = rq; dma_addr = page_pool_get_dma_addr(page) + (xdpf->data - (void *)xdpf); - dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd.len, DMA_TO_DEVICE); + dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd.len, DMA_BIDIRECTIONAL); if (unlikely(xdp_frame_has_frags(xdpf))) { sinfo = xdp_get_shared_info_from_frame(xdpf); @@ -131,7 +131,7 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, skb_frag_off(frag); len = skb_frag_size(frag); dma_sync_single_for_device(sq->pdev, addr, len, - DMA_TO_DEVICE); + DMA_BIDIRECTIONAL); } } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 58084650151f8..a61a43fc8d5c5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -266,7 +266,7 @@ static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, union mlx5e_alloc_uni addr = page_pool_get_dma_addr(au->page); /* Non-XSK always uses PAGE_SIZE. */ - dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE, DMA_FROM_DEVICE); + dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE, rq->buff.map_dir); return true; } @@ -282,8 +282,7 @@ static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, union mlx5e_alloc_u return -ENOMEM; /* Non-XSK always uses PAGE_SIZE. */ - addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE, - rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC); + addr = dma_map_page(rq->pdev, au->page, 0, PAGE_SIZE, rq->buff.map_dir); if (unlikely(dma_mapping_error(rq->pdev, addr))) { page_pool_recycle_direct(rq->page_pool, au->page); au->page = NULL; @@ -427,14 +426,15 @@ mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb, { dma_addr_t addr = page_pool_get_dma_addr(au->page); - dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, + rq->buff.map_dir); page_ref_inc(au->page); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, au->page, frag_offset, len, truesize); } static inline void -mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb, +mlx5e_copy_skb_header(struct mlx5e_rq *rq, struct sk_buff *skb, struct page *page, dma_addr_t addr, int offset_from, int dma_offset, u32 headlen) { @@ -442,7 +442,8 @@ mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb, /* Aligning len to sizeof(long) optimizes memcpy performance */ unsigned int len = ALIGN(headlen, sizeof(long)); - dma_sync_single_for_cpu(pdev, addr + dma_offset, len, DMA_FROM_DEVICE); + dma_sync_single_for_cpu(rq->pdev, addr + dma_offset, len, + rq->buff.map_dir); skb_copy_to_linear_data(skb, from, len); } @@ -1538,7 +1539,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi, addr = page_pool_get_dma_addr(au->page); dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset, - frag_size, DMA_FROM_DEVICE); + frag_size, rq->buff.map_dir); net_prefetch(data); prog = rcu_dereference(rq->xdp_prog); @@ -1587,7 +1588,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi addr = page_pool_get_dma_addr(au->page); dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset, - rq->buff.frame0_sz, DMA_FROM_DEVICE); + rq->buff.frame0_sz, rq->buff.map_dir); net_prefetchw(va); /* xdp_frame data area */ net_prefetch(va + rx_headroom); @@ -1608,7 +1609,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi addr = page_pool_get_dma_addr(au->page); dma_sync_single_for_cpu(rq->pdev, addr + wi->offset, - frag_consumed_bytes, DMA_FROM_DEVICE); + frag_consumed_bytes, rq->buff.map_dir); if (!xdp_buff_has_frags(&xdp)) { /* Init on the first fragment to avoid cold cache access @@ -1905,7 +1906,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset); /* copy header */ addr = page_pool_get_dma_addr(head_au->page); - mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, addr, + mlx5e_copy_skb_header(rq, skb, head_au->page, addr, head_offset, head_offset, headlen); /* skb linear part was allocated with headlen and aligned to long */ skb->tail += headlen; @@ -1939,7 +1940,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, addr = page_pool_get_dma_addr(au->page); dma_sync_single_range_for_cpu(rq->pdev, addr, head_offset, - frag_size, DMA_FROM_DEVICE); + frag_size, rq->buff.map_dir); net_prefetch(data); prog = rcu_dereference(rq->xdp_prog); @@ -1987,7 +1988,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, if (likely(frag_size <= BIT(MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE))) { /* build SKB around header */ - dma_sync_single_range_for_cpu(rq->pdev, head->addr, 0, frag_size, DMA_FROM_DEVICE); + dma_sync_single_range_for_cpu(rq->pdev, head->addr, 0, frag_size, rq->buff.map_dir); prefetchw(hdr); prefetch(data); skb = mlx5e_build_linear_skb(rq, hdr, frag_size, rx_headroom, head_size, 0); @@ -2009,7 +2010,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, } prefetchw(skb->data); - mlx5e_copy_skb_header(rq->pdev, skb, head->page, head->addr, + mlx5e_copy_skb_header(rq, skb, head->page, head->addr, head_offset + rx_headroom, rx_headroom, head_size); /* skb linear part was allocated with headlen and aligned to long */ -- GitLab From 08912ea799cd2a43b8999457e316967fe4e2f327 Mon Sep 17 00:00:00 2001 From: Roi Dayan Date: Wed, 2 Nov 2022 23:55:44 -0700 Subject: [PATCH 1453/1927] net/mlx5e: Fix tc acts array not to be dependent on enum order The tc acts array should not be dependent on kernel internal flow action id enum. Fix the array initialization. Fixes: fad547906980 ("net/mlx5e: Add tc action infrastructure") Signed-off-by: Roi Dayan Reviewed-by: Maor Dickman Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/en/tc/act/act.c | 92 +++++++------------ 1 file changed, 32 insertions(+), 60 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c index 305fde62a78de..3337241cfd84c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c @@ -6,70 +6,42 @@ #include "en/tc_priv.h" #include "mlx5_core.h" -/* Must be aligned with enum flow_action_id. */ static struct mlx5e_tc_act *tc_acts_fdb[NUM_FLOW_ACTIONS] = { - &mlx5e_tc_act_accept, - &mlx5e_tc_act_drop, - &mlx5e_tc_act_trap, - &mlx5e_tc_act_goto, - &mlx5e_tc_act_mirred, - &mlx5e_tc_act_mirred, - &mlx5e_tc_act_redirect_ingress, - NULL, /* FLOW_ACTION_MIRRED_INGRESS, */ - &mlx5e_tc_act_vlan, - &mlx5e_tc_act_vlan, - &mlx5e_tc_act_vlan_mangle, - &mlx5e_tc_act_tun_encap, - &mlx5e_tc_act_tun_decap, - &mlx5e_tc_act_pedit, - &mlx5e_tc_act_pedit, - &mlx5e_tc_act_csum, - NULL, /* FLOW_ACTION_MARK, */ - &mlx5e_tc_act_ptype, - NULL, /* FLOW_ACTION_PRIORITY, */ - NULL, /* FLOW_ACTION_WAKE, */ - NULL, /* FLOW_ACTION_QUEUE, */ - &mlx5e_tc_act_sample, - &mlx5e_tc_act_police, - &mlx5e_tc_act_ct, - NULL, /* FLOW_ACTION_CT_METADATA, */ - &mlx5e_tc_act_mpls_push, - &mlx5e_tc_act_mpls_pop, - NULL, /* FLOW_ACTION_MPLS_MANGLE, */ - NULL, /* FLOW_ACTION_GATE, */ - NULL, /* FLOW_ACTION_PPPOE_PUSH, */ - NULL, /* FLOW_ACTION_JUMP, */ - NULL, /* FLOW_ACTION_PIPE, */ - &mlx5e_tc_act_vlan, - &mlx5e_tc_act_vlan, + [FLOW_ACTION_ACCEPT] = &mlx5e_tc_act_accept, + [FLOW_ACTION_DROP] = &mlx5e_tc_act_drop, + [FLOW_ACTION_TRAP] = &mlx5e_tc_act_trap, + [FLOW_ACTION_GOTO] = &mlx5e_tc_act_goto, + [FLOW_ACTION_REDIRECT] = &mlx5e_tc_act_mirred, + [FLOW_ACTION_MIRRED] = &mlx5e_tc_act_mirred, + [FLOW_ACTION_REDIRECT_INGRESS] = &mlx5e_tc_act_redirect_ingress, + [FLOW_ACTION_VLAN_PUSH] = &mlx5e_tc_act_vlan, + [FLOW_ACTION_VLAN_POP] = &mlx5e_tc_act_vlan, + [FLOW_ACTION_VLAN_MANGLE] = &mlx5e_tc_act_vlan_mangle, + [FLOW_ACTION_TUNNEL_ENCAP] = &mlx5e_tc_act_tun_encap, + [FLOW_ACTION_TUNNEL_DECAP] = &mlx5e_tc_act_tun_decap, + [FLOW_ACTION_MANGLE] = &mlx5e_tc_act_pedit, + [FLOW_ACTION_ADD] = &mlx5e_tc_act_pedit, + [FLOW_ACTION_CSUM] = &mlx5e_tc_act_csum, + [FLOW_ACTION_PTYPE] = &mlx5e_tc_act_ptype, + [FLOW_ACTION_SAMPLE] = &mlx5e_tc_act_sample, + [FLOW_ACTION_POLICE] = &mlx5e_tc_act_police, + [FLOW_ACTION_CT] = &mlx5e_tc_act_ct, + [FLOW_ACTION_MPLS_PUSH] = &mlx5e_tc_act_mpls_push, + [FLOW_ACTION_MPLS_POP] = &mlx5e_tc_act_mpls_pop, + [FLOW_ACTION_VLAN_PUSH_ETH] = &mlx5e_tc_act_vlan, + [FLOW_ACTION_VLAN_POP_ETH] = &mlx5e_tc_act_vlan, }; -/* Must be aligned with enum flow_action_id. */ static struct mlx5e_tc_act *tc_acts_nic[NUM_FLOW_ACTIONS] = { - &mlx5e_tc_act_accept, - &mlx5e_tc_act_drop, - NULL, /* FLOW_ACTION_TRAP, */ - &mlx5e_tc_act_goto, - &mlx5e_tc_act_mirred_nic, - NULL, /* FLOW_ACTION_MIRRED, */ - NULL, /* FLOW_ACTION_REDIRECT_INGRESS, */ - NULL, /* FLOW_ACTION_MIRRED_INGRESS, */ - NULL, /* FLOW_ACTION_VLAN_PUSH, */ - NULL, /* FLOW_ACTION_VLAN_POP, */ - NULL, /* FLOW_ACTION_VLAN_MANGLE, */ - NULL, /* FLOW_ACTION_TUNNEL_ENCAP, */ - NULL, /* FLOW_ACTION_TUNNEL_DECAP, */ - &mlx5e_tc_act_pedit, - &mlx5e_tc_act_pedit, - &mlx5e_tc_act_csum, - &mlx5e_tc_act_mark, - NULL, /* FLOW_ACTION_PTYPE, */ - NULL, /* FLOW_ACTION_PRIORITY, */ - NULL, /* FLOW_ACTION_WAKE, */ - NULL, /* FLOW_ACTION_QUEUE, */ - NULL, /* FLOW_ACTION_SAMPLE, */ - NULL, /* FLOW_ACTION_POLICE, */ - &mlx5e_tc_act_ct, + [FLOW_ACTION_ACCEPT] = &mlx5e_tc_act_accept, + [FLOW_ACTION_DROP] = &mlx5e_tc_act_drop, + [FLOW_ACTION_GOTO] = &mlx5e_tc_act_goto, + [FLOW_ACTION_REDIRECT] = &mlx5e_tc_act_mirred_nic, + [FLOW_ACTION_MANGLE] = &mlx5e_tc_act_pedit, + [FLOW_ACTION_ADD] = &mlx5e_tc_act_pedit, + [FLOW_ACTION_CSUM] = &mlx5e_tc_act_csum, + [FLOW_ACTION_MARK] = &mlx5e_tc_act_mark, + [FLOW_ACTION_CT] = &mlx5e_tc_act_ct, }; /** -- GitLab From 9e06430841363a1d2932d546fdce1cc5edb3c2a0 Mon Sep 17 00:00:00 2001 From: Jianbo Liu Date: Wed, 2 Nov 2022 23:55:45 -0700 Subject: [PATCH 1454/1927] net/mlx5e: TC, Fix wrong rejection of packet-per-second policing In the bellow commit, we added support for PPS policing without removing the check which block offload of such cases. Fix it by removing this check. Fixes: a8d52b024d6d ("net/mlx5e: TC, Support offloading police action") Signed-off-by: Jianbo Liu Reviewed-by: Maor Dickman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index dd6fea9e9a5b7..372dfb89e3969 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -4758,12 +4758,6 @@ int mlx5e_policer_validate(const struct flow_action *action, return -EOPNOTSUPP; } - if (act->police.rate_pkt_ps) { - NL_SET_ERR_MSG_MOD(extack, - "QoS offload not support packets per second"); - return -EOPNOTSUPP; - } - return 0; } -- GitLab From f4f4096b410e8d31c3f07f39de3b17d144edd53d Mon Sep 17 00:00:00 2001 From: Roi Dayan Date: Wed, 2 Nov 2022 23:55:46 -0700 Subject: [PATCH 1455/1927] net/mlx5e: E-Switch, Fix comparing termination table instance The pkt_reformat pointer being saved under flow_act and not dest attribute in the termination table instance. Fix the comparison pointers. Also fix returning success if one pkt_reformat pointer is null and the other is not. Fixes: 249ccc3c95bd ("net/mlx5e: Add support for offloading traffic from uplink to uplink") Signed-off-by: Roi Dayan Reviewed-by: Chris Mi Signed-off-by: Saeed Mahameed --- .../mellanox/mlx5/core/eswitch_offloads_termtbl.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c index ee568bf34ae25..108a3503f413c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c @@ -30,9 +30,9 @@ mlx5_eswitch_termtbl_hash(struct mlx5_flow_act *flow_act, sizeof(dest->vport.num), hash); hash = jhash((const void *)&dest->vport.vhca_id, sizeof(dest->vport.num), hash); - if (dest->vport.pkt_reformat) - hash = jhash(dest->vport.pkt_reformat, - sizeof(*dest->vport.pkt_reformat), + if (flow_act->pkt_reformat) + hash = jhash(flow_act->pkt_reformat, + sizeof(*flow_act->pkt_reformat), hash); return hash; } @@ -53,9 +53,11 @@ mlx5_eswitch_termtbl_cmp(struct mlx5_flow_act *flow_act1, if (ret) return ret; - return dest1->vport.pkt_reformat && dest2->vport.pkt_reformat ? - memcmp(dest1->vport.pkt_reformat, dest2->vport.pkt_reformat, - sizeof(*dest1->vport.pkt_reformat)) : 0; + if (flow_act1->pkt_reformat && flow_act2->pkt_reformat) + return memcmp(flow_act1->pkt_reformat, flow_act2->pkt_reformat, + sizeof(*flow_act1->pkt_reformat)); + + return !(flow_act1->pkt_reformat == flow_act2->pkt_reformat); } static int -- GitLab From 7f1a6d4b9e820b08479a07f6e14c36ccfa641751 Mon Sep 17 00:00:00 2001 From: Roi Dayan Date: Wed, 2 Nov 2022 23:55:47 -0700 Subject: [PATCH 1456/1927] net/mlx5e: TC, Fix slab-out-of-bounds in parse_tc_actions esw_attr is only allocated if namespace is fdb. BUG: KASAN: slab-out-of-bounds in parse_tc_actions+0xdc6/0x10e0 [mlx5_core] Write of size 4 at addr ffff88815f185b04 by task tc/2135 CPU: 5 PID: 2135 Comm: tc Not tainted 6.1.0-rc2+ #2 Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014 Call Trace: dump_stack_lvl+0x57/0x7d print_report+0x170/0x471 ? parse_tc_actions+0xdc6/0x10e0 [mlx5_core] kasan_report+0xbc/0xf0 ? parse_tc_actions+0xdc6/0x10e0 [mlx5_core] parse_tc_actions+0xdc6/0x10e0 [mlx5_core] Fixes: 94d651739e17 ("net/mlx5e: TC, Fix cloned flow attr instance dests are not zeroed") Signed-off-by: Roi Dayan Reviewed-by: Maor Dickman Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_tc.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 372dfb89e3969..5a6aa61ec82af 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -3633,10 +3633,14 @@ mlx5e_clone_flow_attr_for_post_act(struct mlx5_flow_attr *attr, attr2->action = 0; attr2->flags = 0; attr2->parse_attr = parse_attr; - attr2->esw_attr->out_count = 0; - attr2->esw_attr->split_count = 0; attr2->dest_chain = 0; attr2->dest_ft = NULL; + + if (ns_type == MLX5_FLOW_NAMESPACE_FDB) { + attr2->esw_attr->out_count = 0; + attr2->esw_attr->split_count = 0; + } + return attr2; } -- GitLab From 0811664da064c6d7ca64c02f5579f758a007e52d Mon Sep 17 00:00:00 2001 From: Pu Lehui Date: Tue, 8 Nov 2022 20:19:45 +0800 Subject: [PATCH 1457/1927] selftests/bpf: Fix casting error when cross-compiling test_verifier for 32-bit platforms When cross-compiling test_verifier for 32-bit platforms, the casting error is shown below: test_verifier.c:1263:27: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast] 1263 | info.xlated_prog_insns = (__u64)*buf; | ^ cc1: all warnings being treated as errors Fix it by adding zero-extension for it. Fixes: 933ff53191eb ("selftests/bpf: specify expected instructions in test_verifier tests") Signed-off-by: Pu Lehui Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20221108121945.4104644-1-pulehui@huaweicloud.com Signed-off-by: Martin KaFai Lau --- tools/testing/selftests/bpf/test_verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 2dbcbf363c181..b605a70d4f6ba 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -1260,7 +1260,7 @@ static int get_xlated_program(int fd_prog, struct bpf_insn **buf, int *cnt) bzero(&info, sizeof(info)); info.xlated_prog_len = xlated_prog_len; - info.xlated_prog_insns = (__u64)*buf; + info.xlated_prog_insns = (__u64)(unsigned long)*buf; if (bpf_obj_get_info_by_fd(fd_prog, &info, &info_len)) { perror("second bpf_obj_get_info_by_fd failed"); goto out_free_buf; -- GitLab From 5704bc7e8991164b14efb748b5afa0715c25fac3 Mon Sep 17 00:00:00 2001 From: Yang Jihong Date: Tue, 8 Nov 2022 09:58:57 +0800 Subject: [PATCH 1458/1927] selftests/bpf: Fix test_progs compilation failure in 32-bit arch test_progs fails to be compiled in the 32-bit arch, log is as follows: test_progs.c:1013:52: error: format '%ld' expects argument of type 'long int', but argument 3 has type 'size_t' {aka 'unsigned int'} [-Werror=format=] 1013 | sprintf(buf, "MSG_TEST_LOG (cnt: %ld, last: %d)", | ~~^ | | | long int | %d 1014 | strlen(msg->test_log.log_buf), | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | | | size_t {aka unsigned int} Fix it. Fixes: 91b2c0afd00c ("selftests/bpf: Add parallelism to test_progs") Signed-off-by: Yang Jihong Acked-by: Yonghong Song Link: https://lore.kernel.org/r/20221108015857.132457-1-yangjihong1@huawei.com Signed-off-by: Martin KaFai Lau --- tools/testing/selftests/bpf/test_progs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 0e9a47f978908..3fef451d88313 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -1010,7 +1010,7 @@ static inline const char *str_msg(const struct msg *msg, char *buf) msg->subtest_done.have_log); break; case MSG_TEST_LOG: - sprintf(buf, "MSG_TEST_LOG (cnt: %ld, last: %d)", + sprintf(buf, "MSG_TEST_LOG (cnt: %zu, last: %d)", strlen(msg->test_log.log_buf), msg->test_log.is_last); break; -- GitLab From f23df5220d2bf8d5e639f074b76f206a736d09e1 Mon Sep 17 00:00:00 2001 From: Norbert Zulinski Date: Mon, 10 Oct 2022 10:22:22 -0400 Subject: [PATCH 1459/1927] ice: Fix spurious interrupt during removal of trusted VF Previously, during removal of trusted VF when VF is down there was number of spurious interrupt equal to number of queues on VF. Add check if VF already has inactive queues. If VF is disabled and has inactive rx queues then do not disable rx queues. Add check in ice_vsi_stop_tx_ring if it's VF's vsi and if VF is disabled. Fixes: efe41860008e ("ice: Fix memory corruption in VF driver") Signed-off-by: Norbert Zulinski Signed-off-by: Mateusz Palczewski Tested-by: Konrad Jankowski Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/ice/ice_base.c | 2 +- drivers/net/ethernet/intel/ice/ice_lib.c | 25 +++++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_lib.h | 1 + drivers/net/ethernet/intel/ice/ice_vf_lib.c | 5 ++++- 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index 9e36f01dfa4fb..e864634d66bc6 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -958,7 +958,7 @@ ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, * associated to the queue to schedule NAPI handler */ q_vector = ring->q_vector; - if (q_vector) + if (q_vector && !(vsi->vf && ice_is_vf_disabled(vsi->vf))) ice_trigger_sw_intr(hw, q_vector); status = ice_dis_vsi_txq(vsi->port_info, txq_meta->vsi_idx, diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 938ba8c215cb0..7276badfa19ea 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -2239,6 +2239,31 @@ int ice_vsi_stop_xdp_tx_rings(struct ice_vsi *vsi) return ice_vsi_stop_tx_rings(vsi, ICE_NO_RESET, 0, vsi->xdp_rings, vsi->num_xdp_txq); } +/** + * ice_vsi_is_rx_queue_active + * @vsi: the VSI being configured + * + * Return true if at least one queue is active. + */ +bool ice_vsi_is_rx_queue_active(struct ice_vsi *vsi) +{ + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + int i; + + ice_for_each_rxq(vsi, i) { + u32 rx_reg; + int pf_q; + + pf_q = vsi->rxq_map[i]; + rx_reg = rd32(hw, QRX_CTRL(pf_q)); + if (rx_reg & QRX_CTRL_QENA_STAT_M) + return true; + } + + return false; +} + /** * ice_vsi_is_vlan_pruning_ena - check if VLAN pruning is enabled or not * @vsi: VSI to check whether or not VLAN pruning is enabled. diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h index ec4bf0c89857c..dcdf69a693e91 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_lib.h @@ -129,4 +129,5 @@ u16 ice_vsi_num_non_zero_vlans(struct ice_vsi *vsi); bool ice_is_feature_supported(struct ice_pf *pf, enum ice_feature f); void ice_clear_feature_support(struct ice_pf *pf, enum ice_feature f); void ice_init_feature_support(struct ice_pf *pf); +bool ice_vsi_is_rx_queue_active(struct ice_vsi *vsi); #endif /* !_ICE_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c index 0abeed092de1d..1c51778db951b 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c @@ -576,7 +576,10 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags) return -EINVAL; } ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, vf->vf_id); - ice_vsi_stop_all_rx_rings(vsi); + + if (ice_vsi_is_rx_queue_active(vsi)) + ice_vsi_stop_all_rx_rings(vsi); + dev_dbg(dev, "VF is already disabled, there is no need for resetting it, telling VM, all is fine %d\n", vf->vf_id); return 0; -- GitLab From 0e710a3ffd0caaf23b8791b041e8792f252f8e4f Mon Sep 17 00:00:00 2001 From: Michal Jaron Date: Fri, 14 Oct 2022 10:45:37 +0200 Subject: [PATCH 1460/1927] iavf: Fix VF driver counting VLAN 0 filters VF driver mistakenly counts VLAN 0 filters, when no PF driver counts them. Do not count VLAN 0 filters, when VLAN_V2 is engaged. Counting those filters in, will affect filters size by -1, when sending batched VLAN addition message. Fixes: 968996c070ef ("iavf: Fix VLAN_V2 addition/rejection") Signed-off-by: Przemyslaw Patynowski Signed-off-by: Michal Jaron Signed-off-by: Kamil Maziarz Tested-by: Konrad Jankowski Signed-off-by: Tony Nguyen --- drivers/net/ethernet/intel/iavf/iavf_virtchnl.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 5a9e6563923eb..24a701fd140e9 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -2438,6 +2438,8 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, list_for_each_entry(f, &adapter->vlan_filter_list, list) { if (f->is_new_vlan) { f->is_new_vlan = false; + if (!f->vlan.vid) + continue; if (f->vlan.tpid == ETH_P_8021Q) set_bit(f->vlan.vid, adapter->vsi.active_cvlans); -- GitLab From bcde9dad1bfa3754e004c14774d42e1f1210efe9 Mon Sep 17 00:00:00 2001 From: Tim Huang Date: Thu, 3 Nov 2022 11:05:19 +0800 Subject: [PATCH 1461/1927] drm/amd/pm: update SMU IP v13.0.4 msg interface header Some of the unused messages that were used earlier in development have been freed up as spare messages, no intended functional changes. Signed-off-by: Mario Limonciello Signed-off-by: Tim Huang Acked-by: Alex Deucher Reviewed-by: Yifan Zhang Signed-off-by: Alex Deucher --- .../amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h index d9b0cd7522006..f4d6c07b56ea7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h @@ -54,14 +54,14 @@ #define PPSMC_MSG_TestMessage 0x01 ///< To check if PMFW is alive and responding. Requirement specified by PMFW team #define PPSMC_MSG_GetPmfwVersion 0x02 ///< Get PMFW version #define PPSMC_MSG_GetDriverIfVersion 0x03 ///< Get PMFW_DRIVER_IF version -#define PPSMC_MSG_EnableGfxOff 0x04 ///< Enable GFXOFF -#define PPSMC_MSG_DisableGfxOff 0x05 ///< Disable GFXOFF +#define PPSMC_MSG_SPARE0 0x04 ///< SPARE +#define PPSMC_MSG_SPARE1 0x05 ///< SPARE #define PPSMC_MSG_PowerDownVcn 0x06 ///< Power down VCN #define PPSMC_MSG_PowerUpVcn 0x07 ///< Power up VCN; VCN is power gated by default #define PPSMC_MSG_SetHardMinVcn 0x08 ///< For wireless display #define PPSMC_MSG_SetSoftMinGfxclk 0x09 ///< Set SoftMin for GFXCLK, argument is frequency in MHz -#define PPSMC_MSG_ActiveProcessNotify 0x0A ///< Needs update -#define PPSMC_MSG_ForcePowerDownGfx 0x0B ///< Force power down GFX, i.e. enter GFXOFF +#define PPSMC_MSG_SPARE2 0x0A ///< SPARE +#define PPSMC_MSG_SPARE3 0x0B ///< SPARE #define PPSMC_MSG_PrepareMp1ForUnload 0x0C ///< Prepare PMFW for GFX driver unload #define PPSMC_MSG_SetDriverDramAddrHigh 0x0D ///< Set high 32 bits of DRAM address for Driver table transfer #define PPSMC_MSG_SetDriverDramAddrLow 0x0E ///< Set low 32 bits of DRAM address for Driver table transfer @@ -73,8 +73,7 @@ #define PPSMC_MSG_SetSoftMinFclk 0x14 ///< Set hard min for FCLK #define PPSMC_MSG_SetSoftMinVcn 0x15 ///< Set soft min for VCN clocks (VCLK and DCLK) - -#define PPSMC_MSG_EnableGfxImu 0x16 ///< Needs update +#define PPSMC_MSG_EnableGfxImu 0x16 ///< Enable GFX IMU #define PPSMC_MSG_GetGfxclkFrequency 0x17 ///< Get GFX clock frequency #define PPSMC_MSG_GetFclkFrequency 0x18 ///< Get FCLK frequency @@ -102,8 +101,8 @@ #define PPSMC_MSG_SetHardMinIspxclkByFreq 0x2C ///< Set HardMin by frequency for ISPXCLK #define PPSMC_MSG_PowerDownUmsch 0x2D ///< Power down VCN.UMSCH (aka VSCH) scheduler #define PPSMC_MSG_PowerUpUmsch 0x2E ///< Power up VCN.UMSCH (aka VSCH) scheduler -#define PPSMC_Message_IspStutterOn_MmhubPgDis 0x2F ///< ISP StutterOn mmHub PgDis -#define PPSMC_Message_IspStutterOff_MmhubPgEn 0x30 ///< ISP StufferOff mmHub PgEn +#define PPSMC_MSG_IspStutterOn_MmhubPgDis 0x2F ///< ISP StutterOn mmHub PgDis +#define PPSMC_MSG_IspStutterOff_MmhubPgEn 0x30 ///< ISP StufferOff mmHub PgEn #define PPSMC_Message_Count 0x31 ///< Total number of PPSMC messages /** @}*/ -- GitLab From 2d2ccc4cbf440931843e64974da9fcd129d401a8 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 3 Nov 2022 17:01:46 -0400 Subject: [PATCH 1462/1927] drm/amdkfd: Fix error handling in kfd_criu_restore_events mutex_unlock before the exit label because all the error code paths that jump there didn't take that lock. This fixes unbalanced locking errors in case of restore errors. Fixes: 40e8a766a761 ("drm/amdkfd: CRIU checkpoint and restore events") Signed-off-by: Felix Kuehling Reviewed-by: Rajneesh Bhardwaj Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_events.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 83e3ce9f60491..729d26d648af3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -506,6 +506,7 @@ int kfd_criu_restore_event(struct file *devkfd, ret = create_other_event(p, ev, &ev_priv->event_id); break; } + mutex_unlock(&p->event_mutex); exit: if (ret) @@ -513,8 +514,6 @@ int kfd_criu_restore_event(struct file *devkfd, kfree(ev_priv); - mutex_unlock(&p->event_mutex); - return ret; } -- GitLab From a0fba3e078801b9596e61a0687d398664cdc36e6 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 1 Nov 2022 15:02:48 -0400 Subject: [PATCH 1463/1927] drm/amdkfd: Fix error handling in criu_checkpoint Checkpoint BOs last. That way we don't need to close dmabuf FDs if something else fails later. This avoids problematic access to user mode memory in the error handling code path. criu_checkpoint_bos has its own error handling and cleanup that does not depend on access to user memory. In the private data, keep BOs before the remaining objects. This is necessary to restore things in the correct order as restoring events depends on the events-page BO being restored first. Fixes: be072b06c739 ("drm/amdkfd: CRIU export BOs as prime dmabuf objects") Reported-by: Jann Horn CC: Rajneesh Bhardwaj Signed-off-by: Felix Kuehling Reviewed-and-tested-by: Rajneesh Bhardwaj Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 34 +++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 5feaba6a77de4..6d291aa6386bd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1950,7 +1950,7 @@ static int criu_checkpoint(struct file *filep, { int ret; uint32_t num_devices, num_bos, num_objects; - uint64_t priv_size, priv_offset = 0; + uint64_t priv_size, priv_offset = 0, bo_priv_offset; if (!args->devices || !args->bos || !args->priv_data) return -EINVAL; @@ -1994,38 +1994,34 @@ static int criu_checkpoint(struct file *filep, if (ret) goto exit_unlock; - ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos, - (uint8_t __user *)args->priv_data, &priv_offset); - if (ret) - goto exit_unlock; + /* Leave room for BOs in the private data. They need to be restored + * before events, but we checkpoint them last to simplify the error + * handling. + */ + bo_priv_offset = priv_offset; + priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data); if (num_objects) { ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data, &priv_offset); if (ret) - goto close_bo_fds; + goto exit_unlock; ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data, &priv_offset); if (ret) - goto close_bo_fds; + goto exit_unlock; ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset); if (ret) - goto close_bo_fds; + goto exit_unlock; } -close_bo_fds: - if (ret) { - /* If IOCTL returns err, user assumes all FDs opened in criu_dump_bos are closed */ - uint32_t i; - struct kfd_criu_bo_bucket *bo_buckets = (struct kfd_criu_bo_bucket *) args->bos; - - for (i = 0; i < num_bos; i++) { - if (bo_buckets[i].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) - close_fd(bo_buckets[i].dmabuf_fd); - } - } + /* This must be the last thing in this function that can fail. + * Otherwise we leak dmabuf file descriptors. + */ + ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos, + (uint8_t __user *)args->priv_data, &bo_priv_offset); exit_unlock: mutex_unlock(&p->mutex); -- GitLab From f9e69496450352fa0504fd5a8fd9134b31116558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 2 Nov 2022 14:55:13 +0100 Subject: [PATCH 1464/1927] drm/amdgpu: workaround for TLB seq race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It can happen that we query the sequence value before the callback had a chance to run. Workaround that by grabbing the fence lock and releasing it again. Should be replaced by hw handling soon. Signed-off-by: Christian König CC: stable@vger.kernel.org # 5.19+ Fixes: 5255e146c99a6 ("drm/amdgpu: rework TLB flushing") Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2113 Acked-by: Alex Deucher Acked-by: Philip Yang Tested-by: Stefan Springer Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 83acb7bd80feb..1d31771b42308 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -492,6 +492,21 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m); */ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm) { + unsigned long flags; + spinlock_t *lock; + + /* + * Workaround to stop racing between the fence signaling and handling + * the cb. The lock is static after initially setting it up, just make + * sure that the dma_fence structure isn't freed up. + */ + rcu_read_lock(); + lock = vm->last_tlb_flush->lock; + rcu_read_unlock(); + + spin_lock_irqsave(lock, flags); + spin_unlock_irqrestore(lock, flags); + return atomic64_read(&vm->tlb_seq); } -- GitLab From 00efc965a7b7990e63e3506e6d240e6271bd8571 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Mon, 24 Oct 2022 13:39:02 -0400 Subject: [PATCH 1465/1927] drm/amd/display: Wait for VBLANK during pipe programming [Description] - Wait for vblank during front end programming for global sync to ensure all double buffer updates take. - This prevents underflow in some cases. Reviewed-by: Martin Leung Acked-by: Alan Liu Signed-off-by: Alvin Lee Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c index f3334f513eb4f..b465a83bde6f0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c @@ -1663,6 +1663,7 @@ static void dcn20_program_pipe( pipe_ctx->pipe_dlg_param.vupdate_width); if (pipe_ctx->stream->mall_stream_config.type != SUBVP_PHANTOM) { + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); } -- GitLab From ea8626425859cf99de0c7f7e5b2c485d5f19b20e Mon Sep 17 00:00:00 2001 From: Leo Ma Date: Thu, 15 Sep 2022 12:03:04 -0400 Subject: [PATCH 1466/1927] drm/amd/display: Adding HDMI SCDC DEVICE_ID define [Why && How] We will need to differentiate vendor behavior in the future. Reviewed-by: Chris Park Acked-by: Alan Liu Signed-off-by: Leo Ma Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c index 651231387043d..ce8d6a54ca54b 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c @@ -82,6 +82,7 @@ struct dp_hdmi_dongle_signature_data { #define HDMI_SCDC_STATUS_FLAGS 0x40 #define HDMI_SCDC_ERR_DETECT 0x50 #define HDMI_SCDC_TEST_CONFIG 0xC0 +#define HDMI_SCDC_DEVICE_ID 0xD3 union hdmi_scdc_update_read_data { uint8_t byte[2]; -- GitLab From f7085cbf117e4fc2af96790cbc34f5a38a236943 Mon Sep 17 00:00:00 2001 From: Max Tseng Date: Mon, 17 Oct 2022 20:55:36 +0800 Subject: [PATCH 1467/1927] drm/amd/display: Cursor update refactor: PSR-SU support condition [Why] PSR-SU requires extra conditions while cursor update. Reviewed-by: Robin Chen Acked-by: Alan Liu Signed-off-by: Max Tseng Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index 67eef5beab951..4cb912bf400b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -859,11 +859,59 @@ void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv) diag_data.is_cw6_enabled); } +static bool dc_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx) +{ + struct pipe_ctx *test_pipe, *split_pipe; + const struct scaler_data *scl_data = &pipe_ctx->plane_res.scl_data; + struct rect r1 = scl_data->recout, r2, r2_half; + int r1_r = r1.x + r1.width, r1_b = r1.y + r1.height, r2_r, r2_b; + int cur_layer = pipe_ctx->plane_state->layer_index; + + /** + * Disable the cursor if there's another pipe above this with a + * plane that contains this pipe's viewport to prevent double cursor + * and incorrect scaling artifacts. + */ + for (test_pipe = pipe_ctx->top_pipe; test_pipe; + test_pipe = test_pipe->top_pipe) { + // Skip invisible layer and pipe-split plane on same layer + if (!test_pipe->plane_state->visible || test_pipe->plane_state->layer_index == cur_layer) + continue; + + r2 = test_pipe->plane_res.scl_data.recout; + r2_r = r2.x + r2.width; + r2_b = r2.y + r2.height; + split_pipe = test_pipe; + + /** + * There is another half plane on same layer because of + * pipe-split, merge together per same height. + */ + for (split_pipe = pipe_ctx->top_pipe; split_pipe; + split_pipe = split_pipe->top_pipe) + if (split_pipe->plane_state->layer_index == test_pipe->plane_state->layer_index) { + r2_half = split_pipe->plane_res.scl_data.recout; + r2.x = (r2_half.x < r2.x) ? r2_half.x : r2.x; + r2.width = r2.width + r2_half.width; + r2_r = r2.x + r2.width; + break; + } + + if (r1.x >= r2.x && r1.y >= r2.y && r1_r <= r2_r && r1_b <= r2_b) + return true; + } + + return false; +} + static bool dc_dmub_should_update_cursor_data(struct pipe_ctx *pipe_ctx) { if (pipe_ctx->plane_state != NULL) { if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE) return false; + + if (dc_can_pipe_disable_cursor(pipe_ctx)) + return false; } if ((pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1 || -- GitLab From a36021b1fccf1807a5557f35b0b407f1c1a65743 Mon Sep 17 00:00:00 2001 From: Nawwar Ali Date: Wed, 19 Oct 2022 17:57:06 +0800 Subject: [PATCH 1468/1927] drm/amd/display: Update 709 gamma to 2.222 as stated in the standerd [WHY] Previously driver use gamma 2.2 for 709 color space, but the standard is to use gamma of 2.222 [HOW] Change it gamma to 2.222 Reviewed-by: Krunoslav Kovac Acked-by: Alan Liu Signed-off-by: Nawwar Ali Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/modules/color/color_gamma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c index 447a0ec9cbe21..f6034213c700d 100644 --- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c +++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c @@ -61,7 +61,7 @@ static const int32_t numerator01[] = { 31308, 180000, 0, 0, 0}; static const int32_t numerator02[] = { 12920, 4500, 0, 0, 0}; static const int32_t numerator03[] = { 55, 99, 0, 0, 0}; static const int32_t numerator04[] = { 55, 99, 0, 0, 0}; -static const int32_t numerator05[] = { 2400, 2200, 2200, 2400, 2600}; +static const int32_t numerator05[] = { 2400, 2222, 2200, 2400, 2600}; /* one-time setup of X points */ void setup_x_points_distribution(void) -- GitLab From 5abb360ba74f59cc7ea5a4ab149c49d70d869239 Mon Sep 17 00:00:00 2001 From: Wenjing Liu Date: Mon, 24 Oct 2022 12:18:41 -0400 Subject: [PATCH 1469/1927] drm/amd/display: Consider dp cable id only when data is non zero Cable ID is a DP2 feature to identify max certified link rate that a cable can carry. The cable identification method requires both cable and display hardware support. Since the specs comes late, it is anticipated that the first round of DP2 cables and displays may not be fully compatible to reliably return cable ID data. Therefore the decision of our cable id policy is that if the cable can return non zero cable id data, we will take cable's link rate capability into account. However if we get zero data, the cable link rate capability is considered inconclusive. In this case, we will not take cable's capability into account to avoid of over limiting hardware capability from users. The max overall link rate capability is still determined after actual dp pre-training. Cable id is considered as an auxiliary method of determining max link bandwidth capability. Reviewed-by: George Shen Acked-by: Alan Liu Signed-off-by: Wenjing Liu Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/core/dc_link_dp.c | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index e5ab751a5ca1d..b0cfe21871e4d 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -3020,7 +3020,7 @@ static enum dc_link_rate get_lttpr_max_link_rate(struct dc_link *link) static enum dc_link_rate get_cable_max_link_rate(struct dc_link *link) { - enum dc_link_rate cable_max_link_rate = LINK_RATE_HIGH3; + enum dc_link_rate cable_max_link_rate = LINK_RATE_UNKNOWN; if (link->dpcd_caps.cable_id.bits.UHBR10_20_CAPABILITY & DP_UHBR20) cable_max_link_rate = LINK_RATE_UHBR20; @@ -3083,15 +3083,29 @@ struct dc_link_settings dp_get_max_link_cap(struct dc_link *link) max_link_cap.link_spread = link->reported_link_cap.link_spread; - /* Lower link settings based on cable attributes */ + /* Lower link settings based on cable attributes + * Cable ID is a DP2 feature to identify max certified link rate that + * a cable can carry. The cable identification method requires both + * cable and display hardware support. Since the specs comes late, it is + * anticipated that the first round of DP2 cables and displays may not + * be fully compatible to reliably return cable ID data. Therefore the + * decision of our cable id policy is that if the cable can return non + * zero cable id data, we will take cable's link rate capability into + * account. However if we get zero data, the cable link rate capability + * is considered inconclusive. In this case, we will not take cable's + * capability into account to avoid of over limiting hardware capability + * from users. The max overall link rate capability is still determined + * after actual dp pre-training. Cable id is considered as an auxiliary + * method of determining max link bandwidth capability. + */ cable_max_link_rate = get_cable_max_link_rate(link); if (!link->dc->debug.ignore_cable_id && + cable_max_link_rate != LINK_RATE_UNKNOWN && cable_max_link_rate < max_link_cap.link_rate) max_link_cap.link_rate = cable_max_link_rate; - /* - * account for lttpr repeaters cap + /* account for lttpr repeaters cap * notes: repeaters do not snoop in the DPRX Capabilities addresses (3.6.3). */ if (dp_is_lttpr_present(link)) { -- GitLab From 1a2b886b1f242044840147347db1a595cd7f5f56 Mon Sep 17 00:00:00 2001 From: Ryan Lin Date: Wed, 26 Oct 2022 15:12:26 +0800 Subject: [PATCH 1470/1927] drm/amd/display: Waiting for 1 frame to fix the flash issue on PSR1 [Why] Needs more frames waiting before the PSR_Exit sending for the specific TCON. [How] Add relock_delay_frame_cnt to control how many frames waiting are needed before the PSR_Exit sending. The default value is 0. The Driver side can set this variable for specific TCONs. Reviewed-by: Robin Chen Acked-by: Alan Liu Signed-off-by: Ryan Lin Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c | 5 +++++ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c index cda1592c3a5be..2d3201b77d6a0 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c @@ -413,6 +413,11 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub, else copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 0; + //WA for PSR1 on specific TCON, require frame delay for frame re-lock + copy_settings_data->relock_delay_frame_cnt = 0; + if (link->dpcd_caps.sink_dev_id == DP_BRANCH_DEVICE_ID_001CF8) + copy_settings_data->relock_delay_frame_cnt = 2; + dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd); dc_dmub_srv_cmd_execute(dc->dmub_srv); dc_dmub_srv_wait_idle(dc->dmub_srv); diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h index 9df330c86a55b..34b03bc7f8382 100644 --- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h @@ -1876,10 +1876,14 @@ struct dmub_cmd_psr_copy_settings_data { * Use FSM state for PSR power up/down */ uint8_t use_phy_fsm; + /** + * frame delay for frame re-lock + */ + uint8_t relock_delay_frame_cnt; /** * Explicit padding to 2 byte boundary. */ - uint8_t pad3[2]; + uint8_t pad3; }; /** -- GitLab From 4df7e2c8d7783f53863725e21efefde19d85402f Mon Sep 17 00:00:00 2001 From: Nicholas Kazlauskas Date: Tue, 25 Oct 2022 11:26:04 -0400 Subject: [PATCH 1471/1927] drm/amd/display: Update SR watermarks for DCN314 [Why & How] New values requested by hardware after fine-tuning. Update for all memory types. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Nicholas Kazlauskas Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../dc/clk_mgr/dcn314/dcn314_clk_mgr.c | 32 +++++++++---------- .../amd/display/dc/dml/dcn314/dcn314_fpu.c | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c index 1131c6d73f6cd..20a06c04e4a1d 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c @@ -363,32 +363,32 @@ static struct wm_table ddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, } @@ -400,32 +400,32 @@ static struct wm_table lpddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c index cf420ad2b8dcd..34b6c763a4554 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c @@ -146,8 +146,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_14_soc = { }, }, .num_states = 5, - .sr_exit_time_us = 9.0, - .sr_enter_plus_exit_time_us = 11.0, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .sr_exit_z8_time_us = 442.0, .sr_enter_plus_exit_z8_time_us = 560.0, .writeback_latency_us = 12.0, -- GitLab From b17710f42d4b57fa2353b1aeff02920c9560e2e7 Mon Sep 17 00:00:00 2001 From: Nicholas Kazlauskas Date: Tue, 25 Oct 2022 11:31:30 -0400 Subject: [PATCH 1472/1927] drm/amd/display: Allow tuning DCN314 bounding box [Why] We're missing the helpers from dcn20 that would allow overriding these with DC debug options. [How] Use dcn20_patch_bounding_box to support overriding all the relevant values. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Nicholas Kazlauskas Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c index 34b6c763a4554..796c9d19e671a 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c @@ -264,11 +264,8 @@ void dcn314_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p dc->dml.soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2; } - if ((int)(dcn3_14_soc.dram_clock_change_latency_us * 1000) - != dc->debug.dram_clock_change_latency_ns - && dc->debug.dram_clock_change_latency_ns) { - dcn3_14_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000; - } + dcn20_patch_bounding_box(dc, &dcn3_14_soc); + if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) dml_init_instance(&dc->dml, &dcn3_14_soc, &dcn3_14_ip, DML_PROJECT_DCN314); else -- GitLab From a1b6667ef90b38e2480d96f11414fcb923c13ba2 Mon Sep 17 00:00:00 2001 From: Aurabindo Pillai Date: Thu, 20 Oct 2022 17:26:24 -0400 Subject: [PATCH 1473/1927] drm/amd/display: Zeromem mypipe heap struct before using it [Why&How] Bug was caused when moving variable from stack to heap because it was reusable and garbage was left over, so we need to zero mem. Reviewed-by: Martin Leung Acked-by: Alan Liu Signed-off-by: Aurabindo Pillai Signed-off-by: Martin Leung Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 3d184679f129e..ae6e6abc620b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -3192,6 +3192,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.FCLKChangeLatency, mode_lib->vba.UrgLatency[i], mode_lib->vba.SREnterPlusExitTime); + memset(&v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull, 0, sizeof(DmlPipe)); v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe.Dppclk = mode_lib->vba.RequiredDPPCLK[i][j][k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe.Dispclk = mode_lib->vba.RequiredDISPCLK[i][j]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe.PixelClock = mode_lib->vba.PixelClock[k]; -- GitLab From 9e7d03e8b046c84e1b2973a29cd800495a5a2f09 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 20 Oct 2022 18:33:05 -0400 Subject: [PATCH 1474/1927] drm/amd/display: Use min transition for SubVP into MPO [Description] - For SubVP transitioning into MPO, we want to use a minimal transition to prevent transient underflow - Transitioning a phantom pipe directly into a "real" pipe can result in underflow due to the HUBP still having it's "phantom" programming when HUBP is unblanked (have to wait for next VUPDATE of the new OTG) - Also ensure subvp pipe lock is acquired early enough for programming in dc_commit_state_no_check - When disabling phantom planes, enable phantom OTG first so the disable gets the double buffer update Reviewed-by: Aric Cyr Acked-by: Alan Liu Signed-off-by: Alvin Lee Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 43 +++++++++++------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index d446e60989484..da808996e21d1 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1054,6 +1054,7 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context) int i, j; struct dc_state *dangling_context = dc_create_state(dc); struct dc_state *current_ctx; + struct pipe_ctx *pipe; if (dangling_context == NULL) return; @@ -1096,6 +1097,16 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context) } if (should_disable && old_stream) { + pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + /* When disabling plane for a phantom pipe, we must turn on the + * phantom OTG so the disable programming gets the double buffer + * update. Otherwise the pipe will be left in a partially disabled + * state that can result in underflow or hang when enabling it + * again for different use. + */ + if (old_stream->mall_stream_config.type == SUBVP_PHANTOM) { + pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg); + } dc_rem_all_planes_for_stream(dc, old_stream, dangling_context); disable_all_writeback_pipes_for_stream(dc, old_stream, dangling_context); @@ -1749,6 +1760,12 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c context->stream_count == 0) dc->hwss.prepare_bandwidth(dc, context); + /* When SubVP is active, all HW programming must be done while + * SubVP lock is acquired + */ + if (dc->hwss.subvp_pipe_control_lock) + dc->hwss.subvp_pipe_control_lock(dc, context, true, true, NULL, subvp_prev_use); + if (dc->debug.enable_double_buffered_dsc_pg_support) dc->hwss.update_dsc_pg(dc, context, false); @@ -1776,9 +1793,6 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c dc->hwss.wait_for_mpcc_disconnect(dc, dc->res_pool, pipe); } - if (dc->hwss.subvp_pipe_control_lock) - dc->hwss.subvp_pipe_control_lock(dc, context, true, true, NULL, subvp_prev_use); - result = dc->hwss.apply_ctx_to_hw(dc, context); if (result != DC_OK) { @@ -3675,7 +3689,6 @@ static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc, struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream); bool force_minimal_pipe_splitting = false; - uint32_t i; *is_plane_addition = false; @@ -3707,27 +3720,11 @@ static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc, } } - /* For SubVP pipe split case when adding MPO video - * we need to add a minimal transition. In this case - * there will be 2 streams (1 main stream, 1 phantom - * stream). + /* For SubVP when adding MPO video we need to add a minimal transition. */ - if (cur_stream_status && - dc->current_state->stream_count == 2 && - stream->mall_stream_config.type == SUBVP_MAIN) { - bool is_pipe_split = false; - - for (i = 0; i < dc->res_pool->pipe_count; i++) { - if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream && - (dc->current_state->res_ctx.pipe_ctx[i].bottom_pipe || - dc->current_state->res_ctx.pipe_ctx[i].next_odm_pipe)) { - is_pipe_split = true; - break; - } - } - + if (cur_stream_status && stream->mall_stream_config.type == SUBVP_MAIN) { /* determine if minimal transition is required due to SubVP*/ - if (surface_count > 0 && is_pipe_split) { + if (surface_count > 0) { if (cur_stream_status->plane_count > surface_count) { force_minimal_pipe_splitting = true; } else if (cur_stream_status->plane_count < surface_count) { -- GitLab From dc55b106ad477c67f969f3432d9070c6846fb557 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Thu, 27 Oct 2022 10:51:15 -0400 Subject: [PATCH 1475/1927] drm/amd/display: Disable phantom OTG after enable for plane disable [Description] - Need to disable phantom OTG after it's enabled in order to restore it to it's original state. - If it's enabled and then an MCLK switch comes in we may not prefetch the correct data since the phantom OTG could already be in the middle of the frame. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Alvin Lee Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/core/dc.c | 14 +++++++++++++- drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c | 8 ++++++++ .../drm/amd/display/dc/inc/hw/timing_generator.h | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index da808996e21d1..9c3704c4d7e4e 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -1055,6 +1055,7 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context) struct dc_state *dangling_context = dc_create_state(dc); struct dc_state *current_ctx; struct pipe_ctx *pipe; + struct timing_generator *tg; if (dangling_context == NULL) return; @@ -1098,6 +1099,7 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context) if (should_disable && old_stream) { pipe = &dc->current_state->res_ctx.pipe_ctx[i]; + tg = pipe->stream_res.tg; /* When disabling plane for a phantom pipe, we must turn on the * phantom OTG so the disable programming gets the double buffer * update. Otherwise the pipe will be left in a partially disabled @@ -1105,7 +1107,8 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context) * again for different use. */ if (old_stream->mall_stream_config.type == SUBVP_PHANTOM) { - pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg); + if (tg->funcs->enable_crtc) + tg->funcs->enable_crtc(tg); } dc_rem_all_planes_for_stream(dc, old_stream, dangling_context); disable_all_writeback_pipes_for_stream(dc, old_stream, dangling_context); @@ -1122,6 +1125,15 @@ static void disable_dangling_plane(struct dc *dc, struct dc_state *context) dc->hwss.interdependent_update_lock(dc, dc->current_state, false); dc->hwss.post_unlock_program_front_end(dc, dangling_context); } + /* We need to put the phantom OTG back into it's default (disabled) state or we + * can get corruption when transition from one SubVP config to a different one. + * The OTG is set to disable on falling edge of VUPDATE so the plane disable + * will still get it's double buffer update. + */ + if (old_stream->mall_stream_config.type == SUBVP_PHANTOM) { + if (tg->funcs->disable_phantom_crtc) + tg->funcs->disable_phantom_crtc(tg); + } } } diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c index 2b33eeb213e2a..2ee798965bc2b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c @@ -167,6 +167,13 @@ static void optc32_phantom_crtc_post_enable(struct timing_generator *optc) REG_WAIT(OTG_CLOCK_CONTROL, OTG_BUSY, 0, 1, 100000); } +static void optc32_disable_phantom_otg(struct timing_generator *optc) +{ + struct optc *optc1 = DCN10TG_FROM_TG(optc); + + REG_UPDATE(OTG_CONTROL, OTG_MASTER_EN, 0); +} + static void optc32_set_odm_bypass(struct timing_generator *optc, const struct dc_crtc_timing *dc_crtc_timing) { @@ -260,6 +267,7 @@ static struct timing_generator_funcs dcn32_tg_funcs = { .enable_crtc = optc32_enable_crtc, .disable_crtc = optc32_disable_crtc, .phantom_crtc_post_enable = optc32_phantom_crtc_post_enable, + .disable_phantom_crtc = optc32_disable_phantom_otg, /* used by enable_timing_synchronization. Not need for FPGA */ .is_counter_moving = optc1_is_counter_moving, .get_position = optc1_get_position, diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h index 65f18f9dad346..43eb61961e0fd 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h +++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h @@ -184,6 +184,7 @@ struct timing_generator_funcs { bool (*disable_crtc)(struct timing_generator *tg); #ifdef CONFIG_DRM_AMD_DC_DCN void (*phantom_crtc_post_enable)(struct timing_generator *tg); + void (*disable_phantom_crtc)(struct timing_generator *tg); #endif bool (*immediate_disable_crtc)(struct timing_generator *tg); bool (*is_counter_moving)(struct timing_generator *tg); -- GitLab From de020e5fa9ebc6fc32e82ae6ccb0282451ed937c Mon Sep 17 00:00:00 2001 From: Wesley Chalmers Date: Fri, 21 Oct 2022 19:06:10 -0400 Subject: [PATCH 1476/1927] drm/amd/display: Disable DRR actions during state commit [WHY] Committing a state while performing DRR actions can cause underflow. [HOW] Disabled features performing DRR actions during state commit. Need to follow-up on why DRR actions affect state commit. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Wesley Chalmers Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c index 8c50457112649..c20e9f76f0213 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c @@ -992,8 +992,5 @@ void dcn30_prepare_bandwidth(struct dc *dc, dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz); dcn20_prepare_bandwidth(dc, context); - - dc_dmub_srv_p_state_delegate(dc, - context->bw_ctx.bw.dcn.clk.fw_based_mclk_switching, context); } -- GitLab From be9f6b222c529f315a77c0a182a47c988c28f7f1 Mon Sep 17 00:00:00 2001 From: Mustapha Ghaddar Date: Wed, 26 Oct 2022 13:54:27 -0400 Subject: [PATCH 1477/1927] drm/amd/display: Fix fallback issues for DP LL 1.4a tests [WHY] Unlike DP or USBC, the USB4 link does not get its own encoder and has to share therefore verify_caps is skipped. [HOW] Fix the fallback logic for automated tests and take that into consideration for LT and LS. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Mustapha Ghaddar Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/core/dc_link_dp.c | 22 ++++++++++++++++--- .../drm/amd/display/dc/core/dc_link_dpia.c | 15 ++++++++----- drivers/gpu/drm/amd/display/dc/dc_link.h | 1 + 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index b0cfe21871e4d..a3e1aeccd3f5a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -4554,9 +4554,19 @@ void dc_link_dp_handle_link_loss(struct dc_link *link) for (i = 0; i < MAX_PIPES; i++) { pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i]; - if (pipe_ctx && pipe_ctx->stream && !pipe_ctx->stream->dpms_off && - pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) + if (pipe_ctx && pipe_ctx->stream && !pipe_ctx->stream->dpms_off + && pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) { + // Always use max settings here for DP 1.4a LL Compliance CTS + if (link->is_automated) { + pipe_ctx->link_config.dp_link_settings.lane_count = + link->verified_link_cap.lane_count; + pipe_ctx->link_config.dp_link_settings.link_rate = + link->verified_link_cap.link_rate; + pipe_ctx->link_config.dp_link_settings.link_spread = + link->verified_link_cap.link_spread; + } core_link_enable_stream(link->dc->current_state, pipe_ctx); + } } } @@ -4597,6 +4607,8 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link *link, union hpd_irq_data *out_hpd } if (hpd_irq_dpcd_data.bytes.device_service_irq.bits.AUTOMATED_TEST) { + // Workaround for DP 1.4a LL Compliance CTS as USB4 has to share encoders unlike DP and USBC + link->is_automated = true; device_service_clear.bits.AUTOMATED_TEST = 1; core_link_write_dpcd( link, @@ -7240,6 +7252,7 @@ void dp_retrain_link_dp_test(struct dc_link *link, struct pipe_ctx *pipes = &link->dc->current_state->res_ctx.pipe_ctx[0]; unsigned int i; + bool do_fallback = false; for (i = 0; i < MAX_PIPES; i++) { @@ -7272,13 +7285,16 @@ void dp_retrain_link_dp_test(struct dc_link *link, memset(&link->cur_link_settings, 0, sizeof(link->cur_link_settings)); + if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA) + do_fallback = true; + perform_link_training_with_retries( link_setting, skip_video_pattern, LINK_TRAINING_ATTEMPTS, &pipes[i], SIGNAL_TYPE_DISPLAY_PORT, - false); + do_fallback); link->dc->hwss.enable_stream(&pipes[i]); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c index 74e36b34d3f73..d130d58ac08e7 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c @@ -791,10 +791,14 @@ static enum link_training_result dpia_training_eq_transparent( } if (dp_is_ch_eq_done(lane_count, dpcd_lane_status) && - dp_is_symbol_locked(link->cur_link_settings.lane_count, dpcd_lane_status) && - dp_is_interlane_aligned(dpcd_lane_status_updated)) { - result = LINK_TRAINING_SUCCESS; - break; + dp_is_symbol_locked(link->cur_link_settings.lane_count, dpcd_lane_status)) { + /* Take into consideration corner case for DP 1.4a LL Compliance CTS as USB4 + * has to share encoders unlike DP and USBC + */ + if (dp_is_interlane_aligned(dpcd_lane_status_updated) || (link->is_automated && retries_eq)) { + result = LINK_TRAINING_SUCCESS; + break; + } } /* Update VS/PE. */ @@ -1008,7 +1012,8 @@ enum link_training_result dc_link_dpia_perform_link_training( */ if (result == LINK_TRAINING_SUCCESS) { msleep(5); - result = dp_check_link_loss_status(link, <_settings); + if (!link->is_automated) + result = dp_check_link_loss_status(link, <_settings); } else if (result == LINK_TRAINING_ABORT) { dpia_training_abort(link, <_settings, repeater_id); } else { diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h index edb4532eaa39e..dc6afe33bca28 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_link.h +++ b/drivers/gpu/drm/amd/display/dc/dc_link.h @@ -184,6 +184,7 @@ struct dc_link { bool is_dig_mapping_flexible; bool hpd_status; /* HPD status of link without physical HPD pin. */ bool is_hpd_pending; /* Indicates a new received hpd */ + bool is_automated; /* Indicates automated testing */ bool edp_sink_present; -- GitLab From 2f8f9118129dd4603cb425404f1df1f5013b6019 Mon Sep 17 00:00:00 2001 From: Chaitanya Dhere Date: Thu, 27 Oct 2022 09:51:19 -0400 Subject: [PATCH 1478/1927] drm/amd/display: Fix FCLK deviation and tool compile issues [Why] Recent backports from open source do not have header inclusion pattern that is consistent with inclusion style in the rest of the file. This breaks the internal tool builds as well. A recent commit erronously modified the original DML formula for calculating ActiveClockChangeLatencyHidingY. This resulted in a FCLK deviation from the golden values. [How] Change the way in which display_mode_vba.h is included so that it is consistent with the inclusion style in rest of the file which also fixes the tool build. Restore the DML formula to its original state to fix the FCLK deviation. Reviewed-by: Aurabindo Pillai Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Chaitanya Dhere Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c | 2 +- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index 968924c491c18..ab9217732a17f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -4397,7 +4397,7 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( if (v->NumberOfActiveSurfaces > 1) { ActiveClockChangeLatencyHidingY = ActiveClockChangeLatencyHidingY - - (1 - 1 / v->NumberOfActiveSurfaces) * SwathHeightY[k] * v->HTotal[k] + - (1.0 - 1.0 / v->NumberOfActiveSurfaces) * SwathHeightY[k] * v->HTotal[k] / v->PixelClock[k] / v->VRatio[k]; } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index 2c3827546ac77..fdccaa93eb2ef 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -30,7 +30,7 @@ #include "os_types.h" #include "../dc_features.h" #include "../display_mode_structs.h" -#include "dml/display_mode_vba.h" +#include "../display_mode_vba.h" unsigned int dml32_dscceComputeDelay( unsigned int bpc, -- GitLab From 98f9229ff60e39d3d7babd42a13358bb5a392e4b Mon Sep 17 00:00:00 2001 From: Nicholas Kazlauskas Date: Thu, 27 Oct 2022 15:34:33 -0400 Subject: [PATCH 1479/1927] drm/amd/display: Fix reg timeout in enc314_enable_fifo [Why] The link enablement sequence can end up resetting the encoder while the PHY symclk isn't yet on. This means that waiting for symclk on will timeout, along with the reset bit never asserting high. This causes unnecessary delay when enabling the link and produces a warning affecting multiple IGT tests. [How] Don't wait for the symclk to be on here because firmware already does. Don't wait for reset if we know the symclk isn't on. Split the reset into a helper function that checks the bit and decides whether or not a delay is sufficient. Reviewed-by: Roman Li Acked-by: Alan Liu Signed-off-by: Nicholas Kazlauskas Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../dc/dcn314/dcn314_dio_stream_encoder.c | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c index 7e773bf7b895f..38842f938bed0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c @@ -49,18 +49,30 @@ #define CTX \ enc1->base.ctx +static void enc314_reset_fifo(struct stream_encoder *enc, bool reset) +{ + struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); + uint32_t reset_val = reset ? 1 : 0; + uint32_t is_symclk_on; + + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, reset_val); + REG_GET(DIG_FE_CNTL, DIG_SYMCLK_FE_ON, &is_symclk_on); + + if (is_symclk_on) + REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, reset_val, 10, 5000); + else + udelay(10); +} static void enc314_enable_fifo(struct stream_encoder *enc) { struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); - /* TODO: Confirm if we need to wait for DIG_SYMCLK_FE_ON */ - REG_WAIT(DIG_FE_CNTL, DIG_SYMCLK_FE_ON, 1, 10, 5000); REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, 0x7); - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1); - REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 1, 10, 5000); - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 0); - REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 0, 10, 5000); + + enc314_reset_fifo(enc, true); + enc314_reset_fifo(enc, false); + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, 1); } -- GitLab From 1df6b9b07e334cee60afebcaca69297477655325 Mon Sep 17 00:00:00 2001 From: Steve Su Date: Fri, 28 Oct 2022 05:52:32 +0800 Subject: [PATCH 1480/1927] drm/amd/display: Fix gpio port mapping issue [Why] 1. Port of gpio has different mapping. [How] 1. Add a dummy entry in mapping table. 2. Fix incorrect mask bit field access. Reviewed-by: Alvin Lee Acked-by: Alan Liu Signed-off-by: Steve Su Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../amd/display/dc/gpio/dcn32/hw_factory_dcn32.c | 14 ++++++++++++++ drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c | 9 ++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c index d635b73af46fe..0ea52ba5ac827 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c @@ -107,6 +107,13 @@ static const struct ddc_registers ddc_data_regs_dcn[] = { ddc_data_regs_dcn2(3), ddc_data_regs_dcn2(4), ddc_data_regs_dcn2(5), + { + // add a dummy entry for cases no such port + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}, + .ddc_setup = 0, + .phy_aux_cntl = 0, + .dc_gpio_aux_ctrl_5 = 0 + }, { DDC_GPIO_VGA_REG_LIST(DATA), .ddc_setup = 0, @@ -121,6 +128,13 @@ static const struct ddc_registers ddc_clk_regs_dcn[] = { ddc_clk_regs_dcn2(3), ddc_clk_regs_dcn2(4), ddc_clk_regs_dcn2(5), + { + // add a dummy entry for cases no such port + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}, + .ddc_setup = 0, + .phy_aux_cntl = 0, + .dc_gpio_aux_ctrl_5 = 0 + }, { DDC_GPIO_VGA_REG_LIST(CLK), .ddc_setup = 0, diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c index 6fd38cdd68c0c..525bc8881950d 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c @@ -94,11 +94,14 @@ static enum gpio_result set_config( * is required for detection of AUX mode */ if (hw_gpio->base.en != GPIO_DDC_LINE_VIP_PAD) { if (!ddc_data_pd_en || !ddc_clk_pd_en) { - - REG_SET_2(gpio.MASK_reg, regval, + if (hw_gpio->base.en == GPIO_DDC_LINE_DDC_VGA) { + // bit 4 of mask has different usage in some cases + REG_SET(gpio.MASK_reg, regval, DC_GPIO_DDC1DATA_PD_EN, 1); + } else { + REG_SET_2(gpio.MASK_reg, regval, DC_GPIO_DDC1DATA_PD_EN, 1, DC_GPIO_DDC1CLK_PD_EN, 1); - + } if (config_data->type == GPIO_CONFIG_TYPE_I2C_AUX_DUAL_MODE) msleep(3); -- GitLab From ea192af507d99014775a757bbdb5f6e487fc6c7a Mon Sep 17 00:00:00 2001 From: Michael Strauss Date: Thu, 20 Oct 2022 15:57:36 -0400 Subject: [PATCH 1481/1927] drm/amd/display: Only update link settings after successful MST link train [WHY] Currently driver reduces verified link caps on DPIA devices if a link is trained at a link rate below the max rate verified during link detection. This blocks high bandwidth modes after setting a low bandwidth mode. [HOW] Only update link rate after a successful link train if link is MST. Reviewed-by: Mustapha Ghaddar Acked-by: Alan Liu Signed-off-by: Michael Strauss Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c | 8 ++++++++ drivers/gpu/drm/amd/display/dc/core/dc_link.c | 4 ++++ drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c | 7 +++++-- drivers/gpu/drm/amd/display/dc/dm_helpers.h | 5 +++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index f72c013d3a5b0..e47098fa5aac1 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -971,3 +971,11 @@ void dm_helpers_enable_periodic_detection(struct dc_context *ctx, bool enable) { /* TODO: add periodic detection implementation */ } + +void dm_helpers_dp_mst_update_branch_bandwidth( + struct dc_context *ctx, + struct dc_link *link) +{ + // TODO +} + diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index 24ed057414e1e..5304e9daf90a3 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -4663,6 +4663,10 @@ void dc_link_set_preferred_training_settings(struct dc *dc, link->preferred_link_setting.link_rate = LINK_RATE_UNKNOWN; } + if (link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT && + link->type == dc_connection_mst_branch) + dm_helpers_dp_mst_update_branch_bandwidth(dc->ctx, link); + /* Retrain now, or wait until next stream update to apply */ if (skip_immediate_retrain == false) dc_link_set_preferred_link_settings(dc, &link->preferred_link_setting, link); diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c index a3e1aeccd3f5a..b772d7f2301cf 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c @@ -2771,8 +2771,11 @@ bool perform_link_training_with_retries( /* Update verified link settings to current one * Because DPIA LT might fallback to lower link setting. */ - link->verified_link_cap.link_rate = link->cur_link_settings.link_rate; - link->verified_link_cap.lane_count = link->cur_link_settings.lane_count; + if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) { + link->verified_link_cap.link_rate = link->cur_link_settings.link_rate; + link->verified_link_cap.lane_count = link->cur_link_settings.lane_count; + dm_helpers_dp_mst_update_branch_bandwidth(link->ctx, link); + } } } else { status = dc_link_dp_perform_link_training(link, diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h index e3e5c39895a3a..af1c50ed905ab 100644 --- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h +++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h @@ -116,6 +116,11 @@ bool dm_helpers_dp_mst_start_top_mgr( bool dm_helpers_dp_mst_stop_top_mgr( struct dc_context *ctx, struct dc_link *link); + +void dm_helpers_dp_mst_update_branch_bandwidth( + struct dc_context *ctx, + struct dc_link *link); + /** * OS specific aux read callback. */ -- GitLab From 01c0c124b9ecaa905468c6f3b3bf3962b276008b Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Thu, 27 Oct 2022 16:22:26 -0400 Subject: [PATCH 1482/1927] drm/amd/display: Enforce minimum prefetch time for low memclk on DCN32 [WHY?] Data return times when using lowest memclk can be <= 60us, which can cause underflow on high bandwidth displays with a workload. [HOW?] Enforce a minimum prefetch time during validation for low memclk modes. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Dillon Varone Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 + .../gpu/drm/amd/display/dc/dcn32/dcn32_resource.c | 1 + .../gpu/drm/amd/display/dc/dcn321/dcn321_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 2 ++ .../amd/display/dc/dml/dcn32/display_mode_vba_32.c | 4 ++++ .../amd/display/dc/dml/dcn32/display_mode_vba_32.h | 3 +++ .../display/dc/dml/dcn32/display_mode_vba_util_32.c | 12 ++++++++++-- .../display/dc/dml/dcn32/display_mode_vba_util_32.h | 1 + .../gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c | 2 ++ .../drm/amd/display/dc/dml/display_mode_structs.h | 1 + 10 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 84c82d3a6761d..d691218095243 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -864,6 +864,7 @@ struct dc_debug_options { bool enable_dp_dig_pixel_rate_div_policy; enum lttpr_mode lttpr_mode_override; unsigned int dsc_delay_factor_wa_x1000; + unsigned int min_prefetch_in_strobe_ns; }; struct gpu_info_soc_bounding_box_v1_0; diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index 4ba9a86621854..4bd861427b3ca 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -724,6 +724,7 @@ static const struct dc_debug_options debug_defaults_drv = { .enable_dp_dig_pixel_rate_div_policy = 1, .allow_sw_cursor_fallback = false, .alloc_extra_way_for_cursor = true, + .min_prefetch_in_strobe_ns = 60000, // 60us }; static const struct dc_debug_options debug_defaults_diags = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c index 61087f2385a96..6292ac515d1a4 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c @@ -722,6 +722,7 @@ static const struct dc_debug_options debug_defaults_drv = { .enable_dp_dig_pixel_rate_div_policy = 1, .allow_sw_cursor_fallback = false, .alloc_extra_way_for_cursor = true, + .min_prefetch_in_strobe_ns = 60000, // 60us }; static const struct dc_debug_options debug_defaults_diags = { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index 0d704e302d035..853ffb704985f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -2351,6 +2351,8 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa /* DML DSC delay factor workaround */ dcn3_2_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + dcn3_2_ip.min_prefetch_in_strobe_us = dc->debug.min_prefetch_in_strobe_ns / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_2_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index ae6e6abc620b1..244fd15d24b43 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -786,6 +786,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->SwathHeightY[k], v->SwathHeightC[k], TWait, + v->DRAMSpeedPerState[mode_lib->vba.VoltageLevel] <= MEM_STROBE_FREQ_MHZ ? + mode_lib->vba.ip.min_prefetch_in_strobe_us : 0, /* Output */ &v->DSTXAfterScaler[k], &v->DSTYAfterScaler[k], @@ -3245,6 +3247,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->swath_width_chroma_ub_this_state[k], v->SwathHeightYThisState[k], v->SwathHeightCThisState[k], v->TWait, + v->DRAMSpeedPerState[i] <= MEM_STROBE_FREQ_MHZ ? + mode_lib->vba.ip.min_prefetch_in_strobe_us : 0, /* Output */ &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.DSTXAfterScaler[k], diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h index c62e0991358b3..f82e14cd9d8aa 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h @@ -49,6 +49,9 @@ #define BPP_INVALID 0 #define BPP_BLENDED_PIPE 0xffffffff +#define MEM_STROBE_FREQ_MHZ 1600 +#define MEM_STROBE_MAX_DELIVERY_TIME_US 60.0 + struct display_mode_lib; void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib); diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index ab9217732a17f..635fc54338fa2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -3417,6 +3417,7 @@ bool dml32_CalculatePrefetchSchedule( unsigned int SwathHeightY, unsigned int SwathHeightC, double TWait, + double TPreReq, /* Output */ double *DSTXAfterScaler, double *DSTYAfterScaler, @@ -3474,6 +3475,7 @@ bool dml32_CalculatePrefetchSchedule( double min_Lsw; double Tsw_est1 = 0; double Tsw_est3 = 0; + double TPreMargin = 0; if (v->GPUVMEnable == true && v->HostVMEnable == true) HostVMDynamicLevelsTrips = v->HostVMMaxNonCachedPageTableLevels; @@ -3699,6 +3701,8 @@ bool dml32_CalculatePrefetchSchedule( dst_y_prefetch_equ = dml_floor(4.0 * (dst_y_prefetch_equ + 0.125), 1) / 4.0; Tpre_rounded = dst_y_prefetch_equ * LineTime; + + TPreMargin = Tpre_rounded - TPreReq; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: dst_y_prefetch_equ: %f (after round)\n", __func__, dst_y_prefetch_equ); dml_print("DML::%s: LineTime: %f\n", __func__, LineTime); @@ -3726,7 +3730,7 @@ bool dml32_CalculatePrefetchSchedule( *VRatioPrefetchY = 0; *VRatioPrefetchC = 0; *RequiredPrefetchPixDataBWLuma = 0; - if (dst_y_prefetch_equ > 1) { + if (dst_y_prefetch_equ > 1 && TPreMargin > 0.0) { double PrefetchBandwidth1; double PrefetchBandwidth2; double PrefetchBandwidth3; @@ -3872,7 +3876,11 @@ bool dml32_CalculatePrefetchSchedule( } if (dst_y_prefetch_oto < dst_y_prefetch_equ) { - *DestinationLinesForPrefetch = dst_y_prefetch_oto; + if (dst_y_prefetch_oto * LineTime < TPreReq) { + *DestinationLinesForPrefetch = dst_y_prefetch_equ; + } else { + *DestinationLinesForPrefetch = dst_y_prefetch_oto; + } TimeForFetchingMetaPTE = Tvm_oto; TimeForFetchingRowInVBlank = Tr0_oto; *PrefetchBandwidth = prefetch_bw_oto; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index fdccaa93eb2ef..3989c2a28faec 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -743,6 +743,7 @@ bool dml32_CalculatePrefetchSchedule( unsigned int SwathHeightY, unsigned int SwathHeightC, double TWait, + double TPreReq, /* Output */ double *DSTXAfterScaler, double *DSTYAfterScaler, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index ec0486efab147..432b4ecd01a71 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -544,6 +544,8 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p /* DML DSC delay factor workaround */ dcn3_21_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + dcn3_21_ip.min_prefetch_in_strobe_us = dc->debug.min_prefetch_in_strobe_ns / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_21_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index d7be01ac07514..64d602e6412f1 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -367,6 +367,7 @@ struct _vcs_dpi_ip_params_st { /* DM workarounds */ double dsc_delay_factor_wa; // TODO: Remove after implementing root cause fix + double min_prefetch_in_strobe_us; }; struct _vcs_dpi_display_xfc_params_st { -- GitLab From 52b5432cd70a2d6d998cb9b48378c37d5bb31e12 Mon Sep 17 00:00:00 2001 From: Mike Hsieh Date: Tue, 4 Oct 2022 14:58:07 +0800 Subject: [PATCH 1483/1927] drm/amd/display: Set correct EOTF and Gamut flag in VRR info [Why] FreeSync always use G2.2 EOTF and Native gamut [How] Set EOTF and Gamut flags accordingly Reviewed-by: Krunoslav Kovac Acked-by: Alan Liu Signed-off-by: Mike Hsieh Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/modules/freesync/freesync.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c index 0f39ab9dc5b41..c2e00f7b8381e 100644 --- a/drivers/gpu/drm/amd/display/modules/freesync/freesync.c +++ b/drivers/gpu/drm/amd/display/modules/freesync/freesync.c @@ -688,10 +688,10 @@ static void build_vrr_infopacket_fs2_data(enum color_transfer_func app_tf, if (app_tf != TRANSFER_FUNC_UNKNOWN) { infopacket->valid = true; - infopacket->sb[6] |= 0x08; // PB6 = [Bit 3 = Native Color Active] - - if (app_tf == TRANSFER_FUNC_GAMMA_22) { - infopacket->sb[9] |= 0x04; // PB6 = [Bit 2 = Gamma 2.2 EOTF Active] + if (app_tf != TRANSFER_FUNC_PQ2084) { + infopacket->sb[6] |= 0x08; // PB6 = [Bit 3 = Native Color Active] + if (app_tf == TRANSFER_FUNC_GAMMA_22) + infopacket->sb[9] |= 0x04; // PB6 = [Bit 2 = Gamma 2.2 EOTF Active] } } } -- GitLab From 964d641632fa0c8eb52d577c23c1b5b5525a9bc7 Mon Sep 17 00:00:00 2001 From: Alvin Lee Date: Sun, 30 Oct 2022 15:56:53 -0400 Subject: [PATCH 1484/1927] drm/amd/display: Add margin for max vblank time for SubVP + DRR [Description] - Incorporate FW delays as port of max VTOTAL calculated for SubVP + DRR cases (since it is part of the microschedule). - Also add margin for the max VTOTAL possible for SubVP + DRR cases. - Due to rounding errors in FW (integer arithmetic), the microschedule calculation can get pushed to the next frame (incorrectly) in cases where we use the max VTOTAL possible to complete the MCLK switch. - When the rounding error occurs, we are only off by 1-2 lines, use 40us margin which is working consistently. Reviewed-by: Jun Lei Reviewed-by: Aric Cyr Acked-by: Alan Liu Signed-off-by: Alvin Lee Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 + drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c | 12 ++++++++++-- .../gpu/drm/amd/display/dc/dcn32/dcn32_resource.c | 1 + .../gpu/drm/amd/display/dc/dcn321/dcn321_resource.c | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index d691218095243..1ec1b441d5cba 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -261,6 +261,7 @@ struct dc_caps { uint32_t cache_line_size; uint32_t cache_num_ways; uint16_t subvp_fw_processing_delay_us; + uint8_t subvp_drr_max_vblank_margin_us; uint16_t subvp_prefetch_end_to_mall_start_us; uint8_t subvp_swath_height_margin_lines; // subvp start line must be aligned to 2 x swath height uint16_t subvp_pstate_allow_width_us; diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index 4cb912bf400b1..097556f7b32cf 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -477,12 +477,20 @@ static void populate_subvp_cmd_drr_info(struct dc *dc, (((uint64_t)main_timing->pix_clk_100hz * 100))); drr_active_us = div64_u64(((uint64_t)drr_timing->v_addressable * drr_timing->h_total * 1000000), (((uint64_t)drr_timing->pix_clk_100hz * 100))); - max_drr_vblank_us = div64_u64((subvp_active_us - prefetch_us - drr_active_us), 2) + drr_active_us; - max_drr_mallregion_us = subvp_active_us - prefetch_us - mall_region_us; + max_drr_vblank_us = div64_u64((subvp_active_us - prefetch_us - + dc->caps.subvp_fw_processing_delay_us - drr_active_us), 2) + drr_active_us; + max_drr_mallregion_us = subvp_active_us - prefetch_us - mall_region_us - dc->caps.subvp_fw_processing_delay_us; max_drr_supported_us = max_drr_vblank_us > max_drr_mallregion_us ? max_drr_vblank_us : max_drr_mallregion_us; max_vtotal_supported = div64_u64(((uint64_t)drr_timing->pix_clk_100hz * 100 * max_drr_supported_us), (((uint64_t)drr_timing->h_total * 1000000))); + /* When calculating the max vtotal supported for SubVP + DRR cases, add + * margin due to possible rounding errors (being off by 1 line in the + * FW calculation can incorrectly push the P-State switch to wait 1 frame + * longer). + */ + max_vtotal_supported = max_vtotal_supported - dc->caps.subvp_drr_max_vblank_margin_us; + pipe_data->pipe_config.vblank_data.drr_info.min_vtotal_supported = min_vtotal_supported; pipe_data->pipe_config.vblank_data.drr_info.max_vtotal_supported = max_vtotal_supported; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index 4bd861427b3ca..77e40ee488bd6 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -2117,6 +2117,7 @@ static bool dcn32_resource_construct( dc->caps.cache_num_ways = 16; dc->caps.max_cab_allocation_bytes = 67108864; // 64MB = 1024 * 1024 * 64 dc->caps.subvp_fw_processing_delay_us = 15; + dc->caps.subvp_drr_max_vblank_margin_us = 40; dc->caps.subvp_prefetch_end_to_mall_start_us = 15; dc->caps.subvp_swath_height_margin_lines = 16; dc->caps.subvp_pstate_allow_width_us = 20; diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c index 6292ac515d1a4..e5861225f1df9 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c @@ -1704,6 +1704,7 @@ static bool dcn321_resource_construct( dc->caps.cache_num_ways = 16; dc->caps.max_cab_allocation_bytes = 33554432; // 32MB = 1024 * 1024 * 32 dc->caps.subvp_fw_processing_delay_us = 15; + dc->caps.subvp_drr_max_vblank_margin_us = 40; dc->caps.subvp_prefetch_end_to_mall_start_us = 15; dc->caps.subvp_swath_height_margin_lines = 16; dc->caps.subvp_pstate_allow_width_us = 20; -- GitLab From b7ada7ee61d3eb0ebf08089edcefdc272b5f7b1e Mon Sep 17 00:00:00 2001 From: George Shen Date: Thu, 14 Apr 2022 21:29:38 -0400 Subject: [PATCH 1485/1927] drm/amd/display: Populate DP2.0 output type for DML pipe [Why] DCN3.2 DML logic uses a new output type for DP2.0, which will enable validation to pass for higher BW timings that require DP2.0 link rates. [How] Populate the DML pipe with DP2.0 output type if the signal type of the pipe_ctx is 128b/132b. Reviewed-by: Alvin Lee Acked-by: Jasdeep Dhillon Signed-off-by: George Shen Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c index 602e885ed52c4..75dbb7ee193bc 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c @@ -1296,6 +1296,8 @@ int dcn20_populate_dml_pipes_from_context( case SIGNAL_TYPE_DISPLAY_PORT_MST: case SIGNAL_TYPE_DISPLAY_PORT: pipes[pipe_cnt].dout.output_type = dm_dp; + if (is_dp_128b_132b_signal(&res_ctx->pipe_ctx[i])) + pipes[pipe_cnt].dout.output_type = dm_dp2p0; break; case SIGNAL_TYPE_EDP: pipes[pipe_cnt].dout.output_type = dm_edp; -- GitLab From adf65dff5d61d3adad669a1f5c6dd009107eacd7 Mon Sep 17 00:00:00 2001 From: Rajneesh Bhardwaj Date: Wed, 20 Apr 2022 12:28:20 -0400 Subject: [PATCH 1486/1927] drm/amdgpu: Fix the kerneldoc description amdgpu_ttm_tt_set_userptr() is also called by the KFD as part of initializing the user pages for userptr BOs and also while initializing the GPUVM for a KFD process so update the function description. Reviewed-by: Felix Kuehling Signed-off-by: Rajneesh Bhardwaj Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 8c00a7a06c32d..c9ceea46711ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1177,8 +1177,9 @@ int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo, * @addr: The address in the current tasks VM space to use * @flags: Requirements of userptr object. * - * Called by amdgpu_gem_userptr_ioctl() to bind userptr pages - * to current task + * Called by amdgpu_gem_userptr_ioctl() and kfd_ioctl_alloc_memory_of_gpu() to + * bind userptr pages to current task and by kfd_ioctl_acquire_vm() to + * initialize GPU VM for a KFD process. */ int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo, uint64_t addr, uint32_t flags) -- GitLab From f936f535fa70f35ce3369b1418ebae0e657cda6a Mon Sep 17 00:00:00 2001 From: Asher Song Date: Thu, 3 Nov 2022 18:28:40 +0800 Subject: [PATCH 1487/1927] Revert "drm/amdgpu: Revert "drm/amdgpu: getting fan speed pwm for vega10 properly"" This reverts commit 4545ae2ed3f2f7c3f615a53399c9c8460ee5bca7. The origin patch "drm/amdgpu: getting fan speed pwm for vega10 properly" works fine. Test failure is caused by test case self. Signed-off-by: Asher Song Reviewed-by: Guchun Chen Signed-off-by: Alex Deucher --- .../amd/pm/powerplay/hwmgr/vega10_thermal.c | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c index dad3e3741a4e8..190af79f3236f 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c @@ -67,22 +67,21 @@ int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { - uint32_t current_rpm; - uint32_t percent = 0; - - if (hwmgr->thermal_controller.fanInfo.bNoFan) - return 0; + struct amdgpu_device *adev = hwmgr->adev; + uint32_t duty100, duty; + uint64_t tmp64; - if (vega10_get_current_rpm(hwmgr, ¤t_rpm)) - return -1; + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), + CG_FDO_CTRL1, FMAX_DUTY100); + duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS), + CG_THERMAL_STATUS, FDO_PWM_DUTY); - if (hwmgr->thermal_controller. - advanceFanControlParameters.usMaxFanRPM != 0) - percent = current_rpm * 255 / - hwmgr->thermal_controller. - advanceFanControlParameters.usMaxFanRPM; + if (!duty100) + return -EINVAL; - *speed = MIN(percent, 255); + tmp64 = (uint64_t)duty * 255; + do_div(tmp64, duty100); + *speed = MIN((uint32_t)tmp64, 255); return 0; } -- GitLab From e198e55b28202f78bca74feec9d568f4e96c6aab Mon Sep 17 00:00:00 2001 From: Aric Cyr Date: Sun, 30 Oct 2022 19:20:05 -0400 Subject: [PATCH 1488/1927] drm/amd/display: 3.2.211 DC version 3.2.211 brings along the following fixes: - Wait for VBLANK during pipe programming - Adding HDMI SCDC DEVICE_ID define - Cursor update refactor: PSR-SU support condition - Update 709 gamma to 2.222 as stated in the standerd - Consider dp cable id only when data is non zero - Waiting for 1 frame to fix the flash issue on PSR1 - Update SR watermarks for DCN314 - Allow tuning DCN314 bounding box - Zeromem mypipe heap struct before using it - Use min transition for SubVP into MPO - Disable phantom OTG after enable for plane disable - Disable DRR actions during state commit - Fix fallback issues for DP LL 1.4a tests - Fix FCLK deviation and tool compile issues - Fix reg timeout in enc314_enable_fifo - Fix gpio port mapping issue - Only update link settings after successful MST link train - Enforce minimum prefetch time for low memclk on DCN32 - Set correct EOTF and Gamut flag in VRR info - Add margin for max vblank time for SubVP + DRR - Populate DP2.0 output type for DML pipe Acked-by: Alan Liu Reviewed-by: Alan Liu Signed-off-by: Aric Cyr Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 1ec1b441d5cba..caed5597d1dc1 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -47,7 +47,7 @@ struct aux_payload; struct set_config_cmd_payload; struct dmub_notification; -#define DC_VER "3.2.210" +#define DC_VER "3.2.211" #define MAX_SURFACES 3 #define MAX_PLANES 6 -- GitLab From c70e2166967dc4ecb5e3a9b162741a9726755d9b Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 3 Nov 2022 10:24:52 -0400 Subject: [PATCH 1489/1927] drm/amdgpu: Unlock bo_list_mutex after error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get below kernel WARNING backtrace when pressing ctrl-C to kill kfdtest application. If amdgpu_cs_parser_bos returns error after taking bo_list_mutex, as caller amdgpu_cs_ioctl will not unlock bo_list_mutex, this generates the kernel WARNING. Add unlock bo_list_mutex after amdgpu_cs_parser_bos error handling to cleanup bo_list userptr bo. WARNING: kfdtest/2930 still has locks held! 1 lock held by kfdtest/2930: (&list->bo_list_mutex){+.+.}-{3:3}, at: amdgpu_cs_ioctl+0xce5/0x1f10 [amdgpu] stack backtrace: dump_stack_lvl+0x44/0x57 get_signal+0x79f/0xd00 arch_do_signal_or_restart+0x36/0x7b0 exit_to_user_mode_prepare+0xfd/0x1b0 syscall_exit_to_user_mode+0x19/0x40 do_syscall_64+0x40/0x80 Signed-off-by: Philip Yang Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 1bbd39b3b0fc4..d371000a5727e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -992,6 +992,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, kvfree(e->user_pages); e->user_pages = NULL; } + mutex_unlock(&p->bo_list->bo_list_mutex); return r; } -- GitLab From 0bc71adc8b1b65a369d4b8ef098e74ed4aab62d9 Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 2 Nov 2022 16:55:31 -0400 Subject: [PATCH 1490/1927] drm/amdgpu: Drop eviction lock when allocating PT BO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-take the eviction lock immediately again after the allocation is completed, to fix circular locking warning with drm_buddy allocator. Move amdgpu_vm_eviction_lock/unlock/trylock to amdgpu_vm.h as they are called from multiple files. Signed-off-by: Philip Yang Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 26 ----------------------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 26 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 2 ++ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 2291aa14d888c..003aa9e47085e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -143,32 +143,6 @@ int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm, return 0; } -/* - * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS - * happens while holding this lock anywhere to prevent deadlocks when - * an MMU notifier runs in reclaim-FS context. - */ -static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) -{ - mutex_lock(&vm->eviction_lock); - vm->saved_flags = memalloc_noreclaim_save(); -} - -static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) -{ - if (mutex_trylock(&vm->eviction_lock)) { - vm->saved_flags = memalloc_noreclaim_save(); - return 1; - } - return 0; -} - -static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) -{ - memalloc_noreclaim_restore(vm->saved_flags); - mutex_unlock(&vm->eviction_lock); -} - /** * amdgpu_vm_bo_evicted - vm_bo is evicted * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 1d31771b42308..6546e786bf008 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -510,4 +510,30 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm) return atomic64_read(&vm->tlb_seq); } +/* + * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS + * happens while holding this lock anywhere to prevent deadlocks when + * an MMU notifier runs in reclaim-FS context. + */ +static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) +{ + mutex_lock(&vm->eviction_lock); + vm->saved_flags = memalloc_noreclaim_save(); +} + +static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) +{ + if (mutex_trylock(&vm->eviction_lock)) { + vm->saved_flags = memalloc_noreclaim_save(); + return true; + } + return false; +} + +static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) +{ + memalloc_noreclaim_restore(vm->saved_flags); + mutex_unlock(&vm->eviction_lock); +} + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index 358b91243e37b..b5f3bba851db8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -597,7 +597,9 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, if (entry->bo) return 0; + amdgpu_vm_eviction_unlock(vm); r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt); + amdgpu_vm_eviction_lock(vm); if (r) return r; -- GitLab From 407a5bdd5513a3617aa3f8a5c738561e157f8d83 Mon Sep 17 00:00:00 2001 From: Guchun Chen Date: Mon, 7 Nov 2022 16:46:59 +0800 Subject: [PATCH 1491/1927] drm/amdgpu: disable BACO on special BEIGE_GOBY card Still avoid intermittent failure. Signed-off-by: Guchun Chen Reviewed-by: Lijo Lazar Acked-by: Evan Quan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index 74996a8fb6712..6212fd270857c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -377,7 +377,9 @@ static void sienna_cichlid_check_bxco_support(struct smu_context *smu) if (((adev->pdev->device == 0x73A1) && (adev->pdev->revision == 0x00)) || ((adev->pdev->device == 0x73BF) && - (adev->pdev->revision == 0xCF))) + (adev->pdev->revision == 0xCF)) || + ((adev->pdev->device == 0x7422) && + (adev->pdev->revision == 0x00))) smu_baco->platform_support = false; } -- GitLab From 4cc16d64b6cdb179a26fb389cae9dce788e88f5d Mon Sep 17 00:00:00 2001 From: Ma Jun Date: Sun, 6 Nov 2022 20:34:27 +0800 Subject: [PATCH 1492/1927] drm/amdkfd: Fix the memory overrun Fix the memory overrun issue caused by wrong array size. Signed-off-by: Ma Jun Reviewed-by: Felix Kuehling Reported-by: coverity-bot Addresses-Coverity-ID: 1527133 ("Memory - corruptions") Fixes: c0cc999f3c32e6 ("drm/amdkfd: Fix the warning of array-index-out-of-bounds") Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c index 267dd69737fae..af01ba061e1bd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c @@ -1111,7 +1111,7 @@ static int kfd_parse_subtype_cache(struct crat_subtype_cache *cache, props->cache_latency = cache->cache_latency; memcpy(props->sibling_map, cache->sibling_map, - sizeof(props->sibling_map)); + CRAT_SIBLINGMAP_SIZE); /* set the sibling_map_size as 32 for CRAT from ACPI */ props->sibling_map_size = CRAT_SIBLINGMAP_SIZE; -- GitLab From 60cfad329ab877cb62975ea78ed442c2496990ba Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Tue, 8 Nov 2022 08:30:36 +0800 Subject: [PATCH 1493/1927] drm/amd/pm: enable mode1 reset on smu_v13_0_10 enable mode1 reset and prioritize debug port on smu_v13_0_10 as a more reliable message processing v2 - move mode1 reset callback to smu_v13_0_0_ppt.c Signed-off-by: Kenneth Feng Reviewed-by: Yang Wang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/soc21.c | 1 + drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h | 4 ++ .../drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c | 53 ++++++++++++++++++- drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 18 +++++++ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h | 3 ++ 5 files changed, 77 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c index 1d4013ed0d100..b258e9aa0558a 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc21.c +++ b/drivers/gpu/drm/amd/amdgpu/soc21.c @@ -322,6 +322,7 @@ soc21_asic_reset_method(struct amdgpu_device *adev) switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): case IP_VERSION(13, 0, 7): + case IP_VERSION(13, 0, 10): return AMD_RESET_METHOD_MODE1; case IP_VERSION(13, 0, 4): return AMD_RESET_METHOD_MODE2; diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h index e2fa3b066b968..1bc26e93a83c5 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h @@ -568,6 +568,10 @@ struct smu_context u32 param_reg; u32 msg_reg; u32 resp_reg; + + u32 debug_param_reg; + u32 debug_msg_reg; + u32 debug_resp_reg; }; struct i2c_adapter; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 29529328152d0..969a5e80a74c2 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -70,6 +70,26 @@ #define MP0_MP1_DATA_REGION_SIZE_COMBOPPTABLE 0x4000 +#define mmMP1_SMN_C2PMSG_66 0x0282 +#define mmMP1_SMN_C2PMSG_66_BASE_IDX 0 + +#define mmMP1_SMN_C2PMSG_82 0x0292 +#define mmMP1_SMN_C2PMSG_82_BASE_IDX 0 + +#define mmMP1_SMN_C2PMSG_90 0x029a +#define mmMP1_SMN_C2PMSG_90_BASE_IDX 0 + +#define mmMP1_SMN_C2PMSG_75 0x028b +#define mmMP1_SMN_C2PMSG_75_BASE_IDX 0 + +#define mmMP1_SMN_C2PMSG_53 0x0275 +#define mmMP1_SMN_C2PMSG_53_BASE_IDX 0 + +#define mmMP1_SMN_C2PMSG_54 0x0276 +#define mmMP1_SMN_C2PMSG_54_BASE_IDX 0 + +#define DEBUGSMC_MSG_Mode1Reset 2 + static struct cmn2asic_msg_mapping smu_v13_0_0_message_map[SMU_MSG_MAX_COUNT] = { MSG_MAP(TestMessage, PPSMC_MSG_TestMessage, 1), MSG_MAP(GetSmuVersion, PPSMC_MSG_GetSmuVersion, 1), @@ -1763,6 +1783,35 @@ static int smu_v13_0_0_set_df_cstate(struct smu_context *smu, NULL); } +static int smu_v13_0_0_mode1_reset(struct smu_context *smu) +{ + int ret; + struct amdgpu_device *adev = smu->adev; + + if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 10)) + ret = smu_cmn_send_debug_smc_msg(smu, DEBUGSMC_MSG_Mode1Reset); + else + ret = smu_cmn_send_smc_msg(smu, SMU_MSG_Mode1Reset, NULL); + + if (!ret) + msleep(SMU13_MODE1_RESET_WAIT_TIME_IN_MS); + + return ret; +} + +static void smu_v13_0_0_set_smu_mailbox_registers(struct smu_context *smu) +{ + struct amdgpu_device *adev = smu->adev; + + smu->param_reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_82); + smu->msg_reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_66); + smu->resp_reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_90); + + smu->debug_param_reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_53); + smu->debug_msg_reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_75); + smu->debug_resp_reg = SOC15_REG_OFFSET(MP1, 0, mmMP1_SMN_C2PMSG_54); +} + static const struct pptable_funcs smu_v13_0_0_ppt_funcs = { .get_allowed_feature_mask = smu_v13_0_0_get_allowed_feature_mask, .set_default_dpm_table = smu_v13_0_0_set_default_dpm_table, @@ -1830,7 +1879,7 @@ static const struct pptable_funcs smu_v13_0_0_ppt_funcs = { .baco_enter = smu_v13_0_baco_enter, .baco_exit = smu_v13_0_baco_exit, .mode1_reset_is_support = smu_v13_0_0_is_mode1_reset_supported, - .mode1_reset = smu_v13_0_mode1_reset, + .mode1_reset = smu_v13_0_0_mode1_reset, .set_mp1_state = smu_v13_0_0_set_mp1_state, .set_df_cstate = smu_v13_0_0_set_df_cstate, }; @@ -1844,5 +1893,5 @@ void smu_v13_0_0_set_ppt_funcs(struct smu_context *smu) smu->table_map = smu_v13_0_0_table_map; smu->pwr_src_map = smu_v13_0_0_pwr_src_map; smu->workload_map = smu_v13_0_0_workload_map; - smu_v13_0_set_smu_mailbox_registers(smu); + smu_v13_0_0_set_smu_mailbox_registers(smu); } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index e4f8f90ac5aa0..768b6e7dbd771 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -233,6 +233,18 @@ static void __smu_cmn_send_msg(struct smu_context *smu, WREG32(smu->msg_reg, msg); } +static int __smu_cmn_send_debug_msg(struct smu_context *smu, + u32 msg, + u32 param) +{ + struct amdgpu_device *adev = smu->adev; + + WREG32(smu->debug_param_reg, param); + WREG32(smu->debug_msg_reg, msg); + WREG32(smu->debug_resp_reg, 0); + + return 0; +} /** * smu_cmn_send_msg_without_waiting -- send the message; don't wait for status * @smu: pointer to an SMU context @@ -386,6 +398,12 @@ int smu_cmn_send_smc_msg(struct smu_context *smu, read_arg); } +int smu_cmn_send_debug_smc_msg(struct smu_context *smu, + uint32_t msg) +{ + return __smu_cmn_send_debug_msg(smu, msg, 0); +} + int smu_cmn_to_asic_specific_index(struct smu_context *smu, enum smu_cmn2asic_mapping_type type, uint32_t index) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h index 1526ce09c399b..f82cf76dd3a47 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h @@ -42,6 +42,9 @@ int smu_cmn_send_smc_msg(struct smu_context *smu, enum smu_message_type msg, uint32_t *read_arg); +int smu_cmn_send_debug_smc_msg(struct smu_context *smu, + uint32_t msg); + int smu_cmn_wait_for_response(struct smu_context *smu); int smu_cmn_to_asic_specific_index(struct smu_context *smu, -- GitLab From 43fca649e668f9f8f75a9d905de8c61bcf1d0986 Mon Sep 17 00:00:00 2001 From: Ma Jun Date: Sun, 6 Nov 2022 11:43:06 +0800 Subject: [PATCH 1494/1927] drm/amdkfd: Make kfd_fill_cache_non_crat_info() as static kfd_fill_cache_non_crat_info() is only used in kfd_topology.c, so make it as static. Signed-off-by: Ma Jun Reviewed-by: Guchun Chen Reported-by: kernel test robot Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c index 1d9b90d979c42..ef9c6fdfb88db 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c @@ -1723,7 +1723,7 @@ static int fill_in_l2_l3_pcache(struct kfd_cache_properties **props_ext, /* kfd_fill_cache_non_crat_info - Fill GPU cache info using kfd_gpu_cache_info * tables */ -void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_dev *kdev) +static void kfd_fill_cache_non_crat_info(struct kfd_topology_device *dev, struct kfd_dev *kdev) { struct kfd_gpu_cache_info *pcache_info = NULL; int i, j, k; -- GitLab From da858deab88eb561f2196bc99b6dbd2320e56456 Mon Sep 17 00:00:00 2001 From: Luben Tuikov Date: Mon, 7 Nov 2022 12:06:44 -0500 Subject: [PATCH 1495/1927] drm/amdgpu: Remove redundant I2C EEPROM address Remove redundant EEPROM_I2C_MADDR_54H address, since we already have it represented (ARCTURUS), and since we don't include the I2C device type identifier in EEPROM memory addresses, i.e. that high up in the device abstraction--we only use EEPROM memory addresses, as memory is continuously represented by EEPROM device(s) on the I2C bus. Add a comment describing what these memory addresses are, how they come about and how they're usually extracted from the device address byte. Cc: Candice Li Cc: Tao Zhou Cc: Alex Deucher Fixes: c9bdc6c3cf39df ("drm/amdgpu: Add EEPROM I2C address support for ip discovery") Signed-off-by: Luben Tuikov Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_eeprom.c | 2 ++ .../gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c | 24 ++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eeprom.c index 4d9eb0137f8c4..d6c4293829aab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eeprom.c @@ -79,6 +79,8 @@ * That is, for an I2C EEPROM driver everything is controlled by * the "eeprom_addr". * + * See also top of amdgpu_ras_eeprom.c. + * * P.S. If you need to write, lock and read the Identification Page, * (M24M02-DR device only, which we do not use), change the "7" to * "0xF" in the macro below, and let the client set bit 20 to 1 in diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 7268ae65c140c..1bb92a64f24af 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -33,12 +33,30 @@ #include "amdgpu_reset.h" +/* These are memory addresses as would be seen by one or more EEPROM + * chips strung on the I2C bus, usually by manipulating pins 1-3 of a + * set of EEPROM devices. They form a continuous memory space. + * + * The I2C device address includes the device type identifier, 1010b, + * which is a reserved value and indicates that this is an I2C EEPROM + * device. It also includes the top 3 bits of the 19 bit EEPROM memory + * address, namely bits 18, 17, and 16. This makes up the 7 bit + * address sent on the I2C bus with bit 0 being the direction bit, + * which is not represented here, and sent by the hardware directly. + * + * For instance, + * 50h = 1010000b => device type identifier 1010b, bits 18:16 = 000b, address 0. + * 54h = 1010100b => --"--, bits 18:16 = 100b, address 40000h. + * 56h = 1010110b => --"--, bits 18:16 = 110b, address 60000h. + * Depending on the size of the I2C EEPROM device(s), bits 18:16 may + * address memory in a device or a device on the I2C bus, depending on + * the status of pins 1-3. See top of amdgpu_eeprom.c. + */ #define EEPROM_I2C_MADDR_VEGA20 0x0 #define EEPROM_I2C_MADDR_ARCTURUS 0x40000 #define EEPROM_I2C_MADDR_ARCTURUS_D342 0x0 #define EEPROM_I2C_MADDR_SIENNA_CICHLID 0x0 #define EEPROM_I2C_MADDR_ALDEBARAN 0x0 -#define EEPROM_I2C_MADDR_54H (0x54UL << 16) /* * The 2 macros bellow represent the actual size in bytes that @@ -130,7 +148,7 @@ static bool __get_eeprom_i2c_addr_ip_discovery(struct amdgpu_device *adev, switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): case IP_VERSION(13, 0, 10): - control->i2c_address = EEPROM_I2C_MADDR_54H; + control->i2c_address = EEPROM_I2C_MADDR_ARCTURUS; return true; default: return false; @@ -185,7 +203,7 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): - control->i2c_address = EEPROM_I2C_MADDR_54H; + control->i2c_address = EEPROM_I2C_MADDR_ARCTURUS; break; default: -- GitLab From 3b8164f8084ff7888ed24970efa230ff5d36eda8 Mon Sep 17 00:00:00 2001 From: Luben Tuikov Date: Mon, 7 Nov 2022 12:38:03 -0500 Subject: [PATCH 1496/1927] drm/amdgpu: Decouple RAS EEPROM addresses from chips Abstract RAS I2C EEPROM addresses from chip names, and set their macro definition names to the address they set, not the chip they attach to. Since most chips either use I2C EEPROM address 0 or 40000h for the RAS table start offset, this leaves us with only two macro definitions as opposed to five, and removes the redundancy of four. Cc: Candice Li Cc: Tao Zhou Cc: Alex Deucher Signed-off-by: Luben Tuikov Reviewed-by: Kent Russell Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index 1bb92a64f24af..f63bd31e199c8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -51,12 +51,11 @@ * Depending on the size of the I2C EEPROM device(s), bits 18:16 may * address memory in a device or a device on the I2C bus, depending on * the status of pins 1-3. See top of amdgpu_eeprom.c. + * + * The RAS table lives either at address 0 or address 40000h of EEPROM. */ -#define EEPROM_I2C_MADDR_VEGA20 0x0 -#define EEPROM_I2C_MADDR_ARCTURUS 0x40000 -#define EEPROM_I2C_MADDR_ARCTURUS_D342 0x0 -#define EEPROM_I2C_MADDR_SIENNA_CICHLID 0x0 -#define EEPROM_I2C_MADDR_ALDEBARAN 0x0 +#define EEPROM_I2C_MADDR_0 0x0 +#define EEPROM_I2C_MADDR_4 0x40000 /* * The 2 macros bellow represent the actual size in bytes that @@ -135,9 +134,9 @@ static bool __get_eeprom_i2c_addr_arct(struct amdgpu_device *adev, if (strnstr(atom_ctx->vbios_version, "D342", sizeof(atom_ctx->vbios_version))) - control->i2c_address = EEPROM_I2C_MADDR_ARCTURUS_D342; + control->i2c_address = EEPROM_I2C_MADDR_0; else - control->i2c_address = EEPROM_I2C_MADDR_ARCTURUS; + control->i2c_address = EEPROM_I2C_MADDR_4; return true; } @@ -148,7 +147,7 @@ static bool __get_eeprom_i2c_addr_ip_discovery(struct amdgpu_device *adev, switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): case IP_VERSION(13, 0, 10): - control->i2c_address = EEPROM_I2C_MADDR_ARCTURUS; + control->i2c_address = EEPROM_I2C_MADDR_4; return true; default: return false; @@ -180,18 +179,18 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, switch (adev->asic_type) { case CHIP_VEGA20: - control->i2c_address = EEPROM_I2C_MADDR_VEGA20; + control->i2c_address = EEPROM_I2C_MADDR_0; break; case CHIP_ARCTURUS: return __get_eeprom_i2c_addr_arct(adev, control); case CHIP_SIENNA_CICHLID: - control->i2c_address = EEPROM_I2C_MADDR_SIENNA_CICHLID; + control->i2c_address = EEPROM_I2C_MADDR_0; break; case CHIP_ALDEBARAN: - control->i2c_address = EEPROM_I2C_MADDR_ALDEBARAN; + control->i2c_address = EEPROM_I2C_MADDR_0; break; case CHIP_IP_DISCOVERY: @@ -203,7 +202,7 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev, switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): - control->i2c_address = EEPROM_I2C_MADDR_ARCTURUS; + control->i2c_address = EEPROM_I2C_MADDR_4; break; default: -- GitLab From 4b31b92b143f7d209f3d494c56d4c4673e9fc53d Mon Sep 17 00:00:00 2001 From: Harsh Jain Date: Wed, 2 Nov 2022 15:23:08 +0530 Subject: [PATCH 1497/1927] drm/amdgpu: complete gfxoff allow signal during suspend without delay change guarantees that gfxoff is allowed before moving further in s2idle sequence to add more reliablity about gfxoff in amdgpu IP's suspend flow Signed-off-by: Harsh Jain Reviewed-by: Evan Quan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index 9546adc8a76f6..23692e5d4d13b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -583,10 +583,14 @@ void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable) if (adev->gfx.gfx_off_req_count == 0 && !adev->gfx.gfx_off_state) { /* If going to s2idle, no need to wait */ - if (adev->in_s0ix) - delay = GFX_OFF_NO_DELAY; - schedule_delayed_work(&adev->gfx.gfx_off_delay_work, + if (adev->in_s0ix) { + if (!amdgpu_dpm_set_powergating_by_smu(adev, + AMD_IP_BLOCK_TYPE_GFX, true)) + adev->gfx.gfx_off_state = true; + } else { + schedule_delayed_work(&adev->gfx.gfx_off_delay_work, delay); + } } } else { if (adev->gfx.gfx_off_req_count == 0) { -- GitLab From 10276a20be1115e1f76c189330da2992df980eee Mon Sep 17 00:00:00 2001 From: Hanjun Guo Date: Fri, 4 Nov 2022 17:50:02 +0800 Subject: [PATCH 1498/1927] drm/radeon: Add the missed acpi_put_table() to fix memory leak When the radeon driver reads the bios information from ACPI table in radeon_acpi_vfct_bios(), it misses to call acpi_put_table() to release the ACPI memory after the init, so add acpi_put_table() properly to fix the memory leak. v2: fix text formatting (Alex) Fixes: 268ba0a99f89 ("drm/radeon: implement ACPI VFCT vbios fetch (v3)") Signed-off-by: Hanjun Guo Signed-off-by: Alex Deucher --- drivers/gpu/drm/radeon/radeon_bios.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/drivers/gpu/drm/radeon/radeon_bios.c b/drivers/gpu/drm/radeon/radeon_bios.c index 33121655d50bb..1d99c9a2b56ee 100644 --- a/drivers/gpu/drm/radeon/radeon_bios.c +++ b/drivers/gpu/drm/radeon/radeon_bios.c @@ -612,13 +612,14 @@ static bool radeon_acpi_vfct_bios(struct radeon_device *rdev) acpi_size tbl_size; UEFI_ACPI_VFCT *vfct; unsigned offset; + bool r = false; if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr))) return false; tbl_size = hdr->length; if (tbl_size < sizeof(UEFI_ACPI_VFCT)) { DRM_ERROR("ACPI VFCT table present but broken (too short #1)\n"); - return false; + goto out; } vfct = (UEFI_ACPI_VFCT *)hdr; @@ -631,13 +632,13 @@ static bool radeon_acpi_vfct_bios(struct radeon_device *rdev) offset += sizeof(VFCT_IMAGE_HEADER); if (offset > tbl_size) { DRM_ERROR("ACPI VFCT image header truncated\n"); - return false; + goto out; } offset += vhdr->ImageLength; if (offset > tbl_size) { DRM_ERROR("ACPI VFCT image truncated\n"); - return false; + goto out; } if (vhdr->ImageLength && @@ -649,15 +650,18 @@ static bool radeon_acpi_vfct_bios(struct radeon_device *rdev) rdev->bios = kmemdup(&vbios->VbiosContent, vhdr->ImageLength, GFP_KERNEL); + if (rdev->bios) + r = true; - if (!rdev->bios) - return false; - return true; + goto out; } } DRM_ERROR("ACPI VFCT table present but broken (too short #2)\n"); - return false; + +out: + acpi_put_table(hdr); + return r; } #else static inline bool radeon_acpi_vfct_bios(struct radeon_device *rdev) -- GitLab From e50db38342d5d2243e9738a51c9737cbb017e456 Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Mon, 7 Nov 2022 22:32:51 +1300 Subject: [PATCH 1499/1927] drm/amdgpu: Replace 1-element array with flexible-array member One-element arrays are deprecated, and we are replacing them with flexible array members instead. So, replace one-element array with flexible-array member in structs _ATOM_GPIO_PIN_ASSIGNMENT, _ATOM_DISPLAY_OBJECT_PATH, _ATOM_DISPLAY_OBJECT_PATH_TABLE, _ATOM_OBJECT_TABLE and refactor the rest of the code accordingly. Important to mention is that doing a build before/after this patch results in no functional binary output differences. This helps with the ongoing efforts to tighten the FORTIFY_SOURCE routines on memcpy() and help us make progress towards globally enabling -fstrict-flex-arrays=3 [1]. Link: https://github.com/KSPP/linux/issues/79 Link: https://github.com/KSPP/linux/issues/238 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836 [1] Signed-off-by: Paulo Miguel Almeida Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/bios/bios_parser.c | 23 ++++++++++++------- drivers/gpu/drm/amd/include/atombios.h | 8 +++---- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c index 9b8ea6e9a2b96..39dd8b2dc254c 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c @@ -138,7 +138,9 @@ static uint8_t get_number_of_objects(struct bios_parser *bp, uint32_t offset) uint32_t object_table_offset = bp->object_info_tbl_offset + offset; - table = GET_IMAGE(ATOM_OBJECT_TABLE, object_table_offset); + table = ((ATOM_OBJECT_TABLE *) bios_get_image(&bp->base, + object_table_offset, + struct_size(table, asObjects, 1))); if (!table) return 0; @@ -166,8 +168,9 @@ static struct graphics_object_id bios_parser_get_connector_id( uint32_t connector_table_offset = bp->object_info_tbl_offset + le16_to_cpu(bp->object_info_tbl.v1_1->usConnectorObjectTableOffset); - ATOM_OBJECT_TABLE *tbl = - GET_IMAGE(ATOM_OBJECT_TABLE, connector_table_offset); + ATOM_OBJECT_TABLE *tbl = ((ATOM_OBJECT_TABLE *) bios_get_image(&bp->base, + connector_table_offset, + struct_size(tbl, asObjects, 1))); if (!tbl) { dm_error("Can't get connector table from atom bios.\n"); @@ -1789,11 +1792,13 @@ static enum bp_result bios_parser_get_gpio_pin_info( if (!DATA_TABLES(GPIO_Pin_LUT)) return BP_RESULT_BADBIOSTABLE; - header = GET_IMAGE(ATOM_GPIO_PIN_LUT, DATA_TABLES(GPIO_Pin_LUT)); + header = ((ATOM_GPIO_PIN_LUT *) bios_get_image(&bp->base, + DATA_TABLES(GPIO_Pin_LUT), + struct_size(header, asGPIO_Pin, 1))); if (!header) return BP_RESULT_BADBIOSTABLE; - if (sizeof(ATOM_COMMON_TABLE_HEADER) + sizeof(ATOM_GPIO_PIN_LUT) + if (sizeof(ATOM_COMMON_TABLE_HEADER) + struct_size(header, asGPIO_Pin, 1) > le16_to_cpu(header->sHeader.usStructureSize)) return BP_RESULT_BADBIOSTABLE; @@ -1978,7 +1983,8 @@ static ATOM_OBJECT *get_bios_object(struct bios_parser *bp, offset += bp->object_info_tbl_offset; - tbl = GET_IMAGE(ATOM_OBJECT_TABLE, offset); + tbl = ((ATOM_OBJECT_TABLE *) bios_get_image(&bp->base, offset, + struct_size(tbl, asObjects, 1))); if (!tbl) return NULL; @@ -2709,8 +2715,9 @@ static enum bp_result get_bracket_layout_record( genericTableOffset = bp->object_info_tbl_offset + bp->object_info_tbl.v1_3->usMiscObjectTableOffset; - object_table = (ATOM_OBJECT_TABLE *) - GET_IMAGE(ATOM_OBJECT_TABLE, genericTableOffset); + object_table = ((ATOM_OBJECT_TABLE *) bios_get_image(&bp->base, + genericTableOffset, + struct_size(object_table, asObjects, 1))); if (!object_table) return BP_RESULT_FAILURE; diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h index b5b1d073f8e24..55ae93c1e365a 100644 --- a/drivers/gpu/drm/amd/include/atombios.h +++ b/drivers/gpu/drm/amd/include/atombios.h @@ -4386,7 +4386,7 @@ typedef struct _ATOM_GPIO_PIN_ASSIGNMENT typedef struct _ATOM_GPIO_PIN_LUT { ATOM_COMMON_TABLE_HEADER sHeader; - ATOM_GPIO_PIN_ASSIGNMENT asGPIO_Pin[1]; + ATOM_GPIO_PIN_ASSIGNMENT asGPIO_Pin[]; }ATOM_GPIO_PIN_LUT; /****************************************************************************/ @@ -4513,7 +4513,7 @@ typedef struct _ATOM_DISPLAY_OBJECT_PATH USHORT usSize; //the size of ATOM_DISPLAY_OBJECT_PATH USHORT usConnObjectId; //Connector Object ID USHORT usGPUObjectId; //GPU ID - USHORT usGraphicObjIds[1]; //1st Encoder Obj source from GPU to last Graphic Obj destinate to connector. + USHORT usGraphicObjIds[]; //1st Encoder Obj source from GPU to last Graphic Obj destinate to connector. }ATOM_DISPLAY_OBJECT_PATH; typedef struct _ATOM_DISPLAY_EXTERNAL_OBJECT_PATH @@ -4530,7 +4530,7 @@ typedef struct _ATOM_DISPLAY_OBJECT_PATH_TABLE UCHAR ucNumOfDispPath; UCHAR ucVersion; UCHAR ucPadding[2]; - ATOM_DISPLAY_OBJECT_PATH asDispPath[1]; + ATOM_DISPLAY_OBJECT_PATH asDispPath[]; }ATOM_DISPLAY_OBJECT_PATH_TABLE; typedef struct _ATOM_OBJECT //each object has this structure @@ -4545,7 +4545,7 @@ typedef struct _ATOM_OBJECT_TABLE //Above 4 object table { UCHAR ucNumberOfObjects; UCHAR ucPadding[3]; - ATOM_OBJECT asObjects[1]; + ATOM_OBJECT asObjects[]; }ATOM_OBJECT_TABLE; typedef struct _ATOM_SRC_DST_TABLE_FOR_ONE_OBJECT //usSrcDstTableOffset pointing to this structure -- GitLab From fcb4f9193ee550aad05c8d0f8f6637143e567c80 Mon Sep 17 00:00:00 2001 From: LongJun Tang Date: Fri, 4 Nov 2022 17:29:31 +0800 Subject: [PATCH 1500/1927] drm/amd/display: Have risk for memory exhaustion In dcn*_clock_source_create when dcn*_clk_src_construct fails allocated clk_src needs release. A local attack could use this to cause memory exhaustion. Signed-off-by: LongJun Tang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c | 1 + 8 files changed, 8 insertions(+) diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c index af4fe695535ed..e057f8a93377d 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c @@ -1328,6 +1328,7 @@ static struct clock_source *dcn30_clock_source_create( return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c index f04595b750abc..7c1225046544a 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c @@ -1288,6 +1288,7 @@ static struct clock_source *dcn301_clock_source_create( return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c index d3945876aceda..9ca7db0ec7b0b 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c @@ -464,6 +464,7 @@ static struct clock_source *dcn302_clock_source_create(struct dc_context *ctx, s return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c index 7e7f18bef0986..932c5d5706ecf 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c @@ -431,6 +431,7 @@ static struct clock_source *dcn303_clock_source_create(struct dc_context *ctx, s return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c index 8f5e89cb9d3ec..15e73f4d92ab8 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c @@ -1629,6 +1629,7 @@ static struct clock_source *dcn31_clock_source_create( return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c index 96a3d41febffe..2c68862b0d8ed 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c @@ -1627,6 +1627,7 @@ static struct clock_source *dcn31_clock_source_create( return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index 77e40ee488bd6..f7fea3544c31c 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -830,6 +830,7 @@ static struct clock_source *dcn32_clock_source_create( return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c index e5861225f1df9..d7d2c42bdb0f4 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c @@ -829,6 +829,7 @@ static struct clock_source *dcn321_clock_source_create( return &clk_src->base; } + kfree(clk_src); BREAK_TO_DEBUGGER(); return NULL; } -- GitLab From d852871c6dd17b19ae12354be1eaf7037ac79221 Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Wed, 9 Nov 2022 12:06:27 -0500 Subject: [PATCH 1501/1927] drm/amd/display: only fill dirty rectangles when PSR is enabled Currently, we are calling fill_dc_dirty_rects() even if PSR isn't supported by the relevant link in amdgpu_dm_commit_planes(), this is undesirable especially because when drm.debug is enabled we are printing messages in fill_dc_dirty_rects() that are only useful for debugging PSR (and confusing otherwise). So, we can instead limit the filling of dirty rectangles to only when PSR is enabled. Reviewed-by: Leo Li Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 3db79f0b5a8fa..5114726dedf7a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -7684,9 +7684,10 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bundle->surface_updates[planes_count].plane_info = &bundle->plane_infos[planes_count]; - fill_dc_dirty_rects(plane, old_plane_state, new_plane_state, - new_crtc_state, - &bundle->flip_addrs[planes_count]); + if (acrtc_state->stream->link->psr_settings.psr_feature_enabled) + fill_dc_dirty_rects(plane, old_plane_state, + new_plane_state, new_crtc_state, + &bundle->flip_addrs[planes_count]); /* * Only allow immediate flips for fast updates that don't -- GitLab From d1a372af1c3d834c2346b2e822cd6c40abc09866 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Fri, 26 Aug 2022 18:22:35 -0400 Subject: [PATCH 1502/1927] drm/amdgpu: Set MTYPE in PTE based on BO flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same BO may need different MTYPEs and SNOOP flags in PTEs depending on its current location relative to the mapping GPU. Setting MTYPEs from clients ahead of time is not practical for coherent memory sharing. Instead determine the correct MTYPE for the desired coherence model and current BO location when updating the page tables. To maintain backwards compatibility with MTYPE-selection in AMDGPU_VA_OP_MAP, the coherence-model-based MTYPE selection is only applied if it chooses an MTYPE other than MTYPE_NC (the default). Add two AMDGPU_GEM_CREATE_... flags to indicate the coherence model. The default if no flag is specified is non-coherent (i.e. coarse-grained coherent at dispatch boundaries). Update amdgpu_amdkfd_gpuvm.c to use this new method to choose the correct MTYPE depending on the current memory location. v2: * check that bo is not NULL (e.g. PRT mappings) * Fix missing ~ bitmask in gmc_v11_0.c v3: * squash in "drm/amdgpu: Inherit coherence flags on dmabuf import" Suggested-by: Christian König Signed-off-by: Felix Kuehling Acked-by: Christian König Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 59 ++------------ drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 +- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 7 ++ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 7 ++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 79 +++++++++++++++++-- include/uapi/drm/amdgpu_drm.h | 14 ++++ 6 files changed, 110 insertions(+), 60 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index ba72a910d0d59..c5c9bfa2772ef 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -405,63 +405,15 @@ static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) { - struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); - bool coherent = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; - bool uncached = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; - uint32_t mapping_flags; - uint64_t pte_flags; - bool snoop = false; + uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | + AMDGPU_VM_MTYPE_DEFAULT; - mapping_flags = AMDGPU_VM_PAGE_READABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; - switch (adev->ip_versions[GC_HWIP][0]) { - case IP_VERSION(9, 4, 1): - case IP_VERSION(9, 4, 2): - if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { - if (bo_adev == adev) { - if (uncached) - mapping_flags |= AMDGPU_VM_MTYPE_UC; - else if (coherent) - mapping_flags |= AMDGPU_VM_MTYPE_CC; - else - mapping_flags |= AMDGPU_VM_MTYPE_RW; - if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) && - adev->gmc.xgmi.connected_to_cpu) - snoop = true; - } else { - if (uncached || coherent) - mapping_flags |= AMDGPU_VM_MTYPE_UC; - else - mapping_flags |= AMDGPU_VM_MTYPE_NC; - if (amdgpu_xgmi_same_hive(adev, bo_adev)) - snoop = true; - } - } else { - if (uncached || coherent) - mapping_flags |= AMDGPU_VM_MTYPE_UC; - else - mapping_flags |= AMDGPU_VM_MTYPE_NC; - snoop = true; - } - break; - default: - if (uncached || coherent) - mapping_flags |= AMDGPU_VM_MTYPE_UC; - else - mapping_flags |= AMDGPU_VM_MTYPE_NC; - - if (!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) - snoop = true; - } - - pte_flags = amdgpu_gem_va_map_flags(adev, mapping_flags); - pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; - - return pte_flags; + return amdgpu_gem_va_map_flags(adev, mapping_flags); } /** @@ -1673,6 +1625,11 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( } } + if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT) + alloc_flags |= AMDGPU_GEM_CREATE_COHERENT; + if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED) + alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED; + *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); if (!*mem) { ret = -ENOMEM; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c index 7bd8e33b14be5..271e30e34d932 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c @@ -328,7 +328,9 @@ amdgpu_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) if (dma_buf->ops == &amdgpu_dmabuf_ops) { struct amdgpu_bo *other = gem_to_amdgpu_bo(dma_buf->priv); - flags |= other->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC; + flags |= other->flags & (AMDGPU_GEM_CREATE_CPU_GTT_USWC | + AMDGPU_GEM_CREATE_COHERENT | + AMDGPU_GEM_CREATE_UNCACHED); } ret = amdgpu_gem_object_create(adev, dma_buf->size, PAGE_SIZE, diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c index f513e2c2e964f..a83efdc8aa0c5 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c @@ -612,6 +612,8 @@ static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping, uint64_t *flags) { + struct amdgpu_bo *bo = mapping->bo_va->base.bo; + *flags &= ~AMDGPU_PTE_EXECUTABLE; *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; @@ -628,6 +630,11 @@ static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev, *flags |= AMDGPU_PTE_SYSTEM; *flags &= ~AMDGPU_PTE_VALID; } + + if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT | + AMDGPU_GEM_CREATE_UNCACHED)) + *flags = (*flags & ~AMDGPU_PTE_MTYPE_NV10_MASK) | + AMDGPU_PTE_MTYPE_NV10(MTYPE_UC); } static unsigned gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c index 66dfb574cc7d1..16f52049d9863 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c @@ -503,6 +503,8 @@ static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping, uint64_t *flags) { + struct amdgpu_bo *bo = mapping->bo_va->base.bo; + *flags &= ~AMDGPU_PTE_EXECUTABLE; *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; @@ -519,6 +521,11 @@ static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev, *flags |= AMDGPU_PTE_SYSTEM; *flags &= ~AMDGPU_PTE_VALID; } + + if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT | + AMDGPU_GEM_CREATE_UNCACHED)) + *flags = (*flags & ~AMDGPU_PTE_MTYPE_NV10_MASK) | + AMDGPU_PTE_MTYPE_NV10(MTYPE_UC); } static unsigned gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c index 67ca16a8027c7..50386eb2eec8d 100644 --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c @@ -1113,6 +1113,74 @@ static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, } } +static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev, + struct amdgpu_bo *bo, + struct amdgpu_bo_va_mapping *mapping, + uint64_t *flags) +{ + struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); + bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM; + bool coherent = bo->flags & AMDGPU_GEM_CREATE_COHERENT; + bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; + unsigned int mtype; + bool snoop = false; + + switch (adev->ip_versions[GC_HWIP][0]) { + case IP_VERSION(9, 4, 1): + case IP_VERSION(9, 4, 2): + if (is_vram) { + if (bo_adev == adev) { + if (uncached) + mtype = MTYPE_UC; + else if (coherent) + mtype = MTYPE_CC; + else + mtype = MTYPE_RW; + /* FIXME: is this still needed? Or does + * amdgpu_ttm_tt_pde_flags already handle this? + */ + if (adev->ip_versions[GC_HWIP][0] == + IP_VERSION(9, 4, 2) && + adev->gmc.xgmi.connected_to_cpu) + snoop = true; + } else { + if (uncached || coherent) + mtype = MTYPE_UC; + else + mtype = MTYPE_NC; + if (mapping->bo_va->is_xgmi) + snoop = true; + } + } else { + if (uncached || coherent) + mtype = MTYPE_UC; + else + mtype = MTYPE_NC; + /* FIXME: is this still needed? Or does + * amdgpu_ttm_tt_pde_flags already handle this? + */ + snoop = true; + } + break; + default: + if (uncached || coherent) + mtype = MTYPE_UC; + else + mtype = MTYPE_NC; + + /* FIXME: is this still needed? Or does + * amdgpu_ttm_tt_pde_flags already handle this? + */ + if (!is_vram) + snoop = true; + } + + if (mtype != MTYPE_NC) + *flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) | + AMDGPU_PTE_MTYPE_VG10(mtype); + *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; +} + static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, struct amdgpu_bo_va_mapping *mapping, uint64_t *flags) @@ -1128,14 +1196,9 @@ static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, *flags &= ~AMDGPU_PTE_VALID; } - if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || - adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) && - !(*flags & AMDGPU_PTE_SYSTEM) && - mapping->bo_va->is_xgmi) - *flags |= AMDGPU_PTE_SNOOPED; - - if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) - *flags |= mapping->flags & AMDGPU_PTE_SNOOPED; + if (mapping->bo_va->base.bo) + gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo, + mapping, flags); } static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 0d93ec132ebbc..4038abe8505af 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -144,6 +144,20 @@ extern "C" { * content. */ #define AMDGPU_GEM_CREATE_DISCARDABLE (1 << 12) +/* Flag that BO is shared coherently between multiple devices or CPU threads. + * May depend on GPU instructions to flush caches explicitly + * + * This influences the choice of MTYPE in the PTEs on GFXv9 and later GPUs and + * may override the MTYPE selected in AMDGPU_VA_OP_MAP. + */ +#define AMDGPU_GEM_CREATE_COHERENT (1 << 13) +/* Flag that BO should not be cached by GPU. Coherent without having to flush + * GPU caches explicitly + * + * This influences the choice of MTYPE in the PTEs on GFXv9 and later GPUs and + * may override the MTYPE selected in AMDGPU_VA_OP_MAP. + */ +#define AMDGPU_GEM_CREATE_UNCACHED (1 << 14) struct drm_amdgpu_gem_create_in { /** the requested memory size */ -- GitLab From bc66c9ab162d2a633ee3eb864d7bc2369e79c1e4 Mon Sep 17 00:00:00 2001 From: Tim Huang Date: Thu, 3 Nov 2022 11:05:19 +0800 Subject: [PATCH 1503/1927] drm/amd/pm: update SMU IP v13.0.4 msg interface header Some of the unused messages that were used earlier in development have been freed up as spare messages, no intended functional changes. Signed-off-by: Mario Limonciello Signed-off-by: Tim Huang Acked-by: Alex Deucher Reviewed-by: Yifan Zhang Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- .../amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h index d9b0cd7522006..f4d6c07b56ea7 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v13_0_4_ppsmc.h @@ -54,14 +54,14 @@ #define PPSMC_MSG_TestMessage 0x01 ///< To check if PMFW is alive and responding. Requirement specified by PMFW team #define PPSMC_MSG_GetPmfwVersion 0x02 ///< Get PMFW version #define PPSMC_MSG_GetDriverIfVersion 0x03 ///< Get PMFW_DRIVER_IF version -#define PPSMC_MSG_EnableGfxOff 0x04 ///< Enable GFXOFF -#define PPSMC_MSG_DisableGfxOff 0x05 ///< Disable GFXOFF +#define PPSMC_MSG_SPARE0 0x04 ///< SPARE +#define PPSMC_MSG_SPARE1 0x05 ///< SPARE #define PPSMC_MSG_PowerDownVcn 0x06 ///< Power down VCN #define PPSMC_MSG_PowerUpVcn 0x07 ///< Power up VCN; VCN is power gated by default #define PPSMC_MSG_SetHardMinVcn 0x08 ///< For wireless display #define PPSMC_MSG_SetSoftMinGfxclk 0x09 ///< Set SoftMin for GFXCLK, argument is frequency in MHz -#define PPSMC_MSG_ActiveProcessNotify 0x0A ///< Needs update -#define PPSMC_MSG_ForcePowerDownGfx 0x0B ///< Force power down GFX, i.e. enter GFXOFF +#define PPSMC_MSG_SPARE2 0x0A ///< SPARE +#define PPSMC_MSG_SPARE3 0x0B ///< SPARE #define PPSMC_MSG_PrepareMp1ForUnload 0x0C ///< Prepare PMFW for GFX driver unload #define PPSMC_MSG_SetDriverDramAddrHigh 0x0D ///< Set high 32 bits of DRAM address for Driver table transfer #define PPSMC_MSG_SetDriverDramAddrLow 0x0E ///< Set low 32 bits of DRAM address for Driver table transfer @@ -73,8 +73,7 @@ #define PPSMC_MSG_SetSoftMinFclk 0x14 ///< Set hard min for FCLK #define PPSMC_MSG_SetSoftMinVcn 0x15 ///< Set soft min for VCN clocks (VCLK and DCLK) - -#define PPSMC_MSG_EnableGfxImu 0x16 ///< Needs update +#define PPSMC_MSG_EnableGfxImu 0x16 ///< Enable GFX IMU #define PPSMC_MSG_GetGfxclkFrequency 0x17 ///< Get GFX clock frequency #define PPSMC_MSG_GetFclkFrequency 0x18 ///< Get FCLK frequency @@ -102,8 +101,8 @@ #define PPSMC_MSG_SetHardMinIspxclkByFreq 0x2C ///< Set HardMin by frequency for ISPXCLK #define PPSMC_MSG_PowerDownUmsch 0x2D ///< Power down VCN.UMSCH (aka VSCH) scheduler #define PPSMC_MSG_PowerUpUmsch 0x2E ///< Power up VCN.UMSCH (aka VSCH) scheduler -#define PPSMC_Message_IspStutterOn_MmhubPgDis 0x2F ///< ISP StutterOn mmHub PgDis -#define PPSMC_Message_IspStutterOff_MmhubPgEn 0x30 ///< ISP StufferOff mmHub PgEn +#define PPSMC_MSG_IspStutterOn_MmhubPgDis 0x2F ///< ISP StutterOn mmHub PgDis +#define PPSMC_MSG_IspStutterOff_MmhubPgEn 0x30 ///< ISP StufferOff mmHub PgEn #define PPSMC_Message_Count 0x31 ///< Total number of PPSMC messages /** @}*/ -- GitLab From 66f7903779fbbc620bf1040017e4833ef6a0b541 Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Thu, 3 Nov 2022 17:01:46 -0400 Subject: [PATCH 1504/1927] drm/amdkfd: Fix error handling in kfd_criu_restore_events mutex_unlock before the exit label because all the error code paths that jump there didn't take that lock. This fixes unbalanced locking errors in case of restore errors. Fixes: 40e8a766a761 ("drm/amdkfd: CRIU checkpoint and restore events") Signed-off-by: Felix Kuehling Reviewed-by: Rajneesh Bhardwaj Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdkfd/kfd_events.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 83e3ce9f60491..729d26d648af3 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -506,6 +506,7 @@ int kfd_criu_restore_event(struct file *devkfd, ret = create_other_event(p, ev, &ev_priv->event_id); break; } + mutex_unlock(&p->event_mutex); exit: if (ret) @@ -513,8 +514,6 @@ int kfd_criu_restore_event(struct file *devkfd, kfree(ev_priv); - mutex_unlock(&p->event_mutex); - return ret; } -- GitLab From b91c23e099f0b65d62159da13458c5eefa76083f Mon Sep 17 00:00:00 2001 From: Felix Kuehling Date: Tue, 1 Nov 2022 15:02:48 -0400 Subject: [PATCH 1505/1927] drm/amdkfd: Fix error handling in criu_checkpoint Checkpoint BOs last. That way we don't need to close dmabuf FDs if something else fails later. This avoids problematic access to user mode memory in the error handling code path. criu_checkpoint_bos has its own error handling and cleanup that does not depend on access to user memory. In the private data, keep BOs before the remaining objects. This is necessary to restore things in the correct order as restoring events depends on the events-page BO being restored first. Fixes: be072b06c739 ("drm/amdkfd: CRIU export BOs as prime dmabuf objects") Reported-by: Jann Horn CC: Rajneesh Bhardwaj Signed-off-by: Felix Kuehling Reviewed-and-tested-by: Rajneesh Bhardwaj Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 34 +++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 5feaba6a77de4..6d291aa6386bd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1950,7 +1950,7 @@ static int criu_checkpoint(struct file *filep, { int ret; uint32_t num_devices, num_bos, num_objects; - uint64_t priv_size, priv_offset = 0; + uint64_t priv_size, priv_offset = 0, bo_priv_offset; if (!args->devices || !args->bos || !args->priv_data) return -EINVAL; @@ -1994,38 +1994,34 @@ static int criu_checkpoint(struct file *filep, if (ret) goto exit_unlock; - ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos, - (uint8_t __user *)args->priv_data, &priv_offset); - if (ret) - goto exit_unlock; + /* Leave room for BOs in the private data. They need to be restored + * before events, but we checkpoint them last to simplify the error + * handling. + */ + bo_priv_offset = priv_offset; + priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data); if (num_objects) { ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data, &priv_offset); if (ret) - goto close_bo_fds; + goto exit_unlock; ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data, &priv_offset); if (ret) - goto close_bo_fds; + goto exit_unlock; ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset); if (ret) - goto close_bo_fds; + goto exit_unlock; } -close_bo_fds: - if (ret) { - /* If IOCTL returns err, user assumes all FDs opened in criu_dump_bos are closed */ - uint32_t i; - struct kfd_criu_bo_bucket *bo_buckets = (struct kfd_criu_bo_bucket *) args->bos; - - for (i = 0; i < num_bos; i++) { - if (bo_buckets[i].alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) - close_fd(bo_buckets[i].dmabuf_fd); - } - } + /* This must be the last thing in this function that can fail. + * Otherwise we leak dmabuf file descriptors. + */ + ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos, + (uint8_t __user *)args->priv_data, &bo_priv_offset); exit_unlock: mutex_unlock(&p->mutex); -- GitLab From 77c092e054262b594614bad5e5f47e57c5d29639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 2 Nov 2022 14:55:13 +0100 Subject: [PATCH 1506/1927] drm/amdgpu: workaround for TLB seq race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It can happen that we query the sequence value before the callback had a chance to run. Workaround that by grabbing the fence lock and releasing it again. Should be replaced by hw handling soon. Signed-off-by: Christian König CC: stable@vger.kernel.org # 5.19+ Fixes: 5255e146c99a6 ("drm/amdgpu: rework TLB flushing") Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2113 Acked-by: Alex Deucher Acked-by: Philip Yang Tested-by: Stefan Springer Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 83acb7bd80feb..1d31771b42308 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -492,6 +492,21 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m); */ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm) { + unsigned long flags; + spinlock_t *lock; + + /* + * Workaround to stop racing between the fence signaling and handling + * the cb. The lock is static after initially setting it up, just make + * sure that the dma_fence structure isn't freed up. + */ + rcu_read_lock(); + lock = vm->last_tlb_flush->lock; + rcu_read_unlock(); + + spin_lock_irqsave(lock, flags); + spin_unlock_irqrestore(lock, flags); + return atomic64_read(&vm->tlb_seq); } -- GitLab From 632d06985235d988c9d7e6eec8fa655be0761fd0 Mon Sep 17 00:00:00 2001 From: Nicholas Kazlauskas Date: Tue, 25 Oct 2022 11:26:04 -0400 Subject: [PATCH 1507/1927] drm/amd/display: Update SR watermarks for DCN314 [Why & How] New values requested by hardware after fine-tuning. Update for all memory types. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Nicholas Kazlauskas Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- .../dc/clk_mgr/dcn314/dcn314_clk_mgr.c | 32 +++++++++---------- .../amd/display/dc/dml/dcn314/dcn314_fpu.c | 4 +-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c index 1131c6d73f6cd..20a06c04e4a1d 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c @@ -363,32 +363,32 @@ static struct wm_table ddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.72, - .sr_exit_time_us = 9, - .sr_enter_plus_exit_time_us = 11, + .sr_exit_time_us = 12.5, + .sr_enter_plus_exit_time_us = 14.5, .valid = true, }, } @@ -400,32 +400,32 @@ static struct wm_table lpddr5_wm_table = { .wm_inst = WM_A, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, { .wm_inst = WM_B, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, { .wm_inst = WM_C, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, { .wm_inst = WM_D, .wm_type = WM_TYPE_PSTATE_CHG, .pstate_latency_us = 11.65333, - .sr_exit_time_us = 11.5, - .sr_enter_plus_exit_time_us = 14.5, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .valid = true, }, } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c index cf420ad2b8dcd..34b6c763a4554 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c @@ -146,8 +146,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_14_soc = { }, }, .num_states = 5, - .sr_exit_time_us = 9.0, - .sr_enter_plus_exit_time_us = 11.0, + .sr_exit_time_us = 16.5, + .sr_enter_plus_exit_time_us = 18.5, .sr_exit_z8_time_us = 442.0, .sr_enter_plus_exit_z8_time_us = 560.0, .writeback_latency_us = 12.0, -- GitLab From ab4b35008db9b7ae747679250e5c26d7c3a90cea Mon Sep 17 00:00:00 2001 From: Aurabindo Pillai Date: Thu, 20 Oct 2022 17:26:24 -0400 Subject: [PATCH 1508/1927] drm/amd/display: Zeromem mypipe heap struct before using it [Why&How] Bug was caused when moving variable from stack to heap because it was reusable and garbage was left over, so we need to zero mem. Reviewed-by: Martin Leung Acked-by: Alan Liu Signed-off-by: Aurabindo Pillai Signed-off-by: Martin Leung Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index 3d184679f129e..ae6e6abc620b1 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -3192,6 +3192,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l mode_lib->vba.FCLKChangeLatency, mode_lib->vba.UrgLatency[i], mode_lib->vba.SREnterPlusExitTime); + memset(&v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull, 0, sizeof(DmlPipe)); v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe.Dppclk = mode_lib->vba.RequiredDPPCLK[i][j][k]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe.Dispclk = mode_lib->vba.RequiredDISPCLK[i][j]; v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe.PixelClock = mode_lib->vba.PixelClock[k]; -- GitLab From 0d5c5c210a4d4e655feb93b379647f0b179cdafe Mon Sep 17 00:00:00 2001 From: Chaitanya Dhere Date: Thu, 27 Oct 2022 09:51:19 -0400 Subject: [PATCH 1509/1927] drm/amd/display: Fix FCLK deviation and tool compile issues [Why] Recent backports from open source do not have header inclusion pattern that is consistent with inclusion style in the rest of the file. This breaks the internal tool builds as well. A recent commit erronously modified the original DML formula for calculating ActiveClockChangeLatencyHidingY. This resulted in a FCLK deviation from the golden values. [How] Change the way in which display_mode_vba.h is included so that it is consistent with the inclusion style in rest of the file which also fixes the tool build. Restore the DML formula to its original state to fix the FCLK deviation. Reviewed-by: Aurabindo Pillai Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Chaitanya Dhere Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c | 2 +- .../gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index 968924c491c18..ab9217732a17f 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -4397,7 +4397,7 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport( if (v->NumberOfActiveSurfaces > 1) { ActiveClockChangeLatencyHidingY = ActiveClockChangeLatencyHidingY - - (1 - 1 / v->NumberOfActiveSurfaces) * SwathHeightY[k] * v->HTotal[k] + - (1.0 - 1.0 / v->NumberOfActiveSurfaces) * SwathHeightY[k] * v->HTotal[k] / v->PixelClock[k] / v->VRatio[k]; } diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index 2c3827546ac77..fdccaa93eb2ef 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -30,7 +30,7 @@ #include "os_types.h" #include "../dc_features.h" #include "../display_mode_structs.h" -#include "dml/display_mode_vba.h" +#include "../display_mode_vba.h" unsigned int dml32_dscceComputeDelay( unsigned int bpc, -- GitLab From ce62198d8b62734a985d22652e75a649be052390 Mon Sep 17 00:00:00 2001 From: Nicholas Kazlauskas Date: Thu, 27 Oct 2022 15:34:33 -0400 Subject: [PATCH 1510/1927] drm/amd/display: Fix reg timeout in enc314_enable_fifo [Why] The link enablement sequence can end up resetting the encoder while the PHY symclk isn't yet on. This means that waiting for symclk on will timeout, along with the reset bit never asserting high. This causes unnecessary delay when enabling the link and produces a warning affecting multiple IGT tests. [How] Don't wait for the symclk to be on here because firmware already does. Don't wait for reset if we know the symclk isn't on. Split the reset into a helper function that checks the bit and decides whether or not a delay is sufficient. Reviewed-by: Roman Li Acked-by: Alan Liu Signed-off-by: Nicholas Kazlauskas Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org # 6.0.x --- .../dc/dcn314/dcn314_dio_stream_encoder.c | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c index 7e773bf7b895f..38842f938bed0 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c @@ -49,18 +49,30 @@ #define CTX \ enc1->base.ctx +static void enc314_reset_fifo(struct stream_encoder *enc, bool reset) +{ + struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); + uint32_t reset_val = reset ? 1 : 0; + uint32_t is_symclk_on; + + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, reset_val); + REG_GET(DIG_FE_CNTL, DIG_SYMCLK_FE_ON, &is_symclk_on); + + if (is_symclk_on) + REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, reset_val, 10, 5000); + else + udelay(10); +} static void enc314_enable_fifo(struct stream_encoder *enc) { struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc); - /* TODO: Confirm if we need to wait for DIG_SYMCLK_FE_ON */ - REG_WAIT(DIG_FE_CNTL, DIG_SYMCLK_FE_ON, 1, 10, 5000); REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, 0x7); - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1); - REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 1, 10, 5000); - REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 0); - REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 0, 10, 5000); + + enc314_reset_fifo(enc, true); + enc314_reset_fifo(enc, false); + REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_ENABLE, 1); } -- GitLab From c0b2753f5db281b07013899c79b5f06a614055f9 Mon Sep 17 00:00:00 2001 From: Steve Su Date: Fri, 28 Oct 2022 05:52:32 +0800 Subject: [PATCH 1511/1927] drm/amd/display: Fix gpio port mapping issue [Why] 1. Port of gpio has different mapping. [How] 1. Add a dummy entry in mapping table. 2. Fix incorrect mask bit field access. Reviewed-by: Alvin Lee Acked-by: Alan Liu Signed-off-by: Steve Su Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- .../amd/display/dc/gpio/dcn32/hw_factory_dcn32.c | 14 ++++++++++++++ drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c | 9 ++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c b/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c index d635b73af46fe..0ea52ba5ac827 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/dcn32/hw_factory_dcn32.c @@ -107,6 +107,13 @@ static const struct ddc_registers ddc_data_regs_dcn[] = { ddc_data_regs_dcn2(3), ddc_data_regs_dcn2(4), ddc_data_regs_dcn2(5), + { + // add a dummy entry for cases no such port + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}, + .ddc_setup = 0, + .phy_aux_cntl = 0, + .dc_gpio_aux_ctrl_5 = 0 + }, { DDC_GPIO_VGA_REG_LIST(DATA), .ddc_setup = 0, @@ -121,6 +128,13 @@ static const struct ddc_registers ddc_clk_regs_dcn[] = { ddc_clk_regs_dcn2(3), ddc_clk_regs_dcn2(4), ddc_clk_regs_dcn2(5), + { + // add a dummy entry for cases no such port + {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,}, + .ddc_setup = 0, + .phy_aux_cntl = 0, + .dc_gpio_aux_ctrl_5 = 0 + }, { DDC_GPIO_VGA_REG_LIST(CLK), .ddc_setup = 0, diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c index 6fd38cdd68c0c..525bc8881950d 100644 --- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c +++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c @@ -94,11 +94,14 @@ static enum gpio_result set_config( * is required for detection of AUX mode */ if (hw_gpio->base.en != GPIO_DDC_LINE_VIP_PAD) { if (!ddc_data_pd_en || !ddc_clk_pd_en) { - - REG_SET_2(gpio.MASK_reg, regval, + if (hw_gpio->base.en == GPIO_DDC_LINE_DDC_VGA) { + // bit 4 of mask has different usage in some cases + REG_SET(gpio.MASK_reg, regval, DC_GPIO_DDC1DATA_PD_EN, 1); + } else { + REG_SET_2(gpio.MASK_reg, regval, DC_GPIO_DDC1DATA_PD_EN, 1, DC_GPIO_DDC1CLK_PD_EN, 1); - + } if (config_data->type == GPIO_CONFIG_TYPE_I2C_AUX_DUAL_MODE) msleep(3); -- GitLab From ce902d987a8caa5c9a96080e280ecc992414a127 Mon Sep 17 00:00:00 2001 From: Dillon Varone Date: Thu, 27 Oct 2022 16:22:26 -0400 Subject: [PATCH 1512/1927] drm/amd/display: Enforce minimum prefetch time for low memclk on DCN32 [WHY?] Data return times when using lowest memclk can be <= 60us, which can cause underflow on high bandwidth displays with a workload. [HOW?] Enforce a minimum prefetch time during validation for low memclk modes. Reviewed-by: Jun Lei Acked-by: Alan Liu Signed-off-by: Dillon Varone Tested-by: Daniel Wheeler Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dc.h | 1 + .../gpu/drm/amd/display/dc/dcn32/dcn32_resource.c | 1 + .../gpu/drm/amd/display/dc/dcn321/dcn321_resource.c | 1 + drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c | 2 ++ .../amd/display/dc/dml/dcn32/display_mode_vba_32.c | 4 ++++ .../amd/display/dc/dml/dcn32/display_mode_vba_32.h | 3 +++ .../display/dc/dml/dcn32/display_mode_vba_util_32.c | 12 ++++++++++-- .../display/dc/dml/dcn32/display_mode_vba_util_32.h | 1 + .../gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c | 2 ++ .../drm/amd/display/dc/dml/display_mode_structs.h | 1 + 10 files changed, 26 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 737b221ca6890..0598465fd1a1b 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -853,6 +853,7 @@ struct dc_debug_options { bool enable_dp_dig_pixel_rate_div_policy; enum lttpr_mode lttpr_mode_override; unsigned int dsc_delay_factor_wa_x1000; + unsigned int min_prefetch_in_strobe_ns; }; struct gpu_info_soc_bounding_box_v1_0; diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c index a88dd7b3d1c10..d1598e3131f66 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c @@ -724,6 +724,7 @@ static const struct dc_debug_options debug_defaults_drv = { .enable_dp_dig_pixel_rate_div_policy = 1, .allow_sw_cursor_fallback = false, .alloc_extra_way_for_cursor = true, + .min_prefetch_in_strobe_ns = 60000, // 60us }; static const struct dc_debug_options debug_defaults_diags = { diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c index 61087f2385a96..6292ac515d1a4 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c +++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c @@ -722,6 +722,7 @@ static const struct dc_debug_options debug_defaults_drv = { .enable_dp_dig_pixel_rate_div_policy = 1, .allow_sw_cursor_fallback = false, .alloc_extra_way_for_cursor = true, + .min_prefetch_in_strobe_ns = 60000, // 60us }; static const struct dc_debug_options debug_defaults_diags = { diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c index f37c9a6b3b7e0..659323ebd79d8 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c @@ -2364,6 +2364,8 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa /* DML DSC delay factor workaround */ dcn3_2_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + dcn3_2_ip.min_prefetch_in_strobe_us = dc->debug.min_prefetch_in_strobe_ns / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_2_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c index ae6e6abc620b1..244fd15d24b43 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c @@ -786,6 +786,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman v->SwathHeightY[k], v->SwathHeightC[k], TWait, + v->DRAMSpeedPerState[mode_lib->vba.VoltageLevel] <= MEM_STROBE_FREQ_MHZ ? + mode_lib->vba.ip.min_prefetch_in_strobe_us : 0, /* Output */ &v->DSTXAfterScaler[k], &v->DSTYAfterScaler[k], @@ -3245,6 +3247,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l v->swath_width_chroma_ub_this_state[k], v->SwathHeightYThisState[k], v->SwathHeightCThisState[k], v->TWait, + v->DRAMSpeedPerState[i] <= MEM_STROBE_FREQ_MHZ ? + mode_lib->vba.ip.min_prefetch_in_strobe_us : 0, /* Output */ &v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.DSTXAfterScaler[k], diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h index c62e0991358b3..f82e14cd9d8aa 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.h @@ -49,6 +49,9 @@ #define BPP_INVALID 0 #define BPP_BLENDED_PIPE 0xffffffff +#define MEM_STROBE_FREQ_MHZ 1600 +#define MEM_STROBE_MAX_DELIVERY_TIME_US 60.0 + struct display_mode_lib; void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib); diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c index ab9217732a17f..635fc54338fa2 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c @@ -3417,6 +3417,7 @@ bool dml32_CalculatePrefetchSchedule( unsigned int SwathHeightY, unsigned int SwathHeightC, double TWait, + double TPreReq, /* Output */ double *DSTXAfterScaler, double *DSTYAfterScaler, @@ -3474,6 +3475,7 @@ bool dml32_CalculatePrefetchSchedule( double min_Lsw; double Tsw_est1 = 0; double Tsw_est3 = 0; + double TPreMargin = 0; if (v->GPUVMEnable == true && v->HostVMEnable == true) HostVMDynamicLevelsTrips = v->HostVMMaxNonCachedPageTableLevels; @@ -3699,6 +3701,8 @@ bool dml32_CalculatePrefetchSchedule( dst_y_prefetch_equ = dml_floor(4.0 * (dst_y_prefetch_equ + 0.125), 1) / 4.0; Tpre_rounded = dst_y_prefetch_equ * LineTime; + + TPreMargin = Tpre_rounded - TPreReq; #ifdef __DML_VBA_DEBUG__ dml_print("DML::%s: dst_y_prefetch_equ: %f (after round)\n", __func__, dst_y_prefetch_equ); dml_print("DML::%s: LineTime: %f\n", __func__, LineTime); @@ -3726,7 +3730,7 @@ bool dml32_CalculatePrefetchSchedule( *VRatioPrefetchY = 0; *VRatioPrefetchC = 0; *RequiredPrefetchPixDataBWLuma = 0; - if (dst_y_prefetch_equ > 1) { + if (dst_y_prefetch_equ > 1 && TPreMargin > 0.0) { double PrefetchBandwidth1; double PrefetchBandwidth2; double PrefetchBandwidth3; @@ -3872,7 +3876,11 @@ bool dml32_CalculatePrefetchSchedule( } if (dst_y_prefetch_oto < dst_y_prefetch_equ) { - *DestinationLinesForPrefetch = dst_y_prefetch_oto; + if (dst_y_prefetch_oto * LineTime < TPreReq) { + *DestinationLinesForPrefetch = dst_y_prefetch_equ; + } else { + *DestinationLinesForPrefetch = dst_y_prefetch_oto; + } TimeForFetchingMetaPTE = Tvm_oto; TimeForFetchingRowInVBlank = Tr0_oto; *PrefetchBandwidth = prefetch_bw_oto; diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h index fdccaa93eb2ef..3989c2a28faec 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h @@ -743,6 +743,7 @@ bool dml32_CalculatePrefetchSchedule( unsigned int SwathHeightY, unsigned int SwathHeightC, double TWait, + double TPreReq, /* Output */ double *DSTXAfterScaler, double *DSTYAfterScaler, diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c index ec0486efab147..432b4ecd01a71 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c +++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c @@ -544,6 +544,8 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p /* DML DSC delay factor workaround */ dcn3_21_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0; + dcn3_21_ip.min_prefetch_in_strobe_us = dc->debug.min_prefetch_in_strobe_ns / 1000.0; + /* Override dispclk_dppclk_vco_speed_mhz from Clk Mgr */ dcn3_21_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0; diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h index d7be01ac07514..64d602e6412f1 100644 --- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h +++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h @@ -367,6 +367,7 @@ struct _vcs_dpi_ip_params_st { /* DM workarounds */ double dsc_delay_factor_wa; // TODO: Remove after implementing root cause fix + double min_prefetch_in_strobe_us; }; struct _vcs_dpi_display_xfc_params_st { -- GitLab From 30b8e7b8ee3be003e0df85c857c5cd0e0bd58b82 Mon Sep 17 00:00:00 2001 From: Asher Song Date: Thu, 3 Nov 2022 18:28:40 +0800 Subject: [PATCH 1513/1927] Revert "drm/amdgpu: Revert "drm/amdgpu: getting fan speed pwm for vega10 properly"" This reverts commit 4545ae2ed3f2f7c3f615a53399c9c8460ee5bca7. The origin patch "drm/amdgpu: getting fan speed pwm for vega10 properly" works fine. Test failure is caused by test case self. Signed-off-by: Asher Song Reviewed-by: Guchun Chen Signed-off-by: Alex Deucher --- .../amd/pm/powerplay/hwmgr/vega10_thermal.c | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c index dad3e3741a4e8..190af79f3236f 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c @@ -67,22 +67,21 @@ int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr, int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr, uint32_t *speed) { - uint32_t current_rpm; - uint32_t percent = 0; - - if (hwmgr->thermal_controller.fanInfo.bNoFan) - return 0; + struct amdgpu_device *adev = hwmgr->adev; + uint32_t duty100, duty; + uint64_t tmp64; - if (vega10_get_current_rpm(hwmgr, ¤t_rpm)) - return -1; + duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1), + CG_FDO_CTRL1, FMAX_DUTY100); + duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS), + CG_THERMAL_STATUS, FDO_PWM_DUTY); - if (hwmgr->thermal_controller. - advanceFanControlParameters.usMaxFanRPM != 0) - percent = current_rpm * 255 / - hwmgr->thermal_controller. - advanceFanControlParameters.usMaxFanRPM; + if (!duty100) + return -EINVAL; - *speed = MIN(percent, 255); + tmp64 = (uint64_t)duty * 255; + do_div(tmp64, duty100); + *speed = MIN((uint32_t)tmp64, 255); return 0; } -- GitLab From 64f65135c41a75f933d3bca236417ad8e9eb75de Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Thu, 3 Nov 2022 10:24:52 -0400 Subject: [PATCH 1514/1927] drm/amdgpu: Unlock bo_list_mutex after error handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Get below kernel WARNING backtrace when pressing ctrl-C to kill kfdtest application. If amdgpu_cs_parser_bos returns error after taking bo_list_mutex, as caller amdgpu_cs_ioctl will not unlock bo_list_mutex, this generates the kernel WARNING. Add unlock bo_list_mutex after amdgpu_cs_parser_bos error handling to cleanup bo_list userptr bo. WARNING: kfdtest/2930 still has locks held! 1 lock held by kfdtest/2930: (&list->bo_list_mutex){+.+.}-{3:3}, at: amdgpu_cs_ioctl+0xce5/0x1f10 [amdgpu] stack backtrace: dump_stack_lvl+0x44/0x57 get_signal+0x79f/0xd00 arch_do_signal_or_restart+0x36/0x7b0 exit_to_user_mode_prepare+0xfd/0x1b0 syscall_exit_to_user_mode+0x19/0x40 do_syscall_64+0x40/0x80 Signed-off-by: Philip Yang Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 1bbd39b3b0fc4..d371000a5727e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -992,6 +992,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, kvfree(e->user_pages); e->user_pages = NULL; } + mutex_unlock(&p->bo_list->bo_list_mutex); return r; } -- GitLab From e034a0d9aaee5c9129d5dfdfdfcab988a953412d Mon Sep 17 00:00:00 2001 From: Philip Yang Date: Wed, 2 Nov 2022 16:55:31 -0400 Subject: [PATCH 1515/1927] drm/amdgpu: Drop eviction lock when allocating PT BO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-take the eviction lock immediately again after the allocation is completed, to fix circular locking warning with drm_buddy allocator. Move amdgpu_vm_eviction_lock/unlock/trylock to amdgpu_vm.h as they are called from multiple files. Signed-off-by: Philip Yang Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 26 ----------------------- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 26 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 2 ++ 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index 2291aa14d888c..003aa9e47085e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -143,32 +143,6 @@ int amdgpu_vm_set_pasid(struct amdgpu_device *adev, struct amdgpu_vm *vm, return 0; } -/* - * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS - * happens while holding this lock anywhere to prevent deadlocks when - * an MMU notifier runs in reclaim-FS context. - */ -static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) -{ - mutex_lock(&vm->eviction_lock); - vm->saved_flags = memalloc_noreclaim_save(); -} - -static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) -{ - if (mutex_trylock(&vm->eviction_lock)) { - vm->saved_flags = memalloc_noreclaim_save(); - return 1; - } - return 0; -} - -static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) -{ - memalloc_noreclaim_restore(vm->saved_flags); - mutex_unlock(&vm->eviction_lock); -} - /** * amdgpu_vm_bo_evicted - vm_bo is evicted * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h index 1d31771b42308..6546e786bf008 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h @@ -510,4 +510,30 @@ static inline uint64_t amdgpu_vm_tlb_seq(struct amdgpu_vm *vm) return atomic64_read(&vm->tlb_seq); } +/* + * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS + * happens while holding this lock anywhere to prevent deadlocks when + * an MMU notifier runs in reclaim-FS context. + */ +static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) +{ + mutex_lock(&vm->eviction_lock); + vm->saved_flags = memalloc_noreclaim_save(); +} + +static inline bool amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) +{ + if (mutex_trylock(&vm->eviction_lock)) { + vm->saved_flags = memalloc_noreclaim_save(); + return true; + } + return false; +} + +static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) +{ + memalloc_noreclaim_restore(vm->saved_flags); + mutex_unlock(&vm->eviction_lock); +} + #endif diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c index 358b91243e37b..b5f3bba851db8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c @@ -597,7 +597,9 @@ static int amdgpu_vm_pt_alloc(struct amdgpu_device *adev, if (entry->bo) return 0; + amdgpu_vm_eviction_unlock(vm); r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt); + amdgpu_vm_eviction_lock(vm); if (r) return r; -- GitLab From 0c85c067c9d9d7a1b2cc2e01a236d5d0d4a872b5 Mon Sep 17 00:00:00 2001 From: Guchun Chen Date: Mon, 7 Nov 2022 16:46:59 +0800 Subject: [PATCH 1516/1927] drm/amdgpu: disable BACO on special BEIGE_GOBY card Still avoid intermittent failure. Signed-off-by: Guchun Chen Reviewed-by: Lijo Lazar Acked-by: Evan Quan Signed-off-by: Alex Deucher Cc: stable@vger.kernel.org --- drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index 74996a8fb6712..6212fd270857c 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -377,7 +377,9 @@ static void sienna_cichlid_check_bxco_support(struct smu_context *smu) if (((adev->pdev->device == 0x73A1) && (adev->pdev->revision == 0x00)) || ((adev->pdev->device == 0x73BF) && - (adev->pdev->revision == 0xCF))) + (adev->pdev->revision == 0xCF)) || + ((adev->pdev->device == 0x7422) && + (adev->pdev->revision == 0x00))) smu_baco->platform_support = false; } -- GitLab From 675d84621a24490e1de3d59a4992a17fa9ff92b5 Mon Sep 17 00:00:00 2001 From: Hamza Mahfooz Date: Wed, 9 Nov 2022 12:06:27 -0500 Subject: [PATCH 1517/1927] drm/amd/display: only fill dirty rectangles when PSR is enabled Currently, we are calling fill_dc_dirty_rects() even if PSR isn't supported by the relevant link in amdgpu_dm_commit_planes(), this is undesirable especially because when drm.debug is enabled we are printing messages in fill_dc_dirty_rects() that are only useful for debugging PSR (and confusing otherwise). So, we can instead limit the filling of dirty rectangles to only when PSR is enabled. Reviewed-by: Leo Li Signed-off-by: Hamza Mahfooz Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 589bee9acf162..509739d83b5af 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -7622,9 +7622,10 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bundle->surface_updates[planes_count].plane_info = &bundle->plane_infos[planes_count]; - fill_dc_dirty_rects(plane, old_plane_state, new_plane_state, - new_crtc_state, - &bundle->flip_addrs[planes_count]); + if (acrtc_state->stream->link->psr_settings.psr_feature_enabled) + fill_dc_dirty_rects(plane, old_plane_state, + new_plane_state, new_crtc_state, + &bundle->flip_addrs[planes_count]); /* * Only allow immediate flips for fast updates that don't -- GitLab From 39bfcb8138f6dc3375f23b1e62ccfc7c0d83295d Mon Sep 17 00:00:00 2001 From: Jingbo Xu Date: Fri, 21 Oct 2022 10:31:53 +0800 Subject: [PATCH 1518/1927] erofs: fix use-after-free of fsid and domain_id string When erofs instance is remounted with fsid or domain_id mount option specified, the original fsid and domain_id string pointer in sbi->opt is directly overridden with the fsid and domain_id string in the new fs_context, without freeing the original fsid and domain_id string. What's worse, when the new fsid and domain_id string is transferred to sbi, they are not reset to NULL in fs_context, and thus they are freed when remount finishes, while sbi is still referring to these strings. Reconfiguration for fsid and domain_id seems unusual. Thus clarify this restriction explicitly and dump a warning when users are attempting to do this. Besides, to fix the use-after-free issue, move fsid and domain_id from erofs_mount_opts to outside. Fixes: c6be2bd0a5dd ("erofs: register fscache volume") Fixes: 8b7adf1dff3d ("erofs: introduce fscache-based domain") Signed-off-by: Jingbo Xu Reviewed-by: Jia Zhu Reviewed-by: Chao Yu Link: https://lore.kernel.org/r/20221021023153.1330-1-jefflexu@linux.alibaba.com Signed-off-by: Gao Xiang --- fs/erofs/fscache.c | 14 +++++++------- fs/erofs/internal.h | 6 ++++-- fs/erofs/super.c | 39 ++++++++++++++++++++++----------------- fs/erofs/sysfs.c | 8 ++++---- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index 260fa4737fc00..6eaf4a4ab95ca 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -406,13 +406,13 @@ static void erofs_fscache_domain_put(struct erofs_domain *domain) static int erofs_fscache_register_volume(struct super_block *sb) { struct erofs_sb_info *sbi = EROFS_SB(sb); - char *domain_id = sbi->opt.domain_id; + char *domain_id = sbi->domain_id; struct fscache_volume *volume; char *name; int ret = 0; name = kasprintf(GFP_KERNEL, "erofs,%s", - domain_id ? domain_id : sbi->opt.fsid); + domain_id ? domain_id : sbi->fsid); if (!name) return -ENOMEM; @@ -438,7 +438,7 @@ static int erofs_fscache_init_domain(struct super_block *sb) if (!domain) return -ENOMEM; - domain->domain_id = kstrdup(sbi->opt.domain_id, GFP_KERNEL); + domain->domain_id = kstrdup(sbi->domain_id, GFP_KERNEL); if (!domain->domain_id) { kfree(domain); return -ENOMEM; @@ -475,7 +475,7 @@ static int erofs_fscache_register_domain(struct super_block *sb) mutex_lock(&erofs_domain_list_lock); list_for_each_entry(domain, &erofs_domain_list, list) { - if (!strcmp(domain->domain_id, sbi->opt.domain_id)) { + if (!strcmp(domain->domain_id, sbi->domain_id)) { sbi->domain = domain; sbi->volume = domain->volume; refcount_inc(&domain->ref); @@ -612,7 +612,7 @@ struct erofs_fscache *erofs_domain_register_cookie(struct super_block *sb, struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb, char *name, bool need_inode) { - if (EROFS_SB(sb)->opt.domain_id) + if (EROFS_SB(sb)->domain_id) return erofs_domain_register_cookie(sb, name, need_inode); return erofs_fscache_acquire_cookie(sb, name, need_inode); } @@ -644,7 +644,7 @@ int erofs_fscache_register_fs(struct super_block *sb) struct erofs_sb_info *sbi = EROFS_SB(sb); struct erofs_fscache *fscache; - if (sbi->opt.domain_id) + if (sbi->domain_id) ret = erofs_fscache_register_domain(sb); else ret = erofs_fscache_register_volume(sb); @@ -652,7 +652,7 @@ int erofs_fscache_register_fs(struct super_block *sb) return ret; /* acquired domain/volume will be relinquished in kill_sb() on error */ - fscache = erofs_fscache_register_cookie(sb, sbi->opt.fsid, true); + fscache = erofs_fscache_register_cookie(sb, sbi->fsid, true); if (IS_ERR(fscache)) return PTR_ERR(fscache); diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 1701df48c4464..05dc686277220 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -75,8 +75,6 @@ struct erofs_mount_opts { unsigned int max_sync_decompress_pages; #endif unsigned int mount_opt; - char *fsid; - char *domain_id; }; struct erofs_dev_context { @@ -89,6 +87,8 @@ struct erofs_dev_context { struct erofs_fs_context { struct erofs_mount_opts opt; struct erofs_dev_context *devs; + char *fsid; + char *domain_id; }; /* all filesystem-wide lz4 configurations */ @@ -170,6 +170,8 @@ struct erofs_sb_info { struct fscache_volume *volume; struct erofs_fscache *s_fscache; struct erofs_domain *domain; + char *fsid; + char *domain_id; }; #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 2cf96ce1c32ee..1c7dcca702b3e 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -579,9 +579,9 @@ static int erofs_fc_parse_param(struct fs_context *fc, break; case Opt_fsid: #ifdef CONFIG_EROFS_FS_ONDEMAND - kfree(ctx->opt.fsid); - ctx->opt.fsid = kstrdup(param->string, GFP_KERNEL); - if (!ctx->opt.fsid) + kfree(ctx->fsid); + ctx->fsid = kstrdup(param->string, GFP_KERNEL); + if (!ctx->fsid) return -ENOMEM; #else errorfc(fc, "fsid option not supported"); @@ -589,9 +589,9 @@ static int erofs_fc_parse_param(struct fs_context *fc, break; case Opt_domain_id: #ifdef CONFIG_EROFS_FS_ONDEMAND - kfree(ctx->opt.domain_id); - ctx->opt.domain_id = kstrdup(param->string, GFP_KERNEL); - if (!ctx->opt.domain_id) + kfree(ctx->domain_id); + ctx->domain_id = kstrdup(param->string, GFP_KERNEL); + if (!ctx->domain_id) return -ENOMEM; #else errorfc(fc, "domain_id option not supported"); @@ -728,10 +728,12 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_fs_info = sbi; sbi->opt = ctx->opt; - ctx->opt.fsid = NULL; - ctx->opt.domain_id = NULL; sbi->devs = ctx->devs; ctx->devs = NULL; + sbi->fsid = ctx->fsid; + ctx->fsid = NULL; + sbi->domain_id = ctx->domain_id; + ctx->domain_id = NULL; if (erofs_is_fscache_mode(sb)) { sb->s_blocksize = EROFS_BLKSIZ; @@ -820,7 +822,7 @@ static int erofs_fc_get_tree(struct fs_context *fc) { struct erofs_fs_context *ctx = fc->fs_private; - if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && ctx->opt.fsid) + if (IS_ENABLED(CONFIG_EROFS_FS_ONDEMAND) && ctx->fsid) return get_tree_nodev(fc, erofs_fc_fill_super); return get_tree_bdev(fc, erofs_fc_fill_super); @@ -834,6 +836,9 @@ static int erofs_fc_reconfigure(struct fs_context *fc) DBG_BUGON(!sb_rdonly(sb)); + if (ctx->fsid || ctx->domain_id) + erofs_info(sb, "ignoring reconfiguration for fsid|domain_id."); + if (test_opt(&ctx->opt, POSIX_ACL)) fc->sb_flags |= SB_POSIXACL; else @@ -873,8 +878,8 @@ static void erofs_fc_free(struct fs_context *fc) struct erofs_fs_context *ctx = fc->fs_private; erofs_free_dev_context(ctx->devs); - kfree(ctx->opt.fsid); - kfree(ctx->opt.domain_id); + kfree(ctx->fsid); + kfree(ctx->domain_id); kfree(ctx); } @@ -944,8 +949,8 @@ static void erofs_kill_sb(struct super_block *sb) erofs_free_dev_context(sbi->devs); fs_put_dax(sbi->dax_dev, NULL); erofs_fscache_unregister_fs(sb); - kfree(sbi->opt.fsid); - kfree(sbi->opt.domain_id); + kfree(sbi->fsid); + kfree(sbi->domain_id); kfree(sbi); sb->s_fs_info = NULL; } @@ -1098,10 +1103,10 @@ static int erofs_show_options(struct seq_file *seq, struct dentry *root) if (test_opt(opt, DAX_NEVER)) seq_puts(seq, ",dax=never"); #ifdef CONFIG_EROFS_FS_ONDEMAND - if (opt->fsid) - seq_printf(seq, ",fsid=%s", opt->fsid); - if (opt->domain_id) - seq_printf(seq, ",domain_id=%s", opt->domain_id); + if (sbi->fsid) + seq_printf(seq, ",fsid=%s", sbi->fsid); + if (sbi->domain_id) + seq_printf(seq, ",domain_id=%s", sbi->domain_id); #endif return 0; } diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c index 783bb7b21b514..fd476961f742f 100644 --- a/fs/erofs/sysfs.c +++ b/fs/erofs/sysfs.c @@ -210,14 +210,14 @@ int erofs_register_sysfs(struct super_block *sb) int err; if (erofs_is_fscache_mode(sb)) { - if (sbi->opt.domain_id) { - str = kasprintf(GFP_KERNEL, "%s,%s", sbi->opt.domain_id, - sbi->opt.fsid); + if (sbi->domain_id) { + str = kasprintf(GFP_KERNEL, "%s,%s", sbi->domain_id, + sbi->fsid); if (!str) return -ENOMEM; name = str; } else { - name = sbi->opt.fsid; + name = sbi->fsid; } } else { name = sb->s_id; -- GitLab From 6d47b53fb3f363a74538a1dbd09954af3d8d4131 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 9 Nov 2022 09:15:37 +0800 Subject: [PATCH 1519/1927] net: cpsw: disable napi in cpsw_ndo_open() When failed to create xdp rxqs or fill rx channels in cpsw_ndo_open() for opening device, napi isn't disabled. When open cpsw device next time, it will report a invalid opcode issue. Compiled tested only. Fixes: d354eb85d618 ("drivers: net: cpsw: dual_emac: simplify napi usage") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221109011537.96975-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/ti/cpsw.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index 709ca6dd6ecb8..13c9c2d6b79bb 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -854,6 +854,8 @@ static int cpsw_ndo_open(struct net_device *ndev) err_cleanup: if (!cpsw->usage_count) { + napi_disable(&cpsw->napi_rx); + napi_disable(&cpsw->napi_tx); cpdma_ctlr_stop(cpsw->dma); cpsw_destroy_xdp_rxqs(cpsw); } -- GitLab From d75aed1428da787cbe42bc073d76f1354f364d92 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 9 Nov 2022 10:14:51 +0800 Subject: [PATCH 1520/1927] net: cxgb3_main: disable napi when bind qsets failed in cxgb_up() When failed to bind qsets in cxgb_up() for opening device, napi isn't disabled. When open cxgb3 device next time, it will trigger a BUG_ON() in napi_enable(). Compile tested only. Fixes: 48c4b6dbb7e2 ("cxgb3 - fix port up/down error path") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221109021451.121490-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index a52e6b6e28767..9b84c8d8d3097 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -1301,6 +1301,7 @@ static int cxgb_up(struct adapter *adap) if (ret < 0) { CH_ERR(adap, "failed to bind qsets, err %d\n", ret); t3_intr_disable(adap); + quiesce_rx(adap); free_irq_resources(adap); err = ret; goto out; -- GitLab From dcea1a8107c04b9521dee1dd37971757a22db162 Mon Sep 17 00:00:00 2001 From: "Tan, Tee Min" Date: Mon, 7 Nov 2022 21:08:11 -0500 Subject: [PATCH 1521/1927] stmmac: intel: Update PCH PTP clock rate from 200MHz to 204.8MHz Current Intel platform has an output of ~976ms interval when probed on 1 Pulse-per-Second(PPS) hardware pin. The correct PTP clock frequency for PCH GbE should be 204.8MHz instead of 200MHz. PSE GbE PTP clock rate remains at 200MHz. Fixes: 58da0cfa6cf1 ("net: stmmac: create dwmac-intel.c to contain all Intel platform") Signed-off-by: Ling Pei Lee Signed-off-by: Tan, Tee Min Signed-off-by: Voon Weifeng Signed-off-by: Gan Yi Fang Link: https://lore.kernel.org/r/20221108020811.12919-1-yi.fang.gan@intel.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 0a2afc1a3124e..7deb1f817dacc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -629,7 +629,6 @@ static int ehl_common_data(struct pci_dev *pdev, { plat->rx_queues_to_use = 8; plat->tx_queues_to_use = 8; - plat->clk_ptp_rate = 200000000; plat->use_phy_wol = 1; plat->safety_feat_cfg->tsoee = 1; @@ -654,6 +653,8 @@ static int ehl_sgmii_data(struct pci_dev *pdev, plat->serdes_powerup = intel_serdes_powerup; plat->serdes_powerdown = intel_serdes_powerdown; + plat->clk_ptp_rate = 204800000; + return ehl_common_data(pdev, plat); } @@ -667,6 +668,8 @@ static int ehl_rgmii_data(struct pci_dev *pdev, plat->bus_id = 1; plat->phy_interface = PHY_INTERFACE_MODE_RGMII; + plat->clk_ptp_rate = 204800000; + return ehl_common_data(pdev, plat); } @@ -683,6 +686,8 @@ static int ehl_pse0_common_data(struct pci_dev *pdev, plat->bus_id = 2; plat->addr64 = 32; + plat->clk_ptp_rate = 200000000; + intel_mgbe_pse_crossts_adj(intel_priv, EHL_PSE_ART_MHZ); return ehl_common_data(pdev, plat); @@ -722,6 +727,8 @@ static int ehl_pse1_common_data(struct pci_dev *pdev, plat->bus_id = 3; plat->addr64 = 32; + plat->clk_ptp_rate = 200000000; + intel_mgbe_pse_crossts_adj(intel_priv, EHL_PSE_ART_MHZ); return ehl_common_data(pdev, plat); @@ -757,7 +764,7 @@ static int tgl_common_data(struct pci_dev *pdev, { plat->rx_queues_to_use = 6; plat->tx_queues_to_use = 4; - plat->clk_ptp_rate = 200000000; + plat->clk_ptp_rate = 204800000; plat->speed_mode_2500 = intel_speed_mode_2500; plat->safety_feat_cfg->tsoee = 1; -- GitLab From d4072058af4fd8fb4658e7452289042a406a9398 Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Tue, 8 Nov 2022 09:55:17 +0000 Subject: [PATCH 1522/1927] mctp: Fix an error handling path in mctp_init() If mctp_neigh_init() return error, the routes resources should be released in the error handling path. Otherwise some resources leak. Fixes: 4d8b9319282a ("mctp: Add neighbour implementation") Signed-off-by: Wei Yongjun Acked-by: Matt Johnston Link: https://lore.kernel.org/r/20221108095517.620115-1-weiyongjun@huaweicloud.com Signed-off-by: Jakub Kicinski --- net/mctp/af_mctp.c | 4 +++- net/mctp/route.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c index b6b5e496fa403..fc9e728b6333a 100644 --- a/net/mctp/af_mctp.c +++ b/net/mctp/af_mctp.c @@ -665,12 +665,14 @@ static __init int mctp_init(void) rc = mctp_neigh_init(); if (rc) - goto err_unreg_proto; + goto err_unreg_routes; mctp_device_init(); return 0; +err_unreg_routes: + mctp_routes_exit(); err_unreg_proto: proto_unregister(&mctp_proto); err_unreg_sock: diff --git a/net/mctp/route.c b/net/mctp/route.c index 2155f15a074cd..f9a80b82dc511 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -1400,7 +1400,7 @@ int __init mctp_routes_init(void) return register_pernet_subsys(&mctp_net_ops); } -void __exit mctp_routes_exit(void) +void mctp_routes_exit(void) { unregister_pernet_subsys(&mctp_net_ops); rtnl_unregister(PF_MCTP, RTM_DELROUTE); -- GitLab From c6092ea1e6d7bd12acd881f6aa2b5054cd70e096 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 9 Nov 2022 09:21:00 +0800 Subject: [PATCH 1523/1927] cxgb4vf: shut down the adapter when t4vf_update_port_info() failed in cxgb4vf_open() When t4vf_update_port_info() failed in cxgb4vf_open(), resources applied during adapter goes up are not cleared. Fix it. Only be compiled, not be tested. Fixes: 18d79f721e0a ("cxgb4vf: Update port information in cxgb4vf_open()") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221109012100.99132-1-shaozhengchao@huawei.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index 54db79f4dcfe0..63b2bd0841305 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -858,7 +858,7 @@ static int cxgb4vf_open(struct net_device *dev) */ err = t4vf_update_port_info(pi); if (err < 0) - return err; + goto err_unwind; /* * Note that this interface is up and start everything up ... -- GitLab From 79e28f2ab3440e08f5fbf65648b008341c37b496 Mon Sep 17 00:00:00 2001 From: Edson Juliano Drosdeck Date: Wed, 9 Nov 2022 13:17:32 -0400 Subject: [PATCH 1524/1927] ALSA: hda/realtek: Add Positivo C6300 model quirk Positivo Master C6300 (1849:a233) require quirk for anabling headset-mic Signed-off-by: Edson Juliano Drosdeck Cc: Link: https://lore.kernel.org/r/20221109171732.5417-1-edson.drosdeck@gmail.com Signed-off-by: Takashi Iwai --- sound/pci/hda/patch_realtek.c | 1 + 1 file changed, 1 insertion(+) diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index b4f7ff8cfe41b..e18499dd14f08 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -9609,6 +9609,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x511f, "Thinkpad", ALC298_FIXUP_TPT470_DOCK), SND_PCI_QUIRK(0x17aa, 0x9e54, "LENOVO NB", ALC269_FIXUP_LENOVO_EAPD), SND_PCI_QUIRK(0x1849, 0x1233, "ASRock NUC Box 1100", ALC233_FIXUP_NO_AUDIO_JACK), + SND_PCI_QUIRK(0x1849, 0xa233, "Positivo Master C6300", ALC269_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1b35, 0x1235, "CZC B20", ALC269_FIXUP_CZC_B20), -- GitLab From bf990c10231937c0f51e5da5558e08cf5adc6a78 Mon Sep 17 00:00:00 2001 From: Ai Chao Date: Thu, 10 Nov 2022 14:34:52 +0800 Subject: [PATCH 1525/1927] ALSA: usb-audio: add quirk to fix Hamedal C20 disconnect issue For Hamedal C20, the current rate is different from the runtime rate, snd_usb_endpoint stop and close endpoint to resetting rate. if snd_usb_endpoint close the endpoint, sometimes usb will disconnect the device. Signed-off-by: Ai Chao Link: https://lore.kernel.org/r/20221110063452.295110-1-aichao@kylinos.cn Signed-off-by: Takashi Iwai --- sound/usb/endpoint.c | 3 ++- sound/usb/quirks.c | 2 ++ sound/usb/usbaudio.h | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index d0b8d61d1d22b..310cd6fb0038a 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -931,7 +931,8 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip, usb_audio_dbg(chip, "Closing EP 0x%x (count %d)\n", ep->ep_num, ep->opened); - if (!--ep->iface_ref->opened) + if (!--ep->iface_ref->opened && + !(chip->quirk_flags & QUIRK_FLAG_IFACE_SKIP_CLOSE)) endpoint_set_interface(chip, ep, false); if (!--ep->opened) { diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index aedc3987a4cf2..0f4dd3503a6a9 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -2148,6 +2148,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x2b53, 0x0031, /* Fiero SC-01 (firmware v1.1.0) */ QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x0525, 0xa4ad, /* Hamedal C20 usb camero */ + QUIRK_FLAG_IFACE_SKIP_CLOSE), /* Vendor matches */ VENDOR_FLG(0x045e, /* MS Lifecam */ diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 2c6575029b1cd..e97141ef730ad 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -170,6 +170,8 @@ extern bool snd_usb_skip_validation; * Apply the generic implicit feedback sync mode (same as implicit_fb=1 option) * QUIRK_FLAG_SKIP_IMPLICIT_FB * Don't apply implicit feedback sync mode + * QUIRK_FLAG_IFACE_SKIP_CLOSE + * Don't closed interface during setting sample rate */ #define QUIRK_FLAG_GET_SAMPLE_RATE (1U << 0) @@ -191,5 +193,6 @@ extern bool snd_usb_skip_validation; #define QUIRK_FLAG_SET_IFACE_FIRST (1U << 16) #define QUIRK_FLAG_GENERIC_IMPLICIT_FB (1U << 17) #define QUIRK_FLAG_SKIP_IMPLICIT_FB (1U << 18) +#define QUIRK_FLAG_IFACE_SKIP_CLOSE (1U << 19) #endif /* __USBAUDIO_H */ -- GitLab From 764afecbbcd26bea57fcd928680d413b2c851590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Wed, 26 Oct 2022 14:39:06 +0300 Subject: [PATCH 1526/1927] drm/i915: Create resized LUTs for ivb+ split gamma mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently when opeating in split gamma mode we do the "skip ever other sw LUT entry" trick in the low level LUT programming/readout functions. That is very annoying and a big hinderance to revamping the color management uapi. Let's get rid of that problem by making half sized copies of the software LUTs and plugging those into the internal {pre,post}_csc_lut attachment points (instead of the sticking the uapi provide sw LUTs there directly). With this the low level stuff will operate purely in terms the hardware LUT sizes, and all uapi nonsense is contained to the atomic check phase. The one thing we do lose is intel_color_assert_luts() since we no longer have a way to check that the uapi LUTs were correctly used when generating the internal copies. But that seems like a price worth paying. Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/20221026113906.10551-12-ville.syrjala@linux.intel.com Reviewed-by: Ankit Nautiyal --- drivers/gpu/drm/i915/display/intel_color.c | 81 +++++++++++++++++----- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 458e69578da6e..93509cf7bbcc6 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -597,6 +597,30 @@ create_linear_lut(struct drm_i915_private *i915, int lut_size) return blob; } +static struct drm_property_blob * +create_resized_lut(struct drm_i915_private *i915, + const struct drm_property_blob *blob_in, int lut_out_size) +{ + int i, lut_in_size = drm_color_lut_size(blob_in); + struct drm_property_blob *blob_out; + const struct drm_color_lut *lut_in; + struct drm_color_lut *lut_out; + + blob_out = drm_property_create_blob(&i915->drm, + sizeof(lut_out[0]) * lut_out_size, + NULL); + if (IS_ERR(blob_out)) + return blob_out; + + lut_in = blob_in->data; + lut_out = blob_out->data; + + for (i = 0; i < lut_out_size; i++) + lut_out[i] = lut_in[i * (lut_in_size - 1) / (lut_out_size - 1)]; + + return blob_out; +} + static void i9xx_load_lut_8(struct intel_crtc *crtc, const struct drm_property_blob *blob) { @@ -723,19 +747,14 @@ static void ivb_load_lut_10(struct intel_crtc *crtc, u32 prec_index) { struct drm_i915_private *i915 = to_i915(crtc->base.dev); - int hw_lut_size = ivb_lut_10_size(prec_index); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; - for (i = 0; i < hw_lut_size; i++) { - /* We discard half the user entries in split gamma mode */ - const struct drm_color_lut *entry = - &lut[i * (lut_size - 1) / (hw_lut_size - 1)]; - + for (i = 0; i < lut_size; i++) { intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), prec_index++); intel_de_write_fw(i915, PREC_PAL_DATA(pipe), - ilk_lut_10(entry)); + ilk_lut_10(&lut[i])); } /* @@ -751,7 +770,6 @@ static void bdw_load_lut_10(struct intel_crtc *crtc, u32 prec_index) { struct drm_i915_private *i915 = to_i915(crtc->base.dev); - int hw_lut_size = ivb_lut_10_size(prec_index); const struct drm_color_lut *lut = blob->data; int i, lut_size = drm_color_lut_size(blob); enum pipe pipe = crtc->pipe; @@ -759,14 +777,9 @@ static void bdw_load_lut_10(struct intel_crtc *crtc, intel_de_write_fw(i915, PREC_PAL_INDEX(pipe), prec_index | PAL_PREC_AUTO_INCREMENT); - for (i = 0; i < hw_lut_size; i++) { - /* We discard half the user entries in split gamma mode */ - const struct drm_color_lut *entry = - &lut[i * (lut_size - 1) / (hw_lut_size - 1)]; - + for (i = 0; i < lut_size; i++) intel_de_write_fw(i915, PREC_PAL_DATA(pipe), - ilk_lut_10(entry)); - } + ilk_lut_10(&lut[i])); /* * Reset the index, otherwise it prevents the legacy palette to be @@ -1343,7 +1356,7 @@ void intel_color_assert_luts(const struct intel_crtc_state *crtc_state) crtc_state->pre_csc_lut != i915->display.color.glk_linear_degamma_lut); drm_WARN_ON(&i915->drm, crtc_state->post_csc_lut != crtc_state->hw.gamma_lut); - } else { + } else if (crtc_state->gamma_mode != GAMMA_MODE_MODE_SPLIT) { drm_WARN_ON(&i915->drm, crtc_state->pre_csc_lut != crtc_state->hw.degamma_lut && crtc_state->pre_csc_lut != crtc_state->hw.gamma_lut); @@ -1564,6 +1577,38 @@ static u32 ivb_csc_mode(const struct intel_crtc_state *crtc_state) return CSC_POSITION_BEFORE_GAMMA; } +static int ivb_assign_luts(struct intel_crtc_state *crtc_state) +{ + struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); + struct drm_property_blob *degamma_lut, *gamma_lut; + + if (crtc_state->gamma_mode != GAMMA_MODE_MODE_SPLIT) { + ilk_assign_luts(crtc_state); + return 0; + } + + drm_WARN_ON(&i915->drm, drm_color_lut_size(crtc_state->hw.degamma_lut) != 1024); + drm_WARN_ON(&i915->drm, drm_color_lut_size(crtc_state->hw.gamma_lut) != 1024); + + degamma_lut = create_resized_lut(i915, crtc_state->hw.degamma_lut, 512); + if (IS_ERR(degamma_lut)) + return PTR_ERR(degamma_lut); + + gamma_lut = create_resized_lut(i915, crtc_state->hw.gamma_lut, 512); + if (IS_ERR(gamma_lut)) { + drm_property_blob_put(degamma_lut); + return PTR_ERR(gamma_lut); + } + + drm_property_replace_blob(&crtc_state->pre_csc_lut, degamma_lut); + drm_property_replace_blob(&crtc_state->post_csc_lut, gamma_lut); + + drm_property_blob_put(degamma_lut); + drm_property_blob_put(gamma_lut); + + return 0; +} + static int ivb_color_check(struct intel_crtc_state *crtc_state) { struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev); @@ -1599,7 +1644,9 @@ static int ivb_color_check(struct intel_crtc_state *crtc_state) if (ret) return ret; - ilk_assign_luts(crtc_state); + ret = ivb_assign_luts(crtc_state); + if (ret) + return ret; crtc_state->preload_luts = intel_can_preload_luts(crtc_state); -- GitLab From f2d45fdf9a0ed2c94c01c422a0d0add8ffd42099 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 19:46:45 +0800 Subject: [PATCH 1527/1927] stmmac: dwmac-loongson: fix missing pci_disable_msi() while module exiting pci_enable_msi() has been called in loongson_dwmac_probe(), so pci_disable_msi() needs be called in remove path and error path of probe(). Fixes: 30bba69d7db4 ("stmmac: pci: Add dwmac support for Loongson") Signed-off-by: Yang Yingliang Signed-off-by: Paolo Abeni --- .../net/ethernet/stmicro/stmmac/dwmac-loongson.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 79fa7870563b8..16915b4d95057 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -125,6 +125,7 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id if (res.irq < 0) { dev_err(&pdev->dev, "IRQ macirq not found\n"); ret = -ENODEV; + goto err_disable_msi; } res.wol_irq = of_irq_get_byname(np, "eth_wake_irq"); @@ -137,9 +138,18 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id if (res.lpi_irq < 0) { dev_err(&pdev->dev, "IRQ eth_lpi not found\n"); ret = -ENODEV; + goto err_disable_msi; } - return stmmac_dvr_probe(&pdev->dev, plat, &res); + ret = stmmac_dvr_probe(&pdev->dev, plat, &res); + if (ret) + goto err_disable_msi; + + return ret; + +err_disable_msi: + pci_disable_msi(pdev); + return ret; } static void loongson_dwmac_remove(struct pci_dev *pdev) @@ -155,6 +165,7 @@ static void loongson_dwmac_remove(struct pci_dev *pdev) break; } + pci_disable_msi(pdev); pci_disable_device(pdev); } -- GitLab From fe5b3ce8b4377e543960220f539b989a927afd8a Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 19:46:46 +0800 Subject: [PATCH 1528/1927] stmmac: dwmac-loongson: fix missing pci_disable_device() in loongson_dwmac_probe() Add missing pci_disable_device() in the error path in loongson_dwmac_probe(). Fixes: 30bba69d7db4 ("stmmac: pci: Add dwmac support for Loongson") Signed-off-by: Yang Yingliang Signed-off-by: Paolo Abeni --- drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 16915b4d95057..2d480bc49c51d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -97,7 +97,7 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id continue; ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev)); if (ret) - return ret; + goto err_disable_device; break; } @@ -108,7 +108,8 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id phy_mode = device_get_phy_mode(&pdev->dev); if (phy_mode < 0) { dev_err(&pdev->dev, "phy_mode not found\n"); - return phy_mode; + ret = phy_mode; + goto err_disable_device; } plat->phy_interface = phy_mode; @@ -149,6 +150,8 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id err_disable_msi: pci_disable_msi(pdev); +err_disable_device: + pci_disable_device(pdev); return ret; } -- GitLab From 7f94d0498f9c763f37172c08059ae91804c3075a Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 19:46:47 +0800 Subject: [PATCH 1529/1927] stmmac: dwmac-loongson: fix missing of_node_put() while module exiting The node returned by of_get_child_by_name() with refcount decremented, of_node_put() needs be called when finish using it. So add it in the error path in loongson_dwmac_probe() and in loongson_dwmac_remove(). Fixes: 2ae34111fe4e ("stmmac: dwmac-loongson: fix invalid mdio_node") Signed-off-by: Yang Yingliang Signed-off-by: Paolo Abeni --- .../ethernet/stmicro/stmmac/dwmac-loongson.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 2d480bc49c51d..a25c187d31853 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -75,20 +75,24 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id plat->mdio_bus_data = devm_kzalloc(&pdev->dev, sizeof(*plat->mdio_bus_data), GFP_KERNEL); - if (!plat->mdio_bus_data) - return -ENOMEM; + if (!plat->mdio_bus_data) { + ret = -ENOMEM; + goto err_put_node; + } plat->mdio_bus_data->needs_reset = true; } plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), GFP_KERNEL); - if (!plat->dma_cfg) - return -ENOMEM; + if (!plat->dma_cfg) { + ret = -ENOMEM; + goto err_put_node; + } /* Enable pci device */ ret = pci_enable_device(pdev); if (ret) { dev_err(&pdev->dev, "%s: ERROR: failed to enable device\n", __func__); - return ret; + goto err_put_node; } /* Get the base address of device */ @@ -152,13 +156,18 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id pci_disable_msi(pdev); err_disable_device: pci_disable_device(pdev); +err_put_node: + of_node_put(plat->mdio_node); return ret; } static void loongson_dwmac_remove(struct pci_dev *pdev) { + struct net_device *ndev = dev_get_drvdata(&pdev->dev); + struct stmmac_priv *priv = netdev_priv(ndev); int i; + of_node_put(priv->plat->mdio_node); stmmac_dvr_remove(&pdev->dev); for (i = 0; i < PCI_STD_NUM_BARS; i++) { -- GitLab From 1b16b3fdf675cca15a537572bac50cc5354368fc Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Tue, 8 Nov 2022 16:34:58 +0100 Subject: [PATCH 1530/1927] net: phy: mscc: macsec: clear encryption keys when freeing a flow Commit aaab73f8fba4 ("macsec: clear encryption keys from the stack after setting up offload") made sure to clean encryption keys from the stack after setting up offloading, but the MSCC PHY driver made a copy, kept it in the flow data and did not clear it when freeing a flow. Fix this. Fixes: 28c5107aa904 ("net: phy: mscc: macsec support") Signed-off-by: Antoine Tenart Signed-off-by: Paolo Abeni --- drivers/net/phy/mscc/mscc_macsec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/phy/mscc/mscc_macsec.c b/drivers/net/phy/mscc/mscc_macsec.c index ee5b17edca39e..f81b077618f40 100644 --- a/drivers/net/phy/mscc/mscc_macsec.c +++ b/drivers/net/phy/mscc/mscc_macsec.c @@ -632,6 +632,7 @@ static void vsc8584_macsec_free_flow(struct vsc8531_private *priv, list_del(&flow->list); clear_bit(flow->index, bitmap); + memzero_explicit(flow->key, sizeof(flow->key)); kfree(flow); } -- GitLab From 879785def0f5e71d54399de0f8a5cb399db14171 Mon Sep 17 00:00:00 2001 From: Antoine Tenart Date: Tue, 8 Nov 2022 16:34:59 +0100 Subject: [PATCH 1531/1927] net: atlantic: macsec: clear encryption keys from the stack Commit aaab73f8fba4 ("macsec: clear encryption keys from the stack after setting up offload") made sure to clean encryption keys from the stack after setting up offloading, but the atlantic driver made a copy and did not clear it. Fix this. [4 Fixes tags below, all part of the same series, no need to split this] Fixes: 9ff40a751a6f ("net: atlantic: MACSec ingress offload implementation") Fixes: b8f8a0b7b5cb ("net: atlantic: MACSec ingress offload HW bindings") Fixes: 27736563ce32 ("net: atlantic: MACSec egress offload implementation") Fixes: 9d106c6dd81b ("net: atlantic: MACSec egress offload HW bindings") Signed-off-by: Antoine Tenart Signed-off-by: Paolo Abeni --- .../net/ethernet/aquantia/atlantic/aq_macsec.c | 2 ++ .../aquantia/atlantic/macsec/macsec_api.c | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c index a0180811305dc..7eb5851eb95dc 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c @@ -570,6 +570,7 @@ static int aq_update_txsa(struct aq_nic_s *nic, const unsigned int sc_idx, ret = aq_mss_set_egress_sakey_record(hw, &key_rec, sa_idx); + memzero_explicit(&key_rec, sizeof(key_rec)); return ret; } @@ -899,6 +900,7 @@ static int aq_update_rxsa(struct aq_nic_s *nic, const unsigned int sc_idx, ret = aq_mss_set_ingress_sakey_record(hw, &sa_key_record, sa_idx); + memzero_explicit(&sa_key_record, sizeof(sa_key_record)); return ret; } diff --git a/drivers/net/ethernet/aquantia/atlantic/macsec/macsec_api.c b/drivers/net/ethernet/aquantia/atlantic/macsec/macsec_api.c index 36c7cf05630a1..4319249595207 100644 --- a/drivers/net/ethernet/aquantia/atlantic/macsec/macsec_api.c +++ b/drivers/net/ethernet/aquantia/atlantic/macsec/macsec_api.c @@ -757,6 +757,7 @@ set_ingress_sakey_record(struct aq_hw_s *hw, u16 table_index) { u16 packed_record[18]; + int ret; if (table_index >= NUMROWS_INGRESSSAKEYRECORD) return -EINVAL; @@ -789,9 +790,12 @@ set_ingress_sakey_record(struct aq_hw_s *hw, packed_record[16] = rec->key_len & 0x3; - return set_raw_ingress_record(hw, packed_record, 18, 2, - ROWOFFSET_INGRESSSAKEYRECORD + - table_index); + ret = set_raw_ingress_record(hw, packed_record, 18, 2, + ROWOFFSET_INGRESSSAKEYRECORD + + table_index); + + memzero_explicit(packed_record, sizeof(packed_record)); + return ret; } int aq_mss_set_ingress_sakey_record(struct aq_hw_s *hw, @@ -1739,14 +1743,14 @@ static int set_egress_sakey_record(struct aq_hw_s *hw, ret = set_raw_egress_record(hw, packed_record, 8, 2, ROWOFFSET_EGRESSSAKEYRECORD + table_index); if (unlikely(ret)) - return ret; + goto clear_key; ret = set_raw_egress_record(hw, packed_record + 8, 8, 2, ROWOFFSET_EGRESSSAKEYRECORD + table_index - 32); - if (unlikely(ret)) - return ret; - return 0; +clear_key: + memzero_explicit(packed_record, sizeof(packed_record)); + return ret; } int aq_mss_set_egress_sakey_record(struct aq_hw_s *hw, -- GitLab From 91d5c5060ee24fe8da88cd585bb43b843d2f0dce Mon Sep 17 00:00:00 2001 From: Zeng Heng Date: Thu, 10 Nov 2022 16:20:56 +0800 Subject: [PATCH 1532/1927] pinctrl: devicetree: fix null pointer dereferencing in pinctrl_dt_to_map Here is the BUG report by KASAN about null pointer dereference: BUG: KASAN: null-ptr-deref in strcmp+0x2e/0x50 Read of size 1 at addr 0000000000000000 by task python3/2640 Call Trace: strcmp __of_find_property of_find_property pinctrl_dt_to_map kasprintf() would return NULL pointer when kmalloc() fail to allocate. So directly return ENOMEM, if kasprintf() return NULL pointer. Fixes: 57291ce295c0 ("pinctrl: core device tree mapping table parsing support") Signed-off-by: Zeng Heng Link: https://lore.kernel.org/r/20221110082056.2014898-1-zengheng4@huawei.com Signed-off-by: Linus Walleij --- drivers/pinctrl/devicetree.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/pinctrl/devicetree.c b/drivers/pinctrl/devicetree.c index ef898ee8ca6bd..6e0a40962f384 100644 --- a/drivers/pinctrl/devicetree.c +++ b/drivers/pinctrl/devicetree.c @@ -220,6 +220,8 @@ int pinctrl_dt_to_map(struct pinctrl *p, struct pinctrl_dev *pctldev) for (state = 0; ; state++) { /* Retrieve the pinctrl-* property */ propname = kasprintf(GFP_KERNEL, "pinctrl-%d", state); + if (!propname) + return -ENOMEM; prop = of_find_property(np, propname, &size); kfree(propname); if (!prop) { -- GitLab From dba9e3467425800f9d3a14e8b6a0f85c731c1650 Mon Sep 17 00:00:00 2001 From: Gaosheng Cui Date: Thu, 10 Nov 2022 17:44:45 +0800 Subject: [PATCH 1533/1927] drm/vc4: kms: Fix IS_ERR() vs NULL check for vc4_kms The drm_atomic_get_new_private_obj_state() function returns NULL on error path, drm_atomic_get_old_private_obj_state() function returns NULL on error path, too, they does not return error pointers. By the way, vc4_hvs_get_new/old_global_state() should return ERR_PTR(-EINVAL), otherwise there will be null-ptr-defer issue, such as follows: In function vc4_atomic_commit_tail(): |-- old_hvs_state = vc4_hvs_get_old_global_state(state); <-- return NULL |-- if (WARN_ON(IS_ERR(old_hvs_state))) <-- no return |-- unsigned long state_rate = max(old_hvs_state->core_clock_rate, new_hvs_state->core_clock_rate); <-- null-ptr-defer Fixes: 9ec03d7f1ed3 ("drm/vc4: kms: Wait on previous FIFO users before a commit") Signed-off-by: Gaosheng Cui Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20221110094445.2930509-6-cuigaosheng1@huawei.com --- drivers/gpu/drm/vc4/vc4_kms.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c index 4419e810103de..0a6347c05df49 100644 --- a/drivers/gpu/drm/vc4/vc4_kms.c +++ b/drivers/gpu/drm/vc4/vc4_kms.c @@ -197,8 +197,8 @@ vc4_hvs_get_new_global_state(struct drm_atomic_state *state) struct drm_private_state *priv_state; priv_state = drm_atomic_get_new_private_obj_state(state, &vc4->hvs_channels); - if (IS_ERR(priv_state)) - return ERR_CAST(priv_state); + if (!priv_state) + return ERR_PTR(-EINVAL); return to_vc4_hvs_state(priv_state); } @@ -210,8 +210,8 @@ vc4_hvs_get_old_global_state(struct drm_atomic_state *state) struct drm_private_state *priv_state; priv_state = drm_atomic_get_old_private_obj_state(state, &vc4->hvs_channels); - if (IS_ERR(priv_state)) - return ERR_CAST(priv_state); + if (!priv_state) + return ERR_PTR(-EINVAL); return to_vc4_hvs_state(priv_state); } -- GitLab From 0348c1ab980c1d43fb37b758d4b760990c066cb5 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 9 Nov 2022 10:37:41 +0800 Subject: [PATCH 1534/1927] ethernet: s2io: disable napi when start nic failed in s2io_card_up() When failed to start nic or add interrupt service routine in s2io_card_up() for opening device, napi isn't disabled. When open s2io device next time, it will trigger a BUG_ON()in napi_enable(). Compile tested only. Fixes: 5f490c968056 ("S2io: Fixed synchronization between scheduling of napi with card reset and close") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221109023741.131552-1-shaozhengchao@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/neterion/s2io.c | 29 +++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c index dcf8212119f9c..1d3c4474b7cb4 100644 --- a/drivers/net/ethernet/neterion/s2io.c +++ b/drivers/net/ethernet/neterion/s2io.c @@ -7128,9 +7128,8 @@ static int s2io_card_up(struct s2io_nic *sp) if (ret) { DBG_PRINT(ERR_DBG, "%s: Out of memory in Open\n", dev->name); - s2io_reset(sp); - free_rx_buffers(sp); - return -ENOMEM; + ret = -ENOMEM; + goto err_fill_buff; } DBG_PRINT(INFO_DBG, "Buf in ring:%d is %d:\n", i, ring->rx_bufs_left); @@ -7168,18 +7167,16 @@ static int s2io_card_up(struct s2io_nic *sp) /* Enable Rx Traffic and interrupts on the NIC */ if (start_nic(sp)) { DBG_PRINT(ERR_DBG, "%s: Starting NIC failed\n", dev->name); - s2io_reset(sp); - free_rx_buffers(sp); - return -ENODEV; + ret = -ENODEV; + goto err_out; } /* Add interrupt service routine */ if (s2io_add_isr(sp) != 0) { if (sp->config.intr_type == MSI_X) s2io_rem_isr(sp); - s2io_reset(sp); - free_rx_buffers(sp); - return -ENODEV; + ret = -ENODEV; + goto err_out; } timer_setup(&sp->alarm_timer, s2io_alarm_handle, 0); @@ -7199,6 +7196,20 @@ static int s2io_card_up(struct s2io_nic *sp) } return 0; + +err_out: + if (config->napi) { + if (config->intr_type == MSI_X) { + for (i = 0; i < sp->config.rx_ring_num; i++) + napi_disable(&sp->mac_control.rings[i].napi); + } else { + napi_disable(&sp->napi); + } + } +err_fill_buff: + s2io_reset(sp); + free_rx_buffers(sp); + return ret; } /** -- GitLab From f111606b63ff2282428ffbac0447c871eb957b6c Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 9 Nov 2022 10:54:32 +0800 Subject: [PATCH 1535/1927] net: mv643xx_eth: disable napi when init rxq or txq failed in mv643xx_eth_open() When failed to init rxq or txq in mv643xx_eth_open() for opening device, napi isn't disabled. When open mv643xx_eth device next time, it will trigger a BUG_ON() in napi_enable(). Compile tested only. Fixes: 2257e05c1705 ("mv643xx_eth: get rid of receive-side locking") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221109025432.80900-1-shaozhengchao@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/marvell/mv643xx_eth.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index 707993b445d1e..8941f69d93e9c 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -2481,6 +2481,7 @@ static int mv643xx_eth_open(struct net_device *dev) for (i = 0; i < mp->rxq_count; i++) rxq_deinit(mp->rxq + i); out: + napi_disable(&mp->napi); free_irq(dev->irq, dev); return err; -- GitLab From a10234fda4664f9ffe9608abe1e99b51815113f5 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Wed, 9 Nov 2022 10:46:33 +0000 Subject: [PATCH 1536/1927] drm/i915: Partial abandonment of legacy DRM logging macros MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert some usages of legacy DRM logging macros into versions which tell us on which device have the events occurred. v2: * Don't have struct drm_device as local. (Jani, Ville) v3: * Store gt, not i915, in workaround list. (John) Signed-off-by: Tvrtko Ursulin Acked-by: Jani Nikula Cc: Jani Nikula Cc: John Harrison Cc: Ville Syrjälä Reviewed-by: Andrzej Hajda Link: https://patchwork.freedesktop.org/patch/msgid/20221109104633.2579245-1-tvrtko.ursulin@linux.intel.com --- drivers/gpu/drm/i915/gem/i915_gem_context.c | 2 +- .../gpu/drm/i915/gem/i915_gem_execbuffer.c | 26 ++++++++---- .../drm/i915/gt/intel_execlists_submission.c | 13 +++--- drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c | 4 +- drivers/gpu/drm/i915/gt/intel_gt.c | 4 +- drivers/gpu/drm/i915/gt/intel_gt_irq.c | 8 ++-- drivers/gpu/drm/i915/gt/intel_rps.c | 6 ++- drivers/gpu/drm/i915/gt/intel_workarounds.c | 42 +++++++++++-------- .../gpu/drm/i915/gt/intel_workarounds_types.h | 3 ++ .../gpu/drm/i915/gt/selftest_workarounds.c | 4 +- drivers/gpu/drm/i915/i915_debugfs.c | 4 +- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/i915_getparam.c | 2 +- drivers/gpu/drm/i915/i915_irq.c | 12 +++--- drivers/gpu/drm/i915/i915_perf.c | 14 ++++--- drivers/gpu/drm/i915/i915_query.c | 12 +++--- drivers/gpu/drm/i915/i915_sysfs.c | 3 +- drivers/gpu/drm/i915/i915_vma.c | 16 +++---- drivers/gpu/drm/i915/intel_uncore.c | 21 ++++++---- 19 files changed, 117 insertions(+), 81 deletions(-) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 01402f3c58f65..7f2831efc798b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -546,7 +546,7 @@ set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data) } if (intel_engine_uses_guc(master)) { - DRM_DEBUG("bonding extension not supported with GuC submission"); + drm_dbg(&i915->drm, "bonding extension not supported with GuC submission"); return -ENODEV; } diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 1160723c9d2d9..f65fd03f7cf24 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -2148,7 +2148,8 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) return err; } -static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) +static int i915_gem_check_execbuffer(struct drm_i915_private *i915, + struct drm_i915_gem_execbuffer2 *exec) { if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS) return -EINVAL; @@ -2161,7 +2162,7 @@ static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) } if (exec->DR4 == 0xffffffff) { - DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); + drm_dbg(&i915->drm, "UXA submitting garbage DR4, fixing up\n"); exec->DR4 = 0; } if (exec->DR1 || exec->DR4) @@ -2799,7 +2800,8 @@ add_timeline_fence_array(struct i915_execbuffer *eb, syncobj = drm_syncobj_find(eb->file, user_fence.handle); if (!syncobj) { - DRM_DEBUG("Invalid syncobj handle provided\n"); + drm_dbg(&eb->i915->drm, + "Invalid syncobj handle provided\n"); return -ENOENT; } @@ -2807,7 +2809,8 @@ add_timeline_fence_array(struct i915_execbuffer *eb, if (!fence && user_fence.flags && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { - DRM_DEBUG("Syncobj handle has no fence\n"); + drm_dbg(&eb->i915->drm, + "Syncobj handle has no fence\n"); drm_syncobj_put(syncobj); return -EINVAL; } @@ -2816,7 +2819,9 @@ add_timeline_fence_array(struct i915_execbuffer *eb, err = dma_fence_chain_find_seqno(&fence, point); if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { - DRM_DEBUG("Syncobj handle missing requested point %llu\n", point); + drm_dbg(&eb->i915->drm, + "Syncobj handle missing requested point %llu\n", + point); dma_fence_put(fence); drm_syncobj_put(syncobj); return err; @@ -2842,7 +2847,8 @@ add_timeline_fence_array(struct i915_execbuffer *eb, * 0) would break the timeline. */ if (user_fence.flags & I915_EXEC_FENCE_WAIT) { - DRM_DEBUG("Trying to wait & signal the same timeline point.\n"); + drm_dbg(&eb->i915->drm, + "Trying to wait & signal the same timeline point.\n"); dma_fence_put(fence); drm_syncobj_put(syncobj); return -EINVAL; @@ -2913,14 +2919,16 @@ static int add_fence_array(struct i915_execbuffer *eb) syncobj = drm_syncobj_find(eb->file, user_fence.handle); if (!syncobj) { - DRM_DEBUG("Invalid syncobj handle provided\n"); + drm_dbg(&eb->i915->drm, + "Invalid syncobj handle provided\n"); return -ENOENT; } if (user_fence.flags & I915_EXEC_FENCE_WAIT) { fence = drm_syncobj_fence_get(syncobj); if (!fence) { - DRM_DEBUG("Syncobj handle has no fence\n"); + drm_dbg(&eb->i915->drm, + "Syncobj handle has no fence\n"); drm_syncobj_put(syncobj); return -EINVAL; } @@ -3515,7 +3523,7 @@ i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - err = i915_gem_check_execbuffer(args); + err = i915_gem_check_execbuffer(i915, args); if (err) return err; diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index 0187bc72310d6..d92512780467b 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -3921,6 +3921,7 @@ static struct intel_context * execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count, unsigned long flags) { + struct drm_i915_private *i915 = siblings[0]->i915; struct virtual_engine *ve; unsigned int n; int err; @@ -3929,7 +3930,7 @@ execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count, if (!ve) return ERR_PTR(-ENOMEM); - ve->base.i915 = siblings[0]->i915; + ve->base.i915 = i915; ve->base.gt = siblings[0]->gt; ve->base.uncore = siblings[0]->uncore; ve->base.id = -1; @@ -3988,8 +3989,9 @@ execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count, GEM_BUG_ON(!is_power_of_2(sibling->mask)); if (sibling->mask & ve->base.mask) { - DRM_DEBUG("duplicate %s entry in load balancer\n", - sibling->name); + drm_dbg(&i915->drm, + "duplicate %s entry in load balancer\n", + sibling->name); err = -EINVAL; goto err_put; } @@ -4023,8 +4025,9 @@ execlists_create_virtual(struct intel_engine_cs **siblings, unsigned int count, */ if (ve->base.class != OTHER_CLASS) { if (ve->base.class != sibling->class) { - DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n", - sibling->class, ve->base.class); + drm_dbg(&i915->drm, + "invalid mixing of engine class, sibling %d, already %d\n", + sibling->class, ve->base.class); err = -EINVAL; goto err_put; } diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index ea775e601686d..995082d45cb21 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -816,8 +816,8 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, if (obj->bit_17 == NULL) { obj->bit_17 = bitmap_zalloc(page_count, GFP_KERNEL); if (obj->bit_17 == NULL) { - DRM_ERROR("Failed to allocate memory for bit 17 " - "record\n"); + drm_err(&to_i915(obj->base.dev)->drm, + "Failed to allocate memory for bit 17 record\n"); return; } } diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index 8e914c4066ed5..0ba7d6f36b28a 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -190,7 +190,7 @@ int intel_gt_init_hw(struct intel_gt *gt) ret = i915_ppgtt_init_hw(gt); if (ret) { - DRM_ERROR("Enabling PPGTT failed (%d)\n", ret); + drm_err(&i915->drm, "Enabling PPGTT failed (%d)\n", ret); goto out; } @@ -262,7 +262,7 @@ intel_gt_clear_error_registers(struct intel_gt *gt, * some errors might have become stuck, * mask them. */ - DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir); + drm_dbg(>->i915->drm, "EIR stuck: 0x%08x, masking\n", eir); rmw_set(uncore, EMR, eir); intel_uncore_write(uncore, GEN2_IIR, I915_MASTER_ERROR_INTERRUPT); diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c index b197f0e9794f1..4c8ddd074b784 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c +++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c @@ -44,8 +44,9 @@ gen11_gt_engine_identity(struct intel_gt *gt, !time_after32(local_clock() >> 10, timeout_ts)); if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) { - DRM_ERROR("INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", - bank, bit, ident); + drm_err(>->i915->drm, + "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", + bank, bit, ident); return 0; } @@ -364,7 +365,8 @@ void gen6_gt_irq_handler(struct intel_gt *gt, u32 gt_iir) if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT | GT_BSD_CS_ERROR_INTERRUPT | GT_CS_MASTER_ERROR_INTERRUPT)) - DRM_DEBUG("Command parser error, gt_iir 0x%08x\n", gt_iir); + drm_dbg(>->i915->drm, "Command parser error, gt_iir 0x%08x\n", + gt_iir); if (gt_iir & GT_PARITY_ERROR(gt->i915)) gen7_parity_error_irq_handler(gt, gt_iir); diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c index 6c34a83c24b34..effe60ac22cdc 100644 --- a/drivers/gpu/drm/i915/gt/intel_rps.c +++ b/drivers/gpu/drm/i915/gt/intel_rps.c @@ -430,7 +430,8 @@ static int __gen5_rps_set(struct intel_rps *rps, u8 val) rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); if (rgvswctl & MEMCTL_CMD_STS) { - DRM_DEBUG("gpu busy, RCS change rejected\n"); + drm_dbg(&rps_to_i915(rps)->drm, + "gpu busy, RCS change rejected\n"); return -EBUSY; /* still busy with another command */ } @@ -1953,7 +1954,8 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10); if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) - DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); + drm_dbg(&rps_to_i915(rps)->drm, + "Command parser error, pm_iir 0x%08x\n", pm_iir); } void gen5_rps_irq_handler(struct intel_rps *rps) diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c index 3cdf5c24dbc50..07bf115029a02 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds.c +++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c @@ -55,8 +55,10 @@ * - Public functions to init or apply the given workaround type. */ -static void wa_init_start(struct i915_wa_list *wal, const char *name, const char *engine_name) +static void wa_init_start(struct i915_wa_list *wal, struct intel_gt *gt, + const char *name, const char *engine_name) { + wal->gt = gt; wal->name = name; wal->engine_name = engine_name; } @@ -80,13 +82,14 @@ static void wa_init_finish(struct i915_wa_list *wal) if (!wal->count) return; - DRM_DEBUG_DRIVER("Initialized %u %s workarounds on %s\n", - wal->wa_count, wal->name, wal->engine_name); + drm_dbg(&wal->gt->i915->drm, "Initialized %u %s workarounds on %s\n", + wal->wa_count, wal->name, wal->engine_name); } static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) { unsigned int addr = i915_mmio_reg_offset(wa->reg); + struct drm_i915_private *i915 = wal->gt->i915; unsigned int start = 0, end = wal->count; const unsigned int grow = WA_LIST_CHUNK; struct i915_wa *wa_; @@ -99,7 +102,7 @@ static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) list = kmalloc_array(ALIGN(wal->count + 1, grow), sizeof(*wa), GFP_KERNEL); if (!list) { - DRM_ERROR("No space for workaround init!\n"); + drm_err(&i915->drm, "No space for workaround init!\n"); return; } @@ -122,9 +125,10 @@ static void _wa_add(struct i915_wa_list *wal, const struct i915_wa *wa) wa_ = &wal->list[mid]; if ((wa->clr | wa_->clr) && !(wa->clr & ~wa_->clr)) { - DRM_ERROR("Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n", - i915_mmio_reg_offset(wa_->reg), - wa_->clr, wa_->set); + drm_err(&i915->drm, + "Discarding overwritten w/a for reg %04x (clear: %08x, set: %08x)\n", + i915_mmio_reg_offset(wa_->reg), + wa_->clr, wa_->set); wa_->set &= ~wa->clr; } @@ -826,7 +830,7 @@ __intel_engine_init_ctx_wa(struct intel_engine_cs *engine, { struct drm_i915_private *i915 = engine->i915; - wa_init_start(wal, name, engine->name); + wa_init_start(wal, engine->gt, name, engine->name); /* Applies to all engines */ /* @@ -1676,7 +1680,7 @@ void intel_gt_init_workarounds(struct intel_gt *gt) { struct i915_wa_list *wal = >->wa_list; - wa_init_start(wal, "GT", "global"); + wa_init_start(wal, gt, "GT", "global"); gt_init_workarounds(gt, wal); wa_init_finish(wal); } @@ -1698,12 +1702,14 @@ wal_get_fw_for_rmw(struct intel_uncore *uncore, const struct i915_wa_list *wal) } static bool -wa_verify(const struct i915_wa *wa, u32 cur, const char *name, const char *from) +wa_verify(struct intel_gt *gt, const struct i915_wa *wa, u32 cur, + const char *name, const char *from) { if ((cur ^ wa->set) & wa->read) { - DRM_ERROR("%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n", - name, from, i915_mmio_reg_offset(wa->reg), - cur, cur & wa->read, wa->set & wa->read); + drm_err(>->i915->drm, + "%s workaround lost on %s! (reg[%x]=0x%x, relevant bits were 0x%x vs expected 0x%x)\n", + name, from, i915_mmio_reg_offset(wa->reg), + cur, cur & wa->read, wa->set & wa->read); return false; } @@ -1749,7 +1755,7 @@ wa_list_apply(struct intel_gt *gt, const struct i915_wa_list *wal) intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : intel_uncore_read_fw(uncore, wa->reg); - wa_verify(wa, val, wal->name, "application"); + wa_verify(wal->gt, wa, val, wal->name, "application"); } } @@ -1779,7 +1785,7 @@ static bool wa_list_verify(struct intel_gt *gt, intel_uncore_forcewake_get__locked(uncore, fw); for (i = 0, wa = wal->list; i < wal->count; i++, wa++) - ok &= wa_verify(wa, wa->is_mcr ? + ok &= wa_verify(wal->gt, wa, wa->is_mcr ? intel_gt_mcr_read_any_fw(gt, wa->mcr_reg) : intel_uncore_read_fw(uncore, wa->reg), wal->name, from); @@ -2127,7 +2133,7 @@ void intel_engine_init_whitelist(struct intel_engine_cs *engine) struct drm_i915_private *i915 = engine->i915; struct i915_wa_list *w = &engine->whitelist; - wa_init_start(w, "whitelist", engine->name); + wa_init_start(w, engine->gt, "whitelist", engine->name); if (IS_PONTEVECCHIO(i915)) pvc_whitelist_build(engine); @@ -3012,7 +3018,7 @@ void intel_engine_init_workarounds(struct intel_engine_cs *engine) if (GRAPHICS_VER(engine->i915) < 4) return; - wa_init_start(wal, "engine", engine->name); + wa_init_start(wal, engine->gt, "engine", engine->name); engine_init_workarounds(engine, wal); wa_init_finish(wal); } @@ -3193,7 +3199,7 @@ static int engine_wa_list_verify(struct intel_context *ce, if (mcr_range(rq->engine->i915, i915_mmio_reg_offset(wa->reg))) continue; - if (!wa_verify(wa, results[i], wal->name, from)) + if (!wa_verify(wal->gt, wa, results[i], wal->name, from)) err = -ENXIO; } diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds_types.h b/drivers/gpu/drm/i915/gt/intel_workarounds_types.h index 7c8b01d000430..e14188120e660 100644 --- a/drivers/gpu/drm/i915/gt/intel_workarounds_types.h +++ b/drivers/gpu/drm/i915/gt/intel_workarounds_types.h @@ -10,6 +10,8 @@ #include "i915_reg_defs.h" +struct intel_gt; + struct i915_wa { union { i915_reg_t reg; @@ -24,6 +26,7 @@ struct i915_wa { }; struct i915_wa_list { + struct intel_gt *gt; const char *name; const char *engine_name; struct i915_wa *list; diff --git a/drivers/gpu/drm/i915/gt/selftest_workarounds.c b/drivers/gpu/drm/i915/gt/selftest_workarounds.c index 21b1edc052f88..711014bb53d98 100644 --- a/drivers/gpu/drm/i915/gt/selftest_workarounds.c +++ b/drivers/gpu/drm/i915/gt/selftest_workarounds.c @@ -66,14 +66,14 @@ reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) memset(lists, 0, sizeof(*lists)); - wa_init_start(&lists->gt_wa_list, "GT_REF", "global"); + wa_init_start(&lists->gt_wa_list, gt, "GT_REF", "global"); gt_init_workarounds(gt, &lists->gt_wa_list); wa_init_finish(&lists->gt_wa_list); for_each_engine(engine, gt, id) { struct i915_wa_list *wal = &lists->engine[id].wa_list; - wa_init_start(wal, "REF", engine->name); + wa_init_start(wal, gt, "REF", engine->name); engine_init_workarounds(engine, wal); wa_init_finish(wal); diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index ae987e92251dd..6c7ac73b69a52 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -688,8 +688,8 @@ i915_drop_caches_set(void *data, u64 val) unsigned int flags; int ret; - DRM_DEBUG("Dropping caches: 0x%08llx [0x%08llx]\n", - val, val & DROP_ALL); + drm_dbg(&i915->drm, "Dropping caches: 0x%08llx [0x%08llx]\n", + val, val & DROP_ALL); ret = gt_drop_caches(to_gt(i915), val); if (ret) diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 299f94a9fb87b..8132743ca87ef 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -1286,7 +1286,7 @@ int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file) struct i915_drm_client *client; int ret = -ENOMEM; - DRM_DEBUG("\n"); + drm_dbg(&i915->drm, "\n"); file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL); if (!file_priv) diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c index 3047e80e1163d..61ef2d9cfa626 100644 --- a/drivers/gpu/drm/i915/i915_getparam.c +++ b/drivers/gpu/drm/i915/i915_getparam.c @@ -179,7 +179,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, value = i915_perf_oa_timestamp_frequency(i915); break; default: - DRM_DEBUG("Unknown parameter %d\n", param->param); + drm_dbg(&i915->drm, "Unknown parameter %d\n", param->param); return -EINVAL; } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index d68859866bf23..1efe5c19fac11 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -1086,8 +1086,9 @@ static void ivb_parity_work(struct work_struct *work) kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj, KOBJ_CHANGE, parity_event); - DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n", - slice, row, bank, subbank); + drm_dbg(&dev_priv->drm, + "Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n", + slice, row, bank, subbank); kfree(parity_event[4]); kfree(parity_event[3]); @@ -2774,7 +2775,8 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) master_ctl = raw_reg_read(regs, GEN11_GFX_MSTR_IRQ); raw_reg_write(regs, GEN11_GFX_MSTR_IRQ, master_ctl); } else { - DRM_ERROR("Tile not supported: 0x%08x\n", master_tile_ctl); + drm_err(&i915->drm, "Tile not supported: 0x%08x\n", + master_tile_ctl); dg1_master_intr_enable(regs); return IRQ_NONE; } @@ -3940,7 +3942,7 @@ static void i8xx_error_irq_ack(struct drm_i915_private *i915, static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv, u16 eir, u16 eir_stuck) { - DRM_DEBUG("Master Error: EIR 0x%04x\n", eir); + drm_dbg(&dev_priv->drm, "Master Error: EIR 0x%04x\n", eir); if (eir_stuck) drm_dbg(&dev_priv->drm, "EIR stuck: 0x%04x, masked\n", @@ -3975,7 +3977,7 @@ static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv, static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv, u32 eir, u32 eir_stuck) { - DRM_DEBUG("Master Error, EIR 0x%08x\n", eir); + drm_dbg(&dev_priv->drm, "Master Error, EIR 0x%08x\n", eir); if (eir_stuck) drm_dbg(&dev_priv->drm, "EIR stuck: 0x%08x, masked\n", diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 0dd597a7a11f5..9e6f060592d87 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -530,9 +530,9 @@ static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) if (OA_TAKEN(hw_tail, tail) > report_size && __ratelimit(&stream->perf->tail_pointer_race)) - DRM_NOTE("unlanded report(s) head=0x%x " - "tail=0x%x hw_tail=0x%x\n", - head, tail, hw_tail); + drm_notice(&stream->uncore->i915->drm, + "unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n", + head, tail, hw_tail); stream->oa_buffer.tail = gtt_offset + tail; stream->oa_buffer.aging_tail = gtt_offset + hw_tail; @@ -1015,7 +1015,8 @@ static int gen7_append_oa_reports(struct i915_perf_stream *stream, */ if (report32[0] == 0) { if (__ratelimit(&stream->perf->spurious_report_rs)) - DRM_NOTE("Skipping spurious, invalid OA report\n"); + drm_notice(&uncore->i915->drm, + "Skipping spurious, invalid OA report\n"); continue; } @@ -1602,8 +1603,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream) free_noa_wait(stream); if (perf->spurious_report_rs.missed) { - DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n", - perf->spurious_report_rs.missed); + drm_notice(>->i915->drm, + "%d spurious OA report notices suppressed due to ratelimiting\n", + perf->spurious_report_rs.missed); } } diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index 6ec9c9fb7b0d3..00871ef997920 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -250,8 +250,9 @@ static int query_perf_config_data(struct drm_i915_private *i915, return total_size; if (query_item->length < total_size) { - DRM_DEBUG("Invalid query config data item size=%u expected=%u\n", - query_item->length, total_size); + drm_dbg(&i915->drm, + "Invalid query config data item size=%u expected=%u\n", + query_item->length, total_size); return -EINVAL; } @@ -418,9 +419,10 @@ static int query_perf_config_list(struct drm_i915_private *i915, } while (n_configs > alloc); if (query_item->length < sizeof_perf_config_list(n_configs)) { - DRM_DEBUG("Invalid query config list item size=%u expected=%zu\n", - query_item->length, - sizeof_perf_config_list(n_configs)); + drm_dbg(&i915->drm, + "Invalid query config list item size=%u expected=%zu\n", + query_item->length, + sizeof_perf_config_list(n_configs)); kfree(oa_config_ids); return -EINVAL; } diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c index 1e27502108313..595e8b5749907 100644 --- a/drivers/gpu/drm/i915/i915_sysfs.c +++ b/drivers/gpu/drm/i915/i915_sysfs.c @@ -218,7 +218,8 @@ static const struct bin_attribute error_state_attr = { static void i915_setup_error_capture(struct device *kdev) { if (sysfs_create_bin_file(&kdev->kobj, &error_state_attr)) - DRM_ERROR("error_state sysfs setup failed\n"); + drm_err(&kdev_minor_to_i915(kdev)->drm, + "error_state sysfs setup failed\n"); } static void i915_teardown_error_capture(struct device *kdev) diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index c39488eb9eeba..3b969d679c1e2 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -73,14 +73,16 @@ static void vma_print_allocator(struct i915_vma *vma, const char *reason) char buf[512]; if (!vma->node.stack) { - DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n", - vma->node.start, vma->node.size, reason); + drm_dbg(&to_i915(vma->obj->base.dev)->drm + "vma.node [%08llx + %08llx] %s: unknown owner\n", + vma->node.start, vma->node.size, reason); return; } stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0); - DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n", - vma->node.start, vma->node.size, reason, buf); + drm_dbg(&to_i915(vma->obj->base.dev)->drm, + "vma.node [%08llx + %08llx] %s: inserted at %s\n", + vma->node.start, vma->node.size, reason, buf); } #else @@ -782,9 +784,9 @@ i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, * attempt to find space. */ if (size > end) { - DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", - size, flags & PIN_MAPPABLE ? "mappable" : "total", - end); + drm_dbg(&to_i915(vma->obj->base.dev)->drm, + "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", + size, flags & PIN_MAPPABLE ? "mappable" : "total", end); return -ENOSPC; } diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index 2a3e2869fe710..6c25c9e7090ab 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -178,8 +178,9 @@ static inline void fw_domain_wait_ack_clear(const struct intel_uncore_forcewake_domain *d) { if (wait_ack_clear(d, FORCEWAKE_KERNEL)) { - DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n", - intel_uncore_forcewake_domain_to_str(d->id)); + drm_err(&d->uncore->i915->drm, + "%s: timed out waiting for forcewake ack to clear.\n", + intel_uncore_forcewake_domain_to_str(d->id)); add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */ } } @@ -226,11 +227,12 @@ fw_domain_wait_ack_with_fallback(const struct intel_uncore_forcewake_domain *d, fw_clear(d, FORCEWAKE_KERNEL_FALLBACK); } while (!ack_detected && pass++ < 10); - DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n", - intel_uncore_forcewake_domain_to_str(d->id), - type == ACK_SET ? "set" : "clear", - fw_ack(d), - pass); + drm_dbg(&d->uncore->i915->drm, + "%s had to use fallback to %s ack, 0x%x (passes %u)\n", + intel_uncore_forcewake_domain_to_str(d->id), + type == ACK_SET ? "set" : "clear", + fw_ack(d), + pass); return ack_detected ? 0 : -ETIMEDOUT; } @@ -255,8 +257,9 @@ static inline void fw_domain_wait_ack_set(const struct intel_uncore_forcewake_domain *d) { if (wait_ack_set(d, FORCEWAKE_KERNEL)) { - DRM_ERROR("%s: timed out waiting for forcewake ack request.\n", - intel_uncore_forcewake_domain_to_str(d->id)); + drm_err(&d->uncore->i915->drm, + "%s: timed out waiting for forcewake ack request.\n", + intel_uncore_forcewake_domain_to_str(d->id)); add_taint_for_CI(d->uncore->i915, TAINT_WARN); /* CI now unreliable */ } } -- GitLab From 9736a325137b62499d2b4be3fc2d742b131f75da Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Thu, 10 Nov 2022 14:22:16 +0100 Subject: [PATCH 1537/1927] ALSA: memalloc: Don't fall back for SG-buffer with IOMMU When the non-contiguous page allocation for SG buffer allocation fails, the memalloc helper tries to fall back to the old page allocation methods. This would, however, result in the bogus page addresses when IOMMU is enabled. Usually in such a case, the fallback allocation should fail as well, but occasionally it succeeds and hitting a bad access. The fallback was thought for non-IOMMU case, and as the error from dma_alloc_noncontiguous() with IOMMU essentially implies a fatal memory allocation error, we should return the error straightforwardly without fallback. This avoids the corner case like the above. The patch also renames the local variable "dma_ops" with snd_ prefix for avoiding the name conflict. Fixes: a8d302a0b770 ("ALSA: memalloc: Revive x86-specific WC page allocations again") Reported-by: Kai Vehmanen Reviewed-by: Kai Vehmanen Link: https://lore.kernel.org/r/alpine.DEB.2.22.394.2211041541090.3532114@eliteleevi.tm.intel.com Link: https://lore.kernel.org/r/20221110132216.30605-1-tiwai@suse.de Signed-off-by: Takashi Iwai --- sound/core/memalloc.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c index 03cffe7713667..6a81aaab25ab0 100644 --- a/sound/core/memalloc.c +++ b/sound/core/memalloc.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -541,19 +542,20 @@ static void *snd_dma_noncontig_alloc(struct snd_dma_buffer *dmab, size_t size) struct sg_table *sgt; void *p; - sgt = dma_alloc_noncontiguous(dmab->dev.dev, size, dmab->dev.dir, - DEFAULT_GFP, 0); - if (!sgt) { #ifdef CONFIG_SND_DMA_SGBUF + if (!get_dma_ops(dmab->dev.dev)) { if (dmab->dev.type == SNDRV_DMA_TYPE_DEV_WC_SG) dmab->dev.type = SNDRV_DMA_TYPE_DEV_WC_SG_FALLBACK; else dmab->dev.type = SNDRV_DMA_TYPE_DEV_SG_FALLBACK; return snd_dma_sg_fallback_alloc(dmab, size); -#else - return NULL; -#endif } +#endif + + sgt = dma_alloc_noncontiguous(dmab->dev.dev, size, dmab->dev.dir, + DEFAULT_GFP, 0); + if (!sgt) + return NULL; dmab->dev.need_sync = dma_need_sync(dmab->dev.dev, sg_dma_address(sgt->sgl)); @@ -857,7 +859,7 @@ static const struct snd_malloc_ops snd_dma_noncoherent_ops = { /* * Entry points */ -static const struct snd_malloc_ops *dma_ops[] = { +static const struct snd_malloc_ops *snd_dma_ops[] = { [SNDRV_DMA_TYPE_CONTINUOUS] = &snd_dma_continuous_ops, [SNDRV_DMA_TYPE_VMALLOC] = &snd_dma_vmalloc_ops, #ifdef CONFIG_HAS_DMA @@ -883,7 +885,7 @@ static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab) if (WARN_ON_ONCE(!dmab)) return NULL; if (WARN_ON_ONCE(dmab->dev.type <= SNDRV_DMA_TYPE_UNKNOWN || - dmab->dev.type >= ARRAY_SIZE(dma_ops))) + dmab->dev.type >= ARRAY_SIZE(snd_dma_ops))) return NULL; - return dma_ops[dmab->dev.type]; + return snd_dma_ops[dmab->dev.type]; } -- GitLab From 9a5523f72bd2b0d66eef3d58810c6eb7b5ffc143 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Thu, 10 Nov 2022 22:45:39 +0800 Subject: [PATCH 1538/1927] ALSA: hda: fix potential memleak in 'add_widget_node' As 'kobject_add' may allocated memory for 'kobject->name' when return error. And in this function, if call 'kobject_add' failed didn't free kobject. So call 'kobject_put' to recycling resources. Signed-off-by: Ye Bin Cc: Link: https://lore.kernel.org/r/20221110144539.2989354-1-yebin@huaweicloud.com Signed-off-by: Takashi Iwai --- sound/hda/hdac_sysfs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sound/hda/hdac_sysfs.c b/sound/hda/hdac_sysfs.c index e47de49a32e3e..62a9615dcf529 100644 --- a/sound/hda/hdac_sysfs.c +++ b/sound/hda/hdac_sysfs.c @@ -346,8 +346,10 @@ static int add_widget_node(struct kobject *parent, hda_nid_t nid, return -ENOMEM; kobject_init(kobj, &widget_ktype); err = kobject_add(kobj, parent, "%02x", nid); - if (err < 0) + if (err < 0) { + kobject_put(kobj); return err; + } err = sysfs_create_group(kobj, group); if (err < 0) { kobject_put(kobj); -- GitLab From acce40037041f97baad18142bb253064491ebde3 Mon Sep 17 00:00:00 2001 From: Zhengchao Shao Date: Wed, 9 Nov 2022 12:40:16 +0800 Subject: [PATCH 1539/1927] ethernet: tundra: free irq when alloc ring failed in tsi108_open() When alloc tx/rx ring failed in tsi108_open(), it doesn't free irq. Fix it. Fixes: 5e123b844a1c ("[PATCH] Add tsi108/9 On Chip Ethernet device driver support") Signed-off-by: Zhengchao Shao Link: https://lore.kernel.org/r/20221109044016.126866-1-shaozhengchao@huawei.com Signed-off-by: Paolo Abeni --- drivers/net/ethernet/tundra/tsi108_eth.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/tundra/tsi108_eth.c b/drivers/net/ethernet/tundra/tsi108_eth.c index 2cd2afc3fff03..d09d352e1c0af 100644 --- a/drivers/net/ethernet/tundra/tsi108_eth.c +++ b/drivers/net/ethernet/tundra/tsi108_eth.c @@ -1290,12 +1290,15 @@ static int tsi108_open(struct net_device *dev) data->rxring = dma_alloc_coherent(&data->pdev->dev, rxring_size, &data->rxdma, GFP_KERNEL); - if (!data->rxring) + if (!data->rxring) { + free_irq(data->irq_num, dev); return -ENOMEM; + } data->txring = dma_alloc_coherent(&data->pdev->dev, txring_size, &data->txdma, GFP_KERNEL); if (!data->txring) { + free_irq(data->irq_num, dev); dma_free_coherent(&data->pdev->dev, rxring_size, data->rxring, data->rxdma); return -ENOMEM; -- GitLab From 23569b5652ee8e8e55a12f7835f59af6f3cefc30 Mon Sep 17 00:00:00 2001 From: Chuang Wang Date: Wed, 9 Nov 2022 17:07:34 +0800 Subject: [PATCH 1540/1927] net: macvlan: fix memory leaks of macvlan_common_newlink kmemleak reports memory leaks in macvlan_common_newlink, as follows: ip link add link eth0 name .. type macvlan mode source macaddr add kmemleak reports: unreferenced object 0xffff8880109bb140 (size 64): comm "ip", pid 284, jiffies 4294986150 (age 430.108s) hex dump (first 32 bytes): 00 00 00 00 00 00 00 00 b8 aa 5a 12 80 88 ff ff ..........Z..... 80 1b fa 0d 80 88 ff ff 1e ff ac af c7 c1 6b 6b ..............kk backtrace: [] kmem_cache_alloc_trace+0x1c7/0x300 [] macvlan_hash_add_source+0x45/0xc0 [] macvlan_changelink_sources+0xd7/0x170 [] macvlan_common_newlink+0x38c/0x5a0 [] macvlan_newlink+0xe/0x20 [] __rtnl_newlink+0x7af/0xa50 [] rtnl_newlink+0x48/0x70 ... In the scenario where the macvlan mode is configured as 'source', macvlan_changelink_sources() will be execured to reconfigure list of remote source mac addresses, at the same time, if register_netdevice() return an error, the resource generated by macvlan_changelink_sources() is not cleaned up. Using this patch, in the case of an error, it will execute macvlan_flush_sources() to ensure that the resource is cleaned up. Fixes: aa5fd0fb7748 ("driver: macvlan: Destroy new macvlan port if macvlan_common_newlink failed.") Signed-off-by: Chuang Wang Link: https://lore.kernel.org/r/20221109090735.690500-1-nashuiliang@gmail.com Signed-off-by: Paolo Abeni --- drivers/net/macvlan.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index c5cfe85551992..578897aaada08 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1533,8 +1533,10 @@ int macvlan_common_newlink(struct net *src_net, struct net_device *dev, /* the macvlan port may be freed by macvlan_uninit when fail to register. * so we destroy the macvlan port only when it's valid. */ - if (create && macvlan_port_get_rtnl(lowerdev)) + if (create && macvlan_port_get_rtnl(lowerdev)) { + macvlan_flush_sources(port, vlan); macvlan_port_destroy(port->dev); + } return err; } EXPORT_SYMBOL_GPL(macvlan_common_newlink); -- GitLab From 876153ab068b2507a19aa3ef481f5b00a2cc780f Mon Sep 17 00:00:00 2001 From: Aishwarya Kothari Date: Wed, 31 Aug 2022 16:16:22 +0200 Subject: [PATCH 1541/1927] drm/panel: simple: set bpc field for logic technologies displays In case bpc is not set for a panel it then throws a WARN(). Add bpc to the panels logictechno_lt170410_2whc and logictechno_lt161010_2nh. Fixes: 5728fe7fa539 ("drm/panel: simple: add display timings for logic technologies displays") Signed-off-by: Aishwarya Kothari Signed-off-by: Francesco Dolcini Signed-off-by: Douglas Anderson Link: https://patchwork.freedesktop.org/patch/msgid/20220831141622.39605-1-francesco.dolcini@toradex.com --- drivers/gpu/drm/panel/panel-simple.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 2944228a8e2c4..8a3b685c2fcc0 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -2500,6 +2500,7 @@ static const struct display_timing logictechno_lt161010_2nh_timing = { static const struct panel_desc logictechno_lt161010_2nh = { .timings = &logictechno_lt161010_2nh_timing, .num_timings = 1, + .bpc = 6, .size = { .width = 154, .height = 86, @@ -2529,6 +2530,7 @@ static const struct display_timing logictechno_lt170410_2whc_timing = { static const struct panel_desc logictechno_lt170410_2whc = { .timings = &logictechno_lt170410_2whc_timing, .num_timings = 1, + .bpc = 8, .size = { .width = 217, .height = 136, -- GitLab From 36b038791e1e2baea892e9276588815fd14894b4 Mon Sep 17 00:00:00 2001 From: Mel Gorman Date: Thu, 10 Nov 2022 12:44:00 +0000 Subject: [PATCH 1542/1927] x86/fpu: Drop fpregs lock before inheriting FPU permissions Mike Galbraith reported the following against an old fork of preempt-rt but the same issue also applies to the current preempt-rt tree. BUG: sleeping function called from invalid context at kernel/locking/spinlock_rt.c:46 in_atomic(): 1, irqs_disabled(): 0, non_block: 0, pid: 1, name: systemd preempt_count: 1, expected: 0 RCU nest depth: 0, expected: 0 Preemption disabled at: fpu_clone CPU: 6 PID: 1 Comm: systemd Tainted: G E (unreleased) Call Trace: dump_stack_lvl ? fpu_clone __might_resched rt_spin_lock fpu_clone ? copy_thread ? copy_process ? shmem_alloc_inode ? kmem_cache_alloc ? kernel_clone ? __do_sys_clone ? do_syscall_64 ? __x64_sys_rt_sigprocmask ? syscall_exit_to_user_mode ? do_syscall_64 ? syscall_exit_to_user_mode ? do_syscall_64 ? syscall_exit_to_user_mode ? do_syscall_64 ? exc_page_fault ? entry_SYSCALL_64_after_hwframe Mike says: The splat comes from fpu_inherit_perms() being called under fpregs_lock(), and us reaching the spin_lock_irq() therein due to fpu_state_size_dynamic() returning true despite static key __fpu_state_size_dynamic having never been enabled. Mike's assessment looks correct. fpregs_lock on a PREEMPT_RT kernel disables preemption so calling spin_lock_irq() in fpu_inherit_perms() is unsafe. This problem exists since commit 9e798e9aa14c ("x86/fpu: Prepare fpu_clone() for dynamically enabled features"). Even though the original bug report should not have enabled the paths at all, the bug still exists. fpregs_lock is necessary when editing the FPU registers or a task's FP state but it is not necessary for fpu_inherit_perms(). The only write of any FP state in fpu_inherit_perms() is for the new child which is not running yet and cannot context switch or be borrowed by a kernel thread yet. Hence, fpregs_lock is not protecting anything in the new child until clone() completes and can be dropped earlier. The siglock still needs to be acquired by fpu_inherit_perms() as the read of the parent's permissions has to be serialised. [ bp: Cleanup splat. ] Fixes: 9e798e9aa14c ("x86/fpu: Prepare fpu_clone() for dynamically enabled features") Reported-by: Mike Galbraith Signed-off-by: Mel Gorman Signed-off-by: Borislav Petkov Reviewed-by: Thomas Gleixner Cc: Link: https://lore.kernel.org/r/20221110124400.zgymc2lnwqjukgfh@techsingularity.net --- arch/x86/kernel/fpu/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index 3b28c5b25e12c..d00db56a88682 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -605,9 +605,9 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal) if (test_thread_flag(TIF_NEED_FPU_LOAD)) fpregs_restore_userregs(); save_fpregs_to_fpstate(dst_fpu); + fpregs_unlock(); if (!(clone_flags & CLONE_THREAD)) fpu_inherit_perms(dst_fpu); - fpregs_unlock(); /* * Children never inherit PASID state. -- GitLab From 1aeb122d214b92474c86fde00a03d6e2d69381b5 Mon Sep 17 00:00:00 2001 From: Horatiu Vultur Date: Wed, 28 Sep 2022 21:51:12 +0200 Subject: [PATCH 1543/1927] nvmem: lan9662-otp: Fix compatible string The device tree bindings for lan9662-otp expects the compatible string to be one of following compatible strings: microchip,lan9662-otpc microchip,lan9668-otpc The problem is that the lan9662-otp driver contains the microchip,lan9662-otp compatible string instead of microchip,lan9662-otpc. Fix this by updating the compatible string in the driver. Fixes: 9e8f208ad5229d ("nvmem: lan9662-otp: add support") Signed-off-by: Horatiu Vultur Link: https://lore.kernel.org/r/20220928195112.630351-1-horatiu.vultur@microchip.com Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/lan9662-otpc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvmem/lan9662-otpc.c b/drivers/nvmem/lan9662-otpc.c index f6732fd216d80..377bf34c29461 100644 --- a/drivers/nvmem/lan9662-otpc.c +++ b/drivers/nvmem/lan9662-otpc.c @@ -203,7 +203,7 @@ static int lan9662_otp_probe(struct platform_device *pdev) } static const struct of_device_id lan9662_otp_match[] = { - { .compatible = "microchip,lan9662-otp", }, + { .compatible = "microchip,lan9662-otpc", }, { }, }; MODULE_DEVICE_TABLE(of, lan9662_otp_match); -- GitLab From b9c1939627f8185dec8ba6d741e9573a4c7a5834 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Thu, 29 Sep 2022 18:52:02 +0200 Subject: [PATCH 1544/1927] slimbus: stream: correct presence rate frequencies Correct few frequencies in presence rate table - multiplied by 10 (110250 instead of 11025 Hz). Fixes: abb9c9b8b51b ("slimbus: stream: add stream support") Cc: Signed-off-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20220929165202.410937-1-krzysztof.kozlowski@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/slimbus/stream.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/slimbus/stream.c b/drivers/slimbus/stream.c index 75f87b3d8b953..73a2aa3629572 100644 --- a/drivers/slimbus/stream.c +++ b/drivers/slimbus/stream.c @@ -67,10 +67,10 @@ static const int slim_presence_rate_table[] = { 384000, 768000, 0, /* Reserved */ - 110250, - 220500, - 441000, - 882000, + 11025, + 22050, + 44100, + 88200, 176400, 352800, 705600, -- GitLab From 5fddf8962b429b8303c4a654291ecb6e61a7d747 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Tue, 11 Oct 2022 11:14:17 -0600 Subject: [PATCH 1545/1927] docs: update mediator contact information in CoC doc Update mediator contact information in CoC interpretation document. Cc: Signed-off-by: Shuah Khan Link: https://lore.kernel.org/r/20221011171417.34286-1-skhan@linuxfoundation.org Signed-off-by: Greg Kroah-Hartman --- Documentation/process/code-of-conduct-interpretation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/process/code-of-conduct-interpretation.rst b/Documentation/process/code-of-conduct-interpretation.rst index 922e0b547bc39..66b07f14714c2 100644 --- a/Documentation/process/code-of-conduct-interpretation.rst +++ b/Documentation/process/code-of-conduct-interpretation.rst @@ -51,7 +51,7 @@ the Technical Advisory Board (TAB) or other maintainers if you're uncertain how to handle situations that come up. It will not be considered a violation report unless you want it to be. If you are uncertain about approaching the TAB or any other maintainers, please -reach out to our conflict mediator, Joanna Lee . +reach out to our conflict mediator, Joanna Lee . In the end, "be kind to each other" is really what the end goal is for everybody. We know everyone is human and we all fail at times, but the -- GitLab From e54fad8044db18cc400df8d01bfb86cada08b7cb Mon Sep 17 00:00:00 2001 From: Zheng Bin Date: Thu, 27 Oct 2022 17:59:04 +0800 Subject: [PATCH 1546/1927] slimbus: qcom-ngd: Fix build error when CONFIG_SLIM_QCOM_NGD_CTRL=y && CONFIG_QCOM_RPROC_COMMON=m If CONFIG_SLIM_QCOM_NGD_CTRL=y, CONFIG_QCOM_RPROC_COMMON=m, COMPILE_TEST=y, bulding fails: drivers/slimbus/qcom-ngd-ctrl.o: In function `qcom_slim_ngd_ctrl_probe': qcom-ngd-ctrl.c:(.text+0x330): undefined reference to `qcom_register_ssr_notifier' qcom-ngd-ctrl.c:(.text+0x5fc): undefined reference to `qcom_unregister_ssr_notifier' drivers/slimbus/qcom-ngd-ctrl.o: In function `qcom_slim_ngd_remove': qcom-ngd-ctrl.c:(.text+0x90c): undefined reference to `qcom_unregister_ssr_notifier' Make SLIM_QCOM_NGD_CTRL depends on QCOM_RPROC_COMMON || (COMPILE_TEST && !QCOM_RPROC_COMMON) to fix this. Fixes: e291691c6977 ("slimbus: qcom-ngd-ctrl: allow compile testing without QCOM_RPROC_COMMON") Cc: stable Signed-off-by: Zheng Bin Reviewed-by: Krzysztof Kozlowski Link: https://lore.kernel.org/r/20221027095904.3388959-1-zhengbin13@huawei.com Signed-off-by: Greg Kroah-Hartman --- drivers/slimbus/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/slimbus/Kconfig b/drivers/slimbus/Kconfig index 2ed821f75816c..a0fdf9d792cb4 100644 --- a/drivers/slimbus/Kconfig +++ b/drivers/slimbus/Kconfig @@ -23,7 +23,7 @@ config SLIM_QCOM_CTRL config SLIM_QCOM_NGD_CTRL tristate "Qualcomm SLIMbus Satellite Non-Generic Device Component" depends on HAS_IOMEM && DMA_ENGINE && NET - depends on QCOM_RPROC_COMMON || COMPILE_TEST + depends on QCOM_RPROC_COMMON || (COMPILE_TEST && !QCOM_RPROC_COMMON) depends on ARCH_QCOM || COMPILE_TEST select QCOM_QMI_HELPERS select QCOM_PDR_HELPERS -- GitLab From ee424f7d3960152f5f862bbb6943e59828dc7917 Mon Sep 17 00:00:00 2001 From: Christian Lamparter Date: Fri, 4 Nov 2022 17:52:03 +0100 Subject: [PATCH 1547/1927] nvmem: u-boot-env: fix crc32_data_offset on redundant u-boot-env The Western Digital MyBook Live (PowerPC 464/APM82181) has a set of redundant u-boot-env. Loading up the driver the following error: | u_boot_env: Invalid calculated CRC32: 0x4f8f2c86 (expected: 0x98b14514) | u_boot_env: probe of partition@1e000 failed with error -22 Looking up the userspace libubootenv utilities source [0], it looks like the "mark" or "flag" is not part of the crc32 sum... which is unfortunate :( |static int libuboot_load(struct uboot_ctx *ctx) |{ |[...] | if (ctx->redundant) { | [...] | offsetdata = offsetof(struct uboot_env_redund, data); | [...] //-----^^ | } | usable_envsize = ctx->size - offsetdata; | buf[0] = malloc(bufsize); |[...] | for (i = 0; i < copies; i++) { | data = (uint8_t *)(buf[i] + offsetdata); | uint32_t crc; | | ret = devread(ctx, i, buf[i]); | [...] | crc = *(uint32_t *)(buf[i] + offsetcrc); | dev->crc = crc32(0, (uint8_t *)data, usable_envsize); | [0] https://github.com/sbabic/libubootenv/blob/master/src/uboot_env.c#L951 Fixes: d5542923f200 ("nvmem: add driver handling U-Boot environment variables") Signed-off-by: Christian Lamparter Link: https://lore.kernel.org/r/70a16eae113e08db2390b76e174f4837caa135c3.1667580636.git.chunkeey@gmail.com Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/u-boot-env.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvmem/u-boot-env.c b/drivers/nvmem/u-boot-env.c index 8e72d1bbd6490..4fdbdccebda16 100644 --- a/drivers/nvmem/u-boot-env.c +++ b/drivers/nvmem/u-boot-env.c @@ -135,7 +135,7 @@ static int u_boot_env_parse(struct u_boot_env *priv) break; case U_BOOT_FORMAT_REDUNDANT: crc32_offset = offsetof(struct u_boot_env_image_redundant, crc32); - crc32_data_offset = offsetof(struct u_boot_env_image_redundant, mark); + crc32_data_offset = offsetof(struct u_boot_env_image_redundant, data); data_offset = offsetof(struct u_boot_env_image_redundant, data); break; } -- GitLab From 65946690ed8d972fdb91a74ee75ac0f0f0d68321 Mon Sep 17 00:00:00 2001 From: Brian Norris Date: Wed, 19 Oct 2022 18:10:53 -0700 Subject: [PATCH 1548/1927] firmware: coreboot: Register bus in module init The coreboot_table driver registers a coreboot bus while probing a "coreboot_table" device representing the coreboot table memory region. Probing this device (i.e., registering the bus) is a dependency for the module_init() functions of any driver for this bus (e.g., memconsole-coreboot.c / memconsole_driver_init()). With synchronous probe, this dependency works OK, as the link order in the Makefile ensures coreboot_table_driver_init() (and thus, coreboot_table_probe()) completes before a coreboot device driver tries to add itself to the bus. With asynchronous probe, however, coreboot_table_probe() may race with memconsole_driver_init(), and so we're liable to hit one of these two: 1. coreboot_driver_register() eventually hits "[...] the bus was not initialized.", and the memconsole driver fails to register; or 2. coreboot_driver_register() gets past #1, but still races with bus_register() and hits some other undefined/crashing behavior (e.g., in driver_find() [1]) We can resolve this by registering the bus in our initcall, and only deferring "device" work (scanning the coreboot memory region and creating sub-devices) to probe(). [1] Example failure, using 'driver_async_probe=*' kernel command line: [ 0.114217] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000010 ... [ 0.114307] CPU: 1 PID: 1 Comm: swapper/0 Not tainted 6.1.0-rc1 #63 [ 0.114316] Hardware name: Google Scarlet (DT) ... [ 0.114488] Call trace: [ 0.114494] _raw_spin_lock+0x34/0x60 [ 0.114502] kset_find_obj+0x28/0x84 [ 0.114511] driver_find+0x30/0x50 [ 0.114520] driver_register+0x64/0x10c [ 0.114528] coreboot_driver_register+0x30/0x3c [ 0.114540] memconsole_driver_init+0x24/0x30 [ 0.114550] do_one_initcall+0x154/0x2e0 [ 0.114560] do_initcall_level+0x134/0x160 [ 0.114571] do_initcalls+0x60/0xa0 [ 0.114579] do_basic_setup+0x28/0x34 [ 0.114588] kernel_init_freeable+0xf8/0x150 [ 0.114596] kernel_init+0x2c/0x12c [ 0.114607] ret_from_fork+0x10/0x20 [ 0.114624] Code: 5280002b 1100054a b900092a f9800011 (885ffc01) [ 0.114631] ---[ end trace 0000000000000000 ]--- Fixes: b81e3140e412 ("firmware: coreboot: Make bus registration symmetric") Cc: Signed-off-by: Brian Norris Reviewed-by: Guenter Roeck Reviewed-by: Stephen Boyd Link: https://lore.kernel.org/r/20221019180934.1.If29e167d8a4771b0bf4a39c89c6946ed764817b9@changeid Signed-off-by: Greg Kroah-Hartman --- drivers/firmware/google/coreboot_table.c | 37 +++++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index c52bcaa9def60..9ca21feb9d454 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -149,12 +149,8 @@ static int coreboot_table_probe(struct platform_device *pdev) if (!ptr) return -ENOMEM; - ret = bus_register(&coreboot_bus_type); - if (!ret) { - ret = coreboot_table_populate(dev, ptr); - if (ret) - bus_unregister(&coreboot_bus_type); - } + ret = coreboot_table_populate(dev, ptr); + memunmap(ptr); return ret; @@ -169,7 +165,6 @@ static int __cb_dev_unregister(struct device *dev, void *dummy) static int coreboot_table_remove(struct platform_device *pdev) { bus_for_each_dev(&coreboot_bus_type, NULL, NULL, __cb_dev_unregister); - bus_unregister(&coreboot_bus_type); return 0; } @@ -199,6 +194,32 @@ static struct platform_driver coreboot_table_driver = { .of_match_table = of_match_ptr(coreboot_of_match), }, }; -module_platform_driver(coreboot_table_driver); + +static int __init coreboot_table_driver_init(void) +{ + int ret; + + ret = bus_register(&coreboot_bus_type); + if (ret) + return ret; + + ret = platform_driver_register(&coreboot_table_driver); + if (ret) { + bus_unregister(&coreboot_bus_type); + return ret; + } + + return 0; +} + +static void __exit coreboot_table_driver_exit(void) +{ + platform_driver_unregister(&coreboot_table_driver); + bus_unregister(&coreboot_bus_type); +} + +module_init(coreboot_table_driver_init); +module_exit(coreboot_table_driver_exit); + MODULE_AUTHOR("Google, Inc."); MODULE_LICENSE("GPL"); -- GitLab From 1edfe4ea16ca2c9e91ec6cecd446b13636724e4f Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Mon, 10 Oct 2022 13:54:16 -1000 Subject: [PATCH 1549/1927] kernfs: Fix spurious lockdep warning in kernfs_find_and_get_node_by_id() c25491747b21 ("kernfs: Add KERNFS_REMOVING flags") made kernfs_find_and_get_node_by_id() test kernfs_active() instead of KERNFS_ACTIVATED. kernfs_find_and_get_by_id() is called without holding the kernfs_rwsem triggering the following lockdep warning. WARNING: CPU: 1 PID: 6191 at fs/kernfs/dir.c:36 kernfs_active+0xe8/0x120 fs/kernfs/dir.c:38 Modules linked in: CPU: 1 PID: 6191 Comm: syz-executor.1 Not tainted 6.0.0-syzkaller-09413-g4899a36f91a9 #0 Hardware name: linux,dummy-virt (DT) pstate: 10000005 (nzcV daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : kernfs_active+0xe8/0x120 fs/kernfs/dir.c:36 lr : lock_is_held include/linux/lockdep.h:283 [inline] lr : kernfs_active+0x94/0x120 fs/kernfs/dir.c:36 sp : ffff8000182c7a00 x29: ffff8000182c7a00 x28: 0000000000000002 x27: 0000000000000001 x26: ffff00000ee1f6a8 x25: 1fffe00001dc3ed5 x24: 0000000000000000 x23: ffff80000ca1fba0 x22: ffff8000089efcb0 x21: 0000000000000001 x20: ffff0000091181d0 x19: ffff0000091181d0 x18: ffff00006a9e6b88 x17: 0000000000000000 x16: 0000000000000000 x15: ffff00006a9e6bc4 x14: 1ffff00003058f0e x13: 1fffe0000258c816 x12: ffff700003058f39 x11: 1ffff00003058f38 x10: ffff700003058f38 x9 : dfff800000000000 x8 : ffff80000e482f20 x7 : ffff0000091d8058 x6 : ffff80000e482c60 x5 : ffff000009402ee8 x4 : 1ffff00001bd1f46 x3 : 1fffe0000258c6d1 x2 : 0000000000000003 x1 : 00000000000000c0 x0 : 0000000000000000 Call trace: kernfs_active+0xe8/0x120 fs/kernfs/dir.c:38 kernfs_find_and_get_node_by_id+0x6c/0x140 fs/kernfs/dir.c:708 __kernfs_fh_to_dentry fs/kernfs/mount.c:102 [inline] kernfs_fh_to_dentry+0x88/0x1fc fs/kernfs/mount.c:128 exportfs_decode_fh_raw+0x104/0x560 fs/exportfs/expfs.c:435 exportfs_decode_fh+0x10/0x5c fs/exportfs/expfs.c:575 do_handle_to_path fs/fhandle.c:152 [inline] handle_to_path fs/fhandle.c:207 [inline] do_handle_open+0x2a4/0x7b0 fs/fhandle.c:223 __do_compat_sys_open_by_handle_at fs/fhandle.c:277 [inline] __se_compat_sys_open_by_handle_at fs/fhandle.c:274 [inline] __arm64_compat_sys_open_by_handle_at+0x6c/0x9c fs/fhandle.c:274 __invoke_syscall arch/arm64/kernel/syscall.c:38 [inline] invoke_syscall+0x6c/0x260 arch/arm64/kernel/syscall.c:52 el0_svc_common.constprop.0+0xc4/0x254 arch/arm64/kernel/syscall.c:142 do_el0_svc_compat+0x40/0x70 arch/arm64/kernel/syscall.c:212 el0_svc_compat+0x54/0x140 arch/arm64/kernel/entry-common.c:772 el0t_32_sync_handler+0x90/0x140 arch/arm64/kernel/entry-common.c:782 el0t_32_sync+0x190/0x194 arch/arm64/kernel/entry.S:586 irq event stamp: 232 hardirqs last enabled at (231): [] raw_spin_rq_unlock_irq kernel/sched/sched.h:1367 [inline] hardirqs last enabled at (231): [] finish_lock_switch kernel/sched/core.c:4943 [inline] hardirqs last enabled at (231): [] finish_task_switch.isra.0+0x200/0x880 kernel/sched/core.c:5061 hardirqs last disabled at (232): [] el1_dbg+0x24/0x80 arch/arm64/kernel/entry-common.c:404 softirqs last enabled at (228): [] _stext+0x938/0xf58 softirqs last disabled at (207): [] ____do_softirq+0x10/0x20 arch/arm64/kernel/irq.c:79 ---[ end trace 0000000000000000 ]--- The lockdep warning in kernfs_active() is there to ensure that the activated state stays stable for the caller. For kernfs_find_and_get_node_by_id(), all that's needed is ensuring that a node which has never been activated can't be looked up and guaranteeing lookup success when the caller knows the node to be active, both of which can be achieved by testing the active count without holding the kernfs_rwsem. Fix the spurious warning by introducing __kernfs_active() which doesn't have the lockdep annotation. Signed-off-by: Tejun Heo Reported-by: syzbot+590ce62b128e79cf0a35@syzkaller.appspotmail.com Fixes: c25491747b21 ("kernfs: Add KERNFS_REMOVING flags") Cc: Amir Goldstein Cc: Dmitry Vyukov Link: https://lore.kernel.org/r/Y0SwqBsZ9BMmZv6x@slm.duckdns.org Signed-off-by: Greg Kroah-Hartman --- fs/kernfs/dir.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c index 3990f3e270cb6..f33b3baad07cb 100644 --- a/fs/kernfs/dir.c +++ b/fs/kernfs/dir.c @@ -31,10 +31,15 @@ static DEFINE_SPINLOCK(kernfs_idr_lock); /* root->ino_idr */ #define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb) +static bool __kernfs_active(struct kernfs_node *kn) +{ + return atomic_read(&kn->active) >= 0; +} + static bool kernfs_active(struct kernfs_node *kn) { lockdep_assert_held(&kernfs_root(kn)->kernfs_rwsem); - return atomic_read(&kn->active) >= 0; + return __kernfs_active(kn); } static bool kernfs_lockdep(struct kernfs_node *kn) @@ -705,7 +710,12 @@ struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root, goto err_unlock; } - if (unlikely(!kernfs_active(kn) || !atomic_inc_not_zero(&kn->count))) + /* + * We should fail if @kn has never been activated and guarantee success + * if the caller knows that @kn is active. Both can be achieved by + * __kernfs_active() which tests @kn->active without kernfs_rwsem. + */ + if (unlikely(!__kernfs_active(kn) || !atomic_inc_not_zero(&kn->count))) goto err_unlock; spin_unlock(&kernfs_idr_lock); -- GitLab From 3851d25c75ed03117268a8feb34adca5a843a126 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 10 Nov 2022 10:50:55 -0700 Subject: [PATCH 1550/1927] io_uring: check for rollover of buffer ID when providing buffers We already check if the chosen starting offset for the buffer IDs fit within an unsigned short, as 65535 is the maximum value for a provided buffer. But if the caller asks to add N buffers at offset M, and M + N would exceed the size of the unsigned short, we simply add buffers with wrapping around the ID. This is not necessarily a bug and could in fact be a valid use case, but it seems confusing and inconsistent with the initial check for starting offset. Let's check for wrap consistently, and error the addition if we do need to wrap. Reported-by: Olivier Langlois Link: https://github.com/axboe/liburing/issues/726 Cc: stable@vger.kernel.org Signed-off-by: Jens Axboe --- io_uring/kbuf.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index 25cd724ade184..e2c46889d5fab 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -346,6 +346,8 @@ int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe tmp = READ_ONCE(sqe->off); if (tmp > USHRT_MAX) return -E2BIG; + if (tmp + p->nbufs >= USHRT_MAX) + return -EINVAL; p->bid = tmp; return 0; } -- GitLab From 7fdba0011157861892c470995ff586a1871e603f Mon Sep 17 00:00:00 2001 From: Anthony DeRossi Date: Wed, 9 Nov 2022 17:40:25 -0800 Subject: [PATCH 1551/1927] vfio: Fix container device registration life cycle In vfio_device_open(), vfio_device_container_register() is always called when open_count == 1. On error, vfio_device_container_unregister() is only called when open_count == 1 and close_device is set. This leaks a registration for devices without a close_device implementation. In vfio_device_fops_release(), vfio_device_container_unregister() is called unconditionally. This can cause a device to be unregistered multiple times. Treating container device registration/unregistration uniformly (always when open_count == 1) fixes both issues. Fixes: ce4b4657ff18 ("vfio: Replace the DMA unmapping notifier with a callback") Signed-off-by: Anthony DeRossi Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Link: https://lore.kernel.org/r/20221110014027.28780-2-ajderossi@gmail.com Signed-off-by: Alex Williamson --- drivers/vfio/vfio_main.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 2d168793d4e1c..9a4af880e941e 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -801,8 +801,9 @@ static struct file *vfio_device_open(struct vfio_device *device) err_close_device: mutex_lock(&device->dev_set->lock); mutex_lock(&device->group->group_lock); - if (device->open_count == 1 && device->ops->close_device) { - device->ops->close_device(device); + if (device->open_count == 1) { + if (device->ops->close_device) + device->ops->close_device(device); vfio_device_container_unregister(device); } @@ -1017,10 +1018,12 @@ static int vfio_device_fops_release(struct inode *inode, struct file *filep) mutex_lock(&device->dev_set->lock); vfio_assert_device_open(device); mutex_lock(&device->group->group_lock); - if (device->open_count == 1 && device->ops->close_device) - device->ops->close_device(device); + if (device->open_count == 1) { + if (device->ops->close_device) + device->ops->close_device(device); - vfio_device_container_unregister(device); + vfio_device_container_unregister(device); + } mutex_unlock(&device->group->group_lock); device->open_count--; if (device->open_count == 0) -- GitLab From 5cd189e410debedda416fecfc12f4716b5829845 Mon Sep 17 00:00:00 2001 From: Anthony DeRossi Date: Wed, 9 Nov 2022 17:40:26 -0800 Subject: [PATCH 1552/1927] vfio: Export the device set open count The open count of a device set is the sum of the open counts of all devices in the set. Drivers can use this value to determine whether shared resources are in use without tracking them manually or accessing the private open_count in vfio_device. Signed-off-by: Anthony DeRossi Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Reviewed-by: Yi Liu Link: https://lore.kernel.org/r/20221110014027.28780-3-ajderossi@gmail.com Signed-off-by: Alex Williamson --- drivers/vfio/vfio_main.c | 13 +++++++++++++ include/linux/vfio.h | 1 + 2 files changed, 14 insertions(+) diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c index 9a4af880e941e..6e8804fe00953 100644 --- a/drivers/vfio/vfio_main.c +++ b/drivers/vfio/vfio_main.c @@ -125,6 +125,19 @@ static void vfio_release_device_set(struct vfio_device *device) xa_unlock(&vfio_device_set_xa); } +unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set) +{ + struct vfio_device *cur; + unsigned int open_count = 0; + + lockdep_assert_held(&dev_set->lock); + + list_for_each_entry(cur, &dev_set->device_list, dev_set_list) + open_count += cur->open_count; + return open_count; +} +EXPORT_SYMBOL_GPL(vfio_device_set_open_count); + /* * Group objects - create, release, get, put, search */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index e7cebeb875dd1..fdd393f70b198 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -189,6 +189,7 @@ int vfio_register_emulated_iommu_dev(struct vfio_device *device); void vfio_unregister_group_dev(struct vfio_device *device); int vfio_assign_device_set(struct vfio_device *device, void *set_id); +unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set); int vfio_mig_get_next_state(struct vfio_device *device, enum vfio_device_mig_state cur_fsm, -- GitLab From e806e223621e4f5105170df69d7311dc3fb4bbb4 Mon Sep 17 00:00:00 2001 From: Anthony DeRossi Date: Wed, 9 Nov 2022 17:40:27 -0800 Subject: [PATCH 1553/1927] vfio/pci: Check the device set open count on reset vfio_pci_dev_set_needs_reset() inspects the open_count of every device in the set to determine whether a reset is allowed. The current device always has open_count == 1 within vfio_pci_core_disable(), effectively disabling the reset logic. This field is also documented as private in vfio_device, so it should not be used to determine whether other devices in the set are open. Checking for vfio_device_set_open_count() > 1 on the device set fixes both issues. After commit 2cd8b14aaa66 ("vfio/pci: Move to the device set infrastructure"), failure to create a new file for a device would cause the reset to be skipped due to open_count being decremented after calling close_device() in the error path. After commit eadd86f835c6 ("vfio: Remove calls to vfio_group_add_container_user()"), releasing a device would always skip the reset due to an ordering change in vfio_device_fops_release(). Failing to reset the device leaves it in an unknown state, potentially causing errors when it is accessed later or bound to a different driver. This issue was observed with a Radeon RX Vega 56 [1002:687f] (rev c3) assigned to a Windows guest. After shutting down the guest, unbinding the device from vfio-pci, and binding the device to amdgpu: [ 548.007102] [drm:psp_hw_start [amdgpu]] *ERROR* PSP create ring failed! [ 548.027174] [drm:psp_hw_init [amdgpu]] *ERROR* PSP firmware loading failed [ 548.027242] [drm:amdgpu_device_fw_loading [amdgpu]] *ERROR* hw_init of IP block failed -22 [ 548.027306] amdgpu 0000:0a:00.0: amdgpu: amdgpu_device_ip_init failed [ 548.027308] amdgpu 0000:0a:00.0: amdgpu: Fatal error during GPU init Fixes: 2cd8b14aaa66 ("vfio/pci: Move to the device set infrastructure") Fixes: eadd86f835c6 ("vfio: Remove calls to vfio_group_add_container_user()") Signed-off-by: Anthony DeRossi Reviewed-by: Jason Gunthorpe Reviewed-by: Kevin Tian Link: https://lore.kernel.org/r/20221110014027.28780-4-ajderossi@gmail.com Signed-off-by: Alex Williamson --- drivers/vfio/pci/vfio_pci_core.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index badc9d828cac2..e030c2120183e 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -2488,12 +2488,12 @@ static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set) struct vfio_pci_core_device *cur; bool needs_reset = false; - list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) { - /* No VFIO device in the set can have an open device FD */ - if (cur->vdev.open_count) - return false; + /* No other VFIO device in the set can be open. */ + if (vfio_device_set_open_count(dev_set) > 1) + return false; + + list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) needs_reset |= cur->needs_reset; - } return needs_reset; } -- GitLab From 20ea34710f7b20e218a42d457da618d99e3fc9d6 Mon Sep 17 00:00:00 2001 From: Takashi Iwai Date: Mon, 24 Oct 2022 17:40:54 +0200 Subject: [PATCH 1554/1927] drm/radeon: Add HD-audio component notifier support (v6) This patch adds the support for the notification of HD-audio hotplug via the already existing drm_audio_component framework to radeon driver. This allows us more reliable hotplug notification and ELD transfer without accessing HD-audio bus; it's more efficient, and more importantly, it works without waking up the runtime PM. The implementation is rather simplistic: radeon driver provides the get_eld ops for HD-audio, and it notifies the audio hotplug via pin_eld_notify callback upon each radeon_audio_enable() call. The pin->id is referred as the port number passed to the notifier callback, and the corresponding connector is looked through the encoder list in the get_eld callback in turn. The bind and unbind callbacks handle the device-link so that it assures the PM call order. Also, as a gratis bonus, this patch "fixes" the regression by the recent change in HD-audio to be more strict for the HDMI/DP connection, too. Since the HD-audio HDMI/DP codec requires both the connection bit and the valid ELD to be provided, it started failing on some RADEON gfx boards where the ELD update performed instably. As this change switches the communication to a direct way between the audio and the graphics drivers, now the system receives the proper ELD, and the HDMI/DP hotplug starts working again. [ v2: fix the logic in radeon_audio_component_get_eld to walk the connector list since that is where the EDID lives and we can derive the encoder from the connector because the encoder has not been assigned at this point (i.e., during monitor probe). v3: the component binding is moved outside radeon_audio_init() and _fini(), as those are called from suspend/resume, too. Drop modeset lock calls that caused Oops. Moved Kconfig change so that it can be applied on older kernels. v4: revive drm_modeset_lock*() again, add the missing device_link_remove() call at unbinding v5: squash in mutex fix v6: squash in fix for audio get_eld callback ] Link: https://gitlab.freedesktop.org/drm/amd/-/issues/1569 Signed-off-by: Takashi Iwai Signed-off-by: Alex Deucher --- drivers/gpu/drm/Kconfig | 1 + drivers/gpu/drm/radeon/radeon.h | 8 ++ drivers/gpu/drm/radeon/radeon_audio.c | 117 +++++++++++++++++++++++++ drivers/gpu/drm/radeon/radeon_device.c | 4 + 4 files changed, 130 insertions(+) diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 34f5a092c99e7..fa986075e8fb8 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -243,6 +243,7 @@ config DRM_RADEON select DRM_KMS_HELPER select DRM_TTM select DRM_TTM_HELPER + select SND_HDA_COMPONENT if SND_HDA_CORE select POWER_SUPPLY select HWMON select BACKLIGHT_CLASS_DEVICE diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h index 166c18d62f6d7..2e7161acd443c 100644 --- a/drivers/gpu/drm/radeon/radeon.h +++ b/drivers/gpu/drm/radeon/radeon.h @@ -79,6 +79,7 @@ #include #include +#include #include "radeon_family.h" #include "radeon_mode.h" @@ -1796,6 +1797,9 @@ struct r600_audio { struct radeon_audio_funcs *hdmi_funcs; struct radeon_audio_funcs *dp_funcs; struct radeon_audio_basic_funcs *funcs; + struct drm_audio_component *component; + bool component_registered; + struct mutex component_mutex; }; /* @@ -2994,6 +2998,10 @@ void radeon_irq_kms_set_irq_n_enabled(struct radeon_device *rdev, bool enable, const char *name, unsigned n); +/* Audio component binding */ +void radeon_audio_component_init(struct radeon_device *rdev); +void radeon_audio_component_fini(struct radeon_device *rdev); + #include "radeon_object.h" #endif diff --git a/drivers/gpu/drm/radeon/radeon_audio.c b/drivers/gpu/drm/radeon/radeon_audio.c index 7c5e80d03fc90..d6ccaf24ee0c7 100644 --- a/drivers/gpu/drm/radeon/radeon_audio.c +++ b/drivers/gpu/drm/radeon/radeon_audio.c @@ -23,6 +23,7 @@ */ #include +#include #include #include "dce6_afmt.h" @@ -180,6 +181,8 @@ static struct radeon_audio_funcs dce6_dp_funcs = { .dpms = evergreen_dp_enable, }; +static void radeon_audio_component_notify(struct radeon_device *rdev, int port); + static void radeon_audio_enable(struct radeon_device *rdev, struct r600_audio_pin *pin, u8 enable_mask) { @@ -207,6 +210,8 @@ static void radeon_audio_enable(struct radeon_device *rdev, if (rdev->audio.funcs->enable) rdev->audio.funcs->enable(rdev, pin, enable_mask); + + radeon_audio_component_notify(rdev, pin->id); } static void radeon_audio_interface_init(struct radeon_device *rdev) @@ -721,3 +726,115 @@ unsigned int radeon_audio_decode_dfs_div(unsigned int div) else return 0; } + +/* + * Audio component support + */ +static void radeon_audio_component_notify(struct radeon_device *rdev, int port) +{ + struct drm_audio_component *acomp; + + mutex_lock(&rdev->audio.component_mutex); + acomp = rdev->audio.component; + if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) + acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, + port, -1); + mutex_unlock(&rdev->audio.component_mutex); +} + +static int radeon_audio_component_get_eld(struct device *kdev, int port, + int pipe, bool *enabled, + unsigned char *buf, int max_bytes) +{ + struct drm_device *dev = dev_get_drvdata(kdev); + struct radeon_device *rdev = dev->dev_private; + struct drm_encoder *encoder; + struct radeon_encoder *radeon_encoder; + struct radeon_encoder_atom_dig *dig; + struct drm_connector *connector; + int ret = 0; + + *enabled = false; + if (!rdev->audio.enabled || !rdev->mode_info.mode_config_initialized) + return 0; + + list_for_each_entry(encoder, &rdev->ddev->mode_config.encoder_list, head) { + if (!radeon_encoder_is_digital(encoder)) + continue; + radeon_encoder = to_radeon_encoder(encoder); + dig = radeon_encoder->enc_priv; + if (!dig->pin || dig->pin->id != port) + continue; + connector = radeon_get_connector_for_encoder(encoder); + if (!connector) + continue; + *enabled = true; + ret = drm_eld_size(connector->eld); + memcpy(buf, connector->eld, min(max_bytes, ret)); + break; + } + + return ret; +} + +static const struct drm_audio_component_ops radeon_audio_component_ops = { + .get_eld = radeon_audio_component_get_eld, +}; + +static int radeon_audio_component_bind(struct device *kdev, + struct device *hda_kdev, void *data) +{ + struct drm_device *dev = dev_get_drvdata(kdev); + struct radeon_device *rdev = dev->dev_private; + struct drm_audio_component *acomp = data; + + if (WARN_ON(!device_link_add(hda_kdev, kdev, DL_FLAG_STATELESS))) + return -ENOMEM; + + mutex_lock(&rdev->audio.component_mutex); + acomp->ops = &radeon_audio_component_ops; + acomp->dev = kdev; + rdev->audio.component = acomp; + mutex_unlock(&rdev->audio.component_mutex); + + return 0; +} + +static void radeon_audio_component_unbind(struct device *kdev, + struct device *hda_kdev, void *data) +{ + struct drm_device *dev = dev_get_drvdata(kdev); + struct radeon_device *rdev = dev->dev_private; + struct drm_audio_component *acomp = data; + + device_link_remove(hda_kdev, kdev); + + mutex_lock(&rdev->audio.component_mutex); + rdev->audio.component = NULL; + acomp->ops = NULL; + acomp->dev = NULL; + mutex_unlock(&rdev->audio.component_mutex); +} + +static const struct component_ops radeon_audio_component_bind_ops = { + .bind = radeon_audio_component_bind, + .unbind = radeon_audio_component_unbind, +}; + +void radeon_audio_component_init(struct radeon_device *rdev) +{ + if (rdev->audio.component_registered || + !radeon_audio || !radeon_audio_chipset_supported(rdev)) + return; + + if (!component_add(rdev->dev, &radeon_audio_component_bind_ops)) + rdev->audio.component_registered = true; +} + +void radeon_audio_component_fini(struct radeon_device *rdev) +{ + if (rdev->audio.component_registered) { + component_del(rdev->dev, &radeon_audio_component_bind_ops); + rdev->audio.component_registered = false; + } +} diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c index 262e2bcb30c3d..6344454a77217 100644 --- a/drivers/gpu/drm/radeon/radeon_device.c +++ b/drivers/gpu/drm/radeon/radeon_device.c @@ -1312,6 +1312,7 @@ int radeon_device_init(struct radeon_device *rdev, mutex_init(&rdev->pm.mutex); mutex_init(&rdev->gpu_clock_mutex); mutex_init(&rdev->srbm_mutex); + mutex_init(&rdev->audio.component_mutex); init_rwsem(&rdev->pm.mclk_lock); init_rwsem(&rdev->exclusive_lock); init_waitqueue_head(&rdev->irq.vblank_queue); @@ -1451,6 +1452,8 @@ int radeon_device_init(struct radeon_device *rdev, goto failed; } + radeon_audio_component_init(rdev); + r = radeon_ib_ring_tests(rdev); if (r) DRM_ERROR("ib ring test failed (%d).\n", r); @@ -1513,6 +1516,7 @@ void radeon_device_fini(struct radeon_device *rdev) rdev->shutdown = true; /* evict vram memory */ radeon_bo_evict_vram(rdev); + radeon_audio_component_fini(rdev); radeon_fini(rdev); if (!pci_is_thunderbolt_attached(rdev->pdev)) vga_switcheroo_unregister_client(rdev->pdev); -- GitLab From cb20d6508bff89d85319dbe9a74c50983afc3390 Mon Sep 17 00:00:00 2001 From: Andrew Davis Date: Wed, 26 Oct 2022 11:19:35 -0500 Subject: [PATCH 1555/1927] drm: Move radeon and amdgpu Kconfig options into their directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most Kconfig options to enable a driver are in the Kconfig file inside the relevant directory, move these two to the same. Signed-off-by: Andrew Davis Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/Kconfig | 57 ------------------------------ drivers/gpu/drm/amd/amdgpu/Kconfig | 29 +++++++++++++++ drivers/gpu/drm/radeon/Kconfig | 30 ++++++++++++++++ 3 files changed, 59 insertions(+), 57 deletions(-) diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index fa986075e8fb8..9c2d9495cb3c4 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -233,65 +233,8 @@ source "drivers/gpu/drm/i2c/Kconfig" source "drivers/gpu/drm/arm/Kconfig" -config DRM_RADEON - tristate "ATI Radeon" - depends on DRM && PCI && MMU - depends on AGP || !AGP - select FW_LOADER - select DRM_DISPLAY_DP_HELPER - select DRM_DISPLAY_HELPER - select DRM_KMS_HELPER - select DRM_TTM - select DRM_TTM_HELPER - select SND_HDA_COMPONENT if SND_HDA_CORE - select POWER_SUPPLY - select HWMON - select BACKLIGHT_CLASS_DEVICE - select INTERVAL_TREE - # radeon depends on ACPI_VIDEO when ACPI is enabled, for select to work - # ACPI_VIDEO's dependencies must also be selected. - select INPUT if ACPI - select ACPI_VIDEO if ACPI - # On x86 ACPI_VIDEO also needs ACPI_WMI - select X86_PLATFORM_DEVICES if ACPI && X86 - select ACPI_WMI if ACPI && X86 - help - Choose this option if you have an ATI Radeon graphics card. There - are both PCI and AGP versions. You don't need to choose this to - run the Radeon in plain VGA mode. - - If M is selected, the module will be called radeon. - source "drivers/gpu/drm/radeon/Kconfig" -config DRM_AMDGPU - tristate "AMD GPU" - depends on DRM && PCI && MMU - select FW_LOADER - select DRM_DISPLAY_DP_HELPER - select DRM_DISPLAY_HDMI_HELPER - select DRM_DISPLAY_HELPER - select DRM_KMS_HELPER - select DRM_SCHED - select DRM_TTM - select DRM_TTM_HELPER - select POWER_SUPPLY - select HWMON - select BACKLIGHT_CLASS_DEVICE - select INTERVAL_TREE - select DRM_BUDDY - # amdgpu depends on ACPI_VIDEO when ACPI is enabled, for select to work - # ACPI_VIDEO's dependencies must also be selected. - select INPUT if ACPI - select ACPI_VIDEO if ACPI - # On x86 ACPI_VIDEO also needs ACPI_WMI - select X86_PLATFORM_DEVICES if ACPI && X86 - select ACPI_WMI if ACPI && X86 - help - Choose this option if you have a recent AMD Radeon graphics card. - - If M is selected, the module will be called amdgpu. - source "drivers/gpu/drm/amd/amdgpu/Kconfig" source "drivers/gpu/drm/nouveau/Kconfig" diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig index 7777d55275de8..5fcd510f1abba 100644 --- a/drivers/gpu/drm/amd/amdgpu/Kconfig +++ b/drivers/gpu/drm/amd/amdgpu/Kconfig @@ -1,4 +1,33 @@ # SPDX-License-Identifier: MIT + +config DRM_AMDGPU + tristate "AMD GPU" + depends on DRM && PCI && MMU + select FW_LOADER + select DRM_DISPLAY_DP_HELPER + select DRM_DISPLAY_HDMI_HELPER + select DRM_DISPLAY_HELPER + select DRM_KMS_HELPER + select DRM_SCHED + select DRM_TTM + select DRM_TTM_HELPER + select POWER_SUPPLY + select HWMON + select BACKLIGHT_CLASS_DEVICE + select INTERVAL_TREE + select DRM_BUDDY + # amdgpu depends on ACPI_VIDEO when ACPI is enabled, for select to work + # ACPI_VIDEO's dependencies must also be selected. + select INPUT if ACPI + select ACPI_VIDEO if ACPI + # On x86 ACPI_VIDEO also needs ACPI_WMI + select X86_PLATFORM_DEVICES if ACPI && X86 + select ACPI_WMI if ACPI && X86 + help + Choose this option if you have a recent AMD Radeon graphics card. + + If M is selected, the module will be called amdgpu. + config DRM_AMDGPU_SI bool "Enable amdgpu support for SI parts" depends on DRM_AMDGPU diff --git a/drivers/gpu/drm/radeon/Kconfig b/drivers/gpu/drm/radeon/Kconfig index 52819e7f1fca1..97a277f9a25e3 100644 --- a/drivers/gpu/drm/radeon/Kconfig +++ b/drivers/gpu/drm/radeon/Kconfig @@ -1,4 +1,34 @@ # SPDX-License-Identifier: MIT + +config DRM_RADEON + tristate "ATI Radeon" + depends on DRM && PCI && MMU + depends on AGP || !AGP + select FW_LOADER + select DRM_DISPLAY_DP_HELPER + select DRM_DISPLAY_HELPER + select DRM_KMS_HELPER + select DRM_TTM + select DRM_TTM_HELPER + select SND_HDA_COMPONENT if SND_HDA_CORE + select POWER_SUPPLY + select HWMON + select BACKLIGHT_CLASS_DEVICE + select INTERVAL_TREE + # radeon depends on ACPI_VIDEO when ACPI is enabled, for select to work + # ACPI_VIDEO's dependencies must also be selected. + select INPUT if ACPI + select ACPI_VIDEO if ACPI + # On x86 ACPI_VIDEO also needs ACPI_WMI + select X86_PLATFORM_DEVICES if ACPI && X86 + select ACPI_WMI if ACPI && X86 + help + Choose this option if you have an ATI Radeon graphics card. There + are both PCI and AGP versions. You don't need to choose this to + run the Radeon in plain VGA mode. + + If M is selected, the module will be called radeon. + config DRM_RADEON_USERPTR bool "Always enable userptr support" depends on DRM_RADEON -- GitLab From 7701d10a1eb04025b594484975ee09eee11078b1 Mon Sep 17 00:00:00 2001 From: Yifan Zhang Date: Wed, 26 Oct 2022 10:55:10 +0800 Subject: [PATCH 1556/1927] drm/amdgpu: force read discovery file if set discovery=2 If discovery is set to 2 in module parameters explicitly, the intention is to use the discovery file in FW rather than the one in BIOS, usually because the latter is incorrect. This patch to force read discovery file if set discovery=2. Signed-off-by: Yifan Zhang Reviewed-by: Tim Huang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 3993e61349141..5ea9afaaf4f17 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -305,8 +305,13 @@ static int amdgpu_discovery_init(struct amdgpu_device *adev) goto out; } - if (!amdgpu_discovery_verify_binary_signature(adev->mman.discovery_bin)) { - dev_warn(adev->dev, "get invalid ip discovery binary signature from vram\n"); + if (!amdgpu_discovery_verify_binary_signature(adev->mman.discovery_bin) || amdgpu_discovery == 2) { + /* ignore the discovery binary from vram if discovery=2 in kernel module parameter */ + if (amdgpu_discovery == 2) + dev_info(adev->dev,"force read ip discovery binary from file"); + else + dev_warn(adev->dev, "get invalid ip discovery binary signature from vram\n"); + /* retry read ip discovery binary from file */ r = amdgpu_discovery_read_binary_from_file(adev, adev->mman.discovery_bin); if (r) { -- GitLab From a1903b01b0997626a4f2a60accf92bff5d5379a6 Mon Sep 17 00:00:00 2001 From: Kenneth Feng Date: Thu, 10 Nov 2022 10:52:41 +0800 Subject: [PATCH 1557/1927] drm/amd/pm: skip disabling all smu features on smu_v13_0_10 in suspend skip disabling all smu features on smu_v13_0_10 in suspend Signed-off-by: Kenneth Feng Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 4fe75dd2b329d..320cfce1fef56 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1449,6 +1449,7 @@ static int smu_disable_dpms(struct smu_context *smu) switch (adev->ip_versions[MP1_HWIP][0]) { case IP_VERSION(13, 0, 0): case IP_VERSION(13, 0, 7): + case IP_VERSION(13, 0, 10): return 0; default: break; -- GitLab From 3e931368091f7d5d7902cee9d410eb6db2eea419 Mon Sep 17 00:00:00 2001 From: Tao Zhou Date: Thu, 10 Nov 2022 14:37:08 +0800 Subject: [PATCH 1558/1927] drm/amdgpu: remove the check of init status in psp_ras_initialize The initialized status indicates RAS TA is loaded, but in some cases (such as RAS fatal error) RAS TA could be destroyed although it's not unloaded. Hence we load RAS TA unconditionally here. Signed-off-by: Tao Zhou Reviewed-by: Candice Li Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 2fcb5bfbef89b..0a8c30475dda0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -1526,11 +1526,6 @@ int psp_ras_initialize(struct psp_context *psp) if (amdgpu_sriov_vf(adev)) return 0; - if (psp->ras_context.context.initialized) { - dev_warn(adev->dev, "RAS WARN: TA has already been loaded\n"); - return 0; - } - if (!adev->psp.ras_context.context.bin_desc.size_bytes || !adev->psp.ras_context.context.bin_desc.start_addr) { dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n"); @@ -1602,6 +1597,9 @@ int psp_ras_initialize(struct psp_context *psp) else { if (ras_cmd->ras_status) dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status); + + /* fail to load RAS TA */ + psp->ras_context.context.initialized = false; } return ret; -- GitLab From 4864f2ee9ee2acf4a1009b58fbc62f17fa086d4e Mon Sep 17 00:00:00 2001 From: Tong Liu01 Date: Thu, 10 Nov 2022 17:31:36 +0800 Subject: [PATCH 1559/1927] drm/amdgpu: add vram reservation based on vram_usagebyfirmware_v2_2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move TMR region from top of FB to 2MB for FFBM, so we need to reserve TMR region firstly to make sure TMR can be allocated at 2MB Signed-off-by: Tong Liu01 Acked-by: Christian König Signed-off-by: Alex Deucher --- .../gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c | 104 ++++++++++++++---- drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 51 +++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h | 5 + drivers/gpu/drm/amd/include/atomfirmware.h | 63 +++++++++-- 4 files changed, 191 insertions(+), 32 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c index b81b77a9efa61..9b97fa39d47a2 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atomfirmware.c @@ -101,39 +101,97 @@ void amdgpu_atomfirmware_scratch_regs_init(struct amdgpu_device *adev) } } +static int amdgpu_atomfirmware_allocate_fb_v2_1(struct amdgpu_device *adev, + struct vram_usagebyfirmware_v2_1 *fw_usage, int *usage_bytes) +{ + uint32_t start_addr, fw_size, drv_size; + + start_addr = le32_to_cpu(fw_usage->start_address_in_kb); + fw_size = le16_to_cpu(fw_usage->used_by_firmware_in_kb); + drv_size = le16_to_cpu(fw_usage->used_by_driver_in_kb); + + DRM_DEBUG("atom firmware v2_1 requested %08x %dkb fw %dkb drv\n", + start_addr, + fw_size, + drv_size); + + if ((start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) == + (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION << + ATOM_VRAM_OPERATION_FLAGS_SHIFT)) { + /* Firmware request VRAM reservation for SR-IOV */ + adev->mman.fw_vram_usage_start_offset = (start_addr & + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; + adev->mman.fw_vram_usage_size = fw_size << 10; + /* Use the default scratch size */ + *usage_bytes = 0; + } else { + *usage_bytes = drv_size << 10; + } + return 0; +} + +static int amdgpu_atomfirmware_allocate_fb_v2_2(struct amdgpu_device *adev, + struct vram_usagebyfirmware_v2_2 *fw_usage, int *usage_bytes) +{ + uint32_t fw_start_addr, fw_size, drv_start_addr, drv_size; + + fw_start_addr = le32_to_cpu(fw_usage->fw_region_start_address_in_kb); + fw_size = le16_to_cpu(fw_usage->used_by_firmware_in_kb); + + drv_start_addr = le32_to_cpu(fw_usage->driver_region0_start_address_in_kb); + drv_size = le32_to_cpu(fw_usage->used_by_driver_region0_in_kb); + + DRM_DEBUG("atom requested fw start at %08x %dkb and drv start at %08x %dkb\n", + fw_start_addr, + fw_size, + drv_start_addr, + drv_size); + + if ((fw_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) { + /* Firmware request VRAM reservation for SR-IOV */ + adev->mman.fw_vram_usage_start_offset = (fw_start_addr & + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; + adev->mman.fw_vram_usage_size = fw_size << 10; + } + + if ((drv_start_addr & (ATOM_VRAM_BLOCK_NEEDS_NO_RESERVATION << 30)) == 0) { + /* driver request VRAM reservation for SR-IOV */ + adev->mman.drv_vram_usage_start_offset = (drv_start_addr & + (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; + adev->mman.drv_vram_usage_size = drv_size << 10; + } + + *usage_bytes = 0; + return 0; +} + int amdgpu_atomfirmware_allocate_fb_scratch(struct amdgpu_device *adev) { struct atom_context *ctx = adev->mode_info.atom_context; int index = get_index_into_master_table(atom_master_list_of_data_tables_v2_1, vram_usagebyfirmware); - struct vram_usagebyfirmware_v2_1 *firmware_usage; - uint32_t start_addr, size; + struct vram_usagebyfirmware_v2_1 *fw_usage_v2_1; + struct vram_usagebyfirmware_v2_2 *fw_usage_v2_2; uint16_t data_offset; + uint8_t frev, crev; int usage_bytes = 0; - if (amdgpu_atom_parse_data_header(ctx, index, NULL, NULL, NULL, &data_offset)) { - firmware_usage = (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset); - DRM_DEBUG("atom firmware requested %08x %dkb fw %dkb drv\n", - le32_to_cpu(firmware_usage->start_address_in_kb), - le16_to_cpu(firmware_usage->used_by_firmware_in_kb), - le16_to_cpu(firmware_usage->used_by_driver_in_kb)); - - start_addr = le32_to_cpu(firmware_usage->start_address_in_kb); - size = le16_to_cpu(firmware_usage->used_by_firmware_in_kb); - - if ((uint32_t)(start_addr & ATOM_VRAM_OPERATION_FLAGS_MASK) == - (uint32_t)(ATOM_VRAM_BLOCK_SRIOV_MSG_SHARE_RESERVATION << - ATOM_VRAM_OPERATION_FLAGS_SHIFT)) { - /* Firmware request VRAM reservation for SR-IOV */ - adev->mman.fw_vram_usage_start_offset = (start_addr & - (~ATOM_VRAM_OPERATION_FLAGS_MASK)) << 10; - adev->mman.fw_vram_usage_size = size << 10; - /* Use the default scratch size */ - usage_bytes = 0; - } else { - usage_bytes = le16_to_cpu(firmware_usage->used_by_driver_in_kb) << 10; + if (amdgpu_atom_parse_data_header(ctx, index, NULL, &frev, &crev, &data_offset)) { + if (frev == 2 && crev == 1) { + fw_usage_v2_1 = + (struct vram_usagebyfirmware_v2_1 *)(ctx->bios + data_offset); + amdgpu_atomfirmware_allocate_fb_v2_1(adev, + fw_usage_v2_1, + &usage_bytes); + } else if (frev >= 2 && crev >= 2) { + fw_usage_v2_2 = + (struct vram_usagebyfirmware_v2_2 *)(ctx->bios + data_offset); + amdgpu_atomfirmware_allocate_fb_v2_2(adev, + fw_usage_v2_2, + &usage_bytes); } } + ctx->scratch_size_bytes = 0; if (usage_bytes == 0) usage_bytes = 20 * 1024; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index c9ceea46711ef..02a8f25ae1345 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1561,6 +1561,23 @@ static void amdgpu_ttm_fw_reserve_vram_fini(struct amdgpu_device *adev) NULL, &adev->mman.fw_vram_usage_va); } +/* + * Driver Reservation functions + */ +/** + * amdgpu_ttm_drv_reserve_vram_fini - free drv reserved vram + * + * @adev: amdgpu_device pointer + * + * free drv reserved vram if it has been reserved. + */ +static void amdgpu_ttm_drv_reserve_vram_fini(struct amdgpu_device *adev) +{ + amdgpu_bo_free_kernel(&adev->mman.drv_vram_usage_reserved_bo, + NULL, + NULL); +} + /** * amdgpu_ttm_fw_reserve_vram_init - create bo vram reservation from fw * @@ -1587,6 +1604,31 @@ static int amdgpu_ttm_fw_reserve_vram_init(struct amdgpu_device *adev) &adev->mman.fw_vram_usage_va); } +/** + * amdgpu_ttm_drv_reserve_vram_init - create bo vram reservation from driver + * + * @adev: amdgpu_device pointer + * + * create bo vram reservation from drv. + */ +static int amdgpu_ttm_drv_reserve_vram_init(struct amdgpu_device *adev) +{ + uint64_t vram_size = adev->gmc.visible_vram_size; + + adev->mman.drv_vram_usage_reserved_bo = NULL; + + if (adev->mman.drv_vram_usage_size == 0 || + adev->mman.drv_vram_usage_size > vram_size) + return 0; + + return amdgpu_bo_create_kernel_at(adev, + adev->mman.drv_vram_usage_start_offset, + adev->mman.drv_vram_usage_size, + AMDGPU_GEM_DOMAIN_VRAM, + &adev->mman.drv_vram_usage_reserved_bo, + NULL); +} + /* * Memoy training reservation functions */ @@ -1754,6 +1796,14 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) return r; } + /* + *The reserved vram for driver must be pinned to the specified + *place on the VRAM, so reserve it early. + */ + r = amdgpu_ttm_drv_reserve_vram_init(adev); + if (r) + return r; + /* * only NAVI10 and onwards ASIC support for IP discovery. * If IP discovery enabled, a block of memory should be @@ -1879,6 +1929,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_bo_free_kernel(&adev->mman.sdma_access_bo, NULL, &adev->mman.sdma_access_ptr); amdgpu_ttm_fw_reserve_vram_fini(adev); + amdgpu_ttm_drv_reserve_vram_fini(adev); if (drm_dev_enter(adev_to_drm(adev), &idx)) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index 6a70818039dda..7c38843f411e5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -84,6 +84,11 @@ struct amdgpu_mman { struct amdgpu_bo *fw_vram_usage_reserved_bo; void *fw_vram_usage_va; + /* driver VRAM reservation */ + u64 drv_vram_usage_start_offset; + u64 drv_vram_usage_size; + struct amdgpu_bo *drv_vram_usage_reserved_bo; + /* PAGE_SIZE'd BO for process memory r/w over SDMA. */ struct amdgpu_bo *sdma_access_bo; void *sdma_access_ptr; diff --git a/drivers/gpu/drm/amd/include/atomfirmware.h b/drivers/gpu/drm/amd/include/atomfirmware.h index ff855cb21d3f9..bbe1337a8cee3 100644 --- a/drivers/gpu/drm/amd/include/atomfirmware.h +++ b/drivers/gpu/drm/amd/include/atomfirmware.h @@ -705,20 +705,65 @@ struct atom_gpio_pin_lut_v2_1 }; -/* - *************************************************************************** - Data Table vram_usagebyfirmware structure - *************************************************************************** -*/ +/* + * VBIOS/PRE-OS always reserve a FB region at the top of frame buffer. driver should not write + * access that region. driver can allocate their own reservation region as long as it does not + * overlap firwmare's reservation region. + * if (pre-NV1X) atom data table firmwareInfoTable version < 3.3: + * in this case, atom data table vram_usagebyfirmwareTable version always <= 2.1 + * if VBIOS/UEFI GOP is posted: + * VBIOS/UEFIGOP update used_by_firmware_in_kb = total reserved size by VBIOS + * update start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; + * ( total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10) + * driver can allocate driver reservation region under firmware reservation, + * used_by_driver_in_kb = driver reservation size + * driver reservation start address = (start_address_in_kb - used_by_driver_in_kb) + * Comment1[hchan]: There is only one reservation at the beginning of the FB reserved by + * host driver. Host driver would overwrite the table with the following + * used_by_firmware_in_kb = total reserved size for pf-vf info exchange and + * set SRIOV_MSG_SHARE_RESERVATION mask start_address_in_kb = 0 + * else there is no VBIOS reservation region: + * driver must allocate driver reservation region at top of FB. + * driver set used_by_driver_in_kb = driver reservation size + * driver reservation start address = (total_mem_size_in_kb - used_by_driver_in_kb) + * same as Comment1 + * else (NV1X and after): + * if VBIOS/UEFI GOP is posted: + * VBIOS/UEFIGOP update: + * used_by_firmware_in_kb = atom_firmware_Info_v3_3.fw_reserved_size_in_kb; + * start_address_in_kb = total_mem_size_in_kb - used_by_firmware_in_kb; + * (total_mem_size_in_kb = reg(CONFIG_MEMSIZE)<<10) + * if vram_usagebyfirmwareTable version <= 2.1: + * driver can allocate driver reservation region under firmware reservation, + * driver set used_by_driver_in_kb = driver reservation size + * driver reservation start address = start_address_in_kb - used_by_driver_in_kb + * same as Comment1 + * else driver can: + * allocate it reservation any place as long as it does overlap pre-OS FW reservation area + * set used_by_driver_region0_in_kb = driver reservation size + * set driver_region0_start_address_in_kb = driver reservation region start address + * Comment2[hchan]: Host driver can set used_by_firmware_in_kb and start_address_in_kb to + * zero as the reservation for VF as it doesn’t exist. And Host driver should also + * update atom_firmware_Info table to remove the same VBIOS reservation as well. + */ struct vram_usagebyfirmware_v2_1 { - struct atom_common_table_header table_header; - uint32_t start_address_in_kb; - uint16_t used_by_firmware_in_kb; - uint16_t used_by_driver_in_kb; + struct atom_common_table_header table_header; + uint32_t start_address_in_kb; + uint16_t used_by_firmware_in_kb; + uint16_t used_by_driver_in_kb; }; +struct vram_usagebyfirmware_v2_2 { + struct atom_common_table_header table_header; + uint32_t fw_region_start_address_in_kb; + uint16_t used_by_firmware_in_kb; + uint16_t reserved; + uint32_t driver_region0_start_address_in_kb; + uint32_t used_by_driver_region0_in_kb; + uint32_t reserved32[7]; +}; /* *************************************************************************** -- GitLab From ecc9b6e15af2ffc8b9c0e6f15160eb8aaea36af5 Mon Sep 17 00:00:00 2001 From: Gavin Wan Date: Wed, 9 Nov 2022 13:12:42 -0500 Subject: [PATCH 1560/1927] drm/amdgpu: Ignore stop rlc on SRIOV environment. For SRIOV, the guest driver should not do stop rlc. The host handles programing RLC. On SRIOV, the stop rlc will be hang (RLC related registers are blocked by policy) when the RLCG interface is not enabled. Reviewed-by: Yang Wang Acked-by: Alex Deucher Signed-off-by: Gavin Wan Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 320cfce1fef56..9492dd638aea5 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1518,7 +1518,7 @@ static int smu_disable_dpms(struct smu_context *smu) } if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 4, 2) && - adev->gfx.rlc.funcs->stop) + !amdgpu_sriov_vf(adev) && adev->gfx.rlc.funcs->stop) adev->gfx.rlc.funcs->stop(adev); return ret; -- GitLab From 0a204ce0e698e5b94a064de6848b8c4641860f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Syrj=C3=A4l=C3=A4?= Date: Mon, 7 Nov 2022 21:25:39 +0200 Subject: [PATCH 1561/1927] drm/amdgpu: Use drm_mode_init() for on-stack modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initialize on-stack modes with drm_mode_init() to guarantee no stack garbage in the list head, or that we aren't copying over another mode's list head. Based on the following cocci script, with manual fixups: @decl@ identifier M; expression E; @@ - struct drm_display_mode M = E; + struct drm_display_mode M; @@ identifier decl.M; expression decl.E; statement S, S1; @@ struct drm_display_mode M; ... when != S + drm_mode_init(&M, &E); + S1 @@ expression decl.E; @@ - &*E + E Cc: Harry Wentland Cc: Leo Li Cc: Rodrigo Siqueira Cc: Alex Deucher Cc: amd-gfx@lists.freedesktop.org Reviewed-by: Harry Wentland Signed-off-by: Ville Syrjälä Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 5114726dedf7a..65715e6e3dd5e 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -5684,7 +5684,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, const struct drm_connector_state *con_state = dm_state ? &dm_state->base : NULL; struct dc_stream_state *stream = NULL; - struct drm_display_mode mode = *drm_mode; + struct drm_display_mode mode; struct drm_display_mode saved_mode; struct drm_display_mode *freesync_mode = NULL; bool native_mode_found = false; @@ -5699,6 +5699,7 @@ create_stream_for_sink(struct amdgpu_dm_connector *aconnector, struct dc_sink *sink = NULL; + drm_mode_init(&mode, drm_mode); memset(&saved_mode, 0, sizeof(saved_mode)); if (aconnector == NULL) { -- GitLab From 7989d0b731858611289fbfd7c8fb1cdeda30cbc6 Mon Sep 17 00:00:00 2001 From: Paulo Miguel Almeida Date: Wed, 9 Nov 2022 20:33:34 +1300 Subject: [PATCH 1562/1927] drm/amdgpu: Replace one-element array with flex-array member One-element arrays are deprecated, and we are replacing them with flexible array members instead. So, replace one-element array with flexible-array member in structs _ATOM_CONNECTOR_DEVICE_TAG_RECORD, _ATOM_OBJECT_GPIO_CNTL_RECORD, _ATOM_BRACKET_LAYOUT_RECORD, _ATOM_BRACKET_LAYOUT_RECORD, _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3, _ATOM_FUSION_SYSTEM_INFO_V3, _ATOM_I2C_DATA_RECORD, _ATOM_I2C_DEVICE_SETUP_INFO, _ATOM_ASIC_MVDD_INFO and refactor the rest of the code accordingly. While at it, removed a redundant casting. Important to mention is that doing a build before/after this patch results in no binary output differences. This helps with the ongoing efforts to tighten the FORTIFY_SOURCE routines on memcpy() and help us make progress towards globally enabling -fstrict-flex-arrays=3 [1]. Link: https://github.com/KSPP/linux/issues/79 Link: https://github.com/KSPP/linux/issues/238 Link: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836 [1] Signed-off-by: Paulo Miguel Almeida Signed-off-by: Alex Deucher --- .../gpu/drm/amd/display/dc/bios/bios_parser.c | 5 ++--- drivers/gpu/drm/amd/include/atombios.h | 16 ++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c index 39dd8b2dc254c..6b9e64cd43799 100644 --- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c +++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c @@ -2606,8 +2606,7 @@ static enum bp_result update_slot_layout_info( for (;;) { - record_header = (ATOM_COMMON_RECORD_HEADER *) - GET_IMAGE(ATOM_COMMON_RECORD_HEADER, record_offset); + record_header = GET_IMAGE(ATOM_COMMON_RECORD_HEADER, record_offset); if (record_header == NULL) { result = BP_RESULT_BADBIOSTABLE; break; @@ -2621,7 +2620,7 @@ static enum bp_result update_slot_layout_info( if (record_header->ucRecordType == ATOM_BRACKET_LAYOUT_RECORD_TYPE && - sizeof(ATOM_BRACKET_LAYOUT_RECORD) + struct_size(record, asConnInfo, 1) <= record_header->ucRecordSize) { record = (ATOM_BRACKET_LAYOUT_RECORD *) (record_header); diff --git a/drivers/gpu/drm/amd/include/atombios.h b/drivers/gpu/drm/amd/include/atombios.h index 55ae93c1e365a..60c44a8a067ff 100644 --- a/drivers/gpu/drm/amd/include/atombios.h +++ b/drivers/gpu/drm/amd/include/atombios.h @@ -4733,7 +4733,7 @@ typedef struct _ATOM_CONNECTOR_DEVICE_TAG_RECORD ATOM_COMMON_RECORD_HEADER sheader; UCHAR ucNumberOfDevice; UCHAR ucReserved; - ATOM_CONNECTOR_DEVICE_TAG asDeviceTag[1]; //This Id is same as "ATOM_DEVICE_XXX_SUPPORT", 1 is only for allocation + ATOM_CONNECTOR_DEVICE_TAG asDeviceTag[]; //This Id is same as "ATOM_DEVICE_XXX_SUPPORT" }ATOM_CONNECTOR_DEVICE_TAG_RECORD; @@ -4793,7 +4793,7 @@ typedef struct _ATOM_OBJECT_GPIO_CNTL_RECORD ATOM_COMMON_RECORD_HEADER sheader; UCHAR ucFlags; // Future expnadibility UCHAR ucNumberOfPins; // Number of GPIO pins used to control the object - ATOM_GPIO_PIN_CONTROL_PAIR asGpio[1]; // the real gpio pin pair determined by number of pins ucNumberOfPins + ATOM_GPIO_PIN_CONTROL_PAIR asGpio[]; // the real gpio pin pair determined by number of pins ucNumberOfPins }ATOM_OBJECT_GPIO_CNTL_RECORD; //Definitions for GPIO pin state @@ -4982,7 +4982,7 @@ typedef struct _ATOM_BRACKET_LAYOUT_RECORD UCHAR ucWidth; UCHAR ucConnNum; UCHAR ucReserved; - ATOM_CONNECTOR_LAYOUT_INFO asConnInfo[1]; + ATOM_CONNECTOR_LAYOUT_INFO asConnInfo[]; }ATOM_BRACKET_LAYOUT_RECORD; @@ -5161,7 +5161,7 @@ typedef struct _ATOM_GPIO_VOLTAGE_OBJECT_V3 UCHAR ucPhaseDelay; // phase delay in unit of micro second UCHAR ucReserved; ULONG ulGpioMaskVal; // GPIO Mask value - VOLTAGE_LUT_ENTRY_V2 asVolGpioLut[1]; + VOLTAGE_LUT_ENTRY_V2 asVolGpioLut[]; }ATOM_GPIO_VOLTAGE_OBJECT_V3; typedef struct _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 @@ -5171,7 +5171,7 @@ typedef struct _ATOM_LEAKAGE_VOLTAGE_OBJECT_V3 UCHAR ucLeakageEntryNum; // indicate the entry number of LeakageId/Voltage Lut table UCHAR ucReserved[2]; ULONG ulMaxVoltageLevel; - LEAKAGE_VOLTAGE_LUT_ENTRY_V2 asLeakageIdLut[1]; + LEAKAGE_VOLTAGE_LUT_ENTRY_V2 asLeakageIdLut[]; }ATOM_LEAKAGE_VOLTAGE_OBJECT_V3; @@ -6599,7 +6599,7 @@ typedef struct _ATOM_FUSION_SYSTEM_INFO_V3 typedef struct _ATOM_I2C_DATA_RECORD { UCHAR ucNunberOfBytes; //Indicates how many bytes SW needs to write to the external ASIC for one block, besides to "Start" and "Stop" - UCHAR ucI2CData[1]; //I2C data in bytes, should be less than 16 bytes usually + UCHAR ucI2CData[]; //I2C data in bytes, should be less than 16 bytes usually }ATOM_I2C_DATA_RECORD; @@ -6610,14 +6610,14 @@ typedef struct _ATOM_I2C_DEVICE_SETUP_INFO UCHAR ucSSChipID; //SS chip being used UCHAR ucSSChipSlaveAddr; //Slave Address to set up this SS chip UCHAR ucNumOfI2CDataRecords; //number of data block - ATOM_I2C_DATA_RECORD asI2CData[1]; + ATOM_I2C_DATA_RECORD asI2CData[]; }ATOM_I2C_DEVICE_SETUP_INFO; //========================================================================================== typedef struct _ATOM_ASIC_MVDD_INFO { ATOM_COMMON_TABLE_HEADER sHeader; - ATOM_I2C_DEVICE_SETUP_INFO asI2CSetup[1]; + ATOM_I2C_DEVICE_SETUP_INFO asI2CSetup[]; }ATOM_ASIC_MVDD_INFO; //========================================================================================== -- GitLab From fe6d2ca6f3f806f2f03e961d80cdbf27073868c8 Mon Sep 17 00:00:00 2001 From: Liu Jian Date: Wed, 9 Nov 2022 17:31:22 +0800 Subject: [PATCH 1563/1927] drm/amd/display: delete the duplicate .set_odm_bypass initialization in dcn314_tg_funcs Fix below sparse warning: drivers/gpu/drm/amd/amdgpu/../display/dc/dcn314/dcn314_optc.c:244:18: warning: Initializer entry defined twice drivers/gpu/drm/amd/amdgpu/../display/dc/dcn314/dcn314_optc.c:257:18: also defined here Fixes: 5ade1b951dec ("drm/amd/display: Add OTG/ODM functions") Signed-off-by: Liu Jian Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c index 47eb162f1a758..58d38de6a0f89 100644 --- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c +++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c @@ -241,7 +241,6 @@ static struct timing_generator_funcs dcn314_tg_funcs = { .set_dsc_config = optc3_set_dsc_config, .get_dsc_status = optc2_get_dsc_status, .set_dwb_source = NULL, - .set_odm_bypass = optc3_set_odm_bypass, .set_odm_combine = optc314_set_odm_combine, .get_optc_source = optc2_get_optc_source, .set_out_mux = optc3_set_out_mux, -- GitLab From 2ebf61f2cfb9a11bc17db30df3e675a4cd7418d3 Mon Sep 17 00:00:00 2001 From: Dong Chenchen Date: Thu, 10 Nov 2022 22:33:14 +0800 Subject: [PATCH 1564/1927] drm/amdgpu: Fix memory leak in amdgpu_cs_pass1 When p->gang_size equals 0, amdgpu_cs_pass1() will return directly without freeing chunk_array, which will cause a memory leak issue, this patch fixes it. Fixes: 4624459c84d7 ("drm/amdgpu: add gang submit frontend v6") Reviewed-by: Luben Tuikov Signed-off-by: Dong Chenchen Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index d371000a5727e..255d545e55249 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -287,8 +287,10 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, } } - if (!p->gang_size) - return -EINVAL; + if (!p->gang_size) { + ret = -EINVAL; + goto free_partial_kdata; + } for (i = 0; i < p->gang_size; ++i) { ret = amdgpu_job_alloc(p->adev, num_ibs[i], &p->jobs[i], vm); -- GitLab From 6510c78490c490a6636e48b61eeaa6fb65981f4b Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Sat, 29 Oct 2022 19:34:50 +0800 Subject: [PATCH 1565/1927] riscv: process: fix kernel info leakage thread_struct's s[12] may contain random kernel memory content, which may be finally leaked to userspace. This is a security hole. Fix it by clearing the s[12] array in thread_struct when fork. As for kthread case, it's better to clear the s[12] array as well. Fixes: 7db91e57a0ac ("RISC-V: Task implementation") Signed-off-by: Jisheng Zhang Tested-by: Guo Ren Link: https://lore.kernel.org/r/20221029113450.4027-1-jszhang@kernel.org Reviewed-by: Guo Ren Link: https://lore.kernel.org/r/CAJF2gTSdVyAaM12T%2B7kXAdRPGS4VyuO08X1c7paE-n4Fr8OtRA@mail.gmail.com/ Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/process.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index b0c63e8e867ef..8955f2432c2d8 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -164,6 +164,8 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) unsigned long tls = args->tls; struct pt_regs *childregs = task_pt_regs(p); + memset(&p->thread.s, 0, sizeof(p->thread.s)); + /* p->thread holds context to be restored by __switch_to() */ if (unlikely(args->fn)) { /* Kernel thread */ -- GitLab From 6ce3df596be25d95c2cddc9e1b673394948e3732 Mon Sep 17 00:00:00 2001 From: Florian Fainelli Date: Wed, 9 Nov 2022 15:19:07 -0800 Subject: [PATCH 1566/1927] MAINTAINERS: Move Vivien to CREDITS Last patch from Vivien was nearly 3 years ago and he has not reviewed or responded to DSA patches since then, move to CREDITS. Signed-off-by: Florian Fainelli Reviewed-by: Andrew Lunn Link: https://lore.kernel.org/r/20221109231907.621678-1-f.fainelli@gmail.com Signed-off-by: Jakub Kicinski --- CREDITS | 5 +++++ MAINTAINERS | 2 -- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index 1841184c834d2..54672cbcd7191 100644 --- a/CREDITS +++ b/CREDITS @@ -918,6 +918,11 @@ S: Ottawa, Ontario S: K1N 6Z9 S: CANADA +N: Vivien Didelot +E: vivien.didelot@gmail.com +D: DSA framework and MV88E6XXX driver +S: Montreal, Quebec, Canada + N: Jeff Dike E: jdike@karaya.com W: http://user-mode-linux.sourceforge.net diff --git a/MAINTAINERS b/MAINTAINERS index e1bc31a6624b1..239407627be28 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12226,7 +12226,6 @@ F: arch/mips/boot/dts/img/pistachio* MARVELL 88E6XXX ETHERNET SWITCH FABRIC DRIVER M: Andrew Lunn -M: Vivien Didelot L: netdev@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/net/dsa/marvell.txt @@ -14324,7 +14323,6 @@ F: drivers/net/wireless/ NETWORKING [DSA] M: Andrew Lunn -M: Vivien Didelot M: Florian Fainelli M: Vladimir Oltean S: Maintained -- GitLab From 50f4dd657a0fcf90aa8da8dc2794a8100ff4c37c Mon Sep 17 00:00:00 2001 From: Jisheng Zhang Date: Tue, 1 Nov 2022 02:29:43 +0800 Subject: [PATCH 1567/1927] riscv: vdso: fix build with llvm Even after commit 89fd4a1df829 ("riscv: jump_label: mark arguments as const to satisfy asm constraints"), building with CC_OPTIMIZE_FOR_SIZE + LLVM=1 can reproduce below build error: CC arch/riscv/kernel/vdso/vgettimeofday.o In file included from :4: In file included from lib/vdso/gettimeofday.c:5: In file included from include/vdso/datapage.h:17: In file included from include/vdso/processor.h:10: In file included from arch/riscv/include/asm/vdso/processor.h:7: In file included from include/linux/jump_label.h:112: arch/riscv/include/asm/jump_label.h:42:3: error: invalid operand for inline asm constraint 'i' " .option push \n\t" ^ 1 error generated. I think the problem is when "-Os" is passed as CFLAGS, it's removed by "CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os" which is introduced in commit e05d57dcb8c7 ("riscv: Fixup __vdso_gettimeofday broke dynamic ftrace"), thus no optimization at all for vgettimeofday.c arm64 does remove "-Os" as well, but it forces "-O2" after removing "-Os". I compared the generated vgettimeofday.o with "-O2" and "-Os", I think no big performance difference. So let's tell the kbuild not to remove "-Os" rather than follow arm64 style. vdso related performance can be improved a lot when building kernel with CC_OPTIMIZE_FOR_SIZE after this commit, ("-Os" VS no optimization) Fixes: e05d57dcb8c7 ("riscv: Fixup __vdso_gettimeofday broke dynamic ftrace") Signed-off-by: Jisheng Zhang Tested-by: Conor Dooley Link: https://lore.kernel.org/r/20221031182943.2453-1-jszhang@kernel.org Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vdso/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index f2e065671e4d5..84ac0fe612e79 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -30,7 +30,7 @@ obj-y += vdso.o CPPFLAGS_vdso.lds += -P -C -U$(ARCH) # Disable -pg to prevent insert call site -CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os +CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) # Disable profiling and instrumentation for VDSO code GCOV_PROFILE := n -- GitLab From 550b33cfd445296868a478e8413ffb2e963eed32 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 10 Nov 2022 10:36:20 +0100 Subject: [PATCH 1568/1927] arm64: efi: Force the use of SetVirtualAddressMap() on Altra machines Ampere Altra machines are reported to misbehave when the SetTime() EFI runtime service is called after ExitBootServices() but before calling SetVirtualAddressMap(). Given that the latter is horrid, pointless and explicitly documented as optional by the EFI spec, we no longer invoke it at boot if the configured size of the VA space guarantees that the EFI runtime memory regions can remain mapped 1:1 like they are at boot time. On Ampere Altra machines, this results in SetTime() calls issued by the rtc-efi driver triggering synchronous exceptions during boot. We can now recover from those without bringing down the system entirely, due to commit 23715a26c8d81291 ("arm64: efi: Recover from synchronous exceptions occurring in firmware"). However, it would be better to avoid the issue entirely, given that the firmware appears to remain in a funny state after this. So attempt to identify these machines based on the 'family' field in the type #1 SMBIOS record, and call SetVirtualAddressMap() unconditionally in that case. Tested-by: Alexandru Elisei Signed-off-by: Ard Biesheuvel --- drivers/firmware/efi/libstub/Makefile | 2 +- drivers/firmware/efi/libstub/arm64-stub.c | 17 +++++++- drivers/firmware/efi/libstub/efistub.h | 28 +++++++++++++ drivers/firmware/efi/libstub/smbios.c | 48 +++++++++++++++++++++++ include/linux/efi.h | 1 + 5 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 drivers/firmware/efi/libstub/smbios.c diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index b1601aad7e1a8..ef5045a53ce09 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -82,7 +82,7 @@ $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE lib-$(CONFIG_EFI_GENERIC_STUB) += efi-stub.o string.o intrinsics.o systable.o lib-$(CONFIG_ARM) += arm32-stub.o -lib-$(CONFIG_ARM64) += arm64-stub.o +lib-$(CONFIG_ARM64) += arm64-stub.o smbios.o lib-$(CONFIG_X86) += x86-stub.o lib-$(CONFIG_RISCV) += riscv-stub.o lib-$(CONFIG_LOONGARCH) += loongarch-stub.o diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c index 259e4b852d632..f9de5217ea65e 100644 --- a/drivers/firmware/efi/libstub/arm64-stub.c +++ b/drivers/firmware/efi/libstub/arm64-stub.c @@ -15,6 +15,21 @@ #include "efistub.h" +static bool system_needs_vamap(void) +{ + const u8 *type1_family = efi_get_smbios_string(1, family); + + /* + * Ampere Altra machines crash in SetTime() if SetVirtualAddressMap() + * has not been called prior. + */ + if (!type1_family || strcmp(type1_family, "Altra")) + return false; + + efi_warn("Working around broken SetVirtualAddressMap()\n"); + return true; +} + efi_status_t check_platform_features(void) { u64 tg; @@ -24,7 +39,7 @@ efi_status_t check_platform_features(void) * UEFI runtime regions 1:1 and so calling SetVirtualAddressMap() is * unnecessary. */ - if (VA_BITS_MIN >= 48) + if (VA_BITS_MIN >= 48 && !system_needs_vamap()) efi_novamap = true; /* UEFI mandates support for 4 KB granularity, no need to check */ diff --git a/drivers/firmware/efi/libstub/efistub.h b/drivers/firmware/efi/libstub/efistub.h index a30fb5d8ef05a..eb03d5a9aac88 100644 --- a/drivers/firmware/efi/libstub/efistub.h +++ b/drivers/firmware/efi/libstub/efistub.h @@ -975,4 +975,32 @@ efi_enable_reset_attack_mitigation(void) { } void efi_retrieve_tpm2_eventlog(void); +struct efi_smbios_record { + u8 type; + u8 length; + u16 handle; +}; + +struct efi_smbios_type1_record { + struct efi_smbios_record header; + + u8 manufacturer; + u8 product_name; + u8 version; + u8 serial_number; + efi_guid_t uuid; + u8 wakeup_type; + u8 sku_number; + u8 family; +}; + +#define efi_get_smbios_string(__type, __name) ({ \ + int size = sizeof(struct efi_smbios_type ## __type ## _record); \ + int off = offsetof(struct efi_smbios_type ## __type ## _record, \ + __name); \ + __efi_get_smbios_string(__type, off, size); \ +}) + +const u8 *__efi_get_smbios_string(u8 type, int offset, int recsize); + #endif diff --git a/drivers/firmware/efi/libstub/smbios.c b/drivers/firmware/efi/libstub/smbios.c new file mode 100644 index 0000000000000..460418b7f5f5e --- /dev/null +++ b/drivers/firmware/efi/libstub/smbios.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright 2022 Google LLC +// Author: Ard Biesheuvel + +#include + +#include "efistub.h" + +typedef struct efi_smbios_protocol efi_smbios_protocol_t; + +struct efi_smbios_protocol { + efi_status_t (__efiapi *add)(efi_smbios_protocol_t *, efi_handle_t, + u16 *, struct efi_smbios_record *); + efi_status_t (__efiapi *update_string)(efi_smbios_protocol_t *, u16 *, + unsigned long *, u8 *); + efi_status_t (__efiapi *remove)(efi_smbios_protocol_t *, u16); + efi_status_t (__efiapi *get_next)(efi_smbios_protocol_t *, u16 *, u8 *, + struct efi_smbios_record **, + efi_handle_t *); + + u8 major_version; + u8 minor_version; +}; + +const u8 *__efi_get_smbios_string(u8 type, int offset, int recsize) +{ + struct efi_smbios_record *record; + efi_smbios_protocol_t *smbios; + efi_status_t status; + u16 handle = 0xfffe; + const u8 *strtable; + + status = efi_bs_call(locate_protocol, &EFI_SMBIOS_PROTOCOL_GUID, NULL, + (void **)&smbios) ?: + efi_call_proto(smbios, get_next, &handle, &type, &record, NULL); + if (status != EFI_SUCCESS) + return NULL; + + strtable = (u8 *)record + recsize; + for (int i = 1; i < ((u8 *)record)[offset]; i++) { + int len = strlen(strtable); + + if (!len) + return NULL; + strtable += len + 1; + } + return strtable; +} diff --git a/include/linux/efi.h b/include/linux/efi.h index 929d559ad41d2..7603fc58c47cd 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -389,6 +389,7 @@ void efi_native_runtime_setup(void); #define EFI_LOAD_FILE2_PROTOCOL_GUID EFI_GUID(0x4006c0c1, 0xfcb3, 0x403e, 0x99, 0x6d, 0x4a, 0x6c, 0x87, 0x24, 0xe0, 0x6d) #define EFI_RT_PROPERTIES_TABLE_GUID EFI_GUID(0xeb66918a, 0x7eef, 0x402a, 0x84, 0x2e, 0x93, 0x1d, 0x21, 0xc3, 0x8a, 0xe9) #define EFI_DXE_SERVICES_TABLE_GUID EFI_GUID(0x05ad34ba, 0x6f02, 0x4214, 0x95, 0x2e, 0x4d, 0xa0, 0x39, 0x8e, 0x2b, 0xb9) +#define EFI_SMBIOS_PROTOCOL_GUID EFI_GUID(0x03583ff6, 0xcb36, 0x4940, 0x94, 0x7e, 0xb9, 0xb3, 0x9f, 0x4a, 0xfa, 0xf7) #define EFI_IMAGE_SECURITY_DATABASE_GUID EFI_GUID(0xd719b2cb, 0x3d3a, 0x4596, 0xa3, 0xbc, 0xda, 0xd0, 0x0e, 0x67, 0x65, 0x6f) #define EFI_SHIM_LOCK_GUID EFI_GUID(0x605dab50, 0xe046, 0x4300, 0xab, 0xb6, 0x3d, 0xd8, 0x10, 0xdd, 0x8b, 0x23) -- GitLab From 9b9eaee9828fe98b030cf43ac50065a54a2f5d52 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Sun, 6 Nov 2022 15:53:54 +0100 Subject: [PATCH 1569/1927] arm64: efi: Fix handling of misaligned runtime regions and drop warning Currently, when mapping the EFI runtime regions in the EFI page tables, we complain about misaligned regions in a rather noisy way, using WARN(). Not only does this produce a lot of irrelevant clutter in the log, it is factually incorrect, as misaligned runtime regions are actually allowed by the EFI spec as long as they don't require conflicting memory types within the same 64k page. So let's drop the warning, and tweak the code so that we - take both the start and end of the region into account when checking for misalignment - only revert to RWX mappings for non-code regions if misaligned code regions are also known to exist. Cc: Acked-by: Linus Torvalds Signed-off-by: Ard Biesheuvel --- arch/arm64/kernel/efi.c | 52 +++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 8d36e66a6e64c..ee53f2a0aa03f 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -13,6 +13,14 @@ #include +static bool region_is_misaligned(const efi_memory_desc_t *md) +{ + if (PAGE_SIZE == EFI_PAGE_SIZE) + return false; + return !PAGE_ALIGNED(md->phys_addr) || + !PAGE_ALIGNED(md->num_pages << EFI_PAGE_SHIFT); +} + /* * Only regions of type EFI_RUNTIME_SERVICES_CODE need to be * executable, everything else can be mapped with the XN bits @@ -26,14 +34,22 @@ static __init pteval_t create_mapping_protection(efi_memory_desc_t *md) if (type == EFI_MEMORY_MAPPED_IO) return PROT_DEVICE_nGnRE; - if (WARN_ONCE(!PAGE_ALIGNED(md->phys_addr), - "UEFI Runtime regions are not aligned to 64 KB -- buggy firmware?")) + if (region_is_misaligned(md)) { + static bool __initdata code_is_misaligned; + /* - * If the region is not aligned to the page size of the OS, we - * can not use strict permissions, since that would also affect - * the mapping attributes of the adjacent regions. + * Regions that are not aligned to the OS page size cannot be + * mapped with strict permissions, as those might interfere + * with the permissions that are needed by the adjacent + * region's mapping. However, if we haven't encountered any + * misaligned runtime code regions so far, we can safely use + * non-executable permissions for non-code regions. */ - return pgprot_val(PAGE_KERNEL_EXEC); + code_is_misaligned |= (type == EFI_RUNTIME_SERVICES_CODE); + + return code_is_misaligned ? pgprot_val(PAGE_KERNEL_EXEC) + : pgprot_val(PAGE_KERNEL); + } /* R-- */ if ((attr & (EFI_MEMORY_XP | EFI_MEMORY_RO)) == @@ -64,19 +80,16 @@ int __init efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md) bool page_mappings_only = (md->type == EFI_RUNTIME_SERVICES_CODE || md->type == EFI_RUNTIME_SERVICES_DATA); - if (!PAGE_ALIGNED(md->phys_addr) || - !PAGE_ALIGNED(md->num_pages << EFI_PAGE_SHIFT)) { - /* - * If the end address of this region is not aligned to page - * size, the mapping is rounded up, and may end up sharing a - * page frame with the next UEFI memory region. If we create - * a block entry now, we may need to split it again when mapping - * the next region, and support for that is going to be removed - * from the MMU routines. So avoid block mappings altogether in - * that case. - */ + /* + * If this region is not aligned to the page size used by the OS, the + * mapping will be rounded outwards, and may end up sharing a page + * frame with an adjacent runtime memory region. Given that the page + * table descriptor covering the shared page will be rewritten when the + * adjacent region gets mapped, we must avoid block mappings here so we + * don't have to worry about splitting them when that happens. + */ + if (region_is_misaligned(md)) page_mappings_only = true; - } create_pgd_mapping(mm, md->phys_addr, md->virt_addr, md->num_pages << EFI_PAGE_SHIFT, @@ -103,6 +116,9 @@ int __init efi_set_mapping_permissions(struct mm_struct *mm, BUG_ON(md->type != EFI_RUNTIME_SERVICES_CODE && md->type != EFI_RUNTIME_SERVICES_DATA); + if (region_is_misaligned(md)) + return 0; + /* * Calling apply_to_page_range() is only safe on regions that are * guaranteed to be mapped down to pages. Since we are only called -- GitLab From 50e63dd8ed92045eb70a72d7ec725488320fb68b Mon Sep 17 00:00:00 2001 From: Conor Dooley Date: Mon, 7 Nov 2022 15:15:25 +0000 Subject: [PATCH 1570/1927] riscv: fix reserved memory setup Currently, RISC-V sets up reserved memory using the "early" copy of the device tree. As a result, when trying to get a reserved memory region using of_reserved_mem_lookup(), the pointer to reserved memory regions is using the early, pre-virtual-memory address which causes a kernel panic when trying to use the buffer's name: Unable to handle kernel paging request at virtual address 00000000401c31ac Oops [#1] Modules linked in: CPU: 0 PID: 0 Comm: swapper Not tainted 6.0.0-rc1-00001-g0d9d6953d834 #1 Hardware name: Microchip PolarFire-SoC Icicle Kit (DT) epc : string+0x4a/0xea ra : vsnprintf+0x1e4/0x336 epc : ffffffff80335ea0 ra : ffffffff80338936 sp : ffffffff81203be0 gp : ffffffff812e0a98 tp : ffffffff8120de40 t0 : 0000000000000000 t1 : ffffffff81203e28 t2 : 7265736572203a46 s0 : ffffffff81203c20 s1 : ffffffff81203e28 a0 : ffffffff81203d22 a1 : 0000000000000000 a2 : ffffffff81203d08 a3 : 0000000081203d21 a4 : ffffffffffffffff a5 : 00000000401c31ac a6 : ffff0a00ffffff04 a7 : ffffffffffffffff s2 : ffffffff81203d08 s3 : ffffffff81203d00 s4 : 0000000000000008 s5 : ffffffff000000ff s6 : 0000000000ffffff s7 : 00000000ffffff00 s8 : ffffffff80d9821a s9 : ffffffff81203d22 s10: 0000000000000002 s11: ffffffff80d9821c t3 : ffffffff812f3617 t4 : ffffffff812f3617 t5 : ffffffff812f3618 t6 : ffffffff81203d08 status: 0000000200000100 badaddr: 00000000401c31ac cause: 000000000000000d [] vsnprintf+0x1e4/0x336 [] vprintk_store+0xf6/0x344 [] vprintk_emit+0x56/0x192 [] vprintk_default+0x16/0x1e [] vprintk+0x72/0x80 [] _printk+0x36/0x50 [] print_reserved_mem+0x1c/0x24 [] paging_init+0x528/0x5bc [] setup_arch+0xd0/0x592 [] start_kernel+0x82/0x73c early_init_fdt_scan_reserved_mem() takes no arguments as it operates on initial_boot_params, which is populated by early_init_dt_verify(). On RISC-V, early_init_dt_verify() is called twice. Once, directly, in setup_arch() if CONFIG_BUILTIN_DTB is not enabled and once indirectly, very early in the boot process, by parse_dtb() when it calls early_init_dt_scan_nodes(). This first call uses dtb_early_va to set initial_boot_params, which is not usable later in the boot process when early_init_fdt_scan_reserved_mem() is called. On arm64 for example, the corresponding call to early_init_dt_scan_nodes() uses fixmap addresses and doesn't suffer the same fate. Move early_init_fdt_scan_reserved_mem() further along the boot sequence, after the direct call to early_init_dt_verify() in setup_arch() so that the names use the correct virtual memory addresses. The above supposed that CONFIG_BUILTIN_DTB was not set, but should work equally in the case where it is - unflatted_and_copy_device_tree() also updates initial_boot_params. Reported-by: Valentina Fernandez Reported-by: Evgenii Shatokhin Link: https://lore.kernel.org/linux-riscv/f8e67f82-103d-156c-deb0-d6d6e2756f5e@microchip.com/ Fixes: 922b0375fc93 ("riscv: Fix memblock reservation for device tree blob") Signed-off-by: Conor Dooley Tested-by: Evgenii Shatokhin Link: https://lore.kernel.org/r/20221107151524.3941467-1-conor.dooley@microchip.com Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/setup.c | 1 + arch/riscv/mm/init.c | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index ad76bb59b0590..67ec1fadcfe24 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -283,6 +283,7 @@ void __init setup_arch(char **cmdline_p) else pr_err("No DTB found in kernel mappings\n"); #endif + early_init_fdt_scan_reserved_mem(); misc_mem_init(); init_resources(); diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index b56a0a75533fe..50a1b6edd4918 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -262,7 +262,6 @@ static void __init setup_bootmem(void) memblock_reserve(dtb_early_pa, fdt_totalsize(dtb_early_va)); } - early_init_fdt_scan_reserved_mem(); dma_contiguous_reserve(dma32_phys_limit); if (IS_ENABLED(CONFIG_64BIT)) hugetlb_cma_reserve(PUD_SHIFT - PAGE_SHIFT); -- GitLab From 9bae30de136a645e29f632313c0ebb02fc00641b Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Mon, 7 Nov 2022 12:24:10 -0800 Subject: [PATCH 1571/1927] drm/i915/perf: Fix kernel-doc warning Fix kernel-doc issue from a previous commit. Signed-off-by: Umesh Nerlige Ramappa Fixes: 2db609c01495 ("drm/i915/perf: Replace gt->perf.lock with stream->lock for file ops") Reviewed-by: Ashutosh Dixit Signed-off-by: Daniele Ceraolo Spurio Link: https://patchwork.freedesktop.org/patch/msgid/20221107202410.1976895-1-umesh.nerlige.ramappa@intel.com --- drivers/gpu/drm/i915/i915_perf_types.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_perf_types.h b/drivers/gpu/drm/i915/i915_perf_types.h index e0c96b44eda8e..ca150b7af3f29 100644 --- a/drivers/gpu/drm/i915/i915_perf_types.h +++ b/drivers/gpu/drm/i915/i915_perf_types.h @@ -146,8 +146,8 @@ struct i915_perf_stream { */ struct intel_engine_cs *engine; - /* - * Lock associated with operations on stream + /** + * @lock: Lock associated with operations on stream */ struct mutex lock; -- GitLab From fcae44fd36d052e956e69a64642fc03820968d78 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 8 Nov 2022 10:13:23 -0700 Subject: [PATCH 1572/1927] RISC-V: vdso: Do not add missing symbols to version section in linker script Recently, ld.lld moved from '--undefined-version' to '--no-undefined-version' as the default, which breaks the compat vDSO build: ld.lld: error: version script assignment of 'LINUX_4.15' to symbol '__vdso_gettimeofday' failed: symbol not defined ld.lld: error: version script assignment of 'LINUX_4.15' to symbol '__vdso_clock_gettime' failed: symbol not defined ld.lld: error: version script assignment of 'LINUX_4.15' to symbol '__vdso_clock_getres' failed: symbol not defined These symbols are not present in the compat vDSO or the regular vDSO for 32-bit but they are unconditionally included in the version section of the linker script, which is prohibited with '--no-undefined-version'. Fix this issue by only including the symbols that are actually exported in the version section of the linker script. Link: https://github.com/ClangBuiltLinux/linux/issues/1756 Signed-off-by: Nathan Chancellor Tested-by: Conor Dooley Link: https://lore.kernel.org/r/20221108171324.3377226-1-nathan@kernel.org/ Signed-off-by: Palmer Dabbelt --- arch/riscv/kernel/vdso/Makefile | 3 +++ arch/riscv/kernel/vdso/vdso.lds.S | 2 ++ 2 files changed, 5 insertions(+) diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 84ac0fe612e79..db6548509bb3e 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -28,6 +28,9 @@ obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) obj-y += vdso.o CPPFLAGS_vdso.lds += -P -C -U$(ARCH) +ifneq ($(filter vgettimeofday, $(vdso-syms)),) +CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY +endif # Disable -pg to prevent insert call site CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index 01d94aae5bf51..150b1a572e619 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -68,9 +68,11 @@ VERSION LINUX_4.15 { global: __vdso_rt_sigreturn; +#ifdef HAS_VGETTIMEOFDAY __vdso_gettimeofday; __vdso_clock_gettime; __vdso_clock_getres; +#endif __vdso_getcpu; __vdso_flush_icache; local: *; -- GitLab From ff963634f7b2e0dc011349abb3fb81a0d074f443 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Tue, 1 Nov 2022 15:07:15 +0800 Subject: [PATCH 1573/1927] drm/drv: Fix potential memory leak in drm_dev_init() drm_dev_init() will add drm_dev_init_release() as a callback. When drmm_add_action() failed, the release function won't be added. As the result, the ref cnt added by device_get() in drm_dev_init() won't be put by drm_dev_init_release(), which leads to the memleak. Use drmm_add_action_or_reset() instead of drmm_add_action() to prevent memleak. unreferenced object 0xffff88810bc0c800 (size 2048): comm "modprobe", pid 8322, jiffies 4305809845 (age 15.292s) hex dump (first 32 bytes): e8 cc c0 0b 81 88 ff ff ff ff ff ff 00 00 00 00 ................ 20 24 3c 0c 81 88 ff ff 18 c8 c0 0b 81 88 ff ff $<............. backtrace: [<000000007251f72d>] __kmalloc+0x4b/0x1c0 [<0000000045f21f26>] platform_device_alloc+0x2d/0xe0 [<000000004452a479>] platform_device_register_full+0x24/0x1c0 [<0000000089f4ea61>] 0xffffffffa0736051 [<00000000235b2441>] do_one_initcall+0x7a/0x380 [<0000000001a4a177>] do_init_module+0x5c/0x230 [<000000002bf8a8e2>] load_module+0x227d/0x2420 [<00000000637d6d0a>] __do_sys_finit_module+0xd5/0x140 [<00000000c99fc324>] do_syscall_64+0x3f/0x90 [<000000004d85aa77>] entry_SYSCALL_64_after_hwframe+0x63/0xcd Fixes: 2cbf7fc6718b ("drm: Use drmm_ for drm_dev_init cleanup") Signed-off-by: Shang XiaoJing Reviewed-by: Lyude Paul Signed-off-by: Lyude Paul Link: https://patchwork.freedesktop.org/patch/msgid/20221101070716.9189-2-shangxiaojing@huawei.com --- drivers/gpu/drm/drm_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 8214a0b1ab7f1..203bf8d6c34c4 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -615,7 +615,7 @@ static int drm_dev_init(struct drm_device *dev, mutex_init(&dev->clientlist_mutex); mutex_init(&dev->master_mutex); - ret = drmm_add_action(dev, drm_dev_init_release, NULL); + ret = drmm_add_action_or_reset(dev, drm_dev_init_release, NULL); if (ret) return ret; -- GitLab From 4979524f5a2a8210e87fde2f642b0dc060860821 Mon Sep 17 00:00:00 2001 From: Shang XiaoJing Date: Tue, 1 Nov 2022 15:07:16 +0800 Subject: [PATCH 1574/1927] drm: Fix potential null-ptr-deref in drm_vblank_destroy_worker() drm_vblank_init() call drmm_add_action_or_reset() with drm_vblank_init_release() as action. If __drmm_add_action() failed, will directly call drm_vblank_init_release() with the vblank whose worker is NULL. As the resule, a null-ptr-deref will happen in kthread_destroy_worker(). Add the NULL check before calling drm_vblank_destroy_worker(). BUG: null-ptr-deref KASAN: null-ptr-deref in range [0x0000000000000068-0x000000000000006f] CPU: 5 PID: 961 Comm: modprobe Not tainted 6.0.0-11331-gd465bff130bf-dirty RIP: 0010:kthread_destroy_worker+0x25/0xb0 Call Trace: drm_vblank_init_release+0x124/0x220 [drm] ? drm_crtc_vblank_restore+0x8b0/0x8b0 [drm] __drmm_add_action_or_reset+0x41/0x50 [drm] drm_vblank_init+0x282/0x310 [drm] vkms_init+0x35f/0x1000 [vkms] ? 0xffffffffc4508000 ? lock_is_held_type+0xd7/0x130 ? __kmem_cache_alloc_node+0x1c2/0x2b0 ? lock_is_held_type+0xd7/0x130 ? 0xffffffffc4508000 do_one_initcall+0xd0/0x4f0 ... do_syscall_64+0x35/0x80 entry_SYSCALL_64_after_hwframe+0x46/0xb0 Fixes: 5e6c2b4f9161 ("drm/vblank: Add vblank works") Signed-off-by: Shang XiaoJing Reviewed-by: Lyude Paul Signed-off-by: Lyude Paul Link: https://patchwork.freedesktop.org/patch/msgid/20221101070716.9189-3-shangxiaojing@huawei.com --- drivers/gpu/drm/drm_internal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h index 7bb98e6a446d0..5ea5e260118c2 100644 --- a/drivers/gpu/drm/drm_internal.h +++ b/drivers/gpu/drm/drm_internal.h @@ -104,7 +104,8 @@ static inline void drm_vblank_flush_worker(struct drm_vblank_crtc *vblank) static inline void drm_vblank_destroy_worker(struct drm_vblank_crtc *vblank) { - kthread_destroy_worker(vblank->worker); + if (vblank->worker) + kthread_destroy_worker(vblank->worker); } int drm_vblank_worker_init(struct drm_vblank_crtc *vblank); -- GitLab From de91b3197d15172407608b2c357aab7ac1451e2b Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Wed, 9 Nov 2022 15:01:16 +0000 Subject: [PATCH 1575/1927] eth: sp7021: drop free_netdev() from spl2sw_init_netdev() It's not necessary to free netdev allocated with devm_alloc_etherdev() and using free_netdev() leads to double free. Fixes: fd3040b9394c ("net: ethernet: Add driver for Sunplus SP7021") Signed-off-by: Wei Yongjun Link: https://lore.kernel.org/r/20221109150116.2988194-1-weiyongjun@huaweicloud.com Signed-off-by: Jakub Kicinski --- drivers/net/ethernet/sunplus/spl2sw_driver.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/net/ethernet/sunplus/spl2sw_driver.c b/drivers/net/ethernet/sunplus/spl2sw_driver.c index 9be5852372773..c499a14314f13 100644 --- a/drivers/net/ethernet/sunplus/spl2sw_driver.c +++ b/drivers/net/ethernet/sunplus/spl2sw_driver.c @@ -287,7 +287,6 @@ static u32 spl2sw_init_netdev(struct platform_device *pdev, u8 *mac_addr, if (ret) { dev_err(&pdev->dev, "Failed to register net device \"%s\"!\n", ndev->name); - free_netdev(ndev); *r_ndev = NULL; return ret; } -- GitLab From 63fd9437ec81899fc36bb642d558378bc89aa4f9 Mon Sep 17 00:00:00 2001 From: Alexander Stein Date: Tue, 1 Nov 2022 10:30:31 +0100 Subject: [PATCH 1576/1927] arm64: dts: imx8mm-tqma8mqml-mba8mx: Fix USB DR Using extcon USB host mode works properly on DR interface, e.g. enabling/disabling VBUS. But USB device mode is not working. Fix this by switching to usb-role-switch instead. Fixes: dfcd1b6f7620 ("arm64: dts: freescale: add initial device tree for TQMa8MQML with i.MX8MM") Signed-off-by: Alexander Stein Signed-off-by: Shawn Guo --- .../dts/freescale/imx8mm-tqma8mqml-mba8mx.dts | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts index 7e0aeb2db3054..a0aeac6199299 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts +++ b/arch/arm64/boot/dts/freescale/imx8mm-tqma8mqml-mba8mx.dts @@ -34,11 +34,25 @@ reg_usdhc2_vmmc: regulator-vmmc { off-on-delay-us = <12000>; }; - extcon_usbotg1: extcon-usbotg1 { - compatible = "linux,extcon-usb-gpio"; + connector { + compatible = "gpio-usb-b-connector", "usb-b-connector"; + type = "micro"; + label = "X19"; pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_usb1_extcon>; - id-gpio = <&gpio1 10 GPIO_ACTIVE_HIGH>; + pinctrl-0 = <&pinctrl_usb1_connector>; + id-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + usb_dr_connector: endpoint { + remote-endpoint = <&usb1_drd_sw>; + }; + }; + }; }; }; @@ -105,13 +119,19 @@ &usbotg1 { pinctrl-names = "default"; pinctrl-0 = <&pinctrl_usbotg1>; dr_mode = "otg"; - extcon = <&extcon_usbotg1>; srp-disable; hnp-disable; adp-disable; power-active-high; over-current-active-low; + usb-role-switch; status = "okay"; + + port { + usb1_drd_sw: endpoint { + remote-endpoint = <&usb_dr_connector>; + }; + }; }; &usbotg2 { @@ -231,7 +251,7 @@ pinctrl_usbotg1: usbotg1grp { ; }; - pinctrl_usb1_extcon: usb1-extcongrp { + pinctrl_usb1_connector: usb1-connectorgrp { fsl,pins = ; }; -- GitLab From 753395ea1e45c724150070b5785900b6a44bd5fb Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 2 Nov 2022 20:19:45 +0100 Subject: [PATCH 1577/1927] ARM: dts: imx7: Fix NAND controller size-cells The NAND controller size-cells should be 0 per DT bindings. Fix the following warning produces by DT bindings check: " nand-controller@33002000: #size-cells:0:0: 0 was expected nand-controller@33002000: Unevaluated properties are not allowed ('#address-cells', '#size-cells' were unexpected) " Fix the missing space in node name too. Fixes: e7495a45a76de ("ARM: dts: imx7: add GPMI NAND and APBH DMA") Signed-off-by: Marek Vasut Signed-off-by: Shawn Guo --- arch/arm/boot/dts/imx7s.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi index 0fc9e6b8b05dc..03d2e8544a4eb 100644 --- a/arch/arm/boot/dts/imx7s.dtsi +++ b/arch/arm/boot/dts/imx7s.dtsi @@ -1270,10 +1270,10 @@ dma_apbh: dma-apbh@33000000 { clocks = <&clks IMX7D_NAND_USDHC_BUS_RAWNAND_CLK>; }; - gpmi: nand-controller@33002000{ + gpmi: nand-controller@33002000 { compatible = "fsl,imx7d-gpmi-nand"; #address-cells = <1>; - #size-cells = <1>; + #size-cells = <0>; reg = <0x33002000 0x2000>, <0x33004000 0x4000>; reg-names = "gpmi-nand", "bch"; interrupts = ; -- GitLab From 1610233bc2c2cae2dff9e101e6ea5ef69cceb0e9 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 2 Nov 2022 20:19:46 +0100 Subject: [PATCH 1578/1927] arm64: dts: imx8mm: Fix NAND controller size-cells The NAND controller size-cells should be 0 per DT bindings. Fix the following warning produces by DT bindings check: " nand-controller@33002000: #size-cells:0:0: 0 was expected nand-controller@33002000: Unevaluated properties are not allowed ('#address-cells', '#size-cells' were unexpected) " Fix the missing space in node name too. Fixes: a05ea40eb384e ("arm64: dts: imx: Add i.mx8mm dtsi support") Signed-off-by: Marek Vasut Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx8mm.dtsi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi index dabd94dc30c4b..50ef92915c671 100644 --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi @@ -1244,10 +1244,10 @@ dma_apbh: dma-controller@33000000 { clocks = <&clk IMX8MM_CLK_NAND_USDHC_BUS_RAWNAND_CLK>; }; - gpmi: nand-controller@33002000{ + gpmi: nand-controller@33002000 { compatible = "fsl,imx8mm-gpmi-nand", "fsl,imx7d-gpmi-nand"; #address-cells = <1>; - #size-cells = <1>; + #size-cells = <0>; reg = <0x33002000 0x2000>, <0x33004000 0x4000>; reg-names = "gpmi-nand", "bch"; interrupts = ; -- GitLab From 5468e93b5b1083eaa729f98e59da18c85d9c4126 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Wed, 2 Nov 2022 20:19:47 +0100 Subject: [PATCH 1579/1927] arm64: dts: imx8mn: Fix NAND controller size-cells The NAND controller size-cells should be 0 per DT bindings. Fix the following warning produces by DT bindings check: " nand-controller@33002000: #size-cells:0:0: 0 was expected nand-controller@33002000: Unevaluated properties are not allowed ('#address-cells', '#size-cells' were unexpected) " Fixes: 6c3debcbae47a ("arm64: dts: freescale: Add i.MX8MN dtsi support") Signed-off-by: Marek Vasut Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx8mn.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/freescale/imx8mn.dtsi b/arch/arm64/boot/dts/freescale/imx8mn.dtsi index ad0b99adf6911..67b554ba690ca 100644 --- a/arch/arm64/boot/dts/freescale/imx8mn.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8mn.dtsi @@ -1102,7 +1102,7 @@ dma_apbh: dma-controller@33000000 { gpmi: nand-controller@33002000 { compatible = "fsl,imx8mn-gpmi-nand", "fsl,imx7d-gpmi-nand"; #address-cells = <1>; - #size-cells = <1>; + #size-cells = <0>; reg = <0x33002000 0x2000>, <0x33004000 0x4000>; reg-names = "gpmi-nand", "bch"; interrupts = ; -- GitLab From 2db1fdb25d209a88112fd82eb493976d66057d10 Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Fri, 4 Nov 2022 13:49:42 +0800 Subject: [PATCH 1580/1927] arm64: dts: imx93-pinfunc: drop execution permission Drop the header file execution permission Signed-off-by: Peng Fan Fixes: ec8b5b5058ea ("arm64: dts: freescale: Add i.MX93 dtsi support") Signed-off-by: Shawn Guo --- arch/arm64/boot/dts/freescale/imx93-pinfunc.h | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 arch/arm64/boot/dts/freescale/imx93-pinfunc.h diff --git a/arch/arm64/boot/dts/freescale/imx93-pinfunc.h b/arch/arm64/boot/dts/freescale/imx93-pinfunc.h old mode 100755 new mode 100644 -- GitLab From 8c76310740807ade5ecdab5888f70ecb6d35732e Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 21:40:01 +0800 Subject: [PATCH 1581/1927] ata: libata-transport: fix double ata_host_put() in ata_tport_add() In the error path in ata_tport_add(), when calling put_device(), ata_tport_release() is called, it will put the refcount of 'ap->host'. And then ata_host_put() is called again, the refcount is decreased to 0, ata_host_release() is called, all ports are freed and set to null. When unbinding the device after failure, ata_host_stop() is called to release the resources, it leads a null-ptr-deref(), because all the ports all freed and null. Unable to handle kernel NULL pointer dereference at virtual address 0000000000000008 CPU: 7 PID: 18671 Comm: modprobe Kdump: loaded Tainted: G E 6.1.0-rc3+ #8 pstate: 80400009 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : ata_host_stop+0x3c/0x84 [libata] lr : release_nodes+0x64/0xd0 Call trace: ata_host_stop+0x3c/0x84 [libata] release_nodes+0x64/0xd0 devres_release_all+0xbc/0x1b0 device_unbind_cleanup+0x20/0x70 really_probe+0x158/0x320 __driver_probe_device+0x84/0x120 driver_probe_device+0x44/0x120 __driver_attach+0xb4/0x220 bus_for_each_dev+0x78/0xdc driver_attach+0x2c/0x40 bus_add_driver+0x184/0x240 driver_register+0x80/0x13c __pci_register_driver+0x4c/0x60 ahci_pci_driver_init+0x30/0x1000 [ahci] Fix this by removing redundant ata_host_put() in the error path. Fixes: 2623c7a5f279 ("libata: add refcounting to ata_host") Signed-off-by: Yang Yingliang Signed-off-by: Damien Le Moal --- drivers/ata/libata-transport.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index a7e9a75410a35..105da3ec5eaa9 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -317,7 +317,6 @@ int ata_tport_add(struct device *parent, tport_err: transport_destroy_device(dev); put_device(dev); - ata_host_put(ap->host); return error; } -- GitLab From 3613dbe3909dcc637fe6be00e4dc43b4aa0470ee Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 21:40:02 +0800 Subject: [PATCH 1582/1927] ata: libata-transport: fix error handling in ata_tport_add() In ata_tport_add(), the return value of transport_add_device() is not checked. As a result, it causes null-ptr-deref while removing the module, because transport_remove_device() is called to remove the device that was not added. Unable to handle kernel NULL pointer dereference at virtual address 00000000000000d0 CPU: 12 PID: 13605 Comm: rmmod Kdump: loaded Tainted: G W 6.1.0-rc3+ #8 pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : device_del+0x48/0x39c lr : device_del+0x44/0x39c Call trace: device_del+0x48/0x39c attribute_container_class_device_del+0x28/0x40 transport_remove_classdev+0x60/0x7c attribute_container_device_trigger+0x118/0x120 transport_remove_device+0x20/0x30 ata_tport_delete+0x34/0x60 [libata] ata_port_detach+0x148/0x1b0 [libata] ata_pci_remove_one+0x50/0x80 [libata] ahci_remove_one+0x4c/0x8c [ahci] Fix this by checking and handling return value of transport_add_device() in ata_tport_add(). Fixes: d9027470b886 ("[libata] Add ATA transport class") Signed-off-by: Yang Yingliang Signed-off-by: Damien Le Moal --- drivers/ata/libata-transport.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index 105da3ec5eaa9..ef53bdfbcbb24 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -301,7 +301,9 @@ int ata_tport_add(struct device *parent, pm_runtime_enable(dev); pm_runtime_forbid(dev); - transport_add_device(dev); + error = transport_add_device(dev); + if (error) + goto tport_transport_add_err; transport_configure_device(dev); error = ata_tlink_add(&ap->link); @@ -312,6 +314,7 @@ int ata_tport_add(struct device *parent, tport_link_err: transport_remove_device(dev); + tport_transport_add_err: device_del(dev); tport_err: -- GitLab From cf0816f6322c5c37ee52655f928e91ecf32da103 Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 21:40:03 +0800 Subject: [PATCH 1583/1927] ata: libata-transport: fix error handling in ata_tlink_add() In ata_tlink_add(), the return value of transport_add_device() is not checked. As a result, it causes null-ptr-deref while removing the module, because transport_remove_device() is called to remove the device that was not added. Unable to handle kernel NULL pointer dereference at virtual address 00000000000000d0 CPU: 33 PID: 13850 Comm: rmmod Kdump: loaded Tainted: G W 6.1.0-rc3+ #12 pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : device_del+0x48/0x39c lr : device_del+0x44/0x39c Call trace: device_del+0x48/0x39c attribute_container_class_device_del+0x28/0x40 transport_remove_classdev+0x60/0x7c attribute_container_device_trigger+0x118/0x120 transport_remove_device+0x20/0x30 ata_tlink_delete+0x88/0xb0 [libata] ata_tport_delete+0x2c/0x60 [libata] ata_port_detach+0x148/0x1b0 [libata] ata_pci_remove_one+0x50/0x80 [libata] ahci_remove_one+0x4c/0x8c [ahci] Fix this by checking and handling return value of transport_add_device() in ata_tlink_add(). Fixes: d9027470b886 ("[libata] Add ATA transport class") Signed-off-by: Yang Yingliang Signed-off-by: Damien Le Moal --- drivers/ata/libata-transport.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index ef53bdfbcbb24..aac9336e8153b 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -458,7 +458,9 @@ int ata_tlink_add(struct ata_link *link) goto tlink_err; } - transport_add_device(dev); + error = transport_add_device(dev); + if (error) + goto tlink_transport_err; transport_configure_device(dev); ata_for_each_dev(ata_dev, link, ALL) { @@ -473,6 +475,7 @@ int ata_tlink_add(struct ata_link *link) ata_tdev_delete(ata_dev); } transport_remove_device(dev); + tlink_transport_err: device_del(dev); tlink_err: transport_destroy_device(dev); -- GitLab From 1ff36351309e3eadcff297480baf4785e726de9b Mon Sep 17 00:00:00 2001 From: Yang Yingliang Date: Tue, 8 Nov 2022 21:40:04 +0800 Subject: [PATCH 1584/1927] ata: libata-transport: fix error handling in ata_tdev_add() In ata_tdev_add(), the return value of transport_add_device() is not checked. As a result, it causes null-ptr-deref while removing the module, because transport_remove_device() is called to remove the device that was not added. Unable to handle kernel NULL pointer dereference at virtual address 00000000000000d0 CPU: 13 PID: 13603 Comm: rmmod Kdump: loaded Tainted: G W 6.1.0-rc3+ #36 pstate: 60400009 (nZCv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : device_del+0x48/0x3a0 lr : device_del+0x44/0x3a0 Call trace: device_del+0x48/0x3a0 attribute_container_class_device_del+0x28/0x40 transport_remove_classdev+0x60/0x7c attribute_container_device_trigger+0x118/0x120 transport_remove_device+0x20/0x30 ata_tdev_delete+0x24/0x50 [libata] ata_tlink_delete+0x40/0xa0 [libata] ata_tport_delete+0x2c/0x60 [libata] ata_port_detach+0x148/0x1b0 [libata] ata_pci_remove_one+0x50/0x80 [libata] ahci_remove_one+0x4c/0x8c [ahci] Fix this by checking and handling return value of transport_add_device() in ata_tdev_add(). In the error path, device_del() is called to delete the device which was added earlier in this function, and ata_tdev_free() is called to free ata_dev. Fixes: d9027470b886 ("[libata] Add ATA transport class") Signed-off-by: Yang Yingliang Signed-off-by: Damien Le Moal --- drivers/ata/libata-transport.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/ata/libata-transport.c b/drivers/ata/libata-transport.c index aac9336e8153b..e4fb9d1b9b398 100644 --- a/drivers/ata/libata-transport.c +++ b/drivers/ata/libata-transport.c @@ -713,7 +713,13 @@ static int ata_tdev_add(struct ata_device *ata_dev) return error; } - transport_add_device(dev); + error = transport_add_device(dev); + if (error) { + device_del(dev); + ata_tdev_free(ata_dev); + return error; + } + transport_configure_device(dev); return 0; } -- GitLab From da2549576b7be2b465908709dad50810a084cce4 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 9 Nov 2022 17:35:19 +0200 Subject: [PATCH 1585/1927] drm/i915/reg: move masked field helpers to i915_reg_defs.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a more logical place for generic helpers. Cc: Lucas De Marchi Reviewed-by: Ville Syrjälä Signed-off-by: Jani Nikula Link: https://patchwork.freedesktop.org/patch/msgid/5161a0c6d98df206c6c4c1add3fc3f2f408020b1.1668008071.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/i915_reg.h | 13 ------------- drivers/gpu/drm/i915/i915_reg_defs.h | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 5a062ee46ed1d..5955da91bec51 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -171,19 +171,6 @@ INTEL_INFO(dev_priv)->display.cursor_offsets[PIPE_A] + \ DISPLAY_MMIO_BASE(dev_priv) + (reg)) -#define __MASKED_FIELD(mask, value) ((mask) << 16 | (value)) -#define _MASKED_FIELD(mask, value) ({ \ - if (__builtin_constant_p(mask)) \ - BUILD_BUG_ON_MSG(((mask) & 0xffff0000), "Incorrect mask"); \ - if (__builtin_constant_p(value)) \ - BUILD_BUG_ON_MSG((value) & 0xffff0000, "Incorrect value"); \ - if (__builtin_constant_p(mask) && __builtin_constant_p(value)) \ - BUILD_BUG_ON_MSG((value) & ~(mask), \ - "Incorrect value for mask"); \ - __MASKED_FIELD(mask, value); }) -#define _MASKED_BIT_ENABLE(a) ({ typeof(a) _a = (a); _MASKED_FIELD(_a, _a); }) -#define _MASKED_BIT_DISABLE(a) (_MASKED_FIELD((a), 0)) - #define GU_CNTL _MMIO(0x101010) #define LMEM_INIT REG_BIT(7) diff --git a/drivers/gpu/drm/i915/i915_reg_defs.h b/drivers/gpu/drm/i915/i915_reg_defs.h index 8f486f77609f1..7536f1b727781 100644 --- a/drivers/gpu/drm/i915/i915_reg_defs.h +++ b/drivers/gpu/drm/i915/i915_reg_defs.h @@ -98,6 +98,19 @@ */ #define REG_FIELD_GET64(__mask, __val) ((u64)FIELD_GET(__mask, __val)) +#define __MASKED_FIELD(mask, value) ((mask) << 16 | (value)) +#define _MASKED_FIELD(mask, value) ({ \ + if (__builtin_constant_p(mask)) \ + BUILD_BUG_ON_MSG(((mask) & 0xffff0000), "Incorrect mask"); \ + if (__builtin_constant_p(value)) \ + BUILD_BUG_ON_MSG((value) & 0xffff0000, "Incorrect value"); \ + if (__builtin_constant_p(mask) && __builtin_constant_p(value)) \ + BUILD_BUG_ON_MSG((value) & ~(mask), \ + "Incorrect value for mask"); \ + __MASKED_FIELD(mask, value); }) +#define _MASKED_BIT_ENABLE(a) ({ typeof(a) _a = (a); _MASKED_FIELD(_a, _a); }) +#define _MASKED_BIT_DISABLE(a) (_MASKED_FIELD((a), 0)) + typedef struct { u32 reg; } i915_reg_t; -- GitLab From 81e78b13bdf7829bed0ad1a1dfe51293a8f676f3 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 9 Nov 2022 17:35:20 +0200 Subject: [PATCH 1586/1927] drm/i915/reg: move pick even and pick to reg defs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a more logical place for generic helpers. Cc: Lucas De Marchi Reviewed-by: Ville Syrjälä Signed-off-by: Jani Nikula Link: https://patchwork.freedesktop.org/patch/msgid/623327aee73a515300d99c8d65552ca92f3f0721.1668008071.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/i915_reg.h | 15 --------------- drivers/gpu/drm/i915/i915_reg_defs.h | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 5955da91bec51..f0e6da455d5bc 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -117,21 +117,6 @@ #define DISPLAY_MMIO_BASE(dev_priv) (INTEL_INFO(dev_priv)->display.mmio_offset) -/* - * Given the first two numbers __a and __b of arbitrarily many evenly spaced - * numbers, pick the 0-based __index'th value. - * - * Always prefer this over _PICK() if the numbers are evenly spaced. - */ -#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a))) - -/* - * Given the arbitrary numbers in varargs, pick the 0-based __index'th number. - * - * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced. - */ -#define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index]) - /* * Named helper wrappers around _PICK_EVEN() and _PICK(). */ diff --git a/drivers/gpu/drm/i915/i915_reg_defs.h b/drivers/gpu/drm/i915/i915_reg_defs.h index 7536f1b727781..344e5006650e0 100644 --- a/drivers/gpu/drm/i915/i915_reg_defs.h +++ b/drivers/gpu/drm/i915/i915_reg_defs.h @@ -111,6 +111,21 @@ #define _MASKED_BIT_ENABLE(a) ({ typeof(a) _a = (a); _MASKED_FIELD(_a, _a); }) #define _MASKED_BIT_DISABLE(a) (_MASKED_FIELD((a), 0)) +/* + * Given the first two numbers __a and __b of arbitrarily many evenly spaced + * numbers, pick the 0-based __index'th value. + * + * Always prefer this over _PICK() if the numbers are evenly spaced. + */ +#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a))) + +/* + * Given the arbitrary numbers in varargs, pick the 0-based __index'th number. + * + * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced. + */ +#define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index]) + typedef struct { u32 reg; } i915_reg_t; -- GitLab From e563531ace14c6920ff78794c1d9c35a03bc47b1 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 9 Nov 2022 17:35:21 +0200 Subject: [PATCH 1587/1927] drm/i915: split out intel_display_reg_defs.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out the display register helper macros to a separate file. For now, include it from i915_reg.h, but note that there are already files that don't need i915_reg.h, such as intel_audio.c. Cc: Lucas De Marchi Reviewed-by: Ville Syrjälä Signed-off-by: Jani Nikula Link: https://patchwork.freedesktop.org/patch/msgid/3af47193ff5219b6d2cfe353b752ec4bb44de4f1.1668008071.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/icl_dsi_regs.h | 2 +- .../gpu/drm/i915/display/intel_audio_regs.h | 2 +- .../drm/i915/display/intel_backlight_regs.h | 2 +- .../drm/i915/display/intel_display_reg_defs.h | 53 +++++++++++++++++++ .../gpu/drm/i915/display/intel_hdcp_regs.h | 2 +- .../gpu/drm/i915/display/intel_mg_phy_regs.h | 2 +- .../drm/i915/display/intel_snps_phy_regs.h | 2 +- drivers/gpu/drm/i915/display/vlv_dsi_regs.h | 2 +- drivers/gpu/drm/i915/gt/intel_gt_regs.h | 1 + drivers/gpu/drm/i915/i915_reg.h | 42 +-------------- drivers/gpu/drm/i915/i915_reg_defs.h | 2 - 11 files changed, 62 insertions(+), 50 deletions(-) create mode 100644 drivers/gpu/drm/i915/display/intel_display_reg_defs.h diff --git a/drivers/gpu/drm/i915/display/icl_dsi_regs.h b/drivers/gpu/drm/i915/display/icl_dsi_regs.h index f78f28b8dd945..d4845ac65acce 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi_regs.h +++ b/drivers/gpu/drm/i915/display/icl_dsi_regs.h @@ -6,7 +6,7 @@ #ifndef __ICL_DSI_REGS_H__ #define __ICL_DSI_REGS_H__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" /* Gen11 DSI */ #define _MMIO_DSI(tc, dsi0, dsi1) _MMIO_TRANS((tc) - TRANSCODER_DSI_0, \ diff --git a/drivers/gpu/drm/i915/display/intel_audio_regs.h b/drivers/gpu/drm/i915/display/intel_audio_regs.h index 4f432c2eb543d..616e7b1275c4c 100644 --- a/drivers/gpu/drm/i915/display/intel_audio_regs.h +++ b/drivers/gpu/drm/i915/display/intel_audio_regs.h @@ -6,7 +6,7 @@ #ifndef __INTEL_AUDIO_REGS_H__ #define __INTEL_AUDIO_REGS_H__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" #define G4X_AUD_CNTL_ST _MMIO(0x620B4) #define G4X_ELD_VALID REG_BIT(14) diff --git a/drivers/gpu/drm/i915/display/intel_backlight_regs.h b/drivers/gpu/drm/i915/display/intel_backlight_regs.h index 50c1210f6d5de..344eb8096bd2c 100644 --- a/drivers/gpu/drm/i915/display/intel_backlight_regs.h +++ b/drivers/gpu/drm/i915/display/intel_backlight_regs.h @@ -6,7 +6,7 @@ #ifndef __INTEL_BACKLIGHT_REGS_H__ #define __INTEL_BACKLIGHT_REGS_H__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" #define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250) #define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350) diff --git a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h new file mode 100644 index 0000000000000..02605418ff08c --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __INTEL_DISPLAY_REG_DEFS_H__ +#define __INTEL_DISPLAY_REG_DEFS_H__ + +#include "i915_reg_defs.h" + +#define DISPLAY_MMIO_BASE(dev_priv) (INTEL_INFO(dev_priv)->display.mmio_offset) + +#define VLV_DISPLAY_BASE 0x180000 + +/* + * Named helper wrappers around _PICK_EVEN() and _PICK(). + */ +#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b) +#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b) +#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b) +#define _PORT(port, a, b) _PICK_EVEN(port, a, b) +#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b) +#define _PHY(phy, a, b) _PICK_EVEN(phy, a, b) + +#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) +#define _MMIO_PLANE(plane, a, b) _MMIO(_PLANE(plane, a, b)) +#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) +#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) +#define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b)) +#define _MMIO_PHY(phy, a, b) _MMIO(_PHY(phy, a, b)) + +#define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__) + +#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) +#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) +#define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c)) +#define _MMIO_PLL3(pll, ...) _MMIO(_PICK(pll, __VA_ARGS__)) + +/* + * Device info offset array based helpers for groups of registers with unevenly + * spaced base offsets. + */ +#define _MMIO_PIPE2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->display.pipe_offsets[(pipe)] - \ + INTEL_INFO(dev_priv)->display.pipe_offsets[PIPE_A] + \ + DISPLAY_MMIO_BASE(dev_priv) + (reg)) +#define _MMIO_TRANS2(tran, reg) _MMIO(INTEL_INFO(dev_priv)->display.trans_offsets[(tran)] - \ + INTEL_INFO(dev_priv)->display.trans_offsets[TRANSCODER_A] + \ + DISPLAY_MMIO_BASE(dev_priv) + (reg)) +#define _MMIO_CURSOR2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->display.cursor_offsets[(pipe)] - \ + INTEL_INFO(dev_priv)->display.cursor_offsets[PIPE_A] + \ + DISPLAY_MMIO_BASE(dev_priv) + (reg)) + +#endif /* __INTEL_DISPLAY_REG_DEFS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_regs.h b/drivers/gpu/drm/i915/display/intel_hdcp_regs.h index 2a3733e8966c1..8023c85c7fa0e 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp_regs.h +++ b/drivers/gpu/drm/i915/display/intel_hdcp_regs.h @@ -6,7 +6,7 @@ #ifndef __INTEL_HDCP_REGS_H__ #define __INTEL_HDCP_REGS_H__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" /* HDCP Key Registers */ #define HDCP_KEY_CONF _MMIO(0x66c00) diff --git a/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h b/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h index 07978f8d5fb74..0e8248bce52da 100644 --- a/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h +++ b/drivers/gpu/drm/i915/display/intel_mg_phy_regs.h @@ -6,7 +6,7 @@ #ifndef __INTEL_MG_PHY_REGS__ #define __INTEL_MG_PHY_REGS__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" #define MG_PHY_PORT_LN(ln, tc_port, ln0p1, ln0p2, ln1p1) \ _MMIO(_PORT(tc_port, ln0p1, ln0p2) + (ln) * ((ln1p1) - (ln0p1))) diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h b/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h index 0543465aaf140..a04d692169d49 100644 --- a/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h +++ b/drivers/gpu/drm/i915/display/intel_snps_phy_regs.h @@ -6,7 +6,7 @@ #ifndef __INTEL_SNPS_PHY_REGS__ #define __INTEL_SNPS_PHY_REGS__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" #define _SNPS_PHY_A_BASE 0x168000 #define _SNPS_PHY_B_BASE 0x169000 diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h index e065b8f2ee084..abbe427e462e8 100644 --- a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h +++ b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h @@ -6,7 +6,7 @@ #ifndef __VLV_DSI_REGS_H__ #define __VLV_DSI_REGS_H__ -#include "i915_reg_defs.h" +#include "intel_display_reg_defs.h" #define VLV_MIPI_BASE VLV_DISPLAY_BASE #define BXT_MIPI_BASE 0x60000 diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h index 94a0606859227..fb0c61be350de 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h +++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h @@ -7,6 +7,7 @@ #define __INTEL_GT_REGS__ #include "i915_reg_defs.h" +#include "display/intel_display_reg_defs.h" /* VLV_DISPLAY_BASE */ /* RPM unit config (Gen8+) */ #define RPM_CONFIG0 _MMIO(0xd00) diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index f0e6da455d5bc..93f3b7e702abb 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -26,6 +26,7 @@ #define _I915_REG_H_ #include "i915_reg_defs.h" +#include "display/intel_display_reg_defs.h" /** * DOC: The i915 register macro definition style guide @@ -115,47 +116,6 @@ * #define GEN8_BAR _MMIO(0xb888) */ -#define DISPLAY_MMIO_BASE(dev_priv) (INTEL_INFO(dev_priv)->display.mmio_offset) - -/* - * Named helper wrappers around _PICK_EVEN() and _PICK(). - */ -#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b) -#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b) -#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b) -#define _PORT(port, a, b) _PICK_EVEN(port, a, b) -#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b) -#define _PHY(phy, a, b) _PICK_EVEN(phy, a, b) - -#define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b)) -#define _MMIO_PLANE(plane, a, b) _MMIO(_PLANE(plane, a, b)) -#define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b)) -#define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b)) -#define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b)) -#define _MMIO_PHY(phy, a, b) _MMIO(_PHY(phy, a, b)) - -#define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__) - -#define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) -#define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c)) -#define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c)) -#define _MMIO_PLL3(pll, ...) _MMIO(_PICK(pll, __VA_ARGS__)) - - -/* - * Device info offset array based helpers for groups of registers with unevenly - * spaced base offsets. - */ -#define _MMIO_PIPE2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->display.pipe_offsets[(pipe)] - \ - INTEL_INFO(dev_priv)->display.pipe_offsets[PIPE_A] + \ - DISPLAY_MMIO_BASE(dev_priv) + (reg)) -#define _MMIO_TRANS2(tran, reg) _MMIO(INTEL_INFO(dev_priv)->display.trans_offsets[(tran)] - \ - INTEL_INFO(dev_priv)->display.trans_offsets[TRANSCODER_A] + \ - DISPLAY_MMIO_BASE(dev_priv) + (reg)) -#define _MMIO_CURSOR2(pipe, reg) _MMIO(INTEL_INFO(dev_priv)->display.cursor_offsets[(pipe)] - \ - INTEL_INFO(dev_priv)->display.cursor_offsets[PIPE_A] + \ - DISPLAY_MMIO_BASE(dev_priv) + (reg)) - #define GU_CNTL _MMIO(0x101010) #define LMEM_INIT REG_BIT(7) diff --git a/drivers/gpu/drm/i915/i915_reg_defs.h b/drivers/gpu/drm/i915/i915_reg_defs.h index 344e5006650e0..1564f2c72c2a2 100644 --- a/drivers/gpu/drm/i915/i915_reg_defs.h +++ b/drivers/gpu/drm/i915/i915_reg_defs.h @@ -149,6 +149,4 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg) return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG); } -#define VLV_DISPLAY_BASE 0x180000 - #endif /* __I915_REG_DEFS__ */ -- GitLab From f3a72878a3de720661b7ed0d6b7f7c506ddb8a52 Mon Sep 17 00:00:00 2001 From: Jaco Coetzee Date: Wed, 9 Nov 2022 15:27:57 -0500 Subject: [PATCH 1588/1927] nfp: change eeprom length to max length enumerators Extend the size of QSFP EEPROM for types SSF8436 and SFF8636 from 256 to 640 bytes in order to expose all the EEPROM pages by ethtool. For SFF-8636 and SFF-8436 specifications, the driver exposes 256 bytes of EEPROM data for ethtool's get_module_eeprom() callback, resulting in "netlink error: Invalid argument" when an EEPROM read with an offset larger than 256 bytes is attempted. Changing the length enumerators to the _MAX_LEN variants exposes all 640 bytes of the EEPROM allowing upper pages 1, 2 and 3 to be read. Fixes: 96d971e307cc ("ethtool: Add fallback to get_module_eeprom from netlink command") Signed-off-by: Jaco Coetzee Reviewed-by: Louis Peens Signed-off-by: Simon Horman Signed-off-by: David S. Miller --- drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index 22a5d24190840..1775997f9c69c 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -1477,15 +1477,15 @@ nfp_port_get_module_info(struct net_device *netdev, if (data < 0x3) { modinfo->type = ETH_MODULE_SFF_8436; - modinfo->eeprom_len = ETH_MODULE_SFF_8436_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8436_MAX_LEN; } else { modinfo->type = ETH_MODULE_SFF_8636; - modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_MAX_LEN; } break; case NFP_INTERFACE_QSFP28: modinfo->type = ETH_MODULE_SFF_8636; - modinfo->eeprom_len = ETH_MODULE_SFF_8636_LEN; + modinfo->eeprom_len = ETH_MODULE_SFF_8636_MAX_LEN; break; default: netdev_err(netdev, "Unsupported module 0x%x detected\n", -- GitLab From 801543b2593b04eef974a73d3ea03e7efbd5ffae Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 9 Nov 2022 17:35:22 +0200 Subject: [PATCH 1589/1927] drm/i915: stop including i915_irq.h from i915_trace.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turns out many of the files that need i915_reg.h get it implicitly via {display/intel_de.h, gt/intel_context.h} -> i915_trace.h -> i915_irq.h -> i915_reg.h. Since i915_trace.h doesn't actually need i915_irq.h, makes sense to drop it, but that requires adding quite a few new includes all over the place. Prefer including i915_reg.h where needed instead of adding another implicit include, because eventually we'll want to split up i915_reg.h and only include the specific registers at each place. Also some places actually needed i915_irq.h too. Cc: Lucas De Marchi Reviewed-by: Ville Syrjälä Signed-off-by: Jani Nikula Link: https://patchwork.freedesktop.org/patch/msgid/6e78a2e0ac1bffaf5af3b5ccc21dff05e6518cef.1668008071.git.jani.nikula@intel.com --- drivers/gpu/drm/i915/display/g4x_dp.c | 1 + drivers/gpu/drm/i915/display/g4x_hdmi.c | 1 + drivers/gpu/drm/i915/display/i9xx_plane.c | 4 +++- drivers/gpu/drm/i915/display/icl_dsi.c | 1 + drivers/gpu/drm/i915/display/intel_backlight.c | 1 + drivers/gpu/drm/i915/display/intel_cdclk.c | 1 + drivers/gpu/drm/i915/display/intel_color.c | 1 + drivers/gpu/drm/i915/display/intel_combo_phy.c | 1 + drivers/gpu/drm/i915/display/intel_crt.c | 2 ++ drivers/gpu/drm/i915/display/intel_cursor.c | 3 ++- drivers/gpu/drm/i915/display/intel_ddi.c | 1 + drivers/gpu/drm/i915/display/intel_display.c | 1 + drivers/gpu/drm/i915/display/intel_display_debugfs.c | 2 ++ drivers/gpu/drm/i915/display/intel_dp.c | 1 + drivers/gpu/drm/i915/display/intel_dp_aux.c | 1 + drivers/gpu/drm/i915/display/intel_dp_hdcp.c | 1 + drivers/gpu/drm/i915/display/intel_dp_mst.c | 1 + drivers/gpu/drm/i915/display/intel_dpio_phy.c | 1 + drivers/gpu/drm/i915/display/intel_dpll.c | 1 + drivers/gpu/drm/i915/display/intel_dpll_mgr.c | 1 + drivers/gpu/drm/i915/display/intel_drrs.c | 1 + drivers/gpu/drm/i915/display/intel_dsb.c | 1 + drivers/gpu/drm/i915/display/intel_dvo.c | 1 + drivers/gpu/drm/i915/display/intel_fdi.c | 1 + drivers/gpu/drm/i915/display/intel_gmbus.c | 2 ++ drivers/gpu/drm/i915/display/intel_hdmi.c | 1 + drivers/gpu/drm/i915/display/intel_lpe_audio.c | 2 ++ drivers/gpu/drm/i915/display/intel_lspcon.c | 1 + drivers/gpu/drm/i915/display/intel_lvds.c | 1 + drivers/gpu/drm/i915/display/intel_modeset_setup.c | 1 + drivers/gpu/drm/i915/display/intel_panel.c | 1 + drivers/gpu/drm/i915/display/intel_pch_display.c | 1 + drivers/gpu/drm/i915/display/intel_pch_refclk.c | 1 + drivers/gpu/drm/i915/display/intel_pipe_crc.c | 2 ++ drivers/gpu/drm/i915/display/intel_pps.c | 1 + drivers/gpu/drm/i915/display/intel_psr.c | 1 + drivers/gpu/drm/i915/display/intel_sdvo.c | 1 + drivers/gpu/drm/i915/display/intel_snps_phy.c | 1 + drivers/gpu/drm/i915/display/intel_sprite.c | 1 + drivers/gpu/drm/i915/display/intel_tv.c | 2 ++ drivers/gpu/drm/i915/display/intel_vdsc.c | 1 + drivers/gpu/drm/i915/display/intel_vga.c | 1 + drivers/gpu/drm/i915/display/intel_vrr.c | 1 + drivers/gpu/drm/i915/display/skl_scaler.c | 2 ++ drivers/gpu/drm/i915/display/skl_universal_plane.c | 2 ++ drivers/gpu/drm/i915/display/vlv_dsi.c | 1 + drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c | 1 + drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c | 1 + drivers/gpu/drm/i915/gt/intel_engine_cs.c | 2 ++ drivers/gpu/drm/i915/gt/intel_execlists_submission.c | 1 + drivers/gpu/drm/i915/gt/intel_gt.c | 1 + drivers/gpu/drm/i915/gt/intel_gt_pm.c | 1 + drivers/gpu/drm/i915/gt/intel_gtt.c | 1 + drivers/gpu/drm/i915/gt/intel_ring_submission.c | 2 ++ drivers/gpu/drm/i915/gt/intel_workarounds.c | 1 + drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c | 1 + drivers/gpu/drm/i915/gvt/cmd_parser.c | 1 + drivers/gpu/drm/i915/gvt/mmio_context.c | 1 + drivers/gpu/drm/i915/i915_gpu_error.c | 1 + drivers/gpu/drm/i915/i915_perf.c | 1 + drivers/gpu/drm/i915/i915_trace.h | 1 - drivers/gpu/drm/i915/intel_device_info.c | 3 ++- drivers/gpu/drm/i915/intel_uncore.c | 1 + 63 files changed, 76 insertions(+), 4 deletions(-) diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c index e3e3d27ffb53f..3593938dcd87c 100644 --- a/drivers/gpu/drm/i915/display/g4x_dp.c +++ b/drivers/gpu/drm/i915/display/g4x_dp.c @@ -8,6 +8,7 @@ #include #include "g4x_dp.h" +#include "i915_reg.h" #include "intel_audio.h" #include "intel_backlight.h" #include "intel_connector.h" diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c index 8aadf96fa5e9a..75ba24c9785f8 100644 --- a/drivers/gpu/drm/i915/display/g4x_hdmi.c +++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c @@ -6,6 +6,7 @@ */ #include "g4x_hdmi.h" +#include "i915_reg.h" #include "intel_audio.h" #include "intel_connector.h" #include "intel_crtc.h" diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c index 5afbe3e98ee8e..ecaeb7dc196bc 100644 --- a/drivers/gpu/drm/i915/display/i9xx_plane.c +++ b/drivers/gpu/drm/i915/display/i9xx_plane.c @@ -8,6 +8,9 @@ #include #include +#include "i915_irq.h" +#include "i915_reg.h" +#include "i9xx_plane.h" #include "intel_atomic.h" #include "intel_atomic_plane.h" #include "intel_de.h" @@ -15,7 +18,6 @@ #include "intel_fb.h" #include "intel_fbc.h" #include "intel_sprite.h" -#include "i9xx_plane.h" /* Primary plane formats for gen <= 3 */ static const u32 i8xx_primary_formats[] = { diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c index e05e7cd6c4123..d16b30a2dded3 100644 --- a/drivers/gpu/drm/i915/display/icl_dsi.c +++ b/drivers/gpu/drm/i915/display/icl_dsi.c @@ -29,6 +29,7 @@ #include #include +#include "i915_reg.h" #include "icl_dsi.h" #include "icl_dsi_regs.h" #include "intel_atomic.h" diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c index 11a1342d6d37d..0438071f58cf0 100644 --- a/drivers/gpu/drm/i915/display/intel_backlight.c +++ b/drivers/gpu/drm/i915/display/intel_backlight.c @@ -8,6 +8,7 @@ #include #include +#include "i915_reg.h" #include "intel_backlight.h" #include "intel_backlight_regs.h" #include "intel_connector.h" diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c index 8a9031012d744..b74e36d760138 100644 --- a/drivers/gpu/drm/i915/display/intel_cdclk.c +++ b/drivers/gpu/drm/i915/display/intel_cdclk.c @@ -24,6 +24,7 @@ #include #include "hsw_ips.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_atomic_plane.h" #include "intel_audio.h" diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index 93509cf7bbcc6..12f5b976c7957 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -22,6 +22,7 @@ * */ +#include "i915_reg.h" #include "intel_color.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c index 71d7aece1dc61..8b870b2dd4f9d 100644 --- a/drivers/gpu/drm/i915/display/intel_combo_phy.c +++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c @@ -3,6 +3,7 @@ * Copyright © 2018 Intel Corporation */ +#include "i915_reg.h" #include "intel_combo_phy.h" #include "intel_combo_phy_regs.h" #include "intel_de.h" diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c index 94d0a5e1dd030..797ad9489f7ee 100644 --- a/drivers/gpu/drm/i915/display/intel_crt.c +++ b/drivers/gpu/drm/i915/display/intel_crt.c @@ -34,6 +34,8 @@ #include #include "i915_drv.h" +#include "i915_irq.h" +#include "i915_reg.h" #include "intel_connector.h" #include "intel_crt.h" #include "intel_crtc.h" diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c index 96422c98656af..d190fa0d393bc 100644 --- a/drivers/gpu/drm/i915/display/intel_cursor.c +++ b/drivers/gpu/drm/i915/display/intel_cursor.c @@ -10,12 +10,13 @@ #include #include +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_atomic_plane.h" #include "intel_cursor.h" #include "intel_de.h" -#include "intel_display_types.h" #include "intel_display.h" +#include "intel_display_types.h" #include "intel_fb.h" #include "intel_fb_pin.h" #include "intel_frontbuffer.h" diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index e95bde5cf060e..477dd9b72ea3e 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -31,6 +31,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_audio.h" #include "intel_audio_regs.h" #include "intel_backlight.h" diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index c4d31db935fe1..1f642ab422683 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -76,6 +76,7 @@ #include "g4x_hdmi.h" #include "hsw_ips.h" #include "i915_drv.h" +#include "i915_reg.h" #include "i915_utils.h" #include "icl_dsi.h" #include "intel_acpi.h" diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c index cfc056a05bbf1..7bcd90384a46d 100644 --- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c +++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c @@ -9,6 +9,8 @@ #include #include "i915_debugfs.h" +#include "i915_irq.h" +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_debugfs.h" #include "intel_display_power.h" diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 7400d6b4c5876..914161d7d122f 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -46,6 +46,7 @@ #include "g4x_dp.h" #include "i915_debugfs.h" #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_audio.h" #include "intel_backlight.h" diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c index 48c375c65a418..664bebdecea76 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_aux.c +++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c @@ -4,6 +4,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "i915_trace.h" #include "intel_display_types.h" #include "intel_dp_aux.h" diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c index 35360dd543ac8..e0c1771614072 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c @@ -11,6 +11,7 @@ #include #include +#include "i915_reg.h" #include "intel_ddi.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c index cd4e61026d98e..4077a979a9249 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_mst.c +++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c @@ -29,6 +29,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_audio.h" #include "intel_connector.h" diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c index 6fc5b9e581528..7eb7440b31803 100644 --- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c +++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c @@ -21,6 +21,7 @@ * DEALINGS IN THE SOFTWARE. */ +#include "i915_reg.h" #include "intel_ddi.h" #include "intel_ddi_buf_trans.h" #include "intel_de.h" diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c index fdc6fa4f2ed9e..c236aafe9be01 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll.c +++ b/drivers/gpu/drm/i915/display/intel_dpll.c @@ -6,6 +6,7 @@ #include #include +#include "i915_reg.h" #include "intel_crtc.h" #include "intel_de.h" #include "intel_display.h" diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c index 7c6c094a0a01d..79579dca3c7ef 100644 --- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c +++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c @@ -23,6 +23,7 @@ #include +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_dkl_phy.h" diff --git a/drivers/gpu/drm/i915/display/intel_drrs.c b/drivers/gpu/drm/i915/display/intel_drrs.c index e27408efaae2f..5b9e44443814e 100644 --- a/drivers/gpu/drm/i915/display/intel_drrs.c +++ b/drivers/gpu/drm/i915/display/intel_drrs.c @@ -4,6 +4,7 @@ */ #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c index fc9c3e41c3330..1e1c6107d51b4 100644 --- a/drivers/gpu/drm/i915/display/intel_dsb.c +++ b/drivers/gpu/drm/i915/display/intel_dsb.c @@ -7,6 +7,7 @@ #include "gem/i915_gem_internal.h" #include "i915_drv.h" +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_dsb.h" diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c index 5950872889225..c86f9890754d4 100644 --- a/drivers/gpu/drm/i915/display/intel_dvo.c +++ b/drivers/gpu/drm/i915/display/intel_dvo.c @@ -32,6 +32,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_connector.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c index 7f47e5c85c817..063f1da4f229c 100644 --- a/drivers/gpu/drm/i915/display/intel_fdi.c +++ b/drivers/gpu/drm/i915/display/intel_fdi.c @@ -5,6 +5,7 @@ #include +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_crtc.h" #include "intel_ddi.h" diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c index 860e0f8b6b19f..a5840a28a69d5 100644 --- a/drivers/gpu/drm/i915/display/intel_gmbus.c +++ b/drivers/gpu/drm/i915/display/intel_gmbus.c @@ -34,6 +34,8 @@ #include #include "i915_drv.h" +#include "i915_irq.h" +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_gmbus.h" diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index 93519fb23d9dc..28221e12c743b 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -42,6 +42,7 @@ #include "i915_debugfs.h" #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_connector.h" #include "intel_ddi.h" diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c index 6a7ac60e4f766..a482e94ba0789 100644 --- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c +++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c @@ -71,6 +71,8 @@ #include #include "i915_drv.h" +#include "i915_irq.h" +#include "i915_reg.h" #include "intel_de.h" #include "intel_lpe_audio.h" #include "intel_pci_config.h" diff --git a/drivers/gpu/drm/i915/display/intel_lspcon.c b/drivers/gpu/drm/i915/display/intel_lspcon.c index 15d59de8810e1..9ff1c0b223ad4 100644 --- a/drivers/gpu/drm/i915/display/intel_lspcon.c +++ b/drivers/gpu/drm/i915/display/intel_lspcon.c @@ -28,6 +28,7 @@ #include #include +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_dp.h" diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c index 246787bbf5ef0..7bf1bdfd03ec0 100644 --- a/drivers/gpu/drm/i915/display/intel_lvds.c +++ b/drivers/gpu/drm/i915/display/intel_lvds.c @@ -39,6 +39,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_backlight.h" #include "intel_connector.h" diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c index 9d8ca230be392..96395bfbd41df 100644 --- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c +++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c @@ -10,6 +10,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_bw.h" #include "intel_color.h" diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c index 69ce77711b7cf..1640726bfbf6a 100644 --- a/drivers/gpu/drm/i915/display/intel_panel.c +++ b/drivers/gpu/drm/i915/display/intel_panel.c @@ -31,6 +31,7 @@ #include #include +#include "i915_reg.h" #include "intel_backlight.h" #include "intel_connector.h" #include "intel_de.h" diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c index 837152dca063b..cecc0d007cf39 100644 --- a/drivers/gpu/drm/i915/display/intel_pch_display.c +++ b/drivers/gpu/drm/i915/display/intel_pch_display.c @@ -4,6 +4,7 @@ */ #include "g4x_dp.h" +#include "i915_reg.h" #include "intel_crt.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.c b/drivers/gpu/drm/i915/display/intel_pch_refclk.c index a66097cdc1e0b..08a94365b7d13 100644 --- a/drivers/gpu/drm/i915/display/intel_pch_refclk.c +++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.c @@ -3,6 +3,7 @@ * Copyright © 2021 Intel Corporation */ +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_types.h" #include "intel_panel.h" diff --git a/drivers/gpu/drm/i915/display/intel_pipe_crc.c b/drivers/gpu/drm/i915/display/intel_pipe_crc.c index 1c74388c60d7b..673454fbf7841 100644 --- a/drivers/gpu/drm/i915/display/intel_pipe_crc.c +++ b/drivers/gpu/drm/i915/display/intel_pipe_crc.c @@ -29,6 +29,8 @@ #include #include +#include "i915_irq.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c index 22f5e08d396b1..81ee7f3aadf64 100644 --- a/drivers/gpu/drm/i915/display/intel_pps.c +++ b/drivers/gpu/drm/i915/display/intel_pps.c @@ -5,6 +5,7 @@ #include "g4x_dp.h" #include "i915_drv.h" +#include "i915_reg.h" #include "intel_de.h" #include "intel_display_power_well.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index a75b378515041..5b678916e6db5 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -27,6 +27,7 @@ #include "display/intel_dp.h" #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_crtc.h" #include "intel_de.h" diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c index 48b7b1aa37b24..329b9d9af6679 100644 --- a/drivers/gpu/drm/i915/display/intel_sdvo.c +++ b/drivers/gpu/drm/i915/display/intel_sdvo.c @@ -37,6 +37,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_atomic.h" #include "intel_connector.h" #include "intel_crtc.h" diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c index 937cefd6f78f7..c799e891f8b59 100644 --- a/drivers/gpu/drm/i915/display/intel_snps_phy.c +++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c @@ -5,6 +5,7 @@ #include +#include "i915_reg.h" #include "intel_ddi.h" #include "intel_ddi_buf_trans.h" #include "intel_de.h" diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c index 7649c50b54452..e6b4d24b9cd0e 100644 --- a/drivers/gpu/drm/i915/display/intel_sprite.c +++ b/drivers/gpu/drm/i915/display/intel_sprite.c @@ -42,6 +42,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "i915_vgpu.h" #include "i9xx_plane.h" #include "intel_atomic_plane.h" diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c index cf7d5c1ab4069..4d2101ca1692e 100644 --- a/drivers/gpu/drm/i915/display/intel_tv.c +++ b/drivers/gpu/drm/i915/display/intel_tv.c @@ -35,6 +35,8 @@ #include #include "i915_drv.h" +#include "i915_reg.h" +#include "i915_irq.h" #include "intel_connector.h" #include "intel_crtc.h" #include "intel_de.h" diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c index 269f9792390de..9d3b77b41b5c1 100644 --- a/drivers/gpu/drm/i915/display/intel_vdsc.c +++ b/drivers/gpu/drm/i915/display/intel_vdsc.c @@ -10,6 +10,7 @@ #include #include "i915_drv.h" +#include "i915_reg.h" #include "intel_crtc.h" #include "intel_de.h" #include "intel_display_types.h" diff --git a/drivers/gpu/drm/i915/display/intel_vga.c b/drivers/gpu/drm/i915/display/intel_vga.c index b5d058404c147..a69bfcac9a945 100644 --- a/drivers/gpu/drm/i915/display/intel_vga.c +++ b/drivers/gpu/drm/i915/display/intel_vga.c @@ -10,6 +10,7 @@ #include